diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 5f22afe..0000000 --- a/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM clojure:openjdk-16-alpine AS builder -RUN mkdir -p /app -WORKDIR /app -COPY project.clj /app -RUN lein deps -COPY . /app -RUN mv "$(lein uberjar | sed -n 's/^Created \(.*standalone\.jar\)/\1/p')" app-standalone.jar - -FROM openjdk:16 -RUN mkdir -p /app/public -WORKDIR /app -COPY --from=builder /app/app-standalone.jar /app/bdm.jar -# Make sure you mount the public folder -CMD java -jar bdm.jar $TYPE diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..510328c --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +build: + mkdir -p bin + go build -o bin/buchdesmonats cmd/buchdesmonats/main.go + +run: + go run cmd/buchdesmonats/main.go diff --git a/README.mkd b/README.mkd deleted file mode 100644 index 739bd8f..0000000 --- a/README.mkd +++ /dev/null @@ -1,45 +0,0 @@ -# Buch des Monats - -This simple script generate a HTML representation of the "Book of the Month" -list. This includes books and comics. - -## Building - - $ lein uberjar - -## Usage - - $ java -jar target/buchdesmonats-2.0-standalone.jar [book|comic] - -## Docker - -You can use the Dockerfile to generate the static content. This is useful -for a cronjob. You want to mount the target dir, so it is not lost after -the container stops. The Dockerfile is a multi stage dockerfile, which -first compiles the clojure files into a standalone jar file and then use -it to generate the book of the month content. - - $ docker build . -t bdm:latest - -Run this periodically: - - $ docker run -it -v "$PWD/public:/app/public" -e "TYPE=book" bdm:latest - $ docker run -it -v "$PWD/public:/app/public" -e "TYPE=comic" bdm:latest - -If you want to run this in a cronjob, remove the ```-t``` from the docker run -command, because we do not have a tty. - -## Update - - $ git revert . - $ git pull - $ docker build . t bdm:latest - -Keep in mind that the ```git revert .``` is needed because the bdm.jar file -itself will manipulate some of the templates in ```public/``` which will be -conflicted with the version in git. - -## Authors - -* Programming: [Aaron Fischer](https://aaron-fischer.net/) -* Content: [Michael Reutter](https://social.okoyono.de/@mezzo) diff --git a/cmd/buchdesmonats/main.go b/cmd/buchdesmonats/main.go new file mode 100644 index 0000000..bc5f263 --- /dev/null +++ b/cmd/buchdesmonats/main.go @@ -0,0 +1,105 @@ +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "regexp" +) + +type Item struct { + ISBN string + Filename string +} + +func (i Item) imageURL() string { + return "https://medien.ubitweb.de/bildzentrale_original/" + + i.ISBN[0:3] + "/" + + i.ISBN[3:6] + "/" + + i.ISBN[6:9] + "/" + + i.ISBN[9:13] + + ".jpg" +} + +func (i Item) targetFilename() string { + return "covers/" + i.ISBN + ".jpg" +} + +func (i Item) downloadCover() error { + resp, err := http.Get(i.imageURL()) + + if err != nil { + return err + } + + defer resp.Body.Close() + + out, err := os.Create(i.targetFilename()) + if err != nil { + return err + } + + defer out.Close() + + _, err = io.Copy(out, resp.Body) + + return err +} + +var filename string +var force bool + +func getItems(filename string) []Item { + var items []Item + // Get all book URLS + url := "https://git.okoyono.de/mezzo/buch_des_monats/raw/branch/master/" + filename + resp, err := http.Get(url) + + if err != nil { + panic(filename + " is missing") + } + + defer resp.Body.Close() + + content, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic("Can not download the file. Network problem?") + } + + re := regexp.MustCompile(`[0-9]{13}`) + matches := re.FindAllString(string(content), -1) + + for _, isbn := range matches { + items = append(items, Item{ISBN: isbn, Filename: filename}) + } + + return items +} + +func main() { + // Get all items from the git repo + items := getItems(filename) + + for _, item := range items { + _, err := os.Stat(item.targetFilename()) + if os.IsNotExist(err) || force { + fmt.Printf("Downloading %v ...\n", item.imageURL()) + err := item.downloadCover() + + if err != nil { + fmt.Printf("ERROR: File %s not found\n", item.imageURL()) + } + } + } + + // TODO: use template to generate the restulting HTML +} + +func init() { + flag.StringVar(&filename, "filename", "COMIC.mkd", "The filename to use") + flag.BoolVar(&force, "force", false, "Ignore cache, download all covers") + flag.Parse() +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..466732e --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module okoyono.de/buchdesmonats + +go 1.15 diff --git a/project.clj b/project.clj deleted file mode 100644 index 493db06..0000000 --- a/project.clj +++ /dev/null @@ -1,13 +0,0 @@ -(defproject buchdesmonats "1.8" - :description "A simple tool to fetch covers of the month from the okoyono.de project." - :url "https://git.okoyono.de/mezzomix/buch_des_monats" - :license {:name "MIT License" - :url "http://opensource.org/licenses/MIT"} - :dependencies [[org.clojure/clojure "1.10.2"] - [org.clojure/tools.logging "1.1.0"] - [enlive "1.1.6"] - [me.raynes/fs "1.4.6"] ;; R.I.P. Anthony - [clj-http "3.12.1"]] - :main ^:skip-aot buchdesmonats.core - :profiles {:uberjar {:aot :all}}) - diff --git a/public/book.html b/public/book.html index 798defb..0f0219b 100644 --- a/public/book.html +++ b/public/book.html @@ -22,8 +22,9 @@
- Mojoreads + Mojoreads cover
- + + diff --git a/public/comic.html b/public/comic.html index a11bfcf..30690fe 100644 --- a/public/comic.html +++ b/public/comic.html @@ -12,16 +12,16 @@

Comic des Monats

Neues Projekt: Eine Sammlung mit Comics die für Empfehlenswert hält:

- Jeden Monat ein neuer Comic aus seiner Sammlung. Die Buchlinks gehen zu Mojoreads, der - Code von Aaron Fischer. Ein økoyono Projekt.

+ Reutter für Empfehlenswert hält:

+ Jeden Monat ein neuer Comic aus seiner Sammlung. Die Buchlinks gehen zu Mojoreads, der + Code von Aaron Fischer. Ein økoyono Projekt.

- *Eine Seite mit Buchempfehlungen findet Ihr [hier](https://buchdesmonats.okoyono.de)* + *Eine Seite mit Buchempfehlungen findet Ihr [hier](https://buchdesmonats.okoyono.de)*
-
- Mojoreads cover -
+
+ Mojoreads cover +
diff --git a/src/buchdesmonats/core.clj b/src/buchdesmonats/core.clj deleted file mode 100644 index dfd0846..0000000 --- a/src/buchdesmonats/core.clj +++ /dev/null @@ -1,93 +0,0 @@ -;;; Copyright (C) 2014-2021 Aaron Fischer Permission -;;; is hereby granted, free of charge, to any person obtaining a copy of this -;;; software and associated documentation files (the "Software"), to deal in the -;;; Software without restriction, including without limitation the rights to -;;; use, copy, modify, merge, publish, distribute, sublicense, and/or sell -;;; copies of the Software, and to permit persons to whom the Software is -;;; furnished to do so, subject to the following conditions: The above -;;; copyright notice and this permission notice shall be included in all copies -;;; or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -;;; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -;;; FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -;;; COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -;;; IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -;;; CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -(ns buchdesmonats.core - (:gen-class) - (:require [net.cgrand.enlive-html :as html] - [clj-http.client :as http-client] - [clojure.string :as str] - [clojure.java.io :as io] - [clojure.tools.logging :as log] - [me.raynes.fs :as fs])) - -(defn bookurl->isbn [url] - (re-find #"[0-9]+" url)) - -(defn isbn->imageurl [isbn] - (str "https://medien.ubitweb.de/bildzentrale_original/" - (subs isbn 0 3) "/" - (subs isbn 3 6) "/" - (subs isbn 6 9) "/" - (subs isbn 9) ".jpg")) - -(defn imgurl->bytes [url] - (let [isbn (bookurl->isbn url) - url-to-fetch (isbn->imageurl isbn) - stream (http-client/get url-to-fetch {:as :byte-array})] - (:body stream))) - -(defn url->file [url target-dir] - (io/file target-dir (str (bookurl->isbn url) ".jpg"))) - -(defn scrape-book-urls [datasource-url] - (->> (http-client/get datasource-url {:insecure? true}) - :body - str/split-lines - (map #(second (re-find #"^\* .*\[.+\]\((.+)\)" %))) - (remove nil?))) - -(defn scrape-book-cover [url target-dir] - (try - (let [target-file (url->file url target-dir) - bytes (imgurl->bytes url)] - (with-open [out (io/output-stream target-file)] - (.write out bytes))) - (catch Exception e - (log/info "Problem with " url ":" e ". Skip it.")))) - -(defn find-missing-covers [books-url target-dir] - (remove #(fs/exists? (url->file % target-dir)) - (scrape-book-urls books-url))) - -(defn cover-item-model-for-type [public-dir type] -(html/defsnippet cover-item-model (io/file public-dir (str type ".html")) [:div#covers :> :div] - [link title] - [:a] (html/set-attr :href link) - [:img] (html/set-attr :src (url->file link (str type "-covers")) :title title))) - -(defn template-for-type [public-dir type] -(html/deftemplate book-template (io/reader (io/file public-dir (str type ".html"))) - [cover-urls] - [:#covers] (html/content - (map #((cover-item-model-for-type public-dir type) % "zu Lovely Books") - cover-urls)))) - -(defn generate-html [type book-urls public-dir] - (let [content (apply str ((template-for-type public-dir type) book-urls))] - (with-open [out (io/writer (io/file public-dir (str type ".html")))] - (.write out content)))) - -(defn -main [& args] - (if (empty? args) - (do (log/fatal "Please give a cover type (comic/book)") - (System/exit 1))) - (let [type (first args) - datasource-url (str "https://git.okoyono.de/mezzo/buch_des_monats/raw/master/" (clojure.string/upper-case type) ".mkd") - target-dir (io/file "public" (str type "-covers/"))] - (fs/mkdirs target-dir) - (generate-html type (scrape-book-urls datasource-url) "public") - (doall (pmap #(scrape-book-cover % target-dir) - (find-missing-covers datasource-url target-dir))) - (System/exit 0)))