Fresh start with different programming language, WIP
This commit is contained in:
parent
17d8fc8ea6
commit
646659afb8
9 changed files with 124 additions and 174 deletions
14
Dockerfile
14
Dockerfile
|
|
@ -1,14 +0,0 @@
|
|||
FROM clojure:openjdk-16-alpine AS builder
|
||||
RUN mkdir -p /app
|
||||
WORKDIR /app
|
||||
COPY project.clj /app
|
||||
RUN lein deps
|
||||
COPY . /app
|
||||
RUN mv "$(lein uberjar | sed -n 's/^Created \(.*standalone\.jar\)/\1/p')" app-standalone.jar
|
||||
|
||||
FROM openjdk:16
|
||||
RUN mkdir -p /app/public
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/app-standalone.jar /app/bdm.jar
|
||||
# Make sure you mount the public folder
|
||||
CMD java -jar bdm.jar $TYPE
|
||||
6
Makefile
Normal file
6
Makefile
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
build:
|
||||
mkdir -p bin
|
||||
go build -o bin/buchdesmonats cmd/buchdesmonats/main.go
|
||||
|
||||
run:
|
||||
go run cmd/buchdesmonats/main.go
|
||||
45
README.mkd
45
README.mkd
|
|
@ -1,45 +0,0 @@
|
|||
# Buch des Monats
|
||||
|
||||
This simple script generate a HTML representation of the "Book of the Month"
|
||||
list. This includes books and comics.
|
||||
|
||||
## Building
|
||||
|
||||
$ lein uberjar
|
||||
|
||||
## Usage
|
||||
|
||||
$ java -jar target/buchdesmonats-2.0-standalone.jar [book|comic]
|
||||
|
||||
## Docker
|
||||
|
||||
You can use the Dockerfile to generate the static content. This is useful
|
||||
for a cronjob. You want to mount the target dir, so it is not lost after
|
||||
the container stops. The Dockerfile is a multi stage dockerfile, which
|
||||
first compiles the clojure files into a standalone jar file and then use
|
||||
it to generate the book of the month content.
|
||||
|
||||
$ docker build . -t bdm:latest
|
||||
|
||||
Run this periodically:
|
||||
|
||||
$ docker run -it -v "$PWD/public:/app/public" -e "TYPE=book" bdm:latest
|
||||
$ docker run -it -v "$PWD/public:/app/public" -e "TYPE=comic" bdm:latest
|
||||
|
||||
If you want to run this in a cronjob, remove the ```-t``` from the docker run
|
||||
command, because we do not have a tty.
|
||||
|
||||
## Update
|
||||
|
||||
$ git revert .
|
||||
$ git pull
|
||||
$ docker build . t bdm:latest
|
||||
|
||||
Keep in mind that the ```git revert .``` is needed because the bdm.jar file
|
||||
itself will manipulate some of the templates in ```public/``` which will be
|
||||
conflicted with the version in git.
|
||||
|
||||
## Authors
|
||||
|
||||
* Programming: [Aaron Fischer](https://aaron-fischer.net/)
|
||||
* Content: [Michael Reutter](https://social.okoyono.de/@mezzo)
|
||||
105
cmd/buchdesmonats/main.go
Normal file
105
cmd/buchdesmonats/main.go
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
ISBN string
|
||||
Filename string
|
||||
}
|
||||
|
||||
func (i Item) imageURL() string {
|
||||
return "https://medien.ubitweb.de/bildzentrale_original/" +
|
||||
i.ISBN[0:3] + "/" +
|
||||
i.ISBN[3:6] + "/" +
|
||||
i.ISBN[6:9] + "/" +
|
||||
i.ISBN[9:13] +
|
||||
".jpg"
|
||||
}
|
||||
|
||||
func (i Item) targetFilename() string {
|
||||
return "covers/" + i.ISBN + ".jpg"
|
||||
}
|
||||
|
||||
func (i Item) downloadCover() error {
|
||||
resp, err := http.Get(i.imageURL())
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
out, err := os.Create(i.targetFilename())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer out.Close()
|
||||
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
var filename string
|
||||
var force bool
|
||||
|
||||
func getItems(filename string) []Item {
|
||||
var items []Item
|
||||
// Get all book URLS
|
||||
url := "https://git.okoyono.de/mezzo/buch_des_monats/raw/branch/master/" + filename
|
||||
resp, err := http.Get(url)
|
||||
|
||||
if err != nil {
|
||||
panic(filename + " is missing")
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
content, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
panic("Can not download the file. Network problem?")
|
||||
}
|
||||
|
||||
re := regexp.MustCompile(`[0-9]{13}`)
|
||||
matches := re.FindAllString(string(content), -1)
|
||||
|
||||
for _, isbn := range matches {
|
||||
items = append(items, Item{ISBN: isbn, Filename: filename})
|
||||
}
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Get all items from the git repo
|
||||
items := getItems(filename)
|
||||
|
||||
for _, item := range items {
|
||||
_, err := os.Stat(item.targetFilename())
|
||||
if os.IsNotExist(err) || force {
|
||||
fmt.Printf("Downloading %v ...\n", item.imageURL())
|
||||
err := item.downloadCover()
|
||||
|
||||
if err != nil {
|
||||
fmt.Printf("ERROR: File %s not found\n", item.imageURL())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: use template to generate the restulting HTML
|
||||
}
|
||||
|
||||
func init() {
|
||||
flag.StringVar(&filename, "filename", "COMIC.mkd", "The filename to use")
|
||||
flag.BoolVar(&force, "force", false, "Ignore cache, download all covers")
|
||||
flag.Parse()
|
||||
}
|
||||
3
go.mod
Normal file
3
go.mod
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
module okoyono.de/buchdesmonats
|
||||
|
||||
go 1.15
|
||||
13
project.clj
13
project.clj
|
|
@ -1,13 +0,0 @@
|
|||
(defproject buchdesmonats "1.8"
|
||||
:description "A simple tool to fetch covers of the month from the okoyono.de project."
|
||||
:url "https://git.okoyono.de/mezzomix/buch_des_monats"
|
||||
:license {:name "MIT License"
|
||||
:url "http://opensource.org/licenses/MIT"}
|
||||
:dependencies [[org.clojure/clojure "1.10.2"]
|
||||
[org.clojure/tools.logging "1.1.0"]
|
||||
[enlive "1.1.6"]
|
||||
[me.raynes/fs "1.4.6"] ;; R.I.P. Anthony
|
||||
[clj-http "3.12.1"]]
|
||||
:main ^:skip-aot buchdesmonats.core
|
||||
:profiles {:uberjar {:aot :all}})
|
||||
|
||||
|
|
@ -22,8 +22,9 @@
|
|||
|
||||
<div id="covers">
|
||||
<div class="cover-item">
|
||||
<a href="#"><img src="" alt="Mojoreads" title="zu Mojoreads" /></a>
|
||||
<a href="#"><img src="" alt="Mojoreads cover" title="zu Mojoreads" /></a>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
|
|
|||
|
|
@ -12,16 +12,16 @@
|
|||
<body>
|
||||
<h1>Comic des Monats</h1>
|
||||
<p>Neues Projekt: Eine Sammlung mit Comics die <a href="https://social.okoyono.de/@mezzo" rel="author">Michael
|
||||
Reutter</a> für Empfehlenswert hält:<p>
|
||||
Jeden Monat ein neuer Comic aus seiner Sammlung. Die Buchlinks gehen zu <a href="http://mojoreads.de/">Mojoreads</a>, <a href="https://git.okoyono.de/mezzo/buch_des_monats">der
|
||||
Code</a> von <a href="https://aaron-fischer.net/">Aaron Fischer</a>. Ein <a href="https://okoyono.de/">økoyono</a> Projekt.</p>
|
||||
Reutter</a> für Empfehlenswert hält:</p><p>
|
||||
Jeden Monat ein neuer Comic aus seiner Sammlung. Die Buchlinks gehen zu <a href="http://mojoreads.de/">Mojoreads</a>, <a href="https://git.okoyono.de/mezzo/buch_des_monats">der
|
||||
Code</a> von <a href="https://aaron-fischer.net/">Aaron Fischer</a>. Ein <a href="https://okoyono.de/">økoyono</a> Projekt.</p>
|
||||
|
||||
*Eine Seite mit Buchempfehlungen findet Ihr [hier](https://buchdesmonats.okoyono.de)*
|
||||
*Eine Seite mit Buchempfehlungen findet Ihr [hier](https://buchdesmonats.okoyono.de)*
|
||||
|
||||
<div id="covers">
|
||||
<div class="cover-item">
|
||||
<a href="#"><img src="" alt="Mojoreads cover" title="zu Mojoreads" /></a>
|
||||
</div>
|
||||
<div class="cover-item">
|
||||
<a href="#"><img src="" alt="Mojoreads cover" title="zu Mojoreads" /></a>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
;;; Copyright (C) 2014-2021 Aaron Fischer <mail@aaron-fischer.net> Permission
|
||||
;;; is hereby granted, free of charge, to any person obtaining a copy of this
|
||||
;;; software and associated documentation files (the "Software"), to deal in the
|
||||
;;; Software without restriction, including without limitation the rights to
|
||||
;;; use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
;;; copies of the Software, and to permit persons to whom the Software is
|
||||
;;; furnished to do so, subject to the following conditions: The above
|
||||
;;; copyright notice and this permission notice shall be included in all copies
|
||||
;;; or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
;;; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
;;; FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
;;; COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
;;; IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
;;; CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
(ns buchdesmonats.core
|
||||
(:gen-class)
|
||||
(:require [net.cgrand.enlive-html :as html]
|
||||
[clj-http.client :as http-client]
|
||||
[clojure.string :as str]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.tools.logging :as log]
|
||||
[me.raynes.fs :as fs]))
|
||||
|
||||
(defn bookurl->isbn [url]
|
||||
(re-find #"[0-9]+" url))
|
||||
|
||||
(defn isbn->imageurl [isbn]
|
||||
(str "https://medien.ubitweb.de/bildzentrale_original/"
|
||||
(subs isbn 0 3) "/"
|
||||
(subs isbn 3 6) "/"
|
||||
(subs isbn 6 9) "/"
|
||||
(subs isbn 9) ".jpg"))
|
||||
|
||||
(defn imgurl->bytes [url]
|
||||
(let [isbn (bookurl->isbn url)
|
||||
url-to-fetch (isbn->imageurl isbn)
|
||||
stream (http-client/get url-to-fetch {:as :byte-array})]
|
||||
(:body stream)))
|
||||
|
||||
(defn url->file [url target-dir]
|
||||
(io/file target-dir (str (bookurl->isbn url) ".jpg")))
|
||||
|
||||
(defn scrape-book-urls [datasource-url]
|
||||
(->> (http-client/get datasource-url {:insecure? true})
|
||||
:body
|
||||
str/split-lines
|
||||
(map #(second (re-find #"^\* .*\[.+\]\((.+)\)" %)))
|
||||
(remove nil?)))
|
||||
|
||||
(defn scrape-book-cover [url target-dir]
|
||||
(try
|
||||
(let [target-file (url->file url target-dir)
|
||||
bytes (imgurl->bytes url)]
|
||||
(with-open [out (io/output-stream target-file)]
|
||||
(.write out bytes)))
|
||||
(catch Exception e
|
||||
(log/info "Problem with " url ":" e ". Skip it."))))
|
||||
|
||||
(defn find-missing-covers [books-url target-dir]
|
||||
(remove #(fs/exists? (url->file % target-dir))
|
||||
(scrape-book-urls books-url)))
|
||||
|
||||
(defn cover-item-model-for-type [public-dir type]
|
||||
(html/defsnippet cover-item-model (io/file public-dir (str type ".html")) [:div#covers :> :div]
|
||||
[link title]
|
||||
[:a] (html/set-attr :href link)
|
||||
[:img] (html/set-attr :src (url->file link (str type "-covers")) :title title)))
|
||||
|
||||
(defn template-for-type [public-dir type]
|
||||
(html/deftemplate book-template (io/reader (io/file public-dir (str type ".html")))
|
||||
[cover-urls]
|
||||
[:#covers] (html/content
|
||||
(map #((cover-item-model-for-type public-dir type) % "zu Lovely Books")
|
||||
cover-urls))))
|
||||
|
||||
(defn generate-html [type book-urls public-dir]
|
||||
(let [content (apply str ((template-for-type public-dir type) book-urls))]
|
||||
(with-open [out (io/writer (io/file public-dir (str type ".html")))]
|
||||
(.write out content))))
|
||||
|
||||
(defn -main [& args]
|
||||
(if (empty? args)
|
||||
(do (log/fatal "Please give a cover type (comic/book)")
|
||||
(System/exit 1)))
|
||||
(let [type (first args)
|
||||
datasource-url (str "https://git.okoyono.de/mezzo/buch_des_monats/raw/master/" (clojure.string/upper-case type) ".mkd")
|
||||
target-dir (io/file "public" (str type "-covers/"))]
|
||||
(fs/mkdirs target-dir)
|
||||
(generate-html type (scrape-book-urls datasource-url) "public")
|
||||
(doall (pmap #(scrape-book-cover % target-dir)
|
||||
(find-missing-covers datasource-url target-dir)))
|
||||
(System/exit 0)))
|
||||
Loading…
Reference in a new issue