buch_des_monats/src/buchdesmonats/core.clj

;;; Copyright (C) 2014-2021 Aaron Fischer <mail@aaron-fischer.net>  Permission
;;; is hereby granted, free of charge, to any person obtaining a copy of this
;;; software and associated documentation files (the "Software"), to deal in the
;;; Software without restriction, including without limitation the rights to
;;; use, copy, modify, merge, publish, distribute, sublicense, and/or sell
;;; copies of the Software, and to permit persons to whom the Software is
;;; furnished to do so, subject to the following conditions:  The above
;;; copyright notice and this permission notice shall be included in all copies
;;; or substantial portions of the Software.   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
;;; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
;;; FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
;;; COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
;;; IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
;;; CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

(ns buchdesmonats.core
  (:gen-class)
  (:require [net.cgrand.enlive-html :as html]
            [clj-http.client :as http-client]
            [clojure.string :as str]
            [clojure.java.io :as io]
            [clojure.tools.logging :as log]
            [me.raynes.fs :as fs]))

(defn bookurl->isbn [url]
  (re-find #"[0-9]+" url))

(defn isbn->imageurl [isbn]
  (str "https://medien.ubitweb.de/bildzentrale_original/"
       (subs isbn 0 3) "/"
       (subs isbn 3 6) "/"
       (subs isbn 6 9) "/"
       (subs isbn 9) ".jpg"))

(defn imgurl->bytes [url]
  (let [isbn (bookurl->isbn url)
        url-to-fetch (isbn->imageurl isbn)
        stream (http-client/get url-to-fetch {:as :byte-array})]
    (:body stream)))

(defn url->file [url target-dir]
  (io/file target-dir (str (bookurl->isbn url) ".jpg")))

(defn scrape-book-urls [datasource-url]
  (->> (http-client/get datasource-url {:insecure? true})
       :body
       str/split-lines
       (map #(second (re-find #"^\* .*\[.+\]\((.+)\)" %)))
       (remove nil?)))

(defn scrape-book-cover [url target-dir]
  (try
    (let [target-file (url->file url target-dir)
          bytes (imgurl->bytes url)]
      (with-open [out (io/output-stream target-file)]
        (.write out bytes)))
    (catch Exception e
      (log/info "Problem with " url ":" e ". Skip it."))))

(defn find-missing-covers [books-url target-dir]
  (remove #(fs/exists? (url->file % target-dir))
          (scrape-book-urls books-url)))

(defn cover-item-model-for-type [public-dir type]
(html/defsnippet cover-item-model (io/file public-dir (str type ".html")) [:div#covers :> :div]
  [link title]
  [:a] (html/set-attr :href link)
  [:img] (html/set-attr :src (url->file link (str type "-covers")) :title title)))

(defn template-for-type [public-dir type]
(html/deftemplate book-template (io/reader (io/file public-dir (str type ".html")))
  [cover-urls]
  [:#covers] (html/content
              (map #((cover-item-model-for-type public-dir type) % "zu Lovely Books")
                   cover-urls))))

(defn generate-html [type book-urls public-dir]
  (let [content (apply str ((template-for-type public-dir type) book-urls))]
    (with-open [out (io/writer (io/file public-dir (str type ".html")))]
      (.write out content))))

(defn -main [& args]
  (if (empty? args)
    (do (log/fatal "Please give a cover type (comic/book)")
        (System/exit 1)))
  (let [type (first args)
        datasource-url (str "https://git.okoyono.de/mezzo/buch_des_monats/raw/master/" (clojure.string/upper-case type) ".mkd")
        target-dir (io/file "public" (str type "-covers/"))]
    (fs/mkdirs target-dir)
    (generate-html type (scrape-book-urls datasource-url) "public")
    (doall (pmap #(scrape-book-cover % target-dir)
                 (find-missing-covers datasource-url target-dir)))
    (System/exit 0)))