mezzo/buch_des_monats
1
0
Fork 0
Code Issues 7 Pull requests Projects Releases Packages Wiki Activity Actions

File file handling

Browse source
This commit is contained in:
Aaron Mueller 2014-03-24 21:55:07 +01:00
parent 003401b5f1
commit e1d8943c96
2 changed files with 23 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
project.clj
View file

@ -7,4 +7,5 @@
[enlive "1.1.5"]
[me.raynes/fs "1.4.4"]
[clj-http "0.9.1"]]
:main buchdesmonats.core)
:main ^:skip-aot buchdesmonats.core
:profiles {:uberjar {:aot :all}})

31
src/buchdesmonats/core.clj
View file

@ -1,8 +1,13 @@
(ns buchdesmonats.core
(:gen-class)
(:require [net.cgrand.enlive-html :as html]
[clj-http.client :as http-client]
[clojure.string :as str]
[clojure.java.io :as io]))
[clojure.java.io :as io]
[me.raynes.fs :as fs]))
(def config {:books-url "https://raw.github.com/CTHN/wiki-data/master/pages/projects/buch_des_monats.mkd"
:target-dir (io/file "public" "book-covers")})
(defn imgurl->bytes [lovelybooks-url]
(-> (java.net.URL. lovelybooks-url)
@ -14,10 +19,10 @@
(http-client/get {:as :byte-array})
:body))
(defn url->filename [lovelybooks-url]
(defn url->file [lovelybooks-url target-dir]
(let [[_ author title] (re-find #".\/autor\/([^\/]+)\/(.+)-([0-9]+)-.\/$" lovelybooks-url)]
(str/lower-case (str author "_" title ".jpg"))))
(io/file target-dir
(str/lower-case (str author "_" title ".jpg")))))
(defn scrape-book-urls [github-url]
(->> (http-client/get github-url)
@ -26,13 +31,17 @@
(map #(second (re-find #"^\* .*\[.+\]\((.+)\)" %)))
(remove nil?)))
(defn scrape-book-cover [url]
(with-open [out (io/output-stream (url->filename url))]
(.write out (imgurl->bytes url))))
(defn scrape-book-cover [url target-dir]
(let [target-file (url->file url target-dir)]
(with-open [out (io/output-stream target-file)]
(.write out (imgurl->bytes url)))))
(defn find-missing-covers [books-url target-dir]
(remove #(fs/exists? (url->file % target-dir))
(scrape-book-urls books-url)))
(defn -main [& args]
(let [books-url "https://raw.github.com/CTHN/wiki-data/master/pages/projects/buch_des_monats.mkd"]
(doall (pmap #(scrape-book-cover %)
(scrape-book-urls books-url)))
true))
(fs/mkdirs (:target-dir config))
(doall (pmap #(scrape-book-cover % (:target-dir config))
(find-missing-covers (:books-url config) (:target-dir config))))
true)