Find broken URLs
This commit is contained in:
parent
0b5a63ba86
commit
c8c534c9ca
2 changed files with 14 additions and 8 deletions
|
@ -1,9 +1,10 @@
|
|||
(defproject buchdesmonats "1.1"
|
||||
(defproject buchdesmonats "1.2"
|
||||
:description "A simple tool to fetch all books of the month from the okoyono.de project."
|
||||
:url "https://git.okoyono.de/mezzomix/buch_des_monats"
|
||||
:license {:name "MIT License"
|
||||
:url "http://opensource.org/licenses/MIT"}
|
||||
:dependencies [[org.clojure/clojure "1.6.0"]
|
||||
[org.clojure/tools.logging "0.3.1"]
|
||||
[enlive "1.1.5"]
|
||||
[me.raynes/fs "1.4.5"]
|
||||
[clj-http "0.9.2"]]
|
||||
|
|
|
@ -23,6 +23,7 @@
|
|||
[clj-http.client :as http-client]
|
||||
[clojure.string :as str]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.tools.logging :as log]
|
||||
[me.raynes.fs :as fs]))
|
||||
|
||||
(defn imgurl->bytes [lovelybooks-url]
|
||||
|
@ -59,10 +60,14 @@
|
|||
(remove nil?)))
|
||||
|
||||
(defn scrape-book-cover [url target-dir]
|
||||
(let [target-file (url->file url target-dir)
|
||||
encoded-url (encode-url url)]
|
||||
(with-open [out (io/output-stream target-file)]
|
||||
(.write out (imgurl->bytes encoded-url)))))
|
||||
(try
|
||||
(let [target-file (url->file url target-dir)
|
||||
encoded-url (encode-url url)
|
||||
bytes (imgurl->bytes encoded-url)]
|
||||
(with-open [out (io/output-stream target-file)]
|
||||
(.write out bytes)))
|
||||
(catch Exception e
|
||||
(log/info "Problems with " url ", skip it."))))
|
||||
|
||||
(defn find-missing-covers [books-url target-dir]
|
||||
(remove #(fs/exists? (url->file % target-dir))
|
||||
|
@ -85,10 +90,10 @@
|
|||
(.write out content))))
|
||||
|
||||
(defn -main [& args]
|
||||
(let [github-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd"
|
||||
(let [datasource-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd"
|
||||
target-dir (io/file "public" "book-covers")]
|
||||
(fs/mkdirs target-dir)
|
||||
(generate-html (scrape-book-urls github-url) "public")
|
||||
(generate-html (scrape-book-urls datasource-url) "public")
|
||||
(doall (pmap #(scrape-book-cover % target-dir)
|
||||
(find-missing-covers github-url target-dir)))
|
||||
(find-missing-covers datasource-url target-dir)))
|
||||
(System/exit 0)))
|
||||
|
|
Loading…
Reference in a new issue