From c8c534c9ca4652009154d6e78d969c524186fe79 Mon Sep 17 00:00:00 2001 From: Aaron Fischer Date: Wed, 10 Jun 2015 23:16:32 +0200 Subject: [PATCH] Find broken URLs --- project.clj | 3 ++- src/buchdesmonats/core.clj | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/project.clj b/project.clj index c56a670..8111518 100644 --- a/project.clj +++ b/project.clj @@ -1,9 +1,10 @@ -(defproject buchdesmonats "1.1" +(defproject buchdesmonats "1.2" :description "A simple tool to fetch all books of the month from the okoyono.de project." :url "https://git.okoyono.de/mezzomix/buch_des_monats" :license {:name "MIT License" :url "http://opensource.org/licenses/MIT"} :dependencies [[org.clojure/clojure "1.6.0"] + [org.clojure/tools.logging "0.3.1"] [enlive "1.1.5"] [me.raynes/fs "1.4.5"] [clj-http "0.9.2"]] diff --git a/src/buchdesmonats/core.clj b/src/buchdesmonats/core.clj index 74ae32b..f89382b 100644 --- a/src/buchdesmonats/core.clj +++ b/src/buchdesmonats/core.clj @@ -23,6 +23,7 @@ [clj-http.client :as http-client] [clojure.string :as str] [clojure.java.io :as io] + [clojure.tools.logging :as log] [me.raynes.fs :as fs])) (defn imgurl->bytes [lovelybooks-url] @@ -59,10 +60,14 @@ (remove nil?))) (defn scrape-book-cover [url target-dir] - (let [target-file (url->file url target-dir) - encoded-url (encode-url url)] - (with-open [out (io/output-stream target-file)] - (.write out (imgurl->bytes encoded-url))))) + (try + (let [target-file (url->file url target-dir) + encoded-url (encode-url url) + bytes (imgurl->bytes encoded-url)] + (with-open [out (io/output-stream target-file)] + (.write out bytes))) + (catch Exception e + (log/info "Problems with " url ", skip it.")))) (defn find-missing-covers [books-url target-dir] (remove #(fs/exists? (url->file % target-dir)) @@ -85,10 +90,10 @@ (.write out content)))) (defn -main [& args] - (let [github-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd" + (let [datasource-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd" target-dir (io/file "public" "book-covers")] (fs/mkdirs target-dir) - (generate-html (scrape-book-urls github-url) "public") + (generate-html (scrape-book-urls datasource-url) "public") (doall (pmap #(scrape-book-cover % target-dir) - (find-missing-covers github-url target-dir))) + (find-missing-covers datasource-url target-dir))) (System/exit 0)))