From c8c534c9ca4652009154d6e78d969c524186fe79 Mon Sep 17 00:00:00 2001
From: Aaron Fischer <mail@aaron-fischer.net>
Date: Wed, 10 Jun 2015 23:16:32 +0200
Subject: [PATCH] Find broken URLs

---
 project.clj                |  3 ++-
 src/buchdesmonats/core.clj | 19 ++++++++++++-------
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/project.clj b/project.clj
index c56a670..8111518 100644
--- a/project.clj
+++ b/project.clj
@@ -1,9 +1,10 @@
-(defproject buchdesmonats "1.1"
+(defproject buchdesmonats "1.2"
   :description "A simple tool to fetch all books of the month from the okoyono.de project."
   :url "https://git.okoyono.de/mezzomix/buch_des_monats"
   :license {:name "MIT License"
             :url "http://opensource.org/licenses/MIT"}
   :dependencies [[org.clojure/clojure "1.6.0"]
+                 [org.clojure/tools.logging "0.3.1"]
                  [enlive "1.1.5"]
                  [me.raynes/fs "1.4.5"]
                  [clj-http "0.9.2"]]
diff --git a/src/buchdesmonats/core.clj b/src/buchdesmonats/core.clj
index 74ae32b..f89382b 100644
--- a/src/buchdesmonats/core.clj
+++ b/src/buchdesmonats/core.clj
@@ -23,6 +23,7 @@
             [clj-http.client :as http-client]
             [clojure.string :as str]
             [clojure.java.io :as io]
+            [clojure.tools.logging :as log]
             [me.raynes.fs :as fs]))
 
 (defn imgurl->bytes [lovelybooks-url]
@@ -59,10 +60,14 @@
        (remove nil?)))
 
 (defn scrape-book-cover [url target-dir]
-  (let [target-file (url->file url target-dir)
-        encoded-url (encode-url url)]
-    (with-open [out (io/output-stream target-file)]
-      (.write out (imgurl->bytes encoded-url)))))
+  (try
+    (let [target-file (url->file url target-dir)
+          encoded-url (encode-url url)
+          bytes (imgurl->bytes encoded-url)]
+      (with-open [out (io/output-stream target-file)]
+        (.write out bytes)))
+    (catch Exception e
+      (log/info "Problems with " url ", skip it."))))
 
 (defn find-missing-covers [books-url target-dir]
   (remove #(fs/exists? (url->file % target-dir))
@@ -85,10 +90,10 @@
       (.write out content))))
  
 (defn -main [& args]
-  (let [github-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd"
+  (let [datasource-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd"
         target-dir (io/file "public" "book-covers")]
     (fs/mkdirs target-dir)
-    (generate-html (scrape-book-urls github-url) "public")
+    (generate-html (scrape-book-urls datasource-url) "public")
     (doall (pmap #(scrape-book-cover % target-dir)
-                 (find-missing-covers github-url target-dir)))
+                 (find-missing-covers datasource-url target-dir)))
     (System/exit 0)))