From a758553a11c9eb503bfbab7c9aedf8bb98cb8729 Mon Sep 17 00:00:00 2001 From: Aaron Fischer Date: Wed, 10 Jun 2015 23:14:04 +0200 Subject: [PATCH 1/4] Pimp some styles --- public/styles.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/styles.css b/public/styles.css index ca9afcb..3754573 100644 --- a/public/styles.css +++ b/public/styles.css @@ -1,7 +1,7 @@ @import url(http://fonts.googleapis.com/css?family=Gilda+Display); body { - background-color: #CEC3A7; + background-color: #f5efdb; } h1 { From 9e0643e6dea338601893770c1aa5e883c87a9eee Mon Sep 17 00:00:00 2001 From: Aaron Fischer Date: Wed, 10 Jun 2015 23:14:16 +0200 Subject: [PATCH 2/4] Correct the copyright --- src/buchdesmonats/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/buchdesmonats/core.clj b/src/buchdesmonats/core.clj index 40659a3..7550047 100644 --- a/src/buchdesmonats/core.clj +++ b/src/buchdesmonats/core.clj @@ -1,4 +1,4 @@ -;;; Copyright (C) 2014 Aaron Mueller +;;; Copyright (C) 2014-2015 Aaron Fischer ;;; ;;; Permission is hereby granted, free of charge, to any person obtaining a copy of ;;; this software and associated documentation files (the "Software"), to deal in From 0b5a63ba8613169992b5a17602591c84bd0f048d Mon Sep 17 00:00:00 2001 From: Aaron Fischer Date: Wed, 10 Jun 2015 23:15:53 +0200 Subject: [PATCH 3/4] Skip SSL warnings --- src/buchdesmonats/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/buchdesmonats/core.clj b/src/buchdesmonats/core.clj index 7550047..74ae32b 100644 --- a/src/buchdesmonats/core.clj +++ b/src/buchdesmonats/core.clj @@ -51,8 +51,8 @@ (str/replace #"[^a-z0-9-_.]" "") (#(io/file target-dir %))))) -(defn scrape-book-urls [github-url] - (->> (http-client/get github-url) +(defn scrape-book-urls [datasource-url] + (->> (http-client/get datasource-url {:insecure? true}) :body str/split-lines (map #(second (re-find #"^\* .*\[.+\]\((.+)\)" %))) From c8c534c9ca4652009154d6e78d969c524186fe79 Mon Sep 17 00:00:00 2001 From: Aaron Fischer Date: Wed, 10 Jun 2015 23:16:32 +0200 Subject: [PATCH 4/4] Find broken URLs --- project.clj | 3 ++- src/buchdesmonats/core.clj | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/project.clj b/project.clj index c56a670..8111518 100644 --- a/project.clj +++ b/project.clj @@ -1,9 +1,10 @@ -(defproject buchdesmonats "1.1" +(defproject buchdesmonats "1.2" :description "A simple tool to fetch all books of the month from the okoyono.de project." :url "https://git.okoyono.de/mezzomix/buch_des_monats" :license {:name "MIT License" :url "http://opensource.org/licenses/MIT"} :dependencies [[org.clojure/clojure "1.6.0"] + [org.clojure/tools.logging "0.3.1"] [enlive "1.1.5"] [me.raynes/fs "1.4.5"] [clj-http "0.9.2"]] diff --git a/src/buchdesmonats/core.clj b/src/buchdesmonats/core.clj index 74ae32b..f89382b 100644 --- a/src/buchdesmonats/core.clj +++ b/src/buchdesmonats/core.clj @@ -23,6 +23,7 @@ [clj-http.client :as http-client] [clojure.string :as str] [clojure.java.io :as io] + [clojure.tools.logging :as log] [me.raynes.fs :as fs])) (defn imgurl->bytes [lovelybooks-url] @@ -59,10 +60,14 @@ (remove nil?))) (defn scrape-book-cover [url target-dir] - (let [target-file (url->file url target-dir) - encoded-url (encode-url url)] - (with-open [out (io/output-stream target-file)] - (.write out (imgurl->bytes encoded-url))))) + (try + (let [target-file (url->file url target-dir) + encoded-url (encode-url url) + bytes (imgurl->bytes encoded-url)] + (with-open [out (io/output-stream target-file)] + (.write out bytes))) + (catch Exception e + (log/info "Problems with " url ", skip it.")))) (defn find-missing-covers [books-url target-dir] (remove #(fs/exists? (url->file % target-dir)) @@ -85,10 +90,10 @@ (.write out content)))) (defn -main [& args] - (let [github-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd" + (let [datasource-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd" target-dir (io/file "public" "book-covers")] (fs/mkdirs target-dir) - (generate-html (scrape-book-urls github-url) "public") + (generate-html (scrape-book-urls datasource-url) "public") (doall (pmap #(scrape-book-cover % target-dir) - (find-missing-covers github-url target-dir))) + (find-missing-covers datasource-url target-dir))) (System/exit 0)))