diff --git a/project.clj b/project.clj index c56a670..8111518 100644 --- a/project.clj +++ b/project.clj @@ -1,9 +1,10 @@ -(defproject buchdesmonats "1.1" +(defproject buchdesmonats "1.2" :description "A simple tool to fetch all books of the month from the okoyono.de project." :url "https://git.okoyono.de/mezzomix/buch_des_monats" :license {:name "MIT License" :url "http://opensource.org/licenses/MIT"} :dependencies [[org.clojure/clojure "1.6.0"] + [org.clojure/tools.logging "0.3.1"] [enlive "1.1.5"] [me.raynes/fs "1.4.5"] [clj-http "0.9.2"]] diff --git a/public/styles.css b/public/styles.css index ca9afcb..3754573 100644 --- a/public/styles.css +++ b/public/styles.css @@ -1,7 +1,7 @@ @import url(http://fonts.googleapis.com/css?family=Gilda+Display); body { - background-color: #CEC3A7; + background-color: #f5efdb; } h1 { diff --git a/src/buchdesmonats/core.clj b/src/buchdesmonats/core.clj index 40659a3..f89382b 100644 --- a/src/buchdesmonats/core.clj +++ b/src/buchdesmonats/core.clj @@ -1,4 +1,4 @@ -;;; Copyright (C) 2014 Aaron Mueller +;;; Copyright (C) 2014-2015 Aaron Fischer ;;; ;;; Permission is hereby granted, free of charge, to any person obtaining a copy of ;;; this software and associated documentation files (the "Software"), to deal in @@ -23,6 +23,7 @@ [clj-http.client :as http-client] [clojure.string :as str] [clojure.java.io :as io] + [clojure.tools.logging :as log] [me.raynes.fs :as fs])) (defn imgurl->bytes [lovelybooks-url] @@ -51,18 +52,22 @@ (str/replace #"[^a-z0-9-_.]" "") (#(io/file target-dir %))))) -(defn scrape-book-urls [github-url] - (->> (http-client/get github-url) +(defn scrape-book-urls [datasource-url] + (->> (http-client/get datasource-url {:insecure? true}) :body str/split-lines (map #(second (re-find #"^\* .*\[.+\]\((.+)\)" %))) (remove nil?))) (defn scrape-book-cover [url target-dir] - (let [target-file (url->file url target-dir) - encoded-url (encode-url url)] - (with-open [out (io/output-stream target-file)] - (.write out (imgurl->bytes encoded-url))))) + (try + (let [target-file (url->file url target-dir) + encoded-url (encode-url url) + bytes (imgurl->bytes encoded-url)] + (with-open [out (io/output-stream target-file)] + (.write out bytes))) + (catch Exception e + (log/info "Problems with " url ", skip it.")))) (defn find-missing-covers [books-url target-dir] (remove #(fs/exists? (url->file % target-dir)) @@ -85,10 +90,10 @@ (.write out content)))) (defn -main [& args] - (let [github-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd" + (let [datasource-url "https://git.okoyono.de/mezzomix/buch_des_monats/raw/master/README.mkd" target-dir (io/file "public" "book-covers")] (fs/mkdirs target-dir) - (generate-html (scrape-book-urls github-url) "public") + (generate-html (scrape-book-urls datasource-url) "public") (doall (pmap #(scrape-book-cover % target-dir) - (find-missing-covers github-url target-dir))) + (find-missing-covers datasource-url target-dir))) (System/exit 0)))