Fix some huge bugs. The scraper is now stable and usefull

This commit is contained in:
Aaron Mueller 2014-01-31 00:02:26 +01:00
parent a3a0c7af86
commit 86f9a4cb3f
5 changed files with 52 additions and 31 deletions

View file

@ -4,8 +4,10 @@
[ldview.util :as util] [ldview.util :as util]
[ldview.models.entry :as entry])) [ldview.models.entry :as entry]))
; TODO: Make dropdown or something to choose the competition-id
(defn entries-page [] (defn entries-page []
(layout/render "entries.html" {:entries (entry/all)})) (layout/render "entries.html" {:entries (entry/all)
:competition-id 28}))
(defroutes home-routes (defroutes home-routes
(GET "/" [] (entries-page))) (GET "/" [] (entries-page)))

View file

@ -6,10 +6,11 @@
[clojure.java.io :as io] [clojure.java.io :as io]
[noir.io :as noir-io])) [noir.io :as noir-io]))
(def base-path (str (noir-io/resource-path) "img/ld/")) (defn base-path [competition-id]
(str (noir-io/resource-path) "img/ld" competition-id "/"))
(defn image-name [folder entry-id number] (defn image-name [competition-id folder entry-id number]
(str base-path folder "/" entry-id "_" number ".png")) (str (base-path competition-id) folder "/" entry-id "_" number ".png"))
(defn to-square [file new-size] (defn to-square [file new-size]
(let [[width height] (dimensions (utils/buffered-image file))] (let [[width height] (dimensions (utils/buffered-image file))]

View file

@ -4,35 +4,53 @@
[me.raynes.fs :as fs] [me.raynes.fs :as fs]
[ldview.models.entry :as entry] [ldview.models.entry :as entry]
[ldview.models.competition :as competition] [ldview.models.competition :as competition]
[ldview.models.schema :as schema])) [ldview.models.schema :as schema]
[noir.io :as io]))
(defn cleanup! [] (defn delete-database! []
(if (fs/exists? images/base-path) (let [db-file (str (io/resource-path) schema/db-store)]
(fs/delete-dir images/base-path)) (if (fs/exists? db-file) (fs/delete db-file))))
(fs/mkdirs (str images/base-path "/thumbs/"))
(fs/mkdirs (str images/base-path "/fullscreen/"))
(fs/mkdirs (str images/base-path "/raw/")))
(defn save-entry! [new-entry] (defn initialize-file-structure! [competition-id]
(entry/create! new-entry) (let [path (images/base-path competition-id)]
(if (:images new-entry) (if (fs/exists? path) (fs/delete-dir path))
(map (fn [image-url] (fs/mkdirs (str path "/thumbs/"))
(fs/mkdirs (str path "/fullscreen/"))
(fs/mkdirs (str path "/raw/"))))
(defn save-entry! [competition-id new-entry]
(if-not (entry/exists? (:ld_uid new-entry))
(entry/create! new-entry))
(doseq [image-url (:images new-entry)]
(let [id (:ld_uid new-entry) (let [id (:ld_uid new-entry)
number (last (first (re-seq #"shot([0-9]+)" image-url))) number (last (first (re-seq #"shot([0-9]+)" image-url)))
raw-image-path (images/image-name "raw" id number)] raw-image-path (images/image-name competition-id "raw" id number)]
(if-not (fs/exists? raw-image-path)
(do
(scrape/save-image-from-url image-url raw-image-path) (scrape/save-image-from-url image-url raw-image-path)
(images/sourceimage->fullscreen raw-image-path (images/image-name "fullscreen" id number)) (images/sourceimage->fullscreen raw-image-path (images/image-name competition-id "fullscreen" id number))
(images/sourceimage->thumb raw-image-path (images/image-name "thumbs" id number)))) (images/sourceimage->thumb raw-image-path (images/image-name competition-id "thumbs" id number)))))))
(:images new-entry))))
(defn fetch-all-content
([]
(fetch-all-content (competition/latest)))
([competition-id]
(fetch-all-content competition-id (dec (scrape/number-of-pages competition-id))))
([competition-id pages]
; TODO: Make it multithreaded ; TODO: Make it multithreaded
(defn fetch-all-content [competition-id] (doseq [page (range pages)]
(let [pages (range 5)] ;(scrape/number-of-pages competition-id)]
(doseq [page pages]
(doseq [ld-uid (scrape/entries-on-page competition-id (inc page))] (doseq [ld-uid (scrape/entries-on-page competition-id (inc page))]
(if-not (entry/exists? ld-uid) (save-entry! competition-id (scrape/entry-details competition-id ld-uid))))))
(save-entry! (scrape/entry-details competition-id ld-uid)))))))
(defn load-competition [competition-id] (defn load-competition [competition-id]
(if-not (schema/initialized?)
(schema/create-initial-tables))
(initialize-file-structure! competition-id)
(competition/create! competition-id (scrape/theme competition-id)) (competition/create! competition-id (scrape/theme competition-id))
(fetch-all-content competition-id)) (fetch-all-content competition-id))
(defn wipe-all!! []
(delete-database!)
(schema/create-initial-tables))

View file

@ -8,7 +8,7 @@
(str "http://www.ludumdare.com/compo/ludum-dare-" competition-id "/?action=" action)) (str "http://www.ludumdare.com/compo/ludum-dare-" competition-id "/?action=" action))
(defn url-page [competition-id page] (defn url-page [competition-id page]
(str (url-action competition-id "preview") "&start=" page)) (str (url-action competition-id "preview") "&start=" (* page 24)))
(defn url-entry [competition-id entry-id] (defn url-entry [competition-id entry-id]
(str (url-action competition-id "preview") "&uid=" entry-id)) (str (url-action competition-id "preview") "&uid=" entry-id))

View file

@ -8,7 +8,7 @@
<em>{{entry.author.name}}</em> <em>{{entry.author.name}}</em>
</div> </div>
<div class="details"></div> <div class="details"></div>
<a href="http://www.ludumdare.com/compo/ludum-dare-27/?action=preview&uid={{entry.ld_uid}}"><img src="{{servlet-context}}/img/ld/thumbs/{{entry.ld_uid}}_0.png" alt="{{entry.title}}" /></a> <a href="http://www.ludumdare.com/compo/ludum-dare-27/?action=preview&uid={{entry.ld_uid}}"><img src="{{servlet-context}}/img/ld{{competition-id}}/thumbs/{{entry.ld_uid}}_0.png" alt="{{entry.title}}" /></a>
</div> </div>
{% endfor %} {% endfor %}
</div> </div>