arg/ldview
3
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fix some huge bugs. The scraper is now stable and usefull

Browse source
This commit is contained in:
Aaron Mueller 2014-01-31 00:02:26 +01:00
parent a3a0c7af86
commit 86f9a4cb3f
5 changed files with 52 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/ldview/routes/home.clj
View file

@ -4,8 +4,10 @@
[ldview.util :as util]
[ldview.models.entry :as entry]))
; TODO: Make dropdown or something to choose the competition-id
(defn entries-page []
(layout/render "entries.html" {:entries (entry/all)}))
(layout/render "entries.html" {:entries (entry/all)
:competition-id 28}))
(defroutes home-routes
(GET "/" [] (entries-page)))

7
src/ldview/tasks/images.clj
View file

@ -6,10 +6,11 @@
[clojure.java.io :as io]
[noir.io :as noir-io]))
(def base-path (str (noir-io/resource-path) "img/ld/"))
(defn base-path [competition-id]
(str (noir-io/resource-path) "img/ld" competition-id "/"))
(defn image-name [folder entry-id number]
(str base-path folder "/" entry-id "_" number ".png"))
(defn image-name [competition-id folder entry-id number]
(str (base-path competition-id) folder "/" entry-id "_" number ".png"))
(defn to-square [file new-size]
(let [[width height] (dimensions (utils/buffered-image file))]

68
src/ldview/tasks/runner.clj
View file

@ -4,35 +4,53 @@
[me.raynes.fs :as fs]
[ldview.models.entry :as entry]
[ldview.models.competition :as competition]
[ldview.models.schema :as schema]))
[ldview.models.schema :as schema]
[noir.io :as io]))
(defn cleanup! []
(if (fs/exists? images/base-path)
(fs/delete-dir images/base-path))
(fs/mkdirs (str images/base-path "/thumbs/"))
(fs/mkdirs (str images/base-path "/fullscreen/"))
(fs/mkdirs (str images/base-path "/raw/")))
(defn delete-database! []
(let [db-file (str (io/resource-path) schema/db-store)]
(if (fs/exists? db-file) (fs/delete db-file))))
(defn save-entry! [new-entry]
(entry/create! new-entry)
(if (:images new-entry)
(map (fn [image-url]
(let [id (:ld_uid new-entry)
number (last (first (re-seq #"shot([0-9]+)" image-url)))
raw-image-path (images/image-name "raw" id number)]
(scrape/save-image-from-url image-url raw-image-path)
(images/sourceimage->fullscreen raw-image-path (images/image-name "fullscreen" id number))
(images/sourceimage->thumb raw-image-path (images/image-name "thumbs" id number))))
(:images new-entry))))
(defn initialize-file-structure! [competition-id]
(let [path (images/base-path competition-id)]
(if (fs/exists? path) (fs/delete-dir path))
(fs/mkdirs (str path "/thumbs/"))
(fs/mkdirs (str path "/fullscreen/"))
(fs/mkdirs (str path "/raw/"))))
; TODO: Make it multithreaded
(defn fetch-all-content [competition-id]
(let [pages (range 5)] ;(scrape/number-of-pages competition-id)]
(doseq [page pages]
(doseq [ld-uid (scrape/entries-on-page competition-id (inc page))]
(if-not (entry/exists? ld-uid)
(save-entry! (scrape/entry-details competition-id ld-uid)))))))
(defn save-entry! [competition-id new-entry]
(if-not (entry/exists? (:ld_uid new-entry))
(entry/create! new-entry))
(doseq [image-url (:images new-entry)]
(let [id (:ld_uid new-entry)
number (last (first (re-seq #"shot([0-9]+)" image-url)))
raw-image-path (images/image-name competition-id "raw" id number)]
(if-not (fs/exists? raw-image-path)
(do
(scrape/save-image-from-url image-url raw-image-path)
(images/sourceimage->fullscreen raw-image-path (images/image-name competition-id "fullscreen" id number))
(images/sourceimage->thumb raw-image-path (images/image-name competition-id "thumbs" id number)))))))
(defn fetch-all-content
([]
(fetch-all-content (competition/latest)))
([competition-id]
(fetch-all-content competition-id (dec (scrape/number-of-pages competition-id))))
([competition-id pages]
; TODO: Make it multithreaded
(doseq [page (range pages)]
(doseq [ld-uid (scrape/entries-on-page competition-id (inc page))]
(save-entry! competition-id (scrape/entry-details competition-id ld-uid))))))
(defn load-competition [competition-id]
(if-not (schema/initialized?)
(schema/create-initial-tables))
(initialize-file-structure! competition-id)
(competition/create! competition-id (scrape/theme competition-id))
(fetch-all-content competition-id))
(defn wipe-all!! []
(delete-database!)
(schema/create-initial-tables))

2
src/ldview/tasks/scrape.clj
View file

@ -8,7 +8,7 @@
(str "http://www.ludumdare.com/compo/ludum-dare-" competition-id "/?action=" action))
(defn url-page [competition-id page]
(str (url-action competition-id "preview") "&start=" page))
(str (url-action competition-id "preview") "&start=" (* page 24)))
(defn url-entry [competition-id entry-id]
(str (url-action competition-id "preview") "&uid=" entry-id))

2
src/ldview/views/templates/entries.html
View file

@ -8,7 +8,7 @@
<em>{{entry.author.name}}</em>
</div>
<div class="details"></div>
<a href="http://www.ludumdare.com/compo/ludum-dare-27/?action=preview&uid={{entry.ld_uid}}"><img src="{{servlet-context}}/img/ld/thumbs/{{entry.ld_uid}}_0.png" alt="{{entry.title}}" /></a>
<a href="http://www.ludumdare.com/compo/ludum-dare-27/?action=preview&uid={{entry.ld_uid}}"><img src="{{servlet-context}}/img/ld{{competition-id}}/thumbs/{{entry.ld_uid}}_0.png" alt="{{entry.title}}" /></a>
</div>
{% endfor %}
</div>