arg/ldview
3
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Scrape the competition theme from the page

Browse source
This commit is contained in:
Aaron Mueller 2014-01-29 23:58:44 +01:00
parent ab395a49d7
commit ddcfad7fea
1 changed files with 5 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/ldview/tasks/scrape.clj
View file

@ -6,7 +6,7 @@
; Some helper functions to construct proper urls. If the Ludum Date Website ; Some helper functions to construct proper urls. If the Ludum Date Website
; changes some URL stuff, this is the place to crank. ; changes some URL stuff, this is the place to crank.
; TODO: make dynamic ; TODO: make this dynamic and replace all places in the code
(def competition 27) (def competition 27)
(defn url-action [action] (defn url-action [action]
@ -24,7 +24,6 @@
(defn fetch-url [url] (defn fetch-url [url]
(html/html-resource (java.net.URL. url))) (html/html-resource (java.net.URL. url)))
; TODO: bugfix missing images (probably)
(defn save-image-from-url [url target-file] (defn save-image-from-url [url target-file]
(with-open [bodystream (:body (http/get url {:as :stream}))] (with-open [bodystream (:body (http/get url {:as :stream}))]
(io/copy bodystream (io/file target-file)))) (io/copy bodystream (io/file target-file))))
@ -41,6 +40,10 @@
links (map #(:href %1) (map #(:attrs (first (html/select [%1] [:a]))) tds))] links (map #(:href %1) (map #(:attrs (first (html/select [%1] [:a]))) tds))]
(map #(last (split %1 #"=")) links))) (map #(last (split %1 #"=")) links)))
(defn competition-theme [competition]
(let [p (html/select (fetch-url (url-action "preview")) [:div#content :> :div.post :> :div.entry :> :p html/first-child :a])]
(first (:content (first p)))))
(defn links-on-entry [content] (defn links-on-entry [content]
(map (fn [x] {:title (first (:content x)) :url (:href (:attrs x))}) (html/select [content] [:p.links :> :a]))) (map (fn [x] {:title (first (:content x)) :url (:href (:attrs x))}) (html/select [content] [:p.links :> :a])))
@ -64,5 +67,3 @@
:type (format-entry-type unformatted-type) :type (format-entry-type unformatted-type)
:links links :links links
:images images})) :images images}))
; TODO: check for -> scrape sources from game, too (WIN, MAC, LINUX,...)