diff --git a/src/ldview/models/entry.clj b/src/ldview/models/entry.clj index 39667e3..c747254 100644 --- a/src/ldview/models/entry.clj +++ b/src/ldview/models/entry.clj @@ -26,6 +26,7 @@ :ld_uid (:ld_uid fields) :title (:title fields) :description (:description fields) + :type (:type fields) :user_id (:id username) :competition_id competition-id}))) (for [link (:links fields)] diff --git a/src/ldview/models/schema.clj b/src/ldview/models/schema.clj index 62661eb..f0af090 100644 --- a/src/ldview/models/schema.clj +++ b/src/ldview/models/schema.clj @@ -19,6 +19,7 @@ [:ld_uid "INTEGER"] [:title "VARCHAR(255)"] [:description "TEXT"] + [:type "VARCHAR(255)"] [:user_id "INTEGER"] [:competition_id "INTEGER"]) diff --git a/src/ldview/models/source.clj b/src/ldview/models/source.clj index d6d5c80..c7452b5 100644 --- a/src/ldview/models/source.clj +++ b/src/ldview/models/source.clj @@ -2,9 +2,9 @@ (:use korma.core [ldview.models.db])) -(defn create! [entry_id uri description] - (if (empty? (select sources (where {:entry_id entry_id :type description}))) +(defn create! [entry_id uri title] +; (if (empty? (select sources (where {:entry_id entry_id :type title}))) (insert sources (values {:entry_id entry_id :uri uri - :type description})))) + :type title})));) diff --git a/src/ldview/tasks/runner.clj b/src/ldview/tasks/runner.clj index d1c7548..bb880df 100644 --- a/src/ldview/tasks/runner.clj +++ b/src/ldview/tasks/runner.clj @@ -24,6 +24,7 @@ (images/sourceimage->fullscreen raw-image-path (images/image-name "fullscreen" id number)) (images/sourceimage->thumb raw-image-path (images/image-name "thumbs" id number)))) (:images new-entry)))) +; TODO: save 'links' to 'sources' in DB (defn fetch-all-content [] (let [pages (range 1)] ;(scrape/number-of-pages)] diff --git a/src/ldview/tasks/scrape.clj b/src/ldview/tasks/scrape.clj index 6b5e385..e2911b7 100644 --- a/src/ldview/tasks/scrape.clj +++ b/src/ldview/tasks/scrape.clj @@ -6,6 +6,7 @@ ; Some helper functions to construct proper urls. If the Ludum Date Website ; changes some URL stuff, this is the place to crank. +; TODO: make dynamic (def competition 27) (defn url-action [action] @@ -23,6 +24,7 @@ (defn fetch-url [url] (html/html-resource (java.net.URL. url))) +; TODO: bugfix missing images (probably) (defn save-image-from-url [url target-file] (with-open [bodystream (:body (http/get url {:as :stream}))] (io/copy bodystream (io/file target-file)))) @@ -45,10 +47,13 @@ (defn images-on-entry [content] (map #(:href (:attrs %1)) (html/select [content] [:table html/first-child :a]))) +(defn format-entry-type [unformated-type] + (if (= unformated-type "Jam Entry") "jam" "compo")) + (defn entry-details [entry-id] (let [content (first (html/select (fetch-url (url-entry entry-id)) [:div#compo2])) title-parts (first (html/texts (html/select [content] [:h3]))) - [title author] (split title-parts #" - ") + [title author type] (split title-parts #" - ") links (links-on-entry content) description (html/text (nth (html/select [content] [:p]) 2)) images (images-on-entry content)] @@ -56,6 +61,8 @@ :title title :description description :author author + :type (format-entry-type type) :links links :images images})) +; TODO: check for -> scrape sources from game, too (WIN, MAC, LINUX,...)