Fix some huge bugs. The scraper is now stable and usefull

src/ldview/routes/home.clj

@@ -4,8 +4,10 @@
             [ldview.util :as util]
             [ldview.models.entry :as entry]))
 
+; TODO: Make dropdown or something to choose the competition-id
 (defn entries-page []
-  (layout/render "entries.html" {:entries (entry/all)}))
+  (layout/render "entries.html" {:entries (entry/all)
+                                 :competition-id 28}))
 
 (defroutes home-routes
   (GET "/" [] (entries-page)))

src/ldview/tasks/images.clj

@@ -6,10 +6,11 @@
             [clojure.java.io :as io]
             [noir.io :as noir-io]))
 
-(def base-path (str (noir-io/resource-path) "img/ld/"))
+(defn base-path [competition-id]
+  (str (noir-io/resource-path) "img/ld" competition-id "/"))
 
-(defn image-name [folder entry-id number]
+(defn image-name [competition-id folder entry-id number]
-  (str base-path folder "/" entry-id "_" number ".png"))
+  (str (base-path competition-id) folder "/" entry-id "_" number ".png"))
 
 (defn to-square [file new-size]
   (let [[width height] (dimensions (utils/buffered-image file))]

src/ldview/tasks/runner.clj

@@ -4,35 +4,53 @@
             [me.raynes.fs :as fs]
             [ldview.models.entry :as entry]
             [ldview.models.competition :as competition]
-            [ldview.models.schema :as schema]))
+            [ldview.models.schema :as schema]
+            [noir.io :as io]))
 
-(defn cleanup! []
+(defn delete-database! []
-  (if (fs/exists? images/base-path)
+  (let [db-file (str (io/resource-path) schema/db-store)]
-    (fs/delete-dir images/base-path))
+    (if (fs/exists? db-file) (fs/delete db-file))))
-  (fs/mkdirs (str images/base-path "/thumbs/"))
-  (fs/mkdirs (str images/base-path "/fullscreen/"))
-  (fs/mkdirs (str images/base-path "/raw/")))
 
-(defn save-entry! [new-entry]
+(defn initialize-file-structure! [competition-id]
-  (entry/create! new-entry)
+  (let [path (images/base-path competition-id)]
-  (if (:images new-entry)
+    (if (fs/exists? path) (fs/delete-dir path))
-    (map (fn [image-url]
+    (fs/mkdirs (str path "/thumbs/"))
+    (fs/mkdirs (str path "/fullscreen/"))
+    (fs/mkdirs (str path "/raw/"))))
+
+(defn save-entry! [competition-id new-entry]
+  (if-not (entry/exists? (:ld_uid new-entry))
+    (entry/create! new-entry))
+
+  (doseq [image-url (:images new-entry)]
     (let [id (:ld_uid new-entry)
           number (last (first (re-seq #"shot([0-9]+)" image-url)))
-                 raw-image-path (images/image-name "raw" id number)]
+          raw-image-path (images/image-name competition-id "raw" id number)]
+      (if-not (fs/exists? raw-image-path)
+        (do
           (scrape/save-image-from-url image-url raw-image-path)
-             (images/sourceimage->fullscreen raw-image-path (images/image-name "fullscreen" id number))
+          (images/sourceimage->fullscreen raw-image-path (images/image-name competition-id "fullscreen" id number))
-             (images/sourceimage->thumb raw-image-path (images/image-name "thumbs" id number))))
+          (images/sourceimage->thumb raw-image-path (images/image-name competition-id "thumbs" id number)))))))
-         (:images new-entry))))
 
-; TODO: Make it multithreaded
+(defn fetch-all-content
-(defn fetch-all-content [competition-id]
+  ([]
-  (let [pages (range 5)] ;(scrape/number-of-pages competition-id)]
+     (fetch-all-content (competition/latest)))
-    (doseq [page pages]
+  ([competition-id]
+     (fetch-all-content competition-id (dec (scrape/number-of-pages competition-id))))
+  ([competition-id pages]
+     ; TODO: Make it multithreaded
+     (doseq [page (range pages)]
        (doseq [ld-uid (scrape/entries-on-page competition-id (inc page))]
-        (if-not (entry/exists? ld-uid)
+         (save-entry! competition-id (scrape/entry-details competition-id ld-uid))))))
-          (save-entry! (scrape/entry-details competition-id ld-uid)))))))
 
 (defn load-competition [competition-id]
+  (if-not (schema/initialized?)
+    (schema/create-initial-tables))
+  (initialize-file-structure! competition-id)
+
   (competition/create! competition-id (scrape/theme competition-id))
   (fetch-all-content competition-id))
+
+(defn wipe-all!! []
+  (delete-database!)
+  (schema/create-initial-tables))

src/ldview/tasks/scrape.clj

@@ -8,7 +8,7 @@
   (str "http://www.ludumdare.com/compo/ludum-dare-" competition-id "/?action=" action))
 
 (defn url-page [competition-id page]
-  (str (url-action competition-id "preview") "&start=" page))
+  (str (url-action competition-id "preview") "&start=" (* page 24)))
 
 (defn url-entry [competition-id entry-id]
   (str (url-action competition-id "preview") "&uid=" entry-id))

src/ldview/views/templates/entries.html

@@ -8,7 +8,7 @@
       <em>{{entry.author.name}}</em>
     </div>
     <div class="details"></div>
-    <a href="http://www.ludumdare.com/compo/ludum-dare-27/?action=preview&uid={{entry.ld_uid}}"><img src="{{servlet-context}}/img/ld/thumbs/{{entry.ld_uid}}_0.png" alt="{{entry.title}}" /></a>
+    <a href="http://www.ludumdare.com/compo/ludum-dare-27/?action=preview&uid={{entry.ld_uid}}"><img src="{{servlet-context}}/img/ld{{competition-id}}/thumbs/{{entry.ld_uid}}_0.png" alt="{{entry.title}}" /></a>
   </div>
   {% endfor %}
 </div>