Now we are getting somewhere ... scraping ahead

This commit is contained in:
Aaron Mueller 2014-09-28 16:43:29 +02:00
parent 84114633ac
commit 06b298c376
10 changed files with 18 additions and 19 deletions

View file

@ -19,6 +19,7 @@
:init luduverse.handler/init :init luduverse.handler/init
:destroy luduverse.handler/destroy :destroy luduverse.handler/destroy
:open-browser? false} :open-browser? false}
:aliases {"save-competition-30" ["run" "-m" "luduverse.scraper/save-competition" "30"]}
:profiles {:production {:ring {:stacktrace? false}} :profiles {:production {:ring {:stacktrace? false}}
:dev {:ring {:auto-reload? true :dev {:ring {:auto-reload? true
:auto-refresh? true}}}) :auto-refresh? true}}})

View file

@ -1,5 +1,5 @@
(ns luduverse.core (ns luduverse.core
(:require [luduverse.ld-scraper :as scraper]) (:require [luduverse.scraper :as scraper])
(:gen-class)) (:gen-class))
(def -main []) (def -main [])

View file

@ -3,7 +3,7 @@
[korma.db :refer [defdb mysql]])) [korma.db :refer [defdb mysql]]))
(defdb db (mysql {:db "luduverse" (defdb db (mysql {:db "luduverse"
:user "root" :user "luduverse"
:password "dev"})) :password "dev"}))
(declare entities competitions users sources) (declare entities competitions users sources)

View file

@ -3,16 +3,14 @@
[compojure.route :as route] [compojure.route :as route]
[noir.util.middleware :as middleware] [noir.util.middleware :as middleware]
[luduverse.pages :as pages] [luduverse.pages :as pages]
[luduverse.ld-images :as images])) [luduverse.images :as images]))
(defroutes core-routes (defroutes core-routes
(route/resources "/") (route/resources "/")
(GET "/" [] (pages/front-page)) (GET "/" [] (pages/front-page))
(route/not-found "404, sorry.")) (route/not-found "404, sorry."))
(defn init [] (defn init [])
; Create the image resource path if it is missing
(if (not (fs/exists? (images/base-path))) (fs/mkdirs (images/base-path))))
(defn destroy []) (defn destroy [])

View file

@ -1,4 +1,4 @@
(ns luduverse.ld-images (ns luduverse.images
(:require [image-resizer.core :refer :all] (:require [image-resizer.core :refer :all]
[image-resizer.util :as utils] [image-resizer.util :as utils]
[image-resizer.format :refer :all] [image-resizer.format :refer :all]

View file

@ -1,4 +1,4 @@
(ns luduverse.db-models.competition (ns luduverse.models.competition
(:require [korma.core :refer :all] (:require [korma.core :refer :all]
[luduverse.db :refer :all])) [luduverse.db :refer :all]))

View file

@ -1,9 +1,9 @@
(ns luduverse.db-models.entry (ns luduverse.models.entry
(:require [korma.core :refer :all] (:require [korma.core :refer :all]
[luduverse.db :refer :all] [luduverse.db :refer :all]
[luduverse.db-models.user :as user] [luduverse.models.user :as user]
[luduverse.db-models.competition :as competition] [luduverse.models.competition :as competition]
[luduverse.db-models.source :as source])) [luduverse.models.source :as source]))
(defn all [] (defn all []
(select entries)) (select entries))

View file

@ -1,4 +1,4 @@
(ns luduverse.db-models.source (ns luduverse.models.source
(:require [korma.core :refer :all] (:require [korma.core :refer :all]
[luduverse.db :refer :all])) [luduverse.db :refer :all]))

View file

@ -1,4 +1,4 @@
(ns luduverse.db-models.user (ns luduverse.models.user
(:require [korma.core :refer :all] (:require [korma.core :refer :all]
[luduverse.db :refer :all])) [luduverse.db :refer :all]))

View file

@ -1,10 +1,10 @@
(ns luduverse.ld-scraper (ns luduverse.scraper
(:require [net.cgrand.enlive-html :as html] (:require [net.cgrand.enlive-html :as html]
[clojure.java.io :as io] [clojure.java.io :as io]
[clojure.string :refer [split trim]] [clojure.string :refer [split trim]]
[luduverse.ld-images :as image] [luduverse.images :as image]
[luduverse.db-models.entry :as entry] [luduverse.models.entry :as entry]
[luduverse.db-models.competition :as competition])) [luduverse.models.competition :as competition]))
;; NOTE: This part is grabbed from the open source lib ;; NOTE: This part is grabbed from the open source lib
;; https://github.com/arg-games/ldview ;; https://github.com/arg-games/ldview
@ -72,7 +72,7 @@ and simply fetch it from Wikipedia. End of discussion :)"
:images images})) :images images}))
;; Save all what we got ;; Save what we got
(defn save-entry [competition-id entry-id] (defn save-entry [competition-id entry-id]
(let [entry (entry-details competition-id entry-id)] (let [entry (entry-details competition-id entry-id)]
(if-not (competition/exists? competition-id) (competition/create! competition-id (theme competition-id))) (if-not (competition/exists? competition-id) (competition/create! competition-id (theme competition-id)))