Now we are getting somewhere ... scraping ahead

This commit is contained in:
Aaron Mueller 2014-09-28 16:43:29 +02:00
parent 84114633ac
commit 06b298c376
10 changed files with 18 additions and 19 deletions

View file

@ -19,6 +19,7 @@
:init luduverse.handler/init
:destroy luduverse.handler/destroy
:open-browser? false}
:aliases {"save-competition-30" ["run" "-m" "luduverse.scraper/save-competition" "30"]}
:profiles {:production {:ring {:stacktrace? false}}
:dev {:ring {:auto-reload? true
:auto-refresh? true}}})

View file

@ -1,5 +1,5 @@
(ns luduverse.core
(:require [luduverse.ld-scraper :as scraper])
(:require [luduverse.scraper :as scraper])
(:gen-class))
(def -main [])

View file

@ -3,7 +3,7 @@
[korma.db :refer [defdb mysql]]))
(defdb db (mysql {:db "luduverse"
:user "root"
:user "luduverse"
:password "dev"}))
(declare entities competitions users sources)

View file

@ -3,16 +3,14 @@
[compojure.route :as route]
[noir.util.middleware :as middleware]
[luduverse.pages :as pages]
[luduverse.ld-images :as images]))
[luduverse.images :as images]))
(defroutes core-routes
(route/resources "/")
(GET "/" [] (pages/front-page))
(route/not-found "404, sorry."))
(defn init []
; Create the image resource path if it is missing
(if (not (fs/exists? (images/base-path))) (fs/mkdirs (images/base-path))))
(defn init [])
(defn destroy [])

View file

@ -1,4 +1,4 @@
(ns luduverse.ld-images
(ns luduverse.images
(:require [image-resizer.core :refer :all]
[image-resizer.util :as utils]
[image-resizer.format :refer :all]

View file

@ -1,4 +1,4 @@
(ns luduverse.db-models.competition
(ns luduverse.models.competition
(:require [korma.core :refer :all]
[luduverse.db :refer :all]))

View file

@ -1,9 +1,9 @@
(ns luduverse.db-models.entry
(ns luduverse.models.entry
(:require [korma.core :refer :all]
[luduverse.db :refer :all]
[luduverse.db-models.user :as user]
[luduverse.db-models.competition :as competition]
[luduverse.db-models.source :as source]))
[luduverse.models.user :as user]
[luduverse.models.competition :as competition]
[luduverse.models.source :as source]))
(defn all []
(select entries))

View file

@ -1,4 +1,4 @@
(ns luduverse.db-models.source
(ns luduverse.models.source
(:require [korma.core :refer :all]
[luduverse.db :refer :all]))

View file

@ -1,4 +1,4 @@
(ns luduverse.db-models.user
(ns luduverse.models.user
(:require [korma.core :refer :all]
[luduverse.db :refer :all]))

View file

@ -1,10 +1,10 @@
(ns luduverse.ld-scraper
(ns luduverse.scraper
(:require [net.cgrand.enlive-html :as html]
[clojure.java.io :as io]
[clojure.string :refer [split trim]]
[luduverse.ld-images :as image]
[luduverse.db-models.entry :as entry]
[luduverse.db-models.competition :as competition]))
[luduverse.images :as image]
[luduverse.models.entry :as entry]
[luduverse.models.competition :as competition]))
;; NOTE: This part is grabbed from the open source lib
;; https://github.com/arg-games/ldview
@ -72,7 +72,7 @@ and simply fetch it from Wikipedia. End of discussion :)"
:images images}))
;; Save all what we got
;; Save what we got
(defn save-entry [competition-id entry-id]
(let [entry (entry-details competition-id entry-id)]
(if-not (competition/exists? competition-id) (competition/create! competition-id (theme competition-id)))