Now we are getting somewhere ... scraping ahead
This commit is contained in:
parent
84114633ac
commit
06b298c376
10 changed files with 18 additions and 19 deletions
|
@ -19,6 +19,7 @@
|
|||
:init luduverse.handler/init
|
||||
:destroy luduverse.handler/destroy
|
||||
:open-browser? false}
|
||||
:aliases {"save-competition-30" ["run" "-m" "luduverse.scraper/save-competition" "30"]}
|
||||
:profiles {:production {:ring {:stacktrace? false}}
|
||||
:dev {:ring {:auto-reload? true
|
||||
:auto-refresh? true}}})
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
(ns luduverse.core
|
||||
(:require [luduverse.ld-scraper :as scraper])
|
||||
(:require [luduverse.scraper :as scraper])
|
||||
(:gen-class))
|
||||
|
||||
(def -main [])
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
[korma.db :refer [defdb mysql]]))
|
||||
|
||||
(defdb db (mysql {:db "luduverse"
|
||||
:user "root"
|
||||
:user "luduverse"
|
||||
:password "dev"}))
|
||||
|
||||
(declare entities competitions users sources)
|
||||
|
|
|
@ -3,16 +3,14 @@
|
|||
[compojure.route :as route]
|
||||
[noir.util.middleware :as middleware]
|
||||
[luduverse.pages :as pages]
|
||||
[luduverse.ld-images :as images]))
|
||||
[luduverse.images :as images]))
|
||||
|
||||
(defroutes core-routes
|
||||
(route/resources "/")
|
||||
(GET "/" [] (pages/front-page))
|
||||
(route/not-found "404, sorry."))
|
||||
|
||||
(defn init []
|
||||
; Create the image resource path if it is missing
|
||||
(if (not (fs/exists? (images/base-path))) (fs/mkdirs (images/base-path))))
|
||||
(defn init [])
|
||||
|
||||
(defn destroy [])
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
(ns luduverse.ld-images
|
||||
(ns luduverse.images
|
||||
(:require [image-resizer.core :refer :all]
|
||||
[image-resizer.util :as utils]
|
||||
[image-resizer.format :refer :all]
|
|
@ -1,4 +1,4 @@
|
|||
(ns luduverse.db-models.competition
|
||||
(ns luduverse.models.competition
|
||||
(:require [korma.core :refer :all]
|
||||
[luduverse.db :refer :all]))
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
(ns luduverse.db-models.entry
|
||||
(ns luduverse.models.entry
|
||||
(:require [korma.core :refer :all]
|
||||
[luduverse.db :refer :all]
|
||||
[luduverse.db-models.user :as user]
|
||||
[luduverse.db-models.competition :as competition]
|
||||
[luduverse.db-models.source :as source]))
|
||||
[luduverse.models.user :as user]
|
||||
[luduverse.models.competition :as competition]
|
||||
[luduverse.models.source :as source]))
|
||||
|
||||
(defn all []
|
||||
(select entries))
|
|
@ -1,4 +1,4 @@
|
|||
(ns luduverse.db-models.source
|
||||
(ns luduverse.models.source
|
||||
(:require [korma.core :refer :all]
|
||||
[luduverse.db :refer :all]))
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
(ns luduverse.db-models.user
|
||||
(ns luduverse.models.user
|
||||
(:require [korma.core :refer :all]
|
||||
[luduverse.db :refer :all]))
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
(ns luduverse.ld-scraper
|
||||
(ns luduverse.scraper
|
||||
(:require [net.cgrand.enlive-html :as html]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.string :refer [split trim]]
|
||||
[luduverse.ld-images :as image]
|
||||
[luduverse.db-models.entry :as entry]
|
||||
[luduverse.db-models.competition :as competition]))
|
||||
[luduverse.images :as image]
|
||||
[luduverse.models.entry :as entry]
|
||||
[luduverse.models.competition :as competition]))
|
||||
|
||||
;; NOTE: This part is grabbed from the open source lib
|
||||
;; https://github.com/arg-games/ldview
|
||||
|
@ -72,7 +72,7 @@ and simply fetch it from Wikipedia. End of discussion :)"
|
|||
:images images}))
|
||||
|
||||
|
||||
;; Save all what we got
|
||||
;; Save what we got
|
||||
(defn save-entry [competition-id entry-id]
|
||||
(let [entry (entry-details competition-id entry-id)]
|
||||
(if-not (competition/exists? competition-id) (competition/create! competition-id (theme competition-id)))
|
Loading…
Reference in a new issue