Now we are getting somewhere ... scraping ahead
This commit is contained in:
parent
84114633ac
commit
06b298c376
10 changed files with 18 additions and 19 deletions
|
@ -19,6 +19,7 @@
|
||||||
:init luduverse.handler/init
|
:init luduverse.handler/init
|
||||||
:destroy luduverse.handler/destroy
|
:destroy luduverse.handler/destroy
|
||||||
:open-browser? false}
|
:open-browser? false}
|
||||||
|
:aliases {"save-competition-30" ["run" "-m" "luduverse.scraper/save-competition" "30"]}
|
||||||
:profiles {:production {:ring {:stacktrace? false}}
|
:profiles {:production {:ring {:stacktrace? false}}
|
||||||
:dev {:ring {:auto-reload? true
|
:dev {:ring {:auto-reload? true
|
||||||
:auto-refresh? true}}})
|
:auto-refresh? true}}})
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
(ns luduverse.core
|
(ns luduverse.core
|
||||||
(:require [luduverse.ld-scraper :as scraper])
|
(:require [luduverse.scraper :as scraper])
|
||||||
(:gen-class))
|
(:gen-class))
|
||||||
|
|
||||||
(def -main [])
|
(def -main [])
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
[korma.db :refer [defdb mysql]]))
|
[korma.db :refer [defdb mysql]]))
|
||||||
|
|
||||||
(defdb db (mysql {:db "luduverse"
|
(defdb db (mysql {:db "luduverse"
|
||||||
:user "root"
|
:user "luduverse"
|
||||||
:password "dev"}))
|
:password "dev"}))
|
||||||
|
|
||||||
(declare entities competitions users sources)
|
(declare entities competitions users sources)
|
||||||
|
|
|
@ -3,16 +3,14 @@
|
||||||
[compojure.route :as route]
|
[compojure.route :as route]
|
||||||
[noir.util.middleware :as middleware]
|
[noir.util.middleware :as middleware]
|
||||||
[luduverse.pages :as pages]
|
[luduverse.pages :as pages]
|
||||||
[luduverse.ld-images :as images]))
|
[luduverse.images :as images]))
|
||||||
|
|
||||||
(defroutes core-routes
|
(defroutes core-routes
|
||||||
(route/resources "/")
|
(route/resources "/")
|
||||||
(GET "/" [] (pages/front-page))
|
(GET "/" [] (pages/front-page))
|
||||||
(route/not-found "404, sorry."))
|
(route/not-found "404, sorry."))
|
||||||
|
|
||||||
(defn init []
|
(defn init [])
|
||||||
; Create the image resource path if it is missing
|
|
||||||
(if (not (fs/exists? (images/base-path))) (fs/mkdirs (images/base-path))))
|
|
||||||
|
|
||||||
(defn destroy [])
|
(defn destroy [])
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
(ns luduverse.ld-images
|
(ns luduverse.images
|
||||||
(:require [image-resizer.core :refer :all]
|
(:require [image-resizer.core :refer :all]
|
||||||
[image-resizer.util :as utils]
|
[image-resizer.util :as utils]
|
||||||
[image-resizer.format :refer :all]
|
[image-resizer.format :refer :all]
|
|
@ -1,4 +1,4 @@
|
||||||
(ns luduverse.db-models.competition
|
(ns luduverse.models.competition
|
||||||
(:require [korma.core :refer :all]
|
(:require [korma.core :refer :all]
|
||||||
[luduverse.db :refer :all]))
|
[luduverse.db :refer :all]))
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
(ns luduverse.db-models.entry
|
(ns luduverse.models.entry
|
||||||
(:require [korma.core :refer :all]
|
(:require [korma.core :refer :all]
|
||||||
[luduverse.db :refer :all]
|
[luduverse.db :refer :all]
|
||||||
[luduverse.db-models.user :as user]
|
[luduverse.models.user :as user]
|
||||||
[luduverse.db-models.competition :as competition]
|
[luduverse.models.competition :as competition]
|
||||||
[luduverse.db-models.source :as source]))
|
[luduverse.models.source :as source]))
|
||||||
|
|
||||||
(defn all []
|
(defn all []
|
||||||
(select entries))
|
(select entries))
|
|
@ -1,4 +1,4 @@
|
||||||
(ns luduverse.db-models.source
|
(ns luduverse.models.source
|
||||||
(:require [korma.core :refer :all]
|
(:require [korma.core :refer :all]
|
||||||
[luduverse.db :refer :all]))
|
[luduverse.db :refer :all]))
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
(ns luduverse.db-models.user
|
(ns luduverse.models.user
|
||||||
(:require [korma.core :refer :all]
|
(:require [korma.core :refer :all]
|
||||||
[luduverse.db :refer :all]))
|
[luduverse.db :refer :all]))
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
(ns luduverse.ld-scraper
|
(ns luduverse.scraper
|
||||||
(:require [net.cgrand.enlive-html :as html]
|
(:require [net.cgrand.enlive-html :as html]
|
||||||
[clojure.java.io :as io]
|
[clojure.java.io :as io]
|
||||||
[clojure.string :refer [split trim]]
|
[clojure.string :refer [split trim]]
|
||||||
[luduverse.ld-images :as image]
|
[luduverse.images :as image]
|
||||||
[luduverse.db-models.entry :as entry]
|
[luduverse.models.entry :as entry]
|
||||||
[luduverse.db-models.competition :as competition]))
|
[luduverse.models.competition :as competition]))
|
||||||
|
|
||||||
;; NOTE: This part is grabbed from the open source lib
|
;; NOTE: This part is grabbed from the open source lib
|
||||||
;; https://github.com/arg-games/ldview
|
;; https://github.com/arg-games/ldview
|
||||||
|
@ -72,7 +72,7 @@ and simply fetch it from Wikipedia. End of discussion :)"
|
||||||
:images images}))
|
:images images}))
|
||||||
|
|
||||||
|
|
||||||
;; Save all what we got
|
;; Save what we got
|
||||||
(defn save-entry [competition-id entry-id]
|
(defn save-entry [competition-id entry-id]
|
||||||
(let [entry (entry-details competition-id entry-id)]
|
(let [entry (entry-details competition-id entry-id)]
|
||||||
(if-not (competition/exists? competition-id) (competition/create! competition-id (theme competition-id)))
|
(if-not (competition/exists? competition-id) (competition/create! competition-id (theme competition-id)))
|
Loading…
Reference in a new issue