DAmn db stuff .. switch to sqlite and finish up the persistence stuff

This commit is contained in:
Aaron Mueller 2013-09-30 00:37:38 +02:00
parent 146f1bef37
commit 3d4fc79a1f
13 changed files with 102 additions and 90 deletions

View file

@ -10,7 +10,8 @@
[com.taoensso/timbre "2.6.2"] [com.taoensso/timbre "2.6.2"]
[com.postspectacular/rotor "0.1.0"] [com.postspectacular/rotor "0.1.0"]
[com.taoensso/tower "1.7.1"] [com.taoensso/tower "1.7.1"]
[com.h2database/h2 "1.3.172"] ; [com.h2database/h2 "1.3.172"]
[sqlitejdbc "0.5.6"]
[korma "0.3.0-RC5"] [korma "0.3.0-RC5"]
[log4j [log4j
"1.2.17" "1.2.17"

View file

@ -1,5 +1,5 @@
(ns ldview.handler (ns ldview.handler
(:require [compojure.core :refer [defroutes]] (:require [compojure.core :refer [defroutes]]
[ldview.routes.home :refer [home-routes]] [ldview.routes.home :refer [home-routes]]
[noir.util.middleware :as middleware] [noir.util.middleware :as middleware]
[compojure.route :as route] [compojure.route :as route]
@ -24,13 +24,13 @@
:async? false ; should be always false for rotor :async? false ; should be always false for rotor
:max-message-per-msecs nil :max-message-per-msecs nil
:fn rotor/append}) :fn rotor/append})
(timbre/set-config! (timbre/set-config!
[:shared-appender-config :rotor] [:shared-appender-config :rotor]
{:path "ldview.log" :max-size (* 512 1024) :backlog 10}) {:path "ldview.log" :max-size (* 512 1024) :backlog 10})
(if-not (schema/initialized?) (schema/create-tables)) (if-not (schema/initialized?) (schema/create-tables))
(timbre/info "ldview started successfully")) (timbre/info "ldview started successfully"))
(defn destroy (defn destroy

View file

@ -7,3 +7,11 @@
(first (select competitions (first (select competitions
(where {:number *competition*}) (where {:number *competition*})
(limit 1)))) (limit 1))))
(defn exists? [number]
(not (empty? (select competitions
(fields :id)
(where {:number number})))))
(defn create! [number motto]
(insert competitions (values {:number number :motto motto})))

View file

@ -12,7 +12,7 @@
(belongs-to users) (belongs-to users)
(has-many sources)) (has-many sources))
(defentity contests (defentity competitions
(has-many entries)) (has-many entries))
(defentity users (defentity users
@ -21,21 +21,3 @@
(defentity sources (defentity sources
(belongs-to entities)) (belongs-to entities))
; (defn create-entry [entry]
; (insert entries
; (values entry)))
;
; (defn update-entry [id id_uid title description user_id contest_id]
; (update entries
; (set-fields {:first_name first-name
; :last_name last-name
; :email email})
; (where {:id id})))
;
; (defn get-user [id]
; (first (select users
; (where {:id id})
; (limit 1))))

View file

@ -1,25 +0,0 @@
(ns ldview.models.entity
(:use korma.core
[ldview.models.db])
(:require [ldview.models.user :as user]))
(defn all []
(select entries))
(defn exists? [ld-uid]
(not (empty? (select entities
(fields :id)
(where {:ld_uid ld-uid})
(limit 1)))))
(defn create! [fields]
(let [user (user/unique-user (:user fields))
competition-id (competition/latest)]
(if-not (exists? (:ld_uid fields))
(insert entries (values {
:ld_uid (:ld_uid fields)
:title (:title fields)
:description (:description fields)
:user_id (:id user)
:competition_id (:id competition)})))))

View file

@ -0,0 +1,32 @@
(ns ldview.models.entry
(:use korma.core
[ldview.models.db])
(:require [ldview.models.user :as user]
[ldview.models.competition :as competition]
[ldview.models.source :as source]))
(defn all []
(select entries))
(defn by-ld-uid [ld-uid]
(first (select entries
(where {:ld_uid ld-uid}))))
(defn exists? [ld-uid]
(not (empty? (select entries
(fields :id)
(where {:ld_uid ld-uid})
(limit 1)))))
(defn create! [fields]
(let [username (user/unique-user (:author fields))
competition-id (:id (competition/latest))]
(if-not (exists? (:ld_uid fields))
(insert entries (values {
:ld_uid (:ld_uid fields)
:title (:title fields)
:description (:description fields)
:user_id (:id username)
:competition_id competition-id})))
(for [link (:links fields)]
(source/create! (:id (by-ld-uid (:ld_uid fields))) (:url link) (:title link)))))

View file

@ -4,40 +4,36 @@
(def db-store "site.db") (def db-store "site.db")
(def db-spec {:classname "org.h2.Driver" (def db-spec {:classname "org.sqlite.JDBC"
:subprotocol "h2" :subprotocol "sqlite"
:subname (str (io/resource-path) db-store) :subname (str (io/resource-path) db-store)})
:user "sa"
:password ""
:naming {:keys clojure.string/lower-case
:fields clojure.string/upper-case}})
(defn initialized? [] (defn initialized? []
(.exists (new java.io.File (str (io/resource-path) db-store ".h2.db")))) (.exists (new java.io.File (str (io/resource-path) db-store))))
(defn create-initial-tables [] (defn create-initial-tables []
(sql/with-connection db-spec (sql/with-connection db-spec
(sql/create-table :entries (sql/create-table :entries
[:id "INT PRIMARY KEY AUTO_INCREMENT"] [:id "INTEGER PRIMARY KEY AUTOINCREMENT"]
[:ld_uid "INT"] [:ld_uid "INTEGER"]
[:title "VARCHAR(255)"] [:title "VARCHAR(255)"]
[:description "TEXT"] [:description "TEXT"]
[:user_id "INT"] [:user_id "INTEGER"]
[:competition_id "INT"]) [:competition_id "INTEGER"])
(sql/create-table :users (sql/create-table :users
[:id "INT PRIMARY KEY AUTO_INCREMENT"] [:id "INTEGER PRIMARY KEY AUTOINCREMENT"]
[:name "VARCHAR(255)"]) [:name "VARCHAR(255)"])
(sql/create-table :competitions (sql/create-table :competitions
[:id "INT PRIMARY KEY AUTO_INCREMENT"] [:id "INTEGER PRIMARY KEY AUTOINCREMENT"]
[:number "INT"] [:number "INTEGER"]
[:name "VARCHAR(255)"]) [:motto "VARCHAR(255)"])
(sql/create-table :sources (sql/create-table :sources
[:id "INT PRIMARY KEY AUTO_INCREMENT"] [:id "INTEGER PRIMARY KEY AUTOINCREMENT"]
[:entry_id "INT"] [:entry_id "INTEGER"]
[:type "VARCHAR(100)"] [:type "VARCHAR(100)"]
[:uri "VARCHAR(255)"]) [:uri "VARCHAR(255)"])
@ -45,4 +41,5 @@
(sql/do-commands "CREATE INDEX entry_id_index ON sources (entry_id)"))) (sql/do-commands "CREATE INDEX entry_id_index ON sources (entry_id)")))
(defn create-tables [] (defn create-tables []
(create-initial-tables)) (if-not (initialized?)
(create-initial-tables)))

View file

@ -0,0 +1,10 @@
(ns ldview.models.source
(:use korma.core
[ldview.models.db]))
(defn create! [entry_id uri description]
(if (empty? (select sources (where {:entry_id entry_id :type description})))
(insert sources (values {:entry_id entry_id
:uri uri
:type description}))))

View file

@ -1,11 +1,11 @@
(ns ldview.routes.home (ns ldview.routes.home
(:use compojure.core) (:use compojure.core)
(:require [ldview.views.layout :as layout] (:require [ldview.views.layout :as layout]
[ldview.util :as util])) [ldview.util :as util]
[ldview.models.entry :as entry]))
(defn entries-page [] (defn entries-page []
(layout/render (layout/render "entries.html" {:entries (entry/all)}))
"entries.html"))
(defroutes home-routes (defroutes home-routes
(GET "/" [] (entries-page))) (GET "/" [] (entries-page)))

View file

@ -22,8 +22,10 @@
(crop-from resized-img crop-margin 0 new-size new-size))))) (crop-from resized-img crop-margin 0 new-size new-size)))))
(defn sourceimage->thumb [image-path new-image-path] (defn sourceimage->thumb [image-path new-image-path]
(as-file (to-square (io/file image-path) 200) new-image-path)) (with-redefs [image-resizer.fs/new-filename (fn [filepath dimensions] (str filepath))]
(as-file (to-square (io/file image-path) 200) new-image-path)))
(defn sourceimage->fullscreen [image-path new-image-path] (defn sourceimage->fullscreen [image-path new-image-path]
(as-file (resize (io/file image-path) 800 600) new-image-path)) (with-redefs [image-resizer.fs/new-filename (fn [filepath dimensions] (str filepath))]
(as-file (resize (io/file image-path) 800 600) new-image-path)))

View file

@ -1,7 +1,10 @@
(ns ldview.tasks.runner (ns ldview.tasks.runner
(:require [ldview.tasks.scrape :as scrape] (:require [ldview.tasks.scrape :as scrape]
[ldview.tasks.images :as images] [ldview.tasks.images :as images]
[me.raynes.fs :as fs]) [me.raynes.fs :as fs]
[ldview.models.entry :as entry]
[ldview.models.competition :as competition]
[ldview.models.schema :as schema])
(:use [ldview.util])) (:use [ldview.util]))
(defn cleanup [] (defn cleanup []
@ -11,27 +14,30 @@
(fs/mkdirs (str *image-base-path* "/fullscreen/")) (fs/mkdirs (str *image-base-path* "/fullscreen/"))
(fs/mkdirs (str *image-base-path* "/raw/"))) (fs/mkdirs (str *image-base-path* "/raw/")))
(defn save-entry [entry] (defn save-entry [new-entry]
; TODO: Save to Database (entry/create! new-entry)
(if (:images entry) (if (:images new-entry)
(map (fn [image-url] (map (fn [image-url]
(let [id (:id entry) (let [id (:ld_uid new-entry)
number (last (first (re-seq #"shot([0-9]+)" image-url))) number (last (first (re-seq #"shot([0-9]+)" image-url)))
raw-image-path (images/image-name "raw" number)] raw-image-path (images/image-name "raw" id number)]
(scrape/save-image-from-url image-url raw-image-path) (scrape/save-image-from-url image-url raw-image-path)
(images/sourceimage->fullscreen raw-image-path (images/image-name "fullscreen" number)) (images/sourceimage->fullscreen raw-image-path (images/image-name "fullscreen" id number))
(images/sourceimage->thumb raw-image-path (images/image-name "thumbs" number)))) (images/sourceimage->thumb raw-image-path (images/image-name "thumbs" id number))))
(:images entry)))) (:images new-entry))))
; TODO: Make it multithreaded
(defn fetch-all-content [] (defn fetch-all-content []
(let [pages 1] ;(scrape/number-of-pages)] (let [pages (range 1)] ;(scrape/number-of-pages)]
(map (fn [page] (for [page pages]
(map save-entry (scrape/fetch-page 1))) (for [ld-uid (scrape/entries-on-page (inc page))]
(range pages)))) (if-not (entry/exists? ld-uid)
(save-entry (scrape/entry-details ld-uid)))))))
(defn prepare []
(cleanup)
(schema/create-tables)
(competition/create! 27 "10 Seconds"))
(defn start [] (defn start []
(cleanup)
(fetch-all-content)) (fetch-all-content))

View file

@ -58,5 +58,3 @@
:links links :links links
:images images})) :images images}))
(defn fetch-page [page]
(map entry-details (entries-on-page page)))

View file

@ -1,5 +1,6 @@
{% extends "ldview/views/templates/layout.html" %} {% extends "ldview/views/templates/layout.html" %}
{% block content %} {% block content %}
<div class="entry"><img src="{{servlet-context}}/img/sample.png" alt="entry-#" /></div> <div class="entry"><img src="{{servlet-context}}/img/sample.png" alt="entry-#" /></div>
<div class="entry"><img src="{{servlet-context}}/img/sample.png" alt="entry-#" /></div> <div class="entry"><img src="{{servlet-context}}/img/sample.png" alt="entry-#" /></div>
<div class="entry"><img src="{{servlet-context}}/img/sample.png" alt="entry-#" /></div> <div class="entry"><img src="{{servlet-context}}/img/sample.png" alt="entry-#" /></div>