(ns ldview.tasks.runner (:require [ldview.tasks.scrape :as scrape] [ldview.tasks.images :as images] [me.raynes.fs :as fs] [ldview.models.entry :as entry] [ldview.models.competition :as competition] [ldview.models.schema :as schema] [noir.io :as io])) (defn delete-database! [] (let [db-file (str (io/resource-path) schema/db-store)] (if (fs/exists? db-file) (fs/delete db-file)))) (defn initialize-file-structure! [competition-id] (let [path (images/base-path competition-id)] (if (fs/exists? path) (fs/delete-dir path)) (fs/mkdirs (str path "/thumbs/")) (fs/mkdirs (str path "/fullscreen/")) (fs/mkdirs (str path "/raw/")))) (defn save-entry! [competition-id new-entry] (if-not (entry/exists? (:ld_uid new-entry)) (entry/create! new-entry)) (doseq [image-url (:images new-entry)] (let [id (:ld_uid new-entry) number (last (first (re-seq #"shot([0-9]+)" image-url))) raw-image-path (images/image-name competition-id "raw" id number)] (if-not (fs/exists? raw-image-path) (do (scrape/save-image-from-url image-url raw-image-path) (images/sourceimage->fullscreen raw-image-path (images/image-name competition-id "fullscreen" id number)) (images/sourceimage->thumb raw-image-path (images/image-name competition-id "thumbs" id number))))))) (defn fetch-all-content ([] (fetch-all-content (competition/latest))) ([competition-id] (fetch-all-content competition-id (dec (scrape/number-of-pages competition-id)))) ([competition-id pages] ; TODO: Make it multithreaded (doseq [page (range pages)] (doseq [ld-uid (scrape/entries-on-page competition-id (inc page))] (save-entry! competition-id (scrape/entry-details competition-id ld-uid)))))) (defn load-competition [competition-id] (if-not (schema/initialized?) (schema/create-initial-tables)) (initialize-file-structure! competition-id) (competition/create! competition-id (scrape/theme competition-id)) (fetch-all-content competition-id)) (defn reset-all! [] (delete-database!) (fs/delete-dir (images/base-path)) (schema/create-initial-tables))