processor, webserver: move two main subsystems to own dir and make facades

add docstrings

format code

12 files changed, 71 insertions, 52 deletions

diff --git a/src/weepinbell/core.clj b/src/weepinbell/core.clj
index 034bb96..c9cd78e 100644
--- a/src/weepinbell/core.clj
+++ b/src/weepinbell/core.clj
@@ -1,34 +1,25 @@
 (ns weepinbell.core
   "Main component that loads resources and start other components."
-  (:require [org.httpkit.server :refer [run-server]]
-            [clojure.spec.alpha :as s]
-            [weepinbell.processor.starter :refer [start-processor]]
-            [weepinbell.handler :refer [app]]
+  (:require [clojure.spec.alpha :as s]
+            [weepinbell.storage.spec]
             [weepinbell.config :refer [load-config]]
             [weepinbell.webhook :as webhook]
             [weepinbell.storage.edn-storage :as edn]
             [clojure.core.async :refer [chan >!! <!!]]
             [taoensso.timbre :as timbre]
-            [weepinbell.storage.spec])
+            ;; two main subsystems' facades
+            [weepinbell.processor.facade :refer [start-processor]]
+            [weepinbell.webserver.facade :refer [start-webserver]])
   (:import (sun.misc Signal SignalHandler))
-  (:gen-class)) ;; Essential for "lein uberjar"
+  (:gen-class))
 
 ; Handle startup; Callable from REPL
-;; processor starter is in ns weepinbell.proceesor.starter
-;; webserver starter is defined here, temporarily
-(defn start-webserver
-  "Starts web server. Returns a function that stops the server."
-  [server-meta]
-  (let [{:keys [wmc store config]} server-meta
-        port (Integer/parseInt (or (:port config) (System/getenv "PORT") "3000"))]
-    (timbre/info "starting web server, listening on" port)
-    (run-server (app wmc store config) {:port port})))
-
 ;; start whole system
 (defn start-system
   "Start webmention processor and webserver in the background.
    Returns a function that performs a graceful shutdown."
   []
+  {:post [fn?]}
   (let [;; webmention channel 
         ;; Receiver puts webmention requests in. 
         ;; Processor takes them out.
diff --git a/src/weepinbell/processor/starter.clj b/src/weepinbell/processor/facade.clj
index ebf2f17..9acea16 100644
--- a/src/weepinbell/processor/starter.clj
+++ b/src/weepinbell/processor/facade.clj
@@ -1,9 +1,10 @@
-(ns weepinbell.processor.starter

+(ns weepinbell.processor.facade

   (:require [clojure.spec.alpha :as s]

             [clojure.core.async :refer [go-loop <! put!]]

             [taoensso.timbre :as timbre]

             [weepinbell.utils.digest :refer [digest]]

             [weepinbell.storage.spec]

+            ;; pipeline functions:

             [weepinbell.processor.storage-checker :refer [check-storage]]

             [weepinbell.processor.verifier :refer [verify]]

             [weepinbell.processor.janitor :refer [clean]]

@@ -11,30 +12,31 @@
             [weepinbell.processor.writer :refer [write]]))

 

 (s/def ::wmc (partial instance? clojure.core.async.impl.channels.ManyToManyChannel))

-(s/def ::callbacks 

+(s/def ::callbacks

   (s/keys :req-un [::on-receive ::on-create ::on-delete ::on-update]))

 (s/def ::storage (partial s/valid? :weepinbell.storage.spec/vtable))

-(s/def ::config 

+(s/def ::config

   (s/keys :req-un [::base-url ::allowed-target-domains]))

-(s/def ::processor-meta 

+(s/def ::processor-meta

   (s/keys :req-un [::wmc ::callbacks ::storage ::config]))

 

 (defn start-processor

   [metadata]

-  {:pre [(s/valid? ::processor-meta metadata)]}

+  {:pre [(s/valid? ::processor-meta metadata)]

+   :post [fn?]}

   (let [{:keys [wmc callbacks storage config]} metadata]

     ;; start processing loop

     (go-loop [wm (<! wmc)]

       (let [{:keys [s t stop?]} wm

             id (digest (str s t))]

-        (if stop? 

+        (if stop?

           (timbre/warn "processor stopped on stop signal.")

           ;; process found webmention

           (do

             (timbre/info "webmention found in queue, processing.")

             ((:on-receive callbacks) {:webmention {:id id :source s :target t}})

             (-> {:id id

-                 :wm wm 

+                 :wm wm

                  :ok? true}

                 (check-storage storage)

                 verify

diff --git a/src/weepinbell/processor/janitor.clj b/src/weepinbell/processor/janitor.clj
index 905c481..ba75d30 100644
--- a/src/weepinbell/processor/janitor.clj
+++ b/src/weepinbell/processor/janitor.clj
@@ -1,7 +1,7 @@
 (ns weepinbell.processor.janitor

   (:require [taoensso.timbre :as timbre]))

 

-(defn clean 

+(defn clean

   [data storage callbacks]

   (when (:ok? data)

     (let [{:keys [verified? new? id old-data]} data]

diff --git a/src/weepinbell/processor/storage_checker.clj b/src/weepinbell/processor/storage_checker.clj
index c8aacaf..7c5cd7c 100644
--- a/src/weepinbell/processor/storage_checker.clj
+++ b/src/weepinbell/processor/storage_checker.clj
@@ -1,7 +1,7 @@
 (ns weepinbell.processor.storage-checker

   (:require [taoensso.timbre :as timbre]))

 

-(defn check-storage 

+(defn check-storage

   ;; data: {:id ..., :wm {:s ... :t ...}, :ok? true}

   [data storage]

   (when (:ok? data)

@@ -10,7 +10,7 @@
           {:keys [s t]} wm

           old-data (or ((:get storage) id)

                        ((:get storage) s t))]

-      (-> data 

+      (-> data

           (assoc :old-data old-data)

           (assoc :new? (or (nil? old-data) (empty? old-data)))

           (assoc :ok? true)))))

diff --git a/src/weepinbell/processor/verifier.clj b/src/weepinbell/processor/verifier.clj
index 63b9469..90f0786 100644
--- a/src/weepinbell/processor/verifier.clj
+++ b/src/weepinbell/processor/verifier.clj
@@ -10,18 +10,18 @@
             [taoensso.timbre :as timbre]))

 

 (def text-content-types #{"text/html" "text/plain"})

-(defn- match-content-type? 

+(defn- match-content-type?

   "Return true if "

   [types headers]

   (when-let [content-type (:content-type headers)]

     (-> content-type (split #";") first trim lower-case types)))

-(def text-content-types? 

+(def text-content-types?

   (partial match-content-type? text-content-types))

 

-(defn- includes-url? 

+(defn- includes-url?

   "Return true if text includes a certain URL. URLs are normalized before comparison."

   [text url]

-  (some #(url/url= % url) 

+  (some #(url/url= % url)

         (filter url/valid-http-url? (url/extract text))))

 

 (defn- verify-mention

@@ -64,7 +64,7 @@
           source-resp (client/get s {:headers {"Range" "bytes=0-1048575"}})

           target-resp (client/head t)

           verified? (verify-mention id @source-resp @target-resp s t)]

-      (-> data 

+      (-> data

           (assoc :verified? verified?)

           (assoc :source-html (:body @source-resp))

           ;; don't continue if new webmention request is not verified

diff --git a/src/weepinbell/processor/wrapper.clj b/src/weepinbell/processor/wrapper.clj
index 5b0002f..202ac02 100644
--- a/src/weepinbell/processor/wrapper.clj
+++ b/src/weepinbell/processor/wrapper.clj
@@ -72,13 +72,13 @@
       (timbre/warn "webmention" id "data corrupted before writing to storage"))

     data))

 

-(defn wrap 

+(defn wrap

   [data config]

   (when (:ok? data)

     (let [{:keys [source-html wm id]} data

           {:keys [s t]} wm]

       (-> data

-          (assoc :new-data 

+          (assoc :new-data

                  (wrap-webmention id source-html s t (:base-url config)))

           (assoc :ok true)))))

 

diff --git a/src/weepinbell/processor/writer.clj b/src/weepinbell/processor/writer.clj
index 4117c89..7728dbb 100644
--- a/src/weepinbell/processor/writer.clj
+++ b/src/weepinbell/processor/writer.clj
@@ -1,7 +1,7 @@
 (ns weepinbell.processor.writer

   (:require [taoensso.timbre :as timbre]))

 

-(defn write 

+(defn write

   [data storage callbacks]

   (when (:ok? data)

     (let [{:keys [id new? new-data]} data]

@@ -9,7 +9,7 @@
         (do ;; append new webmention

           ((:append storage) new-data)

           ((:on-create callbacks) new-data)

-          (timbre/info "new webmention" id "received.")) 

+          (timbre/info "new webmention" id "received."))

         (do ;; update old webmention

           ((:update storage) id new-data)

           ((:on-update callbacks) new-data)

diff --git a/src/weepinbell/webserver/facade.clj b/src/weepinbell/webserver/facade.clj
new file mode 100644
index 0000000..ba9b8b0
--- /dev/null
+++ b/src/weepinbell/webserver/facade.clj
@@ -0,0 +1,19 @@
+(ns weepinbell.webserver.facade
+  (:require [org.httpkit.server :refer [run-server]]
+            [weepinbell.webserver.handler :refer [app]]
+            [taoensso.timbre :as timbre]
+            [clojure.spec.alpha :as s]))
+
+(s/def ::server-meta
+  (s/keys :req-un [::wmc ::store ::config]))
+
+(defn start-webserver
+  "Starts web server. Returns a function that stops the server."
+  [metadata]
+  {:pre [(s/valid? ::server-meta metadata)]
+   :post [fn?]}
+  (let [{:keys [wmc store config]} metadata
+        port (Integer/parseInt (or (:port config) (System/getenv "PORT") "3000"))]
+    (timbre/info "starting web server, listening on" port)
+    (run-server (app wmc store config) {:port port})))
+
diff --git a/src/weepinbell/feed.clj b/src/weepinbell/webserver/feed.clj
index 85c79a4..12a39ec 100644
--- a/src/weepinbell/feed.clj
+++ b/src/weepinbell/webserver/feed.clj
@@ -1,4 +1,4 @@
-(ns weepinbell.feed
+(ns weepinbell.webserver.feed
   "Handler of route /feed/:domain. Generating RSS feed for each domain."
   (:require [clj-rss.core :as rss]
             [weepinbell.utils.url :refer [get-domain]]
diff --git a/src/weepinbell/handler.clj b/src/weepinbell/webserver/handler.clj
index 079c1e1..b6fac9e 100644
--- a/src/weepinbell/handler.clj
+++ b/src/weepinbell/webserver/handler.clj
@@ -1,4 +1,4 @@
-(ns weepinbell.handler
+(ns weepinbell.webserver.handler
   "Handler define routes and wrap responses. It pass request to other specific handlers.
   Other handlers are receiver, retriever and feed."
   (:require [compojure.core :refer [routes POST GET context]]
@@ -6,13 +6,14 @@
             [ring.middleware.defaults :refer [wrap-defaults api-defaults]]
             [ring.middleware.resource :refer [wrap-resource]]
             [ring.middleware.json :refer [wrap-json-response]]
-            [weepinbell.receiver :refer [receive]]
-            [weepinbell.retriever :as retriever]
-            [weepinbell.feed :as feed]))
+            [weepinbell.webserver.receiver :refer [receive]]
+            [weepinbell.webserver.retriever :as retriever]
+            [weepinbell.webserver.feed :as feed]))
 
-;; Define API routes
-
-(defn make-api [wmc storage config]
+(defn- make-api
+  "Receives webmention channel, storage vtable and config map.
+   Returns a vector of compojure route handlers. Some may be nil."
+  [wmc storage config]
   [;; receiver endpoint
    (POST (:endpoint config) [source target]
      (receive {:s source :t target} wmc config))
@@ -21,16 +22,16 @@
      ;; query webmention list
      (context "/mentions" [page microformats mftype]
        (GET "/" [url]
-         (retriever/by-url url page storage 
+         (retriever/by-url url page storage
                            {:microformats microformats
                             :type mftype}))
        (GET "/:domain" [domain]
-         (retriever/by-domain domain page storage 
+         (retriever/by-domain domain page storage
                               {:microformats microformats
                                :type mftype})))
       ;; query one specific webmention
      (context "/mention" [microformats]
-       (GET "/:id" [id] (retriever/by-id id storage 
+       (GET "/:id" [id] (retriever/by-id id storage
                                          {:microformats microformats}))
        (GET "/" [source target] (retriever/by-st-pair source target storage
                                                       {:microformats microformats}))))
@@ -45,10 +46,16 @@
     ;; 404
    (route/not-found "Not Found")])
 
-(defn- make-handler [api]
+(defn- make-handler
+  "Receive a vector of handlers. Remove nil elements.
+   Combine them in one handler using compojure routes."
+  [api]
   (->> api (remove nil?) (apply routes)))
 
-(defn app [wmc storage config]
+(defn app
+  "Receive webmention channel, storage vtable and config map. 
+   Returns an app handler."
+  [wmc storage config]
   (when (= :clojure.spec.alpha/invalid storage)
     (println "Invalid storage. Check your database configuration or submit an issue.")
     (System/exit 1))
diff --git a/src/weepinbell/receiver.clj b/src/weepinbell/webserver/receiver.clj
index 99665ab..144f3db 100644
--- a/src/weepinbell/receiver.clj
+++ b/src/weepinbell/webserver/receiver.clj
@@ -1,4 +1,4 @@
-(ns weepinbell.receiver
+(ns weepinbell.webserver.receiver
   "Handler of route /endpoint or any other route the user configured. 
   Receiver receives Webmention request (a source and a target); verify the request; return error message if any; push Webmention into webbmention channel for processor to take."
   (:require [clojure.core.async :refer [put!]]
diff --git a/src/weepinbell/retriever.clj b/src/weepinbell/webserver/retriever.clj
index 4018dbd..a8c175d 100644
--- a/src/weepinbell/retriever.clj
+++ b/src/weepinbell/webserver/retriever.clj
@@ -1,4 +1,4 @@
-(ns weepinbell.retriever
+(ns weepinbell.webserver.retriever
   "Handler of routes /api/mention/:id, /api/mentions/...
   Retriever gets webmention form storage and wraps it in a response."
   (:require [weepinbell.utils.url :refer [get-domain url=]]
@@ -45,8 +45,8 @@
   (let [result {:webmention (:webmention data)}]
     (cond-> result
       (not (empty? (:microformats data)))
-      (assoc :microformats (cond-> data 
-                             true :microformats 
+      (assoc :microformats (cond-> data
+                             true :microformats
                              (not= "raw" (:microformats opts)) reduce-mf))
       (:extra data)
       (assoc :extra (:extra data)))))