processor: refactor processor; replace mf2j with mf2clj

- replace mf2j with mf2clj
- remove utils.clj completely
- refactored processor, use Chain of Responsibilities pattern and pass
around maps to reuse data (so we don't have to make a separate HTTP
request just to parse microformats, etc.)
- test: add more test cases to verifier test
- test: add basic test cases to all processing steps
- add more logging to each processing steps. good for debugging.

10 files changed, 190 insertions, 188 deletions

diff --git a/test/weepinbell/janitor_test.clj b/test/weepinbell/janitor_test.clj
new file mode 100644
index 0000000..99efcfb
--- /dev/null
+++ b/test/weepinbell/janitor_test.clj
@@ -0,0 +1,43 @@
+(ns weepinbell.janitor-test

+  (:require [clojure.test :refer [deftest testing is]]

+            [weepinbell.processor.janitor :refer [clean]]))

+

+(def old-unverified {:ok? true

+                     :verified? false

+                     :new? false

+                     :id "1234"

+                     :old-data {}})

+(def new-unverified {:ok? true

+                     :verified? false

+                     :new? true

+                     :id "1234"

+                     :old-data {}})

+(def old-verified {:ok? true

+                   :verified? true

+                   :new? false

+                   :id "1234"

+                   :old-data {}})

+(def new-verified {:ok? true

+                   :verified? true

+                   :new? true

+                   :id "1234"

+                   :old-data {}})

+

+(defn- not-if [condition value]

+  (if condition (not value) value))

+

+(defn- test-clean [testdata should-clean?]

+  (let [deleted (promise)

+        storage {:delete (fn [_] (deliver deleted true))}

+        callbacked (promise)

+        callbacks {:on-delete (fn [_] (deliver callbacked true))}

+        cleaned (clean testdata storage callbacks)]

+    (is (not-if should-clean? (:ok? cleaned)))

+    (is (not-if should-clean? (not (deref deleted 10 false))))

+    (is (not-if should-clean? (not (deref callbacked 10 false))))))

+

+(deftest janitor-test

+  (testing "janitor cleans old unverified webmention"

+    (test-clean old-unverified true))

+  (testing "janitor does nothing for new unverified and all verified webmentions"

+    (doall (map #(test-clean % false) [old-verified new-verified new-unverified]))))

diff --git a/test/weepinbell/processor_starter_test.clj b/test/weepinbell/processor_starter_test.clj
new file mode 100644
index 0000000..d19ac2a
--- /dev/null
+++ b/test/weepinbell/processor_starter_test.clj
@@ -0,0 +1,25 @@
+(ns weepinbell.processor-starter-test

+  (:require [clojure.test :refer [deftest testing is]]

+            [clojure.core.async :refer [chan put!]]

+            [weepinbell.processor.starter :refer [start-processor]]))

+

+(def test-wmc (chan))

+(def received (promise))

+(def processor-meta {:wmc test-wmc

+                     :callbacks {:on-receive (fn [_] (deliver received true))

+                                 :on-create (fn [_])

+                                 :on-delete (fn [_])

+                                 :on-update (fn [_])}

+                     :storage {:get (fn [& _])

+                               :delete (fn [& _])

+                               :append (fn [& _])

+                               :update (fn [& _])

+                               :get-by-domain (fn [& _])}

+                     :config {:base-url "https://test.com"

+                              :allowed-target-domains ["test.com"]}})

+

+(deftest start-processor-test 

+  (testing "starting processor and putting a fake webmention in queue"

+    (start-processor processor-meta)

+    (put! test-wmc {:s "s" :t "t"})

+    (is (deref received 50 false))))

diff --git a/test/weepinbell/processor_test.clj b/test/weepinbell/processor_test.clj
deleted file mode 100644
index 3756f4f..0000000
--- a/test/weepinbell/processor_test.clj
+++ /dev/null
@@ -1,80 +0,0 @@
-(ns weepinbell.processor-test
-  (:require [clojure.test :refer [deftest is testing]]
-            [weepinbell.test-utility :refer [with-webmention]]))
-
-;; Testing process-new-webmention
-;; though we're calling a proxy function (process-webmention)
-;; But the arguments and storage vtables that test cases provide
-;; should result in a call to process-new-webmention
-
-(deftest new-webmention-test
-  (testing "Creating a new webmention."
-    (with-webmention :new ["/source" {:status 200
-                                      :headers {"Content-Type" "text/plain"}
-                                      :body (str "Oh I love this post: "
-                                                 "http://localhost:38888/target")}
-                           "/target" {:status 200
-                                      :headers {"Content-Type" "text/plain"}
-                                      :body "Nothing here"}]
-      (is (deref created 100 nil)))))
-
-(deftest new-webmention-content-type-fail-test
-  (testing "One of source and target has unsupported Content-Type"
-    (with-webmention :new ["/source" {:status 200
-                                      :headers {"Content-Type" "text/plain"}
-                                      :body (str "Oh I love this post: "
-                                                 "http://localhost:38888/target")}
-                           "/target" {:status 200
-                                      :headers {"Content-Type" "application/json"}
-                                      :body "Nothing here"}]
-      (is (not (deref created 100 nil))))))
-
-(deftest new-webmention-no-target-url-fail-test
-  (testing "The source does not contain target URL"
-    (with-webmention :new ["/source" {:status 200
-                                      :headers {"Content-Type" "text/plain"}
-                                      :body (str "Oh I love this post "
-                                                 "but I don't mention it.")}
-                           "/target" {:status 200
-                                      :headers {"Content-Type" "application/json"}
-                                      :body "Nothing here"}]
-      (is (not (deref created 100 nil))))))
-
-;; Testing process-existing-webmention
-
-(deftest old-webmention-update-test
-  (testing "Updating exisiting webmention if links are still valid."
-    (with-webmention :old ["/source" {:status 200
-                                      :headers {"Content-Type" "text/plain"}
-                                      :body (str "Oh I love this post "
-                                                 "http://localhost:38888/target")}
-                           "/target" {:status 200
-                                      :headers {"Content-Type" "text/plain"}
-                                      :body "Nothing here"}]
-      (is (deref updated 100 nil)))))
-
-(deftest old-webmention-no-target-link-delete-test
-  (testing "Deleting exisiting webmention if source doest no mention target anymore."
-    (with-webmention :old ["/source" {:status 200
-                                      :headers {"Content-Type" "text/plain"}
-                                      :body "Nothing Here"}
-                           "/target" {:status 200
-                                      :headers {"Content-Type" "text/plain"}}]
-      (is (deref deleted 100 nil)))))
-0
-(deftest old-webmention-source-not-found-delete-test
-  (testing "Deleting exisiting webmention if source is not there."
-    (with-webmention :old ["/source" {:status 404
-                                      :headers {"Content-Type" "text/plain"}}
-                           "/target" {:status 200
-                                      :headers {"Content-Type" "text/plain"}}]
-      (is (deref deleted 100 nil)))))
-
-(deftest old-webmention-target-not-found-delete-test
-  (testing "Deleting exisiting webmention if source is not there."
-    (with-webmention :old ["/source" {:status 200
-                                      :headers {"Content-Type" "text/plain"}}
-                           "/target" {:status 404
-                                      :headers {"Content-Type" "text/plain"}}]
-      (is (deref deleted 100 nil)))))
-
diff --git a/test/weepinbell/storage_checker_test.clj b/test/weepinbell/storage_checker_test.clj
new file mode 100644
index 0000000..a2d8981
--- /dev/null
+++ b/test/weepinbell/storage_checker_test.clj
@@ -0,0 +1,26 @@
+(ns weepinbell.storage-checker-test

+  (:require [clojure.test :refer [deftest testing is]]

+            [weepinbell.processor.storage-checker :refer [check-storage]]))

+

+(deftest process-handler-storaga-checker-test

+  (testing "nil when not OK"

+    (is (nil? (check-storage {:ok? false} {}))))

+  (testing "no such webmention stored"

+    (let [data {:ok? true

+                :id "1234"

+                :wm {:s "s" :t "t"}}]

+      (is (= (check-storage data {:get (fn [& _] [])})

+             (-> data

+                 (assoc :old-data [])

+                 (assoc :new? true)

+                 (assoc :ok? true))))))

+  (testing "webmention exists"

+    (let [data {:ok? true

+                :id "1234"

+                :wm {:s "s" :t "t"}}]

+      (is (= (check-storage data {:get (fn [& _] [{}])})

+             (-> data

+                 (assoc :old-data [{}])

+                 (assoc :new? false)

+                 (assoc :ok? true)))))))

+

diff --git a/test/weepinbell/storage_test.clj b/test/weepinbell/storage_test.clj
index 625cd07..41b92a2 100644
--- a/test/weepinbell/storage_test.clj
+++ b/test/weepinbell/storage_test.clj
@@ -2,11 +2,11 @@
   (:require [clojure.test :refer [testing is deftest]]

             [weepinbell.storage.edn-storage :as edns]

             [clojure.spec.alpha :as s]

-            [weepinbell.utils]

+            [weepinbell.storage.edn-storage]

             [weepinbell.storage.spec]))

 

 (deftest edn-storage-conformity-test

   (testing "edn-storage should conform storage spec."

-    (with-redefs [weepinbell.utils/file-exists? (fn [_] true)]

+    (with-redefs [weepinbell.storage.edn-storage/file-exists? (fn [_] true)]

       (is (s/valid? :weepinbell.storage.spec/vtable

                     (edns/use-store "source.edn"))))))

diff --git a/test/weepinbell/test_utility.clj b/test/weepinbell/test_utility.clj
index b92f3a9..73b831a 100644
--- a/test/weepinbell/test_utility.clj
+++ b/test/weepinbell/test_utility.clj
@@ -1,12 +1,6 @@
 (ns weepinbell.test-utility
   (:require [compojure.core :refer [GET routes]]
-            [org.httpkit.server :refer [run-server]]
-            [weepinbell.processor :as p]))
-
-(def mock-callbacks {:on-create  (fn [_])
-                     :on-delete  (fn [_])
-                     :on-update  (fn [_])
-                     :on-receive (fn [_])})
+            [org.httpkit.server :refer [run-server]]))
 
 (defmacro with-webmention [existance [source-route source-resp target-route target-resp] & body]
   (let [source-url (str "http://localhost:38888" source-route)
@@ -27,10 +21,6 @@
                                                  :target ~target-url}}))}
            ~'server (run-server handler# {:port 38888 :join? false})]
        (try
-         (p/process-webmention {:s ~source-url :t ~target-url}
-                               mock-callbacks
-                               ~'storage
-                               {})
          ~@body
          (finally (~'server))))))
 
diff --git a/test/weepinbell/utils_test.clj b/test/weepinbell/utils_test.clj
deleted file mode 100644
index 88968c0..0000000
--- a/test/weepinbell/utils_test.clj
+++ /dev/null
@@ -1,59 +0,0 @@
-(ns weepinbell.utils-test
-  (:require [weepinbell.utils :as utils]
-            [weepinbell.utils.url :refer [valid-http-url? get-domain get-path]]
-            [clojure.test :refer [deftest testing is]]))
-
-(deftest get-domain-test
-  (testing "Testing utils/get-domain"
-    (is (= (get-domain "https://example.com/era23#123")
-           "example.com"))
-    (is (= (get-domain "http://sub.example.com/era23?quer#123")
-           "sub.example.com"))
-    (is (= (get-domain "https://eada.xasdw/wada")
-           "eada.xasdw"))))
-
-(deftest get-path-test
-  (testing "Testing utils/get-path"
-    (is (= (get-path "https://neodb.social/tv/season/1m09Jn4wPIyzC2m2J1ERGE")
-           "/tv/season/1m09Jn4wPIyzC2m2J1ERGE"))
-    (is (= (get-path "https://eltr.ac/")
-           "/"))
-    (is (= (get-path "https://www.example.com/path/some.html?flag=true")
-           "/path/some.html"))))
-
-(deftest valid-http-url-test
-  (testing "Testing valid-http-url?"
-    ;; invalid cases
-    (is (= (valid-http-url? "ssh://example.com/http/")
-           false))
-    (is (= (valid-http-url? "shttp://example.com/")
-           false))
-    (is (= (valid-http-url? "http//example.com")
-           false))
-    (is (= (valid-http-url? "example.com/path/to/post")
-           false))
-    (is (= (valid-http-url? "http://localhost/path")
-           false))
-    (is (= (valid-http-url? "https://www.example")
-           false))
-    ;; valid cases
-    (is (= (valid-http-url? "https://www.example.com/path?query=string#anchor")
-           true))
-    (is (= (valid-http-url? "http://simple.top")
-           true))
-    (is (= (valid-http-url? "http://中文.域名")
-           true))
-    (is (= (valid-http-url? "https://example.com/%2F%E5%8C%85%E5%90%AB%E9%9D%9EASCII%E5%AD%97%E7%AC%A6%E7%9A%84%E4%BE%8B%E5%AD%90")
-           true))))
-
-(deftest format-list-test
-  (testing "Testing utils.format-list"
-    (is (= (utils/format-list ["abc", "213", "$#@"])
-           "abc, 213 and $#@"))
-    (is (= (utils/format-list ["abc"])
-           "abc"))
-    (is (= (utils/format-list ["123", 456])
-           "123 and 456"))
-    (is (= (utils/format-list [])
-           ""))))
-
diff --git a/test/weepinbell/verifier_test.clj b/test/weepinbell/verifier_test.clj
index e82ebb0..5780abd 100644
--- a/test/weepinbell/verifier_test.clj
+++ b/test/weepinbell/verifier_test.clj
@@ -1,8 +1,15 @@
 (ns weepinbell.verifier-test
   (:require [clojure.test :refer [deftest testing is]]
-            [weepinbell.verifier :as v]
+            [weepinbell.processor.verifier :as v]
             [weepinbell.test-utility :refer [with-webmention]]))
 
+(defn- test-verify [source target]
+  (-> {:wm {:s source :t target}
+       :ok? true
+       :id "123"}
+      v/verify
+      :verified?))
+
 (deftest verification-happy-paths-test
   (testing "Test if processor can correctly verifies a Webmention."
     (with-webmention :new ["/source" {:status 200
@@ -12,9 +19,9 @@
                                                  "       this is very clever"
                                                  " and I find it inspiring")}
                            "/target" {:status 200
-                                      :headers {"Content-Type" "text/html"}}]
-      (is (= true (v/verify-mention "http://localhost:38888/source"
-                                    "http://localhost:38888/target")))))
+                                      :headers {"Content-Type" "text/plain"}}]
+      (is (= true (test-verify "http://localhost:38888/source"
+                               "http://localhost:38888/target")))))
   (testing "Test if processor can correctly verifies a Webmention."
     (with-webmention :new ["/source" {:status 206
                                       :headers {"Content-Type" "text/html"}
@@ -23,8 +30,8 @@
                                                  "       this is very clev")}
                            "/target" {:status 200
                                       :headers {"Content-Type" "text/plain"}}]
-      (is (= true (v/verify-mention "http://localhost:38888/source"
-                                    "http://localhost:38888/target"))))))
+      (is (= true (test-verify "http://localhost:38888/source"
+                               "http://localhost:38888/target"))))))
 
 (deftest verification-exceptional-cases-test
   (testing "source is not found"
@@ -33,24 +40,24 @@
                                       :body "Not Found."}
                            "/target" {:status 200
                                       :headers {"Content-Type" "text/plain"}}]
-      (is (not (v/verify-mention "http://localhost:38888/source"
-                                 "http://localhost:38888/target")))))
+      (is (not (test-verify "http://localhost:38888/source"
+                            "http://localhost:38888/target")))))
   (testing "source returns error status 502"
     (with-webmention :new ["/source" {:status 502
                                       :headers {"Content-Type" "text/plain"}
                                       :body "Bad Gateway"}
                            "/target" {:status 200
                                       :headers {"Content-Type" "text/plain"}}]
-      (is (not (v/verify-mention "http://localhost:38888/source"
-                                 "http://localhost:38888/target")))))
+      (is (not (test-verify "http://localhost:38888/source"
+                            "http://localhost:38888/target")))))
   (testing "source body does not contain any URL"
     (with-webmention :new ["/source" {:status 200
                                       :headers {"Content-Type" "text/plain"}
                                       :body "I didn't mention anyone"}
                            "/target" {:status 200
                                       :headers {"Content-Type" "text/plain"}}]
-      (is (not (v/verify-mention "http://localhost:38888/source"
-                                 "http://localhost:38888/target")))))
+      (is (not (test-verify "http://localhost:38888/source"
+                            "http://localhost:38888/target")))))
   (testing "source body does not contain target URL"
     (with-webmention :new ["/source" {:status 200
                                       :headers {"Content-Type" "text/plain"}
@@ -58,40 +65,38 @@
                                                  "https://chris.blog/archive/xxx")}
                            "/target" {:status 200
                                       :headers {"Content-Type" "text/plain"}}]
-      (is (not (v/verify-mention "http://localhost:38888/source"
-                                 "http://localhost:38888/target")))))
+      (is (not (test-verify "http://localhost:38888/source"
+                            "http://localhost:38888/target")))))
   (testing "source response has no Content-Type header"
     (with-webmention :new ["/source" {:status 200
                                       :body (str "I mentioned alice's other post\n"
                                                  "https://alice.blog/article/efg")}
                            "/target" {:status 200
                                       :headers {"Content-Type" "text/plain"}}]
-      (is (not (v/verify-mention "http://localhost:38888/source"
-                                 "http://localhost:38888/target")))))
+      (is (not (test-verify "http://localhost:38888/source"
+                            "http://localhost:38888/target")))))
   (testing "target response has no Content-Type header"
     (with-webmention :new ["/source" {:status 200
                                       :headers {"Content-Type" "text/plain"}
                                       :body (str "I mentioned alice's other post\n"
                                                  "https://alice.blog/article/efg")}
                            "/target" {:status 200}]
-      (is (not (v/verify-mention "http://localhost:38888/source"
-                                 "http://localhost:38888/target")))))
-  (testing "source response has unsupported Content-Type"
-    (with-webmention :new ["/source" {:status 200
-                                      :headers {"Content-Type" "image/png"}}
-                           "/target" {:status 200}]
-      (is (not (v/verify-mention "http://localhost:38888/source"
-                                 "http://localhost:38888/target"))))))
-
-(deftest verification-error-test 
-  (testing "Host is not found"
-    (with-webmention :new ["/source" {}
-                           "/target" {:status 200}]
-      (is (not (v/verify-mention "http://localh/source" ;; invalid host
-                                 "http://localhost:38888/target")))))
-  (testing "Host is not found"
-    (with-webmention :new ["/source" {}
-                           "/target" {:status 200}]
-      (is (not (v/verify-mention "http://localh/source" ;; invalid host
-                                 "http://localhost:38888/target"))))))
+      (is (not (test-verify "http://localhost:38888/source"
+                            "http://localhost:38888/target")))))
+  (testing "responses have no status"
+    (with-webmention :new ["/source" {:status nil
+                                      :headers {"Content-Type" "text/plain"}
+                                      :body (str "I mentioned alice's other post\n"
+                                                 "https://alice.blog/article/efg")}
+                           "/target" {:headers {"Content-Type" "text/plain"}}]
+      (is (not (test-verify "http://localhost:38888/source"
+                            "http://localhost:38888/target")))))
+  (testing "HTTP not complete, 100 status"
+    (with-webmention :new ["/source" {:status 100
+                                      :headers {"Content-Type" "text/plain"}
+                                      :body "oops"}
+                           "/target" {:status 200
+                                      :headers {"Content-Type" "text/plain"}}]
+      (is (not (test-verify "http://localhost:38888/source"
+                            "http://localhost:38888/target"))))))
 
diff --git a/test/weepinbell/wrapper_test.clj b/test/weepinbell/wrapper_test.clj
new file mode 100644
index 0000000..cfb6a73
--- /dev/null
+++ b/test/weepinbell/wrapper_test.clj
@@ -0,0 +1,25 @@
+(ns weepinbell.wrapper-test

+  (:require [clojure.test :refer [deftest testing is]]

+            [clojure.string :refer [includes?]]

+            [weepinbell.utils.digest :refer [digest]]

+            [weepinbell.processor.wrapper :refer [wrap]]))

+

+(def normal-source-html 

+  "<div class=h-entry>

+    <div class=e-content>test</div>

+    <div class=\"p-author h-card\">

+      <img alt=test src=\"https://source.net/favicon.png\" />

+    </div>

+  </div>")

+

+(deftest happy-path-wrap 

+  (testing "Wrapping a normal webmention"

+    (let [wrapped (:new-data (wrap {:ok? true 

+           :wm {:s "https://source.net"

+                :t "https://target.org"}

+           :source-html normal-source-html}

+          {:base-url "https://resource.com"}))]

+      (is (= "https://source.net" (get-in wrapped [:webmention :source])))

+      (is (= "https://target.org" (get-in wrapped [:webmention :target])))

+      (is (= "b36497f812ccfde649996e21f8df83a6" (get-in wrapped [:webmention :id])))

+      (is (includes? (get-in wrapped [:extra :proxied-photo]) (digest "https://source.net/favicon.png"))))))

diff --git a/test/weepinbell/writer_test.clj b/test/weepinbell/writer_test.clj
new file mode 100644
index 0000000..1d9b14a
--- /dev/null
+++ b/test/weepinbell/writer_test.clj
@@ -0,0 +1,27 @@
+(ns weepinbell.writer-test

+  (:require [clojure.test :refer [deftest testing is]]

+            [weepinbell.processor.writer :refer [write]]))

+

+(defn test-write [data behavior]

+  (let [appended (promise)

+        updated (promise)

+        append-callbacked (promise)

+        update-callbacked (promise)

+        storage {:append (fn [& _] (deliver appended true))

+                 :update (fn [& _] (deliver updated true))}

+        callbacks {:on-create (fn [_] (deliver append-callbacked true))

+                   :on-update (fn [_] (deliver update-callbacked true))}

+        written (write data storage callbacks)]

+    (is (:ok? written))

+    (when (= behavior :should-append)

+      (is (deref appended 10 false)

+          (deref append-callbacked 10 false)))

+    (when (= behavior :should-update)

+      (is (deref updated 10 false)

+          (deref update-callbacked 10 false)))))

+

+(deftest writer-test 

+  (testing "Append new webmentin"

+    (test-write {:ok? true :new? true :id "1" :new-data {}} :should-append))

+  (testing "Update old webmentin"

+    (test-write {:ok? true :new? false :id "1" :new-data {}} :should-update)))