retriever, test: refactor retriever; accurate MF type matching

- microformats type inferring is more accurate
  - before: check if microformats h-entry has a type key (:like-of,
:in-reply-to...)
  - now: check if such key's value equals to target URL, so it makes
sure this like/reply/repost/rsvp is sent to this target
- use ring.util.http-response to simplify response
- `?mftype=` filter no longer depends on microformats reduction
- retriever.clj is overall cleaner

6 files changed, 113 insertions, 96 deletions

diff --git a/project.clj b/project.clj
index c5d4d1b..e599db2 100644
--- a/project.clj
+++ b/project.clj
@@ -12,6 +12,7 @@
                  [compojure "1.7.2"]
                  [ring/ring-core "1.15.5"]
                  [ring/ring-defaults "0.7.1"]
+                 [metosin/ring-http-response "0.9.5"]
                  [http-kit "2.9.0-beta4"]
                  [ring/ring-json "0.5.1"]
                  [cheshire "6.2.0"]
diff --git a/src/weepinbell/microformats/reducer.clj b/src/weepinbell/microformats/reducer.clj
index 552b0d1..f718451 100644
--- a/src/weepinbell/microformats/reducer.clj
+++ b/src/weepinbell/microformats/reducer.clj
@@ -1,22 +1,15 @@
 (ns weepinbell.microformats.reducer

-  (:require [weepinbell.microformats.searcher :refer [use-items find-h-card find-h-entry 

-                                                      search-photo use-prop]]

+  (:require [weepinbell.microformats.searcher :refer [use-items find-h-card find-h-entry

+                                                      search-photo use-prop search-mftype]]

             [clojure.spec.alpha :as s]))

 

-(s/def ::author 

+(s/def ::author

   (s/keys :req-un [::name]

           :opt-un [::url ::photo]))

 (s/def ::reduced-microformats

   (s/keys :req-un [::author ::type ::published ::content]))

 

-(defn find-type [h-entry]

-  (or 

-    (->> [:like-of :repost-of :in-reply-to]

-         (filter #(use-prop % h-entry))

-         first)

-    :mention))

-

-(defn reduce-mf [mf]

+(defn reduce-mf [target mf]

   (let [items (use-items mf)

         entry (find-h-entry items)

         card (find-h-card items)

@@ -24,10 +17,10 @@
                    card)

         photo (or (search-photo author)

                   (search-photo card))]

-    {:author {:name  (use-prop :name author) 

-                       :url   (use-prop :url author)

-                       :photo photo}

-              :type (find-type entry)

-              :published (use-prop :published entry)

-              :content (use-prop :content entry)}))

+    {:author {:name  (use-prop :name author)

+              :url   (use-prop :url author)

+              :photo photo}

+     :type (search-mftype target entry)

+     :published (use-prop :published entry)

+     :content (use-prop :content entry)}))

 

diff --git a/src/weepinbell/microformats/searcher.clj b/src/weepinbell/microformats/searcher.clj
index 480b6ad..50ebd01 100644
--- a/src/weepinbell/microformats/searcher.clj
+++ b/src/weepinbell/microformats/searcher.clj
@@ -1,7 +1,8 @@
 (ns weepinbell.microformats.searcher

   "This namespace contains functions used for searching in a microformats JSON.

   Microformats are indeterministic and dynamic. A specific properties could be 

-  in different places and many forms, thus will require extra effort finding it.")

+  in different places and many forms, thus will require extra effort finding it."

+  (:require [weepinbell.utils.url :refer [url=]]))

 

 ;; glossary

 ;;

@@ -76,3 +77,11 @@
       (if author-photo author-photo

           (when-let [card (use-prop :org node)]

             (search-photo card))))))

+

+(defn search-mftype [target h-entry]

+  (or

+   (->> [:like-of :repost-of :in-reply-to]

+        (filter #(url= (use-prop % h-entry) target))

+        first)

+   :mention))

+

diff --git a/src/weepinbell/webserver/retriever.clj b/src/weepinbell/webserver/retriever.clj
index 4a21b02..eac4053 100644
--- a/src/weepinbell/webserver/retriever.clj
+++ b/src/weepinbell/webserver/retriever.clj
@@ -1,103 +1,113 @@
 (ns weepinbell.webserver.retriever
   "Service getting webmention form storage and wrapping it in a response."
-  (:require [weepinbell.utils.url :refer [get-domain url=]]
-            [weepinbell.microformats.reducer :refer [reduce-mf]]))
+  (:require [weepinbell.utils.url :refer [get-domain get-path]]
+            [weepinbell.microformats.reducer :refer [reduce-mf]]
+            [weepinbell.microformats.searcher :as mfsr]
+            [ring.util.http-response :refer [bad-request not-found ok]]
+            [clojure.spec.alpha :as s]))
 
 (def default-page-size 12)
 
-;; utilities
-(defn- partition-pages
-  [page size data]
-  (let [pages (partition-all size data)
-        length (count pages)
-        page (cond (keyword? page) page
-                   (number? page) page
-                   (= page "")     0
-                   (string? page)  (Integer/parseInt page)
-                   :else           false)]
-    (cond (= page :last) (last pages)
-          (number? page) (if (>= page length) '()
-                             (nth pages page))
-          :else          false)))
+;; response utilities 
 
 (defn- respond-webmention-json [data]
-  (let [base-resp {:status 200
-                   :headers {"Content-Type" "application/json; charset=utf-8"}
-                   :body nil}]
-    (if (and data (some? (:webmention data)))
-      (assoc base-resp :body data)
-      (-> base-resp
-          (assoc :status 404)
-          (assoc :body {:message "The requested webmention does not exist."})))))
+  (if (and data (some? (:webmention data)))
+    (ok data)
+    (not-found {:message "The requested webmention does not exist."})))
 
 (defn- respond-webmention-list-json [data]
-  (let [base-resp {:status 200
-                   :headers {"Content-Type" "application/json; charset=utf-8"}
-                   :body nil}]
-    (cond
-      (false? data) (-> base-resp
-                        (assoc :status 400)
-                        (assoc :body "Page number is invalid. Either too large or not a number."))
-      (empty? data) (-> base-resp
-                        (assoc :status 404)
-                        (assoc :body {:message "The requested webmention list does not exist."}))
-      :else (assoc base-resp :body data))))
+  (cond
+    (false? data) (bad-request {:message "Page number is invalid. Either too large or not a number."})
+    (empty? data) (not-found {:message "The requested webmention list does not exist."})
+    :else         (ok data)))
 
-(defn- wrap-response-object [data opts]
-  (let [result {:webmention (:webmention data)}]
-    (cond-> result
-      (not (empty? (:microformats data)))
-      (assoc :microformats (cond-> data
-                             true :microformats
-                             (not= "raw" (:microformats opts)) reduce-mf))
-      (:extra data)
-      (assoc :extra (:extra data)))))
+;; query list of webmentions
 
-(defn- wrap-response-list [opts data]
-  (map #(wrap-response-object % opts) data))
+(defn filter-by-mftype 
+  [mftype data]
+  (if (nil? mftype) data 
+    (->> data
+         (map (fn [item]
+                [(get-in item [:webmention :target])
+                 (get item :microformats)]))
+         (filter (fn [[target mf]]
+                   (->> mf mfsr/use-items 
+                     mfsr/find-h-entry 
+                     (mfsr/search-mftype target)
+                     (= mftype)))))))
 
-(defn- filter-response-by-mftype [typ data]
-  (filter #(= (keyword typ) 
-              (get-in % [:microformats :type])) data))
-;; end utilities
+(defn query-list 
+  "Get a list of webmentions by certain filters."
+  {:pre [(s/keys :req-un [::domain ::path ::page ::page-size ::storage ::mftype])]}
+  [opts]
+  (let [{:keys [domain path page page-size storage mftype]} opts
+        reduce-mf? (not= (:microformats opts) "raw")
+        page-size (or page-size default-page-size)
+        data ((:get-by-domain storage path) domain)
+        data (filter-by-mftype mftype data)
+        pages (vec (partition-all page-size data))
+        page-number (if (number? page) page 
+                      (- (count pages) 1))
+        page-selected (get pages page-number)
+        result (if reduce-mf? (map #(assoc % :microformats 
+                            (reduce-mf (get-in % [:webmention :target])
+                                       (:microformats %))) 
+                    page-selected)
+                 page-selected)]
+    (respond-webmention-list-json result)))
 
 (defn by-domain
   "Get webmentions sent to a particular domain."
   [domain page storage opts]
-  (cond->> ((:get-by-domain storage) domain)
-    true (wrap-response-list opts)
-    (:type opts) (filter-response-by-mftype (:type opts))
-    page (partition-pages page default-page-size)
-    true respond-webmention-list-json))
+  (query-list {:domain domain
+               :path nil
+               :page page 
+               :page-size default-page-size
+               :storage storage 
+               :mftype (:type opts)
+               :microformats (:microformats opts)}))
 
 (defn by-url
   "Get webmention sent to a particular URL."
   [url page storage opts]
   (if url
-    (cond->> (get-domain url)
-      true ((:get-by-domain storage))
-      true (filter #(url= url (get-in % [:webmention :target])))
-      true (wrap-response-list opts)
-      (:type opts) (filter-response-by-mftype (:type opts))
-      page (partition-pages page default-page-size)
-      true respond-webmention-list-json)
-    ;; if no url query is passed
-    {:status 400
-     :headers {"Content-Type" "application/json; charset=utf-8"}
-     :body {:status 400
-            :message "Missing URL. Specify target URL whose webmentions you are querying."}}))
+    (query-list {:domain (get-domain url)
+                 :path (get-path url)
+                 :page page 
+                 :page-size default-page-size
+                 :storage storage 
+                 :mftype (:type opts)
+                 :microformats (:microformats opts)})
+    {:status 400 
+     :body {:message "Missing URL. Specify target URL whose webmentions you are querying."}}))
+
+;; query one webmention
+
+(defn query-one 
+  "Get one specific webmention by id or by its source and target."
+  [opts]
+  {:post [(s/keys :req-un [::identifiers ::storage ::microformats])]}
+  (let [{:keys [identifiers storage microformats]} opts
+        reduce-mf? (not= "raw" microformats)
+        data (apply (:get storage) identifiers)
+        result (if (and (:microformats data) reduce-mf?)
+               (assoc data :microformats 
+                      (reduce-mf (get-in (:microformats data) [:webmention :target])
+                                 (:microformats data)))
+               data)]
+    (respond-webmention-json result)))
 
 (defn by-id
   "Get one specific webmention by ID."
   [id storage opts]
-  (-> ((:get storage) id)
-      (wrap-response-object opts)
-      respond-webmention-json))
+  (query-one {:identifiers [id]
+              :storage storage 
+              :microformats (:microformats opts)}))
 
 (defn by-st-pair
   "Get one specific webmention by source and target."
   [source target storage opts]
-  (-> ((:get storage) source target)
-      (wrap-response-object opts)
-      respond-webmention-json))
+  (query-one {:identifiers [source target]
+              :storage storage 
+              :microformats (:microformats opts)}))
 
diff --git a/test/weepinbell/microformats_test.clj b/test/weepinbell/microformats_test.clj
index a307c75..c7b41de 100644
--- a/test/weepinbell/microformats_test.clj
+++ b/test/weepinbell/microformats_test.clj
@@ -5,7 +5,8 @@
             [clojure.test :refer [deftest testing is]]))

 

 (def testdata

-  [{:in {:items [{:type ["h-entry"]

+  [{:target "https://test.com/"

+    :in {:items [{:type ["h-entry"]

                   :properties {:content [{:html "<div>test</div>"

                                           :text "test"}]

                                :in-reply-to ["https://test.com"]

@@ -23,7 +24,8 @@
           :type :in-reply-to,

           :published 52341243232,

           :content {:html "<div>test</div>", :text "test"}}}

-   {:in {:items [{:type ["h-entry"]

+   {:target ""

+    :in {:items [{:type ["h-entry"]

                   :properties {:content [{:html "<div>test</div>"

                                           :text "test"}]

                                :published [52341243232]}}

@@ -45,7 +47,7 @@
 (deftest reducer-test

   (testing "Normal microformats reduction"

     (->> testdata

-         (map #(let [reduced (reduce-mf (:in %))]

+         (map #(let [reduced (reduce-mf (:target %) (:in %))]

                  (is (= reduced (:out %)))

                  (is (spec-check reduced))))

          doall))

diff --git a/test/weepinbell/retriever_test.clj b/test/weepinbell/retriever_test.clj
index 7b5614d..c4044ed 100644
--- a/test/weepinbell/retriever_test.clj
+++ b/test/weepinbell/retriever_test.clj
@@ -13,13 +13,15 @@
             resp1 (ret/by-domain "" 1 mock-store {})]

         (is (= page-size (count (:body resp0))))

         (is (= page-size (count (:body resp1)))))))

-  (testing "by-domain handle invalid page number"

+  (testing "by-domain: if page is not number, return the last page frame"

     (with-redefs [weepinbell.webserver.retriever/default-page-size page-size]

       (let [mock-data [{:webmention {:source "1" :target "1"}}

                        {:webmention {:source "2" :target "2"}}]

             mock-store {:get-by-domain (fn [& _] mock-data)}

             resp (ret/by-domain "" true mock-store {})]

-        (is (= 400 (:status resp))))))

+        (is (= 200 (:status resp)))

+        (is (= (-> mock-data last :webmention)

+               (-> resp :body last :webmention))))))

   (testing "by-domain return 404 when list is empty"

     (let [mock-store {:get-by-domain (fn [& _] [])}

           resp (ret/by-domain "" nil mock-store {})]