fix: falsely receive webmention for non-HTML/text documents

2 files changed, 54 insertions, 22 deletions

diff --git a/src/weepinbell/processor.clj b/src/weepinbell/processor.clj
index 080ea40..3d5032c 100644
--- a/src/weepinbell/processor.clj
+++ b/src/weepinbell/processor.clj
@@ -3,7 +3,8 @@
             [clojure.string :refer [includes? lower-case]]
             [clj-http.client :as client]
             [weepinbell.storage :as storage]
-            [clojure.walk :as walk])
+            [clojure.walk :as walk]
+            [weepinbell.utils :refer [validate-content-types]])
   (:import [com.kylewm.mf2j Mf2Parser]
            [java.net URI]))
 
@@ -69,36 +70,39 @@
     {:webmention wm :microformats mf}))
 
 (defn process-new-webmention
-  "If a webmention is verified, fetch microformats data and store it."
+  "Store new webmention if verifed. Do nothing otherwise."
   [source target]
-  ;; log
-  (println "\n🙋‍♂️ Valid webmention received:")
-  (println "from " source)
-  (println "to " target "\n")
-  ;; wrap and store data
-  (let [data (wrap-webmention source target)]
-    (storage/append-wm data)))
+  (when (verify-mention source target)
+    (println "\n🙋‍♂️ Valid webmention received:")
+    (println "from " source)
+    (println "to " target "\n")
+    (let [data (wrap-webmention source target)]
+      (storage/append-wm data))))
 
 (defn process-existing-webmention
   "Update existing Webmention when verified, delete it otherwise."
-  [verified? source target]
-  (if verified?
+  [source target]
+  (if (verify-mention source target)
     (storage/update-wm source target
-                    (wrap-webmention source target))
+                       (wrap-webmention source target))
     (storage/delete-wm source target)))
 
 (defn process-webmention
-  "Retrieve webmention from queue and process it."
   [wmc]
-  (println "Processor started.")
   (go-loop []
-    (when-let [wm (<! wmc)]
+    (when-let [webmention (<! wmc)]
       (println "Webmention found in queue.")
-      (let [{:keys [s t]} wm
-            verified? (verify-mention s t)]
-        ;; handle existing webmention and new webmention differently
-        (if (storage/get-wm s t)
-          (process-existing-webmention verified? s t)
-          (when verified? (process-new-webmention s t)))))
+      (let [{:keys [s t]} webmention]
+       ;; first check Content-Types of both urls
+       ;; only proceed when it's HTML or plain text
+        (if (validate-content-types s t)
+         ;; second check if such webmention exists
+         ;; if so, update it or delete it
+         ;; if not, write to data
+          (if (storage/get-wm s t)
+            (process-existing-webmention s t)
+            (process-new-webmention s t))
+          ;; if content-type check fails
+          (println "Source or target is not HTML or plain text; or error occurred trying to get content type."))))
     (recur)))
 
diff --git a/src/weepinbell/utils.clj b/src/weepinbell/utils.clj
index a7e897b..7704231 100644
--- a/src/weepinbell/utils.clj
+++ b/src/weepinbell/utils.clj
@@ -1,7 +1,9 @@
 (ns weepinbell.utils
   (:import [org.apache.commons.validator.routines UrlValidator]
            [java.net URI])
-  (:require [weepinbell.config :refer [get-config]]))
+  (:require [weepinbell.config :refer [get-config]]
+            [clj-http.client :as client]
+            [clojure.string :as string]))
 
 (defn get-domain [url]
   (-> (URI/create url) .getHost))
@@ -22,3 +24,29 @@
         target-domain (get-domain t)]
     (some #{target-domain} allow-list)))
 
+(defn get-content-type [url]
+  (try
+    (let [response (client/head url)]
+      (get-in response [:headers "Content-Type"]))
+    (catch clojure.lang.ExceptionInfo e
+      (let [status (:status (ex-data e))]
+        (println status " error trying to get content type of " url)
+        nil))
+    (catch Exception e
+        (println "Error trying to get content type of " url)
+        (println (.getMessage e))
+      nil)))
+
+(defn plain-or-html? [content-type]
+  (cond
+    (nil? content-type) false
+    (string/includes? content-type "text/html") :html
+    (string/includes? content-type "text/plain") :text
+    :else false))
+
+(defn validate-content-types [source target]
+  (let [validate #(-> % get-content-type plain-or-html? boolean)
+        source-valid (validate source)
+        target-valid (validate target)]
+    (and source-valid target-valid)))
+