Merge pull request 'refactor(receiver, webhook, utils): replace commons validator #9' (#16) from refact_commons_validator into master

Reviewed-on: https://codeberg.org/eltrac/weepinbell/pulls/16

9 files changed, 49 insertions, 41 deletions

diff --git a/project.clj b/project.clj
index ee612b6..efc5474 100644
--- a/project.clj
+++ b/project.clj
@@ -10,7 +10,6 @@
                  [http-kit "2.3.0"]
                  [ring/ring-json "0.5.1"]
                  [cheshire "6.1.0"]
-                 [commons-validator "1.10.1"]
                  [com.kylewm/mf2j "0.0.4"]
                  [com.taoensso/timbre "6.8.0"]
                  [hiccup "2.0.0"]
diff --git a/src/weepinbell/feed.clj b/src/weepinbell/feed.clj
index c28e0f9..7f1963d 100644
--- a/src/weepinbell/feed.clj
+++ b/src/weepinbell/feed.clj
@@ -1,6 +1,6 @@
 (ns weepinbell.feed
   (:require [clj-rss.core :as rss]
-            [weepinbell.utils :refer [get-domain]]
+            [weepinbell.utils.url :refer [get-domain]]
             [weepinbell.utils.digest :refer [md5-hash]])
   (:import [java.time Instant]))
 
diff --git a/src/weepinbell/receiver.clj b/src/weepinbell/receiver.clj
index 21433ee..bf4bce7 100644
--- a/src/weepinbell/receiver.clj
+++ b/src/weepinbell/receiver.clj
@@ -2,7 +2,7 @@
   "Receiver receives Webmention request (a source and a target); verify the request; return error message if any; push Webmention into queue of processor."
   (:require [clojure.string :as string]
             [clojure.core.async :refer [put!]]
-            [weepinbell.utils :refer [valid-http-url? get-domain]]
+            [weepinbell.utils.url :refer [get-domain valid-http-url?]]
             [weepinbell.utils.digest :refer [md5-hash]]))
 
 ;; utilities
diff --git a/src/weepinbell/retriever.clj b/src/weepinbell/retriever.clj
index ce9fcfd..1bcd959 100644
--- a/src/weepinbell/retriever.clj
+++ b/src/weepinbell/retriever.clj
@@ -1,5 +1,5 @@
 (ns weepinbell.retriever
-  (:require [weepinbell.utils :refer [get-domain]]
+  (:require [weepinbell.utils.url :refer [get-domain]]
             [weepinbell.microformats-wrapper :as mf-wrapper]))
 
 (def default-page-size 12)
diff --git a/src/weepinbell/storage/edn_storage.clj b/src/weepinbell/storage/edn_storage.clj
index 436d2be..ca2d9b7 100644
--- a/src/weepinbell/storage/edn_storage.clj
+++ b/src/weepinbell/storage/edn_storage.clj
@@ -1,6 +1,6 @@
 (ns weepinbell.storage.edn-storage
   (:require [clojure.edn :as edn]
-            [weepinbell.utils :refer [get-domain get-target get-path]]
+            [weepinbell.utils.url :refer [get-domain get-target get-path]]
             [clojure.java.io :as io]
             [clojure.string :refer [join]]))
 
diff --git a/src/weepinbell/utils.clj b/src/weepinbell/utils.clj
index 78068ef..9c279b7 100644
--- a/src/weepinbell/utils.clj
+++ b/src/weepinbell/utils.clj
@@ -1,28 +1,10 @@
 (ns weepinbell.utils
-  (:import [org.apache.commons.validator.routines UrlValidator]
-           [java.net URI])
   (:require [org.httpkit.client :as hk]
             [clojure.string :as string]
             [clojure.walk :as walk]
             [clojure.java.io :as io]
             [taoensso.timbre :as timbre]))
 
-(defn get-domain [url]
-  (when (some? url)
-    (-> url URI/create .getHost)))
-
-(defn get-path [url]
-  (when (some? url)
-    (-> url URI/create .getPath)))
-
-(defn get-target [wm]
-  (get-in wm [:webmention :target]))
-
-(defn valid-http-url? [url]
-  (let [schemes (into-array String ["http" "https"])
-        validator (UrlValidator. schemes)]
-    (.isValid validator url)))
-
 (defn javamap->map [map]
   (->> (seq map) (into {}) walk/keywordize-keys))
 
diff --git a/src/weepinbell/utils/url.clj b/src/weepinbell/utils/url.clj
new file mode 100644
index 0000000..1e47b82
--- /dev/null
+++ b/src/weepinbell/utils/url.clj
@@ -0,0 +1,26 @@
+(ns weepinbell.utils.url

+  (:import [java.net URL URI MalformedURLException URISyntaxException IDN])

+  (:require [clojure.string :as string]))

+

+(defn valid-http-url? [url-string]

+  (try 

+    (let [url (URL. url-string)

+          uri (.toURI url)

+          scheme (.getScheme uri)

+          host (. IDN toASCII (-> url .getHost (string/replace "www." "")))]

+      (and (string/includes? host ".") ;; do not accept localhost and other non-TLD names

+           (or (= scheme "http") (= scheme "https"))))

+    (catch MalformedURLException _ false)

+    (catch URISyntaxException _ false)))

+

+(defn get-domain [url]

+  (when (some? url)

+    (-> url URI/create .getHost)))

+

+(defn get-path [url]

+  (when (some? url)

+    (-> url URI/create .getPath)))

+

+(defn get-target [wm]

+  (get-in wm [:webmention :target]))

+

diff --git a/src/weepinbell/webhook.clj b/src/weepinbell/webhook.clj
index 0090243..58583b0 100644
--- a/src/weepinbell/webhook.clj
+++ b/src/weepinbell/webhook.clj
@@ -1,7 +1,7 @@
 (ns weepinbell.webhook
   (:require [org.httpkit.client :as hk]
             [taoensso.timbre :as timbre]
-            [weepinbell.utils :refer [valid-http-url?]]))
+            [weepinbell.utils.url :refer [valid-http-url?]]))
 
 (defn trigger
   "Send requests to a list of webhooks."
diff --git a/test/weepinbell/utils_test.clj b/test/weepinbell/utils_test.clj
index 48172a9..88968c0 100644
--- a/test/weepinbell/utils_test.clj
+++ b/test/weepinbell/utils_test.clj
@@ -1,48 +1,49 @@
 (ns weepinbell.utils-test
   (:require [weepinbell.utils :as utils]
+            [weepinbell.utils.url :refer [valid-http-url? get-domain get-path]]
             [clojure.test :refer [deftest testing is]]))
 
 (deftest get-domain-test
   (testing "Testing utils/get-domain"
-    (is (= (utils/get-domain "https://example.com/era23#123")
+    (is (= (get-domain "https://example.com/era23#123")
            "example.com"))
-    (is (= (utils/get-domain "http://sub.example.com/era23?quer#123")
+    (is (= (get-domain "http://sub.example.com/era23?quer#123")
            "sub.example.com"))
-    (is (= (utils/get-domain "https://eada.xasdw/wada")
+    (is (= (get-domain "https://eada.xasdw/wada")
            "eada.xasdw"))))
 
 (deftest get-path-test
   (testing "Testing utils/get-path"
-    (is (= (utils/get-path "https://neodb.social/tv/season/1m09Jn4wPIyzC2m2J1ERGE")
+    (is (= (get-path "https://neodb.social/tv/season/1m09Jn4wPIyzC2m2J1ERGE")
            "/tv/season/1m09Jn4wPIyzC2m2J1ERGE"))
-    (is (= (utils/get-path "https://eltr.ac/")
+    (is (= (get-path "https://eltr.ac/")
            "/"))
-    (is (= (utils/get-path "https://www.example.com/path/some.html?flag=true")
+    (is (= (get-path "https://www.example.com/path/some.html?flag=true")
            "/path/some.html"))))
 
 (deftest valid-http-url-test
   (testing "Testing valid-http-url?"
     ;; invalid cases
-    (is (= (utils/valid-http-url? "ssh://example.com/http/")
+    (is (= (valid-http-url? "ssh://example.com/http/")
            false))
-    (is (= (utils/valid-http-url? "shttp://example.com/")
+    (is (= (valid-http-url? "shttp://example.com/")
            false))
-    (is (= (utils/valid-http-url? "http//example.com")
+    (is (= (valid-http-url? "http//example.com")
            false))
-    (is (= (utils/valid-http-url? "example.com/path/to/post")
+    (is (= (valid-http-url? "example.com/path/to/post")
            false))
-    (is (= (utils/valid-http-url? "https://www.example")
+    (is (= (valid-http-url? "http://localhost/path")
            false))
-    (is (= (utils/valid-http-url? "http://localhost/path")
-           false))
-    (is (= (utils/valid-http-url? "http://example.com/测试")
+    (is (= (valid-http-url? "https://www.example")
            false))
     ;; valid cases
-    (is (= (utils/valid-http-url? "https://www.example.com/path?query=string#anchor")
+    (is (= (valid-http-url? "https://www.example.com/path?query=string#anchor")
+           true))
+    (is (= (valid-http-url? "http://simple.top")
            true))
-    (is (= (utils/valid-http-url? "http://simple.top")
+    (is (= (valid-http-url? "http://中文.域名")
            true))
-    (is (= (utils/valid-http-url? "https://example.com/%2F%E5%8C%85%E5%90%AB%E9%9D%9EASCII%E5%AD%97%E7%AC%A6%E7%9A%84%E4%BE%8B%E5%AD%90")
+    (is (= (valid-http-url? "https://example.com/%2F%E5%8C%85%E5%90%AB%E9%9D%9EASCII%E5%AD%97%E7%AC%A6%E7%9A%84%E4%BE%8B%E5%AD%90")
            true))))
 
 (deftest format-list-test