verifier, utils: fix verification failure when page contains invalid URL

- URISyntaxException was not caught in url/normalize, causing
normalization failure which leads to verification failure when there is
one invalid URL, even though valid URL exists
- skip non-HTTP(S) urls when verifying

2 files changed, 8 insertions, 4 deletions

diff --git a/src/weepinbell/processor/verifier.clj b/src/weepinbell/processor/verifier.clj
index b868201..63b9469 100644
--- a/src/weepinbell/processor/verifier.clj
+++ b/src/weepinbell/processor/verifier.clj
@@ -21,7 +21,8 @@
 (defn- includes-url? 

   "Return true if text includes a certain URL. URLs are normalized before comparison."

   [text url]

-  (some #(url/url= % url) (url/extract text)))

+  (some #(url/url= % url) 

+        (filter url/valid-http-url? (url/extract text))))

 

 (defn- verify-mention

   "Get source and follow redirects to check if target link is present."

diff --git a/src/weepinbell/utils/url.clj b/src/weepinbell/utils/url.clj
index 30cbdf9..b8d53e8 100644
--- a/src/weepinbell/utils/url.clj
+++ b/src/weepinbell/utils/url.clj
@@ -42,13 +42,16 @@
   "Normalize URL by removing trailing slashes; removing anchor;

    replacing . and ..; ordering query params."

   [^String url]

-  (let [base   (URI. url)

+  (try 

+    (let [base   (URI. url)

         scheme (.getScheme base)

         auth   (.getAuthority base)

         path   (normlaize-path (.getPath base))

         query  (normalize-query (.getQuery base))]

-    (str scheme "://" auth path

-      (when (seq query) (str "?" query)))))

+      (str scheme "://" auth path

+        (when (seq query) (str "?" query))))

+    ;; fall back to original string if syntax is invalid

+    (catch URISyntaxException _ url)))

 

 (defn url= [& urls]

   (apply = (map normalize urls)))