From f356729a846da50c71d1fd3ce98db88fa3a3976b Mon Sep 17 00:00:00 2001 From: eltrac Date: Wed, 7 Oct 2026 16:11:04 +0800 Subject: verifier, utils: fix verification failure when page contains invalid URL - URISyntaxException was not caught in url/normalize, causing normalization failure which leads to verification failure when there is one invalid URL, even though valid URL exists - skip non-HTTP(S) urls when verifying --- src/weepinbell/processor/verifier.clj | 3 ++- src/weepinbell/utils/url.clj | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/weepinbell/processor/verifier.clj b/src/weepinbell/processor/verifier.clj index b868201..63b9469 100644 --- a/src/weepinbell/processor/verifier.clj +++ b/src/weepinbell/processor/verifier.clj @@ -21,7 +21,8 @@ (defn- includes-url? "Return true if text includes a certain URL. URLs are normalized before comparison." [text url] - (some #(url/url= % url) (url/extract text))) + (some #(url/url= % url) + (filter url/valid-http-url? (url/extract text)))) (defn- verify-mention "Get source and follow redirects to check if target link is present." diff --git a/src/weepinbell/utils/url.clj b/src/weepinbell/utils/url.clj index 30cbdf9..b8d53e8 100644 --- a/src/weepinbell/utils/url.clj +++ b/src/weepinbell/utils/url.clj @@ -42,13 +42,16 @@ "Normalize URL by removing trailing slashes; removing anchor; replacing . and ..; ordering query params." [^String url] - (let [base (URI. url) + (try + (let [base (URI. url) scheme (.getScheme base) auth (.getAuthority base) path (normlaize-path (.getPath base)) query (normalize-query (.getQuery base))] - (str scheme "://" auth path - (when (seq query) (str "?" query))))) + (str scheme "://" auth path + (when (seq query) (str "?" query)))) + ;; fall back to original string if syntax is invalid + (catch URISyntaxException _ url))) (defn url= [& urls] (apply = (map normalize urls))) -- cgit v1.2.3