From 87e57727ebfdb51d970f65a1d332aef1cab7c8c9 Mon Sep 17 00:00:00 2001 From: eltrac Date: Tue, 6 Oct 2026 20:36:39 +0800 Subject: receiver, verifier, retriever, storage: URL normalization utils: add url utilities for accurate comparison - add url/normalize taht reconstructs URL, strips unnecessary components, reorders query string and ensures determinism - add url/url= that uses url/normalize receiver: normalize URLs before checking and putting in queue - use url= and normalize from utils.url - rename verify-request to check-request to avoid confusion with verifier.clj verifier: check if source includes target URL more accurately - implement url/extract with robinst/autolink-java, which extracts a vector a URLs from string - check if source includes target by 1. extracting all links from source body 2. normalizing all links before comparing with target link (with url= func) - refactor verify-headers -> match-content-types, with a higher order function and clearer description retriever, storage: use url= instead of = when comparing urls --- src/weepinbell/processor/verifier.clj | 28 ++++++++++++--------- src/weepinbell/receiver.clj | 27 ++++++-------------- src/weepinbell/retriever.clj | 4 +-- src/weepinbell/storage/edn_storage.clj | 6 ++--- src/weepinbell/utils/url.clj | 45 +++++++++++++++++++++++++++++++++- 5 files changed, 73 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/weepinbell/processor/verifier.clj b/src/weepinbell/processor/verifier.clj index 9a561ed..b868201 100644 --- a/src/weepinbell/processor/verifier.clj +++ b/src/weepinbell/processor/verifier.clj @@ -4,18 +4,24 @@ 1) source contains target URL. 2) source respond with 2** status code. 3) source's Content-Type is either HTML or plain text." - (:require [clojure.string :refer [includes? trim lower-case split]] + (:require [clojure.string :refer [trim lower-case split]] + [weepinbell.utils.url :as url] [weepinbell.http.client :as client] [taoensso.timbre :as timbre])) -(defn- verify-headers [headers] +(def text-content-types #{"text/html" "text/plain"}) +(defn- match-content-type? + "Return true if " + [types headers] (when-let [content-type (:content-type headers)] - (-> content-type - (split #";") - first - trim - lower-case - (#{"text/html" "text/plain"})))) + (-> content-type (split #";") first trim lower-case types))) +(def text-content-types? + (partial match-content-type? text-content-types)) + +(defn- includes-url? + "Return true if text includes a certain URL. URLs are normalized before comparison." + [text url] + (some #(url/url= % url) (url/extract text))) (defn- verify-mention "Get source and follow redirects to check if target link is present." @@ -37,14 +43,12 @@ (timbre/warn "invalid source of" id ", status:" status) false) ;; verify Content-Type - (or (not (verify-headers headers)) - (not (verify-headers target-headers))) + (not (every? text-content-types? [target-headers headers])) (do (timbre/warn "source or target of" id "is neither HTML nor plain text content.") false) ;; verify inclusion of target URL - ;; TODO needs to be more robust and performant - (not (includes? (str body) target)) + (not (includes-url? (str body) target)) (do (timbre/warn "invalid source of" id ": target not found in source body.") false) diff --git a/src/weepinbell/receiver.clj b/src/weepinbell/receiver.clj index 7e62a16..99665ab 100644 --- a/src/weepinbell/receiver.clj +++ b/src/weepinbell/receiver.clj @@ -1,19 +1,10 @@ (ns weepinbell.receiver "Handler of route /endpoint or any other route the user configured. Receiver receives Webmention request (a source and a target); verify the request; return error message if any; push Webmention into webbmention channel for processor to take." - (:require [clojure.string :as string] - [clojure.core.async :refer [put!]] - [weepinbell.utils.url :refer [get-domain valid-http-url?]] + (:require [clojure.core.async :refer [put!]] + [weepinbell.utils.url :refer [get-domain valid-http-url? normalize url=]] [weepinbell.utils.digest :refer [digest]])) -;; utilities -(defn- remove-anchor - "Takes a URL string. Removes anchor (#...)" - [url] - (when (string? url) - (string/replace url #"#.*" ""))) -;; end utilities - ;; validation and verification (defn- validate-urls [s t] @@ -23,7 +14,7 @@ (let [target-domain (get-domain t)] (some #{target-domain} allow-list))) -(defn- verify-request +(defn- check-request "Verify source and target based on URLs, return error message (string) if any. Return nil if none." [source target opts] (cond @@ -33,7 +24,7 @@ "Invalid URLs. Source and target have to be valid http(s) urls." ;; The receiver MUST reject the request ;; if the source URL is the same as the target URL. - (= source target) + (url= source target) "Source and target are identical." ;; The receiver only accetps webmentions on behalf of configured domains. (not (validate-target target opts)) @@ -46,14 +37,12 @@ Return JSON response of pending notification, rejection or error." [wm wmc config] (let [{:keys [s t]} wm - source (remove-anchor s) - target (remove-anchor t) + source (normalize s) + target (normalize t) new-wm {:s source :t target} - allowed-domains (:allowed-target-domains config)] + allowlist (:allowed-target-domains config)] ;; Request verification - (if-let [error-msg (verify-request source - target - allowed-domains)] + (if-let [error-msg (check-request source target allowlist)] ;; respond if request verification failed {:status 400 :headers {"Content-Type" "application/json; charset=utf-8"} diff --git a/src/weepinbell/retriever.clj b/src/weepinbell/retriever.clj index 864f839..4018dbd 100644 --- a/src/weepinbell/retriever.clj +++ b/src/weepinbell/retriever.clj @@ -1,7 +1,7 @@ (ns weepinbell.retriever "Handler of routes /api/mention/:id, /api/mentions/... Retriever gets webmention form storage and wraps it in a response." - (:require [weepinbell.utils.url :refer [get-domain]] + (:require [weepinbell.utils.url :refer [get-domain url=]] [weepinbell.microformats.reducer :refer [reduce-mf]])) (def default-page-size 12) @@ -73,7 +73,7 @@ (if url (cond->> (get-domain url) true ((:get-by-domain storage)) - true (filter #(= url (get-in % [:webmention :target]))) + true (filter #(url= url (get-in % [:webmention :target]))) true (wrap-response-list opts) (:type opts) (filter-response-by-mftype (:type opts)) page (partition-pages page default-page-size) diff --git a/src/weepinbell/storage/edn_storage.clj b/src/weepinbell/storage/edn_storage.clj index 5f07884..9d6da1b 100644 --- a/src/weepinbell/storage/edn_storage.clj +++ b/src/weepinbell/storage/edn_storage.clj @@ -3,7 +3,7 @@ Entry function is use-store, which makes a valid storage vtable." (:require [clojure.edn :as edn] [clojure.java.io :as io] - [weepinbell.utils.url :refer [get-domain get-target get-path]] + [weepinbell.utils.url :refer [get-domain get-target get-path url=]] [clojure.string :refer [join]])) (defn unload-data @@ -14,8 +14,8 @@ ;; predicates used for filtering webmention (defn- st-pair-equal-pred [source target wm] - (and (= source (get-in wm [:webmention :source])) - (= target (get-in wm [:webmention :target])))) + (and (url= source (get-in wm [:webmention :source])) + (url= target (get-in wm [:webmention :target])))) (defn- id-equal-pred [id wm] (= id (get-in wm [:webmention :id]))) (defn- get-predicate [source target] diff --git a/src/weepinbell/utils/url.clj b/src/weepinbell/utils/url.clj index 88c9fea..30cbdf9 100644 --- a/src/weepinbell/utils/url.clj +++ b/src/weepinbell/utils/url.clj @@ -1,5 +1,7 @@ (ns weepinbell.utils.url - (:import [java.net URL URI MalformedURLException URISyntaxException IDN]) + (:import [java.net URL URI MalformedURLException URISyntaxException IDN] + [org.nibor.autolink LinkExtractor LinkType] + [java.util EnumSet]) (:require [clojure.string :as string])) (defn valid-http-url? [url-string] @@ -25,3 +27,44 @@ (defn get-target [wm] (get-in wm [:webmention :target])) +(defn- normlaize-path [^String path*] + (let [path (or path* "") + deslashed (if (and (> (count path) 1) (.endsWith path "/")) + (subs path 0 (dec (count path))) + path)] + (str (.normalize (URI. deslashed))))) +(defn- normalize-query [^String query] + (some-> query + (string/split #"&") + sort + (->> (string/join "&")))) +(defn normalize + "Normalize URL by removing trailing slashes; removing anchor; + replacing . and ..; ordering query params." + [^String url] + (let [base (URI. url) + scheme (.getScheme base) + auth (.getAuthority base) + path (normlaize-path (.getPath base)) + query (normalize-query (.getQuery base))] + (str scheme "://" auth path + (when (seq query) (str "?" query))))) + +(defn url= [& urls] + (apply = (map normalize urls))) + +(defn extract + "Given plain text. Returns a vector of URLs that appears in it." + [^String text] + {:pre [(string? text)] + :post [#(every? valid-http-url? %)]} + (let [extractor (-> (LinkExtractor/builder) + (.linkTypes (EnumSet/of LinkType/URL)) + .build) + indexes (->> (.extractLinks extractor text) + (map (fn [link] + [(.getBeginIndex link) + (.getEndIndex link)]))) + urls (map #(subs text (first %) (last %)) indexes)] + (vec urls))) + -- cgit v1.2.3