diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/weepinbell/processor/verifier.clj | 28 | ||||
| -rw-r--r-- | src/weepinbell/receiver.clj | 27 | ||||
| -rw-r--r-- | src/weepinbell/retriever.clj | 4 | ||||
| -rw-r--r-- | src/weepinbell/storage/edn_storage.clj | 6 | ||||
| -rw-r--r-- | src/weepinbell/utils/url.clj | 45 |
5 files changed, 73 insertions, 37 deletions
diff --git a/src/weepinbell/processor/verifier.clj b/src/weepinbell/processor/verifier.clj index 9a561ed..b868201 100644 --- a/src/weepinbell/processor/verifier.clj +++ b/src/weepinbell/processor/verifier.clj @@ -4,18 +4,24 @@ 1) source contains target URL.
2) source respond with 2** status code.
3) source's Content-Type is either HTML or plain text."
- (:require [clojure.string :refer [includes? trim lower-case split]]
+ (:require [clojure.string :refer [trim lower-case split]]
+ [weepinbell.utils.url :as url]
[weepinbell.http.client :as client]
[taoensso.timbre :as timbre]))
-(defn- verify-headers [headers]
+(def text-content-types #{"text/html" "text/plain"})
+(defn- match-content-type?
+ "Return true if "
+ [types headers]
(when-let [content-type (:content-type headers)]
- (-> content-type
- (split #";")
- first
- trim
- lower-case
- (#{"text/html" "text/plain"}))))
+ (-> content-type (split #";") first trim lower-case types)))
+(def text-content-types?
+ (partial match-content-type? text-content-types))
+
+(defn- includes-url?
+ "Return true if text includes a certain URL. URLs are normalized before comparison."
+ [text url]
+ (some #(url/url= % url) (url/extract text)))
(defn- verify-mention
"Get source and follow redirects to check if target link is present."
@@ -37,14 +43,12 @@ (timbre/warn "invalid source of" id ", status:" status)
false)
;; verify Content-Type
- (or (not (verify-headers headers))
- (not (verify-headers target-headers)))
+ (not (every? text-content-types? [target-headers headers]))
(do
(timbre/warn "source or target of" id "is neither HTML nor plain text content.")
false)
;; verify inclusion of target URL
- ;; TODO needs to be more robust and performant
- (not (includes? (str body) target))
+ (not (includes-url? (str body) target))
(do
(timbre/warn "invalid source of" id ": target not found in source body.")
false)
diff --git a/src/weepinbell/receiver.clj b/src/weepinbell/receiver.clj index 7e62a16..99665ab 100644 --- a/src/weepinbell/receiver.clj +++ b/src/weepinbell/receiver.clj @@ -1,19 +1,10 @@ (ns weepinbell.receiver "Handler of route /endpoint or any other route the user configured. Receiver receives Webmention request (a source and a target); verify the request; return error message if any; push Webmention into webbmention channel for processor to take." - (:require [clojure.string :as string] - [clojure.core.async :refer [put!]] - [weepinbell.utils.url :refer [get-domain valid-http-url?]] + (:require [clojure.core.async :refer [put!]] + [weepinbell.utils.url :refer [get-domain valid-http-url? normalize url=]] [weepinbell.utils.digest :refer [digest]])) -;; utilities -(defn- remove-anchor - "Takes a URL string. Removes anchor (#...)" - [url] - (when (string? url) - (string/replace url #"#.*" ""))) -;; end utilities - ;; validation and verification (defn- validate-urls [s t] @@ -23,7 +14,7 @@ (let [target-domain (get-domain t)] (some #{target-domain} allow-list))) -(defn- verify-request +(defn- check-request "Verify source and target based on URLs, return error message (string) if any. Return nil if none." [source target opts] (cond @@ -33,7 +24,7 @@ "Invalid URLs. Source and target have to be valid http(s) urls." ;; The receiver MUST reject the request ;; if the source URL is the same as the target URL. - (= source target) + (url= source target) "Source and target are identical." ;; The receiver only accetps webmentions on behalf of configured domains. (not (validate-target target opts)) @@ -46,14 +37,12 @@ Return JSON response of pending notification, rejection or error." [wm wmc config] (let [{:keys [s t]} wm - source (remove-anchor s) - target (remove-anchor t) + source (normalize s) + target (normalize t) new-wm {:s source :t target} - allowed-domains (:allowed-target-domains config)] + allowlist (:allowed-target-domains config)] ;; Request verification - (if-let [error-msg (verify-request source - target - allowed-domains)] + (if-let [error-msg (check-request source target allowlist)] ;; respond if request verification failed {:status 400 :headers {"Content-Type" "application/json; charset=utf-8"} diff --git a/src/weepinbell/retriever.clj b/src/weepinbell/retriever.clj index 864f839..4018dbd 100644 --- a/src/weepinbell/retriever.clj +++ b/src/weepinbell/retriever.clj @@ -1,7 +1,7 @@ (ns weepinbell.retriever "Handler of routes /api/mention/:id, /api/mentions/... Retriever gets webmention form storage and wraps it in a response." - (:require [weepinbell.utils.url :refer [get-domain]] + (:require [weepinbell.utils.url :refer [get-domain url=]] [weepinbell.microformats.reducer :refer [reduce-mf]])) (def default-page-size 12) @@ -73,7 +73,7 @@ (if url (cond->> (get-domain url) true ((:get-by-domain storage)) - true (filter #(= url (get-in % [:webmention :target]))) + true (filter #(url= url (get-in % [:webmention :target]))) true (wrap-response-list opts) (:type opts) (filter-response-by-mftype (:type opts)) page (partition-pages page default-page-size) diff --git a/src/weepinbell/storage/edn_storage.clj b/src/weepinbell/storage/edn_storage.clj index 5f07884..9d6da1b 100644 --- a/src/weepinbell/storage/edn_storage.clj +++ b/src/weepinbell/storage/edn_storage.clj @@ -3,7 +3,7 @@ Entry function is use-store, which makes a valid storage vtable." (:require [clojure.edn :as edn] [clojure.java.io :as io] - [weepinbell.utils.url :refer [get-domain get-target get-path]] + [weepinbell.utils.url :refer [get-domain get-target get-path url=]] [clojure.string :refer [join]])) (defn unload-data @@ -14,8 +14,8 @@ ;; predicates used for filtering webmention (defn- st-pair-equal-pred [source target wm] - (and (= source (get-in wm [:webmention :source])) - (= target (get-in wm [:webmention :target])))) + (and (url= source (get-in wm [:webmention :source])) + (url= target (get-in wm [:webmention :target])))) (defn- id-equal-pred [id wm] (= id (get-in wm [:webmention :id]))) (defn- get-predicate [source target] diff --git a/src/weepinbell/utils/url.clj b/src/weepinbell/utils/url.clj index 88c9fea..30cbdf9 100644 --- a/src/weepinbell/utils/url.clj +++ b/src/weepinbell/utils/url.clj @@ -1,5 +1,7 @@ (ns weepinbell.utils.url
- (:import [java.net URL URI MalformedURLException URISyntaxException IDN])
+ (:import [java.net URL URI MalformedURLException URISyntaxException IDN]
+ [org.nibor.autolink LinkExtractor LinkType]
+ [java.util EnumSet])
(:require [clojure.string :as string]))
(defn valid-http-url? [url-string]
@@ -25,3 +27,44 @@ (defn get-target [wm]
(get-in wm [:webmention :target]))
+(defn- normlaize-path [^String path*]
+ (let [path (or path* "")
+ deslashed (if (and (> (count path) 1) (.endsWith path "/"))
+ (subs path 0 (dec (count path)))
+ path)]
+ (str (.normalize (URI. deslashed)))))
+(defn- normalize-query [^String query]
+ (some-> query
+ (string/split #"&")
+ sort
+ (->> (string/join "&"))))
+(defn normalize
+ "Normalize URL by removing trailing slashes; removing anchor;
+ replacing . and ..; ordering query params."
+ [^String url]
+ (let [base (URI. url)
+ scheme (.getScheme base)
+ auth (.getAuthority base)
+ path (normlaize-path (.getPath base))
+ query (normalize-query (.getQuery base))]
+ (str scheme "://" auth path
+ (when (seq query) (str "?" query)))))
+
+(defn url= [& urls]
+ (apply = (map normalize urls)))
+
+(defn extract
+ "Given plain text. Returns a vector of URLs that appears in it."
+ [^String text]
+ {:pre [(string? text)]
+ :post [#(every? valid-http-url? %)]}
+ (let [extractor (-> (LinkExtractor/builder)
+ (.linkTypes (EnumSet/of LinkType/URL))
+ .build)
+ indexes (->> (.extractLinks extractor text)
+ (map (fn [link]
+ [(.getBeginIndex link)
+ (.getEndIndex link)])))
+ urls (map #(subs text (first %) (last %)) indexes)]
+ (vec urls)))
+
|