blob: e6415f82d37734a642f58315fc7a41dd8e7dd787 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
(ns weepinbell.processor.verifier
"Verifies in-coming webmentions after receiver checks URL.
This makes sure:
1) source contains target URL.
2) source respond with 2** status code.
3) source's Content-Type is either HTML or plain text."
(:require [clojure.string :refer [trim lower-case split]]
[weepinbell.utils.url :as url]
[weepinbell.http.client :as client]
[taoensso.timbre :as timbre]))
(def text-content-types #{"text/html" "text/plain"})
(defn- match-content-type?
"Return true if "
[types headers]
(when-let [content-type (:content-type headers)]
(-> content-type (split #";") first trim lower-case types)))
(def text-content-types?
(partial match-content-type? text-content-types))
(defn- includes-url?
"Return true if text includes a certain URL. URLs are normalized before comparison."
[text url]
(some #(url/url= % url) (url/extract text)))
(defn- verify-mention
"Get source and follow redirects to check if target link is present."
[id source-resp target-resp source target]
;; get first 1MB of source URL and verify
(let [{:keys [status body headers error]} source-resp
target-headers (:headers target-resp)]
(timbre/info "verifying webmention" id
(str "(" "source: " source ", target: " target ")"))
(cond
;; handle error
error
(do
(timbre/error "error verifying webmention:" error)
false)
;; verify HTTP status
(or (not status) (< status 200) (>= status 300))
(do
(timbre/warn "invalid source of" id ", status:" status)
false)
;; verify Content-Type
(not (every? text-content-types? [target-headers headers]))
(do
(timbre/warn "source or target of" id "is neither HTML nor plain text content.")
false)
;; verify inclusion of target URL
(not (includes-url? (str body) target))
(do
(timbre/warn "invalid source of" id ": target not found in source body.")
false)
;; true if all verified
:else true)))
(def retry-times 3)
(defn verify
[data]
(when (:ok? data)
(let [{:keys [id wm new?]} data
{:keys [s t]} wm
source-resp (client/with-retry retry-times 5
(client/get s {:headers {"Range" "bytes=0-1048575"}}))
target-resp (client/with-retry retry-times 5 (client/head t))
verified? (verify-mention id @source-resp @target-resp s t)]
(-> data
(assoc :verified? verified?)
(assoc :source-html (:body @source-resp))
;; don't continue if new webmention request is not verified
(assoc :ok? (or (not new?) verified?))))))
|
