blob: 49c8535599e94744cc4bdf70dd0f043d8da00919 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
(ns weepinbell.utils.url
(:import [java.net URL URI MalformedURLException URISyntaxException IDN]
[org.nibor.autolink LinkExtractor LinkType]
[java.util EnumSet])
(:require [clojure.string :as string]))
(defn valid-http-url? [url-string]
(try
(let [url (URL. url-string)
uri (.toURI url)
scheme (.getScheme uri)
host (. IDN toASCII (-> url .getHost (string/replace "www." "")))]
(and (string/includes? host ".") ;; do not accept localhost and other non-TLD names
(or (= scheme "http") (= scheme "https"))))
(catch ClassCastException _ false) ;; not string
(catch MalformedURLException _ false)
(catch URISyntaxException _ false)))
(defn relative? [url]
(or (string/starts-with? url "/")
(string/starts-with? url ".")))
(defn get-domain [url]
(when (some? url)
(-> url URI/create .getHost)))
(defn get-path [url]
(when (some? url)
(-> url URI/create .getPath)))
(defn get-base [url]
(let [uri (URI. url)]
(str (.getScheme uri) "://" (.getAuthority uri))))
(defn get-target [wm]
(get-in wm [:webmention :target]))
(defn- normlaize-path [^String path*]
(let [path (or path* "")
deslashed (if (and (pos? (count path)) (.endsWith path "/"))
(subs path 0 (dec (count path)))
path)]
(str (.normalize (URI. deslashed)))))
(defn- normalize-query [^String query]
(some-> query
(string/split #"&")
sort
(->> (string/join "&"))))
(defn normalize
"Normalize URL by removing trailing slashes; removing anchor;
replacing . and ..; ordering query params."
[^String url]
(try
(let [base (URI. url)
scheme (.getScheme base)
auth (.getAuthority base)
path (normlaize-path (.getPath base))
query (normalize-query (.getQuery base))]
(str scheme "://" auth path
(when (seq query) (str "?" query))))
;; return nil is url is nil
(catch NullPointerException _ nil)
;; fallback to original string if syntax is invalid
(catch URISyntaxException _ url)))
(defn url= [& urls]
(apply = (map normalize urls)))
(defn extract
"Given plain text. Returns a vector of URLs that appears in it."
[^String text]
{:pre [(string? text)]
:post [(partial every? string?)]}
(let [extractor (-> (LinkExtractor/builder)
(.linkTypes (EnumSet/of LinkType/URL))
.build)
indexes (->> (.extractLinks extractor text)
(map (fn [link]
[(.getBeginIndex link)
(.getEndIndex link)])))
urls (map #(subs text (first %) (last %)) indexes)]
(vec urls)))
|
