summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authoreltrac <hi@eltr.ac>2026-10-29 12:42:50 +0100
committerEltrac <codeberg@eltr.ac>2026-10-29 12:42:50 +0100
commit573e5f513387ebd34ad9df4b7fe4dc83d7323c8d (patch)
tree70e60433aab15f65f19a90a44aa15b7ca3242273
parent77ab58d1ed5686b3c1c1685a6cc0bdd68603ab93 (diff)
all: use JF2 as a single source of truth, compile microformats/JSON-LD/<meta> to JF2 (#80)
closes #75 This is a large squash commit. For details, see https://codeberg.org/eltrac/weepinbell/pulls/80
Diffstat
-rw-r--r--doc/manual/API.md41
-rw-r--r--doc/manual/installation.md13
-rw-r--r--doc/manual/migration.md31
-rw-r--r--doc/manual/storage.md8
-rw-r--r--project.clj6
-rw-r--r--resources/sql/h2_init.sql6
-rw-r--r--src/weepinbell/core.clj1
-rw-r--r--src/weepinbell/metadata/html_head.clj91
-rw-r--r--src/weepinbell/metadata/jf2.clj32
-rw-r--r--src/weepinbell/metadata/jf2_spec.clj42
-rw-r--r--src/weepinbell/metadata/json_ld.clj33
-rw-r--r--src/weepinbell/metadata/microformats.clj132
-rw-r--r--src/weepinbell/microformats/reducer.clj26
-rw-r--r--src/weepinbell/microformats/searcher.clj91
-rw-r--r--src/weepinbell/migration.clj49
-rw-r--r--src/weepinbell/processor/wrapper.clj126
-rw-r--r--src/weepinbell/processor/writer.clj6
-rw-r--r--src/weepinbell/storage/h2.clj41
-rw-r--r--src/weepinbell/storage/spec.clj36
-rw-r--r--src/weepinbell/utils/url.clj4
-rw-r--r--src/weepinbell/webserver/handler.clj24
-rw-r--r--src/weepinbell/webserver/retriever.clj111
-rw-r--r--test/resources/webmention-jf2-test.edn17
-rw-r--r--test/resources/webmention-test.edn17
-rw-r--r--test/weepinbell/jf2_test.clj279
-rw-r--r--test/weepinbell/microformats_test.clj122
-rw-r--r--test/weepinbell/migration_test.clj39
-rw-r--r--test/weepinbell/retriever_test.clj93
-rw-r--r--test/weepinbell/storage_test.clj4
-rw-r--r--test/weepinbell/wrapper_test.clj100
30 files changed, 1124 insertions, 497 deletions
diff --git a/doc/manual/API.md b/doc/manual/API.md
index cd78f02..8d3f9f1 100644
--- a/doc/manual/API.md
+++ b/doc/manual/API.md
@@ -16,12 +16,12 @@ Receiver endpoint is where people send their webmentions to. It's a route that a
| Method | Path |
| ------ | ----------------------------------------------------- |
-| `POST` | `/endpoint` (Configurable, see [configuration](./configuration.md)) |
+| `POST` | `/endpoint` (can be configured using option `:endpoint`, see `config.edn`) |
| Post Params | Description | Default |
| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
| `source` | Where webmention is coming from. The page that mentioned `target`. | None |
-| `target` | Where webmention is sent to. The page that was mentioned by `source`. Target's domain should be in the allow list of receiver. (see [configuration](./configuration.md)) | None |
+| `target` | Where webmention is sent to. The page that was mentioned by `source`. Target's domain should be in the allow list of receiver. (allow list is `:allowed-target-domains` in `config.edn`) | None |
The endpoint responds with status code `201` and a `Location` header pointing to the webmention. It'll take some time for weepinbell to verify, process and store webmention. Senders should check the `Location` URL a few minutes after.
@@ -63,10 +63,6 @@ The message themselves should be informative enough.
| `GET` | `/api/mention/{id}` |
| `GET` | `/api/mention/?source={source}&target={target}` |
-| Query Params | Description | Default |
-| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
-| `microformats`* | Only takes value `"raw"`. By default, microformats are reduced to deterministic structure. If `"raw"`, return original unmodified microformats data. | None |
-
An example response:
```json
@@ -77,18 +73,28 @@ An example response:
"id": "<32_digit_SHA_256_hash>",
"modified": <timestamp_of_last_modified>
},
- "microformats": {
+ "jf2": {
+ "type": "entry", // it's always "entry"
+ "name": "Title of this entry",
+ "url": "URL to this entry",
"author": {
- "name": "<author_name>",
- "url": "<author_url>",
- "photo": "<author_photo>"
+ "type": "card", // it's always "card"
+ "name": "Name of author",
+ "url": "URL to this author",
+ "photo": "Photo URL" // optional
},
- "type": "<mention | in-reply-to | like-of | repost-of>",
- "published": "<date>",
+ "published": "Date string of published date",
"content": {
- "html": "<html>",
- "text": "<text>"
- }
+ "html": "<p>content of this entry</p>",
+ "text": "content of this entry"
+ },
+ "summary": "Summary or description of this entry", // optional
+ "wm-property": "mention", // or "like-of", "repost-of", "in-reply-to", "rsvp"
+ // below are fields that only exists when the coresponding "wm-property" is present
+ "like-of": "URL that this webmention represents a like of",
+ "repost-of": "URL that this webmention represents a repost of",
+ "in-reply-to": "URL that this webmention replies to",
+ "rsvp": "URL to the event that this webmention represents an RSVP for"
},
"extra": {
"proxied-photo": "<config:base_url>/avatars/<photo_path>"
@@ -106,11 +112,10 @@ An example response:
| Query Params | Description | Default |
| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
-| `microformats`* | Only takes value `"raw"`. By default, microformats are reduced to deterministic structure. If `"raw"`, return original unmodified microformats data. | None |
| `page`* | Page number starting from `0`. **If no page number, weepinbell returns a full list.** If not a number (e.g. `last`, a string), weepinbell returns the last page. | None |
-| `mftype`* | Filters webmentions by their microformats type. Available types are `like-of`, `repost-of`, `in-reply-to` and `rsvp`. If no mftype, no filter is applied. | None |
+| `jf2type`* | Filters webmentions by their JF2 type (.i.e the `wm-property`). Available types are `like-of`, `repost-of`, `in-reply-to` and `rsvp`. If absent, no filter is applied. | None |
-The responses are almost the same as `/api/mention/{id}` except that there are multiple webmention items wrapped inside a JSON array.
+The responses are mostly the same as `/api/mention/{id}` except that there are multiple webmention items wrapped inside a JSON array.
### Couting a list of webmentions
diff --git a/doc/manual/installation.md b/doc/manual/installation.md
index 442dfc5..321c2fc 100644
--- a/doc/manual/installation.md
+++ b/doc/manual/installation.md
@@ -60,6 +60,8 @@ For weepinbell to return the right URLs to webmentions and proxied photos, set `
> [!warning]
> You have to **restart** weepinbell for configuration to take effects.
+You may want to know how and where your webmentions are stored by default and how to change it, see [storage](./storage.md).
+
## 3. Run the app
### The command
@@ -74,6 +76,17 @@ When you see this line of log, you're good to go:
2026-09-28T08:20:45.680Z MACHINE INFO [weepinbell.core:43] - weepinbell started, listening on 3000
```
+> [!warning]
+> If you're just trying weepinbell, running it with a command is fine. It's recommended to use a service manager or a container. See below.
+
+### `systemd` (Linux)
+
+TODO
+
+### `rc` (OpenBSD)
+
+TODO
+
### Docker
> [!warning]
diff --git a/doc/manual/migration.md b/doc/manual/migration.md
index 8ef7d38..6edb020 100644
--- a/doc/manual/migration.md
+++ b/doc/manual/migration.md
@@ -1,6 +1,35 @@
# Migration
-## Migrating from `webmention.io`
+## Migrating from an older version
+
+### Migrate from before `v1.0.0`
+
+Before `v1.0.0`, weepinbell stored microformats rather than JF2, and did not support H2 database.
+
+To convert microformats in your `.edn` file, back up your data and run this:
+
+```shell
+# change .jar name to what you have; use absolute paths
+java -jar weepinbell-v1.x.x-standalone.jar edn-jf2ify /path/to/old.edn /path/to/new.edn
+
+# or, if you have Leiningen installed, you can run this in the top directory of the repository
+lein run edn-jf2ify /path/to/old.edn /path/to/new.edn
+```
+
+To migrate from EDN file-based storage to H2 database, run this (it automatically converts microformats to JF2 as well):
+
+```shell
+java -jar weepinbell-v1.x.x-standalone.jar edn-jf2ify /path/to/old.edn /path/to/database
+
+# or
+lein run edn-jf2ify /path/to/old.edn /path/to/new.edn
+```
+
+Also see [storage](./storage.md).
+
+## Migrating from another service
+
+### `webmention.io`
You can migrate from [webmention.io](https://webmention.io/) with [this Babashka script](script/migrate-from-webmentionio.clj). Make sure you have [Babashka](https://babashka.org/) installed and:
diff --git a/doc/manual/storage.md b/doc/manual/storage.md
index 20f905b..3aa04f5 100644
--- a/doc/manual/storage.md
+++ b/doc/manual/storage.md
@@ -18,7 +18,7 @@ By default, weepinbell uses EDN storage and writes data to `data/webmention.edn`
> [!note]
> If you're not familiar with this format, see [edn](https://github.com/edn-format/edn). In short, it's Clojure's equivalent of JSON, but more extensible.
-When `:storage-type` is set to `:edn`, weepinbell writes to and reads from a `.edn` file, but more accurately, it's in "EDN lines" format, like JSONL. Every line is a valid EDN data. Every line is a webmention entry. Every entry is a map containing the following keys: `:webmention`, `:microformats`, `:extra`.
+When `:storage-type` is set to `:edn`, weepinbell writes to and reads from a `.edn` file, but more accurately, it's in "EDN lines" format, like JSONL. Every line is a valid EDN data. Every line is a webmention entry. Every entry is a map containing the following keys: `:webmention`, `:jf2`, `:extra`.
It's not recommended to edit data file directly by hand, but if you must, back up before you do. Also, it's considered good practice to set up a cron job to routinely back up data file.
@@ -26,9 +26,9 @@ If you don't have much data, using a single EDN would be enough, though the curr
## H2 database
-H2 is a file-based SQL database in pure Java. Like SQLite, it's lightweight and fast. Unlike SQLite, it does not need a native C library, which is slower but more suitable for Java and Clojure.
+H2 is a file-based SQL database in pure Java. Like SQLite, it's lightweight and fast. Unlike SQLite, it does not need a native C library, so it's slower but more suitable for Java and Clojure.
-When `:storage-type` is set to `:h2`, weepinbell uses a H2 database. The database is usually two files. The `:data-path` is the name of database, not the filename like EDN file-based storage. Just set `:data-path` to a name you want and weepinbell will create and use relevant files for you.
+When `:storage-type` is set to `:h2`, weepinbell uses a H2 database. The `:data-path` is the path to the database file but **without extension name**, so instead of `"data/webmention.db"`, you put `"data/webmention"`.
-This is the recommended storage type.
+H2 database is more performant and faster, and it's lightweight enough.
diff --git a/project.clj b/project.clj
index 6983e7d..524f54c 100644
--- a/project.clj
+++ b/project.clj
@@ -23,11 +23,13 @@
[io.github.tonsky/fast-edn "1.2.0"]
[com.github.seancorfield/next.jdbc "1.3.828"]
[com.h2database/h2 "2.3.232"]
- [nrepl "1.8.0"]]
+ [nrepl "1.8.0"]
+ [org.clj-commons/hickory "0.7.7"]]
:managed-dependencies [[org.clojure/tools.reader "1.6.0"]
[org.ow2.asm/asm "9.11.1"]]
:profiles {:uberjar {:aot :all}
- :test {:dependencies [[ring/ring-mock "0.6.2"]]}
+ :test {:dependencies [[ring/ring-mock "0.6.2"]]
+ :resource-paths ["test/resources"]}
:dev {:source-paths ["dev"]
:repl-options {:init-ns user
:init (do (start) (watch))}
diff --git a/resources/sql/h2_init.sql b/resources/sql/h2_init.sql
index 4674842..0d959e6 100644
--- a/resources/sql/h2_init.sql
+++ b/resources/sql/h2_init.sql
@@ -10,11 +10,11 @@ create table if not exists webmention (
lastmod bigint not null
);
-create table if not exists microformats (
+create table if not exists jf2 (
id integer auto_increment primary key,
wm_id integer not null,
- mf_type text not null,
- mf_data text not null, /* schema on read, store EDN string here */
+ jf2_type text not null,
+ jf2_data text not null, /* schema on read, store EDN string here */
FOREIGN KEY (wm_id)
references webmention(id)
diff --git a/src/weepinbell/core.clj b/src/weepinbell/core.clj
index 903d228..0cc7bda 100644
--- a/src/weepinbell/core.clj
+++ b/src/weepinbell/core.clj
@@ -86,5 +86,6 @@
(defn -main [& args]
(case (first args)
"migrate-db" (apply m/edn->h2 (rest args))
+ "edn-jf2ify" (apply m/edn-jf2ify (rest args))
(start-engine)))
diff --git a/src/weepinbell/metadata/html_head.clj b/src/weepinbell/metadata/html_head.clj
new file mode 100644
index 0000000..9a020df
--- /dev/null
+++ b/src/weepinbell/metadata/html_head.clj
@@ -0,0 +1,91 @@
+ (ns weepinbell.metadata.html-head
+ "HTML <head> parser that takes a hickory document, then tries
+ to gather information from basic <meta> properties, Open Graph
+ protocol, Twitter <meta> tags and <link> with rel attribute,
+ and finally compiles to a JF2 object.
+
+ This namespace also contains some utility functions to manipulate
+ a hickory document."
+ (:require [weepinbell.metadata.jf2-spec]))
+
+(defn get-child
+ "Get a specific type of tag inside a hickory object."
+ [tag hickory]
+ (->> hickory :content (filter #(= (:tag %) tag)) first))
+
+(defn get-meta
+ "Gets <meta> tags from a hickory document. Returns a seq of maps.
+ e.g. ({:name \"description\", :content \"A sentence.\"})"
+ [hickory]
+ (->> hickory :content
+ (filter #(= (:tag %) :meta))
+ (map #(get-in % [:attrs]))
+ (filter #(and (or (:name %) (:property %))
+ (:content %)))))
+(defn get-rels
+ "Gets <link> tags with rel attribute from a hickory document.
+ Returns a seq of maps, e.g. ({:rel ..., :href ...})"
+ [hickory]
+ (->> hickory :content
+ (filter #(= (:tag %) :link))
+ (map #(get-in % [:attrs]))
+ (filter #(and (:rel %)
+ (:href %)))))
+(defn attrs->map
+ "Converts a seq of maps to a regular map, like this:
+ ({:name \"a\", :content \"a\"} {:property \"b\", :content \"b\"})
+ becomes {\"a\" \"a\", \"b\" \"b\"}"
+ [meta-attrs]
+ (let [{:keys [property name content rel href]} meta-attrs]
+ {(or name property rel) (or content href)}))
+
+(defn get-metadata
+ "Takes a hickory data object, returns a map of <meta> name/content"
+ [hickory]
+ (->> hickory (get-child :html) (get-child :head)
+ get-meta (map attrs->map) (reduce merge)))
+
+(defn get-title
+ "Takes a hickory data object, returns content of <title>."
+ [hickory]
+ (->> hickory (get-child :html) (get-child :head)
+ :content (filter #(= (:tag %) :title)) first
+ :content first))
+
+(defn get-reldata
+ "Takes a hickory data object, returns a map of <link> rel/hrefs"
+ [hickory]
+ (->> hickory (get-child :html) (get-child :head)
+ get-rels (map attrs->map) (reduce merge)))
+
+(defn parse-meta
+ "Takes hickory document, parses <meta> <link> <title> tags,
+ returns a jf2 map. Supports Open Graph protocol and more."
+ [hickory]
+ (let [data (get-metadata hickory)
+ rels (get-reldata hickory)]
+ {:type "entry"
+ :name (or (get data "og:title")
+ (get data "twitter:title")
+ (get-title hickory))
+ :url (or (get data "og:url")
+ (get data "twitter:url")
+ (get rels "canonical"))
+ :author {:type "card"
+ :name (or (get data "article:author")
+ (get data "twitter:creator")
+ (get data "profile:username")
+ (let [fullname
+ (str (get data "profile:first_name")
+ " " (get data "profile:last_name"))]
+ (when (not= fullname " ") fullname)))
+ :url nil
+ :photo (get rels "icon")}
+ :published (get data "article:published_time")
+ :content ""
+ :wm-property :mention
+ :summary (or (get data "og:description")
+ (get data "description")
+ (get data "twitter:description"))}))
+
+
diff --git a/src/weepinbell/metadata/jf2.clj b/src/weepinbell/metadata/jf2.clj
new file mode 100644
index 0000000..758c381
--- /dev/null
+++ b/src/weepinbell/metadata/jf2.clj
@@ -0,0 +1,32 @@
+(ns weepinbell.metadata.jf2
+ "JF2 parser from various data formats (eg. microformats) to JF2 format."
+ (:require [hickory.core :as hickory]
+ [mf2clj.core :as mf]
+ [weepinbell.metadata.microformats :refer [parse-microformats]]
+ [weepinbell.metadata.html-head :refer [parse-meta]]
+ [weepinbell.metadata.json-ld :refer [parse-json-ld]]))
+
+(defn supplement-merge
+ "Deep merge maps A and B where B never overrides A.
+ Supplements A with values A does not have but B has."
+ [a b]
+ (let [supplement (fn [acc k v]
+ (assoc acc k (supplement-merge (get acc k) v)))]
+ (cond
+ (nil? a) b
+ (not (and (map? a) (map? b))) a
+ :else (reduce-kv supplement a b))))
+
+(defn parse
+ "Takes HTML string, return a JF2 data object compiled from
+ mircorformats and more sources."
+ [^String html-string]
+ {:pre [(string? html-string)]}
+ (let [mf (mf/parse html-string)
+ mf->jf2 (parse-microformats mf)
+ hick (-> html-string hickory/parse hickory/as-hickory)
+ more [mf->jf2
+ (parse-meta hick)
+ (parse-json-ld hick)]]
+ (reduce supplement-merge more)))
+
diff --git a/src/weepinbell/metadata/jf2_spec.clj b/src/weepinbell/metadata/jf2_spec.clj
new file mode 100644
index 0000000..f179244
--- /dev/null
+++ b/src/weepinbell/metadata/jf2_spec.clj
@@ -0,0 +1,42 @@
+(ns weepinbell.metadata.jf2-spec
+ (:require [clojure.spec.alpha :as s]
+ [weepinbell.utils.url :refer [valid-http-url?]]))
+
+(s/def ::type string?)
+
+(s/def ::name string?)
+
+(s/def ::url
+ (s/or :has-value valid-http-url?
+ :lacks-value nil?))
+
+(s/def ::published
+ (s/or :date-string string?
+ :timestamp int?
+ :no-date nil?))
+
+(s/def ::author
+ (s/keys :req-un [::type ::name ::url]
+ :opt-un [::photo]))
+
+(s/def ::content
+ (s/or :content-map (s/keys :req-un [::html ::text])
+ :string string?))
+
+(s/def ::wm-property
+ #{:like-of :in-reply-to :repost-of :rsvp :mention
+ "like-of" "in-reply-to" "repost-of" "rsvp" "mention"})
+
+(s/def ::summary
+ (s/or :has-summary string?
+ :no-summary nil?))
+
+(s/def ::like-of valid-http-url?)
+(s/def ::in-reply-to valid-http-url?)
+(s/def ::repost-of valid-http-url?)
+(s/def ::rsvp valid-http-url?)
+
+(s/def ::jf2
+ (s/keys :req-un [::type ::name ::url ::author ::published ::content ::wm-property]
+ :opt-un [::summary ::like-of ::in-reply-to ::repost-of ::rsvp]))
+
diff --git a/src/weepinbell/metadata/json_ld.clj b/src/weepinbell/metadata/json_ld.clj
new file mode 100644
index 0000000..5ff098a
--- /dev/null
+++ b/src/weepinbell/metadata/json_ld.clj
@@ -0,0 +1,33 @@
+(ns weepinbell.metadata.json-ld
+ "JSON-LD parser that finds JSON-LD string in a hickroy document
+ and then tries to convert it to a JF2 object."
+ (:require [weepinbell.metadata.html-head :refer [get-child]]
+ [cheshire.core :as json]))
+
+(defn- find-json-ld [hickory]
+ (->> hickory (get-child :html) (get-child :head) :content
+ (filter #(and (= (:tag %) :script)
+ (= "application/ld+json"
+ (get-in % [:attrs :type]))))
+ first :content first))
+
+(defn parse-json-ld
+ "Takes hickory document, finds <script type=\"application/ld+json\">,
+ parses JSON, returns a JF2 map."
+ [hickory]
+ (let [data (-> hickory find-json-ld json/parse-string)
+ author (get data "author")]
+ {:type "entry"
+ :name (or (get data "headline")
+ (get data "name"))
+ :url (get data "url")
+ :author (when author
+ {:type "card"
+ :name (get author "name")
+ :url (get author "url")
+ :photo (get-in author ["image" "url"])})
+ :published (get data "datePublished")
+ :content ""
+ :wm-property :mention
+ :summary (get data "description")}))
+
diff --git a/src/weepinbell/metadata/microformats.clj b/src/weepinbell/metadata/microformats.clj
new file mode 100644
index 0000000..81317cc
--- /dev/null
+++ b/src/weepinbell/metadata/microformats.clj
@@ -0,0 +1,132 @@
+(ns weepinbell.metadata.microformats
+ "Microformats parser that converts a microformats map to a JF2 equivalent.
+ This namespace also contains utilities for handling microformats."
+ (:require [weepinbell.utils.url :refer [normalize]]
+ [weepinbell.metadata.jf2-spec]))
+
+(defn- get-key [data k]
+ (cond
+ (not (map? data)) nil
+ (contains? data k) (get data k)
+ (keyword? k) (get data (name k))
+ (string? k) (get data (keyword k))
+ :else nil))
+
+(defn- has-key? [data k]
+ (cond
+ (not (map? data)) false
+ (contains? data k) true
+ (keyword? k) (contains? data (name k))
+ (string? k) (contains? data (keyword k))
+ :else false))
+
+(defn- first-value [value]
+ (if (and (coll? value) (not (map? value)))
+ (first value)
+ value))
+
+(defn- type-vector [item]
+ (let [t (get-key item :type)]
+ (cond
+ (nil? t) []
+ (coll? t) t
+ (string? t) [t]
+ (keyword? t) [(name t)]
+ :else [t])))
+
+(defn use-items [data]
+ (cond
+ (not (coll? data)) nil
+ (empty? data) nil
+ (has-key? data :items) (get-key data :items)
+ (sequential? data) data
+ (has-key? data :type) [data]
+ :else nil))
+
+(defn use-prop-all [property data]
+ (cond
+ (not (map? data)) nil
+ (has-key? data :properties) (get-key (get-key data :properties) property)
+ (has-key? data property) (get-key data property)
+ :else nil))
+
+(defn use-prop [property data]
+ (first-value (use-prop-all property data)))
+
+(defn find-type [mftype items]
+ (concat (filter #(some #{mftype} (type-vector %)) items)
+ (mapcat #(find-type mftype (:children %)) items)))
+(def find-h-entries (partial find-type "h-entry"))
+(def find-h-cards (partial find-type "h-card"))
+
+(defn find-h-entry [items]
+ (-> items find-h-entries first))
+(defn find-h-card [items]
+ (-> items find-h-cards first))
+
+(defn search-photo [node]
+ (if-let [photo (use-prop :photo node)]
+ (if (map? photo) (get-key photo :value) photo)
+ (let [author-photo (when-let [card (use-prop :author node)]
+ (search-photo card))]
+ (if author-photo author-photo
+ (when-let [card (use-prop :org node)]
+ (search-photo card))))))
+
+(defn- assoc-wm-property
+ "Associate like-of, in-reply-to, repost-of and rsvp properties
+ to J2F base map, and add an additional :wm-property key."
+ [base h-entry]
+ (let [data (select-keys (:properties h-entry)
+ [:like-of :repost-of :in-reply-to :rsvp])
+ props (keys data)]
+ (if (empty? props)
+ (assoc base :wm-property :mention)
+ (-> base (merge data) (assoc :wm-property (first props))))))
+
+(defn- assoc-content
+ "Associate content and sumary to J2F base map."
+ [base h-entry]
+ (let [content (use-prop :content h-entry)
+ summary (use-prop :summary h-entry)
+ get-value (fn [data]
+ ;; convert microformats "content.value"
+ ;; to JF2 "content.text", weirdly necessary
+ (if (and (map? data) (:value data))
+ (dissoc (assoc data :text (:value data)) :value)
+ data))]
+ (cond-> base
+ content (assoc :content (get-value content))
+ summary (assoc :summary (get-value summary)))))
+
+(defn- assoc-author
+ "Associate author map to J2F base map."
+ [base h-card photo]
+ (let [author {:type "card"
+ :name (use-prop :name h-card)
+ :url (normalize (use-prop :url h-card))}
+ author (if photo
+ (assoc author :photo photo)
+ author)]
+ (assoc base :author author)))
+
+(defn parse-microformats
+ "Convert microformats h-entry to JF2 format/"
+ [mf]
+ {:pre [(coll? mf)]}
+ (let [items (use-items mf)
+ entry (find-h-entry items)
+ card (find-h-card items)
+ author (or (use-prop :author entry) card)
+ photo (or (search-photo author)
+ (search-photo card))
+ base {:type "entry"
+ :name (use-prop :name entry)
+ :url (let [raw-url (use-prop :url entry)]
+ (normalize
+ (if (map? raw-url) (:value raw-url) raw-url)))
+ :published (use-prop :published entry)}]
+ (-> base
+ (assoc-author author photo)
+ (assoc-content entry)
+ (assoc-wm-property entry))))
diff --git a/src/weepinbell/microformats/reducer.clj b/src/weepinbell/microformats/reducer.clj
deleted file mode 100644
index f718451..0000000
--- a/src/weepinbell/microformats/reducer.clj
+++ /dev/null
@@ -1,26 +0,0 @@
-(ns weepinbell.microformats.reducer
- (:require [weepinbell.microformats.searcher :refer [use-items find-h-card find-h-entry
- search-photo use-prop search-mftype]]
- [clojure.spec.alpha :as s]))
-
-(s/def ::author
- (s/keys :req-un [::name]
- :opt-un [::url ::photo]))
-(s/def ::reduced-microformats
- (s/keys :req-un [::author ::type ::published ::content]))
-
-(defn reduce-mf [target mf]
- (let [items (use-items mf)
- entry (find-h-entry items)
- card (find-h-card items)
- author (or (->> entry (use-prop :author))
- card)
- photo (or (search-photo author)
- (search-photo card))]
- {:author {:name (use-prop :name author)
- :url (use-prop :url author)
- :photo photo}
- :type (search-mftype target entry)
- :published (use-prop :published entry)
- :content (use-prop :content entry)}))
-
diff --git a/src/weepinbell/microformats/searcher.clj b/src/weepinbell/microformats/searcher.clj
deleted file mode 100644
index e156eb9..0000000
--- a/src/weepinbell/microformats/searcher.clj
+++ /dev/null
@@ -1,91 +0,0 @@
-(ns weepinbell.microformats.searcher
- "This namespace contains functions used for searching in a microformats JSON.
- Microformats are indeterministic and dynamic. A specific properties could be
- in different places and many forms, thus will require extra effort finding it."
- (:require [weepinbell.utils.url :refer [url=]]))
-
-;; glossary
-;;
-;; find: finding elements of a specific type in a vector or list
-;; search: searching for s specific key in a map
-;; use: make sure the data node passed to it is indeed of a specific type, reduce it or return nil otherwise
-
-(defn- get-key [data k]
- (cond
- (not (map? data)) nil
- (contains? data k) (get data k)
- (keyword? k) (get data (name k))
- (string? k) (get data (keyword k))
- :else nil))
-
-(defn- has-key? [data k]
- (cond
- (not (map? data)) false
- (contains? data k) true
- (keyword? k) (contains? data (name k))
- (string? k) (contains? data (keyword k))
- :else false))
-
-;; a property value is usually a vector; unwrap it but leave scalars and
-;; unwrapped maps as they are.
-(defn- first-value [value]
- (if (and (coll? value) (not (map? value)))
- (first value)
- value))
-
-(defn- type-vector [item]
- (let [t (get-key item :type)]
- (cond
- (nil? t) []
- (coll? t) t
- (string? t) [t]
- (keyword? t) [(name t)]
- :else [t])))
-
-(defn use-items [data]
- (cond
- (not (coll? data)) nil
- (empty? data) nil
- (has-key? data :items) (get-key data :items)
- (sequential? data) data
- (has-key? data :type) [data]
- :else nil))
-
-(defn use-prop-all [property data]
- (cond
- (not (map? data)) nil
- (has-key? data :properties) (get-key (get-key data :properties) property)
- (has-key? data property) (get-key data property)
- :else nil))
-
-(defn use-prop [property data]
- (first-value (use-prop-all property data)))
-
-(defn find-type [mftype items]
- (concat (filter #(some #{mftype} (type-vector %)) items)
- (mapcat #(find-type mftype (:children %)) items)))
-(def find-h-entries (partial find-type "h-entry"))
-(def find-h-cards (partial find-type "h-card"))
-
-(defn find-h-entry [items]
- (-> items find-h-entries first))
-(defn find-h-card [items]
- (-> items find-h-cards first))
-
-(defn search-photo [node]
- (if-let [photo (use-prop :photo node)]
- (if (map? photo) (get-key photo :value) photo)
- (let [author-photo (when-let [card (use-prop :author node)]
- (search-photo card))]
- (if author-photo author-photo
- (when-let [card (use-prop :org node)]
- (search-photo card))))))
-
-(defn search-mftype [target h-entry]
- (or
- (->> [:like-of :repost-of :in-reply-to :rsvp]
- (filter #(some (fn [x] (url= x target))
- (use-prop-all % h-entry)))
- first)
- :mention))
-
diff --git a/src/weepinbell/migration.clj b/src/weepinbell/migration.clj
index 6579d84..c029b56 100644
--- a/src/weepinbell/migration.clj
+++ b/src/weepinbell/migration.clj
@@ -1,12 +1,29 @@
(ns weepinbell.migration
+ "Migration utilities that is meant to be used as a command in the -main
+ function. Not to be used inside the system.
+ Features may be removed when they are not needed."
(:require [weepinbell.storage.h2 :as h2]
- [weepinbell.storage.edn-storage :as edns])
- #_(:import [org.h2 SQLiteException]))
+ [weepinbell.storage.edn-storage :as edns]
+ [weepinbell.storage.spec]
+ [weepinbell.metadata.microformats :refer [parse-microformats]]
+ [clojure.java.io :as io]
+ [fast-edn.core :as edn]))
+
+(defn mf->jf2
+ "Given a Webmention object with microformats, convert it to
+ jf2, and remove original microformats."
+ [wm]
+ {:pre [(map? wm) (contains? wm :microformats)]}
+ (-> wm
+ (assoc :jf2 (parse-microformats (:microformats wm)))
+ (dissoc :microformats)))
(defn edn->h2
- "Given a EDN file and a SQLite DB path, migrate data from that
+ "Given an EDN file and a SQLite DB path, migrate data from that
edn file to SQLite DB. It appends data to DB so the destination
- db can be empty or it can has some data."
+ db can be empty or it can has some data.
+ This also converts microformats to jf2, as the new H2 schema is
+ incompatible with microformats."
[edn-file h2-db]
{:pre [(some? edn-file) (some? h2-db)]}
(let [ds (h2/make-datasource h2-db)]
@@ -16,12 +33,26 @@
(doseq [wm wms]
(let [id (get-in wm [:webmention :id])]
(println "Appending webmention" id "to database")
- (try
- (h2/insert-webmention ds wm)
-;; (catch SQLiteException e
-;; (println "Something wrong with data or database")
-;; (println (.getMessage e)))
+ (try
+ (let [wm (if (contains? wm :microformats)
+ (mf->jf2 wm) wm)]
+ (h2/insert-webmention ds wm))
(catch Exception e
(println "Something unexpected happened.")
(println (.getMessage e))))))))
(println "Done")))
+
+(defn edn-jf2ify
+ "Given an EDN file with deprecated microformats data, converts
+ microformats to jf2 and generate a new file in new format."
+ [old-edn new-edn]
+ {:pre [(string? old-edn) (string? new-edn)]}
+ (io/make-parents new-edn)
+ (.createNewFile (io/file new-edn))
+ (with-open [rdr (io/reader old-edn)]
+ (doseq [wm (map edn/read-string (line-seq rdr))]
+ (println (get-in wm [:webmention :id]))
+ (let [new-data (if (contains? wm :microformats)
+ (mf->jf2 wm) wm)]
+ (spit new-edn (prn-str new-data) :append true)))))
+
diff --git a/src/weepinbell/processor/wrapper.clj b/src/weepinbell/processor/wrapper.clj
index 8e2a8e2..be43a5b 100644
--- a/src/weepinbell/processor/wrapper.clj
+++ b/src/weepinbell/processor/wrapper.clj
@@ -1,90 +1,74 @@
(ns weepinbell.processor.wrapper
- "Wrapper component wraps a qualified webmention while processing, used by processor.
- It adds microformats and extra data and return complete webmention map ready for storage."
+ "Wrapper component wraps a qualified webmention while processing, part of the pipeline.
+ It adds metadata and extra, and return complete webmention map ready for storage."
(:require [clojure.java.io :as io]
[weepinbell.http.downloader :refer [download-file]]
- [weepinbell.utils.url :refer [valid-http-url? get-base relative? normalize]]
- [weepinbell.microformats.searcher :refer [use-items search-photo find-h-entry find-h-card]]
- [mf2clj.core :as mf-parser]
+ [weepinbell.utils.url :refer [get-base relative? normalize]]
+ [weepinbell.metadata.jf2 :as jf2]
+ [weepinbell.metadata.jf2-spec]
[taoensso.timbre :as timbre]
[weepinbell.utils.digest :refer [digest]]
- [clojure.spec.alpha :as s]))
+ [clojure.string :refer [split]]))
-;;;; spec definitions
-(s/def ::source valid-http-url?)
-(s/def ::target valid-http-url?)
-(s/def ::id (s/and string?
- #(let [len (count %)]
- (or (= len 32)
- (= len 8)))))
-(s/def ::modified int?)
-(s/def ::webmention
- (s/keys :req-un [::source ::target ::id ::modified]))
-
-;; NOTE current microformats parser is unstable
-;; will have to postpone microformats spec definition
-(s/def ::microformats map?)
-
-(s/def ::proxied-photo valid-http-url?)
-(s/def ::extra
- (s/keys :opt-un [::proxied-photo]))
-
-(s/def ::webmention-wrapped
- (s/keys :req-un [::webmention]
- :opt-un [::microformats ::extra]))
-
-;;;; utilities
-(defn- find-mf-photo [mf]
- (or (->> mf use-items find-h-entry search-photo)
- (->> mf use-items find-h-card search-photo)))
-
-(defn- proxy-mf-photo [photo-base raw-photo]
- (if raw-photo
- (let [photo-url (str raw-photo)
- file-ext (or (re-find #"\.(?:jpg|jpeg|png|gif|webp)" photo-url) ".jpg")
- filename (str (digest photo-url) file-ext)
- local-path (str "data/avatars/" filename)
- public-path (str "/avatars/" filename)]
+(defn- proxy-photo
+ "Download a photo to /data/avatars to serve from our server."
+ [photo-base raw-photo]
+ (let [photo-url (str raw-photo)
+ ;; TODO might need to use MIME-type to get ext
+ file-ext (str "." (last (split photo-url #"\.")))
+ filename (str (digest photo-url) file-ext)
+ local-path (str "data/avatars/" filename)
+ public-path (str "/avatars/" filename)]
;; download photo if not exists
- (when-not (.exists (io/file local-path))
- (timbre/info "Downloading microformats photo to proxy external image.")
- (download-file photo-url local-path))
- (str photo-base public-path))
- nil))
+ (when-not (.exists (io/file local-path))
+ (timbre/info "Downloading author photo to serve locally")
+ (download-file photo-url local-path))
+ (str photo-base public-path)))
-(defn complete-url [source-base url]
+(defn- complete-url
+ "Prepend base URL if a URL is relative."
+ [source-base url]
(if (relative? url)
(normalize (str source-base url))
url))
-(defn- wrap-webmention
- "Parse microformats and collect essential info.
- Wrap source, target and collected data into a map and return it."
- [id source-html source target photo-base]
- (timbre/info "parsing microformats and wrapping up webmention" id)
- (let [mf (-> source-html mf-parser/parse :items)
- wm {:source source
- :target target
- :id (digest (str source target))
- :modified (System/currentTimeMillis)}
- proxied-photo (->> mf
- find-mf-photo
- (complete-url (get-base source))
- (proxy-mf-photo photo-base))
- data {:webmention wm
- :microformats mf
- :extra {:proxied-photo proxied-photo}}]
- (when-not (s/valid? ::webmention-wrapped data)
- (timbre/warn "webmention" id "data corrupted before writing to storage"))
- data))
+;; wrappers
+(defn- wrap-webmention-basics
+ "Wrap basic webmention data, :source :target :id and :modified.
+ Will use current system time in milliseconds for :modified."
+ [{:keys [s t]} id]
+ {:webmention {:source s, :target t,
+ :id (or id (digest (str s t))),
+ :modified (System/currentTimeMillis)}})
+
+(defn- wrap-jf2
+ "Parses JF2 from source html, and associate :jf2 key to it."
+ [data source-html]
+ (assoc data :jf2 (-> source-html jf2/parse)))
+
+(defn- wrap-extra
+ "Wrap extra data, i.e. proxied photo."
+ [data photo-base]
+ (let [{:keys [webmention jf2]} data]
+ (if-let [photo (get-in jf2 [:author :photo])]
+ (assoc data :extra
+ {:proxied-photo
+ (->> photo
+ (complete-url (get-base (:source webmention)))
+ (proxy-photo photo-base))})
+ data)))
(defn wrap
+ "Part of the processor pipeline. Takes :source-html from last
+ handler of the pipeline, parses data from it, wraps data
+ to :new-data in the data map, and proceed with ok signal."
[data config]
(when (:ok? data)
(let [{:keys [source-html wm id]} data
- {:keys [s t]} wm]
+ new-data (-> wm (wrap-webmention-basics id)
+ (wrap-jf2 source-html)
+ (wrap-extra (:base-url config)))]
(-> data
- (assoc :new-data
- (wrap-webmention id source-html s t (:base-url config)))
- (assoc :ok true)))))
+ (assoc :new-data new-data)
+ (assoc :ok? true)))))
diff --git a/src/weepinbell/processor/writer.clj b/src/weepinbell/processor/writer.clj
index 02043f1..5bc6725 100644
--- a/src/weepinbell/processor/writer.clj
+++ b/src/weepinbell/processor/writer.clj
@@ -1,5 +1,7 @@
(ns weepinbell.processor.writer
- (:require [taoensso.timbre :as timbre]))
+ (:require [taoensso.timbre :as timbre]
+ [weepinbell.storage.spec]
+ [clojure.spec.alpha :as s]))
(defn write
[data storage callbacks]
@@ -14,4 +16,6 @@
(timbre/info "webmention" id "exists, updating...")
((:update storage) id new-data)
((:on-update callbacks) new-data)))
+ (when (not (s/valid? :weepinbell.storage.spec/webmention-wrapped new-data))
+ (timbre/warn "webmention" id "data does not match spec before writing to storage"))
(-> data (assoc :ok? true)))))
diff --git a/src/weepinbell/storage/h2.clj b/src/weepinbell/storage/h2.clj
index 8d6c2af..4994412 100644
--- a/src/weepinbell/storage/h2.clj
+++ b/src/weepinbell/storage/h2.clj
@@ -5,22 +5,21 @@
[next.jdbc.sql :as sql]
[clojure.java.io :as io]
[fast-edn.core :as edn]
- [weepinbell.utils.url :refer [get-domain get-path]]
- [weepinbell.microformats.searcher :refer [use-items find-h-entry search-mftype]]))
+ [weepinbell.utils.url :refer [get-domain get-path]]))
;; data trasnformation utilities
(defn schema->map
[{:keys [wm_source wm_target hash lastmod
- mf_data photo_url]}]
+ jf2_data photo_url]}]
(let [parse #(if (string? %) (edn/read-string %) %)
- mf (parse mf_data)
+ jf2 (parse jf2_data)
photo photo_url]
(cond-> {:webmention {:source wm_source
:target wm_target
:id hash
:modified lastmod}}
- (some? mf) (assoc :microformats mf)
+ (some? jf2) (assoc :jf2 jf2)
(some? photo) (assoc :extra {:proxied-photo photo}))))
(defn wm->schema [data]
@@ -35,39 +34,39 @@
:wm_target_path (get-path target)
:lastmod (or modified (System/currentTimeMillis))}))
-(defn mf->schema [wm_id data]
- (let [mf (:microformats data)
- target (get-in data [:webmention :target])]
+(defn jf2->schema [wm_id data]
+ (let [jf2 (:jf2 data)
+ get-name #(if % (name %) "mention")]
{:wm_id wm_id
- :mf_type (->> mf use-items find-h-entry (search-mftype target) name)
+ :jf2_type (->> jf2 :wm-property get-name)
;; conver EDN to raw string
- :mf_data (pr-str mf)}))
+ :jf2_data (pr-str jf2)}))
;; sql
(defn insert-webmention [ds data]
(jdbc/with-transaction [tx ds]
(let [tx (jdbc/with-options tx {:builder-fn as-unqualified-lower-maps})
- mf (:microformats data)
+ jf2 (:jf2 data)
photo (get-in data [:extra :proxied-photo])
wm_result (sql/insert! tx :webmention (wm->schema data))
wm_id (:id wm_result)]
- (when mf
- (sql/insert! tx :microformats (mf->schema wm_id data)))
+ (when jf2
+ (sql/insert! tx :jf2 (jf2->schema wm_id data)))
(when photo
(sql/insert! tx :proxied_photo
{:wm_id wm_id
:url photo})))))
(defn delete-webmention [ds hashcode]
- ;; microformats and photo will be deleted by
+ ;; jf2 data and photo will be deleted by
;; cascade rule by SQL, no need to handle it
(sql/delete! ds :webmention {:hash hashcode}))
-(def base-sql "SELECT w.*, m.mf_data, m.mf_type, p.url AS photo_url
+(def base-sql "SELECT w.*, j.jf2_data, j.jf2_type, p.url AS photo_url
FROM webmention w
- LEFT JOIN microformats m
- ON w.id = m.wm_id
+ LEFT JOIN jf2 j
+ ON w.id = j.wm_id
LEFT JOIN proxied_photo p
ON w.id = p.wm_id")
@@ -107,11 +106,11 @@
(map schema->map r))))
(defn- update-related! [ds wm_id data]
- (let [mf (:microformats data)
+ (let [jf2 (:jf2 data)
photo (get-in data [:extra :proxied-photo])]
- (sql/delete! ds :microformats {:wm_id wm_id})
- (when mf
- (sql/insert! ds :microformats (mf->schema wm_id data)))
+ (sql/delete! ds :jf2 {:wm_id wm_id})
+ (when jf2
+ (sql/insert! ds jf2 (jf2->schema wm_id data)))
(sql/delete! ds :proxied_photo {:wm_id wm_id})
(when photo
(sql/insert! ds :proxied_photo {:wm_id wm_id :url photo}))))
diff --git a/src/weepinbell/storage/spec.clj b/src/weepinbell/storage/spec.clj
index d7d6469..6a09b29 100644
--- a/src/weepinbell/storage/spec.clj
+++ b/src/weepinbell/storage/spec.clj
@@ -1,10 +1,40 @@
(ns weepinbell.storage.spec
"Spec of storage middle layer, which is a vtable/map of storage functions.
The spec makes sure any map is a valid storage vtable."
- (:require [clojure.spec.alpha :as s]))
+ (:require [clojure.spec.alpha :as s]
+ [weepinbell.utils.url :refer [valid-http-url?]]))
+
+;; storage vtable
+
+(s/def ::append fn?)
+(s/def ::update fn?)
+(s/def ::delete fn?)
+(s/def ::get fn?)
+(s/def ::get-by-domain fn?)
(s/def ::vtable
(s/keys :req-un [::append ::update ::delete ::get ::get-by-domain]))
-;; TODO Needs to define specs for ::append ::update and other keys.
-;; make sure they're functions and has the right paramters.
+;; storage schema
+
+(s/def ::source valid-http-url?)
+(s/def ::target valid-http-url?)
+(s/def ::id (s/and string?
+ #(let [len (count %)]
+ (or (= len 32)
+ (= len 8)))))
+(s/def ::modified int?)
+(s/def ::webmention
+ (s/keys :req-un [::source ::target ::id ::modified]))
+
+(s/def ::jf2
+ #(s/valid? :weepinbell.metadata.jf2-spec/jf2 %))
+
+(s/def ::proxied-photo valid-http-url?)
+(s/def ::extra
+ (s/keys :opt-un [::proxied-photo]))
+
+(s/def ::webmention-wrapped
+ (s/keys :req-un [::webmention]
+ :opt-un [::jf2 ::extra]))
+
diff --git a/src/weepinbell/utils/url.clj b/src/weepinbell/utils/url.clj
index 9b25cd2..49c8535 100644
--- a/src/weepinbell/utils/url.clj
+++ b/src/weepinbell/utils/url.clj
@@ -58,8 +58,8 @@
query (normalize-query (.getQuery base))]
(str scheme "://" auth path
(when (seq query) (str "?" query))))
- ;; fallback to empty string is url is nil
- (catch NullPointerException _ "")
+ ;; return nil is url is nil
+ (catch NullPointerException _ nil)
;; fallback to original string if syntax is invalid
(catch URISyntaxException _ url)))
diff --git a/src/weepinbell/webserver/handler.clj b/src/weepinbell/webserver/handler.clj
index 7f40f57..70c26d2 100644
--- a/src/weepinbell/webserver/handler.clj
+++ b/src/weepinbell/webserver/handler.clj
@@ -35,30 +35,26 @@
(POST "/send" [source target]
(receive {:s source :t target} wmc config))
;; query webmention list
- (context "/mentions" [page microformats mftype]
+ (context "/mentions" [page jf2type]
(GET "/" [url]
(retrieve-by-url url page store
- {:microformats microformats
- :type mftype}))
+ {:type jf2type}))
(GET "/:domain" [domain]
(retrieve-by-domain domain page store
- {:microformats microformats
- :type mftype})))
+ {:type jf2type})))
;; query count of webmention list
- (context "/count" [mftype]
+ (context "/count" [jf2type]
(GET "/" [url]
- (count-by-url url store {:type mftype}))
+ (count-by-url url store {:type jf2type}))
(GET "/:domain" [domain]
- (count-by-domain domain store {:type mftype})))
+ (count-by-domain domain store {:type jf2type})))
;; query one specific webmention
- (context "/mention" [microformats]
- (GET "/:id" [id] (retrieve-by-id id store
- {:microformats microformats}))
- (GET "/" [source target] (retrieve-by-pair source target store
- {:microformats microformats}))))
+ (context "/mention" []
+ (GET "/:id" [id] (retrieve-by-id id store))
+ (GET "/" [source target] (retrieve-by-pair source target store))))
;; RSS feed
(GET "/feed/:domain" [domain] (gen-feed domain store config))
- ;; host proxied avatars (microformats photos)
+ ;; host proxied avatars (jf2 photos)
(route/files "/avatars/" {:root "data/avatars"})
;; 404
(route/not-found "Not Found")]))
diff --git a/src/weepinbell/webserver/retriever.clj b/src/weepinbell/webserver/retriever.clj
index e8b4485..986db8e 100644
--- a/src/weepinbell/webserver/retriever.clj
+++ b/src/weepinbell/webserver/retriever.clj
@@ -1,10 +1,7 @@
(ns weepinbell.webserver.retriever
"Service getting webmention form storage and wrapping it in a response."
(:require [weepinbell.utils.url :refer [get-domain get-path]]
- [weepinbell.microformats.reducer :refer [reduce-mf]]
- [weepinbell.microformats.searcher :as mfsr]
- [ring.util.http-response :refer [bad-request not-found ok]]
- [clojure.spec.alpha :as s]))
+ [ring.util.http-response :refer [bad-request not-found ok]]))
(def default-page-size 12)
@@ -17,63 +14,46 @@
(defn- respond-webmention-list-json [data]
(cond
- (false? data) (bad-request {:message "Page number is invalid. Either too large or not a number."})
+ (= data :OOB) (bad-request {:message "Page number is too large."})
(empty? data) (not-found {:message "The requested webmention list does not exist."})
:else (ok data)))
;; query list of webmentions
-(defn- mftype= [& types]
- (apply = (map keyword types)))
-(defn- filter-by-mftype
- [mftype data]
- (if (nil? mftype) data
- (->> data
- (map (fn [item]
- [(get-in item [:webmention :target])
- item]))
- (filter (fn [[target item]]
- (->> item
- :microformats
- mfsr/use-items
- mfsr/find-h-entry
- (mfsr/search-mftype target)
- (mftype= mftype))))
- (map last))))
-
-(defn- paging [^Long page ^Long page-size data]
- (if (nil? page) data
- (let [pages (vec (partition-all
- (or page-size default-page-size)
- data))
- page-number (if (number? page) page
- (- (count pages) 1))]
- (get pages page-number))))
-
-(defn- reduce-helper [wm]
- (if-let [mf (:microformats wm)]
- (if (empty? mf)
- (dissoc wm :microformats)
- (let [target (get-in wm [:webmention :target])
- reduced (reduce-mf target mf)]
- (assoc wm :microformats reduced)))
- wm))
-(defn- reduce-microformats [mf-opt data]
- (if (= mf-opt "raw") data
- (map reduce-helper data)))
+(defn- filter-by-jf2type
+ [jf2type data]
+ (if (nil? jf2type) data
+ (filter #(= jf2type (get-in % [:jf2 :wm-property])) data)))
+
+(defn- paging
+ "Paging utility, given a full set of data, partition it and return
+ the requested frame.
+ Returns the last frame if page number is not a number, as a fallback.
+ Returns :OOB (Out Of Bounds) if page number is too large."
+ [^Long page ^Long page-size data]
+ (if (nil? page)
+ ;; if no page number, return the whole query-list
+ data
+ ;; otherwise, partition all and return the requested frame
+ (let [pages (->> data (partition-all page-size) vec)
+ npages (count pages)
+ page-number (if (number? page) page
+ (- (count pages) 1))]
+ (if (>= page-number npages)
+ :OOB ; Out Of Bounds
+ (get pages page-number)))))
(defn query-list
"Get a list of webmentions by certain filters."
- {:pre [(s/keys :req-un [::domain ::path ::page ::page-size ::storage ::mftype])]}
- [opts]
- (let [{:keys [domain path page page-size storage mftype]} opts
- ;; parse to number if possible, use original if not
+ [{:keys [^String domain, ^String path, ^Long page, ^Long page-size
+ storage mftype]}]
+ {:pre [(contains? storage :get-by-domain)]}
+ (let [;; parse to number if possible, use original if not
page (try (Long/parseLong page) (catch Exception _ page))
;; data trasformations
data ((:get-by-domain storage path) domain)
- data (filter-by-mftype mftype data)
- data (paging page page-size data)
- data (reduce-microformats (:microformats opts) data)]
+ data (filter-by-jf2type mftype data)
+ data (paging page page-size data)]
(respond-webmention-list-json data)))
(defn by-domain
@@ -84,8 +64,7 @@
:page page
:page-size default-page-size
:storage storage
- :mftype (:type opts)
- :microformats (:microformats opts)}))
+ :mftype (:type opts)}))
(defn by-url
"Get webmention sent to a particular URL."
@@ -96,8 +75,7 @@
:page page
:page-size default-page-size
:storage storage
- :mftype (:type opts)
- :microformats (:microformats opts)})
+ :mftype (:type opts)})
{:status 400
:body {:message "Missing URL. Specify target URL whose webmentions you are querying."}}))
@@ -106,28 +84,21 @@
(defn query-one
"Get one specific webmention by id or by its source and target."
[opts]
- {:post [(s/keys :req-un [::identifiers ::storage ::microformats])]}
- (let [{:keys [identifiers storage microformats]} opts
- reduce-mf? (not= "raw" microformats)
- data (apply (:get storage) identifiers)
- result (if (and (:microformats data) reduce-mf?)
- (assoc data :microformats
- (reduce-mf (get-in (:microformats data) [:webmention :target])
- (:microformats data)))
- data)]
- (respond-webmention-json result)))
+ {:post [(vector? (:identifiers opts))
+ (contains? (:storage opts) :get)]}
+ (let [{:keys [identifiers storage]} opts
+ data (apply (:get storage) identifiers)]
+ (respond-webmention-json data)))
(defn by-id
"Get one specific webmention by ID."
- [id storage opts]
+ [id storage]
(query-one {:identifiers [id]
- :storage storage
- :microformats (:microformats opts)}))
+ :storage storage}))
(defn by-st-pair
"Get one specific webmention by source and target."
- [source target storage opts]
+ [source target storage]
(query-one {:identifiers [source target]
- :storage storage
- :microformats (:microformats opts)}))
+ :storage storage}))
diff --git a/test/resources/webmention-jf2-test.edn b/test/resources/webmention-jf2-test.edn
new file mode 100644
index 0000000..5ec6a0d
--- /dev/null
+++ b/test/resources/webmention-jf2-test.edn
@@ -0,0 +1,17 @@
+{:webmention {:source "https://alice.example/posts/hello", :target "https://example.com/blog/first-post", :id "0a1b2c3d4e5f60718293a4b5c6d7e8f9", :modified 1700000000000}}
+{:webmention {:source "https://bob.example/notes/re-first-post", :target "https://example.com/blog/first-post", :id "1b2c3d4e5f60718293a4b5c6d7e8f90a", :modified 1700000100000}, :jf2 {:type "entry", :name "Re: first post", :url nil, :published "2023-11-14T22:13:20+00:00", :author {:type "card", :name "Bob", :url "https://bob.example"}, :content {:html "Really enjoyed <em>this</em>.", :text "Really enjoyed this."}, :wm-property :mention}}
+{:webmention {:source "https://carol.example/likes/first-post", :target "https://example.com/blog/first-post", :id "2c3d4e5f60718293a4b5c6d7e8f90a1b", :modified 1700000200000}, :extra {:proxied-photo "http://localhost:3000/avatars/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png"}, :jf2 {:type "entry", :name nil, :url nil, :published nil, :author {:type "card", :name "Carol", :url "https://carol.example", :photo "https://carol.example/avatar.png"}, :like-of ["https://example.com/blog/first-post"], :wm-property :like-of}}
+{:webmention {:source "https://dan.example/reposts", :target "https://example.com/blog/first-post", :id "3d4e5f60718293a4b5c6d7e8f90a1b2c", :modified 1700000300000}, :jf2 {:type "entry", :name nil, :url nil, :published nil, :author {:type "card", :name "Dan", :url "https://dan.example"}, :repost-of ["https://example.com/blog/first-post"], :wm-property :repost-of}}
+{:webmention {:source "https://erin.example/replies/42", :target "https://example.com/blog/second-post", :id "4e5f60718293a4b5c6d7e8f90a1b2c3d", :modified 1700000400000}, :extra {:proxied-photo "http://localhost:3000/avatars/0123456789abcdef0123456789abcdef.jpg"}, :jf2 {:type "entry", :name nil, :url nil, :published nil, :author {:type "card", :name "Erin", :url "https://erin.example", :photo "https://erin.example/media/erin.jpg"}, :content {:text "I disagree, but interesting take."}, :in-reply-to ["https://example.com/blog/second-post"], :wm-property :in-reply-to}}
+{:webmention {:source "https://frank.example/events/attend", :target "https://example.com/events/meetup-2024", :id "5f60718293a4b5c6d7e8f90a1b2c3d4e", :modified 1700000500000}, :jf2 {:type "entry", :name "I will be there", :url nil, :published nil, :author {:type "card", :name "Frank", :url nil}, :rsvp ["yes"], :wm-property :rsvp}}
+{:webmention {:source "https://grace.example/blog/a-link", :target "https://example.com/blog/post-a", :id "60718293a4b5c6d7e8f90a1b2c3d4e5f", :modified 1700000600000}, :jf2 {:type "entry", :name "A link worth sharing", :url nil, :published nil, :author {:type "card", :name "Grace", :url "https://grace.example"}, :wm-property :mention}}
+{:webmention {:source "https://heidi.example/blog/another-link", :target "https://example.com/blog/post-b", :id "718293a4b5c6d7e8f90a1b2c3d4e5f60", :modified 1700000700000}, :jf2 {:type "entry", :name "Also worth sharing", :url nil, :published nil, :author {:type "card", :name "Heidi", :url "https://heidi.example"}, :wm-property :mention}}
+{:webmention {:source "https://ivan.example/about", :target "https://example.com/blog/post-a", :id "8293a4b5c6d7e8f90a1b2c3d4e5f6071", :modified 1700000800000}, :extra {:proxied-photo "http://localhost:3000/avatars/fedcba9876543210fedcba9876543210.webp"}, :jf2 {:type "entry", :name nil, :url nil, :published nil, :author {:type "card", :name "Ivan", :url "https://ivan.example", :photo "https://ivan.example/photo.webp"}, :wm-property :mention}}
+{:webmention {:source "https://judy.example/empty", :target "https://example.com/blog/post-b", :id "93a4b5c6d7e8f90a1b2c3d4e5f607182", :modified 1700000900000}, :jf2 {:type "entry", :name nil, :url nil, :published nil, :author {:type "card", :name nil, :url nil}, :wm-property :mention}}
+{:webmention {:source "https://karl.example/notes/nested", :target "https://example.com/blog/post-c", :id "a4b5c6d7e8f90a1b2c3d4e5f60718293", :modified 1700001000000}, :jf2 {:type "entry", :name "Nested children", :url nil, :published nil, :author {:type "card", :name "Karl", :url nil}, :wm-property :mention}}
+{:webmention {:source "https://laura.example/feed", :target "https://example.com/blog/post-c", :id "b5c6d7e8f90a1b2c3d4e5f60718293a4", :modified 1700001100000}, :jf2 {:type "entry", :name nil, :url nil, :published nil, :author {:type "card", :name "Laura", :url nil}, :like-of ["https://example.com/blog/post-c/"], :wm-property :like-of}}
+{:webmention {:source "https://mallory.example/posts/trailing", :target "https://example.com/blog/post-d/?ref=reader", :id "c6d7e8f90a1b2c3d4e5f60718293a4b5", :modified 1700001200000}, :jf2 {:type "entry", :name nil, :url nil, :published nil, :author {:type "card", :name nil, :url nil}, :content {:text "Citation with trailing slash weblog."}, :in-reply-to ["https://example.com/blog/post-d"], :wm-property :in-reply-to}}
+{:webmention {:source "https://niaj.example/mentions/no-mf", :target "https://example.com/blog/post-d/", :id "d7e8f90a1b2c3d4e5f60718293a4b5c6", :modified 1700001300000}, :extra {:proxied-photo "http://localhost:3000/avatars/mnopqrstuvwxyz0123456789abcdef01.png"}}
+{:webmention {:source "https://olivia.example/notes/older-id", :target "https://example.com/blog/first-post", :id "e8f90a1b2c3d4e5f60718293a4b5c6d7", :modified 1699999999000}, :jf2 {:type "entry", :name nil, :url nil, :published nil, :author {:type "card", :name "Olivia", :url "https://olivia.example"}, :like-of ["https://example.com/blog/first-post"], :wm-property :like-of}}
+{:webmention {:source "https://peggy.example/legacy-hash", :target "https://example.org/notes/legacy", :id "deadbeef", :modified 1699999998000}, :jf2 {:type "entry", :name nil, :url nil, :published nil, :author {:type "card", :name nil, :url nil}, :repost-of ["https://example.org/notes/legacy"], :wm-property :repost-of}}
+{:webmention {:source "https://quinn.example/posts/long-id", :target "https://example.org/notes/legacy", :id "1234567890abcdef1234567890abcdef", :modified 1699999997000}}
diff --git a/test/resources/webmention-test.edn b/test/resources/webmention-test.edn
new file mode 100644
index 0000000..e655daf
--- /dev/null
+++ b/test/resources/webmention-test.edn
@@ -0,0 +1,17 @@
+{:webmention {:source "https://alice.example/posts/hello" :target "https://example.com/blog/first-post" :id "0a1b2c3d4e5f60718293a4b5c6d7e8f9" :modified 1700000000000}}
+{:webmention {:source "https://bob.example/notes/re-first-post" :target "https://example.com/blog/first-post" :id "1b2c3d4e5f60718293a4b5c6d7e8f90a" :modified 1700000100000} :microformats [{:type ["h-entry"] :properties {:name ["Re: first post"] :content [{:value "Really enjoyed this." :html "Really enjoyed <em>this</em>."}] :author [{:type ["h-card"] :properties {:name ["Bob"] :url ["https://bob.example"]}}] :published ["2023-11-14T22:13:20+00:00"]}}]}
+{:webmention {:source "https://carol.example/likes/first-post" :target "https://example.com/blog/first-post" :id "2c3d4e5f60718293a4b5c6d7e8f90a1b" :modified 1700000200000} :microformats [{:type ["h-entry"] :properties {:like-of ["https://example.com/blog/first-post"] :author [{:type ["h-card"] :properties {:name ["Carol"] :photo ["https://carol.example/avatar.png"] :url ["https://carol.example"]}}]}}] :extra {:proxied-photo "http://localhost:3000/avatars/9f8e7d6c5b4a39281706f5e4d3c2b1a0.png"}}
+{:webmention {:source "https://dan.example/reposts" :target "https://example.com/blog/first-post" :id "3d4e5f60718293a4b5c6d7e8f90a1b2c" :modified 1700000300000} :microformats [{:type ["h-entry"] :properties {:repost-of ["https://example.com/blog/first-post"] :author [{:type ["h-card"] :properties {:name ["Dan"] :url ["https://dan.example"]}}]}}]}
+{:webmention {:source "https://erin.example/replies/42" :target "https://example.com/blog/second-post" :id "4e5f60718293a4b5c6d7e8f90a1b2c3d" :modified 1700000400000} :microformats [{:type ["h-entry"] :properties {:in-reply-to ["https://example.com/blog/second-post"] :content [{:value "I disagree, but interesting take."}] :author [{:type ["h-card"] :properties {:name ["Erin"] :photo [{:value "https://erin.example/media/erin.jpg" :alt "Erin"} ] :url ["https://erin.example"]}}]}}] :extra {:proxied-photo "http://localhost:3000/avatars/0123456789abcdef0123456789abcdef.jpg"}}
+{:webmention {:source "https://frank.example/events/attend" :target "https://example.com/events/meetup-2024" :id "5f60718293a4b5c6d7e8f90a1b2c3d4e" :modified 1700000500000} :microformats [{:type ["h-entry"] :properties {:rsvp ["yes"] :name ["I will be there"] :author [{:type ["h-card"] :properties {:name ["Frank"]}}]}}]}
+{:webmention {:source "https://grace.example/blog/a-link" :target "https://example.com/blog/post-a" :id "60718293a4b5c6d7e8f90a1b2c3d4e5f" :modified 1700000600000} :microformats [{:type ["h-entry"] :properties {:name ["A link worth sharing"] :author [{:type ["h-card"] :properties {:name ["Grace"] :url ["https://grace.example"]}}]}}]}
+{:webmention {:source "https://heidi.example/blog/another-link" :target "https://example.com/blog/post-b" :id "718293a4b5c6d7e8f90a1b2c3d4e5f60" :modified 1700000700000} :microformats [{:type ["h-entry"] :properties {:name ["Also worth sharing"] :author [{:type ["h-card"] :properties {:name ["Heidi"] :url ["https://heidi.example"]}}]}}]}
+{:webmention {:source "https://ivan.example/about" :target "https://example.com/blog/post-a" :id "8293a4b5c6d7e8f90a1b2c3d4e5f6071" :modified 1700000800000} :microformats [{:type ["h-card"] :properties {:name ["Ivan"] :url ["https://ivan.example"] :photo ["https://ivan.example/photo.webp"]}}] :extra {:proxied-photo "http://localhost:3000/avatars/fedcba9876543210fedcba9876543210.webp"}}
+{:webmention {:source "https://judy.example/empty" :target "https://example.com/blog/post-b" :id "93a4b5c6d7e8f90a1b2c3d4e5f607182" :modified 1700000900000} :microformats []}
+{:webmention {:source "https://karl.example/notes/nested" :target "https://example.com/blog/post-c" :id "a4b5c6d7e8f90a1b2c3d4e5f60718293" :modified 1700001000000} :microformats [{:type ["h-entry"] :properties {:name ["Nested children"] :author [{:type ["h-card"] :properties {:name ["Karl"]}}]} :children [{:type ["h-cite"] :properties {:name ["Cited work"] :url ["https://somewhere.example/paper"]}}]}]}
+{:webmention {:source "https://laura.example/feed" :target "https://example.com/blog/post-c" :id "b5c6d7e8f90a1b2c3d4e5f60718293a4" :modified 1700001100000} :microformats [{:type ["h-feed"] :properties {:name ["Laura's feed"]} :children [{:type ["h-entry"] :properties {:like-of ["https://example.com/blog/post-c/"] :author [{:type "h-card" :properties {:name ["Laura"]}}]}}]}]}
+{:webmention {:source "https://mallory.example/posts/trailing" :target "https://example.com/blog/post-d/?ref=reader" :id "c6d7e8f90a1b2c3d4e5f60718293a4b5" :modified 1700001200000} :microformats [{:type ["h-entry"] :properties {:in-reply-to ["https://example.com/blog/post-d"] :content [{:value "Citation with trailing slash weblog."}]}}]}
+{:webmention {:source "https://niaj.example/mentions/no-mf" :target "https://example.com/blog/post-d/" :id "d7e8f90a1b2c3d4e5f60718293a4b5c6" :modified 1700001300000} :extra {:proxied-photo "http://localhost:3000/avatars/mnopqrstuvwxyz0123456789abcdef01.png"}}
+{:webmention {:source "https://olivia.example/notes/older-id" :target "https://example.com/blog/first-post" :id "e8f90a1b2c3d4e5f60718293a4b5c6d7" :modified 1699999999000} :microformats [{:type ["h-entry"] :properties {:like-of ["https://example.com/blog/first-post"] :author [{:type ["h-card"] :properties {:name ["Olivia"] :url ["https://olivia.example"]}}]} :children []}]}
+{:webmention {:source "https://peggy.example/legacy-hash" :target "https://example.org/notes/legacy" :id "deadbeef" :modified 1699999998000} :microformats [{:type ["h-entry"] :properties {:repost-of ["https://example.org/notes/legacy"]}}]}
+{:webmention {:source "https://quinn.example/posts/long-id" :target "https://example.org/notes/legacy" :id "1234567890abcdef1234567890abcdef" :modified 1699999997000}}
diff --git a/test/weepinbell/jf2_test.clj b/test/weepinbell/jf2_test.clj
new file mode 100644
index 0000000..acaaaca
--- /dev/null
+++ b/test/weepinbell/jf2_test.clj
@@ -0,0 +1,279 @@
+(ns weepinbell.jf2-test
+ (:require [clojure.test :refer [deftest testing is]]
+ [hickory.core :as hickory]
+ [weepinbell.metadata.microformats :refer [parse-microformats]]
+ [weepinbell.metadata.json-ld :refer [parse-json-ld]]
+ [weepinbell.metadata.html-head :refer [parse-meta]]
+ [weepinbell.metadata.jf2 :refer [parse supplement-merge]]))
+
+(def to-hick (comp hickory/as-hickory hickory/parse))
+
+;; microformats -> JF2
+
+(def test-data
+ [{:in {:items [{:type ["h-entry"], :properties {:author [{:type ["h-card"], :properties {:name ["A. Developer"], :url ["https://example.com/"]}, :value "A. Developer"}], :name ["Hello World"], :summary ["Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus imperdiet ultrices pulvinar."], :url ["https://example.com/2015/10/21"], :published ["2015-10-21T12:00:00-0700"], :content [{:html "<p>Donec dapibus enim lacus, <i>a vehicula magna bibendum non</i>. Phasellus id lacinia felis, vitae pellentesque enim. Sed at quam dui. Suspendisse accumsan, est id pulvinar consequat, urna ex tincidunt enim, nec sodales lectus nulla et augue. Cras venenatis vehicula molestie. Donec sagittis elit orci, sit amet egestas ex pharetra in.</p>", :value "Donec dapibus enim lacus, a vehicula magna bibendum non. Phasellus id lacinia felis, vitae pellentesque enim. Sed at quam dui. Suspendisse accumsan, est id pulvinar consequat, urna ex tincidunt enim, nec sodales lectus nulla et augue. Cras venenatis vehicula molestie. Donec sagittis elit orci, sit amet egestas ex pharetra in."}]}}]}
+ :out {:type "entry", :author {:type "card", :url "https://example.com", :name "A. Developer"}, :url "https://example.com/2015/10/21", :published "2015-10-21T12:00:00-0700", :name "Hello World", :summary "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus imperdiet ultrices pulvinar.", :content {:html "<p>Donec dapibus enim lacus, <i>a vehicula magna bibendum non</i>. Phasellus id lacinia felis, vitae pellentesque enim. Sed at quam dui. Suspendisse accumsan, est id pulvinar consequat, urna ex tincidunt enim, nec sodales lectus nulla et augue. Cras venenatis vehicula molestie. Donec sagittis elit orci, sit amet egestas ex pharetra in.</p>", :text "Donec dapibus enim lacus, a vehicula magna bibendum non. Phasellus id lacinia felis, vitae pellentesque enim. Sed at quam dui. Suspendisse accumsan, est id pulvinar consequat, urna ex tincidunt enim, nec sodales lectus nulla et augue. Cras venenatis vehicula molestie. Donec sagittis elit orci, sit amet egestas ex pharetra in."}, :wm-property :mention}}])
+
+(deftest jf2-parse-microformats-test
+ (testing "Parsing example microformats to J2F, from https://jf2.spec.indieweb.org/"
+ (doseq [data test-data]
+ (is (= (:out data)
+ (parse-microformats (:in data)))))))
+
+;; JSON-LD -> JF2
+
+(def test-jsonld "
+ <script type=\"application/ld+json\">{\n\t\"@context\": \"https://schema.org\",\n\t\"@type\": \"Article\",\n\t\"publisher\": {\n\t\t\"@type\": \"Organization\",\n\t\t\"name\": \"freeCodeCamp.org\",\n\t\t\"url\": \"https://www.freecodecamp.org/news/\",\n\t\t\"logo\": {\n\t\t\t\"@type\": \"ImageObject\",\n\t\t\t\"url\": \"https://cdn.freecodecamp.org/platform/universal/fcc_primary.svg\",\n\t\t\t\"width\": 2100,\n\t\t\t\"height\": 240\n\t\t}\n\t},\n\t\"image\": {\n\t\t\"@type\": \"ImageObject\",\n\t\t\"url\": \"https://www.freecodecamp.org/news/content/images/2020/03/open-graph.jpg\",\n\t\t\"width\": 600,\n\t\t\"height\": 400\n\t},\n\t\"url\": \"https://www.freecodecamp.org/news/what-is-open-graph-and-how-can-i-use-it-for-my-website/\",\n\t\"mainEntityOfPage\": {\n\t\t\"@type\": \"WebPage\",\n\t\t\"@id\": \"https://www.freecodecamp.org/news/\"\n\t},\n\t\"datePublished\": \"2020-03-26T14:51:33.000Z\",\n\t\"dateModified\": \"2020-03-26T14:51:33.000Z\",\n\t\"keywords\": \"Digital Marketing , HTML, marketing, open graph, social media, tech , Web Development\",\n\t\"description\": \"It can take a lot of time to build content and maintain a website. How can we make sure our content stands out when getting shared on social feeds around the internet?\\n\\nWhat is Open Graph?\\nWhy do I need it?\\nWhat happens if I don’t have it?\\nStarting w...\",\n\t\"headline\": \"What is Open Graph and how can I use it for my website?\",\n\t\"author\": {\n\t\t\"@type\": \"Person\",\n\t\t\"name\": \"Colby Fayock\",\n\t\t\"url\": \"https://www.freecodecamp.org/news/author/colbyfayock/\",\n\t\t\"sameAs\": [\n\t\t\t\"http://colbyfayock.com/\",\n\t\t\t\"https://x.com/colbyfayock\"\n\t\t],\n\t\t\"image\": {\n\t\t\t\"@type\": \"ImageObject\",\n\t\t\t\"url\": \"https://cdn.hashnode.com/res/hashnode/image/upload/v1601829804572/1eKdHWWIz.jpeg\",\n\t\t\t\"width\": 600,\n\t\t\t\"height\": 600\n\t\t}\n\t}\n}</script>
+ ")
+
+(def jsonld->jf2
+ {:type "entry", :name "What is Open Graph and how can I use it for my website?", :url "https://www.freecodecamp.org/news/what-is-open-graph-and-how-can-i-use-it-for-my-website/", :author {:type "card", :name "Colby Fayock", :url "https://www.freecodecamp.org/news/author/colbyfayock/", :photo "https://cdn.hashnode.com/res/hashnode/image/upload/v1601829804572/1eKdHWWIz.jpeg"}, :published "2020-03-26T14:51:33.000Z", :content "", :wm-property :mention, :summary "It can take a lot of time to build content and maintain a website. How can we make sure our content stands out when getting shared on social feeds around the internet?\n\nWhat is Open Graph?\nWhy do I need it?\nWhat happens if I don’t have it?\nStarting w..."})
+
+(def test-jsonld-html
+ (str "<html><head>" test-jsonld "</head></html>"))
+
+(deftest jf2-parse-jsonld-test
+ (testing "parse JSON-LD from a document"
+ (is (= (-> test-jsonld-html to-hick parse-json-ld)
+ jsonld->jf2)))
+ (testing "parse JSON-LD from raw script element"
+ (is (= (-> test-jsonld to-hick parse-json-ld)
+ jsonld->jf2))))
+
+;; Open Graph, <head> tags -> JF2
+
+(def link-and-meta-tags
+ {:open-graph
+ "<html><head>
+ <title>Fallback title</title>
+ <link rel=\"icon\" href=\"https://example.com/favicon.ico\" type=\"image/png\">
+ <link rel=\"canonical\" href=\"https://example.com/open-graph\">
+ <meta name=\"description\" content=\"Fallback description\">
+ <meta property=\"og:title\" content=\"Open Graph title\">
+ <meta property=\"og:url\" content=\"https://example.com/open-graph\">
+ <meta property=\"og:description\" content=\"Open Graph description\">
+ <meta property=\"article:author\" content=\"Open Graph author\">
+ <meta property=\"article:published_time\" content=\"2024-01-02T03:04:05Z\">
+ </head></html>"
+
+ :twitter
+ "<html><head>
+ <title>Title fallback</title>
+ <link rel=\"canonical\" href=\"https://example.com/twitter\">
+ <meta name=\"twitter:title\" content=\"Twitter title\">
+ <meta name=\"twitter:url\" content=\"https://example.com/twitter\">
+ <meta name=\"twitter:description\" content=\"Twitter description\">
+ <meta name=\"twitter:creator\" content=\"@writer\">
+ </head></html>"
+
+ :profile-username
+ "<html><head>
+ <meta property=\"og:title\" content=\"Profile title\">
+ <meta property=\"profile:username\" content=\"profile-username\">
+ </head></html>"
+
+ :profile-fullname
+ "<html><head>
+ <meta property=\"profile:first_name\" content=\"Rich\">
+ <meta property=\"profile:last_name\" content=\"Hickey\">
+ </head></html>"
+
+ :basic
+ "<html><head>
+ <title>Document title</title>
+ <link rel=\"canonical\" href=\"https://example.com/cononical\">
+ <meta name=\"description\" content=\"Document description\">
+ </head></html>"})
+
+(deftest jf2-parse-head-test
+ (testing "Open Graph to JF2"
+ (is (= (parse-meta (to-hick (:open-graph link-and-meta-tags)))
+ {:type "entry"
+ :name "Open Graph title"
+ :url "https://example.com/open-graph"
+ :author {:type "card"
+ :name "Open Graph author"
+ :url nil
+ :photo "https://example.com/favicon.ico"}
+ :published "2024-01-02T03:04:05Z"
+ :content ""
+ :wm-property :mention
+ :summary "Open Graph description"}))
+ (is (= (parse-meta (to-hick (:twitter link-and-meta-tags)))
+ {:type "entry"
+ :name "Twitter title"
+ :url "https://example.com/twitter"
+ :author {:type "card"
+ :name "@writer"
+ :url nil
+ :photo nil}
+ :published nil
+ :content ""
+ :wm-property :mention
+ :summary "Twitter description"}))
+ (is (= (parse-meta (to-hick (:profile-username link-and-meta-tags)))
+ {:type "entry"
+ :name "Profile title"
+ :url nil
+ :author {:type "card"
+ :name "profile-username"
+ :url nil
+ :photo nil}
+ :published nil
+ :content ""
+ :wm-property :mention
+ :summary nil}))
+ (is (= (parse-meta (to-hick (:profile-fullname link-and-meta-tags)))
+ {:type "entry"
+ :name nil
+ :url nil
+ :author {:type "card"
+ :name "Rich Hickey"
+ :url nil
+ :photo nil}
+ :published nil
+ :content ""
+ :wm-property :mention
+ :summary nil}))
+ (is (= (parse-meta (to-hick (:basic link-and-meta-tags)))
+ {:type "entry"
+ :name "Document title"
+ :url "https://example.com/cononical"
+ :author {:type "card"
+ :name nil
+ :url nil
+ :photo nil}
+ :published nil
+ :content ""
+ :wm-property :mention
+ :summary "Document description"}))))
+
+;; utilities
+
+(deftest supplement-merge-test
+ (testing "supplement-merge supplements A with values A lacks but B has"
+ (is (= (supplement-merge
+ {:a nil :b 1} {:a 1 :b 3})
+ {:a 1 :b 1})))
+ (testing "supplement-merge never overrides existing values, except nil"
+ (is (= (supplement-merge {:a "hi"} {:a "bye"}) {:a "hi"})))
+ (testing "supplement-merge does deep merge as well"
+ (is (= (supplement-merge {:a {:aa "hello" :ab nil}
+ :b nil}
+ {:a {:aa "byebye" :ab "yes"}
+ :b nil})
+ {:a {:aa "hello" :ab "yes"} :b nil})))
+ (testing "supplement-merge adds missing keys"
+ (is (= (supplement-merge {:a "n"} {:a "m" :b "s"})
+ {:a "n" :b "s"}))))
+
+;; compile all sources
+
+(def compiling-test-data
+ [{:html "<html><head></head><body>
+ <article class=\"h-entry\">
+ <h1 class=\"p-name\">Microformats title</h1>
+ <a class=\"u-url\" href=\"https://example.com/microformats\">Permalink</a>
+ <time class=\"dt-published\" datetime=\"2024-02-03T04:05:06Z\">February 3</time>
+ <div class=\"p-author h-card\">
+ <span class=\"p-name\">Microformats author</span>
+ <a class=\"u-url\" href=\"https://example.com/author\">Author</a>
+ </div>
+ <div class=\"e-content\">Microformats content</div>
+ <a class=\"u-in-reply-to\" href=\"https://example.com/original\">Original</a>
+ </article>
+ </body></html>"
+ :out {:type "entry"
+ :name "Microformats title"
+ :url "https://example.com/microformats"
+ :author {:type "card"
+ :name "Microformats author"
+ :url "https://example.com/author"
+ :photo nil}
+ :published "2024-02-03T04:05:06Z"
+ :content {:html "Microformats content"
+ :text "Microformats content"}
+ :wm-property :in-reply-to
+ :in-reply-to ["https://example.com/original"]
+ :summary nil}}
+ {:html "<html><head>
+ <title>Document title</title>
+ <link rel=\"canonical\" href=\"https://example.com/head-url\">
+ <meta property=\"og:title\" content=\"Open Graph title\">
+ <meta property=\"og:url\" content=\"https://example.com/og-url\">
+ <meta property=\"og:description\" content=\"Open Graph summary\">
+ <meta property=\"article:author\" content=\"Head author\">
+ <meta property=\"article:published_time\" content=\"2024-03-04T05:06:07Z\">
+ <script type=\"application/ld+json\">{
+ \"@context\": \"https://schema.org\",
+ \"@type\": \"Article\",
+ \"headline\": \"JSON-LD title\",
+ \"url\": \"https://example.com/json-ld-url\",
+ \"datePublished\": \"2024-04-05T06:07:08Z\",
+ \"description\": \"JSON-LD summary\",
+ \"author\": {
+ \"@type\": \"Person\",
+ \"name\": \"JSON-LD author\",
+ \"url\": \"https://example.com/json-ld-author\",
+ \"image\": {\"url\": \"https://example.com/author.jpg\"}
+ }
+ }</script>
+ </head><body>
+ <article class=\"h-entry\">
+ <h1 class=\"p-name\">Microformats title</h1>
+ <a class=\"u-url\" href=\"https://example.com/microformats-url\">Permalink</a>
+ <time class=\"dt-published\" datetime=\"2024-01-02T03:04:05Z\">January 2</time>
+ <div class=\"p-author h-card\">
+ <span class=\"p-name\">Microformats author</span>
+ <a class=\"u-url\" href=\"https://example.com/microformats-author\">Author</a>
+ </div>
+ <div class=\"e-content\">Microformats content</div>
+ </article>
+ </body></html>"
+ :out {:type "entry"
+ :name "Microformats title"
+ :url "https://example.com/microformats-url"
+ :author {:type "card"
+ :name "Microformats author"
+ :url "https://example.com/microformats-author"
+ :photo "https://example.com/author.jpg"}
+ :published "2024-01-02T03:04:05Z"
+ :content {:html "Microformats content"
+ :text "Microformats content"}
+ :wm-property :mention
+ :summary "Open Graph summary"}}
+ {:html "<html><head>
+ <meta name=\"description\" content=\"Head description\">
+ <meta name=\"twitter:title\" content=\"Twitter title\">
+ <meta name=\"twitter:url\" content=\"https://example.com/twitter-url\">
+ <meta name=\"twitter:creator\" content=\"@head-author\">
+ <script type=\"application/ld+json\">{
+ \"@context\": \"https://schema.org\",
+ \"@type\": \"Article\",
+ \"headline\": \"JSON-LD title\",
+ \"url\": \"https://example.com/json-ld-url\",
+ \"datePublished\": \"2024-05-06T07:08:09Z\",
+ \"author\": {
+ \"@type\": \"Person\",
+ \"name\": \"JSON-LD author\",
+ \"url\": \"https://example.com/json-ld-author\"
+ },
+ \"description\": \"JSON-LD description\"
+ }</script>
+ </head><body></body></html>"
+ :out {:type "entry"
+ :name "Twitter title"
+ :url "https://example.com/twitter-url"
+ :author {:type "card"
+ :name "@head-author"
+ :url "https://example.com/json-ld-author"
+ :photo nil}
+ :published "2024-05-06T07:08:09Z"
+ :content ""
+ :wm-property :mention
+ :summary "Head description"}}])
+
+(deftest jf2-parse-test
+ (testing "compiles JF2 from all available metadata sources"
+ (doseq [{:keys [html out]} compiling-test-data]
+ (is (= out (parse html))))))
diff --git a/test/weepinbell/microformats_test.clj b/test/weepinbell/microformats_test.clj
deleted file mode 100644
index 5f17a5b..0000000
--- a/test/weepinbell/microformats_test.clj
+++ /dev/null
@@ -1,122 +0,0 @@
-(ns weepinbell.microformats-test
- (:require [weepinbell.microformats.reducer :refer [reduce-mf]]
- [weepinbell.microformats.searcher :as sr]
- [clojure.spec.alpha :as s]
- [clojure.test :refer [deftest testing is]]))
-
-(def testdata
- [{:target "https://test.com/"
- :in {:items [{:type ["h-entry"]
- :properties {:content [{:html "<div>test</div>"
- :text "test"}]
- :in-reply-to ["https://test.com"]
- :published [52341243232]
- :author [{:type ["h-card"]
- :properties {:name ["test"]
- :nickname ["testie"]
- :photo [{:alt "testie's selfie"
- :value "https://test.com/photo.png"}]
- :url ["https://test.com"]}}]}}]}
- :out {:author
- {:name "test",
- :url "https://test.com",
- :photo "https://test.com/photo.png"},
- :type :in-reply-to,
- :published 52341243232,
- :content {:html "<div>test</div>", :text "test"}}}
- {:target ""
- :in {:items [{:type ["h-entry"]
- :properties {:content [{:html "<div>test</div>"
- :text "test"}]
- :published [52341243232]}}
- {:type ["h-card"]
- :properties {:name ["test"]
- :nickname ["testie"]
- :photo [{:alt "testie's selfie"
- :value "https://test.com/photo.png"}]
- :url ["https://test.com"]}}]}
- :out {:author
- {:name "test",
- :url "https://test.com",
- :photo "https://test.com/photo.png"},
- :type :mention,
- :published 52341243232,
- :content {:html "<div>test</div>", :text "test"}}}])
-
-(def spec-check (partial s/valid? :weepinbell.microformats.reducer/reduced-microformats))
-(deftest reducer-test
- (testing "Normal microformats reduction"
- (->> testdata
- (map #(let [reduced (reduce-mf (:target %) (:in %))]
- (is (= reduced (:out %)))
- (is (spec-check reduced))))
- doall))
- (testing "spec check invalid reduced microformats"
- (is (not (spec-check {:author nil :type :mention :published 1 :content nil})))
- (is (not (spec-check [])))))
-
-(deftest searcher-test
- (testing "use-items gets a list of microformats items"
- (->> testdata
- (map #(is (= (sr/use-items (:in %))
- (sr/use-items (:items (:in %)))
- (get-in % [:in :items]))))
- doall))
- (testing "use-items returns nil if not items or cannot get items in it"
- (is (nil? (sr/use-items {:no-items nil})))
- (is (nil? (sr/use-items "no-items")))
- (is (nil? (sr/use-items [])))
- (is (nil? (sr/use-items 1))))
- (testing "use-props get properties of a microformats item"
- (->> testdata
- (map #(is (= (->> % sr/use-items sr/find-h-entry (sr/use-prop :content))
- (->> % sr/use-items sr/find-h-entry (sr/use-prop "content"))
- (get-in % [:items first :properties :content first]))))
- doall))
- (testing "use-props works with out :properties key as well"
- (is (= (sr/use-prop :key {:properties {:key ["value"]}})
- (sr/use-prop :key {:key ["value"]})
- "value")))
- (testing "find-type should find items with a specific type given a collection of items"
- (is (= ["h-entry"]
- (->> testdata first :in sr/use-items sr/find-h-entry :type)))
- (is (= ["h-card"]
- (->> testdata second :in sr/use-items sr/find-h-card :type))))
- (testing "find-type should recursively find qualified items inside items' children"
- (let [testdata-a {:items [{:type ["h-feed"]
- :children [{:type ["h-entry"]
- :properties {:content [{:html "<div>test</div>"
- :text "test"}]
- :published [52341243232]}}]}]}]
- (is (= ["h-entry"]
- (->> testdata-a sr/use-items sr/find-h-entry :type))))
- (testing "find-type should find all matched items on different layers"
- (let [testdata-a {:items [{:type ["h-feed"]
- :children [{:type ["h-entry"]
- :children [{:type ["h-card"]}]}
- {:type ["h-card"]}]}
- {:type ["h-card"]}]}]
- (is (= 3 (->> testdata-a sr/use-items sr/find-h-cards count)))))
- (testing "find-type should not return empty seq if nothing's matched"
- (let [testdata-a {:items [{:type ["h-feed"]
- :children []}]}]
- (is (->> testdata-a sr/use-items sr/find-h-entry empty?)))))
- (testing "search photos in h-card"
- (let [photo-url "https://test.com/photo.png"
- testdata-a {:type ["h-card"]
- :properties {:photo [photo-url]}}
- testdata-b {:type ["h-card"]
- :properties {:photo [{:alt "photo"
- :value photo-url}]}}
- testdata-c {:type ["h-entry"]
- :properties {:author [{:type ["h-card"]
- :properties {:photo [photo-url]}}]}}
- testdata-d {:type ["h-card"]
- :properties {:author [{:type ["h-card"]
- :properties {}}]
- :org [{:type ["h-card"]
- :properties {:photo [photo-url]}}]}}]
- (is (= photo-url (sr/search-photo testdata-a)))
- (is (= photo-url (sr/search-photo testdata-b)))
- (is (= photo-url (sr/search-photo testdata-c)))
- (is (= photo-url (sr/search-photo testdata-d))))))
diff --git a/test/weepinbell/migration_test.clj b/test/weepinbell/migration_test.clj
new file mode 100644
index 0000000..6cf06e0
--- /dev/null
+++ b/test/weepinbell/migration_test.clj
@@ -0,0 +1,39 @@
+(ns weepinbell.migration-test
+ (:require [weepinbell.migration :as m]
+ [weepinbell.metadata.jf2-spec]
+ [fast-edn.core :as edn]
+ [weepinbell.storage.h2 :as h2]
+ [next.jdbc :as jdbc]
+ [clojure.spec.alpha :as s]
+ [clojure.test :refer [deftest testing is]]
+ [clojure.java.io :as io]))
+
+(def webmention-test-edn
+ (with-open [rdr (io/reader (io/resource "webmention-test.edn"))]
+ (->> rdr line-seq (map edn/read-string) doall)))
+
+(deftest test-migration-from-mf-to-jf2
+ (testing "Convert a Webmention object with microformats to one with jf2"
+ (let [data (->> webmention-test-edn
+ (filter #(some? (:microformats %)))
+ first)
+ converted (m/mf->jf2 data)]
+ (is (contains? converted :jf2))
+ (is (not (contains? converted :microformats)))
+ (is (s/valid? :weepinbell.metadata.jf2-spec/jf2
+ (:jf2 converted))))))
+
+(deftest test-migration-from-edn-to-h2
+ (testing "Migrating EDN file storage to H2 database storage"
+ (let [source-edn (io/resource "webmention-test.edn")
+ target-h2 "tmp/webmention-test-h2"]
+ ;; assert migration succeed with no error
+ (is (nil? (try
+ (m/edn->h2 source-edn target-h2)
+ (catch Throwable _ :fail))))
+ ;; assert data count
+ (let [ds (h2/make-datasource target-h2)
+ result (jdbc/execute! ds ["SELECT count(*) FROM webmention"])
+ len (get (first result) (keyword "count(*)"))]
+ (is (= len (count webmention-test-edn))))
+ (io/delete-file (str target-h2 ".mv.db")))))
diff --git a/test/weepinbell/retriever_test.clj b/test/weepinbell/retriever_test.clj
index 98102ef..72a5bf7 100644
--- a/test/weepinbell/retriever_test.clj
+++ b/test/weepinbell/retriever_test.clj
@@ -1,35 +1,27 @@
(ns weepinbell.retriever-test
(:require [clojure.test :refer [deftest testing is]]
+ [weepinbell.utils.digest :refer [digest]]
[weepinbell.webserver.retriever :as ret]))
(def page-size 1)
+
+(defn- make-bare-wm [number]
+ {:webmention {:source (str number) :target (str number)
+ :id (digest (str number))
+ :modified (System/currentTimeMillis)}})
+(defn- make-typed-wm [type number]
+ (-> (make-bare-wm number)
+ (assoc :jf2 {:wm-property type})))
+
(deftest retrieve-by-domain-test
(testing "by-domain list paging"
(with-redefs [weepinbell.webserver.retriever/default-page-size page-size]
- (let [mock-data [{:webmention {:source "1" :target "1"}}
- {:webmention {:source "2" :target "2"}}]
+ (let [mock-data (mapv make-bare-wm (range 2))
mock-store {:get-by-domain (fn [& _] mock-data)}
resp0 (ret/by-domain "" 0 mock-store {})
resp1 (ret/by-domain "" 1 mock-store {})]
(is (= page-size (count (:body resp0))))
(is (= page-size (count (:body resp1)))))))
- (testing "by-domain: if page is a string that can be parsed to number, it should work"
- (with-redefs [weepinbell.webserver.retriever/default-page-size page-size]
- (let [mock-data [{:webmention {:source "1" :target "1"}}
- {:webmention {:source "2" :target "2"}}]
- mock-store {:get-by-domain (fn [& _] mock-data)}
- resp0 (ret/by-domain "" "0" mock-store {})
- resp1 (ret/by-domain "" 0 mock-store {})]
- (is (= resp0 resp1)))))
- (testing "by-domain: if page is not number, return the last page frame"
- (with-redefs [weepinbell.webserver.retriever/default-page-size page-size]
- (let [mock-data [{:webmention {:source "1" :target "1"}}
- {:webmention {:source "2" :target "2"}}]
- mock-store {:get-by-domain (fn [& _] mock-data)}
- resp (ret/by-domain "" true mock-store {})]
- (is (= 200 (:status resp)))
- (is (= (-> mock-data last :webmention)
- (-> resp :body last :webmention))))))
(testing "by-domain return 404 when list is empty"
(let [mock-store {:get-by-domain (fn [& _] [])}
resp (ret/by-domain "" nil mock-store {})]
@@ -40,8 +32,7 @@
(deftest retrieve-by-url-test
(testing "by-url list paging"
(with-redefs [weepinbell.webserver.retriever/default-page-size page-size]
- (let [mock-data [{:webmention {:source "1" :target "2"}}
- {:webmention {:source "1" :target "2"}}]
+ (let [mock-data (mapv make-bare-wm (range 2))
mock-store {:get-by-domain (fn [& _] mock-data)}
resp0 (ret/by-url "2" 0 mock-store {})
resp1 (ret/by-url "2" 1 mock-store {})]
@@ -57,15 +48,63 @@
(testing "by-url return 404 when list is empty"
(with-redefs [weepinbell.webserver.retriever/default-page-size page-size]
(let [mock-store {:get-by-domain (fn [& _])}
- resp (ret/by-url "" 0 mock-store {})]
+ resp (ret/by-url "" nil mock-store {})]
(is (= 404 (:status resp)))
(is (= "The requested webmention list does not exist."
(get-in resp [:body :message])))))))
-(deftest retrieve-by-id-test
- (testing "by-id return 404 when no webmention is matched"
- (let [mock-store {:get (fn [& _] [])}
- resp (ret/by-id "" mock-store {})]
+(defn- should-404 [data]
+ (let [mock-store {:get (fn [& _] data)}
+ resp (ret/by-id "" mock-store)]
(is (= 404 (:status resp)))
(is (= "The requested webmention does not exist."
- (get-in resp [:body :message]))))))
+ (get-in resp [:body :message])))))
+(deftest retrieve-by-id-test
+ (testing "by-id return 404 when no webmention is matched"
+ (doseq [no-data [{} [] nil]]
+ (should-404 no-data))))
+
+(deftest filter-by-jf2type-test
+ (testing "by-domain ?jf2type filter"
+ (let [like-wms (mapv (partial make-typed-wm :like-of) (range 2))
+ reply-wms (mapv (partial make-typed-wm :in-reply-to) (range 3))
+ mock-data (reduce into [] [like-wms reply-wms])
+ mock-store {:get-by-domain (fn [& _] mock-data)}
+ ;; responses
+ resp-like (ret/by-domain "" 0 mock-store {:type :like-of})
+ resp-reply (ret/by-domain "" 0 mock-store {:type :in-reply-to})]
+ (is (= reply-wms (:body resp-reply)))
+ (is (= like-wms (:body resp-like)))))
+ (testing "by-url ?jf2type filter"
+ (let [like-wms (mapv (partial make-typed-wm :like-of) (range 2))
+ reply-wms (mapv (partial make-typed-wm :in-reply-to) (range 3))
+ mock-data (reduce into [] [like-wms reply-wms])
+ mock-store {:get-by-domain (fn [& _] mock-data)}
+ ;; responses
+ resp-like (ret/by-url "" 0 mock-store {:type :like-of})
+ resp-reply (ret/by-url "" 0 mock-store {:type :in-reply-to})]
+ (is (= reply-wms (:body resp-reply)))
+ (is (= like-wms (:body resp-like))))))
+
+(deftest paging-edge-cases-test
+ (testing "if page is a string that can be parsed to number, it should work"
+ (with-redefs [weepinbell.webserver.retriever/default-page-size page-size]
+ (let [mock-data (mapv make-bare-wm (range 2))
+ mock-store {:get-by-domain (fn [& _] mock-data)}
+ resp0 (ret/by-domain "" "0" mock-store {})
+ resp1 (ret/by-domain "" 0 mock-store {})]
+ (is (= resp0 resp1)))))
+ (testing "if page is not number, return the last page frame"
+ (with-redefs [weepinbell.webserver.retriever/default-page-size page-size]
+ (let [mock-data (mapv make-bare-wm (range 2))
+ mock-store {:get-by-domain (fn [& _] mock-data)}
+ resp (ret/by-domain "" true mock-store {})]
+ (is (= 200 (:status resp)))
+ (is (= (-> mock-data last :webmention)
+ (-> resp :body last :webmention))))))
+ (testing "if page is larger than total amount of pages, return error"
+ (with-redefs [weepinbell.webserver.retriever/default-page-size page-size]
+ (let [mock-data [(make-bare-wm page-size)]
+ mock-store {:get-by-domain (fn [& _] mock-data)}
+ resp (ret/by-domain "" (inc page-size) mock-store {})]
+ (is (= 400 (:status resp)))))))
diff --git a/test/weepinbell/storage_test.clj b/test/weepinbell/storage_test.clj
index 34fb104..a25634a 100644
--- a/test/weepinbell/storage_test.clj
+++ b/test/weepinbell/storage_test.clj
@@ -130,13 +130,13 @@
:source "https://source.net"
:target "https://target.net"
:modified (System/currentTimeMillis)}
- :microformats mf
+ :jf2 mf
:extra {:proxied-photo photo}}]
((:append storage) data)
(wait-til! ((:get storage) id))
(let [retrieved ((:get storage) id)]
(is (= photo (get-in retrieved [:extra :proxied-photo])))
- (is (= mf (:microformats retrieved))))
+ (is (= mf (:jf2 retrieved))))
((:delete storage) id)
(wait-til! (not ((:get storage) id)))))
diff --git a/test/weepinbell/wrapper_test.clj b/test/weepinbell/wrapper_test.clj
index 74bfbcf..e405d4c 100644
--- a/test/weepinbell/wrapper_test.clj
+++ b/test/weepinbell/wrapper_test.clj
@@ -1,25 +1,105 @@
(ns weepinbell.wrapper-test
(:require [clojure.test :refer [deftest testing is]]
- [clojure.string :refer [includes?]]
+ [clojure.string :refer [includes? replace]]
+ [clojure.spec.alpha :as s]
[weepinbell.utils.digest :refer [digest]]
- [weepinbell.processor.wrapper :refer [wrap]]))
+ [weepinbell.processor.wrapper :refer [wrap]]
+ [weepinbell.metadata.jf2-spec]
+ [weepinbell.http.client]
+ [weepinbell.utils.url :refer [url=]]
+ [org.httpkit.server :refer [run-server]]
+ [clojure.java.io :as io]))
+
+(def photo-server-port 23232)
+
+(def fake-photo (str "http://localhost:" photo-server-port "/favicon.avif"))
(def normal-source-html
- "<div class=h-entry>
+ (str "<html><head><title>Name</title></head>
+ <div class=h-entry>
<div class=e-content>test</div>
<div class=\"p-author h-card\">
- <img alt=test src=\"https://source.net/favicon.png\" />
+ <img alt=test src=\"" fake-photo "\" />
</div>
- </div>")
+ </div>
+ </html>"))
(deftest happy-path-wrap
(testing "Wrapping a normal webmention"
- (let [wrapped (:new-data (wrap {:ok? true
+ (with-redefs [weepinbell.http.client/default-options {:timeout 100}]
+ (let [;; serve local fake photo
+ photo-digest (digest fake-photo)
+ local-photo (str "data/avatars/" photo-digest ".avif")
+ photo-handler (fn [_] {:status 200
+ :headers {"Content-Type" "image/png"}
+ :body "ReaLly4IMagE"})
+ photo-server (run-server photo-handler {:port photo-server-port})
+ ;; test wrapping
+ test-wrap (fn [] (wrap {:ok? true
:wm {:s "https://source.net"
:t "https://target.org"}
:source-html normal-source-html}
+ {:base-url "https://resource.com"}))
+ wrapped (:new-data (test-wrap))]
+ (is (= "https://source.net" (get-in wrapped [:webmention :source])))
+ (is (= "https://target.org" (get-in wrapped [:webmention :target])))
+ (is (= "b36497f812ccfde649996e21f8df83a6" (get-in wrapped [:webmention :id])))
+ (is (s/valid? :weepinbell.metadata.jf2-spec/jf2 (:jf2 wrapped)))
+ (is (.exists (io/file local-photo)))
+ (is (includes? (get-in wrapped [:extra :proxied-photo]) photo-digest))
+ ;; wrap again to test if using local avatars
+ (is (includes? (get-in (test-wrap) [:new-data :extra :proxied-photo]) photo-digest))
+ ;; clean up
+ (io/delete-file local-photo)
+ (photo-server))))
+ (testing "Wrapping a normal webmention with relative photo-url"
+ (with-redefs [weepinbell.http.client/default-options {:timeout 100}]
+ (let [;; serve local fake photo
+ photo-digest (digest fake-photo)
+ local-photo (str "data/avatars/" photo-digest ".avif")
+ photo-handler (fn [_] {:status 200
+ :headers {"Content-Type" "image/png"}
+ :body "ReaLly4IMagE"})
+ photo-server (run-server photo-handler {:port photo-server-port})
+ ;; test wrapping
+ source-html (replace normal-source-html fake-photo "/favicon.avif")
+ wrapped (:new-data (wrap {:ok? true
+ :wm {:s (str "http://localhost:" photo-server-port)
+ :t "https://target.org"}
+ :source-html source-html}
{:base-url "https://resource.com"}))]
- (is (= "https://source.net" (get-in wrapped [:webmention :source])))
- (is (= "https://target.org" (get-in wrapped [:webmention :target])))
- (is (= "b36497f812ccfde649996e21f8df83a6" (get-in wrapped [:webmention :id])))
- (is (includes? (get-in wrapped [:extra :proxied-photo]) (digest "https://source.net/favicon.png"))))))
+ (is (includes? (get-in wrapped [:extra :proxied-photo]) photo-digest))
+ ;; clean up
+ (io/delete-file local-photo)
+ (photo-server))))
+ (testing "Wrapping a webmention with no photo to proxy, checking JF2"
+ (let [source-html "<html><head><title>Name</title></head>
+ <div class=h-entry>
+ <div class=e-content>test</div>
+ <div class=\"p-author h-card\">
+ <a href=\"https://author.com/\" class=\"p-name u-url\">Author</a>
+ </div>
+ </div>
+ </html>"
+ wrapped (:new-data (wrap {:ok? true
+ :wm {:s "https://source.net"
+ :t "https://target.org"}
+ :source-html source-html}
+ {:base-url "https://resource.com"}))]
+ (is (= "Name" (get-in wrapped [:jf2 :name])))
+ (is (= "Author" (get-in wrapped [:jf2 :author :name])))
+ (is (url= "https://author.com/" (get-in wrapped [:jf2 :author :url])))
+ (is (= "test" (get-in wrapped [:jf2 :content :text])))))
+ (testing "Wrapper uses provided id"
+ (with-redefs [weepinbell.http.client/default-options {:timeout 100}]
+ (let [id "totally-accurate-SHA256-hash"
+ wrapped (:new-data (wrap {:ok? true
+ :id id
+ :wm {:s "https://source.net"
+ :t "https://target.org"}
+ :source-html normal-source-html}
+ {:base-url "https://resource.com"}))]
+ (is (= id (get-in wrapped [:webmention :id])))))))
+
+
+