From 65f6195d709cffa54dfd78c059c0ec342564c4da Mon Sep 17 00:00:00 2001 From: eltrac Date: Wed, 8 Apr 2026 17:41:15 +0800 Subject: feat: add microformats wrapper and rename microformats.clj -> microformats_parser.clj --- src/weepinbell/microformats.clj | 35 ---------------------- src/weepinbell/microformats_parser.clj | 34 +++++++++++++++++++++ src/weepinbell/microformats_wrapper.clj | 52 +++++++++++++++++++++++++++++++++ src/weepinbell/wrapper.clj | 4 +-- 4 files changed, 88 insertions(+), 37 deletions(-) delete mode 100644 src/weepinbell/microformats.clj create mode 100644 src/weepinbell/microformats_parser.clj create mode 100644 src/weepinbell/microformats_wrapper.clj (limited to 'src') diff --git a/src/weepinbell/microformats.clj b/src/weepinbell/microformats.clj deleted file mode 100644 index 9184d9a..0000000 --- a/src/weepinbell/microformats.clj +++ /dev/null @@ -1,35 +0,0 @@ -(ns weepinbell.microformats - (:require [clojure.string :refer [includes? lower-case]] - [taoensso.timbre :as timbre]) - (:import [com.kylewm.mf2j Mf2Parser] - [java.net URI])) - -(defn parse - "Parse microformats from a URL." - [url] - (let [max-retries 3] - (loop [attempt 1] - (let [result (try - (let [parser (doto (Mf2Parser.) - (.setIncludeAlternates true) - (.setIncludeRelUrls true))] - {:ok true :data (.parse parser (URI. url))}) - (catch Exception e - {:ok false :exception e}))] - (if (:ok result) - (:data result) - (let [e (:exception result) - msg (.getMessage e)] - (if (and (<= attempt max-retries) - (or (instance? java.net.SocketTimeoutException e) - (includes? (lower-case (str msg)) "timeout"))) - (let [wait-time (* attempt 2000)] - (timbre/warn (format "MF2 Parsing timeout for %s (attempt %d/%d). Retrying in %dms..." - url attempt max-retries wait-time)) - (Thread/sleep wait-time) - (recur (inc attempt))) - (do - (timbre/warn (format "MF2 Parsing failed for %s after %d attempts. Error: %s" - url attempt (or msg "Unknown error"))) - nil)))))))) - diff --git a/src/weepinbell/microformats_parser.clj b/src/weepinbell/microformats_parser.clj new file mode 100644 index 0000000..4bf915c --- /dev/null +++ b/src/weepinbell/microformats_parser.clj @@ -0,0 +1,34 @@ +(ns weepinbell.microformats-parser + (:require [clojure.string :refer [includes? lower-case]] + [taoensso.timbre :as timbre]) + (:import [com.kylewm.mf2j Mf2Parser] + [java.net URI])) + +(defn parse + "Parse microformats from a URL." + [url] + (let [max-retries 3] + (loop [attempt 1] + (let [result (try + (let [parser (doto (Mf2Parser.) + (.setIncludeAlternates true) + (.setIncludeRelUrls true))] + {:ok true :data (.parse parser (URI. url))}) + (catch Exception e + {:ok false :exception e}))] + (if (:ok result) + (:data result) + (let [e (:exception result) + msg (.getMessage e)] + (if (and (<= attempt max-retries) + (or (instance? java.net.SocketTimeoutException e) + (includes? (lower-case (str msg)) "timeout"))) + (let [wait-time (* attempt 2000)] + (timbre/warn (format "MF2 Parsing timeout for %s (attempt %d/%d). Retrying in %dms..." + url attempt max-retries wait-time)) + (Thread/sleep wait-time) + (recur (inc attempt))) + (do + (timbre/warn (format "MF2 Parsing failed for %s after %d attempts. Error: %s" + url attempt (or msg "Unknown error"))) + nil)))))))) diff --git a/src/weepinbell/microformats_wrapper.clj b/src/weepinbell/microformats_wrapper.clj new file mode 100644 index 0000000..05a49f4 --- /dev/null +++ b/src/weepinbell/microformats_wrapper.clj @@ -0,0 +1,52 @@ +(ns weepinbell.microformats-wrapper + (:require [weepinbell.utils :refer [javamap->map]] + [weepinbell.storage :as store])) + +(defn- find-item + "Find specific type of items (like h-entry, h-card) in parsed microformats." + [typ mf] + (let [items (get mf "items") + get-type #(first (get % "type"))] + (filter #(= (get-type %) typ) items))) + +(defn- get-prop + "Use keyword get get a property value from a microformats item." + [entry prop] + (-> entry + javamap->map + :properties + javamap->map + (get prop))) + +(defn- get-prop-first + "Alias of (first (get-prop ...))" + [entry prop] + (first (get-prop entry prop))) + +(defn- get-entry-type + "Get type of an h-entry item in keyword. (:like-of, :repost-of, :in-reply-to)" + [entry] + ;; rsvp type is not supported yet + (or + (->> [:like-of :repost-of :in-reply-to] + (filter #(get-prop entry %)) + first) + :mention)) + +(defn- get-card-info + "Get essential information of an h-card item and flatten it into a map." + [card] + {:name (get-prop-first card :name) + :nickname (get-prop-first card :nickname) + :photo (get-prop-first card :photo) + :url (get-prop-first card :url)}) + +(defn wrap-entry + "Wrap parsed raw microformats (h-entry) data. If no h-entry found, return nil." + [entry] + (when entry + {:author (get-card-info (get-prop-first entry :author)) + :type (get-entry-type entry) + :published (get-prop-first entry :published) + :content (javamap->map (get-prop-first entry :content))})) + diff --git a/src/weepinbell/wrapper.clj b/src/weepinbell/wrapper.clj index 10cad52..97d81cf 100644 --- a/src/weepinbell/wrapper.clj +++ b/src/weepinbell/wrapper.clj @@ -3,7 +3,7 @@ file-exists? download-file]] [weepinbell.config :refer [get-config]] - [weepinbell.microformats :as microformats] + [weepinbell.microformats-parser :as mf-parser] [taoensso.timbre :as timbre] [md5.core :refer [string->md5-hex]])) @@ -33,7 +33,7 @@ "Parse microformats and collect essential info. Wrap source, target and collected data into a Clojure map and return it." [source target] (let [mf (-> source - microformats/parse + mf-parser/parse javamap->map :items first javamap->map) -- cgit v1.2.3