summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rw-r--r--scripts/fetch_webmentions.go116
1 files changed, 0 insertions, 116 deletions
diff --git a/scripts/fetch_webmentions.go b/scripts/fetch_webmentions.go
deleted file mode 100644
index d568af31..00000000
--- a/scripts/fetch_webmentions.go
+++ /dev/null
@@ -1,116 +0,0 @@
-package main
-
-import (
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "os"
- "regexp"
- "sort"
-)
-
-type JF2Response struct {
- Children []map[string]interface{} `json:"children"`
-}
-
-func mustEnv(key string) string {
- v := os.Getenv(key)
- if v == "" {
- fmt.Fprintf(os.Stderr, "missing env: %s\n", key)
- os.Exit(1)
- }
- return v
-}
-
-func main() {
- token := mustEnv("WEBMENTION_IO_TOKEN")
- domain := mustEnv("WEBMENTION_IO_DOMAIN")
-
- var all []map[string]interface{}
- client := &http.Client{}
- page := 0
-
- for {
- url := fmt.Sprintf(
- "https://webmention.io/api/mentions.jf2?domain=%s&token=%s&page=%d",
- domain, token, page,
- )
-
- resp, err := client.Get(url)
- if err != nil {
- panic(err)
- }
- body, _ := io.ReadAll(resp.Body)
- resp.Body.Close()
-
- var jf2 JF2Response
- if err := json.Unmarshal(body, &jf2); err != nil {
- panic(err)
- }
-
- if len(jf2.Children) == 0 {
- break
- }
-
- all = append(all, jf2.Children...)
- page++
- }
-
- reDomain := regexp.MustCompile(`^https?://[^/]+`)
-
- for _, c := range all {
- if t, ok := c["wm-target"].(string); ok {
- c["wm-target"] = reDomain.ReplaceAllString(t, "")
- }
- }
-
- sort.Slice(all, func(i, j int) bool {
- ti, _ := all[i]["wm-target"].(string)
- tj, _ := all[j]["wm-target"].(string)
- return ti < tj
- })
-
- grouped := map[string][]map[string]interface{}{}
-
- for _, c := range all {
- target, _ := c["wm-target"].(string)
-
- item := map[string]interface{}{
- "type": c["wm-property"],
- "author": c["author"],
- "url": c["url"],
- "source": c["wm-source"],
- "published": c["published"],
- "received": c["wm-received"],
- "content": c["content"],
- }
-
- grouped[target] = append(grouped[target], item)
- }
-
- for target, items := range grouped {
- sort.Slice(items, func(i, j int) bool {
- timeI, okI := items[i]["published"].(string)
- timeJ, okJ := items[j]["published"].(string)
-
- if okI && okJ {
- return timeI < timeJ
- }
-
- receivedI, _ := items[i]["received"].(string)
- receivedJ, _ := items[j]["received"].(string)
- return receivedI < receivedJ
- })
- grouped[target] = items
- }
-
- if err := os.MkdirAll("data", 0755); err != nil {
- panic(err)
- }
-
- out, _ := json.MarshalIndent(grouped, "", " ")
- if err := os.WriteFile("data/webmentions.json", out, 0644); err != nil {
- panic(err)
- }
-}