refactor: rewrite webmention fetching in Go and pagethrough whole list

2 files changed, 111 insertions, 58 deletions

diff --git a/.forgejo/workflows/fetch-webmentions.yml b/.forgejo/workflows/fetch-webmentions.yml
index a1100ba3..b42c9a80 100644
--- a/.forgejo/workflows/fetch-webmentions.yml
+++ b/.forgejo/workflows/fetch-webmentions.yml
@@ -2,73 +2,26 @@ name: Fetch Webmentions
 
 on:
   schedule:
-    - cron: "*/30 * * * *" # every 30 minutes
+    - cron: "*/30 * * * *"
   workflow_dispatch:
 
 jobs:
   fetch:
-    runs-on: docker
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
 
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+
       - name: Fetch webmentions
         env:
-          TOKEN: ${{ secrets.WEBMENTION_IO_TOKEN }}
-          DOMAIN: ${{ secrets.WEBMENTION_IO_DOMAIN }}
+          WEBMENTION_IO_TOKEN: ${{ secrets.WEBMENTION_IO_TOKEN }}
+          WEBMENTION_IO_DOMAIN: ${{ secrets.WEBMENTION_IO_DOMAIN }}
         run: |
-          mkdir -p data
-
-          curl -s "https://webmention.io/api/mentions.jf2?domain=$DOMAIN&token=$TOKEN" > data/webmentions_raw.json
-
-          cat > /tmp/process_webmentions.py << 'PYTHON_SCRIPT'
-          import json
-          import re
-          import sys
-
-          with open('data/webmentions_raw.json', 'r') as f:
-              data = json.load(f)
-
-          children = data.get('children', [])
-
-          # Process children
-          processed = []
-          for child in children:
-              # Remove domain from wm-target
-              wm_target = child.get('wm-target', '')
-              wm_target = re.sub(r'^https?://[^/]+', '', wm_target)
-              child['wm-target'] = wm_target
-              processed.append(child)
-
-          # Sort by wm-target
-          processed.sort(key=lambda x: x.get('wm-target', ''))
-
-          # Group by wm-target
-          from itertools import groupby
-          result = {}
-          for target, group in groupby(processed, key=lambda x: x.get('wm-target', '')):
-              items = []
-              for item in group:
-                  processed_item = {
-                      'type': item.get('wm-property'),
-                      'author': item.get('author'),
-                      'url': item.get('url'),
-                      'source': item.get('wm-source'),
-                      'published': item.get('published'),
-                      'received': item.get('wm-received'),
-                      'content': item.get('content') if 'content' in item else None
-                  }
-                  items.append(processed_item)
-              result[target] = items
-
-          with open('data/webmentions.json', 'w') as f:
-              json.dump(result, f, indent=2)
-
-          import os
-          os.remove('data/webmentions_raw.json')
-          PYTHON_SCRIPT
-
-          python3 /tmp/process_webmentions.py
+          go run fetch_webmentions.go
 
       - name: Commit if changed
         run: |
@@ -77,8 +30,8 @@ jobs:
             exit 0
           fi
 
-          git config user.name "github-actions"
-          git config user.email "github-actions@github.com"
+          git config user.name "forgejo-actions"
+          git config user.email "forgejo-actions@eltr.ac"
           git add data/webmentions.json
           git commit -m "chore: update webmentions"
           git push
diff --git a/scripts/fetch-webmentions.go b/scripts/fetch-webmentions.go
new file mode 100644
index 00000000..ca03995c
--- /dev/null
+++ b/scripts/fetch-webmentions.go
@@ -0,0 +1,100 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"regexp"
+	"sort"
+)
+
+type JF2Response struct {
+	Children []map[string]interface{} `json:"children"`
+}
+
+func mustEnv(key string) string {
+	v := os.Getenv(key)
+	if v == "" {
+		fmt.Fprintf(os.Stderr, "missing env: %s\n", key)
+		os.Exit(1)
+	}
+	return v
+}
+
+func main() {
+	token := mustEnv("WEBMENTION_IO_TOKEN")
+	domain := mustEnv("WEBMENTION_IO_DOMAIN")
+
+	var all []map[string]interface{}
+	client := &http.Client{}
+	page := 0
+
+	for {
+		url := fmt.Sprintf(
+			"https://webmention.io/api/mentions.jf2?domain=%s&token=%s&page=%d",
+			domain, token, page,
+		)
+
+		resp, err := client.Get(url)
+		if err != nil {
+			panic(err)
+		}
+		body, _ := io.ReadAll(resp.Body)
+		resp.Body.Close()
+
+		var jf2 JF2Response
+		if err := json.Unmarshal(body, &jf2); err != nil {
+			panic(err)
+		}
+
+		if len(jf2.Children) == 0 {
+			break
+		}
+
+		all = append(all, jf2.Children...)
+		page++
+	}
+
+	reDomain := regexp.MustCompile(`^https?://[^/]+`)
+
+	for _, c := range all {
+		if t, ok := c["wm-target"].(string); ok {
+			c["wm-target"] = reDomain.ReplaceAllString(t, "")
+		}
+	}
+
+	sort.Slice(all, func(i, j int) bool {
+		ti, _ := all[i]["wm-target"].(string)
+		tj, _ := all[j]["wm-target"].(string)
+		return ti < tj
+	})
+
+	grouped := map[string][]map[string]interface{}{}
+
+	for _, c := range all {
+		target, _ := c["wm-target"].(string)
+
+		item := map[string]interface{}{
+			"type":      c["wm-property"],
+			"author":    c["author"],
+			"url":       c["url"],
+			"source":    c["wm-source"],
+			"published": c["published"],
+			"received":  c["wm-received"],
+			"content":   c["content"],
+		}
+
+		grouped[target] = append(grouped[target], item)
+	}
+
+	if err := os.MkdirAll("data", 0755); err != nil {
+		panic(err)
+	}
+
+	out, _ := json.MarshalIndent(grouped, "", "  ")
+	if err := os.WriteFile("data/webmentions.json", out, 0644); err != nil {
+		panic(err)
+	}
+}