refactor: rewrite webmention fetching in Go and pagethrough whole list

1 files changed, 11 insertions, 58 deletions

diff --git a/.forgejo/workflows/fetch-webmentions.yml b/.forgejo/workflows/fetch-webmentions.yml
index a1100ba3..b42c9a80 100644
--- a/.forgejo/workflows/fetch-webmentions.yml
+++ b/.forgejo/workflows/fetch-webmentions.yml
@@ -2,73 +2,26 @@ name: Fetch Webmentions
 
 on:
   schedule:
-    - cron: "*/30 * * * *" # every 30 minutes
+    - cron: "*/30 * * * *"
   workflow_dispatch:
 
 jobs:
   fetch:
-    runs-on: docker
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
 
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+
       - name: Fetch webmentions
         env:
-          TOKEN: ${{ secrets.WEBMENTION_IO_TOKEN }}
-          DOMAIN: ${{ secrets.WEBMENTION_IO_DOMAIN }}
+          WEBMENTION_IO_TOKEN: ${{ secrets.WEBMENTION_IO_TOKEN }}
+          WEBMENTION_IO_DOMAIN: ${{ secrets.WEBMENTION_IO_DOMAIN }}
         run: |
-          mkdir -p data
-
-          curl -s "https://webmention.io/api/mentions.jf2?domain=$DOMAIN&token=$TOKEN" > data/webmentions_raw.json
-
-          cat > /tmp/process_webmentions.py << 'PYTHON_SCRIPT'
-          import json
-          import re
-          import sys
-
-          with open('data/webmentions_raw.json', 'r') as f:
-              data = json.load(f)
-
-          children = data.get('children', [])
-
-          # Process children
-          processed = []
-          for child in children:
-              # Remove domain from wm-target
-              wm_target = child.get('wm-target', '')
-              wm_target = re.sub(r'^https?://[^/]+', '', wm_target)
-              child['wm-target'] = wm_target
-              processed.append(child)
-
-          # Sort by wm-target
-          processed.sort(key=lambda x: x.get('wm-target', ''))
-
-          # Group by wm-target
-          from itertools import groupby
-          result = {}
-          for target, group in groupby(processed, key=lambda x: x.get('wm-target', '')):
-              items = []
-              for item in group:
-                  processed_item = {
-                      'type': item.get('wm-property'),
-                      'author': item.get('author'),
-                      'url': item.get('url'),
-                      'source': item.get('wm-source'),
-                      'published': item.get('published'),
-                      'received': item.get('wm-received'),
-                      'content': item.get('content') if 'content' in item else None
-                  }
-                  items.append(processed_item)
-              result[target] = items
-
-          with open('data/webmentions.json', 'w') as f:
-              json.dump(result, f, indent=2)
-
-          import os
-          os.remove('data/webmentions_raw.json')
-          PYTHON_SCRIPT
-
-          python3 /tmp/process_webmentions.py
+          go run fetch_webmentions.go
 
       - name: Commit if changed
         run: |
@@ -77,8 +30,8 @@ jobs:
             exit 0
           fi
 
-          git config user.name "github-actions"
-          git config user.email "github-actions@github.com"
+          git config user.name "forgejo-actions"
+          git config user.email "forgejo-actions@eltr.ac"
           git add data/webmentions.json
           git commit -m "chore: update webmentions"
           git push