diff options
| -rw-r--r-- | .forgejo/workflows/fetch-webmentions.yml | 69 | ||||
| -rw-r--r-- | scripts/fetch-webmentions.go | 100 |
2 files changed, 111 insertions, 58 deletions
diff --git a/.forgejo/workflows/fetch-webmentions.yml b/.forgejo/workflows/fetch-webmentions.yml index a1100ba3..b42c9a80 100644 --- a/.forgejo/workflows/fetch-webmentions.yml +++ b/.forgejo/workflows/fetch-webmentions.yml @@ -2,73 +2,26 @@ name: Fetch Webmentions on: schedule: - - cron: "*/30 * * * *" # every 30 minutes + - cron: "*/30 * * * *" workflow_dispatch: jobs: fetch: - runs-on: docker + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.22" + - name: Fetch webmentions env: - TOKEN: ${{ secrets.WEBMENTION_IO_TOKEN }} - DOMAIN: ${{ secrets.WEBMENTION_IO_DOMAIN }} + WEBMENTION_IO_TOKEN: ${{ secrets.WEBMENTION_IO_TOKEN }} + WEBMENTION_IO_DOMAIN: ${{ secrets.WEBMENTION_IO_DOMAIN }} run: | - mkdir -p data - - curl -s "https://webmention.io/api/mentions.jf2?domain=$DOMAIN&token=$TOKEN" > data/webmentions_raw.json - - cat > /tmp/process_webmentions.py << 'PYTHON_SCRIPT' - import json - import re - import sys - - with open('data/webmentions_raw.json', 'r') as f: - data = json.load(f) - - children = data.get('children', []) - - # Process children - processed = [] - for child in children: - # Remove domain from wm-target - wm_target = child.get('wm-target', '') - wm_target = re.sub(r'^https?://[^/]+', '', wm_target) - child['wm-target'] = wm_target - processed.append(child) - - # Sort by wm-target - processed.sort(key=lambda x: x.get('wm-target', '')) - - # Group by wm-target - from itertools import groupby - result = {} - for target, group in groupby(processed, key=lambda x: x.get('wm-target', '')): - items = [] - for item in group: - processed_item = { - 'type': item.get('wm-property'), - 'author': item.get('author'), - 'url': item.get('url'), - 'source': item.get('wm-source'), - 'published': item.get('published'), - 'received': item.get('wm-received'), - 'content': item.get('content') if 'content' in item else None - } - items.append(processed_item) - result[target] = items - - with open('data/webmentions.json', 'w') as f: - json.dump(result, f, indent=2) - - import os - os.remove('data/webmentions_raw.json') - PYTHON_SCRIPT - - python3 /tmp/process_webmentions.py + go run fetch_webmentions.go - name: Commit if changed run: | @@ -77,8 +30,8 @@ jobs: exit 0 fi - git config user.name "github-actions" - git config user.email "github-actions@github.com" + git config user.name "forgejo-actions" + git config user.email "forgejo-actions@eltr.ac" git add data/webmentions.json git commit -m "chore: update webmentions" git push diff --git a/scripts/fetch-webmentions.go b/scripts/fetch-webmentions.go new file mode 100644 index 00000000..ca03995c --- /dev/null +++ b/scripts/fetch-webmentions.go @@ -0,0 +1,100 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "regexp" + "sort" +) + +type JF2Response struct { + Children []map[string]interface{} `json:"children"` +} + +func mustEnv(key string) string { + v := os.Getenv(key) + if v == "" { + fmt.Fprintf(os.Stderr, "missing env: %s\n", key) + os.Exit(1) + } + return v +} + +func main() { + token := mustEnv("WEBMENTION_IO_TOKEN") + domain := mustEnv("WEBMENTION_IO_DOMAIN") + + var all []map[string]interface{} + client := &http.Client{} + page := 0 + + for { + url := fmt.Sprintf( + "https://webmention.io/api/mentions.jf2?domain=%s&token=%s&page=%d", + domain, token, page, + ) + + resp, err := client.Get(url) + if err != nil { + panic(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + var jf2 JF2Response + if err := json.Unmarshal(body, &jf2); err != nil { + panic(err) + } + + if len(jf2.Children) == 0 { + break + } + + all = append(all, jf2.Children...) + page++ + } + + reDomain := regexp.MustCompile(`^https?://[^/]+`) + + for _, c := range all { + if t, ok := c["wm-target"].(string); ok { + c["wm-target"] = reDomain.ReplaceAllString(t, "") + } + } + + sort.Slice(all, func(i, j int) bool { + ti, _ := all[i]["wm-target"].(string) + tj, _ := all[j]["wm-target"].(string) + return ti < tj + }) + + grouped := map[string][]map[string]interface{}{} + + for _, c := range all { + target, _ := c["wm-target"].(string) + + item := map[string]interface{}{ + "type": c["wm-property"], + "author": c["author"], + "url": c["url"], + "source": c["wm-source"], + "published": c["published"], + "received": c["wm-received"], + "content": c["content"], + } + + grouped[target] = append(grouped[target], item) + } + + if err := os.MkdirAll("data", 0755); err != nil { + panic(err) + } + + out, _ := json.MarshalIndent(grouped, "", " ") + if err := os.WriteFile("data/webmentions.json", out, 0644); err != nil { + panic(err) + } +} |