1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"regexp"
"sort"
)
type JF2Response struct {
Children []map[string]interface{} `json:"children"`
}
func mustEnv(key string) string {
v := os.Getenv(key)
if v == "" {
fmt.Fprintf(os.Stderr, "missing env: %s\n", key)
os.Exit(1)
}
return v
}
func main() {
token := mustEnv("WEBMENTION_IO_TOKEN")
domain := mustEnv("WEBMENTION_IO_DOMAIN")
var all []map[string]interface{}
client := &http.Client{}
page := 0
for {
url := fmt.Sprintf(
"https://webmention.io/api/mentions.jf2?domain=%s&token=%s&page=%d",
domain, token, page,
)
resp, err := client.Get(url)
if err != nil {
panic(err)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
var jf2 JF2Response
if err := json.Unmarshal(body, &jf2); err != nil {
panic(err)
}
if len(jf2.Children) == 0 {
break
}
all = append(all, jf2.Children...)
page++
}
reDomain := regexp.MustCompile(`^https?://[^/]+`)
for _, c := range all {
if t, ok := c["wm-target"].(string); ok {
c["wm-target"] = reDomain.ReplaceAllString(t, "")
}
}
sort.Slice(all, func(i, j int) bool {
ti, _ := all[i]["wm-target"].(string)
tj, _ := all[j]["wm-target"].(string)
return ti < tj
})
grouped := map[string][]map[string]interface{}{}
for _, c := range all {
target, _ := c["wm-target"].(string)
item := map[string]interface{}{
"type": c["wm-property"],
"author": c["author"],
"url": c["url"],
"source": c["wm-source"],
"published": c["published"],
"received": c["wm-received"],
"content": c["content"],
}
grouped[target] = append(grouped[target], item)
}
for target, items := range grouped {
sort.Slice(items, func(i, j int) bool {
timeI, okI := items[i]["published"].(string)
timeJ, okJ := items[j]["published"].(string)
if okI && okJ {
return timeI < timeJ
}
receivedI, _ := items[i]["received"].(string)
receivedJ, _ := items[j]["received"].(string)
return receivedI < receivedJ
})
grouped[target] = items
}
if err := os.MkdirAll("data", 0755); err != nil {
panic(err)
}
out, _ := json.MarshalIndent(grouped, "", " ")
if err := os.WriteFile("data/webmentions.json", out, 0644); err != nil {
panic(err)
}
}
|
