package main import ( "encoding/json" "fmt" "io" "net/http" "os" "regexp" "sort" ) type JF2Response struct { Children []map[string]interface{} `json:"children"` } func mustEnv(key string) string { v := os.Getenv(key) if v == "" { fmt.Fprintf(os.Stderr, "missing env: %s\n", key) os.Exit(1) } return v } func main() { token := mustEnv("WEBMENTION_IO_TOKEN") domain := mustEnv("WEBMENTION_IO_DOMAIN") var all []map[string]interface{} client := &http.Client{} page := 0 for { url := fmt.Sprintf( "https://webmention.io/api/mentions.jf2?domain=%s&token=%s&page=%d", domain, token, page, ) resp, err := client.Get(url) if err != nil { panic(err) } body, _ := io.ReadAll(resp.Body) resp.Body.Close() var jf2 JF2Response if err := json.Unmarshal(body, &jf2); err != nil { panic(err) } if len(jf2.Children) == 0 { break } all = append(all, jf2.Children...) page++ } reDomain := regexp.MustCompile(`^https?://[^/]+`) for _, c := range all { if t, ok := c["wm-target"].(string); ok { c["wm-target"] = reDomain.ReplaceAllString(t, "") } } sort.Slice(all, func(i, j int) bool { ti, _ := all[i]["wm-target"].(string) tj, _ := all[j]["wm-target"].(string) return ti < tj }) grouped := map[string][]map[string]interface{}{} for _, c := range all { target, _ := c["wm-target"].(string) item := map[string]interface{}{ "type": c["wm-property"], "author": c["author"], "url": c["url"], "source": c["wm-source"], "published": c["published"], "received": c["wm-received"], "content": c["content"], } grouped[target] = append(grouped[target], item) } if err := os.MkdirAll("data", 0755); err != nil { panic(err) } out, _ := json.MarshalIndent(grouped, "", " ") if err := os.WriteFile("data/webmentions.json", out, 0644); err != nil { panic(err) } }