refactor(build): separate link scanning logic

Generation of both internal link graph and external link rank used to be done in one single script. Now it's separated for maintenance.


Former-commit-id: 7d308beee2359afe39582ed6182f085250768f3f

3 files changed, 174 insertions, 300 deletions

diff --git a/scripts/extract_links.go b/scripts/extract_links.go
deleted file mode 100644
index 657cb738..00000000
--- a/scripts/extract_links.go
+++ /dev/null
@@ -1,300 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"io/fs"
-	"net/url"
-	"os"
-	"path/filepath"
-	"regexp"
-	"sort"
-	"strings"
-)
-
-type Frontmatter struct {
-	Slug           string
-	Aliases        []string
-	Draft          bool
-	Title          string
-	RelatedCards   []string
-	MentionedBooks []string
-}
-
-type Node struct {
-	ID    string `json:"id"`
-	Label string `json:"label"`
-	URL   string `json:"url"`
-}
-
-type Edge struct {
-	From string `json:"from"`
-	To   string `json:"to"`
-}
-
-var (
-	contentDir           = "content"
-	internalLinkPattern   = regexp.MustCompile(`$begin:math:display$[^$end:math:display$]*\]$begin:math:text$(/[^)]+)$end:math:text$`)
-	markdownLinkPattern   = regexp.MustCompile(`$begin:math:display$.*?$end:math:display$$begin:math:text$(http.*?)$end:math:text$`)
-	urlMap                = make(map[string]string)       // file path -> canonical url
-	aliasMap              = make(map[string]string)       // alias url -> canonical url
-	linkGraph             = make(map[string][]string)     // canonical url -> links
-	titleMap              = make(map[string]string)       // canonical url -> title
-	externalLinks         = make(map[string]int)          // domain -> count
-	excludedExternalHosts = map[string]bool{"image.guhub.cn": true, "www.geedea.pro": true}
-)
-
-func parseFrontmatter(filepath string) (Frontmatter, int) {
-	file, err := os.Open(filepath)
-	if err != nil {
-		return Frontmatter{}, 0
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	if !scanner.Scan() || strings.TrimSpace(scanner.Text()) != "---" {
-		return Frontmatter{}, 0
-	}
-
-	fm := Frontmatter{}
-	lineNum := 1
-
-	for scanner.Scan() {
-		lineNum++
-		line := strings.TrimSpace(scanner.Text())
-		if line == "---" {
-			break
-		}
-
-		if strings.Contains(line, ":") {
-			parts := strings.SplitN(line, ":", 2)
-			key := strings.TrimSpace(parts[0])
-			value := strings.Trim(strings.TrimSpace(parts[1]), `"'`)
-
-			switch key {
-			case "aliases":
-				for scanner.Scan() {
-					next := strings.TrimSpace(scanner.Text())
-					lineNum++
-					if !strings.HasPrefix(next, "- ") {
-						break
-					}
-					fm.Aliases = append(fm.Aliases, strings.TrimSpace(next[2:]))
-				}
-			case "slug":
-				fm.Slug = value
-			case "draft":
-				fm.Draft = strings.ToLower(value) == "true"
-			case "title":
-				fm.Title = value
-			case "related_cards":
-				for scanner.Scan() {
-					next := strings.TrimSpace(scanner.Text())
-					lineNum++
-					if !strings.HasPrefix(next, "- ") {
-						break
-					}
-					fm.RelatedCards = append(fm.RelatedCards, strings.TrimSpace(next[2:]))
-				}
-			case "mentioned_books":
-				for scanner.Scan() {
-					next := strings.TrimSpace(scanner.Text())
-					lineNum++
-					if !strings.HasPrefix(next, "- ") {
-						break
-					}
-					fm.MentionedBooks = append(fm.MentionedBooks, strings.TrimSpace(next[2:]))
-				}
-			}
-		}
-	}
-	return fm, lineNum
-}
-
-func getCanonicalURL(path string, fm Frontmatter) string {
-	rel, _ := filepathRel(contentDir, path)
-	dir := filepath.Dir(rel)
-	base := strings.TrimSuffix(filepath.Base(path), ".md")
-	slug := fm.Slug
-	if slug == "" {
-		slug = strings.ToLower(base)
-	}
-
-	if dir != "." {
-		return fmt.Sprintf("/%s/%s/", filepathToSlash(dir), slug)
-	}
-	return fmt.Sprintf("/%s/", slug)
-}
-
-func resolveURL(u string) string {
-	if resolved, ok := aliasMap[u]; ok {
-		return resolved
-	}
-	return u
-}
-
-func resolveURLFromDir(link, subdir string) string {
-	targetDir := filepath.Join(contentDir, subdir)
-	for path, canonical := range urlMap {
-		if strings.HasPrefix(path, targetDir) {
-			name := strings.TrimSuffix(filepath.Base(path), ".md")
-			if name == link {
-				return canonical
-			}
-		}
-	}
-	return ""
-}
-
-func getIcon(u string) string {
-	switch {
-	case strings.HasPrefix(u, "/cards/"):
-		return "📄"
-	case strings.HasPrefix(u, "/posts/"):
-		return "📜"
-	case strings.HasPrefix(u, "/library/"):
-		return "📖"
-	case strings.HasPrefix(u, "/fictions/"):
-		return "🧙‍♀️"
-	case strings.HasPrefix(u, "/en/"):
-		return "🇬🇧"
-	default:
-		return "📁"
-	}
-}
-
-func collectURLs() {
-	filepath.WalkDir(contentDir, func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-		if d.IsDir() || !strings.HasSuffix(d.Name(), ".md") || d.Name() == "_index.md" {
-			return nil
-		}
-
-		fm, _ := parseFrontmatter(path)
-		if fm.Draft {
-			return nil
-		}
-
-		canonical := getCanonicalURL(path, fm)
-		urlMap[path] = canonical
-		linkGraph[canonical] = []string{}
-
-		title := fm.Title
-		if title == "" {
-			title = strings.Trim(canonical, "/")
-			if parts := strings.Split(title, "/"); len(parts) > 0 {
-				title = parts[len(parts)-1]
-			}
-		}
-		titleMap[canonical] = fmt.Sprintf("%s %s", getIcon(canonical), title)
-
-		for _, alias := range fm.Aliases {
-			aliasMap[alias] = canonical
-		}
-
-		return nil
-	})
-}
-
-func buildGraphAndExternalLinks() {
-	for path, canonical := range urlMap {
-		content, err := os.ReadFile(path)
-		if err != nil {
-			continue
-		}
-		text := string(content)
-
-		// Internal links
-		for _, m := range internalLinkPattern.FindAllStringSubmatch(text, -1) {
-			link := resolveURL(m[1])
-			if _, ok := linkGraph[link]; ok {
-				linkGraph[canonical] = append(linkGraph[canonical], link)
-			}
-		}
-
-		// External links
-		for _, m := range markdownLinkPattern.FindAllStringSubmatch(text, -1) {
-			u, err := url.Parse(m[1])
-			if err == nil && u.Host != "" && !excludedExternalHosts[u.Host] {
-				externalLinks[u.Host]++
-			}
-		}
-
-		// Related cards and books
-		fm, _ := parseFrontmatter(path)
-		for _, card := range fm.RelatedCards {
-			if target := resolveURLFromDir(card, "cards"); target != "" {
-				linkGraph[canonical] = append(linkGraph[canonical], target)
-			}
-		}
-		for _, book := range fm.MentionedBooks {
-			if target := resolveURLFromDir(book, "library"); target != "" {
-				linkGraph[canonical] = append(linkGraph[canonical], target)
-			}
-		}
-	}
-}
-
-func main() {
-	collectURLs()
-	buildGraphAndExternalLinks()
-
-	os.MkdirAll("public", os.ModePerm)
-	os.MkdirAll("data", os.ModePerm)
-
-	// Write link-graph.json
-	var nodes []Node
-	var edges []Edge
-	for url, targets := range linkGraph {
-		nodes = append(nodes, Node{ID: url, Label: titleMap[url], URL: url})
-		for _, tgt := range targets {
-			edges = append(edges, Edge{From: url, To: tgt})
-		}
-	}
-	writeJSON("public/link-graph.json", map[string]interface{}{"nodes": nodes, "edges": edges})
-	fmt.Println("✅ Link graph generated: public/link-graph.json")
-
-	// Write externallinks.json
-	type kv struct {
-		Key   string
-		Count int
-	}
-	var sorted []kv
-	for domain, count := range externalLinks {
-		sorted = append(sorted, kv{domain, count})
-	}
-	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Count > sorted[j].Count })
-
-	writeJSON("data/externallinks.json", sorted)
-	fmt.Println("✅ External link data written to data/externallinks.json")
-}
-
-func writeJSON(path string, data interface{}) {
-	file, err := os.Create(path)
-	if err != nil {
-		fmt.Println("❌ Error creating file:", err)
-		return
-	}
-	defer file.Close()
-
-	enc := json.NewEncoder(file)
-	enc.SetIndent("", "  ")
-	if err := enc.Encode(data); err != nil {
-		fmt.Println("❌ Error writing JSON:", err)
-	}
-}
-
-func filepathRel(base, target string) (string, error) {
-	rel, err := filepath.Rel(base, target)
-	if err != nil {
-		return "", err
-	}
-	return rel, nil
-}
-
-func filepathToSlash(p string) string {
-	return strings.ReplaceAll(p, string(os.PathSeparator), "/")
-}
diff --git a/scripts/get_external_link_rank.go b/scripts/get_external_link_rank.go
new file mode 100644
index 00000000..780bb5ec
--- /dev/null
+++ b/scripts/get_external_link_rank.go
@@ -0,0 +1,73 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"net/url"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+func main() {
+	externalLinks := countExternalLinks("content")
+	writeJSON("data/externallinks.json", externalLinks)
+	fmt.Println("✅ External link rank data generated.")
+}
+
+// countExternalLinks scans markdown files and counts occurrences of each external domain.
+func countExternalLinks(root string) [][2]interface{} {
+	linkRegex := regexp.MustCompile(`https?://[^\s)>"']+`)
+	counts := make(map[string]int)
+
+	filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() || !strings.HasSuffix(path, ".md") {
+			return nil
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return nil
+		}
+		matches := linkRegex.FindAllString(string(data), -1)
+		for _, match := range matches {
+			u, err := url.Parse(match)
+			if err == nil && u.Host != "" && u.Host != "image.guhub.cn" {
+				counts[u.Host]++
+			}
+		}
+		return nil
+	})
+
+	// convert map to sorted array
+	type kv struct {
+		Key   string
+		Value int
+	}
+	var list []kv
+	for k, v := range counts {
+		list = append(list, kv{k, v})
+	}
+	sort.Slice(list, func(i, j int) bool { return list[i].Value > list[j].Value })
+
+	result := make([][2]interface{}, len(list))
+	for i, kv := range list {
+		result[i] = [2]interface{}{kv.Key, kv.Value}
+	}
+	return result
+}
+
+// writeJSON writes the result to file with indentation.
+func writeJSON(path string, data interface{}) {
+	file, err := os.Create(path)
+	if err != nil {
+		fmt.Println("Error:", err)
+		return
+	}
+	defer file.Close()
+	enc := json.NewEncoder(file)
+	enc.SetIndent("", "  ")
+	enc.Encode(data)
+}
diff --git a/scripts/get_internal_link_graph.go b/scripts/get_internal_link_graph.go
new file mode 100644
index 00000000..2f7c8b12
--- /dev/null
+++ b/scripts/get_internal_link_graph.go
@@ -0,0 +1,101 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+type Node struct {
+	ID    string `json:"id"`
+	Label string `json:"label"`
+	URL   string `json:"url"`
+}
+
+type Edge struct {
+	From string `json:"from"`
+	To   string `json:"to"`
+}
+
+func main() {
+	root := "content"
+
+	nodes := map[string]Node{}
+	edges := make([]Edge, 0) // non-nil so it becomes [] not null
+
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() || !strings.HasSuffix(strings.ToLower(path), ".md") {
+			return nil
+		}
+
+		rel, err := filepath.Rel(root, path)
+		if err != nil {
+			return nil
+		}
+		rel = filepath.ToSlash(rel)
+		urlPath := "/" + strings.TrimSuffix(rel, ".md") + "/"
+		label := strings.TrimSuffix(filepath.Base(path), ".md")
+
+		nodes[urlPath] = Node{ID: urlPath, Label: label, URL: urlPath}
+
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return nil
+		}
+		content := string(data)
+
+		linkPattern := regexp.MustCompile(`\[[^\]]*\]\((/[^)]+)\)`)
+		matches := linkPattern.FindAllStringSubmatch(content, -1)
+		for _, m := range matches {
+			link := strings.TrimSuffix(m[1], ".md")
+			if !strings.HasSuffix(link, "/") {
+				link += "/"
+			}
+			edges = append(edges, Edge{From: urlPath, To: link})
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		fmt.Println("WalkDir error:", err)
+		return
+	}
+
+	nodeKeys := make([]string, 0, len(nodes))
+	for k := range nodes {
+		nodeKeys = append(nodeKeys, k)
+	}
+	sort.Strings(nodeKeys)
+	nodeList := make([]Node, 0, len(nodeKeys))
+	for _, k := range nodeKeys {
+		nodeList = append(nodeList, nodes[k])
+	}
+
+	os.MkdirAll("public", os.ModePerm)
+	output := map[string]interface{}{
+		"nodes": nodeList,
+		"edges": edges,
+	}
+
+	outfile := "public/link-graph.json"
+	f, err := os.Create(outfile)
+	if err != nil {
+		fmt.Println("create file error:", err)
+		return
+	}
+	defer f.Close()
+
+	enc := json.NewEncoder(f)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(output); err != nil {
+		fmt.Println("json encode error:", err)
+	}
+
+	fmt.Printf("✅ Link graph written to %s (%d nodes, %d edges)\n", outfile, len(nodeList), len(edges))
+}