From e1bc41f5014565d1f3a2f26deccce1932fbd24fd Mon Sep 17 00:00:00 2001 From: Eltrac Date: Tue, 21 Oct 2025 11:12:16 +0800 Subject: chore(CI/CD): rewrite pre-build Python scripts with Go Former-commit-id: fbd878474f1af38605090307571c03b8409a077c --- build.sh | 9 +- scripts/analyze_links.py | 170 ------------------------ scripts/extract_highlights.go | 120 +++++++++++++++++ scripts/extract_links.go | 300 ++++++++++++++++++++++++++++++++++++++++++ scripts/highlights.py | 61 --------- scripts/run_scripts.py | 12 -- 6 files changed, 426 insertions(+), 246 deletions(-) delete mode 100644 scripts/analyze_links.py create mode 100644 scripts/extract_highlights.go create mode 100644 scripts/extract_links.go delete mode 100644 scripts/highlights.py delete mode 100644 scripts/run_scripts.py diff --git a/build.sh b/build.sh index 41ad7777..1e59e673 100644 --- a/build.sh +++ b/build.sh @@ -19,7 +19,7 @@ main() { # ===== Install All Dependencies ===== # - echo "🏗️ Installing all dependencies...\n" + echo "🏗️ Installing all dependencies..." # Install Python echo "Installing Python ${PYTHON_VERSION}..." @@ -59,8 +59,11 @@ main() { # ===== Execute Pre-Hugo Actions ===== # - echo "🏗️ Executing Python Scripts..." - python3 scripts/run_scripts.py + echo "🏗️ Building and Running Custom Go Scripts" + go build -o extract_highlights extract_highlights.go + go build -o extract_links extract_links.go + ./extract_highlights + ./extract_links echo "🏗️ Building UnoCSS..." npm run build:uno:prod diff --git a/scripts/analyze_links.py b/scripts/analyze_links.py deleted file mode 100644 index 8364100e..00000000 --- a/scripts/analyze_links.py +++ /dev/null @@ -1,170 +0,0 @@ -import os -import re -import json -from urllib.parse import urlparse -from collections import Counter - -content_dir = "content" -internal_link_pattern = re.compile(r"\[[^\]]*\]\((/[^)]+)\)") -markdown_link_re = re.compile(r"\[.*?\]\((http.*?)\)") - -url_map = {} # file path -> canonical url -alias_map = {} # alias url -> canonical url -link_graph = {} # canonical url -> list of canonical urls -title_map = {} # canonical url -> title -external_links = Counter() - -def parse_frontmatter(filepath): - with open(filepath, encoding="utf-8") as f: - lines = f.readlines() - - if not lines or lines[0].strip() != "---": - return {}, 0 - - frontmatter = {} - aliases, related_cards, mentioned_books = [], [], [] - slug, draft, title = None, False, None - i = 1 - while i < len(lines): - line = lines[i].strip() - if line == "---": - break - if ":" in line: - key, value = line.split(":", 1) - key = key.strip() - value = value.strip().strip('"').strip("'") - - if key == "aliases": - aliases = [] - i += 1 - while i < len(lines) and lines[i].startswith(" - "): - aliases.append(lines[i].strip()[3:].strip()) - i += 1 - continue - elif key == "slug": - slug = value - elif key == "draft": - draft = value.lower() == "true" - elif key == "title": - title = value - elif key == "related_cards": - related_cards = [] - i += 1 - while i < len(lines) and lines[i].strip().startswith("- "): - related_cards.append(lines[i].strip()[2:].strip()) - i += 1 - continue - elif key == "mentioned_books": - mentioned_books = [] - i += 1 - while i < len(lines) and lines[i].strip().startswith("- "): - mentioned_books.append(lines[i].strip()[2:].strip()) - i += 1 - continue - i += 1 - - return { - "slug": slug, - "aliases": aliases, - "draft": draft, - "title": title, - "related_cards": related_cards, - "mentioned_books": mentioned_books - }, i + 1 - -def get_canonical_url(filepath, frontmatter): - rel_path = os.path.relpath(filepath, content_dir) - dirname, filename = os.path.split(rel_path) - name = os.path.splitext(filename)[0].lower() - slug = frontmatter.get("slug") or name - parts = dirname.split(os.sep) - return f"/{'/'.join(parts)}/{slug}/" if parts != [""] else f"/{slug}/" - -def resolve_url(url): - return alias_map.get(url, url) - -def resolve_url_from_dir(link, subdir): - target_dir = os.path.join(content_dir, subdir) - for path, canonical in url_map.items(): - if path.startswith(target_dir): - name = os.path.splitext(os.path.basename(path))[0] - if name == link: - return canonical - return None - -def get_icon(url): - if url.startswith("/cards/"): return "📄" - if url.startswith("/posts/"): return "📜" - if url.startswith("/library/"): return "📖" - if url.startswith("/fictions/"): return "🧙‍♀️" - if url.startswith("/en/"): return "🇬🇧" - return "📁" - -def collect_urls(): - for root, _, files in os.walk(content_dir): - for file in files: - if not file.endswith(".md") or file == "_index.md": - continue - path = os.path.join(root, file) - frontmatter, _ = parse_frontmatter(path) - if frontmatter.get("draft"): - continue - - canonical_url = get_canonical_url(path, frontmatter) - url_map[path] = canonical_url - link_graph[canonical_url] = [] - title = frontmatter.get("title") or canonical_url.strip("/").split("/")[-1] - title_map[canonical_url] = f"{get_icon(canonical_url)} {title}" - - for alias in frontmatter.get("aliases", []): - alias_map[alias] = canonical_url - -def build_link_graph_and_external_links(): - for path, canonical_url in url_map.items(): - with open(path, encoding="utf-8") as f: - content = f.read() - - # 内部链接 - for match in internal_link_pattern.finditer(content): - link = match.group(1) - normalized = resolve_url(link) - if normalized and normalized in link_graph: - link_graph[canonical_url].append(normalized) - - # 外部链接 - for match in markdown_link_re.findall(content): - domain = urlparse(match).netloc - if domain and domain not in {"image.guhub.cn", "www.geedea.pro"}: - external_links[domain] += 1 - - # 相关卡片和图书 - frontmatter, _ = parse_frontmatter(path) - for card in frontmatter.get("related_cards", []): - target = resolve_url_from_dir(card, "cards") - if target and target in link_graph: - link_graph[canonical_url].append(target) - for book in frontmatter.get("mentioned_books", []): - target = resolve_url_from_dir(book, "library") - if target and target in link_graph: - link_graph[canonical_url].append(target) - -def main(): - collect_urls() - build_link_graph_and_external_links() - - # 写入内部链接图 - os.makedirs("public", exist_ok=True) - nodes = [{"id": url, "label": title_map[url], "url": url} for url in link_graph] - edges = [{"from": src, "to": tgt} for src, targets in link_graph.items() for tgt in targets] - with open("public/link-graph.json", "w", encoding="utf-8") as f: - json.dump({"nodes": nodes, "edges": edges}, f, indent=2, ensure_ascii=False) - print("✅ Link graph generated: public/link-graph.json") - - # 写入外部链接统计 - os.makedirs("data", exist_ok=True) - with open("data/externallinks.json", "w", encoding="utf-8") as f: - json.dump(external_links.most_common(), f, indent=2, ensure_ascii=False) - print("✅ External link data written to data/externallinks.json") - -if __name__ == "__main__": - main() diff --git a/scripts/extract_highlights.go b/scripts/extract_highlights.go new file mode 100644 index 00000000..c7253d5f --- /dev/null +++ b/scripts/extract_highlights.go @@ -0,0 +1,120 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/fs" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" +) + +type HighlightLink struct { + Text string `json:"text"` + Link string `json:"link"` +} + +type Result struct { + Title string `json:"title"` + URL string `json:"url"` + Highlights []HighlightLink `json:"highlights"` +} + +func main() { + const contentDir = "content" + const outputFile = "data/highlights.json" + + highlightPattern := regexp.MustCompile(`==(.+?)==`) + titlePattern := regexp.MustCompile(`(?m)^title:\s*["']?(.+?)["']?\s*$`) + slugPattern := regexp.MustCompile(`(?m)^slug:\s*["']?(.+?)["']?\s*$`) + + var results []Result + + err := filepath.WalkDir(contentDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(d.Name(), ".md") { + return nil + } + + contentBytes, err := os.ReadFile(path) + if err != nil { + return err + } + content := string(contentBytes) + + // Title + title := filepath.Base(path) + if m := titlePattern.FindStringSubmatch(content); len(m) > 1 { + title = m[1] + } + + // Slug and URL + urlPath := "" + if m := slugPattern.FindStringSubmatch(content); len(m) > 1 { + slug := strings.Trim(m[1], "/") + relDir, _ := filepath.Rel(contentDir, filepath.Dir(path)) + relDir = filepath.ToSlash(relDir) + if relDir != "." { + urlPath = fmt.Sprintf("/%s/%s/", relDir, slug) + } else { + urlPath = fmt.Sprintf("/%s/", slug) + } + } else { + rel, _ := filepath.Rel(contentDir, path) + rel = filepath.ToSlash(rel) + urlPath = "/" + strings.TrimSuffix(rel, ".md") + "/" + } + + // Highlights + matches := highlightPattern.FindAllStringSubmatch(content, -1) + if len(matches) == 0 { + return nil + } + + var highlightLinks []HighlightLink + for _, m := range matches { + if len(m) > 1 { + h := m[1] + encoded := url.QueryEscape(h) + link := fmt.Sprintf("%s#:~:text=%s", urlPath, encoded) + highlightLinks = append(highlightLinks, HighlightLink{Text: h, Link: link}) + } + } + + results = append(results, Result{ + Title: title, + URL: urlPath, + Highlights: highlightLinks, + }) + + return nil + }) + + if err != nil { + fmt.Println("❌ Error:", err) + return + } + + // Write JSON + if err := os.MkdirAll(filepath.Dir(outputFile), os.ModePerm); err != nil { + fmt.Println("❌ Error creating output directory:", err) + return + } + + jsonData, err := json.MarshalIndent(results, "", " ") + if err != nil { + fmt.Println("❌ Error encoding JSON:", err) + return + } + + if err := os.WriteFile(outputFile, jsonData, 0644); err != nil { + fmt.Println("❌ Error writing JSON file:", err) + return + } + + fmt.Printf("✔ 提取完成，共 %d 篇文章有高亮，已写入 %s\n", len(results), outputFile) +} diff --git a/scripts/extract_links.go b/scripts/extract_links.go new file mode 100644 index 00000000..2b7b243a --- /dev/null +++ b/scripts/extract_links.go @@ -0,0 +1,300 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io/fs" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +type Frontmatter struct { + Slug string + Aliases []string + Draft bool + Title string + RelatedCards []string + MentionedBooks []string +} + +type Node struct { + ID string `json:"id"` + Label string `json:"label"` + URL string `json:"url"` +} + +type Edge struct { + From string `json:"from"` + To string `json:"to"` +} + +var ( + contentDir = "content" + internalLinkPattern = regexp.MustCompile(`$begin:math:display$[^$end:math:display$]*\]$begin:math:text$(/[^)]+)$end:math:text$`) + markdownLinkPattern = regexp.MustCompile(`$begin:math:display$.*?$end:math:display$$begin:math:text$(http.*?)$end:math:text$`) + urlMap = make(map[string]string) // file path -> canonical url + aliasMap = make(map[string]string) // alias url -> canonical url + linkGraph = make(map[string][]string) // canonical url -> links + titleMap = make(map[string]string) // canonical url -> title + externalLinks = make(map[string]int) // domain -> count + excludedExternalHosts = map[string]bool{"image.guhub.cn": true, "www.geedea.pro": true} +) + +func parseFrontmatter(filepath string) (Frontmatter, int) { + file, err := os.Open(filepath) + if err != nil { + return Frontmatter{}, 0 + } + defer file.Close() + + scanner := bufio.NewScanner(file) + if !scanner.Scan() || strings.TrimSpace(scanner.Text()) != "---" { + return Frontmatter{}, 0 + } + + fm := Frontmatter{} + lineNum := 1 + + for scanner.Scan() { + lineNum++ + line := strings.TrimSpace(scanner.Text()) + if line == "---" { + break + } + + if strings.Contains(line, ":") { + parts := strings.SplitN(line, ":", 2) + key := strings.TrimSpace(parts[0]) + value := strings.Trim(strings.TrimSpace(parts[1]), `"'`) + + switch key { + case "aliases": + for scanner.Scan() { + next := strings.TrimSpace(scanner.Text()) + lineNum++ + if !strings.HasPrefix(next, "- ") { + break + } + fm.Aliases = append(fm.Aliases, strings.TrimSpace(next[2:])) + } + case "slug": + fm.Slug = value + case "draft": + fm.Draft = strings.ToLower(value) == "true" + case "title": + fm.Title = value + case "related_cards": + for scanner.Scan() { + next := strings.TrimSpace(scanner.Text()) + lineNum++ + if !strings.HasPrefix(next, "- ") { + break + } + fm.RelatedCards = append(fm.RelatedCards, strings.TrimSpace(next[2:])) + } + case "mentioned_books": + for scanner.Scan() { + next := strings.TrimSpace(scanner.Text()) + lineNum++ + if !strings.HasPrefix(next, "- ") { + break + } + fm.MentionedBooks = append(fm.MentionedBooks, strings.TrimSpace(next[2:])) + } + } + } + } + return fm, lineNum +} + +func getCanonicalURL(filepath string, fm Frontmatter) string { + rel, _ := filepathRel(contentDir, filepath) + dir := filepath.Dir(rel) + base := strings.TrimSuffix(filepath.Base(filepath), ".md") + slug := fm.Slug + if slug == "" { + slug = strings.ToLower(base) + } + + if dir != "." { + return fmt.Sprintf("/%s/%s/", filepathToSlash(dir), slug) + } + return fmt.Sprintf("/%s/", slug) +} + +func resolveURL(u string) string { + if resolved, ok := aliasMap[u]; ok { + return resolved + } + return u +} + +func resolveURLFromDir(link, subdir string) string { + targetDir := filepath.Join(contentDir, subdir) + for path, canonical := range urlMap { + if strings.HasPrefix(path, targetDir) { + name := strings.TrimSuffix(filepath.Base(path), ".md") + if name == link { + return canonical + } + } + } + return "" +} + +func getIcon(u string) string { + switch { + case strings.HasPrefix(u, "/cards/"): + return "📄" + case strings.HasPrefix(u, "/posts/"): + return "📜" + case strings.HasPrefix(u, "/library/"): + return "📖" + case strings.HasPrefix(u, "/fictions/"): + return "🧙‍♀️" + case strings.HasPrefix(u, "/en/"): + return "🇬🇧" + default: + return "📁" + } +} + +func collectURLs() { + filepath.WalkDir(contentDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(d.Name(), ".md") || d.Name() == "_index.md" { + return nil + } + + fm, _ := parseFrontmatter(path) + if fm.Draft { + return nil + } + + canonical := getCanonicalURL(path, fm) + urlMap[path] = canonical + linkGraph[canonical] = []string{} + + title := fm.Title + if title == "" { + title = strings.Trim(canonical, "/") + if parts := strings.Split(title, "/"); len(parts) > 0 { + title = parts[len(parts)-1] + } + } + titleMap[canonical] = fmt.Sprintf("%s %s", getIcon(canonical), title) + + for _, alias := range fm.Aliases { + aliasMap[alias] = canonical + } + + return nil + }) +} + +func buildGraphAndExternalLinks() { + for path, canonical := range urlMap { + content, err := os.ReadFile(path) + if err != nil { + continue + } + text := string(content) + + // Internal links + for _, m := range internalLinkPattern.FindAllStringSubmatch(text, -1) { + link := resolveURL(m[1]) + if _, ok := linkGraph[link]; ok { + linkGraph[canonical] = append(linkGraph[canonical], link) + } + } + + // External links + for _, m := range markdownLinkPattern.FindAllStringSubmatch(text, -1) { + u, err := url.Parse(m[1]) + if err == nil && u.Host != "" && !excludedExternalHosts[u.Host] { + externalLinks[u.Host]++ + } + } + + // Related cards and books + fm, _ := parseFrontmatter(path) + for _, card := range fm.RelatedCards { + if target := resolveURLFromDir(card, "cards"); target != "" { + linkGraph[canonical] = append(linkGraph[canonical], target) + } + } + for _, book := range fm.MentionedBooks { + if target := resolveURLFromDir(book, "library"); target != "" { + linkGraph[canonical] = append(linkGraph[canonical], target) + } + } + } +} + +func main() { + collectURLs() + buildGraphAndExternalLinks() + + os.MkdirAll("public", os.ModePerm) + os.MkdirAll("data", os.ModePerm) + + // Write link-graph.json + var nodes []Node + var edges []Edge + for url, targets := range linkGraph { + nodes = append(nodes, Node{ID: url, Label: titleMap[url], URL: url}) + for _, tgt := range targets { + edges = append(edges, Edge{From: url, To: tgt}) + } + } + writeJSON("public/link-graph.json", map[string]interface{}{"nodes": nodes, "edges": edges}) + fmt.Println("✅ Link graph generated: public/link-graph.json") + + // Write externallinks.json + type kv struct { + Key string + Count int + } + var sorted []kv + for domain, count := range externalLinks { + sorted = append(sorted, kv{domain, count}) + } + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Count > sorted[j].Count }) + + writeJSON("data/externallinks.json", sorted) + fmt.Println("✅ External link data written to data/externallinks.json") +} + +func writeJSON(path string, data interface{}) { + file, err := os.Create(path) + if err != nil { + fmt.Println("❌ Error creating file:", err) + return + } + defer file.Close() + + enc := json.NewEncoder(file) + enc.SetIndent("", " ") + if err := enc.Encode(data); err != nil { + fmt.Println("❌ Error writing JSON:", err) + } +} + +func filepathRel(base, target string) (string, error) { + rel, err := filepath.Rel(base, target) + if err != nil { + return "", err + } + return rel, nil +} + +func filepathToSlash(p string) string { + return strings.ReplaceAll(p, string(os.PathSeparator), "/") +} diff --git a/scripts/highlights.py b/scripts/highlights.py deleted file mode 100644 index 102ff02b..00000000 --- a/scripts/highlights.py +++ /dev/null @@ -1,61 +0,0 @@ -import os -import re -import json -import urllib.parse - -CONTENT_DIR = "content" -OUTPUT_FILE = "data/highlights.json" - -# 匹配 Hugo 文章中的 ==高亮== -highlight_pattern = re.compile(r"==(.+?)==") - -# 匹配 front matter（支持 yaml 格式） -title_pattern = re.compile(r'^title:\s*["\']?(.+?)["\']?\s*$', re.MULTILINE) -slug_pattern = re.compile(r'^slug:\s*["\']?(.+?)["\']?\s*$', re.MULTILINE) - -results = [] - -for root, _, files in os.walk(CONTENT_DIR): - for f in files: - if f.endswith(".md"): - path = os.path.join(root, f) - - with open(path, "r", encoding="utf-8") as file: - content = file.read() - - # 提取 title - title_match = title_pattern.search(content) - title = title_match.group(1) if title_match else os.path.basename(path) - - # 提取 slug（如果有就替换默认 URL） - slug_match = slug_pattern.search(content) - if slug_match: - slug = slug_match.group(1).strip("/") - # 用目录推断 section，比如 content/post/abc.md => /post/slug/ - rel_dir = os.path.relpath(root, CONTENT_DIR).replace("\\", "/") - url = f"/{rel_dir}/{slug}/" if rel_dir != "." else f"/{slug}/" - else: - # 默认 Hugo URL 规则：相对路径去掉 .md - url = "/" + os.path.relpath(path, CONTENT_DIR).replace("\\", "/").replace(".md", "/") - - # 提取高亮 - highlights = highlight_pattern.findall(content) - if highlights: - highlight_links = [] - for h in highlights: - encoded = urllib.parse.quote(h, safe="") - link = f"{url}#:~:text={encoded}" - highlight_links.append({"text": h, "link": link}) - - results.append({ - "title": title, - "url": url, - "highlights": highlight_links - }) - -# 写入 data 文件夹 -os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True) -with open(OUTPUT_FILE, "w", encoding="utf-8") as f: - json.dump(results, f, ensure_ascii=False, indent=2) - -print(f"✔ 提取完成，共 {len(results)} 篇文章有高亮，已写入 {OUTPUT_FILE}") diff --git a/scripts/run_scripts.py b/scripts/run_scripts.py deleted file mode 100644 index 71764a11..00000000 --- a/scripts/run_scripts.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -import subprocess -import sys - -SCRIPTS_DIR = "scripts" -SELF = os.path.basename(__file__) - -for filename in sorted(os.listdir(SCRIPTS_DIR)): - if filename.endswith(".py") and filename != SELF: - path = os.path.join(SCRIPTS_DIR, filename) - print(f"▶ Running {path} ...") - subprocess.run([sys.executable, path], check=True) -- cgit v1.2.3