From 60d68be1b4ea81b235b73924d2a98be67d1ebfc9 Mon Sep 17 00:00:00 2001 From: Eltrac Date: Fri, 23 May 2025 16:09:01 +0800 Subject: feat: list external links ranking also move python scripts to one folder --- extractLinks.py | 187 ----------------------------------- layouts/_default/external-links.html | 8 ++ scripts/extractLinks.py | 187 +++++++++++++++++++++++++++++++++++ scripts/listExternalLinks.py | 53 ++++++++++ 4 files changed, 248 insertions(+), 187 deletions(-) delete mode 100644 extractLinks.py create mode 100644 layouts/_default/external-links.html create mode 100644 scripts/extractLinks.py create mode 100644 scripts/listExternalLinks.py diff --git a/extractLinks.py b/extractLinks.py deleted file mode 100644 index eb172353..00000000 --- a/extractLinks.py +++ /dev/null @@ -1,187 +0,0 @@ -import os -import re -import json - -content_dir = "content" -internal_link_pattern = re.compile(r"\[[^\]]*\]\((/[^)]+)\)") - -url_map = {} # file path -> canonical url -alias_map = {} # alias url -> canonical url -link_graph = {} # canonical url -> list of canonical urls -title_map = {} # canonical url -> title - - -def parse_frontmatter(filepath): - with open(filepath, encoding="utf-8") as f: - lines = f.readlines() - - if not lines or lines[0].strip() != "---": - return {}, 0 - - frontmatter = {} - aliases = [] - slug = None - draft = False - title = None - related_cards = [] - mentioned_books = [] - i = 1 - while i < len(lines): - line = lines[i].strip() - if line == "---": - break - if ":" in line: - key, value = line.split(":", 1) - key = key.strip() - value = value.strip().strip('"').strip("'") - - if key == "aliases": - aliases = [] - i += 1 - while i < len(lines) and lines[i].startswith(" - "): - alias = lines[i].strip()[3:].strip() - aliases.append(alias) - i += 1 - continue - elif key == "slug": - slug = value - elif key == "draft": - draft = value.lower() == "true" - elif key == "title": - title = value - elif key == "related_cards": - related_cards = [] - i += 1 - while i < len(lines) and lines[i].strip().startswith("- "): - related_cards.append(lines[i].strip()[2:].strip()) - i += 1 - continue - elif key == "mentioned_books": - mentioned_books = [] - i += 1 - while i < len(lines) and lines[i].strip().startswith("- "): - mentioned_books.append(lines[i].strip()[2:].strip()) - i += 1 - continue - i += 1 - - return { - "slug": slug, - "aliases": aliases, - "draft": draft, - "title": title, - "related_cards": related_cards, - "mentioned_books": mentioned_books - }, i + 1 - - -def get_canonical_url(filepath, frontmatter): - rel_path = os.path.relpath(filepath, content_dir) - dirname, filename = os.path.split(rel_path) - name = os.path.splitext(filename)[0] - slug = frontmatter.get("slug") or name - parts = dirname.split(os.sep) - if parts == [""]: - return f"/{slug}/" - return f"/{'/'.join(parts)}/{slug}/" - - -def resolve_url(url): - return alias_map.get(url, url) - - -def resolve_url_from_dir(link, subdir): - """Only search for link in a specific content subdir (like 'cards', 'library')""" - target_dir = os.path.join(content_dir, subdir) - for path, canonical in url_map.items(): - if path.startswith(target_dir): - name = os.path.splitext(os.path.basename(path))[0] - if name == link: - return canonical - return None - - -def get_icon(url): - if url.startswith("/cards/"): - return "πŸ“„" - elif url.startswith("/posts/"): - return "πŸ“œ" - elif url.startswith("/library/"): - return "πŸ“–" - elif url.startswith("/fictions/"): - return "πŸ§™β€β™€οΈ" - elif url.startswith("/en/"): - return "πŸ‡¬πŸ‡§" - return "πŸ“" - - -def collect_urls(): - for root, _, files in os.walk(content_dir): - for file in files: - if not file.endswith(".md") or file == "_index.md": - continue - path = os.path.join(root, file) - frontmatter, _ = parse_frontmatter(path) - if frontmatter.get("draft"): - continue - - canonical_url = get_canonical_url(path, frontmatter) - url_map[path] = canonical_url - link_graph[canonical_url] = [] - title = frontmatter.get("title") or canonical_url.strip("/").split("/")[-1] - title_map[canonical_url] = f"{get_icon(canonical_url)} {title}" - - for alias in frontmatter.get("aliases", []): - alias_map[alias] = canonical_url - - -def build_link_graph(): - for path, canonical_url in url_map.items(): - with open(path, encoding="utf-8") as f: - content = f.read() - - # markdown links - for match in internal_link_pattern.finditer(content): - link = match.group(1) - normalized = resolve_url(link) - if normalized and normalized in link_graph: - link_graph[canonical_url].append(normalized) - - frontmatter, _ = parse_frontmatter(path) - - # related_cards β†’ only from cards - for card in frontmatter.get("related_cards", []): - target = resolve_url_from_dir(card, "cards") - if target and target in link_graph: - link_graph[canonical_url].append(target) - - # mentioned_books β†’ only from library - for book in frontmatter.get("mentioned_books", []): - target = resolve_url_from_dir(book, "library") - if target and target in link_graph: - link_graph[canonical_url].append(target) - - -def main(): - collect_urls() - build_link_graph() - - nodes = [ - {"id": url, "label": title_map[url], "url": url} - for url in link_graph - ] - - edges = [] - for source, targets in link_graph.items(): - for target in targets: - edges.append({"from": source, "to": target}) - - graph = {"nodes": nodes, "edges": edges} - - output_path = os.path.join("public", "link-graph.json") - with open(output_path, "w", encoding="utf-8") as f: - json.dump(graph, f, indent=2, ensure_ascii=False) - print(f"βœ… Link graph generated: {output_path}") - -if __name__ == "__main__": - main() diff --git a/layouts/_default/external-links.html b/layouts/_default/external-links.html new file mode 100644 index 00000000..c07fdcb9 --- /dev/null +++ b/layouts/_default/external-links.html @@ -0,0 +1,8 @@ +{{ define "main" }} +

Top External Links

+
    + {{ range .Site.Data.externallinks }} +
  1. {{ index . 0 }} β€” {{ index . 1 }} links
    2. + {{ end }} +
+{{ end }} diff --git a/scripts/extractLinks.py b/scripts/extractLinks.py new file mode 100644 index 00000000..eb172353 --- /dev/null +++ b/scripts/extractLinks.py @@ -0,0 +1,187 @@ +import os +import re +import json + +content_dir = "content" +internal_link_pattern = re.compile(r"\[[^\]]*\]\((/[^)]+)\)") + +url_map = {} # file path -> canonical url +alias_map = {} # alias url -> canonical url +link_graph = {} # canonical url -> list of canonical urls +title_map = {} # canonical url -> title + + +def parse_frontmatter(filepath): + with open(filepath, encoding="utf-8") as f: + lines = f.readlines() + + if not lines or lines[0].strip() != "---": + return {}, 0 + + frontmatter = {} + aliases = [] + slug = None + draft = False + title = None + related_cards = [] + mentioned_books = [] + i = 1 + while i < len(lines): + line = lines[i].strip() + if line == "---": + break + if ":" in line: + key, value = line.split(":", 1) + key = key.strip() + value = value.strip().strip('"').strip("'") + + if key == "aliases": + aliases = [] + i += 1 + while i < len(lines) and lines[i].startswith(" - "): + alias = lines[i].strip()[3:].strip() + aliases.append(alias) + i += 1 + continue + elif key == "slug": + slug = value + elif key == "draft": + draft = value.lower() == "true" + elif key == "title": + title = value + elif key == "related_cards": + related_cards = [] + i += 1 + while i < len(lines) and lines[i].strip().startswith("- "): + related_cards.append(lines[i].strip()[2:].strip()) + i += 1 + continue + elif key == "mentioned_books": + mentioned_books = [] + i += 1 + while i < len(lines) and lines[i].strip().startswith("- "): + mentioned_books.append(lines[i].strip()[2:].strip()) + i += 1 + continue + i += 1 + + return { + "slug": slug, + "aliases": aliases, + "draft": draft, + "title": title, + "related_cards": related_cards, + "mentioned_books": mentioned_books + }, i + 1 + + +def get_canonical_url(filepath, frontmatter): + rel_path = os.path.relpath(filepath, content_dir) + dirname, filename = os.path.split(rel_path) + name = os.path.splitext(filename)[0] + slug = frontmatter.get("slug") or name + parts = dirname.split(os.sep) + if parts == [""]: + return f"/{slug}/" + return f"/{'/'.join(parts)}/{slug}/" + + +def resolve_url(url): + return alias_map.get(url, url) + + +def resolve_url_from_dir(link, subdir): + """Only search for link in a specific content subdir (like 'cards', 'library')""" + target_dir = os.path.join(content_dir, subdir) + for path, canonical in url_map.items(): + if path.startswith(target_dir): + name = os.path.splitext(os.path.basename(path))[0] + if name == link: + return canonical + return None + + +def get_icon(url): + if url.startswith("/cards/"): + return "πŸ“„" + elif url.startswith("/posts/"): + return "πŸ“œ" + elif url.startswith("/library/"): + return "πŸ“–" + elif url.startswith("/fictions/"): + return "πŸ§™β€β™€οΈ" + elif url.startswith("/en/"): + return "πŸ‡¬πŸ‡§" + return "πŸ“" + + +def collect_urls(): + for root, _, files in os.walk(content_dir): + for file in files: + if not file.endswith(".md") or file == "_index.md": + continue + path = os.path.join(root, file) + frontmatter, _ = parse_frontmatter(path) + if frontmatter.get("draft"): + continue + + canonical_url = get_canonical_url(path, frontmatter) + url_map[path] = canonical_url + link_graph[canonical_url] = [] + title = frontmatter.get("title") or canonical_url.strip("/").split("/")[-1] + title_map[canonical_url] = f"{get_icon(canonical_url)} {title}" + + for alias in frontmatter.get("aliases", []): + alias_map[alias] = canonical_url + + +def build_link_graph(): + for path, canonical_url in url_map.items(): + with open(path, encoding="utf-8") as f: + content = f.read() + + # markdown links + for match in internal_link_pattern.finditer(content): + link = match.group(1) + normalized = resolve_url(link) + if normalized and normalized in link_graph: + link_graph[canonical_url].append(normalized) + + frontmatter, _ = parse_frontmatter(path) + + # related_cards β†’ only from cards + for card in frontmatter.get("related_cards", []): + target = resolve_url_from_dir(card, "cards") + if target and target in link_graph: + link_graph[canonical_url].append(target) + + # mentioned_books β†’ only from library + for book in frontmatter.get("mentioned_books", []): + target = resolve_url_from_dir(book, "library") + if target and target in link_graph: + link_graph[canonical_url].append(target) + + +def main(): + collect_urls() + build_link_graph() + + nodes = [ + {"id": url, "label": title_map[url], "url": url} + for url in link_graph + ] + + edges = [] + for source, targets in link_graph.items(): + for target in targets: + edges.append({"from": source, "to": target}) + + graph = {"nodes": nodes, "edges": edges} + + output_path = os.path.join("public", "link-graph.json") + with open(output_path, "w", encoding="utf-8") as f: + json.dump(graph, f, indent=2, ensure_ascii=False) + print(f"βœ… Link graph generated: {output_path}") + +if __name__ == "__main__": + main() diff --git a/scripts/listExternalLinks.py b/scripts/listExternalLinks.py new file mode 100644 index 00000000..6b52ab76 --- /dev/null +++ b/scripts/listExternalLinks.py @@ -0,0 +1,53 @@ +import os +from urllib.parse import urlparse +from collections import Counter +import json +import re + +content_dir = "content" +external_links = Counter() + +# εŒΉι… Markdown δΈ­ηš„ URL ι“ΎζŽ₯ [text](url) +markdown_link_re = re.compile(r'\[.*?\]\((http.*?)\)') + +def extract_front_matter_and_content(path): + with open(path, 'r', encoding='utf-8') as f: + lines = f.readlines() + + if not lines or lines[0].strip() != '---': + return {}, ''.join(lines) + + front_matter = {} + end_index = None + for i in range(1, len(lines)): + if lines[i].strip() == '---': + end_index = i + break + key_value = lines[i].split(':', 1) + if len(key_value) == 2: + key, value = key_value + front_matter[key.strip()] = value.strip() + + content = ''.join(lines[end_index+1:]) if end_index else ''.join(lines) + return front_matter, content + +def extract_links_from_file(path): + _, content = extract_front_matter_and_content(path) + + # 提取 Markdown 格式链ζŽ₯ + for match in markdown_link_re.findall(content): + domain = urlparse(match).netloc + if domain: + external_links[domain] += 1 + +for root, dirs, files in os.walk(content_dir): + for file in files: + if file.endswith(".md"): + extract_links_from_file(os.path.join(root, file)) + +# 写ε…₯ JSON +os.makedirs("data", exist_ok=True) +with open("data/externallinks.json", "w", encoding="utf-8") as f: + json.dump(external_links.most_common(), f, indent=2, ensure_ascii=False) + +print("βœ… External link data written to data/externallinks.json") -- cgit v1.2.3