feat(vault): Phase 3 tools + batch citation/relation enrichment

This commit is contained in:
2026-07-01 15:20:40 +08:00
parent 8273017082
commit 407a9213bf
548 changed files with 53410 additions and 2113 deletions
+207
View File
@@ -0,0 +1,207 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
enrich-citations.py — Phase 3 P0: 批量为零引用页面补溯源链接
策略:
1. **来源节补全**:无 `## 来源` 节的页面,自动添加,包含指向 raw source 的 wikilink
2. **首段引用注入**:若 raw source 文件存在,提取页面标题关键词,
在 raw 文件中定位匹配行,注入首条 `[raw:source:行号]` 作为溯源种子
3. **存根页标记**body < 50 字的页面添加 `> [!warning] 存根页面` callout
安全措施:
- 幂等:已有 `## 来源` 节或 `[raw:` 引用则跳过
- --dry-run / --apply
- 仅处理 wiki/*.md(排除 index/log/archive
用法:
python tools/scripts/enrich-citations.py --dry-run
python tools/scripts/enrich-citations.py --apply
"""
import argparse
import re
import sys
from pathlib import Path
try:
import yaml
except ImportError:
yaml = None
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
WIKI = VAULT / "wiki"
RAW = VAULT / "raw"
EXCLUDE = {"index.md", "log.md"}
def read_frontmatter(fp: Path) -> tuple[dict, str, str]:
content = fp.read_text(encoding="utf-8")
m = re.match(r"^---\n(.*?)\n---\n?(.*)", content, re.DOTALL)
if not m:
return {}, content, ""
fm_text = m.group(1)
body = m.group(2)
if yaml:
try:
front = yaml.safe_load(fm_text) or {}
except Exception:
front = {}
else:
front = {}
return front, body, fm_text
def get_source_filename(front: dict) -> str:
"""Extract raw filename from source field"""
source = front.get("source", "")
if not source:
return ""
s = str(source).strip().strip('"').strip("'")
s = re.sub(r"\[\[(?:raw/)?(.+?)(?:\|.*)?\]\]", r"\1", s)
s = re.sub(r"\.md$", "", s)
return s
def find_raw_file(source_name: str) -> Path | None:
"""Try to locate the raw file"""
candidates = [
RAW / f"{source_name}.md",
RAW / source_name,
]
# Also search subdirectories
for pattern in [f"**/{source_name}.md", f"**/{source_name}/index.md"]:
matches = list(RAW.glob(pattern))
if matches:
candidates.append(matches[0])
for c in candidates:
if c.exists() and c.is_file():
return c
return None
def find_keyword_line(raw_path: Path, page_name: str) -> str:
"""Find a line in raw file matching the page name or key keyword"""
try:
raw_text = raw_path.read_text(encoding="utf-8")
except Exception:
return ""
lines = raw_text.splitlines()
# Strategy: search for page name, then aliases
keywords = [page_name]
# Also try shorter keywords
if len(page_name) > 4:
keywords.append(page_name[:4])
for kw in keywords:
for i, line in enumerate(lines, 1):
if kw in line and len(line.strip()) > 5:
return f"{i}"
return ""
def has_source_section(body: str) -> bool:
return bool(re.search(r"^##\s*来源", body, re.MULTILINE))
def has_raw_citations(body: str) -> bool:
return "[raw:" in body
def is_stub(body: str) -> bool:
clean = re.sub(r"!\[\[.*?\]\]", "", body)
clean = re.sub(r"```.*?```", "", clean, flags=re.DOTALL)
clean = re.sub(r"\|.*?\|", "", clean)
clean = re.sub(r"\s+", "", clean)
return len(clean) < 150
def main():
parser = argparse.ArgumentParser(description="Enrich wiki pages with source citations")
parser.add_argument("--dry-run", action="store_true", help="Preview without writing")
parser.add_argument("--apply", action="store_true", help="Write changes to files")
args = parser.parse_args()
if not args.dry_run and not args.apply:
args.dry_run = True
stats = {"source_section": 0, "citation_seed": 0, "stub_flag": 0, "skip_has_citation": 0, "skip_no_source": 0}
for fp in sorted(WIKI.glob("*.md")):
if fp.name in EXCLUDE:
continue
front, body, fm_text = read_frontmatter(fp)
page_name = fp.stem
if has_raw_citations(body):
stats["skip_has_citation"] += 1
continue
source_name = get_source_filename(front)
raw_path = find_raw_file(source_name) if source_name else None
modified = False
additions = []
# 1. Add citation seed if raw file exists
if raw_path:
line_num = find_keyword_line(raw_path, page_name)
if line_num:
# Find first paragraph in body and append citation
first_para_match = re.search(r"^(.+?)(?:\n\n|\n#)", body, re.DOTALL)
if first_para_match:
insert_point = first_para_match.end()
cite = f"[raw:{source_name}:{line_num}]"
if cite not in body:
body = body[:insert_point] + f" {cite}" + body[insert_point:]
modified = True
stats["citation_seed"] += 1
additions.append(f"citation seed [raw:{source_name}:{line_num}]")
# 2. Add/fix source section
if not has_source_section(body):
source_link = front.get("source", "")
if source_link:
source_section = f"\n\n## 来源\n\n> **溯源规则**:所有数字/百分比/具体结论必须标注 `[raw:{{文件名}}:{{行号}}]` 格式。\n\n- {source_link}"
body = body.rstrip() + source_section
modified = True
stats["source_section"] += 1
additions.append("source section")
# 3. Flag stub pages
if is_stub(body) and not args.dry_run:
if "> [!warning] 存根页面" not in body:
stub_callout = "\n\n> [!warning] 存根页面\n> 本页内容不足,待扩充。参考同类黄金标准页面(如 [[DSpark]]、[[何长工]])补充来源引用和正文内容。\n"
body = body.rstrip() + stub_callout
modified = True
stats["stub_flag"] += 1
additions.append("stub callout")
if not modified:
if not source_name:
stats["skip_no_source"] += 1
continue
if args.dry_run:
print(f" [[{page_name}]]: {', '.join(additions)}")
continue
if args.apply:
# Reconstruct file
if yaml:
new_yaml = yaml.dump(front, allow_unicode=True, default_flow_style=False, sort_keys=False)
else:
new_yaml = fm_text
fp.write_text(f"---\n{new_yaml}---\n{body}", encoding="utf-8")
mode = "DRY RUN" if args.dry_run else "APPLIED"
print(f"\n=== {mode} ===")
print(f" Citation seeds added: {stats['citation_seed']}")
print(f" Source sections added: {stats['source_section']}")
print(f" Stub flags added: {stats['stub_flag']}")
print(f" Skipped (has citation): {stats['skip_has_citation']}")
print(f" Skipped (no source): {stats['skip_no_source']}")
if __name__ == "__main__":
main()
+199
View File
@@ -0,0 +1,199 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
fix-yaml-frontmatter.py — 批量修复 wiki frontmatter YAML 错误
修复类型:
1. Windows 反斜杠路径 → 正斜杠(source 字段 \ → /
2. 缺空格的 tag 列表项(-tag → - tag
3. 断裂的 type 字段(type: place + 孤儿 - "Region" → 合并为列表)
4. relations 字段格式标准化(Obsidian 兼容:缩进列表 + 双引号)
5. frontmatter 后空行规范化
用法:
python tools/scripts/fix-yaml-frontmatter.py --dry-run
python tools/scripts/fix-yaml-frontmatter.py --apply
"""
import argparse
import re
import sys
from pathlib import Path
try:
import yaml
except ImportError:
print("ERROR: pyyaml required"); sys.exit(1)
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
WIKI = VAULT / "wiki"
EXCLUDE = {"index.md", "log.md"}
class IndentedDumper(yaml.Dumper):
"""Custom dumper that indents sequences under their key (Obsidian-friendly)"""
def increase_indent(self, flow=False, indentless=False):
return super().increase_indent(flow, False)
def yaml_dump(front: dict) -> str:
"""Dump frontmatter with Obsidian-compatible formatting"""
return yaml.dump(
front,
Dumper=IndentedDumper,
allow_unicode=True,
default_flow_style=False,
sort_keys=False,
width=1000, # prevent line wrapping
)
def split_frontmatter(content: str):
"""Split into (frontmatter_text, body) preserving original formatting"""
m = re.match(r"^(---\n)(.*?)(\n---\n?)(.*)", content, re.DOTALL)
if not m:
return None, content, None
return m.group(2), m.group(4), m
def fix_backslashes(fm_text: str) -> tuple[str, bool]:
"""Fix Windows backslash paths in source field"""
changed = False
def replacer(m):
nonlocal changed
val = m.group(0)
if '\\' in val:
changed = True
return val.replace('\\', '/')
return val
result = re.sub(r'source:\s*".*?"', replacer, fm_text)
return result, changed
def fix_dash_nospace(fm_text: str) -> tuple[str, bool]:
"""Fix list items missing space after dash: -tag → - tag"""
changed = False
lines = fm_text.split('\n')
for i, line in enumerate(lines):
m = re.match(r'^(\s+)-(\S)', line)
if m:
lines[i] = m.group(1) + '- ' + m.group(2) + line[m.end():]
changed = True
return '\n'.join(lines), changed
def fix_broken_type(fm_text: str) -> tuple[str, bool]:
"""Fix orphaned type values: type: place\n- "Region" → type list"""
changed = False
lines = fm_text.split('\n')
result = []
i = 0
while i < len(lines):
line = lines[i]
type_match = re.match(r'^(\s*)type:\s*(\S+)', line)
if type_match and i + 1 < len(lines):
next_stripped = lines[i + 1].strip()
if next_stripped.startswith('- ') or next_stripped.startswith('-"'):
indent = type_match.group(1)
main_type = type_match.group(2)
extra = re.match(r'^-\s*"?(.+?)"?$', next_stripped)
extra_val = extra.group(1) if extra else next_stripped.lstrip('- ')
result.append(f'{indent}type:')
result.append(f'{indent} - "{main_type}"')
result.append(f'{indent} - "{extra_val}"')
changed = True
i += 2
continue
result.append(line)
i += 1
return '\n'.join(result), changed
def process_file(fp: Path, apply: bool) -> dict:
content = fp.read_text(encoding="utf-8")
m = re.match(r"^(---\n)(.*?)(\n---\n?)(.*)", content, re.DOTALL)
if not m:
return {"file": fp.name, "skipped": "no frontmatter"}
fm_text = m.group(2)
body = m.group(4)
fixes = []
# Phase 1: Regex fixes for known error patterns
fm_text, changed = fix_backslashes(fm_text)
if changed:
fixes.append("backslash")
fm_text, changed = fix_dash_nospace(fm_text)
if changed:
fixes.append("dash_nospace")
fm_text, changed = fix_broken_type(fm_text)
if changed:
fixes.append("broken_type")
# Phase 2: Parse → Re-dump with Obsidian-compatible formatting
try:
front = yaml.safe_load(fm_text)
except Exception as e:
return {"file": fp.name, "error": f"YAML still invalid after regex fixes: {e}"}
if not isinstance(front, dict):
return {"file": fp.name, "skipped": "frontmatter not a dict"}
# Re-dump with IndentedDumper (proper sequence indentation + double quotes)
new_fm = yaml_dump(front)
# Check if re-dumping changed anything (format normalization)
if new_fm.rstrip() != fm_text.rstrip():
fixes.append("format_normalize")
# Validate the re-dumped YAML
try:
yaml.safe_load(new_fm)
except Exception as e:
return {"file": fp.name, "error": f"Re-dumped YAML invalid: {e}"}
if not fixes:
return {"file": fp.name, "skipped": "no fixes needed"}
# Reconstruct file
new_content = f"---\n{new_fm}---\n\n{body.lstrip()}"
if apply:
fp.write_text(new_content, encoding="utf-8")
return {"file": fp.name, "fixes": fixes}
def main():
parser = argparse.ArgumentParser(description="Fix YAML frontmatter errors in wiki")
parser.add_argument("--dry-run", action="store_true", help="Preview without writing")
parser.add_argument("--apply", action="store_true", help="Write fixes")
args = parser.parse_args()
if not args.dry_run and not args.apply:
args.dry_run = True
mode = "DRY RUN" if args.dry_run else "APPLYING"
print(f"=== {mode} ===\n")
stats = {}
for fp in sorted(WIKI.glob("*.md")):
if fp.name in EXCLUDE:
continue
result = process_file(fp, args.apply)
if "fixes" in result:
print(f" {result['file']}: {', '.join(result['fixes'])}")
for f in result["fixes"]:
stats[f] = stats.get(f, 0) + 1
elif "error" in result:
print(f" ERROR: {result['file']}: {result['error']}")
print(f"\n=== Summary ===")
for fix, count in sorted(stats.items()):
print(f" {fix}: {count} files")
print(f" Total files fixed: {sum(stats.values())}")
if __name__ == "__main__":
main()
+46 -5
View File
@@ -5,14 +5,17 @@ graph-search.py — 图感知搜索
流程:
1. qmd vsearch 获取语义匹配结果(Top 10)
2. 对每个结果,读取 relations 字段 → 获取相邻节点
2. 对每个结果,扩展相邻节点
- 默认:读取各页面 relations 字段(实时,慢但零状态)
- --use-graph:读取预构建的 knowledge-graph.json(快,需先 build
3. 去重后返回(直接关联 + 关系扩展)
用法:
python tools/scripts/graph-search.py "查询词" [--count 15] [--depth 1]
python tools/scripts/graph-search.py "查询词" [--count 15] [--depth 1] [--use-graph]
依赖:
- qmd 已安装(通过 node
- --use-graph 需先运行 knowledge-graph.py build
"""
import argparse
import json
@@ -28,6 +31,7 @@ except ImportError:
QMD = r'node "C:\Users\hhhh2024\AppData\Roaming\npm\node_modules\@tobilu\qmd\dist\cli\qmd.js"'
WIKI = Path(r"D:\Applications\app\kepano-obsidian-main\wiki")
GRAPH_DB = Path(r"D:\Applications\app\kepano-obsidian-main\tools\data\knowledge-graph.json")
def qmd_search(query: str, count: int = 10) -> list[str]:
@@ -57,7 +61,7 @@ def get_relations(page_name: str) -> list[dict]:
def expand_nodes(nodes: list[str], depth: int = 1) -> list[str]:
"""从起始节点出发,沿 relations 扩展相邻节点"""
"""从起始节点出发,沿 relations 扩展相邻节点(实时读取页面 frontmatter"""
expanded = list(nodes)
frontier = list(nodes)
for _ in range(depth):
@@ -75,11 +79,38 @@ def expand_nodes(nodes: list[str], depth: int = 1) -> list[str]:
return expanded
def expand_nodes_via_graph(nodes: list[str], depth: int = 1, min_conf: int = 3) -> list[str]:
"""通过预构建的 knowledge-graph.json 扩展(更快,无 IO"""
if not GRAPH_DB.exists():
return list(nodes)
graph = json.loads(GRAPH_DB.read_text(encoding="utf-8"))
adj: dict[str, list] = {}
for e in graph["edges"]:
adj.setdefault(e["source"], []).append((e["target"], e.get("confidence", 3)))
adj.setdefault(e["target"], []).append((e["source"], e.get("confidence", 3)))
expanded = list(nodes)
frontier = list(nodes)
for _ in range(depth):
next_frontier = []
for node in frontier:
for target, conf in adj.get(node, []):
if conf >= min_conf and target not in expanded:
expanded.append(target)
next_frontier.append(target)
frontier = next_frontier
if not frontier:
break
return expanded
def main():
parser = argparse.ArgumentParser(description="Graph-aware search for wiki pages")
parser.add_argument("query", help="Search query")
parser.add_argument("--count", type=int, default=15, help="Max results")
parser.add_argument("--depth", type=int, default=1, help="Graph expansion depth")
parser.add_argument("--use-graph", action="store_true",
help="Use prebuilt knowledge-graph.json instead of live frontmatter")
args = parser.parse_args()
print(f"=== Searching: {args.query} ===")
@@ -92,10 +123,20 @@ def main():
for r in results:
print(f" [[{r}]]")
expanded = expand_nodes(results, depth=args.depth)
if args.use_graph and not GRAPH_DB.exists():
print(f"\n (note: {GRAPH_DB.name} not found, run `knowledge-graph.py build`; "
f"falling back to live frontmatter)")
if args.use_graph and GRAPH_DB.exists():
expanded = expand_nodes_via_graph(results, depth=args.depth)
source = "knowledge-graph.json"
else:
expanded = expand_nodes(results, depth=args.depth)
source = "live frontmatter"
new = [e for e in expanded if e not in results]
if new:
print(f"\nGraph-expanded (via relations, depth={args.depth}):")
print(f"\nGraph-expanded (via {source}, depth={args.depth}):")
for n in new[:args.count]:
print(f" [[{n}]]")
if len(new) > args.count:
+288
View File
@@ -0,0 +1,288 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
knowledge-graph.py — Phase 3 图存储层 + 图遍历搜索
维护 wiki/ 的 typed relationships 图。存储为 JSONtools/data/knowledge-graph.json),
支持图构建、证据链反向追溯、以及融合搜索(qmd 直接匹配 + graph 关系扩展)。
轻量级方案(Sqlite + JSON 混合):当前仅用 JSON,无需外部数据库依赖。
用法:
python tools/scripts/knowledge-graph.py build # 从 relations 重建图
python tools/scripts/knowledge-graph.py stats # 图统计
python tools/scripts/knowledge-graph.py evidence <页面> # 反向追溯证据链(caused_by/supports
python tools/scripts/knowledge-graph.py neighbors <页面> # 直接邻接节点
python tools/scripts/knowledge-graph.py search "<查询>" # 融合搜索(需 qmd
python tools/scripts/knowledge-graph.py export-dot # 输出 Graphviz DOT
"""
import argparse
import json
import re
import subprocess
import sys
from collections import deque
from pathlib import Path
try:
import yaml
except ImportError:
yaml = None
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
WIKI = VAULT / "wiki"
GRAPH_DB = VAULT / "tools" / "data" / "knowledge-graph.json"
QMD = r'node "C:\Users\hhhh2024\AppData\Roaming\npm\node_modules\@tobilu\qmd\dist\cli\qmd.js"'
EVIDENCE_EDGES = {"caused_by", "supports", "extends"}
def read_frontmatter(fp: Path) -> dict | None:
content = fp.read_text(encoding="utf-8")
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
if not m or not yaml:
return None
try:
return yaml.safe_load(m.group(1)) or {}
except Exception:
return None
def build_graph() -> dict:
"""扫描所有 wiki 页面的 relations 字段,重建完整图"""
graph = {"nodes": [], "edges": [], "meta": {}}
seen = set()
edges_seen = set()
for fp in sorted(WIKI.glob("*.md")):
if fp.name in ("index.md", "log.md"):
continue
front = read_frontmatter(fp)
if front is None:
front = {}
name = fp.stem
if name not in seen:
graph["nodes"].append({
"id": name,
"type": front.get("type", "unknown"),
"confidence": front.get("confidence", 3),
"status": front.get("status", "active"),
})
seen.add(name)
for rel in front.get("relations", []) or []:
if not isinstance(rel, dict):
continue
target = str(rel.get("target", "")).strip("[]")
if not target or target == name:
continue
if target not in seen:
graph["nodes"].append({
"id": target,
"type": "unknown",
"confidence": 3,
"status": "unknown",
})
seen.add(target)
edge_key = (name, target, rel.get("type", "related"))
if edge_key in edges_seen:
continue
edges_seen.add(edge_key)
graph["edges"].append({
"source": name,
"target": target,
"type": rel.get("type", "related"),
"confidence": rel.get("confidence", 3),
})
graph["meta"] = {
"node_count": len(graph["nodes"]),
"edge_count": len(graph["edges"]),
"edge_types": {},
}
for e in graph["edges"]:
t = e["type"]
graph["meta"]["edge_types"][t] = graph["meta"]["edge_types"].get(t, 0) + 1
GRAPH_DB.parent.mkdir(parents=True, exist_ok=True)
GRAPH_DB.write_text(
json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8"
)
return graph
def load_graph() -> dict | None:
if not GRAPH_DB.exists():
return None
return json.loads(GRAPH_DB.read_text(encoding="utf-8"))
def adjacency(graph: dict, directed: bool = True) -> dict:
"""构建邻接表。directed=False 时加入反向边(标注 inverse_ 前缀)"""
adj: dict[str, list] = {}
for e in graph["edges"]:
adj.setdefault(e["source"], []).append((e["target"], e["type"], e.get("confidence", 3)))
if not directed:
adj.setdefault(e["target"], []).append(
(e["source"], "inverse_" + e["type"], e.get("confidence", 3))
)
return adj
def find_evidence_chain(target_page: str, max_depth: int = 5) -> list[str]:
"""从 target 反向追溯所有 caused_by/supports 关系的源头(BFS"""
graph = load_graph()
if not graph:
return []
reverse_adj: dict[str, list] = {}
for e in graph["edges"]:
reverse_adj.setdefault(e["target"], []).append((e["source"], e["type"]))
chain: list[str] = []
visited = {target_page}
dq = deque([(target_page, 0)])
while dq:
node, depth = dq.popleft()
for src, rel in reverse_adj.get(node, []):
if rel not in EVIDENCE_EDGES:
continue
if src in visited:
continue
visited.add(src)
chain.append((" " * (depth + 1)) + f"← [[{src}]] ({rel})")
if depth + 1 < max_depth:
dq.append((src, depth + 1))
return chain
def graph_expanded_search(query: str, count: int = 15, min_conf: int = 3) -> list[dict]:
"""融合搜索:qmd BM25/向量结果 → graph 关系扩展 → 去重重排序"""
cmd = f'{QMD} vsearch "{query}" -c wiki -n 10'
result = subprocess.run(cmd, capture_output=True, text=True, shell=True, timeout=30)
direct = [ln.strip() for ln in result.stdout.splitlines() if ln.strip()]
graph = load_graph()
if not graph:
return [{"page": p, "relevance": "direct"} for p in direct[:count]]
adj = adjacency(graph, directed=True)
expanded = list(direct)
for page in direct:
for target, rel_type, conf in adj.get(page, []):
if target not in expanded and conf >= min_conf:
expanded.append(target)
return [{"page": p, "relevance": "direct" if p in direct else "graph"} for p in expanded[:count]]
def cmd_stats(args):
graph = load_graph()
if not graph:
print("Graph DB not found. Run 'build' first.")
return
meta = graph["meta"]
print(f"=== Knowledge Graph Stats ===")
print(f" Nodes: {meta['node_count']}")
print(f" Edges: {meta['edge_count']}")
print(f"\n Edge type breakdown:")
for t, c in sorted(meta["edge_types"].items(), key=lambda x: -x[1]):
print(f" {t}: {c}")
typed_nodes = sum(1 for n in graph["nodes"] if n["type"] != "unknown")
print(f"\n Typed nodes: {typed_nodes} / {meta['node_count']}")
def cmd_evidence(args):
chain = find_evidence_chain(args.page)
print(f"=== Evidence chain for [[{args.page}]] ===")
if not chain:
print(" No evidence edges (caused_by/supports/extends) found.")
return
for line in chain:
print(f" {line}")
print(f"\n Total upstream evidence: {len(chain)}")
def cmd_neighbors(args):
graph = load_graph()
if not graph:
print("Graph DB not found. Run 'build' first.")
return
found = False
for e in graph["edges"]:
if e["source"] == args.page:
print(f" --{e['type']}--> [[{e['target']}]] (conf={e.get('confidence', 3)})")
found = True
for e in graph["edges"]:
if e["target"] == args.page:
print(f" <--{e['type']}-- [[{e['source']}]] (conf={e.get('confidence', 3)})")
found = True
if not found:
print(f" [[{args.page}]] has no relations in the graph.")
def cmd_search(args):
print(f"=== Fusion search: {args.query} ===")
results = graph_expanded_search(args.query, count=args.count)
direct = [r for r in results if r["relevance"] == "direct"]
graph_hits = [r for r in results if r["relevance"] == "graph"]
if direct:
print(f"\nDirect matches ({len(direct)}):")
for r in direct:
print(f" [[{r['page']}]]")
if graph_hits:
print(f"\nGraph-expanded (via relations):")
for r in graph_hits:
print(f" [[{r['page']}]]")
print(f"\nTotal: {len(results)}")
def cmd_export_dot(args):
graph = load_graph()
if not graph:
print("Graph DB not found. Run 'build' first.")
return
print("digraph wiki {")
print(' rankdir=LR;')
for n in graph["nodes"]:
print(f' "{n["id"]}";')
for e in graph["edges"]:
print(f' "{e["source"]}" -> "{e["target"]}" [label="{e["type"]}"];')
print("}")
def main():
parser = argparse.ArgumentParser(description="Phase 3 knowledge graph storage + traversal")
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("build", help="Rebuild graph from relations")
sub.add_parser("stats", help="Print graph statistics")
p_ev = sub.add_parser("evidence", help="Trace evidence chain (caused_by/supports)")
p_ev.add_argument("page", help="Target page name")
p_nb = sub.add_parser("neighbors", help="Direct neighbors of a page")
p_nb.add_argument("page", help="Page name")
p_se = sub.add_parser("search", help="Fusion search (qmd + graph expansion)")
p_se.add_argument("query", help="Search query")
p_se.add_argument("--count", type=int, default=15)
sub.add_parser("export-dot", help="Export Graphviz DOT format")
args = parser.parse_args()
if args.command == "build":
graph = build_graph()
m = graph["meta"]
print(f"Graph built: {m['node_count']} nodes, {m['edge_count']} edges")
print(f"Saved to {GRAPH_DB}")
elif args.command == "stats":
cmd_stats(args)
elif args.command == "evidence":
cmd_evidence(args)
elif args.command == "neighbors":
cmd_neighbors(args)
elif args.command == "search":
cmd_search(args)
elif args.command == "export-dot":
cmd_export_dot(args)
if __name__ == "__main__":
main()
+291
View File
@@ -0,0 +1,291 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
populate-relations.py — Phase 3 P1: 批量填充 wiki 页面的 relations 字段
策略(自动推断,无 LLM 依赖):
1. **Source-grouping**:共享同一 raw source 的页面群,互为 part_of 关系
- 每组中被引用最多的页面成为 hub
- 其余页面 part_of → hub
2. **Type-based**
- entity (人物) → example_of 对应 concept(如 毛泽东 example_of 教员的终极答案)
- place → part_of 上级地点(如 西安 part_of 陕西省)
3. **Cross-link inference**:页面正文中的 [[wikilink]] 若目标存在,
且源页面 type=concept,推断 extends 关系(仅取 top 2 避免噪声)
安全措施:
- 幂等:已有 relations 不重复添加
- 仅处理 wiki/*.md(排除 index/log/archive
- --dry-run 预览,--apply 执行
- 每页最多添加 5 条 relations(防爆裂)
用法:
python tools/scripts/populate-relations.py --dry-run # 预览
python tools/scripts/populate-relations.py --apply # 执行
python tools/scripts/populate-relations.py --apply --max-per-page 3
"""
import argparse
import json
import re
import sys
from collections import defaultdict
from pathlib import Path
try:
import yaml
except ImportError:
print("ERROR: pyyaml required"); sys.exit(1)
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
WIKI = VAULT / "wiki"
EXCLUDE = {"index.md", "log.md"}
MAX_RELATIONS_DEFAULT = 5
def read_frontmatter(fp: Path) -> tuple[dict, str]:
content = fp.read_text(encoding="utf-8")
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
if not m:
return {}, content
try:
front = yaml.safe_load(m.group(1)) or {}
except Exception:
return {}, content
return front, content
def write_frontmatter(fp: Path, front: dict, body: str):
# Strip any existing frontmatter from body to avoid duplication
body_clean = re.sub(r"^---\n.*?\n---\n?", "", body, count=1, flags=re.DOTALL).lstrip()
new_yaml = yaml.dump(front, allow_unicode=True, default_flow_style=False, sort_keys=False)
fp.write_text(f"---\n{new_yaml}---\n\n{body_clean}", encoding="utf-8")
def get_source_key(front: dict) -> str:
"""Extract a normalized source key for grouping"""
source = front.get("source", "")
if not source:
return ""
s = str(source).strip().strip('"').strip("'")
# Remove [[ ]] and raw/ prefix
s = re.sub(r"\[\[(?:raw/)?(.+?)(?:\|.*)?\]\]", r"\1", s)
# Remove .md extension
s = re.sub(r"\.md$", "", s)
# Remove path prefixes
s = s.split("/")[-1]
return s
def get_inbound_counts() -> dict[str, int]:
"""Count inbound wikilinks for each wiki page"""
counts = defaultdict(int)
all_content = ""
for fp in VAULT.rglob("*.md"):
if ".git" in str(fp):
continue
try:
all_content += fp.read_text(encoding="utf-8") + "\n"
except Exception:
continue
for m in re.finditer(r"\[\[([^\]|#]+?)(?:[#\]|]|$)", all_content):
name = m.group(1).strip()
if name and not name.startswith("http"):
counts[name] += 1
return counts
def extract_wikilinks(content: str) -> list[str]:
"""Extract wikilink targets from page body (not frontmatter)"""
# Remove frontmatter
body = re.sub(r"^---\n.*?\n---", "", content, count=1, flags=re.DOTALL)
links = []
for m in re.finditer(r"\[\[([^\]|#]+?)(?:[#\]|])", body):
target = m.group(1).strip()
if target and not target.startswith("http") and not target.startswith("raw/"):
links.append(target)
return links
def dedup_existing(front: dict) -> set[tuple[str, str]]:
"""Get existing relation (type, target) pairs"""
existing = set()
for r in front.get("relations", []) or []:
if isinstance(r, dict):
t = r.get("target", "").strip("[]")
existing.add((r.get("type", ""), t))
return existing
def main():
parser = argparse.ArgumentParser(description="Batch populate relations for wiki pages")
parser.add_argument("--dry-run", action="store_true", help="Preview without writing")
parser.add_argument("--apply", action="store_true", help="Write changes to files")
parser.add_argument("--max-per-page", type=int, default=MAX_RELATIONS_DEFAULT)
args = parser.parse_args()
if not args.dry_run and not args.apply:
args.dry_run = True # default to dry-run for safety
# Phase 1: Read all pages
pages = {}
for fp in sorted(WIKI.glob("*.md")):
if fp.name in EXCLUDE:
continue
front, content = read_frontmatter(fp)
if not front:
continue
pages[fp.stem] = {"fp": fp, "front": front, "content": content}
print(f"Loaded {len(pages)} wiki pages")
# Phase 2: Compute inbound counts
inbound = get_inbound_counts()
print(f"Computed inbound counts for {len(inbound)} link targets")
# Phase 3: Group by source
source_groups = defaultdict(list)
for name, info in pages.items():
src = get_source_key(info["front"])
if src:
source_groups[src].append(name)
# Phase 4: Infer relations
additions = defaultdict(list) # page -> list of {type, target, ...}
# 4a: Source-grouping → part_of hub
multi_page_groups = {s: g for s, g in source_groups.items() if len(g) > 1}
print(f"\nSource groups with >1 page: {len(multi_page_groups)}")
for src, group in multi_page_groups.items():
# Hub = most referenced page in group
hub = max(group, key=lambda p: inbound.get(p, 0))
for page in group:
if page == hub:
continue
additions[page].append({
"type": "part_of",
"target": f"[[{hub}]]",
"description": f"同源:{src}",
"confidence": 3,
})
# 4b: Single-source pages → extends the source concept if source is a wiki page
single_source = {s: g[0] for s, g in source_groups.items() if len(g) == 1}
for src, page in single_source.items():
info = pages.get(page)
if not info:
continue
page_type = info["front"].get("type", "")
if isinstance(page_type, list):
page_type = page_type[0] if page_type else ""
page_type = str(page_type).lower()
# If source matches a wiki page name, add extends
if src in pages and src != page:
src_type = pages[src]["front"].get("type", "")
if isinstance(src_type, list):
src_type = src_type[0] if src_type else ""
src_type = str(src_type).lower()
if page_type == "entity" and src_type in ("concept", "reference"):
additions[page].append({
"type": "example_of",
"target": f"[[{src}]]",
"confidence": 3,
})
elif page_type == "concept" and src_type == "concept":
additions[page].append({
"type": "extends",
"target": f"[[{src}]]",
"confidence": 3,
})
elif page_type in ("place",) and src_type in ("place", "concept"):
additions[page].append({
"type": "part_of",
"target": f"[[{src}]]",
"confidence": 3,
})
# 4c: Cross-link inference for concept pages (top 2 wikilinks)
for name, info in pages.items():
page_type = info["front"].get("type", "")
if isinstance(page_type, list):
page_type = page_type[0] if page_type else ""
if str(page_type).lower() not in ("concept", "method"):
continue
links = extract_wikilinks(info["content"])
seen = set()
added = 0
for target in links:
if added >= 2:
break
if target == name or target in seen:
continue
if target not in pages:
continue
seen.add(target)
additions[name].append({
"type": "extends",
"target": f"[[{target}]]",
"confidence": 2,
})
added += 1
# Phase 5: Apply (single pass — dry-run prints, apply writes)
total_added = 0
pages_modified = 0
for name in sorted(pages.keys()):
pending = additions.get(name, [])
if not pending:
continue
info = pages[name]
front = info["front"]
existing = dedup_existing(front)
new_rels = list(front.get("relations", []) or [])
added = 0
for rel in pending:
key = (rel["type"], rel["target"].strip("[]"))
if key in existing:
continue
if len(new_rels) >= args.max_per_page:
break
new_rels.append(rel)
existing.add(key)
added += 1
if added == 0:
continue
front["relations"] = new_rels
pages_modified += 1
total_added += added
if args.dry_run:
print(f" [[{name}]]: +{added}")
for rel in new_rels[-added:]:
print(f" --{rel['type']}--> {rel['target']}")
mode = "DRY RUN" if args.dry_run else "APPLYING"
print(f"\n=== {mode} ===")
print(f" Pages to modify: {pages_modified}")
print(f" Relations to add: {total_added}")
if args.apply and pages_modified > 0:
written = 0
for name in sorted(pages.keys()):
info = pages[name]
front = info["front"]
rels = front.get("relations", [])
if not rels:
continue
# Only write if relations were newly populated (not pre-existing)
# Check by seeing if this page was in our additions map
if name not in additions:
continue
write_frontmatter(info["fp"], front, info["content"])
written += 1
print(f" Written to {written} files")
if __name__ == "__main__":
main()
+224
View File
@@ -0,0 +1,224 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
review-pages.py — Phase 3 wiki 页面评审器
两种模式:
1. LLM 评审(需 Ollama 后端):调用本地模型审核内容准确性,检测声明与 raw 来源
不匹配、跨页面矛盾、过时信息。
2. 静态评审(LLM 不可用时自动降级):基于 frontmatter 规则与 raw 来源比对,
检测低置信度、缺 source、断链、过时等结构问题。无外部依赖。
B-3.3 低置信度自动建议:confidence <= 2 的页面生成"需要更多证据"提示。
B-3.1 LLM 评审:可检测语义偏差(被 AI 过度概括/扭曲的结论)。
用法:
python tools/scripts/review-pages.py --stale # 审查过时页面
python tools/scripts/review-pages.py --random 5 # 随机抽 5 页
python tools/scripts/review-pages.py --page "LLM-Wiki-v2" # 指定单页
python tools/scripts/review-pages.py --confidence-low # 低置信度页面
python tools/scripts/review-pages.py --random 3 --llm # 强制启用 LLM 评审
python tools/scripts/review-pages.py --json # JSON 输出
"""
import argparse
import json
import random
import re
import shutil
import subprocess
import sys
from datetime import date
from pathlib import Path
try:
import yaml
except ImportError:
yaml = None
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
WIKI = VAULT / "wiki"
RAW = VAULT / "raw"
TODAY = date.today()
LLM_MODEL = "qwen2.5:7b"
REVIEW_PROMPT = """请审核以下 Obsidian wiki 页面,输出 JSON(仅 JSON,无多余文字):
{
"accuracy_score": 1到5的整数,
"issues": ["具体问题描述"],
"confidence_match": true或false,
"suggested_updates": ["可执行的改进建议"]
}
重点关注:数字/百分比是否标注来源行号、结论是否被过度概括、frontmatter 是否完整。
页面内容(截断至 4000 字符):
{content}
"""
def read_frontmatter(fp: Path) -> tuple[dict, str]:
content = fp.read_text(encoding="utf-8")
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
if not m or not yaml:
return {}, content
try:
return yaml.safe_load(m.group(1)) or {}, content
except Exception:
return {}, content
def ollama_available() -> bool:
return shutil.which("ollama") is not None
def llm_review(page_name: str, content: str) -> dict | None:
"""调用 Ollama 审核单页,返回结构化结果"""
if not ollama_available():
return None
prompt = REVIEW_PROMPT.format(content=content[:4000])
try:
result = subprocess.run(
["ollama", "run", LLM_MODEL, prompt],
capture_output=True, text=True, timeout=180, encoding="utf-8",
)
out = result.stdout.strip()
start, end = out.find("{"), out.rfind("}")
if start != -1 and end != -1:
return json.loads(out[start:end + 1])
except Exception:
pass
return None
def static_review(fp: Path) -> dict:
"""静态规则评审:无 LLM 依赖的结构化检查"""
front, content = read_frontmatter(fp)
issues = []
suggestions = []
cats = front.get("categories", []) or []
if not any("LLM Wiki" in str(c) for c in cats):
issues.append("categories 缺少 [[LLM Wiki]]")
tags = front.get("tags", []) or []
if "wiki" not in tags:
issues.append("tags 缺少 wiki")
if not front.get("type"):
issues.append("缺少 type 字段")
source = front.get("source", "")
if not source or str(source).strip('[]"') == "":
issues.append("缺少 sourcewiki 层必填)")
suggestions.append(f"补充 source 指向 raw/ 来源,参考同类页面:{fp.stem}")
conf = front.get("confidence")
if conf is not None and isinstance(conf, (int, float)) and conf <= 2:
issues.append(f"低置信度 (confidence={conf})")
suggestions.append("需要更多证据:补充多源交叉验证或将 confidence 提升至 3+")
last = str(front.get("last_reviewed", ""))
if last:
try:
overdue = (TODAY - date.fromisoformat(last)).days
interval = front.get("review_interval_days", 180)
if overdue > interval:
issues.append(f"过时(上次审查 {last},超期 {overdue - interval} 天)")
suggestions.append(f"复查内容并更新 last_reviewed 为 {TODAY.isoformat()}")
except ValueError:
issues.append(f"last_reviewed 格式无效:{last}")
no_line = re.findall(r"\[raw:[^:\]\[]+\]", content)
if no_line:
issues.append(f"{len(no_line)} 条 raw 引用缺少行号")
suggestions.append("运行 python tools/scripts/fix-raw-citations.py 自动补全行号")
score = 5 - min(4, len([i for i in issues if i.startswith(("低置信", "过时", "缺少 source"))]))
return {
"page": fp.stem,
"mode": "static",
"accuracy_score": max(1, score),
"issues": issues,
"confidence_match": (conf is None or conf >= 3),
"suggested_updates": suggestions,
}
def select_pages(args) -> list[Path]:
all_pages = [p for p in sorted(WIKI.glob("*.md"))
if p.name not in ("index.md", "log.md")]
if args.page:
fp = WIKI / f"{args.page}.md"
return [fp] if fp.exists() else []
if args.confidence_low:
return [p for p in all_pages
if (lambda c: c is not None and isinstance(c, (int, float)) and c <= 2)(
read_frontmatter(p)[0].get("confidence"))]
if args.stale:
selected = []
for p in all_pages:
front, _ = read_frontmatter(p)
last = str(front.get("last_reviewed", ""))
interval = front.get("review_interval_days", 180)
if not last:
selected.append(p)
continue
try:
if (TODAY - date.fromisoformat(last)).days > interval:
selected.append(p)
except ValueError:
selected.append(p)
return selected
if args.random:
return random.sample(all_pages, min(args.random, len(all_pages)))
return all_pages
def main():
parser = argparse.ArgumentParser(description="Review wiki pages (LLM or static fallback)")
sel = parser.add_mutually_exclusive_group()
sel.add_argument("--stale", action="store_true", help="Review stale pages only")
sel.add_argument("--random", type=int, metavar="N", help="Randomly sample N pages")
sel.add_argument("--page", help="Review a single page")
sel.add_argument("--confidence-low", action="store_true", help="Review confidence<=2 pages")
parser.add_argument("--llm", action="store_true", help="Force LLM review (requires Ollama)")
parser.add_argument("--json", action="store_true", help="Output JSON")
args = parser.parse_args()
pages = select_pages(args)
if not pages:
print("No pages matched selection criteria.")
return
use_llm = args.llm or (not args.confidence_low)
llm_on = use_llm and ollama_available()
if use_llm and not llm_on:
print("(Ollama 不可用,降级为静态评审模式。安装 Ollama 以启用 LLM 语义审核。)\n")
reports = []
for fp in pages:
front, content = read_frontmatter(fp)
if llm_on:
llm = llm_review(fp.stem, content)
report = llm if llm else static_review(fp)
if report:
report["mode"] = "llm"
else:
report = static_review(fp)
report.setdefault("page", fp.stem)
reports.append(report)
if args.json:
print(json.dumps(reports, ensure_ascii=False, indent=2))
return
print(f"=== Review Report ({'LLM' if llm_on else 'static'} mode, {len(reports)} pages) ===\n")
needs_action = [r for r in reports if r["issues"]]
for r in reports:
flag = "OK " if not r["issues"] else "FIX"
print(f"[{flag}] [[{r['page']}]] — score {r['accuracy_score']}/5")
for issue in r["issues"]:
print(f"{issue}")
for sug in r["suggested_updates"]:
print(f"{sug}")
print(f"\nSummary: {len(needs_action)}/{len(reports)} pages need attention")
if __name__ == "__main__":
main()
+143 -43
View File
@@ -12,14 +12,23 @@ sync_home_wiki.py — home-wiki 知识页面 → kepano LLM Wiki 同步脚本
concepts/ + entities/ + syntheses/ (排除 index.md
Daily/ MyNotes/ reports/ 不在知识同步范围。
转换规则(home-wiki frontmatter → kepano Wiki 规范):
categories: 加 [[LLM Wiki]]entity/person 额外加 [[People]]
tags: 前缀 [wiki, {people|concept}]subtype→concept/{subtype}(仅 concept),追加原 tags
created: 保留
source: sources[0] → 文件路径取 stem 做 wikilink;URL/标识符原样;空则留空
type: 保留(concept/entity/synthesis
aliases: title 或 name
正文: 保留,移除 home-wiki 特有的 openclaw 自动段落(## Related、注释标记行)
转换规则(home-wiki frontmatter → kepano Wiki 规范Phase 1 v2 升级版):
categories: 加 [[LLM Wiki]]entity/person 额外加 [[People]]
tags: 前缀 [wiki, {people|concept}]subtype→concept/{subtype}(仅 concept),追加原 tags
created: 保留
source: sources[0] → 文件路径取 stem 做 wikilink;URL/标识符原样;空则留空
type: 保留(concept/entity/synthesis
aliases: title 或 name
confidence: home-wiki 已有则沿用,否则默认 3
status: 默认 active
last_reviewed: 同步日期(YYYY-MM-DD
review_interval_days: 180(技术类 90
relations: 从 home-wiki `related: [[wikilink]]` 字段提取 → `[{type: extends, target: [[x]]}]`
正文: 保留,移除 home-wiki 特有的 openclaw 自动段落(## Related、注释标记行)
YAML 输出格式(Obsidian 兼容):
使用 IndentedDumper(子类)确保序列正确缩进(列表项 2 空格缩进),
wikilink 字符串加双引号,避免 Obsidian 解析异常。
index 自动维护(--index):
用标记块 <!-- BEGIN/END home-wiki-sync --> 界定 index.md 的「概念页」「实体页」
@@ -32,7 +41,7 @@ index 自动维护(--index):
python tools/scripts/sync_home_wiki.py --dry-run # 仅诊断差异
python tools/scripts/sync_home_wiki.py # 同步(写入变化的页面)
python tools/scripts/sync_home_wiki.py --index --log # 一站式:同步+更新index+记日志
python tools/scripts/sync_home_wiki.py --lint # 体检(孤儿/断链/source空值)
python tools/scripts/sync_home_wiki.py --lint # 体检(孤儿/断链/source空值/缺失lifecycle字段
退出码:0 成功;1 源目录缺失;2 有错误。
"""
@@ -50,6 +59,24 @@ except ImportError:
sys.stderr.write("ERROR: PyYAML 未安装,请运行 pip install pyyaml\n")
sys.exit(2)
class IndentedDumper(yaml.Dumper):
"""Custom dumper that indents sequences under their key (Obsidian-friendly)"""
def increase_indent(self, flow=False, indentless=False):
return super().increase_indent(flow, False)
def yaml_dump(data) -> str:
"""Dump data with Obsidian-compatible formatting (indented sequences, double quotes)"""
return yaml.dump(
data,
Dumper=IndentedDumper,
allow_unicode=True,
default_flow_style=False,
sort_keys=False,
width=1000,
)
# ---------------------------------------------------------------------------
# 路径配置(两库位于不同位置,必须使用绝对路径)
# ---------------------------------------------------------------------------
@@ -103,7 +130,7 @@ def _robust_parse_fm(fm_text):
m = re.search(rf"^[ \t]*{re.escape(key)}[ \t]*:[ \t]*(.+?)[ \t]*$", fm_text, re.M)
if m and m.group(1).strip() not in ("", "[]"):
fm[key] = m.group(1).strip().strip("\"'")
for key in ("tags", "sources"):
for key in ("tags", "sources", "related"):
vals = _extract_list_field(fm_text, key)
if vals:
fm[key] = vals
@@ -157,8 +184,38 @@ def _as_list(v):
return [v]
def _extract_related(fm_text):
"""提取 home-wiki 的 `related: [[x]]` 字段为 relations 列表。
home-wiki 用 `related: [[wikilink]]` 记录关联页面(YAML 解析时 [[ 会失败),
转为 kepano Wiki 的 `relations: [{type: extends, target: "[[x]]"}]` 格式。
"""
rels = []
# 匹配 related: 后跟 [[wikilink]],支持多个
for m in re.finditer(r"^[ \t]*related[ \t]*:[ \t]*\[?\[([^\]|#]+)", fm_text, re.M):
tgt = m.group(1).strip()
if tgt:
rels.append({"type": "extends", "target": f"[[{tgt}]]", "confidence": 2})
# 也支持 related: 换行后列表
block = re.search(r"^[ \t]*related[ \t]*:[ \t]*\n((?:[ \t]+-\s*[^\n]+\n?)+)", fm_text, re.M)
if block:
for line in block.group(1).splitlines():
m = re.search(r"\[\[([^\]|#]+)", line)
if m:
rels.append({"type": "extends", "target": f"[[{m.group(1).strip()}]]", "confidence": 2})
return rels
def convert_frontmatter(fm, default_name):
"""home-wiki frontmatter → kepano Wiki frontmatter(有序)。"""
"""home-wiki frontmatter → kepano Wiki frontmatter(有序 dict)。
Phase 1 lifecycle 字段(2026-07-01 新增):
- confidence: 1-5(默认 3home-wiki 有则沿用)
- status: active(默认)
- last_reviewed: 同步日期
- review_interval_days: 180(技术类 90
- relations: 从 home-wiki related: 字段映射
"""
fm_type = str(fm.get("type", "concept")).strip()
subtype = fm.get("subtype")
if isinstance(subtype, list):
@@ -178,8 +235,6 @@ def convert_frontmatter(fm, default_name):
type_tag = "concept"
tags = ["wiki", type_tag]
# 仅 concept 类型把 subtype 转为 concept/{subtype} 标签
# entity/person 用 people 标签 + [[People]] category,不加 concept/person
if subtype and fm_type == "concept":
tags.append(f"concept/{subtype}")
for t in _as_list(fm.get("tags")):
@@ -196,37 +251,41 @@ def convert_frontmatter(fm, default_name):
else:
src_val = ""
return [
("categories", categories),
("tags", tags),
("created", str(fm.get("created", ""))),
("source", src_val),
("type", fm_type),
("aliases", [str(title)]),
]
today = date.today().isoformat()
confidence = fm.get("confidence", 3)
try:
confidence = int(confidence)
except (ValueError, TypeError):
confidence = 3
# 技术类(type=tool)默认 90 天审查周期
review_interval = 90 if fm_type == "tool" else 180
# 用普通 dictPython 3.7+ 保持插入顺序),避免 OrderedDict YAML 序列化问题
pairs = {
"categories": categories,
"tags": tags,
"created": str(fm.get("created", "")),
"source": src_val,
"type": fm_type,
"aliases": [str(title)],
"confidence": confidence,
"status": "active",
"last_reviewed": today,
"review_interval_days": review_interval,
}
def _yaml_quote(val):
"""如果值包含 YAML 特殊字符(如 [[ wikilink 的方括号),加双引号。"""
s = str(val)
if not s:
return '""'
if '[' in s or ']' in s or '{' in s or '}' in s or ':' in s or '#' in s:
return f'"{s}"'
return s
return pairs
def dump_frontmatter(pairs):
out = ["---"]
for k, v in pairs:
if isinstance(v, list):
out.append(f"{k}:")
for item in v:
out.append(f" - {_yaml_quote(item)}")
else:
out.append(f"{k}: {_yaml_quote(v)}")
out.append("---")
return "\n".join(out) + "\n"
"""输出 Obsidian 兼容的 frontmatter(使用 IndentedDumper 缩进序列)。"""
if isinstance(pairs, list):
from collections import OrderedDict
pairs = OrderedDict(pairs)
# 转为普通 dict(避免 OrderedDict 触发 python/object 标签)
data = dict(pairs) if not isinstance(pairs, dict) else pairs
yaml_str = yaml_dump(data)
return f"---\n{yaml_str}---\n\n"
# ---------------------------------------------------------------------------
@@ -256,9 +315,23 @@ def build_page(src_path, stem):
text = src_path.read_text(encoding="utf-8")
fm, body = parse_doc(text)
pairs = convert_frontmatter(fm, stem)
# 提取 relations(从原始 frontmatter 文本中)
m = re.match(r"^---\s*\n(.*?)\n---\s*\n?", text, re.S)
if m:
rels = _extract_related(m.group(1))
if rels:
# 去重:同 (type, target) 只保留第一个
seen = set()
unique_rels = []
for r in rels:
key = (r["type"], r["target"])
if key not in seen:
seen.add(key)
unique_rels.append(r)
pairs["relations"] = unique_rels[:5] # 限制最多 5 条
body_clean = strip_openclaw(body)
# 确保正文与 frontmatter 间有空行
return dump_frontmatter(pairs) + "\n" + body_clean.lstrip("\n")
return dump_frontmatter(pairs) + body_clean.lstrip("\n")
def _norm_full(text):
@@ -507,6 +580,27 @@ def lint():
if m and not m.group(1).strip():
empty_source.append(s)
# Phase 1 lifecycle 字段缺失检查(2026-07-01 新增)
LIFECYCLE_FIELDS = ("confidence", "status", "last_reviewed", "review_interval_days")
missing_lifecycle = []
for s in synced:
t = (DST_WIKI / f"{s}.md").read_text(encoding="utf-8")
for field in LIFECYCLE_FIELDS:
if not re.search(rf"^[ \t]*{re.escape(field)}[ \t]*:", t, re.M):
missing_lifecycle.append(f"{s} (缺 {field})")
break # 一页只报一次
# relations 字段检查(可选,但建议)
no_relations = []
for s in synced:
t = (DST_WIKI / f"{s}.md").read_text(encoding="utf-8")
m = re.match(r"^---\s*\n(.*?)\n---\s*\n?", t, re.S)
if m:
fm_text = m.group(1)
if not re.search(r"^[ \t]*relations[ \t]*:[ \t]*\n[ \t]+-", fm_text, re.M) and \
not re.search(r"^[ \t]*relations[ \t]*:[ \t]*\[", fm_text, re.M):
no_relations.append(s)
# 断链:同步页面引用的 [[x]] 在 wiki 是否存在(排除 raw 来源类、aliases、category
valid = allfiles | set(aliases_map.keys()) | set(synced)
skip = {"People", "LLM Wiki", "wikilink"}
@@ -540,10 +634,16 @@ def lint():
print(f"\n[未入 index] {len(not_in_index)} 个(概念/实体未在跨库章节)")
for s in not_in_index:
print(f" - {s}")
print(f"\n[缺 lifecycle 字段] {len(missing_lifecycle)} 个(Phase 1 必填:confidence/status/last_reviewed/review_interval_days")
for s in missing_lifecycle:
print(f" - {s}")
print(f"\n[无 relations] {len(no_relations)} 个(建议补充以激活图遍历)")
for s in no_relations:
print(f" - {s}")
issues = len(orphans) + len(empty_source) + len(not_in_index)
print(f"\n硬性问题(孤儿+空source+未入index: {issues}")
print(f"软性问题(断链,多为待创建概念/raw来源): {len(broken)}")
issues = len(orphans) + len(empty_source) + len(not_in_index) + len(missing_lifecycle)
print(f"\n硬性问题(孤儿+空source+未入index+缺lifecycle: {issues}")
print(f"软性问题(断链/无relations,多为待创建概念/raw来源): {len(broken) + len(no_relations)}")
return issues
+12 -5
View File
@@ -65,15 +65,22 @@ def validate_one(fp: Path) -> list[str]:
errors.append("Missing 'wiki' in tags")
# type must exist and be valid
# Convention: type can be scalar (e.g. 'concept') or list (e.g. ['entity', 'People', 'Author'])
# When list, the FIRST element is the canonical type; subsequent are role/subtype annotations
ptype = front.get("type")
if not ptype:
errors.append("Missing type")
elif isinstance(ptype, str) and ptype not in VALID_TYPES:
errors.append(f"Invalid type: '{ptype}' (valid: {', '.join(VALID_TYPES)})")
elif isinstance(ptype, str):
if ptype not in VALID_TYPES:
errors.append(f"Invalid type: '{ptype}' (valid: {', '.join(VALID_TYPES)})")
elif isinstance(ptype, list):
for t in ptype:
if t not in VALID_TYPES:
errors.append(f"Invalid type in list: '{t}'")
if not ptype:
errors.append("type list is empty")
else:
primary = ptype[0]
if primary not in VALID_TYPES:
errors.append(f"Invalid primary type: '{primary}' (valid: {', '.join(VALID_TYPES)})")
# Role/subtype annotations (2nd+ elements) are not validated against VALID_TYPES
# source must exist
if not front.get("source"):