feat(vault): Phase 3 tools + batch citation/relation enrichment
This commit is contained in:
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
knowledge-graph.py — Phase 3 图存储层 + 图遍历搜索
|
||||
|
||||
维护 wiki/ 的 typed relationships 图。存储为 JSON（tools/data/knowledge-graph.json），
|
||||
支持图构建、证据链反向追溯、以及融合搜索（qmd 直接匹配 + graph 关系扩展）。
|
||||
|
||||
轻量级方案（Sqlite + JSON 混合）：当前仅用 JSON，无需外部数据库依赖。
|
||||
|
||||
用法：
|
||||
python tools/scripts/knowledge-graph.py build # 从 relations 重建图
|
||||
python tools/scripts/knowledge-graph.py stats # 图统计
|
||||
python tools/scripts/knowledge-graph.py evidence <页面> # 反向追溯证据链（caused_by/supports）
|
||||
python tools/scripts/knowledge-graph.py neighbors <页面> # 直接邻接节点
|
||||
python tools/scripts/knowledge-graph.py search "<查询>" # 融合搜索（需 qmd）
|
||||
python tools/scripts/knowledge-graph.py export-dot # 输出 Graphviz DOT
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||||
WIKI = VAULT / "wiki"
|
||||
GRAPH_DB = VAULT / "tools" / "data" / "knowledge-graph.json"
|
||||
QMD = r'node "C:\Users\hhhh2024\AppData\Roaming\npm\node_modules\@tobilu\qmd\dist\cli\qmd.js"'
|
||||
|
||||
EVIDENCE_EDGES = {"caused_by", "supports", "extends"}
|
||||
|
||||
|
||||
def read_frontmatter(fp: Path) -> dict | None:
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m or not yaml:
|
||||
return None
|
||||
try:
|
||||
return yaml.safe_load(m.group(1)) or {}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def build_graph() -> dict:
|
||||
"""扫描所有 wiki 页面的 relations 字段，重建完整图"""
|
||||
graph = {"nodes": [], "edges": [], "meta": {}}
|
||||
seen = set()
|
||||
edges_seen = set()
|
||||
|
||||
for fp in sorted(WIKI.glob("*.md")):
|
||||
if fp.name in ("index.md", "log.md"):
|
||||
continue
|
||||
front = read_frontmatter(fp)
|
||||
if front is None:
|
||||
front = {}
|
||||
name = fp.stem
|
||||
if name not in seen:
|
||||
graph["nodes"].append({
|
||||
"id": name,
|
||||
"type": front.get("type", "unknown"),
|
||||
"confidence": front.get("confidence", 3),
|
||||
"status": front.get("status", "active"),
|
||||
})
|
||||
seen.add(name)
|
||||
|
||||
for rel in front.get("relations", []) or []:
|
||||
if not isinstance(rel, dict):
|
||||
continue
|
||||
target = str(rel.get("target", "")).strip("[]")
|
||||
if not target or target == name:
|
||||
continue
|
||||
if target not in seen:
|
||||
graph["nodes"].append({
|
||||
"id": target,
|
||||
"type": "unknown",
|
||||
"confidence": 3,
|
||||
"status": "unknown",
|
||||
})
|
||||
seen.add(target)
|
||||
edge_key = (name, target, rel.get("type", "related"))
|
||||
if edge_key in edges_seen:
|
||||
continue
|
||||
edges_seen.add(edge_key)
|
||||
graph["edges"].append({
|
||||
"source": name,
|
||||
"target": target,
|
||||
"type": rel.get("type", "related"),
|
||||
"confidence": rel.get("confidence", 3),
|
||||
})
|
||||
|
||||
graph["meta"] = {
|
||||
"node_count": len(graph["nodes"]),
|
||||
"edge_count": len(graph["edges"]),
|
||||
"edge_types": {},
|
||||
}
|
||||
for e in graph["edges"]:
|
||||
t = e["type"]
|
||||
graph["meta"]["edge_types"][t] = graph["meta"]["edge_types"].get(t, 0) + 1
|
||||
|
||||
GRAPH_DB.parent.mkdir(parents=True, exist_ok=True)
|
||||
GRAPH_DB.write_text(
|
||||
json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
return graph
|
||||
|
||||
|
||||
def load_graph() -> dict | None:
|
||||
if not GRAPH_DB.exists():
|
||||
return None
|
||||
return json.loads(GRAPH_DB.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def adjacency(graph: dict, directed: bool = True) -> dict:
|
||||
"""构建邻接表。directed=False 时加入反向边（标注 inverse_ 前缀）"""
|
||||
adj: dict[str, list] = {}
|
||||
for e in graph["edges"]:
|
||||
adj.setdefault(e["source"], []).append((e["target"], e["type"], e.get("confidence", 3)))
|
||||
if not directed:
|
||||
adj.setdefault(e["target"], []).append(
|
||||
(e["source"], "inverse_" + e["type"], e.get("confidence", 3))
|
||||
)
|
||||
return adj
|
||||
|
||||
|
||||
def find_evidence_chain(target_page: str, max_depth: int = 5) -> list[str]:
|
||||
"""从 target 反向追溯所有 caused_by/supports 关系的源头（BFS）"""
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
return []
|
||||
reverse_adj: dict[str, list] = {}
|
||||
for e in graph["edges"]:
|
||||
reverse_adj.setdefault(e["target"], []).append((e["source"], e["type"]))
|
||||
|
||||
chain: list[str] = []
|
||||
visited = {target_page}
|
||||
dq = deque([(target_page, 0)])
|
||||
while dq:
|
||||
node, depth = dq.popleft()
|
||||
for src, rel in reverse_adj.get(node, []):
|
||||
if rel not in EVIDENCE_EDGES:
|
||||
continue
|
||||
if src in visited:
|
||||
continue
|
||||
visited.add(src)
|
||||
chain.append((" " * (depth + 1)) + f"← [[{src}]] ({rel})")
|
||||
if depth + 1 < max_depth:
|
||||
dq.append((src, depth + 1))
|
||||
return chain
|
||||
|
||||
|
||||
def graph_expanded_search(query: str, count: int = 15, min_conf: int = 3) -> list[dict]:
|
||||
"""融合搜索：qmd BM25/向量结果 → graph 关系扩展 → 去重重排序"""
|
||||
cmd = f'{QMD} vsearch "{query}" -c wiki -n 10'
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, shell=True, timeout=30)
|
||||
direct = [ln.strip() for ln in result.stdout.splitlines() if ln.strip()]
|
||||
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
return [{"page": p, "relevance": "direct"} for p in direct[:count]]
|
||||
|
||||
adj = adjacency(graph, directed=True)
|
||||
expanded = list(direct)
|
||||
for page in direct:
|
||||
for target, rel_type, conf in adj.get(page, []):
|
||||
if target not in expanded and conf >= min_conf:
|
||||
expanded.append(target)
|
||||
|
||||
return [{"page": p, "relevance": "direct" if p in direct else "graph"} for p in expanded[:count]]
|
||||
|
||||
|
||||
def cmd_stats(args):
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
print("Graph DB not found. Run 'build' first.")
|
||||
return
|
||||
meta = graph["meta"]
|
||||
print(f"=== Knowledge Graph Stats ===")
|
||||
print(f" Nodes: {meta['node_count']}")
|
||||
print(f" Edges: {meta['edge_count']}")
|
||||
print(f"\n Edge type breakdown:")
|
||||
for t, c in sorted(meta["edge_types"].items(), key=lambda x: -x[1]):
|
||||
print(f" {t}: {c}")
|
||||
typed_nodes = sum(1 for n in graph["nodes"] if n["type"] != "unknown")
|
||||
print(f"\n Typed nodes: {typed_nodes} / {meta['node_count']}")
|
||||
|
||||
|
||||
def cmd_evidence(args):
|
||||
chain = find_evidence_chain(args.page)
|
||||
print(f"=== Evidence chain for [[{args.page}]] ===")
|
||||
if not chain:
|
||||
print(" No evidence edges (caused_by/supports/extends) found.")
|
||||
return
|
||||
for line in chain:
|
||||
print(f" {line}")
|
||||
print(f"\n Total upstream evidence: {len(chain)}")
|
||||
|
||||
|
||||
def cmd_neighbors(args):
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
print("Graph DB not found. Run 'build' first.")
|
||||
return
|
||||
found = False
|
||||
for e in graph["edges"]:
|
||||
if e["source"] == args.page:
|
||||
print(f" --{e['type']}--> [[{e['target']}]] (conf={e.get('confidence', 3)})")
|
||||
found = True
|
||||
for e in graph["edges"]:
|
||||
if e["target"] == args.page:
|
||||
print(f" <--{e['type']}-- [[{e['source']}]] (conf={e.get('confidence', 3)})")
|
||||
found = True
|
||||
if not found:
|
||||
print(f" [[{args.page}]] has no relations in the graph.")
|
||||
|
||||
|
||||
def cmd_search(args):
|
||||
print(f"=== Fusion search: {args.query} ===")
|
||||
results = graph_expanded_search(args.query, count=args.count)
|
||||
direct = [r for r in results if r["relevance"] == "direct"]
|
||||
graph_hits = [r for r in results if r["relevance"] == "graph"]
|
||||
if direct:
|
||||
print(f"\nDirect matches ({len(direct)}):")
|
||||
for r in direct:
|
||||
print(f" [[{r['page']}]]")
|
||||
if graph_hits:
|
||||
print(f"\nGraph-expanded (via relations):")
|
||||
for r in graph_hits:
|
||||
print(f" [[{r['page']}]]")
|
||||
print(f"\nTotal: {len(results)}")
|
||||
|
||||
|
||||
def cmd_export_dot(args):
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
print("Graph DB not found. Run 'build' first.")
|
||||
return
|
||||
print("digraph wiki {")
|
||||
print(' rankdir=LR;')
|
||||
for n in graph["nodes"]:
|
||||
print(f' "{n["id"]}";')
|
||||
for e in graph["edges"]:
|
||||
print(f' "{e["source"]}" -> "{e["target"]}" [label="{e["type"]}"];')
|
||||
print("}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Phase 3 knowledge graph storage + traversal")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
sub.add_parser("build", help="Rebuild graph from relations")
|
||||
sub.add_parser("stats", help="Print graph statistics")
|
||||
p_ev = sub.add_parser("evidence", help="Trace evidence chain (caused_by/supports)")
|
||||
p_ev.add_argument("page", help="Target page name")
|
||||
p_nb = sub.add_parser("neighbors", help="Direct neighbors of a page")
|
||||
p_nb.add_argument("page", help="Page name")
|
||||
p_se = sub.add_parser("search", help="Fusion search (qmd + graph expansion)")
|
||||
p_se.add_argument("query", help="Search query")
|
||||
p_se.add_argument("--count", type=int, default=15)
|
||||
sub.add_parser("export-dot", help="Export Graphviz DOT format")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "build":
|
||||
graph = build_graph()
|
||||
m = graph["meta"]
|
||||
print(f"Graph built: {m['node_count']} nodes, {m['edge_count']} edges")
|
||||
print(f"Saved to {GRAPH_DB}")
|
||||
elif args.command == "stats":
|
||||
cmd_stats(args)
|
||||
elif args.command == "evidence":
|
||||
cmd_evidence(args)
|
||||
elif args.command == "neighbors":
|
||||
cmd_neighbors(args)
|
||||
elif args.command == "search":
|
||||
cmd_search(args)
|
||||
elif args.command == "export-dot":
|
||||
cmd_export_dot(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user