289 lines
9.9 KiB
Python
289 lines
9.9 KiB
Python
#!/usr/bin/env python
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
knowledge-graph.py — Phase 3 图存储层 + 图遍历搜索
|
||
|
||
维护 wiki/ 的 typed relationships 图。存储为 JSON（tools/data/knowledge-graph.json），
|
||
支持图构建、证据链反向追溯、以及融合搜索（qmd 直接匹配 + graph 关系扩展）。
|
||
|
||
轻量级方案（Sqlite + JSON 混合）：当前仅用 JSON，无需外部数据库依赖。
|
||
|
||
用法：
|
||
python tools/scripts/knowledge-graph.py build # 从 relations 重建图
|
||
python tools/scripts/knowledge-graph.py stats # 图统计
|
||
python tools/scripts/knowledge-graph.py evidence <页面> # 反向追溯证据链（caused_by/supports）
|
||
python tools/scripts/knowledge-graph.py neighbors <页面> # 直接邻接节点
|
||
python tools/scripts/knowledge-graph.py search "<查询>" # 融合搜索（需 qmd）
|
||
python tools/scripts/knowledge-graph.py export-dot # 输出 Graphviz DOT
|
||
"""
|
||
import argparse
|
||
import json
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
from collections import deque
|
||
from pathlib import Path
|
||
|
||
try:
|
||
import yaml
|
||
except ImportError:
|
||
yaml = None
|
||
|
||
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||
WIKI = VAULT / "wiki"
|
||
GRAPH_DB = VAULT / "tools" / "data" / "knowledge-graph.json"
|
||
QMD = r'node "C:\Users\hhhh2024\AppData\Roaming\npm\node_modules\@tobilu\qmd\dist\cli\qmd.js"'
|
||
|
||
EVIDENCE_EDGES = {"caused_by", "supports", "extends"}
|
||
|
||
|
||
def read_frontmatter(fp: Path) -> dict | None:
|
||
content = fp.read_text(encoding="utf-8")
|
||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||
if not m or not yaml:
|
||
return None
|
||
try:
|
||
return yaml.safe_load(m.group(1)) or {}
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def build_graph() -> dict:
|
||
"""扫描所有 wiki 页面的 relations 字段，重建完整图"""
|
||
graph = {"nodes": [], "edges": [], "meta": {}}
|
||
seen = set()
|
||
edges_seen = set()
|
||
|
||
for fp in sorted(WIKI.glob("*.md")):
|
||
if fp.name in ("index.md", "log.md"):
|
||
continue
|
||
front = read_frontmatter(fp)
|
||
if front is None:
|
||
front = {}
|
||
name = fp.stem
|
||
if name not in seen:
|
||
graph["nodes"].append({
|
||
"id": name,
|
||
"type": front.get("type", "unknown"),
|
||
"confidence": front.get("confidence", 3),
|
||
"status": front.get("status", "active"),
|
||
})
|
||
seen.add(name)
|
||
|
||
for rel in front.get("relations", []) or []:
|
||
if not isinstance(rel, dict):
|
||
continue
|
||
target = str(rel.get("target", "")).strip("[]")
|
||
if not target or target == name:
|
||
continue
|
||
if target not in seen:
|
||
graph["nodes"].append({
|
||
"id": target,
|
||
"type": "unknown",
|
||
"confidence": 3,
|
||
"status": "unknown",
|
||
})
|
||
seen.add(target)
|
||
edge_key = (name, target, rel.get("type", "related"))
|
||
if edge_key in edges_seen:
|
||
continue
|
||
edges_seen.add(edge_key)
|
||
graph["edges"].append({
|
||
"source": name,
|
||
"target": target,
|
||
"type": rel.get("type", "related"),
|
||
"confidence": rel.get("confidence", 3),
|
||
})
|
||
|
||
graph["meta"] = {
|
||
"node_count": len(graph["nodes"]),
|
||
"edge_count": len(graph["edges"]),
|
||
"edge_types": {},
|
||
}
|
||
for e in graph["edges"]:
|
||
t = e["type"]
|
||
graph["meta"]["edge_types"][t] = graph["meta"]["edge_types"].get(t, 0) + 1
|
||
|
||
GRAPH_DB.parent.mkdir(parents=True, exist_ok=True)
|
||
GRAPH_DB.write_text(
|
||
json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8"
|
||
)
|
||
return graph
|
||
|
||
|
||
def load_graph() -> dict | None:
|
||
if not GRAPH_DB.exists():
|
||
return None
|
||
return json.loads(GRAPH_DB.read_text(encoding="utf-8"))
|
||
|
||
|
||
def adjacency(graph: dict, directed: bool = True) -> dict:
|
||
"""构建邻接表。directed=False 时加入反向边（标注 inverse_ 前缀）"""
|
||
adj: dict[str, list] = {}
|
||
for e in graph["edges"]:
|
||
adj.setdefault(e["source"], []).append((e["target"], e["type"], e.get("confidence", 3)))
|
||
if not directed:
|
||
adj.setdefault(e["target"], []).append(
|
||
(e["source"], "inverse_" + e["type"], e.get("confidence", 3))
|
||
)
|
||
return adj
|
||
|
||
|
||
def find_evidence_chain(target_page: str, max_depth: int = 5) -> list[str]:
|
||
"""从 target 反向追溯所有 caused_by/supports 关系的源头（BFS）"""
|
||
graph = load_graph()
|
||
if not graph:
|
||
return []
|
||
reverse_adj: dict[str, list] = {}
|
||
for e in graph["edges"]:
|
||
reverse_adj.setdefault(e["target"], []).append((e["source"], e["type"]))
|
||
|
||
chain: list[str] = []
|
||
visited = {target_page}
|
||
dq = deque([(target_page, 0)])
|
||
while dq:
|
||
node, depth = dq.popleft()
|
||
for src, rel in reverse_adj.get(node, []):
|
||
if rel not in EVIDENCE_EDGES:
|
||
continue
|
||
if src in visited:
|
||
continue
|
||
visited.add(src)
|
||
chain.append((" " * (depth + 1)) + f"← [[{src}]] ({rel})")
|
||
if depth + 1 < max_depth:
|
||
dq.append((src, depth + 1))
|
||
return chain
|
||
|
||
|
||
def graph_expanded_search(query: str, count: int = 15, min_conf: int = 3) -> list[dict]:
|
||
"""融合搜索：qmd BM25/向量结果 → graph 关系扩展 → 去重重排序"""
|
||
cmd = f'{QMD} vsearch "{query}" -c wiki -n 10'
|
||
result = subprocess.run(cmd, capture_output=True, text=True, shell=True, timeout=30)
|
||
direct = [ln.strip() for ln in result.stdout.splitlines() if ln.strip()]
|
||
|
||
graph = load_graph()
|
||
if not graph:
|
||
return [{"page": p, "relevance": "direct"} for p in direct[:count]]
|
||
|
||
adj = adjacency(graph, directed=True)
|
||
expanded = list(direct)
|
||
for page in direct:
|
||
for target, rel_type, conf in adj.get(page, []):
|
||
if target not in expanded and conf >= min_conf:
|
||
expanded.append(target)
|
||
|
||
return [{"page": p, "relevance": "direct" if p in direct else "graph"} for p in expanded[:count]]
|
||
|
||
|
||
def cmd_stats(args):
|
||
graph = load_graph()
|
||
if not graph:
|
||
print("Graph DB not found. Run 'build' first.")
|
||
return
|
||
meta = graph["meta"]
|
||
print(f"=== Knowledge Graph Stats ===")
|
||
print(f" Nodes: {meta['node_count']}")
|
||
print(f" Edges: {meta['edge_count']}")
|
||
print(f"\n Edge type breakdown:")
|
||
for t, c in sorted(meta["edge_types"].items(), key=lambda x: -x[1]):
|
||
print(f" {t}: {c}")
|
||
typed_nodes = sum(1 for n in graph["nodes"] if n["type"] != "unknown")
|
||
print(f"\n Typed nodes: {typed_nodes} / {meta['node_count']}")
|
||
|
||
|
||
def cmd_evidence(args):
|
||
chain = find_evidence_chain(args.page)
|
||
print(f"=== Evidence chain for [[{args.page}]] ===")
|
||
if not chain:
|
||
print(" No evidence edges (caused_by/supports/extends) found.")
|
||
return
|
||
for line in chain:
|
||
print(f" {line}")
|
||
print(f"\n Total upstream evidence: {len(chain)}")
|
||
|
||
|
||
def cmd_neighbors(args):
|
||
graph = load_graph()
|
||
if not graph:
|
||
print("Graph DB not found. Run 'build' first.")
|
||
return
|
||
found = False
|
||
for e in graph["edges"]:
|
||
if e["source"] == args.page:
|
||
print(f" --{e['type']}--> [[{e['target']}]] (conf={e.get('confidence', 3)})")
|
||
found = True
|
||
for e in graph["edges"]:
|
||
if e["target"] == args.page:
|
||
print(f" <--{e['type']}-- [[{e['source']}]] (conf={e.get('confidence', 3)})")
|
||
found = True
|
||
if not found:
|
||
print(f" [[{args.page}]] has no relations in the graph.")
|
||
|
||
|
||
def cmd_search(args):
|
||
print(f"=== Fusion search: {args.query} ===")
|
||
results = graph_expanded_search(args.query, count=args.count)
|
||
direct = [r for r in results if r["relevance"] == "direct"]
|
||
graph_hits = [r for r in results if r["relevance"] == "graph"]
|
||
if direct:
|
||
print(f"\nDirect matches ({len(direct)}):")
|
||
for r in direct:
|
||
print(f" [[{r['page']}]]")
|
||
if graph_hits:
|
||
print(f"\nGraph-expanded (via relations):")
|
||
for r in graph_hits:
|
||
print(f" [[{r['page']}]]")
|
||
print(f"\nTotal: {len(results)}")
|
||
|
||
|
||
def cmd_export_dot(args):
|
||
graph = load_graph()
|
||
if not graph:
|
||
print("Graph DB not found. Run 'build' first.")
|
||
return
|
||
print("digraph wiki {")
|
||
print(' rankdir=LR;')
|
||
for n in graph["nodes"]:
|
||
print(f' "{n["id"]}";')
|
||
for e in graph["edges"]:
|
||
print(f' "{e["source"]}" -> "{e["target"]}" [label="{e["type"]}"];')
|
||
print("}")
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Phase 3 knowledge graph storage + traversal")
|
||
sub = parser.add_subparsers(dest="command", required=True)
|
||
|
||
sub.add_parser("build", help="Rebuild graph from relations")
|
||
sub.add_parser("stats", help="Print graph statistics")
|
||
p_ev = sub.add_parser("evidence", help="Trace evidence chain (caused_by/supports)")
|
||
p_ev.add_argument("page", help="Target page name")
|
||
p_nb = sub.add_parser("neighbors", help="Direct neighbors of a page")
|
||
p_nb.add_argument("page", help="Page name")
|
||
p_se = sub.add_parser("search", help="Fusion search (qmd + graph expansion)")
|
||
p_se.add_argument("query", help="Search query")
|
||
p_se.add_argument("--count", type=int, default=15)
|
||
sub.add_parser("export-dot", help="Export Graphviz DOT format")
|
||
|
||
args = parser.parse_args()
|
||
|
||
if args.command == "build":
|
||
graph = build_graph()
|
||
m = graph["meta"]
|
||
print(f"Graph built: {m['node_count']} nodes, {m['edge_count']} edges")
|
||
print(f"Saved to {GRAPH_DB}")
|
||
elif args.command == "stats":
|
||
cmd_stats(args)
|
||
elif args.command == "evidence":
|
||
cmd_evidence(args)
|
||
elif args.command == "neighbors":
|
||
cmd_neighbors(args)
|
||
elif args.command == "search":
|
||
cmd_search(args)
|
||
elif args.command == "export-dot":
|
||
cmd_export_dot(args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|