feat(vault): Phase 3 tools + batch citation/relation enrichment
This commit is contained in:
@@ -33,6 +33,10 @@
|
||||
"价值驱动": {
|
||||
"summary": "郭朝晖:以终为始在数字化场景的落地(手段≠目的/投入产出比/口号陷阱)"
|
||||
},
|
||||
"军事系统工程": {
|
||||
"summary": "来源:钱学森+王寿云+柴本良 1979 年 7 月 24 日解放军总部机关领导同志学习会讲演稿 钱学森主讲,王寿云+柴本良合著,许国志协助",
|
||||
"type": "concept"
|
||||
},
|
||||
"十大类生产性服务业": {
|
||||
"summary": "来源:黄奇帆 2026 年演讲(笔记侠 ID: Notesman)",
|
||||
"type": "concept"
|
||||
@@ -60,6 +64,14 @@
|
||||
"山西12个原始古村落": {
|
||||
"summary": "山西古村落考察实践记录"
|
||||
},
|
||||
"工程系统工程": {
|
||||
"summary": "来源:钱学森《论科学技术研究的组织管理与科研系统工程》(1980)第二节",
|
||||
"type": "concept"
|
||||
},
|
||||
"情报信息系统工程": {
|
||||
"summary": "来源:钱学森《论科学技术研究的组织管理与科研系统工程》(1980)第四节",
|
||||
"type": "concept"
|
||||
},
|
||||
"教学过程核心模型": {
|
||||
"summary": "专业教学过程的核心模型(课程设计与教学环节结构化)"
|
||||
},
|
||||
@@ -69,9 +81,21 @@
|
||||
"教育研究核验提示词库": {
|
||||
"summary": "核验教育研究文献的 AI 智能体提示词模板,配套教学过程核心模型佐证研究"
|
||||
},
|
||||
"教育系统工程": {
|
||||
"summary": "来源:钱学森《论科学技术研究的组织管理与科研系统工程》(1980)第二节",
|
||||
"type": "concept"
|
||||
},
|
||||
"新范式": {
|
||||
"summary": "陆奇2024元框架:三位一体/三拐点/模型=知识/边际成本转向固定成本"
|
||||
},
|
||||
"柴本良": {
|
||||
"summary": "我们是其中少数几篇论文的合作者。",
|
||||
"type": "entity"
|
||||
},
|
||||
"标准系统工程": {
|
||||
"summary": "来源:钱学森《论科学技术研究的组织管理与科研系统工程》(1980)第四节",
|
||||
"type": "concept"
|
||||
},
|
||||
"涌现": {
|
||||
"summary": "大量简单个体在临界条件下自发诞生不可还原的全新宏观秩序"
|
||||
},
|
||||
@@ -86,10 +110,22 @@
|
||||
"summary": "来源:黄奇帆 2026 年演讲",
|
||||
"type": "concept"
|
||||
},
|
||||
"科学学": {
|
||||
"summary": "来源:钱学森《论科学技术研究的组织管理与科研系统工程》(1980)第一节",
|
||||
"type": "concept"
|
||||
},
|
||||
"科技创新五阶段论": {
|
||||
"summary": "来源:黄奇帆 2026 年演讲",
|
||||
"type": "concept"
|
||||
},
|
||||
"科研社会化": {
|
||||
"summary": "来源:钱学森《论科学技术研究的组织管理与科研系统工程》(1980)第三节",
|
||||
"type": "concept"
|
||||
},
|
||||
"科研系统工程": {
|
||||
"summary": "来源:钱学森《论科学技术研究的组织管理与科研系统工程》(1980)",
|
||||
"type": "concept"
|
||||
},
|
||||
"系统工程工程实践": {
|
||||
"summary": "来源:钱学森 + 王寿云 1980s CCTV 系统工程讲座第一讲(第二节)",
|
||||
"type": "concept"
|
||||
@@ -101,6 +137,14 @@
|
||||
"老红军长寿现象的精神分析": {
|
||||
"summary": "革命信念与精神状态对生命力的深层影响"
|
||||
},
|
||||
"计量系统工程": {
|
||||
"summary": "来源:钱学森《论科学技术研究的组织管理与科研系统工程》(1980)第四节",
|
||||
"type": "concept"
|
||||
},
|
||||
"许国志": {
|
||||
"summary": "1955 年钱学森同志在和我们中的许国志同志讨论问题时,表示了把运筹学和社会主义计划经济结合起来的想法。",
|
||||
"type": "entity"
|
||||
},
|
||||
"郭朝晖": {
|
||||
"summary": "技术创新管理专家,曾任职宝钢,著《我的科研经历》"
|
||||
},
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -874,6 +874,8 @@ def auto_suggest_low_confidence():
|
||||
| `.githooks/pre-commit` | P2 | Git hook |
|
||||
| `.githooks/post-merge` | P2 | Git hook |
|
||||
| `tools/data/knowledge-graph.json` | P3 | 图数据库文件 |
|
||||
| `tools/scripts/populate-relations.py` | P1+ | 批量 relations 推断(source-grouping + cross-link) |
|
||||
| `tools/scripts/enrich-citations.py` | P0+ | 批量溯源注入(citation seed + 来源节 + 存根标记) |
|
||||
|
||||
### 修改文件
|
||||
|
||||
@@ -882,6 +884,11 @@ def auto_suggest_low_confidence():
|
||||
| `AGENTS.md` | P0/P1 | 修复冗余 + 新增 v2 特性定义 + 更新模板 |
|
||||
| `wiki/index.md` | P0 | 更新指标数字 |
|
||||
| `wiki/log.md` | P0/P1/P2 | 追加每次操作日志 |
|
||||
| `Templates/Wiki Page Template.md` | P2+ | 纳入 Phase 1 lifecycle 字段 |
|
||||
| `Templates/People Template.md` | P2+ | 纳入 Phase 1 lifecycle 字段 |
|
||||
| `Templates/Author Template.md` | P2+ | 纳入 Phase 1 lifecycle 字段 |
|
||||
| `wiki/*.md`(375 页) | P1 | 批量填充 relations 字段 |
|
||||
| `wiki/*.md`(284 页) | P0 | 注入 `[raw:source:line]` 种子引用 |
|
||||
|
||||
### 依赖安装
|
||||
|
||||
@@ -890,3 +897,17 @@ pip install pyyaml # 已安装,确认版本
|
||||
# Phase 3 可能需要:
|
||||
pip install networkx # 轻量图分析(替代 Neo4j)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 执行状态总览(2026-07-01)
|
||||
|
||||
| 阶段 | 状态 | 关键指标 |
|
||||
|------|------|----------|
|
||||
| **Phase 0** — 即时修复 | ✅ 完成 | AGENTS.md 冗余清除;行号覆盖率 56%→100%;index.md 指标修正 |
|
||||
| **Phase 1** — 短期升级 | ✅ 完成 | relations 字段定义;manage-relations/graph-search/check-staleness/detect-conflicts 脚本就位 |
|
||||
| **Phase 2** — 中期自动化 | ✅ 完成 | pre-commit/post-merge hooks;validate-frontmatter/weekly-lint/promote-knowledge 脚本;结晶机制+知识提级入 AGENTS.md |
|
||||
| **Phase 3** — 长期进阶 | ✅ 完成 | knowledge-graph.py(505节点/646边);review-pages.py(LLM+静态双模式);graph-search.py --use-graph |
|
||||
| **P0 普及溯源** | ✅ 完成 | enrich-citations.py:引用覆盖 79→405 页(15.6%→80.2%) |
|
||||
| **P1 激活关系图** | ✅ 完成 | populate-relations.py:375 页 +644 条 relations,图 2→646 边 |
|
||||
| **P2 落地生命周期** | ✅ 完成 | 3 个模板更新,Phase 1 字段入模板规范 |
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
enrich-citations.py — Phase 3 P0: 批量为零引用页面补溯源链接
|
||||
|
||||
策略:
|
||||
1. **来源节补全**:无 `## 来源` 节的页面,自动添加,包含指向 raw source 的 wikilink
|
||||
2. **首段引用注入**:若 raw source 文件存在,提取页面标题关键词,
|
||||
在 raw 文件中定位匹配行,注入首条 `[raw:source:行号]` 作为溯源种子
|
||||
3. **存根页标记**:body < 50 字的页面添加 `> [!warning] 存根页面` callout
|
||||
|
||||
安全措施:
|
||||
- 幂等:已有 `## 来源` 节或 `[raw:` 引用则跳过
|
||||
- --dry-run / --apply
|
||||
- 仅处理 wiki/*.md(排除 index/log/archive)
|
||||
|
||||
用法:
|
||||
python tools/scripts/enrich-citations.py --dry-run
|
||||
python tools/scripts/enrich-citations.py --apply
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||||
WIKI = VAULT / "wiki"
|
||||
RAW = VAULT / "raw"
|
||||
EXCLUDE = {"index.md", "log.md"}
|
||||
|
||||
|
||||
def read_frontmatter(fp: Path) -> tuple[dict, str, str]:
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---\n?(.*)", content, re.DOTALL)
|
||||
if not m:
|
||||
return {}, content, ""
|
||||
fm_text = m.group(1)
|
||||
body = m.group(2)
|
||||
if yaml:
|
||||
try:
|
||||
front = yaml.safe_load(fm_text) or {}
|
||||
except Exception:
|
||||
front = {}
|
||||
else:
|
||||
front = {}
|
||||
return front, body, fm_text
|
||||
|
||||
|
||||
def get_source_filename(front: dict) -> str:
|
||||
"""Extract raw filename from source field"""
|
||||
source = front.get("source", "")
|
||||
if not source:
|
||||
return ""
|
||||
s = str(source).strip().strip('"').strip("'")
|
||||
s = re.sub(r"\[\[(?:raw/)?(.+?)(?:\|.*)?\]\]", r"\1", s)
|
||||
s = re.sub(r"\.md$", "", s)
|
||||
return s
|
||||
|
||||
|
||||
def find_raw_file(source_name: str) -> Path | None:
|
||||
"""Try to locate the raw file"""
|
||||
candidates = [
|
||||
RAW / f"{source_name}.md",
|
||||
RAW / source_name,
|
||||
]
|
||||
# Also search subdirectories
|
||||
for pattern in [f"**/{source_name}.md", f"**/{source_name}/index.md"]:
|
||||
matches = list(RAW.glob(pattern))
|
||||
if matches:
|
||||
candidates.append(matches[0])
|
||||
for c in candidates:
|
||||
if c.exists() and c.is_file():
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def find_keyword_line(raw_path: Path, page_name: str) -> str:
|
||||
"""Find a line in raw file matching the page name or key keyword"""
|
||||
try:
|
||||
raw_text = raw_path.read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
return ""
|
||||
lines = raw_text.splitlines()
|
||||
|
||||
# Strategy: search for page name, then aliases
|
||||
keywords = [page_name]
|
||||
# Also try shorter keywords
|
||||
if len(page_name) > 4:
|
||||
keywords.append(page_name[:4])
|
||||
|
||||
for kw in keywords:
|
||||
for i, line in enumerate(lines, 1):
|
||||
if kw in line and len(line.strip()) > 5:
|
||||
return f"{i}"
|
||||
return ""
|
||||
|
||||
|
||||
def has_source_section(body: str) -> bool:
|
||||
return bool(re.search(r"^##\s*来源", body, re.MULTILINE))
|
||||
|
||||
|
||||
def has_raw_citations(body: str) -> bool:
|
||||
return "[raw:" in body
|
||||
|
||||
|
||||
def is_stub(body: str) -> bool:
|
||||
clean = re.sub(r"!\[\[.*?\]\]", "", body)
|
||||
clean = re.sub(r"```.*?```", "", clean, flags=re.DOTALL)
|
||||
clean = re.sub(r"\|.*?\|", "", clean)
|
||||
clean = re.sub(r"\s+", "", clean)
|
||||
return len(clean) < 150
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Enrich wiki pages with source citations")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Preview without writing")
|
||||
parser.add_argument("--apply", action="store_true", help="Write changes to files")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.dry_run and not args.apply:
|
||||
args.dry_run = True
|
||||
|
||||
stats = {"source_section": 0, "citation_seed": 0, "stub_flag": 0, "skip_has_citation": 0, "skip_no_source": 0}
|
||||
|
||||
for fp in sorted(WIKI.glob("*.md")):
|
||||
if fp.name in EXCLUDE:
|
||||
continue
|
||||
front, body, fm_text = read_frontmatter(fp)
|
||||
page_name = fp.stem
|
||||
|
||||
if has_raw_citations(body):
|
||||
stats["skip_has_citation"] += 1
|
||||
continue
|
||||
|
||||
source_name = get_source_filename(front)
|
||||
raw_path = find_raw_file(source_name) if source_name else None
|
||||
|
||||
modified = False
|
||||
additions = []
|
||||
|
||||
# 1. Add citation seed if raw file exists
|
||||
if raw_path:
|
||||
line_num = find_keyword_line(raw_path, page_name)
|
||||
if line_num:
|
||||
# Find first paragraph in body and append citation
|
||||
first_para_match = re.search(r"^(.+?)(?:\n\n|\n#)", body, re.DOTALL)
|
||||
if first_para_match:
|
||||
insert_point = first_para_match.end()
|
||||
cite = f"[raw:{source_name}:{line_num}]"
|
||||
if cite not in body:
|
||||
body = body[:insert_point] + f" {cite}" + body[insert_point:]
|
||||
modified = True
|
||||
stats["citation_seed"] += 1
|
||||
additions.append(f"citation seed [raw:{source_name}:{line_num}]")
|
||||
|
||||
# 2. Add/fix source section
|
||||
if not has_source_section(body):
|
||||
source_link = front.get("source", "")
|
||||
if source_link:
|
||||
source_section = f"\n\n## 来源\n\n> **溯源规则**:所有数字/百分比/具体结论必须标注 `[raw:{{文件名}}:{{行号}}]` 格式。\n\n- {source_link}"
|
||||
body = body.rstrip() + source_section
|
||||
modified = True
|
||||
stats["source_section"] += 1
|
||||
additions.append("source section")
|
||||
|
||||
# 3. Flag stub pages
|
||||
if is_stub(body) and not args.dry_run:
|
||||
if "> [!warning] 存根页面" not in body:
|
||||
stub_callout = "\n\n> [!warning] 存根页面\n> 本页内容不足,待扩充。参考同类黄金标准页面(如 [[DSpark]]、[[何长工]])补充来源引用和正文内容。\n"
|
||||
body = body.rstrip() + stub_callout
|
||||
modified = True
|
||||
stats["stub_flag"] += 1
|
||||
additions.append("stub callout")
|
||||
|
||||
if not modified:
|
||||
if not source_name:
|
||||
stats["skip_no_source"] += 1
|
||||
continue
|
||||
|
||||
if args.dry_run:
|
||||
print(f" [[{page_name}]]: {', '.join(additions)}")
|
||||
continue
|
||||
|
||||
if args.apply:
|
||||
# Reconstruct file
|
||||
if yaml:
|
||||
new_yaml = yaml.dump(front, allow_unicode=True, default_flow_style=False, sort_keys=False)
|
||||
else:
|
||||
new_yaml = fm_text
|
||||
fp.write_text(f"---\n{new_yaml}---\n{body}", encoding="utf-8")
|
||||
|
||||
mode = "DRY RUN" if args.dry_run else "APPLIED"
|
||||
print(f"\n=== {mode} ===")
|
||||
print(f" Citation seeds added: {stats['citation_seed']}")
|
||||
print(f" Source sections added: {stats['source_section']}")
|
||||
print(f" Stub flags added: {stats['stub_flag']}")
|
||||
print(f" Skipped (has citation): {stats['skip_has_citation']}")
|
||||
print(f" Skipped (no source): {stats['skip_no_source']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
fix-yaml-frontmatter.py — 批量修复 wiki frontmatter YAML 错误
|
||||
|
||||
修复类型:
|
||||
1. Windows 反斜杠路径 → 正斜杠(source 字段 \ → /)
|
||||
2. 缺空格的 tag 列表项(-tag → - tag)
|
||||
3. 断裂的 type 字段(type: place + 孤儿 - "Region" → 合并为列表)
|
||||
4. relations 字段格式标准化(Obsidian 兼容:缩进列表 + 双引号)
|
||||
5. frontmatter 后空行规范化
|
||||
|
||||
用法:
|
||||
python tools/scripts/fix-yaml-frontmatter.py --dry-run
|
||||
python tools/scripts/fix-yaml-frontmatter.py --apply
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("ERROR: pyyaml required"); sys.exit(1)
|
||||
|
||||
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||||
WIKI = VAULT / "wiki"
|
||||
EXCLUDE = {"index.md", "log.md"}
|
||||
|
||||
|
||||
class IndentedDumper(yaml.Dumper):
|
||||
"""Custom dumper that indents sequences under their key (Obsidian-friendly)"""
|
||||
def increase_indent(self, flow=False, indentless=False):
|
||||
return super().increase_indent(flow, False)
|
||||
|
||||
|
||||
def yaml_dump(front: dict) -> str:
|
||||
"""Dump frontmatter with Obsidian-compatible formatting"""
|
||||
return yaml.dump(
|
||||
front,
|
||||
Dumper=IndentedDumper,
|
||||
allow_unicode=True,
|
||||
default_flow_style=False,
|
||||
sort_keys=False,
|
||||
width=1000, # prevent line wrapping
|
||||
)
|
||||
|
||||
|
||||
def split_frontmatter(content: str):
|
||||
"""Split into (frontmatter_text, body) preserving original formatting"""
|
||||
m = re.match(r"^(---\n)(.*?)(\n---\n?)(.*)", content, re.DOTALL)
|
||||
if not m:
|
||||
return None, content, None
|
||||
return m.group(2), m.group(4), m
|
||||
|
||||
|
||||
def fix_backslashes(fm_text: str) -> tuple[str, bool]:
|
||||
"""Fix Windows backslash paths in source field"""
|
||||
changed = False
|
||||
def replacer(m):
|
||||
nonlocal changed
|
||||
val = m.group(0)
|
||||
if '\\' in val:
|
||||
changed = True
|
||||
return val.replace('\\', '/')
|
||||
return val
|
||||
result = re.sub(r'source:\s*".*?"', replacer, fm_text)
|
||||
return result, changed
|
||||
|
||||
|
||||
def fix_dash_nospace(fm_text: str) -> tuple[str, bool]:
|
||||
"""Fix list items missing space after dash: -tag → - tag"""
|
||||
changed = False
|
||||
lines = fm_text.split('\n')
|
||||
for i, line in enumerate(lines):
|
||||
m = re.match(r'^(\s+)-(\S)', line)
|
||||
if m:
|
||||
lines[i] = m.group(1) + '- ' + m.group(2) + line[m.end():]
|
||||
changed = True
|
||||
return '\n'.join(lines), changed
|
||||
|
||||
|
||||
def fix_broken_type(fm_text: str) -> tuple[str, bool]:
|
||||
"""Fix orphaned type values: type: place\n- "Region" → type list"""
|
||||
changed = False
|
||||
lines = fm_text.split('\n')
|
||||
result = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
type_match = re.match(r'^(\s*)type:\s*(\S+)', line)
|
||||
if type_match and i + 1 < len(lines):
|
||||
next_stripped = lines[i + 1].strip()
|
||||
if next_stripped.startswith('- ') or next_stripped.startswith('-"'):
|
||||
indent = type_match.group(1)
|
||||
main_type = type_match.group(2)
|
||||
extra = re.match(r'^-\s*"?(.+?)"?$', next_stripped)
|
||||
extra_val = extra.group(1) if extra else next_stripped.lstrip('- ')
|
||||
result.append(f'{indent}type:')
|
||||
result.append(f'{indent} - "{main_type}"')
|
||||
result.append(f'{indent} - "{extra_val}"')
|
||||
changed = True
|
||||
i += 2
|
||||
continue
|
||||
result.append(line)
|
||||
i += 1
|
||||
return '\n'.join(result), changed
|
||||
|
||||
|
||||
def process_file(fp: Path, apply: bool) -> dict:
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^(---\n)(.*?)(\n---\n?)(.*)", content, re.DOTALL)
|
||||
if not m:
|
||||
return {"file": fp.name, "skipped": "no frontmatter"}
|
||||
|
||||
fm_text = m.group(2)
|
||||
body = m.group(4)
|
||||
fixes = []
|
||||
|
||||
# Phase 1: Regex fixes for known error patterns
|
||||
fm_text, changed = fix_backslashes(fm_text)
|
||||
if changed:
|
||||
fixes.append("backslash")
|
||||
|
||||
fm_text, changed = fix_dash_nospace(fm_text)
|
||||
if changed:
|
||||
fixes.append("dash_nospace")
|
||||
|
||||
fm_text, changed = fix_broken_type(fm_text)
|
||||
if changed:
|
||||
fixes.append("broken_type")
|
||||
|
||||
# Phase 2: Parse → Re-dump with Obsidian-compatible formatting
|
||||
try:
|
||||
front = yaml.safe_load(fm_text)
|
||||
except Exception as e:
|
||||
return {"file": fp.name, "error": f"YAML still invalid after regex fixes: {e}"}
|
||||
|
||||
if not isinstance(front, dict):
|
||||
return {"file": fp.name, "skipped": "frontmatter not a dict"}
|
||||
|
||||
# Re-dump with IndentedDumper (proper sequence indentation + double quotes)
|
||||
new_fm = yaml_dump(front)
|
||||
|
||||
# Check if re-dumping changed anything (format normalization)
|
||||
if new_fm.rstrip() != fm_text.rstrip():
|
||||
fixes.append("format_normalize")
|
||||
|
||||
# Validate the re-dumped YAML
|
||||
try:
|
||||
yaml.safe_load(new_fm)
|
||||
except Exception as e:
|
||||
return {"file": fp.name, "error": f"Re-dumped YAML invalid: {e}"}
|
||||
|
||||
if not fixes:
|
||||
return {"file": fp.name, "skipped": "no fixes needed"}
|
||||
|
||||
# Reconstruct file
|
||||
new_content = f"---\n{new_fm}---\n\n{body.lstrip()}"
|
||||
|
||||
if apply:
|
||||
fp.write_text(new_content, encoding="utf-8")
|
||||
|
||||
return {"file": fp.name, "fixes": fixes}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Fix YAML frontmatter errors in wiki")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Preview without writing")
|
||||
parser.add_argument("--apply", action="store_true", help="Write fixes")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.dry_run and not args.apply:
|
||||
args.dry_run = True
|
||||
|
||||
mode = "DRY RUN" if args.dry_run else "APPLYING"
|
||||
print(f"=== {mode} ===\n")
|
||||
|
||||
stats = {}
|
||||
for fp in sorted(WIKI.glob("*.md")):
|
||||
if fp.name in EXCLUDE:
|
||||
continue
|
||||
result = process_file(fp, args.apply)
|
||||
if "fixes" in result:
|
||||
print(f" {result['file']}: {', '.join(result['fixes'])}")
|
||||
for f in result["fixes"]:
|
||||
stats[f] = stats.get(f, 0) + 1
|
||||
elif "error" in result:
|
||||
print(f" ERROR: {result['file']}: {result['error']}")
|
||||
|
||||
print(f"\n=== Summary ===")
|
||||
for fix, count in sorted(stats.items()):
|
||||
print(f" {fix}: {count} files")
|
||||
print(f" Total files fixed: {sum(stats.values())}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -5,14 +5,17 @@ graph-search.py — 图感知搜索
|
||||
|
||||
流程:
|
||||
1. qmd vsearch 获取语义匹配结果(Top 10)
|
||||
2. 对每个结果,读取 relations 字段 → 获取相邻节点
|
||||
2. 对每个结果,扩展相邻节点:
|
||||
- 默认:读取各页面 relations 字段(实时,慢但零状态)
|
||||
- --use-graph:读取预构建的 knowledge-graph.json(快,需先 build)
|
||||
3. 去重后返回(直接关联 + 关系扩展)
|
||||
|
||||
用法:
|
||||
python tools/scripts/graph-search.py "查询词" [--count 15] [--depth 1]
|
||||
python tools/scripts/graph-search.py "查询词" [--count 15] [--depth 1] [--use-graph]
|
||||
|
||||
依赖:
|
||||
- qmd 已安装(通过 node)
|
||||
- --use-graph 需先运行 knowledge-graph.py build
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
@@ -28,6 +31,7 @@ except ImportError:
|
||||
|
||||
QMD = r'node "C:\Users\hhhh2024\AppData\Roaming\npm\node_modules\@tobilu\qmd\dist\cli\qmd.js"'
|
||||
WIKI = Path(r"D:\Applications\app\kepano-obsidian-main\wiki")
|
||||
GRAPH_DB = Path(r"D:\Applications\app\kepano-obsidian-main\tools\data\knowledge-graph.json")
|
||||
|
||||
|
||||
def qmd_search(query: str, count: int = 10) -> list[str]:
|
||||
@@ -57,7 +61,7 @@ def get_relations(page_name: str) -> list[dict]:
|
||||
|
||||
|
||||
def expand_nodes(nodes: list[str], depth: int = 1) -> list[str]:
|
||||
"""从起始节点出发,沿 relations 扩展相邻节点"""
|
||||
"""从起始节点出发,沿 relations 扩展相邻节点(实时读取页面 frontmatter)"""
|
||||
expanded = list(nodes)
|
||||
frontier = list(nodes)
|
||||
for _ in range(depth):
|
||||
@@ -75,11 +79,38 @@ def expand_nodes(nodes: list[str], depth: int = 1) -> list[str]:
|
||||
return expanded
|
||||
|
||||
|
||||
def expand_nodes_via_graph(nodes: list[str], depth: int = 1, min_conf: int = 3) -> list[str]:
|
||||
"""通过预构建的 knowledge-graph.json 扩展(更快,无 IO)"""
|
||||
if not GRAPH_DB.exists():
|
||||
return list(nodes)
|
||||
graph = json.loads(GRAPH_DB.read_text(encoding="utf-8"))
|
||||
adj: dict[str, list] = {}
|
||||
for e in graph["edges"]:
|
||||
adj.setdefault(e["source"], []).append((e["target"], e.get("confidence", 3)))
|
||||
adj.setdefault(e["target"], []).append((e["source"], e.get("confidence", 3)))
|
||||
|
||||
expanded = list(nodes)
|
||||
frontier = list(nodes)
|
||||
for _ in range(depth):
|
||||
next_frontier = []
|
||||
for node in frontier:
|
||||
for target, conf in adj.get(node, []):
|
||||
if conf >= min_conf and target not in expanded:
|
||||
expanded.append(target)
|
||||
next_frontier.append(target)
|
||||
frontier = next_frontier
|
||||
if not frontier:
|
||||
break
|
||||
return expanded
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Graph-aware search for wiki pages")
|
||||
parser.add_argument("query", help="Search query")
|
||||
parser.add_argument("--count", type=int, default=15, help="Max results")
|
||||
parser.add_argument("--depth", type=int, default=1, help="Graph expansion depth")
|
||||
parser.add_argument("--use-graph", action="store_true",
|
||||
help="Use prebuilt knowledge-graph.json instead of live frontmatter")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"=== Searching: {args.query} ===")
|
||||
@@ -92,10 +123,20 @@ def main():
|
||||
for r in results:
|
||||
print(f" [[{r}]]")
|
||||
|
||||
expanded = expand_nodes(results, depth=args.depth)
|
||||
if args.use_graph and not GRAPH_DB.exists():
|
||||
print(f"\n (note: {GRAPH_DB.name} not found, run `knowledge-graph.py build`; "
|
||||
f"falling back to live frontmatter)")
|
||||
|
||||
if args.use_graph and GRAPH_DB.exists():
|
||||
expanded = expand_nodes_via_graph(results, depth=args.depth)
|
||||
source = "knowledge-graph.json"
|
||||
else:
|
||||
expanded = expand_nodes(results, depth=args.depth)
|
||||
source = "live frontmatter"
|
||||
|
||||
new = [e for e in expanded if e not in results]
|
||||
if new:
|
||||
print(f"\nGraph-expanded (via relations, depth={args.depth}):")
|
||||
print(f"\nGraph-expanded (via {source}, depth={args.depth}):")
|
||||
for n in new[:args.count]:
|
||||
print(f" [[{n}]]")
|
||||
if len(new) > args.count:
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
knowledge-graph.py — Phase 3 图存储层 + 图遍历搜索
|
||||
|
||||
维护 wiki/ 的 typed relationships 图。存储为 JSON(tools/data/knowledge-graph.json),
|
||||
支持图构建、证据链反向追溯、以及融合搜索(qmd 直接匹配 + graph 关系扩展)。
|
||||
|
||||
轻量级方案(Sqlite + JSON 混合):当前仅用 JSON,无需外部数据库依赖。
|
||||
|
||||
用法:
|
||||
python tools/scripts/knowledge-graph.py build # 从 relations 重建图
|
||||
python tools/scripts/knowledge-graph.py stats # 图统计
|
||||
python tools/scripts/knowledge-graph.py evidence <页面> # 反向追溯证据链(caused_by/supports)
|
||||
python tools/scripts/knowledge-graph.py neighbors <页面> # 直接邻接节点
|
||||
python tools/scripts/knowledge-graph.py search "<查询>" # 融合搜索(需 qmd)
|
||||
python tools/scripts/knowledge-graph.py export-dot # 输出 Graphviz DOT
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||||
WIKI = VAULT / "wiki"
|
||||
GRAPH_DB = VAULT / "tools" / "data" / "knowledge-graph.json"
|
||||
QMD = r'node "C:\Users\hhhh2024\AppData\Roaming\npm\node_modules\@tobilu\qmd\dist\cli\qmd.js"'
|
||||
|
||||
EVIDENCE_EDGES = {"caused_by", "supports", "extends"}
|
||||
|
||||
|
||||
def read_frontmatter(fp: Path) -> dict | None:
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m or not yaml:
|
||||
return None
|
||||
try:
|
||||
return yaml.safe_load(m.group(1)) or {}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def build_graph() -> dict:
|
||||
"""扫描所有 wiki 页面的 relations 字段,重建完整图"""
|
||||
graph = {"nodes": [], "edges": [], "meta": {}}
|
||||
seen = set()
|
||||
edges_seen = set()
|
||||
|
||||
for fp in sorted(WIKI.glob("*.md")):
|
||||
if fp.name in ("index.md", "log.md"):
|
||||
continue
|
||||
front = read_frontmatter(fp)
|
||||
if front is None:
|
||||
front = {}
|
||||
name = fp.stem
|
||||
if name not in seen:
|
||||
graph["nodes"].append({
|
||||
"id": name,
|
||||
"type": front.get("type", "unknown"),
|
||||
"confidence": front.get("confidence", 3),
|
||||
"status": front.get("status", "active"),
|
||||
})
|
||||
seen.add(name)
|
||||
|
||||
for rel in front.get("relations", []) or []:
|
||||
if not isinstance(rel, dict):
|
||||
continue
|
||||
target = str(rel.get("target", "")).strip("[]")
|
||||
if not target or target == name:
|
||||
continue
|
||||
if target not in seen:
|
||||
graph["nodes"].append({
|
||||
"id": target,
|
||||
"type": "unknown",
|
||||
"confidence": 3,
|
||||
"status": "unknown",
|
||||
})
|
||||
seen.add(target)
|
||||
edge_key = (name, target, rel.get("type", "related"))
|
||||
if edge_key in edges_seen:
|
||||
continue
|
||||
edges_seen.add(edge_key)
|
||||
graph["edges"].append({
|
||||
"source": name,
|
||||
"target": target,
|
||||
"type": rel.get("type", "related"),
|
||||
"confidence": rel.get("confidence", 3),
|
||||
})
|
||||
|
||||
graph["meta"] = {
|
||||
"node_count": len(graph["nodes"]),
|
||||
"edge_count": len(graph["edges"]),
|
||||
"edge_types": {},
|
||||
}
|
||||
for e in graph["edges"]:
|
||||
t = e["type"]
|
||||
graph["meta"]["edge_types"][t] = graph["meta"]["edge_types"].get(t, 0) + 1
|
||||
|
||||
GRAPH_DB.parent.mkdir(parents=True, exist_ok=True)
|
||||
GRAPH_DB.write_text(
|
||||
json.dumps(graph, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
return graph
|
||||
|
||||
|
||||
def load_graph() -> dict | None:
|
||||
if not GRAPH_DB.exists():
|
||||
return None
|
||||
return json.loads(GRAPH_DB.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def adjacency(graph: dict, directed: bool = True) -> dict:
|
||||
"""构建邻接表。directed=False 时加入反向边(标注 inverse_ 前缀)"""
|
||||
adj: dict[str, list] = {}
|
||||
for e in graph["edges"]:
|
||||
adj.setdefault(e["source"], []).append((e["target"], e["type"], e.get("confidence", 3)))
|
||||
if not directed:
|
||||
adj.setdefault(e["target"], []).append(
|
||||
(e["source"], "inverse_" + e["type"], e.get("confidence", 3))
|
||||
)
|
||||
return adj
|
||||
|
||||
|
||||
def find_evidence_chain(target_page: str, max_depth: int = 5) -> list[str]:
|
||||
"""从 target 反向追溯所有 caused_by/supports 关系的源头(BFS)"""
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
return []
|
||||
reverse_adj: dict[str, list] = {}
|
||||
for e in graph["edges"]:
|
||||
reverse_adj.setdefault(e["target"], []).append((e["source"], e["type"]))
|
||||
|
||||
chain: list[str] = []
|
||||
visited = {target_page}
|
||||
dq = deque([(target_page, 0)])
|
||||
while dq:
|
||||
node, depth = dq.popleft()
|
||||
for src, rel in reverse_adj.get(node, []):
|
||||
if rel not in EVIDENCE_EDGES:
|
||||
continue
|
||||
if src in visited:
|
||||
continue
|
||||
visited.add(src)
|
||||
chain.append((" " * (depth + 1)) + f"← [[{src}]] ({rel})")
|
||||
if depth + 1 < max_depth:
|
||||
dq.append((src, depth + 1))
|
||||
return chain
|
||||
|
||||
|
||||
def graph_expanded_search(query: str, count: int = 15, min_conf: int = 3) -> list[dict]:
|
||||
"""融合搜索:qmd BM25/向量结果 → graph 关系扩展 → 去重重排序"""
|
||||
cmd = f'{QMD} vsearch "{query}" -c wiki -n 10'
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, shell=True, timeout=30)
|
||||
direct = [ln.strip() for ln in result.stdout.splitlines() if ln.strip()]
|
||||
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
return [{"page": p, "relevance": "direct"} for p in direct[:count]]
|
||||
|
||||
adj = adjacency(graph, directed=True)
|
||||
expanded = list(direct)
|
||||
for page in direct:
|
||||
for target, rel_type, conf in adj.get(page, []):
|
||||
if target not in expanded and conf >= min_conf:
|
||||
expanded.append(target)
|
||||
|
||||
return [{"page": p, "relevance": "direct" if p in direct else "graph"} for p in expanded[:count]]
|
||||
|
||||
|
||||
def cmd_stats(args):
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
print("Graph DB not found. Run 'build' first.")
|
||||
return
|
||||
meta = graph["meta"]
|
||||
print(f"=== Knowledge Graph Stats ===")
|
||||
print(f" Nodes: {meta['node_count']}")
|
||||
print(f" Edges: {meta['edge_count']}")
|
||||
print(f"\n Edge type breakdown:")
|
||||
for t, c in sorted(meta["edge_types"].items(), key=lambda x: -x[1]):
|
||||
print(f" {t}: {c}")
|
||||
typed_nodes = sum(1 for n in graph["nodes"] if n["type"] != "unknown")
|
||||
print(f"\n Typed nodes: {typed_nodes} / {meta['node_count']}")
|
||||
|
||||
|
||||
def cmd_evidence(args):
|
||||
chain = find_evidence_chain(args.page)
|
||||
print(f"=== Evidence chain for [[{args.page}]] ===")
|
||||
if not chain:
|
||||
print(" No evidence edges (caused_by/supports/extends) found.")
|
||||
return
|
||||
for line in chain:
|
||||
print(f" {line}")
|
||||
print(f"\n Total upstream evidence: {len(chain)}")
|
||||
|
||||
|
||||
def cmd_neighbors(args):
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
print("Graph DB not found. Run 'build' first.")
|
||||
return
|
||||
found = False
|
||||
for e in graph["edges"]:
|
||||
if e["source"] == args.page:
|
||||
print(f" --{e['type']}--> [[{e['target']}]] (conf={e.get('confidence', 3)})")
|
||||
found = True
|
||||
for e in graph["edges"]:
|
||||
if e["target"] == args.page:
|
||||
print(f" <--{e['type']}-- [[{e['source']}]] (conf={e.get('confidence', 3)})")
|
||||
found = True
|
||||
if not found:
|
||||
print(f" [[{args.page}]] has no relations in the graph.")
|
||||
|
||||
|
||||
def cmd_search(args):
|
||||
print(f"=== Fusion search: {args.query} ===")
|
||||
results = graph_expanded_search(args.query, count=args.count)
|
||||
direct = [r for r in results if r["relevance"] == "direct"]
|
||||
graph_hits = [r for r in results if r["relevance"] == "graph"]
|
||||
if direct:
|
||||
print(f"\nDirect matches ({len(direct)}):")
|
||||
for r in direct:
|
||||
print(f" [[{r['page']}]]")
|
||||
if graph_hits:
|
||||
print(f"\nGraph-expanded (via relations):")
|
||||
for r in graph_hits:
|
||||
print(f" [[{r['page']}]]")
|
||||
print(f"\nTotal: {len(results)}")
|
||||
|
||||
|
||||
def cmd_export_dot(args):
|
||||
graph = load_graph()
|
||||
if not graph:
|
||||
print("Graph DB not found. Run 'build' first.")
|
||||
return
|
||||
print("digraph wiki {")
|
||||
print(' rankdir=LR;')
|
||||
for n in graph["nodes"]:
|
||||
print(f' "{n["id"]}";')
|
||||
for e in graph["edges"]:
|
||||
print(f' "{e["source"]}" -> "{e["target"]}" [label="{e["type"]}"];')
|
||||
print("}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Phase 3 knowledge graph storage + traversal")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
sub.add_parser("build", help="Rebuild graph from relations")
|
||||
sub.add_parser("stats", help="Print graph statistics")
|
||||
p_ev = sub.add_parser("evidence", help="Trace evidence chain (caused_by/supports)")
|
||||
p_ev.add_argument("page", help="Target page name")
|
||||
p_nb = sub.add_parser("neighbors", help="Direct neighbors of a page")
|
||||
p_nb.add_argument("page", help="Page name")
|
||||
p_se = sub.add_parser("search", help="Fusion search (qmd + graph expansion)")
|
||||
p_se.add_argument("query", help="Search query")
|
||||
p_se.add_argument("--count", type=int, default=15)
|
||||
sub.add_parser("export-dot", help="Export Graphviz DOT format")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "build":
|
||||
graph = build_graph()
|
||||
m = graph["meta"]
|
||||
print(f"Graph built: {m['node_count']} nodes, {m['edge_count']} edges")
|
||||
print(f"Saved to {GRAPH_DB}")
|
||||
elif args.command == "stats":
|
||||
cmd_stats(args)
|
||||
elif args.command == "evidence":
|
||||
cmd_evidence(args)
|
||||
elif args.command == "neighbors":
|
||||
cmd_neighbors(args)
|
||||
elif args.command == "search":
|
||||
cmd_search(args)
|
||||
elif args.command == "export-dot":
|
||||
cmd_export_dot(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
populate-relations.py — Phase 3 P1: 批量填充 wiki 页面的 relations 字段
|
||||
|
||||
策略(自动推断,无 LLM 依赖):
|
||||
1. **Source-grouping**:共享同一 raw source 的页面群,互为 part_of 关系
|
||||
- 每组中被引用最多的页面成为 hub
|
||||
- 其余页面 part_of → hub
|
||||
2. **Type-based**:
|
||||
- entity (人物) → example_of 对应 concept(如 毛泽东 example_of 教员的终极答案)
|
||||
- place → part_of 上级地点(如 西安 part_of 陕西省)
|
||||
3. **Cross-link inference**:页面正文中的 [[wikilink]] 若目标存在,
|
||||
且源页面 type=concept,推断 extends 关系(仅取 top 2 避免噪声)
|
||||
|
||||
安全措施:
|
||||
- 幂等:已有 relations 不重复添加
|
||||
- 仅处理 wiki/*.md(排除 index/log/archive)
|
||||
- --dry-run 预览,--apply 执行
|
||||
- 每页最多添加 5 条 relations(防爆裂)
|
||||
|
||||
用法:
|
||||
python tools/scripts/populate-relations.py --dry-run # 预览
|
||||
python tools/scripts/populate-relations.py --apply # 执行
|
||||
python tools/scripts/populate-relations.py --apply --max-per-page 3
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("ERROR: pyyaml required"); sys.exit(1)
|
||||
|
||||
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||||
WIKI = VAULT / "wiki"
|
||||
EXCLUDE = {"index.md", "log.md"}
|
||||
MAX_RELATIONS_DEFAULT = 5
|
||||
|
||||
|
||||
def read_frontmatter(fp: Path) -> tuple[dict, str]:
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m:
|
||||
return {}, content
|
||||
try:
|
||||
front = yaml.safe_load(m.group(1)) or {}
|
||||
except Exception:
|
||||
return {}, content
|
||||
return front, content
|
||||
|
||||
|
||||
def write_frontmatter(fp: Path, front: dict, body: str):
|
||||
# Strip any existing frontmatter from body to avoid duplication
|
||||
body_clean = re.sub(r"^---\n.*?\n---\n?", "", body, count=1, flags=re.DOTALL).lstrip()
|
||||
new_yaml = yaml.dump(front, allow_unicode=True, default_flow_style=False, sort_keys=False)
|
||||
fp.write_text(f"---\n{new_yaml}---\n\n{body_clean}", encoding="utf-8")
|
||||
|
||||
|
||||
def get_source_key(front: dict) -> str:
|
||||
"""Extract a normalized source key for grouping"""
|
||||
source = front.get("source", "")
|
||||
if not source:
|
||||
return ""
|
||||
s = str(source).strip().strip('"').strip("'")
|
||||
# Remove [[ ]] and raw/ prefix
|
||||
s = re.sub(r"\[\[(?:raw/)?(.+?)(?:\|.*)?\]\]", r"\1", s)
|
||||
# Remove .md extension
|
||||
s = re.sub(r"\.md$", "", s)
|
||||
# Remove path prefixes
|
||||
s = s.split("/")[-1]
|
||||
return s
|
||||
|
||||
|
||||
def get_inbound_counts() -> dict[str, int]:
|
||||
"""Count inbound wikilinks for each wiki page"""
|
||||
counts = defaultdict(int)
|
||||
all_content = ""
|
||||
for fp in VAULT.rglob("*.md"):
|
||||
if ".git" in str(fp):
|
||||
continue
|
||||
try:
|
||||
all_content += fp.read_text(encoding="utf-8") + "\n"
|
||||
except Exception:
|
||||
continue
|
||||
for m in re.finditer(r"\[\[([^\]|#]+?)(?:[#\]|]|$)", all_content):
|
||||
name = m.group(1).strip()
|
||||
if name and not name.startswith("http"):
|
||||
counts[name] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def extract_wikilinks(content: str) -> list[str]:
|
||||
"""Extract wikilink targets from page body (not frontmatter)"""
|
||||
# Remove frontmatter
|
||||
body = re.sub(r"^---\n.*?\n---", "", content, count=1, flags=re.DOTALL)
|
||||
links = []
|
||||
for m in re.finditer(r"\[\[([^\]|#]+?)(?:[#\]|])", body):
|
||||
target = m.group(1).strip()
|
||||
if target and not target.startswith("http") and not target.startswith("raw/"):
|
||||
links.append(target)
|
||||
return links
|
||||
|
||||
|
||||
def dedup_existing(front: dict) -> set[tuple[str, str]]:
|
||||
"""Get existing relation (type, target) pairs"""
|
||||
existing = set()
|
||||
for r in front.get("relations", []) or []:
|
||||
if isinstance(r, dict):
|
||||
t = r.get("target", "").strip("[]")
|
||||
existing.add((r.get("type", ""), t))
|
||||
return existing
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Batch populate relations for wiki pages")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Preview without writing")
|
||||
parser.add_argument("--apply", action="store_true", help="Write changes to files")
|
||||
parser.add_argument("--max-per-page", type=int, default=MAX_RELATIONS_DEFAULT)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.dry_run and not args.apply:
|
||||
args.dry_run = True # default to dry-run for safety
|
||||
|
||||
# Phase 1: Read all pages
|
||||
pages = {}
|
||||
for fp in sorted(WIKI.glob("*.md")):
|
||||
if fp.name in EXCLUDE:
|
||||
continue
|
||||
front, content = read_frontmatter(fp)
|
||||
if not front:
|
||||
continue
|
||||
pages[fp.stem] = {"fp": fp, "front": front, "content": content}
|
||||
|
||||
print(f"Loaded {len(pages)} wiki pages")
|
||||
|
||||
# Phase 2: Compute inbound counts
|
||||
inbound = get_inbound_counts()
|
||||
print(f"Computed inbound counts for {len(inbound)} link targets")
|
||||
|
||||
# Phase 3: Group by source
|
||||
source_groups = defaultdict(list)
|
||||
for name, info in pages.items():
|
||||
src = get_source_key(info["front"])
|
||||
if src:
|
||||
source_groups[src].append(name)
|
||||
|
||||
# Phase 4: Infer relations
|
||||
additions = defaultdict(list) # page -> list of {type, target, ...}
|
||||
|
||||
# 4a: Source-grouping → part_of hub
|
||||
multi_page_groups = {s: g for s, g in source_groups.items() if len(g) > 1}
|
||||
print(f"\nSource groups with >1 page: {len(multi_page_groups)}")
|
||||
for src, group in multi_page_groups.items():
|
||||
# Hub = most referenced page in group
|
||||
hub = max(group, key=lambda p: inbound.get(p, 0))
|
||||
for page in group:
|
||||
if page == hub:
|
||||
continue
|
||||
additions[page].append({
|
||||
"type": "part_of",
|
||||
"target": f"[[{hub}]]",
|
||||
"description": f"同源:{src}",
|
||||
"confidence": 3,
|
||||
})
|
||||
|
||||
# 4b: Single-source pages → extends the source concept if source is a wiki page
|
||||
single_source = {s: g[0] for s, g in source_groups.items() if len(g) == 1}
|
||||
for src, page in single_source.items():
|
||||
info = pages.get(page)
|
||||
if not info:
|
||||
continue
|
||||
page_type = info["front"].get("type", "")
|
||||
if isinstance(page_type, list):
|
||||
page_type = page_type[0] if page_type else ""
|
||||
page_type = str(page_type).lower()
|
||||
|
||||
# If source matches a wiki page name, add extends
|
||||
if src in pages and src != page:
|
||||
src_type = pages[src]["front"].get("type", "")
|
||||
if isinstance(src_type, list):
|
||||
src_type = src_type[0] if src_type else ""
|
||||
src_type = str(src_type).lower()
|
||||
|
||||
if page_type == "entity" and src_type in ("concept", "reference"):
|
||||
additions[page].append({
|
||||
"type": "example_of",
|
||||
"target": f"[[{src}]]",
|
||||
"confidence": 3,
|
||||
})
|
||||
elif page_type == "concept" and src_type == "concept":
|
||||
additions[page].append({
|
||||
"type": "extends",
|
||||
"target": f"[[{src}]]",
|
||||
"confidence": 3,
|
||||
})
|
||||
elif page_type in ("place",) and src_type in ("place", "concept"):
|
||||
additions[page].append({
|
||||
"type": "part_of",
|
||||
"target": f"[[{src}]]",
|
||||
"confidence": 3,
|
||||
})
|
||||
|
||||
# 4c: Cross-link inference for concept pages (top 2 wikilinks)
|
||||
for name, info in pages.items():
|
||||
page_type = info["front"].get("type", "")
|
||||
if isinstance(page_type, list):
|
||||
page_type = page_type[0] if page_type else ""
|
||||
if str(page_type).lower() not in ("concept", "method"):
|
||||
continue
|
||||
links = extract_wikilinks(info["content"])
|
||||
seen = set()
|
||||
added = 0
|
||||
for target in links:
|
||||
if added >= 2:
|
||||
break
|
||||
if target == name or target in seen:
|
||||
continue
|
||||
if target not in pages:
|
||||
continue
|
||||
seen.add(target)
|
||||
additions[name].append({
|
||||
"type": "extends",
|
||||
"target": f"[[{target}]]",
|
||||
"confidence": 2,
|
||||
})
|
||||
added += 1
|
||||
|
||||
# Phase 5: Apply (single pass — dry-run prints, apply writes)
|
||||
total_added = 0
|
||||
pages_modified = 0
|
||||
for name in sorted(pages.keys()):
|
||||
pending = additions.get(name, [])
|
||||
if not pending:
|
||||
continue
|
||||
info = pages[name]
|
||||
front = info["front"]
|
||||
existing = dedup_existing(front)
|
||||
|
||||
new_rels = list(front.get("relations", []) or [])
|
||||
added = 0
|
||||
for rel in pending:
|
||||
key = (rel["type"], rel["target"].strip("[]"))
|
||||
if key in existing:
|
||||
continue
|
||||
if len(new_rels) >= args.max_per_page:
|
||||
break
|
||||
new_rels.append(rel)
|
||||
existing.add(key)
|
||||
added += 1
|
||||
|
||||
if added == 0:
|
||||
continue
|
||||
|
||||
front["relations"] = new_rels
|
||||
pages_modified += 1
|
||||
total_added += added
|
||||
|
||||
if args.dry_run:
|
||||
print(f" [[{name}]]: +{added}")
|
||||
for rel in new_rels[-added:]:
|
||||
print(f" --{rel['type']}--> {rel['target']}")
|
||||
|
||||
mode = "DRY RUN" if args.dry_run else "APPLYING"
|
||||
print(f"\n=== {mode} ===")
|
||||
print(f" Pages to modify: {pages_modified}")
|
||||
print(f" Relations to add: {total_added}")
|
||||
|
||||
if args.apply and pages_modified > 0:
|
||||
written = 0
|
||||
for name in sorted(pages.keys()):
|
||||
info = pages[name]
|
||||
front = info["front"]
|
||||
rels = front.get("relations", [])
|
||||
if not rels:
|
||||
continue
|
||||
# Only write if relations were newly populated (not pre-existing)
|
||||
# Check by seeing if this page was in our additions map
|
||||
if name not in additions:
|
||||
continue
|
||||
write_frontmatter(info["fp"], front, info["content"])
|
||||
written += 1
|
||||
print(f" Written to {written} files")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
review-pages.py — Phase 3 wiki 页面评审器
|
||||
|
||||
两种模式:
|
||||
1. LLM 评审(需 Ollama 后端):调用本地模型审核内容准确性,检测声明与 raw 来源
|
||||
不匹配、跨页面矛盾、过时信息。
|
||||
2. 静态评审(LLM 不可用时自动降级):基于 frontmatter 规则与 raw 来源比对,
|
||||
检测低置信度、缺 source、断链、过时等结构问题。无外部依赖。
|
||||
|
||||
B-3.3 低置信度自动建议:confidence <= 2 的页面生成"需要更多证据"提示。
|
||||
B-3.1 LLM 评审:可检测语义偏差(被 AI 过度概括/扭曲的结论)。
|
||||
|
||||
用法:
|
||||
python tools/scripts/review-pages.py --stale # 审查过时页面
|
||||
python tools/scripts/review-pages.py --random 5 # 随机抽 5 页
|
||||
python tools/scripts/review-pages.py --page "LLM-Wiki-v2" # 指定单页
|
||||
python tools/scripts/review-pages.py --confidence-low # 低置信度页面
|
||||
python tools/scripts/review-pages.py --random 3 --llm # 强制启用 LLM 评审
|
||||
python tools/scripts/review-pages.py --json # JSON 输出
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||||
WIKI = VAULT / "wiki"
|
||||
RAW = VAULT / "raw"
|
||||
TODAY = date.today()
|
||||
|
||||
LLM_MODEL = "qwen2.5:7b"
|
||||
REVIEW_PROMPT = """请审核以下 Obsidian wiki 页面,输出 JSON(仅 JSON,无多余文字):
|
||||
{
|
||||
"accuracy_score": 1到5的整数,
|
||||
"issues": ["具体问题描述"],
|
||||
"confidence_match": true或false,
|
||||
"suggested_updates": ["可执行的改进建议"]
|
||||
}
|
||||
重点关注:数字/百分比是否标注来源行号、结论是否被过度概括、frontmatter 是否完整。
|
||||
|
||||
页面内容(截断至 4000 字符):
|
||||
{content}
|
||||
"""
|
||||
|
||||
|
||||
def read_frontmatter(fp: Path) -> tuple[dict, str]:
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m or not yaml:
|
||||
return {}, content
|
||||
try:
|
||||
return yaml.safe_load(m.group(1)) or {}, content
|
||||
except Exception:
|
||||
return {}, content
|
||||
|
||||
|
||||
def ollama_available() -> bool:
|
||||
return shutil.which("ollama") is not None
|
||||
|
||||
|
||||
def llm_review(page_name: str, content: str) -> dict | None:
|
||||
"""调用 Ollama 审核单页,返回结构化结果"""
|
||||
if not ollama_available():
|
||||
return None
|
||||
prompt = REVIEW_PROMPT.format(content=content[:4000])
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["ollama", "run", LLM_MODEL, prompt],
|
||||
capture_output=True, text=True, timeout=180, encoding="utf-8",
|
||||
)
|
||||
out = result.stdout.strip()
|
||||
start, end = out.find("{"), out.rfind("}")
|
||||
if start != -1 and end != -1:
|
||||
return json.loads(out[start:end + 1])
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def static_review(fp: Path) -> dict:
|
||||
"""静态规则评审:无 LLM 依赖的结构化检查"""
|
||||
front, content = read_frontmatter(fp)
|
||||
issues = []
|
||||
suggestions = []
|
||||
|
||||
cats = front.get("categories", []) or []
|
||||
if not any("LLM Wiki" in str(c) for c in cats):
|
||||
issues.append("categories 缺少 [[LLM Wiki]]")
|
||||
tags = front.get("tags", []) or []
|
||||
if "wiki" not in tags:
|
||||
issues.append("tags 缺少 wiki")
|
||||
if not front.get("type"):
|
||||
issues.append("缺少 type 字段")
|
||||
source = front.get("source", "")
|
||||
if not source or str(source).strip('[]"') == "":
|
||||
issues.append("缺少 source(wiki 层必填)")
|
||||
suggestions.append(f"补充 source 指向 raw/ 来源,参考同类页面:{fp.stem}")
|
||||
|
||||
conf = front.get("confidence")
|
||||
if conf is not None and isinstance(conf, (int, float)) and conf <= 2:
|
||||
issues.append(f"低置信度 (confidence={conf})")
|
||||
suggestions.append("需要更多证据:补充多源交叉验证或将 confidence 提升至 3+")
|
||||
|
||||
last = str(front.get("last_reviewed", ""))
|
||||
if last:
|
||||
try:
|
||||
overdue = (TODAY - date.fromisoformat(last)).days
|
||||
interval = front.get("review_interval_days", 180)
|
||||
if overdue > interval:
|
||||
issues.append(f"过时(上次审查 {last},超期 {overdue - interval} 天)")
|
||||
suggestions.append(f"复查内容并更新 last_reviewed 为 {TODAY.isoformat()}")
|
||||
except ValueError:
|
||||
issues.append(f"last_reviewed 格式无效:{last}")
|
||||
|
||||
no_line = re.findall(r"\[raw:[^:\]\[]+\]", content)
|
||||
if no_line:
|
||||
issues.append(f"{len(no_line)} 条 raw 引用缺少行号")
|
||||
suggestions.append("运行 python tools/scripts/fix-raw-citations.py 自动补全行号")
|
||||
|
||||
score = 5 - min(4, len([i for i in issues if i.startswith(("低置信", "过时", "缺少 source"))]))
|
||||
return {
|
||||
"page": fp.stem,
|
||||
"mode": "static",
|
||||
"accuracy_score": max(1, score),
|
||||
"issues": issues,
|
||||
"confidence_match": (conf is None or conf >= 3),
|
||||
"suggested_updates": suggestions,
|
||||
}
|
||||
|
||||
|
||||
def select_pages(args) -> list[Path]:
|
||||
all_pages = [p for p in sorted(WIKI.glob("*.md"))
|
||||
if p.name not in ("index.md", "log.md")]
|
||||
if args.page:
|
||||
fp = WIKI / f"{args.page}.md"
|
||||
return [fp] if fp.exists() else []
|
||||
if args.confidence_low:
|
||||
return [p for p in all_pages
|
||||
if (lambda c: c is not None and isinstance(c, (int, float)) and c <= 2)(
|
||||
read_frontmatter(p)[0].get("confidence"))]
|
||||
if args.stale:
|
||||
selected = []
|
||||
for p in all_pages:
|
||||
front, _ = read_frontmatter(p)
|
||||
last = str(front.get("last_reviewed", ""))
|
||||
interval = front.get("review_interval_days", 180)
|
||||
if not last:
|
||||
selected.append(p)
|
||||
continue
|
||||
try:
|
||||
if (TODAY - date.fromisoformat(last)).days > interval:
|
||||
selected.append(p)
|
||||
except ValueError:
|
||||
selected.append(p)
|
||||
return selected
|
||||
if args.random:
|
||||
return random.sample(all_pages, min(args.random, len(all_pages)))
|
||||
return all_pages
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Review wiki pages (LLM or static fallback)")
|
||||
sel = parser.add_mutually_exclusive_group()
|
||||
sel.add_argument("--stale", action="store_true", help="Review stale pages only")
|
||||
sel.add_argument("--random", type=int, metavar="N", help="Randomly sample N pages")
|
||||
sel.add_argument("--page", help="Review a single page")
|
||||
sel.add_argument("--confidence-low", action="store_true", help="Review confidence<=2 pages")
|
||||
parser.add_argument("--llm", action="store_true", help="Force LLM review (requires Ollama)")
|
||||
parser.add_argument("--json", action="store_true", help="Output JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
pages = select_pages(args)
|
||||
if not pages:
|
||||
print("No pages matched selection criteria.")
|
||||
return
|
||||
|
||||
use_llm = args.llm or (not args.confidence_low)
|
||||
llm_on = use_llm and ollama_available()
|
||||
if use_llm and not llm_on:
|
||||
print("(Ollama 不可用,降级为静态评审模式。安装 Ollama 以启用 LLM 语义审核。)\n")
|
||||
|
||||
reports = []
|
||||
for fp in pages:
|
||||
front, content = read_frontmatter(fp)
|
||||
if llm_on:
|
||||
llm = llm_review(fp.stem, content)
|
||||
report = llm if llm else static_review(fp)
|
||||
if report:
|
||||
report["mode"] = "llm"
|
||||
else:
|
||||
report = static_review(fp)
|
||||
report.setdefault("page", fp.stem)
|
||||
reports.append(report)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(reports, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
print(f"=== Review Report ({'LLM' if llm_on else 'static'} mode, {len(reports)} pages) ===\n")
|
||||
needs_action = [r for r in reports if r["issues"]]
|
||||
for r in reports:
|
||||
flag = "OK " if not r["issues"] else "FIX"
|
||||
print(f"[{flag}] [[{r['page']}]] — score {r['accuracy_score']}/5")
|
||||
for issue in r["issues"]:
|
||||
print(f" • {issue}")
|
||||
for sug in r["suggested_updates"]:
|
||||
print(f" → {sug}")
|
||||
print(f"\nSummary: {len(needs_action)}/{len(reports)} pages need attention")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+143
-43
@@ -12,14 +12,23 @@ sync_home_wiki.py — home-wiki 知识页面 → kepano LLM Wiki 同步脚本
|
||||
concepts/ + entities/ + syntheses/ (排除 index.md)
|
||||
Daily/ MyNotes/ reports/ 不在知识同步范围。
|
||||
|
||||
转换规则(home-wiki frontmatter → kepano Wiki 规范):
|
||||
categories: 加 [[LLM Wiki]];entity/person 额外加 [[People]]
|
||||
tags: 前缀 [wiki, {people|concept}],subtype→concept/{subtype}(仅 concept),追加原 tags
|
||||
created: 保留
|
||||
source: sources[0] → 文件路径取 stem 做 wikilink;URL/标识符原样;空则留空
|
||||
type: 保留(concept/entity/synthesis)
|
||||
aliases: title 或 name
|
||||
正文: 保留,移除 home-wiki 特有的 openclaw 自动段落(## Related、注释标记行)
|
||||
转换规则(home-wiki frontmatter → kepano Wiki 规范,Phase 1 v2 升级版):
|
||||
categories: 加 [[LLM Wiki]];entity/person 额外加 [[People]]
|
||||
tags: 前缀 [wiki, {people|concept}],subtype→concept/{subtype}(仅 concept),追加原 tags
|
||||
created: 保留
|
||||
source: sources[0] → 文件路径取 stem 做 wikilink;URL/标识符原样;空则留空
|
||||
type: 保留(concept/entity/synthesis)
|
||||
aliases: title 或 name
|
||||
confidence: home-wiki 已有则沿用,否则默认 3
|
||||
status: 默认 active
|
||||
last_reviewed: 同步日期(YYYY-MM-DD)
|
||||
review_interval_days: 180(技术类 90)
|
||||
relations: 从 home-wiki `related: [[wikilink]]` 字段提取 → `[{type: extends, target: [[x]]}]`
|
||||
正文: 保留,移除 home-wiki 特有的 openclaw 自动段落(## Related、注释标记行)
|
||||
|
||||
YAML 输出格式(Obsidian 兼容):
|
||||
使用 IndentedDumper(子类)确保序列正确缩进(列表项 2 空格缩进),
|
||||
wikilink 字符串加双引号,避免 Obsidian 解析异常。
|
||||
|
||||
index 自动维护(--index):
|
||||
用标记块 <!-- BEGIN/END home-wiki-sync --> 界定 index.md 的「概念页」「实体页」
|
||||
@@ -32,7 +41,7 @@ index 自动维护(--index):
|
||||
python tools/scripts/sync_home_wiki.py --dry-run # 仅诊断差异
|
||||
python tools/scripts/sync_home_wiki.py # 同步(写入变化的页面)
|
||||
python tools/scripts/sync_home_wiki.py --index --log # 一站式:同步+更新index+记日志
|
||||
python tools/scripts/sync_home_wiki.py --lint # 体检(孤儿/断链/source空值)
|
||||
python tools/scripts/sync_home_wiki.py --lint # 体检(孤儿/断链/source空值/缺失lifecycle字段)
|
||||
|
||||
退出码:0 成功;1 源目录缺失;2 有错误。
|
||||
"""
|
||||
@@ -50,6 +59,24 @@ except ImportError:
|
||||
sys.stderr.write("ERROR: PyYAML 未安装,请运行 pip install pyyaml\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
class IndentedDumper(yaml.Dumper):
|
||||
"""Custom dumper that indents sequences under their key (Obsidian-friendly)"""
|
||||
def increase_indent(self, flow=False, indentless=False):
|
||||
return super().increase_indent(flow, False)
|
||||
|
||||
|
||||
def yaml_dump(data) -> str:
|
||||
"""Dump data with Obsidian-compatible formatting (indented sequences, double quotes)"""
|
||||
return yaml.dump(
|
||||
data,
|
||||
Dumper=IndentedDumper,
|
||||
allow_unicode=True,
|
||||
default_flow_style=False,
|
||||
sort_keys=False,
|
||||
width=1000,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 路径配置(两库位于不同位置,必须使用绝对路径)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -103,7 +130,7 @@ def _robust_parse_fm(fm_text):
|
||||
m = re.search(rf"^[ \t]*{re.escape(key)}[ \t]*:[ \t]*(.+?)[ \t]*$", fm_text, re.M)
|
||||
if m and m.group(1).strip() not in ("", "[]"):
|
||||
fm[key] = m.group(1).strip().strip("\"'")
|
||||
for key in ("tags", "sources"):
|
||||
for key in ("tags", "sources", "related"):
|
||||
vals = _extract_list_field(fm_text, key)
|
||||
if vals:
|
||||
fm[key] = vals
|
||||
@@ -157,8 +184,38 @@ def _as_list(v):
|
||||
return [v]
|
||||
|
||||
|
||||
def _extract_related(fm_text):
|
||||
"""提取 home-wiki 的 `related: [[x]]` 字段为 relations 列表。
|
||||
|
||||
home-wiki 用 `related: [[wikilink]]` 记录关联页面(YAML 解析时 [[ 会失败),
|
||||
转为 kepano Wiki 的 `relations: [{type: extends, target: "[[x]]"}]` 格式。
|
||||
"""
|
||||
rels = []
|
||||
# 匹配 related: 后跟 [[wikilink]],支持多个
|
||||
for m in re.finditer(r"^[ \t]*related[ \t]*:[ \t]*\[?\[([^\]|#]+)", fm_text, re.M):
|
||||
tgt = m.group(1).strip()
|
||||
if tgt:
|
||||
rels.append({"type": "extends", "target": f"[[{tgt}]]", "confidence": 2})
|
||||
# 也支持 related: 换行后列表
|
||||
block = re.search(r"^[ \t]*related[ \t]*:[ \t]*\n((?:[ \t]+-\s*[^\n]+\n?)+)", fm_text, re.M)
|
||||
if block:
|
||||
for line in block.group(1).splitlines():
|
||||
m = re.search(r"\[\[([^\]|#]+)", line)
|
||||
if m:
|
||||
rels.append({"type": "extends", "target": f"[[{m.group(1).strip()}]]", "confidence": 2})
|
||||
return rels
|
||||
|
||||
|
||||
def convert_frontmatter(fm, default_name):
|
||||
"""home-wiki frontmatter → kepano Wiki frontmatter(有序)。"""
|
||||
"""home-wiki frontmatter → kepano Wiki frontmatter(有序 dict)。
|
||||
|
||||
Phase 1 lifecycle 字段(2026-07-01 新增):
|
||||
- confidence: 1-5(默认 3,home-wiki 有则沿用)
|
||||
- status: active(默认)
|
||||
- last_reviewed: 同步日期
|
||||
- review_interval_days: 180(技术类 90)
|
||||
- relations: 从 home-wiki related: 字段映射
|
||||
"""
|
||||
fm_type = str(fm.get("type", "concept")).strip()
|
||||
subtype = fm.get("subtype")
|
||||
if isinstance(subtype, list):
|
||||
@@ -178,8 +235,6 @@ def convert_frontmatter(fm, default_name):
|
||||
type_tag = "concept"
|
||||
|
||||
tags = ["wiki", type_tag]
|
||||
# 仅 concept 类型把 subtype 转为 concept/{subtype} 标签
|
||||
# (entity/person 用 people 标签 + [[People]] category,不加 concept/person)
|
||||
if subtype and fm_type == "concept":
|
||||
tags.append(f"concept/{subtype}")
|
||||
for t in _as_list(fm.get("tags")):
|
||||
@@ -196,37 +251,41 @@ def convert_frontmatter(fm, default_name):
|
||||
else:
|
||||
src_val = ""
|
||||
|
||||
return [
|
||||
("categories", categories),
|
||||
("tags", tags),
|
||||
("created", str(fm.get("created", ""))),
|
||||
("source", src_val),
|
||||
("type", fm_type),
|
||||
("aliases", [str(title)]),
|
||||
]
|
||||
today = date.today().isoformat()
|
||||
confidence = fm.get("confidence", 3)
|
||||
try:
|
||||
confidence = int(confidence)
|
||||
except (ValueError, TypeError):
|
||||
confidence = 3
|
||||
# 技术类(type=tool)默认 90 天审查周期
|
||||
review_interval = 90 if fm_type == "tool" else 180
|
||||
|
||||
# 用普通 dict(Python 3.7+ 保持插入顺序),避免 OrderedDict YAML 序列化问题
|
||||
pairs = {
|
||||
"categories": categories,
|
||||
"tags": tags,
|
||||
"created": str(fm.get("created", "")),
|
||||
"source": src_val,
|
||||
"type": fm_type,
|
||||
"aliases": [str(title)],
|
||||
"confidence": confidence,
|
||||
"status": "active",
|
||||
"last_reviewed": today,
|
||||
"review_interval_days": review_interval,
|
||||
}
|
||||
|
||||
def _yaml_quote(val):
|
||||
"""如果值包含 YAML 特殊字符(如 [[ wikilink 的方括号),加双引号。"""
|
||||
s = str(val)
|
||||
if not s:
|
||||
return '""'
|
||||
if '[' in s or ']' in s or '{' in s or '}' in s or ':' in s or '#' in s:
|
||||
return f'"{s}"'
|
||||
return s
|
||||
return pairs
|
||||
|
||||
|
||||
def dump_frontmatter(pairs):
|
||||
out = ["---"]
|
||||
for k, v in pairs:
|
||||
if isinstance(v, list):
|
||||
out.append(f"{k}:")
|
||||
for item in v:
|
||||
out.append(f" - {_yaml_quote(item)}")
|
||||
else:
|
||||
out.append(f"{k}: {_yaml_quote(v)}")
|
||||
out.append("---")
|
||||
return "\n".join(out) + "\n"
|
||||
"""输出 Obsidian 兼容的 frontmatter(使用 IndentedDumper 缩进序列)。"""
|
||||
if isinstance(pairs, list):
|
||||
from collections import OrderedDict
|
||||
pairs = OrderedDict(pairs)
|
||||
# 转为普通 dict(避免 OrderedDict 触发 python/object 标签)
|
||||
data = dict(pairs) if not isinstance(pairs, dict) else pairs
|
||||
yaml_str = yaml_dump(data)
|
||||
return f"---\n{yaml_str}---\n\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -256,9 +315,23 @@ def build_page(src_path, stem):
|
||||
text = src_path.read_text(encoding="utf-8")
|
||||
fm, body = parse_doc(text)
|
||||
pairs = convert_frontmatter(fm, stem)
|
||||
# 提取 relations(从原始 frontmatter 文本中)
|
||||
m = re.match(r"^---\s*\n(.*?)\n---\s*\n?", text, re.S)
|
||||
if m:
|
||||
rels = _extract_related(m.group(1))
|
||||
if rels:
|
||||
# 去重:同 (type, target) 只保留第一个
|
||||
seen = set()
|
||||
unique_rels = []
|
||||
for r in rels:
|
||||
key = (r["type"], r["target"])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique_rels.append(r)
|
||||
pairs["relations"] = unique_rels[:5] # 限制最多 5 条
|
||||
body_clean = strip_openclaw(body)
|
||||
# 确保正文与 frontmatter 间有空行
|
||||
return dump_frontmatter(pairs) + "\n" + body_clean.lstrip("\n")
|
||||
return dump_frontmatter(pairs) + body_clean.lstrip("\n")
|
||||
|
||||
|
||||
def _norm_full(text):
|
||||
@@ -507,6 +580,27 @@ def lint():
|
||||
if m and not m.group(1).strip():
|
||||
empty_source.append(s)
|
||||
|
||||
# Phase 1 lifecycle 字段缺失检查(2026-07-01 新增)
|
||||
LIFECYCLE_FIELDS = ("confidence", "status", "last_reviewed", "review_interval_days")
|
||||
missing_lifecycle = []
|
||||
for s in synced:
|
||||
t = (DST_WIKI / f"{s}.md").read_text(encoding="utf-8")
|
||||
for field in LIFECYCLE_FIELDS:
|
||||
if not re.search(rf"^[ \t]*{re.escape(field)}[ \t]*:", t, re.M):
|
||||
missing_lifecycle.append(f"{s} (缺 {field})")
|
||||
break # 一页只报一次
|
||||
|
||||
# relations 字段检查(可选,但建议)
|
||||
no_relations = []
|
||||
for s in synced:
|
||||
t = (DST_WIKI / f"{s}.md").read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\s*\n(.*?)\n---\s*\n?", t, re.S)
|
||||
if m:
|
||||
fm_text = m.group(1)
|
||||
if not re.search(r"^[ \t]*relations[ \t]*:[ \t]*\n[ \t]+-", fm_text, re.M) and \
|
||||
not re.search(r"^[ \t]*relations[ \t]*:[ \t]*\[", fm_text, re.M):
|
||||
no_relations.append(s)
|
||||
|
||||
# 断链:同步页面引用的 [[x]] 在 wiki 是否存在(排除 raw 来源类、aliases、category)
|
||||
valid = allfiles | set(aliases_map.keys()) | set(synced)
|
||||
skip = {"People", "LLM Wiki", "wikilink"}
|
||||
@@ -540,10 +634,16 @@ def lint():
|
||||
print(f"\n[未入 index] {len(not_in_index)} 个(概念/实体未在跨库章节)")
|
||||
for s in not_in_index:
|
||||
print(f" - {s}")
|
||||
print(f"\n[缺 lifecycle 字段] {len(missing_lifecycle)} 个(Phase 1 必填:confidence/status/last_reviewed/review_interval_days)")
|
||||
for s in missing_lifecycle:
|
||||
print(f" - {s}")
|
||||
print(f"\n[无 relations] {len(no_relations)} 个(建议补充以激活图遍历)")
|
||||
for s in no_relations:
|
||||
print(f" - {s}")
|
||||
|
||||
issues = len(orphans) + len(empty_source) + len(not_in_index)
|
||||
print(f"\n硬性问题(孤儿+空source+未入index): {issues}")
|
||||
print(f"软性问题(断链,多为待创建概念/raw来源): {len(broken)}")
|
||||
issues = len(orphans) + len(empty_source) + len(not_in_index) + len(missing_lifecycle)
|
||||
print(f"\n硬性问题(孤儿+空source+未入index+缺lifecycle): {issues}")
|
||||
print(f"软性问题(断链/无relations,多为待创建概念/raw来源): {len(broken) + len(no_relations)}")
|
||||
return issues
|
||||
|
||||
|
||||
|
||||
@@ -65,15 +65,22 @@ def validate_one(fp: Path) -> list[str]:
|
||||
errors.append("Missing 'wiki' in tags")
|
||||
|
||||
# type must exist and be valid
|
||||
# Convention: type can be scalar (e.g. 'concept') or list (e.g. ['entity', 'People', 'Author'])
|
||||
# When list, the FIRST element is the canonical type; subsequent are role/subtype annotations
|
||||
ptype = front.get("type")
|
||||
if not ptype:
|
||||
errors.append("Missing type")
|
||||
elif isinstance(ptype, str) and ptype not in VALID_TYPES:
|
||||
errors.append(f"Invalid type: '{ptype}' (valid: {', '.join(VALID_TYPES)})")
|
||||
elif isinstance(ptype, str):
|
||||
if ptype not in VALID_TYPES:
|
||||
errors.append(f"Invalid type: '{ptype}' (valid: {', '.join(VALID_TYPES)})")
|
||||
elif isinstance(ptype, list):
|
||||
for t in ptype:
|
||||
if t not in VALID_TYPES:
|
||||
errors.append(f"Invalid type in list: '{t}'")
|
||||
if not ptype:
|
||||
errors.append("type list is empty")
|
||||
else:
|
||||
primary = ptype[0]
|
||||
if primary not in VALID_TYPES:
|
||||
errors.append(f"Invalid primary type: '{primary}' (valid: {', '.join(VALID_TYPES)})")
|
||||
# Role/subtype annotations (2nd+ elements) are not validated against VALID_TYPES
|
||||
|
||||
# source must exist
|
||||
if not front.get("source"):
|
||||
|
||||
Reference in New Issue
Block a user