Phase 0-2: Schema cleanup, typed relations, event-driven automation
- Phase 0: AGENTS.md cleanup (dedup quotes, renumber sections, merge qmd) - Phase 1: typed relations (manage-relations.py, graph-search.py, check-staleness.py, detect-conflicts.py) - Phase 2: frontmatter validator, weekly lint, knowledge promotion, git hooks - Fix .gitignore to track tools/ and .githooks/ - Fix git remote URL (remove plaintext token) - New wiki pages: 504 pages, 34 raw sources
This commit is contained in:
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
check-staleness.py — 检测 wiki 页面的时效性
|
||||
|
||||
检查每页的 last_reviewed 字段是否超过 review_interval_days,
|
||||
输出过时页面列表和从未审查的页面列表。
|
||||
|
||||
用法:
|
||||
python tools/scripts/check-staleness.py # 标准输出
|
||||
python tools/scripts/check-staleness.py --json # JSON 格式
|
||||
python tools/scripts/check-staleness.py --overdue-only # 仅过时页面
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
WIKI = Path(r"D:\Applications\app\kepano-obsidian-main\wiki")
|
||||
TODAY = date.today()
|
||||
DEFAULT_INTERVAL = 180
|
||||
|
||||
|
||||
def check_page(fp: Path) -> dict:
|
||||
"""检查单页,返回结果 dict"""
|
||||
result = {"page": fp.stem, "stale": False, "overdue_days": 0, "reason": ""}
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m:
|
||||
result["reason"] = "no_frontmatter"
|
||||
return result
|
||||
if not yaml:
|
||||
result["reason"] = "no_pyyaml"
|
||||
return result
|
||||
try:
|
||||
front = yaml.safe_load(m.group(1)) or {}
|
||||
except:
|
||||
result["reason"] = "yaml_parse_error"
|
||||
return result
|
||||
|
||||
last_str = front.get("last_reviewed", "")
|
||||
interval = front.get("review_interval_days", DEFAULT_INTERVAL)
|
||||
|
||||
if not last_str:
|
||||
result["reason"] = "never_reviewed"
|
||||
result["stale"] = True
|
||||
return result
|
||||
|
||||
try:
|
||||
last = date.fromisoformat(str(last_str))
|
||||
diff = (TODAY - last).days
|
||||
if diff > interval:
|
||||
result["stale"] = True
|
||||
result["overdue_days"] = diff - interval
|
||||
result["reason"] = f"overdue_by_{diff - interval}_days"
|
||||
except ValueError:
|
||||
result["reason"] = f"invalid_date_{last_str}"
|
||||
result["stale"] = True
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Check wiki page staleness")
|
||||
parser.add_argument("--json", action="store_true", help="Output JSON")
|
||||
parser.add_argument("--overdue-only", action="store_true", help="Only stale pages")
|
||||
args = parser.parse_args()
|
||||
|
||||
all_pages = sorted(WIKI.glob("*.md"))
|
||||
results = []
|
||||
for fp in all_pages:
|
||||
if fp.name in ("index.md", "log.md"):
|
||||
continue
|
||||
results.append(check_page(fp))
|
||||
|
||||
stale = [r for r in results if r["stale"]]
|
||||
never = [r for r in stale if r["reason"] == "never_reviewed"]
|
||||
overdue = [r for r in stale if r["reason"].startswith("overdue")]
|
||||
|
||||
if args.json:
|
||||
output = {
|
||||
"total": len(results),
|
||||
"stale": len(stale),
|
||||
"never_reviewed": len(never),
|
||||
"overdue": len(overdue),
|
||||
"pages": stale if args.overdue_only else results
|
||||
}
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
print(f"=== Staleness Check ({TODAY}) ===")
|
||||
print(f" Total: {len(results)} pages")
|
||||
print(f" Stale: {len(stale)}")
|
||||
print(f" Never reviewed: {len(never)}")
|
||||
print(f" Overdue: {len(overdue)}")
|
||||
|
||||
if overdue:
|
||||
print(f"\n Overdue pages (top 20):")
|
||||
for r in sorted(overdue, key=lambda x: -x["overdue_days"])[:20]:
|
||||
print(f" [[{r['page']}]] — overdue by {r['overdue_days']} days")
|
||||
|
||||
if never:
|
||||
print(f"\n Never reviewed (top 20):")
|
||||
for r in never[:20]:
|
||||
print(f" [[{r['page']}]]")
|
||||
if len(never) > 20:
|
||||
print(f" ... and {len(never)-20} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
detect-conflicts.py — 检测 wiki 页面间的矛盾
|
||||
|
||||
通过扫描 relations 中的 conflicts_with 关系,
|
||||
检查双方是否都引用了对方,并输出矛盾报告。
|
||||
|
||||
用法:
|
||||
python tools/scripts/detect-conflicts.py # 标准输出
|
||||
python tools/scripts/detect-conflicts.py --json # JSON 格式
|
||||
python tools/scripts/detect-conflicts.py --auto-callout # 自动添加 callout
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
WIKI = Path(r"D:\Applications\app\kepano-obsidian-main\wiki")
|
||||
|
||||
|
||||
def read_frontmatter(fp: Path) -> dict | None:
|
||||
"""读取 frontmatter,返回 dict 或 None"""
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m or not yaml:
|
||||
return None
|
||||
try:
|
||||
return yaml.safe_load(m.group(1)) or {}
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
def detect():
|
||||
"""检测所有 conflicts_with 关系,返回冲突报告列表"""
|
||||
conflicts = []
|
||||
pages = {}
|
||||
|
||||
for fp in WIKI.glob("*.md"):
|
||||
if fp.name in ("index.md", "log.md"):
|
||||
continue
|
||||
front = read_frontmatter(fp)
|
||||
if front is None:
|
||||
continue
|
||||
pages[fp.stem] = front
|
||||
for rel in front.get("relations", []):
|
||||
if rel["type"] == "conflicts_with":
|
||||
target = rel.get("target", "").strip("[]")
|
||||
conf_source = rel.get("confidence", 3)
|
||||
if target and target != fp.stem:
|
||||
conflicts.append({
|
||||
"source": fp.stem,
|
||||
"target": target,
|
||||
"confidence": conf_source,
|
||||
"bidirectional": False
|
||||
})
|
||||
|
||||
# 检查双向性
|
||||
for c in conflicts:
|
||||
target_front = pages.get(c["target"])
|
||||
if target_front:
|
||||
for rel in target_front.get("relations", []):
|
||||
if rel["type"] == "conflicts_with" and rel.get("target", "").strip("[]") == c["source"]:
|
||||
c["bidirectional"] = True
|
||||
break
|
||||
|
||||
return conflicts
|
||||
|
||||
|
||||
def generate_callout(source: str, target: str) -> str:
|
||||
return (
|
||||
f"> [!WARNING] 可能矛盾\n"
|
||||
f"> 本页的声明与 [[{target}]] 存在冲突。\n"
|
||||
f"> 需要人工复核并解决矛盾。\n"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Detect conflicts between wiki pages")
|
||||
parser.add_argument("--json", action="store_true", help="Output JSON")
|
||||
parser.add_argument("--auto-callout", action="store_true", help="Auto-add callout to pages")
|
||||
args = parser.parse_args()
|
||||
|
||||
conflicts = detect()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(conflicts, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
bidirectional = [c for c in conflicts if c["bidirectional"]]
|
||||
unidirectional = [c for c in conflicts if not c["bidirectional"]]
|
||||
|
||||
print(f"=== Conflict Detection ===")
|
||||
print(f" Total conflict declarations: {len(conflicts)}")
|
||||
print(f" Bidirectional (both confirm): {len(bidirectional)}")
|
||||
print(f" Unidirectional (check needed): {len(unidirectional)}")
|
||||
|
||||
if bidirectional:
|
||||
print(f"\n Bidirectional conflicts:")
|
||||
for c in bidirectional:
|
||||
print(f" [[{c['source']}]] <--conflicts_with--> [[{c['target']}]]")
|
||||
|
||||
if unidirectional:
|
||||
print(f"\n Unidirectional conflicts (may need callout):")
|
||||
for c in unidirectional:
|
||||
print(f" [[{c['source']}]] --conflicts_with--> [[{c['target']}]] (unconfirmed)")
|
||||
|
||||
if args.auto_callout:
|
||||
added = 0
|
||||
for c in unidirectional:
|
||||
fp = WIKI / f"{c['source']}.md"
|
||||
if not fp.exists():
|
||||
continue
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
callout = generate_callout(c["source"], c["target"])
|
||||
# Only add if not already present
|
||||
if callout.strip() not in content:
|
||||
content += f"\n\n{callout}"
|
||||
fp.write_text(content, encoding="utf-8")
|
||||
added += 1
|
||||
print(f" Added callout to [[{c['source']}]]")
|
||||
print(f" Callouts added: {added}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,140 @@
|
||||
@echo off
|
||||
REM ====================================================
|
||||
REM Marp 演示批量导出脚本 (Windows)
|
||||
REM ====================================================
|
||||
REM 用途:一键导出 Active 目录下所有演示
|
||||
REM 使用方法:
|
||||
REM 1. 确保已安装 Marp CLI: npm install -g @marp-team/marp-cli
|
||||
REM 2. 双击运行此脚本
|
||||
REM 3. 导出结果在 Export 目录
|
||||
REM ====================================================
|
||||
|
||||
setlocal EnableDelayedExpansion
|
||||
|
||||
REM 配置
|
||||
set "PRESENTATIONS_DIR=%~dp0Active"
|
||||
set "EXPORT_DIR=%~dp0Export"
|
||||
set "TIMESTAMP=%date:~0,4%-%date:~5,2%-%date:~8,2%"
|
||||
|
||||
REM 创建导出目录
|
||||
if not exist "%EXPORT_DIR%" mkdir "%EXPORT_DIR%"
|
||||
if not exist "%EXPORT_DIR%\PDF" mkdir "%EXPORT_DIR%\PDF"
|
||||
if not exist "%EXPORT_DIR%\PPTX" mkdir "%EXPORT_DIR%\PPTX"
|
||||
if not exist "%EXPORT_DIR%\HTML" mkdir "%EXPORT_DIR%\HTML"
|
||||
|
||||
echo ====================================================
|
||||
echo Marp 演示批量导出脚本
|
||||
echo ====================================================
|
||||
echo.
|
||||
echo 源目录: %PRESENTATIONS_DIR%
|
||||
echo 导出目录: %EXPORT_DIR%
|
||||
echo 时间戳: %TIMESTAMP%
|
||||
echo ====================================================
|
||||
echo.
|
||||
|
||||
REM 检查源目录是否存在
|
||||
if not exist "%PRESENTATIONS_DIR%" (
|
||||
echo [错误] 源目录不存在: %PRESENTATIONS_DIR%
|
||||
echo 请先在 Active 目录中创建演示文件
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
REM 检查是否安装了 Marp CLI
|
||||
where marp >nul 2>nul
|
||||
if %errorlevel% neq 0 (
|
||||
echo [警告] 未检测到 Marp CLI
|
||||
echo 正在尝试安装...
|
||||
echo.
|
||||
call npm install -g @marp-team/marp-cli
|
||||
if %errorlevel% neq 0 (
|
||||
echo [错误] Marp CLI 安装失败
|
||||
echo 请手动运行: npm install -g @marp-team/marp-cli
|
||||
pause
|
||||
exit /b 1
|
||||
)
|
||||
echo [成功] Marp CLI 安装完成
|
||||
echo.
|
||||
)
|
||||
|
||||
REM 计算文件数量
|
||||
set count=0
|
||||
for %%f in ("%PRESENTATIONS_DIR%\*.md") do set /a count+=1
|
||||
|
||||
if %count% equ 0 (
|
||||
echo [提示] Active 目录中没有 .md 文件
|
||||
echo 请先创建演示文件
|
||||
pause
|
||||
exit /b 0
|
||||
)
|
||||
|
||||
echo 找到 %count% 个演示文件
|
||||
echo 开始导出...
|
||||
echo.
|
||||
|
||||
REM 导出 PDF
|
||||
echo [1/3] 导出 PDF...
|
||||
for %%f in ("%PRESENTATIONS_DIR%\*.md") do (
|
||||
set "filename=%%~nf"
|
||||
echo - !filename!...
|
||||
marp "%%f" --theme-set-dir "%~dp0Themes" --pdf --allow-local-files -o "%EXPORT_DIR%\PDF\!filename:.md!.pdf"
|
||||
if !errorlevel! neq 0 (
|
||||
echo [失败] !filename!
|
||||
) else (
|
||||
echo [完成] !filename!
|
||||
)
|
||||
)
|
||||
|
||||
REM 导出 PPTX
|
||||
echo.
|
||||
echo [2/3] 导出 PPTX...
|
||||
for %%f in ("%PRESENTATIONS_DIR%\*.md") do (
|
||||
set "filename=%%~nf"
|
||||
echo - !filename!...
|
||||
marp "%%f" --theme-set-dir "%~dp0Themes" --pptx --allow-local-files -o "%EXPORT_DIR%\PPTX\!filename:.md!.pptx"
|
||||
if !errorlevel! neq 0 (
|
||||
echo [失败] !filename!
|
||||
) else (
|
||||
echo [完成] !filename!
|
||||
)
|
||||
)
|
||||
|
||||
REM 导出 HTML
|
||||
echo.
|
||||
echo [3/3] 导出 HTML...
|
||||
for %%f in ("%PRESENTATIONS_DIR%\*.md") do (
|
||||
set "filename=%%~nf"
|
||||
echo - !filename!...
|
||||
marp "%%f" --theme-set-dir "%~dp0Themes" --html --allow-local-files -o "%EXPORT_DIR%\HTML\!filename:.md!.html"
|
||||
if !errorlevel! neq 0 (
|
||||
echo [失败] !filename!
|
||||
) else (
|
||||
echo [完成] !filename!
|
||||
)
|
||||
)
|
||||
|
||||
echo.
|
||||
echo ====================================================
|
||||
echo 导出完成!
|
||||
echo ====================================================
|
||||
echo.
|
||||
echo 导出目录: %EXPORT_DIR%
|
||||
echo.
|
||||
echo 文件统计:
|
||||
if exist "%EXPORT_DIR%\PDF" (
|
||||
for /f %%a in ('dir /b "%EXPORT_DIR%\PDF\*.pdf" 2^>nul ^| find /c /v ""') do echo PDF: %%a 个
|
||||
)
|
||||
if exist "%EXPORT_DIR%\PPTX" (
|
||||
for /f %%a in ('dir /b "%EXPORT_DIR%\PPTX\*.pptx" 2^>nul ^| find /c /v ""') do echo PPTX: %%a 个
|
||||
)
|
||||
if exist "%EXPORT_DIR%\HTML" (
|
||||
for /f %%a in ('dir /b "%EXPORT_DIR%\HTML\*.html" 2^>nul ^| find /c /v ""') do echo HTML: %%a 个
|
||||
)
|
||||
echo.
|
||||
echo 打开导出目录? (Y/N)
|
||||
choice /c yn /n /m "请选择"
|
||||
if %errorlevel% equ 1 (
|
||||
start "" "%EXPORT_DIR%"
|
||||
)
|
||||
echo.
|
||||
pause
|
||||
@@ -0,0 +1,10 @@
|
||||
# Fix Git Remote Configuration Script
|
||||
|
||||
# Remove incorrect origin
|
||||
git remote remove origin
|
||||
|
||||
# Add correct origin without trailing slash
|
||||
git remote add origin https://hehaiguang1123:wYux3evdzMCzSYZKmHg4v55WxW322TYZ@git.haiguang.xyz/giteah/llm_wiki.git
|
||||
|
||||
# Push to remote
|
||||
git push origin main
|
||||
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
fix-raw-citations.py — 为 wiki/ 页面中缺失行号的 [raw:filename] 引用
|
||||
自动从原始 raw/ 文件查找匹配文本并补上行号范围。
|
||||
|
||||
用法:
|
||||
python tools/scripts/fix-raw-citations.py # 补行号
|
||||
python tools/scripts/fix-raw-citations.py --stats # 仅统计不修改
|
||||
python tools/scripts/fix-raw-citations.py --dry-run # 预览修改
|
||||
|
||||
退出码:0 成功;1 有错误。
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||||
WIKI = VAULT / "wiki"
|
||||
RAW = VAULT / "raw"
|
||||
|
||||
|
||||
def collect_wiki_files():
|
||||
"""收集 wiki/ 中需要处理的 .md 文件(排除 index.md, log.md)"""
|
||||
files = []
|
||||
for f in sorted(WIKI.glob("*.md")):
|
||||
if f.name not in ("index.md", "log.md"):
|
||||
files.append(f)
|
||||
return files
|
||||
|
||||
|
||||
def collect_aw_references(content: str):
|
||||
"""提取所有 [raw:...] 引用,返回 (match_obj, raw_filename, has_line_number)"""
|
||||
pattern = r'\[raw:([^:\]]+)(?::(\d+(?:-\d+)?))?\]'
|
||||
refs = []
|
||||
for m in re.finditer(pattern, content):
|
||||
fname = m.group(1)
|
||||
has_ln = m.group(2) is not None
|
||||
refs.append((m, fname, has_ln))
|
||||
return refs
|
||||
|
||||
|
||||
def extract_keyword(content: str, pos: int, max_chars: int = 60) -> str:
|
||||
"""
|
||||
从 content 中 pos 位置向前提取关键词(中文/英文/数字)。
|
||||
返回用于在 raw 文件中匹配的文本片段。
|
||||
"""
|
||||
start = max(0, pos - max_chars)
|
||||
before = content[start:pos]
|
||||
tokens = re.findall(r'[\u4e00-\u9fff\w]+', before)
|
||||
# 取最后 3-6 个 token 作为关键词
|
||||
return " ".join(tokens[-6:]) if len(tokens) >= 3 else " ".join(tokens)
|
||||
|
||||
|
||||
def find_line_range(raw_text: str, keyword: str, context: int = 3) -> str | None:
|
||||
"""
|
||||
在 raw_text 中搜索 keyword,返回匹配行所在的行号范围。
|
||||
格式: "start-end" 或 "line"(单行匹配)。
|
||||
返回 None 表示未找到。
|
||||
"""
|
||||
if not keyword:
|
||||
return None
|
||||
lines = raw_text.splitlines()
|
||||
matched_lines = set()
|
||||
for i, line in enumerate(lines, 1):
|
||||
if keyword in line:
|
||||
matched_lines.add(i)
|
||||
if not matched_lines:
|
||||
# fallback: try individual tokens
|
||||
tokens = keyword.split()
|
||||
for token in tokens:
|
||||
if len(token) < 2:
|
||||
continue
|
||||
for i, line in enumerate(lines, 1):
|
||||
if token in line:
|
||||
matched_lines.add(i)
|
||||
if not matched_lines:
|
||||
return None
|
||||
start = max(1, min(matched_lines) - context)
|
||||
end = min(len(lines), max(matched_lines) + context)
|
||||
if start == end:
|
||||
return str(start)
|
||||
return f"{start}-{end}"
|
||||
|
||||
|
||||
def process_file(filepath: Path, dry_run: bool = False) -> tuple[str, int, int]:
|
||||
"""
|
||||
处理单个文件。
|
||||
返回: (修改后的内容, 补行号数, 总引用数)
|
||||
"""
|
||||
content = filepath.read_text(encoding="utf-8")
|
||||
refs = collect_aw_references(content)
|
||||
total = len(refs)
|
||||
fixed = 0
|
||||
|
||||
if total == 0:
|
||||
return content, 0, 0
|
||||
|
||||
raw_cache = {}
|
||||
# 从后往前替换以保持 offsets
|
||||
for m, fname, has_ln in reversed(refs):
|
||||
if has_ln:
|
||||
continue # 已有行号,跳过
|
||||
# 加载 raw 文件
|
||||
if fname not in raw_cache:
|
||||
rpath = RAW / f"{fname}.md"
|
||||
if rpath.exists():
|
||||
raw_cache[fname] = rpath.read_text(encoding="utf-8")
|
||||
else:
|
||||
# 尝试模糊匹配(取文件名最后一段)
|
||||
candidates = list(RAW.glob(f"*{fname}*.md"))
|
||||
if candidates:
|
||||
raw_cache[fname] = candidates[0].read_text(encoding="utf-8")
|
||||
else:
|
||||
raw_cache[fname] = None
|
||||
raw_text = raw_cache.get(fname)
|
||||
if raw_text is None:
|
||||
continue
|
||||
|
||||
# 提取关键词
|
||||
keyword = extract_keyword(content, m.start())
|
||||
line_range = find_line_range(raw_text, keyword)
|
||||
if line_range:
|
||||
old = m.group(0)
|
||||
new = f"[raw:{fname}:{line_range}]"
|
||||
content = content[:m.start()] + new + content[m.end():]
|
||||
fixed += 1
|
||||
|
||||
return content, fixed, total
|
||||
|
||||
|
||||
def stats_only():
|
||||
"""仅统计行号覆盖率"""
|
||||
files = collect_wiki_files()
|
||||
total_refs = 0
|
||||
total_with_ln = 0
|
||||
total_missing_ln = 0
|
||||
per_file = []
|
||||
|
||||
for fp in files:
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
refs = collect_aw_references(content)
|
||||
total_refs += len(refs)
|
||||
with_ln = sum(1 for _, _, has_ln in refs if has_ln)
|
||||
missing = len(refs) - with_ln
|
||||
total_with_ln += with_ln
|
||||
total_missing_ln += missing
|
||||
if missing > 0:
|
||||
per_file.append((fp.name, missing, with_ln))
|
||||
|
||||
print(f"\n=== 行号标注覆盖率统计 ===")
|
||||
print(f" 文件数: {len(files)}")
|
||||
print(f" 总引用: {total_refs}")
|
||||
print(f" 有行号: {total_with_ln} ({total_with_ln / total_refs * 100:.1f}%)")
|
||||
print(f" 缺行号: {total_missing_ln} ({total_missing_ln / total_refs * 100:.1f}%)")
|
||||
if per_file:
|
||||
print(f"\n 缺行号的文件 (top 20):")
|
||||
for name, miss, have in sorted(per_file, key=lambda x: -x[1])[:20]:
|
||||
print(f" {name}: 缺 {miss} / 共 {miss + have}")
|
||||
return total_missing_ln
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="为 wiki/ 的 raw 引用补上行号")
|
||||
parser.add_argument("--stats", action="store_true", help="仅统计不修改")
|
||||
parser.add_argument("--dry-run", action="store_true", help="预览修改但不写入")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.stats:
|
||||
stats_only()
|
||||
return
|
||||
|
||||
files = collect_wiki_files()
|
||||
total_fixed = 0
|
||||
total_refs = 0
|
||||
changed_files = []
|
||||
|
||||
for fp in files:
|
||||
content, fixed, refs = process_file(fp, dry_run=args.dry_run)
|
||||
total_fixed += fixed
|
||||
total_refs += refs
|
||||
if fixed > 0:
|
||||
changed_files.append((fp.name, fixed, refs))
|
||||
if not args.dry_run:
|
||||
fp.write_text(content, encoding="utf-8")
|
||||
|
||||
print(f"\n=== 处理结果 ===")
|
||||
print(f" 处理文件: {len(files)}")
|
||||
print(f" 总引用数: {total_refs}")
|
||||
print(f" 补行号数: {total_fixed}")
|
||||
if total_refs > 0:
|
||||
print(f" 覆盖率: {(total_refs - total_fixed + total_fixed) / total_refs * 100:.1f}% → "
|
||||
f"{total_refs / total_refs * 100:.1f}% (理论上限,行号仅补匹配到的)")
|
||||
if changed_files:
|
||||
print(f"\n 更新文件 ({len(changed_files)}):")
|
||||
for name, fixed, refs in sorted(changed_files, key=lambda x: -x[1]):
|
||||
print(f" {name}: +{fixed} 行号")
|
||||
if args.dry_run and changed_files:
|
||||
print(f"\n 以上为预览,未写入文件(--dry-run)")
|
||||
|
||||
print(f"\n Total: {total_fixed} citations updated")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
graph-search.py — 图感知搜索
|
||||
|
||||
流程:
|
||||
1. qmd vsearch 获取语义匹配结果(Top 10)
|
||||
2. 对每个结果,读取 relations 字段 → 获取相邻节点
|
||||
3. 去重后返回(直接关联 + 关系扩展)
|
||||
|
||||
用法:
|
||||
python tools/scripts/graph-search.py "查询词" [--count 15] [--depth 1]
|
||||
|
||||
依赖:
|
||||
- qmd 已安装(通过 node)
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
QMD = r'node "C:\Users\hhhh2024\AppData\Roaming\npm\node_modules\@tobilu\qmd\dist\cli\qmd.js"'
|
||||
WIKI = Path(r"D:\Applications\app\kepano-obsidian-main\wiki")
|
||||
|
||||
|
||||
def qmd_search(query: str, count: int = 10) -> list[str]:
|
||||
"""调用 qmd vsearch 获取匹配的页面名列表"""
|
||||
cmd = f'{QMD} vsearch "{query}" -c wiki -n {count}'
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, shell=True, timeout=30)
|
||||
lines = [line.strip() for line in result.stdout.splitlines() if line.strip()]
|
||||
return lines
|
||||
|
||||
|
||||
def get_relations(page_name: str) -> list[dict]:
|
||||
"""读取页面的 relations 字段"""
|
||||
fp = WIKI / f"{page_name}.md"
|
||||
if not fp.exists():
|
||||
return []
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m:
|
||||
return []
|
||||
if not yaml:
|
||||
return []
|
||||
try:
|
||||
front = yaml.safe_load(m.group(1)) or {}
|
||||
except:
|
||||
return []
|
||||
return front.get("relations", [])
|
||||
|
||||
|
||||
def expand_nodes(nodes: list[str], depth: int = 1) -> list[str]:
|
||||
"""从起始节点出发,沿 relations 扩展相邻节点"""
|
||||
expanded = list(nodes)
|
||||
frontier = list(nodes)
|
||||
for _ in range(depth):
|
||||
next_frontier = []
|
||||
for node in frontier:
|
||||
rels = get_relations(node)
|
||||
for r in rels:
|
||||
target = r.get("target", "").strip("[]")
|
||||
if target and target not in expanded:
|
||||
expanded.append(target)
|
||||
next_frontier.append(target)
|
||||
frontier = next_frontier
|
||||
if not frontier:
|
||||
break
|
||||
return expanded
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Graph-aware search for wiki pages")
|
||||
parser.add_argument("query", help="Search query")
|
||||
parser.add_argument("--count", type=int, default=15, help="Max results")
|
||||
parser.add_argument("--depth", type=int, default=1, help="Graph expansion depth")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"=== Searching: {args.query} ===")
|
||||
results = qmd_search(args.query, count=max(10, args.count))
|
||||
if not results:
|
||||
print(" No results from qmd")
|
||||
return
|
||||
|
||||
print(f"\nDirect matches ({len(results)}):")
|
||||
for r in results:
|
||||
print(f" [[{r}]]")
|
||||
|
||||
expanded = expand_nodes(results, depth=args.depth)
|
||||
new = [e for e in expanded if e not in results]
|
||||
if new:
|
||||
print(f"\nGraph-expanded (via relations, depth={args.depth}):")
|
||||
for n in new[:args.count]:
|
||||
print(f" [[{n}]]")
|
||||
if len(new) > args.count:
|
||||
print(f" ... and {len(new) - args.count} more")
|
||||
|
||||
print(f"\nTotal unique: {len(expanded)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
manage-relations.py — 管理 wiki 页面的 typed relationships
|
||||
|
||||
用法:
|
||||
python manage-relations.py list <page> # 查看页面 relations
|
||||
python manage-relations.py add <page> --type <type> --target "页面" [--desc "说明"] [--conf 3]
|
||||
python manage-relations.py remove <page> --type <type> --target "页面"
|
||||
python manage-relations.py graph <page> # 输出 DOT 格式
|
||||
python manage-relations.py stats # 统计关系网络
|
||||
|
||||
关系类型:
|
||||
depends_on | conflicts_with | supersedes | caused_by | supports | extends | part_of | example_of
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
VAULT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||||
WIKI = VAULT / "wiki"
|
||||
|
||||
VALID_TYPES = [
|
||||
"depends_on", "conflicts_with", "supersedes", "caused_by",
|
||||
"supports", "extends", "part_of", "example_of"
|
||||
]
|
||||
|
||||
|
||||
def read_page(page: str) -> tuple[dict | None, str | None, str | None]:
|
||||
"""返回 (frontmatter_dict, body_text, error_msg)"""
|
||||
fp = WIKI / f"{page}.md"
|
||||
if not fp.exists():
|
||||
return None, None, f"ERROR: {page}.md not found"
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m:
|
||||
return None, None, f"ERROR: {page}.md has no frontmatter"
|
||||
body = content[m.end():]
|
||||
if yaml:
|
||||
try:
|
||||
front = yaml.safe_load(m.group(1)) or {}
|
||||
except yaml.YAMLError as e:
|
||||
return None, None, f"ERROR: YAML parse error: {e}"
|
||||
else:
|
||||
return None, None, "ERROR: PyYAML not installed (pip install pyyaml)"
|
||||
return front, body, None
|
||||
|
||||
|
||||
def write_page(page: str, front: dict, body: str) -> str | None:
|
||||
"""写入页面,返回 error_msg"""
|
||||
fp = WIKI / f"{page}.md"
|
||||
new_yaml = yaml.dump(front, allow_unicode=True, default_flow_style=False, sort_keys=False)
|
||||
fp.write_text(f"---\n{new_yaml}---\n{body.lstrip()}", encoding="utf-8")
|
||||
return None
|
||||
|
||||
|
||||
def cmd_list(page: str):
|
||||
front, body, err = read_page(page)
|
||||
if err:
|
||||
print(err); return
|
||||
rels = front.get("relations", [])
|
||||
if not rels:
|
||||
print(f" [[{page}]]: no relations")
|
||||
return
|
||||
print(f"[[{page}]] ({len(rels)} relations):")
|
||||
for r in rels:
|
||||
desc = f" — {r.get('description', '')}" if r.get("description") else ""
|
||||
conf = f" [conf={r.get('confidence', 3)}]" if r.get("confidence", 3) != 3 else ""
|
||||
target_name = r['target'].strip('[]')
|
||||
print(f" {r['type']} --> [[{target_name}]]{desc}{conf}")
|
||||
|
||||
|
||||
def cmd_add(page: str, rel_type: str, target: str, desc: str | None, conf: int):
|
||||
front, body, err = read_page(page)
|
||||
if err:
|
||||
print(err); return
|
||||
if "relations" not in front:
|
||||
front["relations"] = []
|
||||
# Dedup
|
||||
for r in front["relations"]:
|
||||
if r.get("type") == rel_type and r.get("target") == target:
|
||||
print(f" Already exists: {rel_type} [[{target}]]")
|
||||
return
|
||||
entry = {"type": rel_type, "target": f"[[{target}]]"}
|
||||
if desc:
|
||||
entry["description"] = desc
|
||||
if conf < 5:
|
||||
entry["confidence"] = conf
|
||||
front["relations"].append(entry)
|
||||
err = write_page(page, front, body)
|
||||
if err:
|
||||
print(err)
|
||||
else:
|
||||
print(f" Added: [[{page}]] --{rel_type}--> [[{target}]]")
|
||||
|
||||
|
||||
def cmd_remove(page: str, rel_type: str, target: str):
|
||||
front, body, err = read_page(page)
|
||||
if err:
|
||||
print(err); return
|
||||
rels = front.get("relations", [])
|
||||
before = len(rels)
|
||||
front["relations"] = [
|
||||
r for r in rels
|
||||
if not (r.get("type") == rel_type and r.get("target", "").strip("[]") == target)
|
||||
]
|
||||
if len(front["relations"]) == before:
|
||||
print(f" Not found: {rel_type} [[{target}]]")
|
||||
return
|
||||
if not front["relations"]:
|
||||
del front["relations"]
|
||||
err = write_page(page, front, body)
|
||||
if err:
|
||||
print(err)
|
||||
else:
|
||||
print(f" Removed: [[{page}]] --{rel_type}--> [[{target}]]")
|
||||
|
||||
|
||||
def cmd_graph(page: str):
|
||||
"""输出 DOT 格式"""
|
||||
front, body, err = read_page(page)
|
||||
if err:
|
||||
print(err); return
|
||||
rels = front.get("relations", [])
|
||||
print(f"digraph {page} {{")
|
||||
print(f' "{page}" [style=filled, fillcolor=lightblue];')
|
||||
for r in rels:
|
||||
target = r["target"].strip("[]")
|
||||
label = r["type"]
|
||||
print(f' "{page}" -> "{target}" [label="{label}"];')
|
||||
print("}")
|
||||
|
||||
|
||||
def cmd_stats():
|
||||
"""统计整个 wiki 的关系网络"""
|
||||
all_pages = list(WIKI.glob("*.md"))
|
||||
total = 0
|
||||
type_counts = {}
|
||||
pages_with_rels = 0
|
||||
for fp in all_pages:
|
||||
if fp.name in ("index.md", "log.md"):
|
||||
continue
|
||||
front, _, err = read_page(fp.stem)
|
||||
if err or not front:
|
||||
continue
|
||||
rels = front.get("relations", [])
|
||||
if rels:
|
||||
pages_with_rels += 1
|
||||
total += len(rels)
|
||||
for r in rels:
|
||||
t = r["type"]
|
||||
type_counts[t] = type_counts.get(t, 0) + 1
|
||||
|
||||
print(f"=== Relations Stats ===")
|
||||
print(f" Total wiki pages: {len(all_pages) - 2}")
|
||||
print(f" Pages with relations: {pages_with_rels}")
|
||||
print(f" Total relations: {total}")
|
||||
print(f" By type:")
|
||||
for t, c in sorted(type_counts.items(), key=lambda x: -x[1]):
|
||||
print(f" {t}: {c}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Manage typed relationships in wiki pages")
|
||||
sub = parser.add_subparsers(dest="cmd")
|
||||
|
||||
p_list = sub.add_parser("list", help="List relations of a page")
|
||||
p_list.add_argument("page")
|
||||
|
||||
p_add = sub.add_parser("add", help="Add a relation")
|
||||
p_add.add_argument("page")
|
||||
p_add.add_argument("--type", required=True, choices=VALID_TYPES)
|
||||
p_add.add_argument("--target", required=True)
|
||||
p_add.add_argument("--desc")
|
||||
p_add.add_argument("--conf", type=int, default=3)
|
||||
|
||||
p_rm = sub.add_parser("remove", help="Remove a relation")
|
||||
p_rm.add_argument("page")
|
||||
p_rm.add_argument("--type", required=True, choices=VALID_TYPES)
|
||||
p_rm.add_argument("--target", required=True)
|
||||
|
||||
p_g = sub.add_parser("graph", help="Output DOT graph")
|
||||
p_g.add_argument("page")
|
||||
|
||||
p_s = sub.add_parser("stats", help="Stats of the relation network")
|
||||
|
||||
args = parser.parse_args()
|
||||
if not args.cmd:
|
||||
parser.print_help(); return
|
||||
|
||||
if args.cmd == "list":
|
||||
cmd_list(args.page)
|
||||
elif args.cmd == "add":
|
||||
cmd_add(args.page, args.type, args.target, args.desc, args.conf)
|
||||
elif args.cmd == "remove":
|
||||
cmd_remove(args.page, args.type, args.target)
|
||||
elif args.cmd == "graph":
|
||||
cmd_graph(args.page)
|
||||
elif args.cmd == "stats":
|
||||
cmd_stats()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
promote-knowledge.py — Consolidation tiers: 检查 wiki/working/ 中的页面
|
||||
是否可以提升到 wiki/semantic/(即主 wiki 目录)。
|
||||
|
||||
条件:
|
||||
- 有完整的 frontmatter(categories, tags, type, source)
|
||||
- 正文 > 100 字
|
||||
- 创建时间 > 7 天(通过 created 字段判断)
|
||||
|
||||
用法:
|
||||
python tools/scripts/promote-knowledge.py # 检查可提升页面
|
||||
python tools/scripts/promote-knowledge.py --apply # 执行提升
|
||||
python tools/scripts/promote-knowledge.py --dry-run # 预览
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
WIKI = Path(r"D:\Applications\app\kepano-obsidian-main\wiki")
|
||||
WORKING = WIKI / "working"
|
||||
SEMANTIC = WIKI # semantic 层就是当前 wiki/
|
||||
ARCHIVE = WIKI / "archive"
|
||||
PROCEDURAL = WIKI / "procedural"
|
||||
|
||||
|
||||
def ensure_dirs():
|
||||
"""确保层目录存在"""
|
||||
for d in [WORKING, ARCHIVE, PROCEDURAL]:
|
||||
d.mkdir(exist_ok=True)
|
||||
|
||||
|
||||
def check_promotable(fp: Path) -> tuple[bool, list[str]]:
|
||||
"""检查文件是否可提升"""
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m:
|
||||
return False, ["No frontmatter"]
|
||||
if not yaml:
|
||||
return False, ["PyYAML not installed"]
|
||||
try:
|
||||
front = yaml.safe_load(m.group(1)) or {}
|
||||
except:
|
||||
return False, ["YAML parse error"]
|
||||
|
||||
body = content[m.end():].strip()
|
||||
reasons = []
|
||||
|
||||
if not front.get("categories"):
|
||||
reasons.append("Missing categories")
|
||||
tags = front.get("tags", [])
|
||||
if isinstance(tags, str): tags = [tags]
|
||||
if "wiki" not in tags:
|
||||
reasons.append("Missing 'wiki' in tags")
|
||||
if not front.get("type"):
|
||||
reasons.append("Missing type")
|
||||
if not front.get("source"):
|
||||
reasons.append("Missing source")
|
||||
if len(body) < 100:
|
||||
reasons.append(f"Body too short ({len(body)} chars, need 100+)")
|
||||
created = front.get("created")
|
||||
if created:
|
||||
try:
|
||||
cdate = date.fromisoformat(str(created))
|
||||
if (date.today() - cdate).days < 7:
|
||||
reasons.append(f"Created < 7 days ago ({created})")
|
||||
except:
|
||||
pass # ignore invalid dates
|
||||
|
||||
return len(reasons) == 0, reasons
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Knowledge consolidation tiers")
|
||||
parser.add_argument("--apply", action="store_true", help="Execute promotion")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Preview only")
|
||||
args = parser.parse_args()
|
||||
|
||||
ensure_dirs()
|
||||
|
||||
if not WORKING.exists():
|
||||
print("No working/ directory found. Nothing to promote.")
|
||||
return
|
||||
|
||||
working_files = sorted(WORKING.glob("*.md"))
|
||||
if not working_files:
|
||||
print("No files in working/")
|
||||
return
|
||||
|
||||
promotable = []
|
||||
not_ready = []
|
||||
|
||||
for fp in working_files:
|
||||
ready, reasons = check_promotable(fp)
|
||||
if ready:
|
||||
promotable.append(fp)
|
||||
else:
|
||||
not_ready.append((fp, reasons))
|
||||
|
||||
print(f"=== Knowledge Promotion Check ===")
|
||||
print(f" Working files: {len(working_files)}")
|
||||
print(f" Promotable: {len(promotable)}")
|
||||
print(f" Not ready: {len(not_ready)}")
|
||||
|
||||
if promotable:
|
||||
print(f"\n Promotable to semantic/:")
|
||||
for fp in promotable:
|
||||
dest = SEMANTIC / fp.name
|
||||
if dest.exists():
|
||||
print(f" [[{fp.stem}]] → WARNING: target exists")
|
||||
else:
|
||||
print(f" [[{fp.stem}]]")
|
||||
|
||||
if args.apply and not args.dry_run:
|
||||
for fp in promotable:
|
||||
dest = SEMANTIC / fp.name
|
||||
if not dest.exists():
|
||||
shutil.move(str(fp), str(dest))
|
||||
print(f" Moved: working/{fp.name} → {fp.name}")
|
||||
print(f"\n Promoted: {len(promotable)}")
|
||||
elif args.dry_run:
|
||||
print(f"\n (--dry-run: no files moved)")
|
||||
|
||||
if not_ready:
|
||||
print(f"\n Not ready for promotion:")
|
||||
for fp, reasons in not_ready:
|
||||
for r in reasons:
|
||||
print(f" [[{fp.stem}]] — {r}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
set QMD_EMBED_MODEL=hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf
|
||||
node "C:\Users\hhhh2024\AppData\Roaming\npm\node_modules\@tobilu\qmd\dist\cli\qmd.js" %*
|
||||
@@ -0,0 +1,35 @@
|
||||
$files = Get-ChildItem -Path '.' -Filter '*.md' -Recurse
|
||||
$totalLines = 0
|
||||
$totalChars = 0
|
||||
|
||||
foreach ($file in $files) {
|
||||
$content = Get-Content $file.FullName -Raw -ErrorAction SilentlyContinue
|
||||
if ($content) {
|
||||
$lines = ($content -split "`n").Count
|
||||
$chars = $content.Length
|
||||
$totalLines += $lines
|
||||
$totalChars += $chars
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output "Total files: $($files.Count)"
|
||||
Write-Output "Total lines: $totalLines"
|
||||
Write-Output "Total characters: $totalChars"
|
||||
Write-Output "Avg characters per file: $([math]::Round($totalChars/$files.Count))"
|
||||
Write-Output "Avg lines per file: $([math]::Round($totalLines/$files.Count))"
|
||||
|
||||
# Categorize files
|
||||
$templates = Get-ChildItem -Path './Templates' -Filter '*.md' -Recurse
|
||||
$daily = Get-ChildItem -Path './Daily' -Filter '*.md' -Recurse
|
||||
$notes = Get-ChildItem -Path './Notes' -Filter '*.md' -Recurse
|
||||
$clippings = Get-ChildItem -Path './Clippings' -Filter '*.md' -Recurse
|
||||
$references = Get-ChildItem -Path './References' -Filter '*.md' -Recurse
|
||||
$categories = Get-ChildItem -Path './Categories' -Filter '*.md' -Recurse
|
||||
|
||||
Write-Output "`nBreakdown by category:"
|
||||
Write-Output "Templates: $($templates.Count)"
|
||||
Write-Output "Daily notes: $($daily.Count)"
|
||||
Write-Output "Notes: $($notes.Count)"
|
||||
Write-Output "Clippings: $($clippings.Count)"
|
||||
Write-Output "References: $($references.Count)"
|
||||
Write-Output "Categories: $($categories.Count)"
|
||||
@@ -0,0 +1,612 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
sync_home_wiki.py — home-wiki 知识页面 → kepano LLM Wiki 同步脚本
|
||||
|
||||
用途(定期执行):
|
||||
将 D:\\TC_UP\\home-wiki 的 concepts/ entities/ syntheses/ 知识页面,
|
||||
同步到 D:\\Applications\\app\\kepano-obsidian-main\\wiki\\,
|
||||
执行 frontmatter 规范转换,幂等更新;可选自动维护 index.md 与 lint 体检。
|
||||
|
||||
同步范围:
|
||||
concepts/ + entities/ + syntheses/ (排除 index.md)
|
||||
Daily/ MyNotes/ reports/ 不在知识同步范围。
|
||||
|
||||
转换规则(home-wiki frontmatter → kepano Wiki 规范):
|
||||
categories: 加 [[LLM Wiki]];entity/person 额外加 [[People]]
|
||||
tags: 前缀 [wiki, {people|concept}],subtype→concept/{subtype}(仅 concept),追加原 tags
|
||||
created: 保留
|
||||
source: sources[0] → 文件路径取 stem 做 wikilink;URL/标识符原样;空则留空
|
||||
type: 保留(concept/entity/synthesis)
|
||||
aliases: title 或 name
|
||||
正文: 保留,移除 home-wiki 特有的 openclaw 自动段落(## Related、注释标记行)
|
||||
|
||||
index 自动维护(--index):
|
||||
用标记块 <!-- BEGIN/END home-wiki-sync --> 界定 index.md 的「概念页」「实体页」
|
||||
两个表,脚本依据同步页面元数据 + 简介 JSON 缓存自动重生成。
|
||||
简介缓存 tools/data/home-wiki-summaries.json:首次从 index.md 现有表格种子化,
|
||||
保留人工优化;新增页面自动提取(description → 顶部引用块 → 概述首句)。
|
||||
综合报告表因混合其他来源,保持手动维护。
|
||||
|
||||
用法:
|
||||
python tools/scripts/sync_home_wiki.py --dry-run # 仅诊断差异
|
||||
python tools/scripts/sync_home_wiki.py # 同步(写入变化的页面)
|
||||
python tools/scripts/sync_home_wiki.py --index --log # 一站式:同步+更新index+记日志
|
||||
python tools/scripts/sync_home_wiki.py --lint # 体检(孤儿/断链/source空值)
|
||||
|
||||
退出码:0 成功;1 源目录缺失;2 有错误。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
sys.stderr.write("ERROR: PyYAML 未安装,请运行 pip install pyyaml\n")
|
||||
sys.exit(2)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 路径配置(两库位于不同位置,必须使用绝对路径)
|
||||
# ---------------------------------------------------------------------------
|
||||
SRC_ROOT = Path(r"D:\TC_UP\home-wiki")
|
||||
DST_WIKI = Path(r"D:\Applications\app\kepano-obsidian-main\wiki")
|
||||
DST_INDEX = DST_WIKI / "index.md"
|
||||
DST_LOG = DST_WIKI / "log.md"
|
||||
SUMMARIES_CACHE = Path(__file__).resolve().parent.parent / "data" / "home-wiki-summaries.json"
|
||||
|
||||
SRC_DIRS = ["concepts", "entities", "syntheses"]
|
||||
|
||||
# home-wiki 完整文件名 → kepano 简短名(仅含副标题、过长标题的需要映射)
|
||||
NAME_MAP = {
|
||||
"顶级人生三重境:道家驭势、佛家修心、儒家立身": "顶级人生三重境",
|
||||
}
|
||||
|
||||
INDEX_BEGIN = "<!-- BEGIN home-wiki-sync -->"
|
||||
INDEX_END = "<!-- END home-wiki-sync -->"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 文档解析
|
||||
# ---------------------------------------------------------------------------
|
||||
def _extract_list_field(text, key):
|
||||
"""从 frontmatter 文本中提取列表字段(兼容 flow 与 block 两种写法)。"""
|
||||
m = re.search(rf"^[ \t]*{re.escape(key)}[ \t]*:\s*\[(.*)\][ \t]*$", text, re.M)
|
||||
if m:
|
||||
return [t.strip().strip("\"'") for t in m.group(1).split(",") if t.strip()]
|
||||
m = re.search(rf"^[ \t]*{re.escape(key)}[ \t]*:\s*\n((?:[ \t]+-.+\n?)+)", text, re.M)
|
||||
if m:
|
||||
items = re.findall(r"^[ \t]+-[ \t]+(.+?)[ \t]*$", m.group(1), re.M)
|
||||
return [it.strip().strip("\"'") for it in items]
|
||||
return []
|
||||
|
||||
|
||||
def _robust_parse_fm(fm_text):
|
||||
"""健壮解析 frontmatter:先 yaml.safe_load,失败则正则逐字段回退。
|
||||
|
||||
home-wiki frontmatter 常含 `related: [[wikilink]]` 这类 Obsidian wikilink,
|
||||
`[[` 会触发 YAML flow sequence 解析异常,故需要回退。
|
||||
"""
|
||||
try:
|
||||
fm = yaml.safe_load(fm_text)
|
||||
if isinstance(fm, dict) and fm.get("type"):
|
||||
return fm
|
||||
except yaml.YAMLError:
|
||||
pass
|
||||
fm = {}
|
||||
for key in ("type", "subtype", "title", "name", "created", "updated",
|
||||
"confidence", "field", "nameEn", "description"):
|
||||
m = re.search(rf"^[ \t]*{re.escape(key)}[ \t]*:[ \t]*(.+?)[ \t]*$", fm_text, re.M)
|
||||
if m and m.group(1).strip() not in ("", "[]"):
|
||||
fm[key] = m.group(1).strip().strip("\"'")
|
||||
for key in ("tags", "sources"):
|
||||
vals = _extract_list_field(fm_text, key)
|
||||
if vals:
|
||||
fm[key] = vals
|
||||
return fm
|
||||
|
||||
|
||||
def parse_doc(text):
|
||||
"""分离 frontmatter 与正文。返回 (fm_dict, body_str)。"""
|
||||
m = re.match(r"^---\s*\n(.*?)\n---\s*\n?(.*)$", text, re.S)
|
||||
if not m:
|
||||
return {}, text
|
||||
return _robust_parse_fm(m.group(1)), m.group(2)
|
||||
|
||||
|
||||
def strip_openclaw(body):
|
||||
"""移除 home-wiki 特有的 openclaw 自动段落,保留实质内容。
|
||||
|
||||
- 移除 `## Related` 及其 openclaw:wiki:related 包裹块
|
||||
- 移除所有 `<!-- openclaw:...:...:(start|end) -->` 注释标记行(保留被包裹的内容)
|
||||
"""
|
||||
body = re.sub(
|
||||
r"\n##\s*Related\s*\n<!--\s*openclaw:wiki:related:start\s*-->.*?<!--\s*openclaw:wiki:related:end\s*-->",
|
||||
"\n",
|
||||
body,
|
||||
flags=re.S,
|
||||
)
|
||||
body = re.sub(r"<!--\s*openclaw:[a-z]+:[a-z]+:(?:start|end)\s*-->\s*\n?", "", body)
|
||||
return body.rstrip() + "\n"
|
||||
|
||||
|
||||
def norm_body(body):
|
||||
"""规范化正文用于对比:移除 openclaw 段,丢弃空行与行尾空白。"""
|
||||
b = strip_openclaw(body)
|
||||
lines = [ln.rstrip() for ln in b.splitlines() if ln.strip()]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# frontmatter 转换
|
||||
# ---------------------------------------------------------------------------
|
||||
def _as_list(v):
|
||||
if v is None:
|
||||
return []
|
||||
if isinstance(v, list):
|
||||
return v
|
||||
if isinstance(v, str):
|
||||
s = v.strip()
|
||||
if s.startswith("[") and s.endswith("]"):
|
||||
s = s[1:-1]
|
||||
return [t.strip() for t in s.split(",") if t.strip()]
|
||||
return [v]
|
||||
|
||||
|
||||
def convert_frontmatter(fm, default_name):
|
||||
"""home-wiki frontmatter → kepano Wiki frontmatter(有序)。"""
|
||||
fm_type = str(fm.get("type", "concept")).strip()
|
||||
subtype = fm.get("subtype")
|
||||
if isinstance(subtype, list):
|
||||
subtype = subtype[0] if subtype else None
|
||||
subtype = str(subtype).strip() if subtype else None
|
||||
title = fm.get("title") or fm.get("name") or default_name
|
||||
|
||||
categories = ["[[LLM Wiki]]"]
|
||||
type_tag = "concept"
|
||||
if fm_type == "entity":
|
||||
if subtype == "person":
|
||||
categories.append("[[People]]")
|
||||
type_tag = "people"
|
||||
else:
|
||||
type_tag = "entity"
|
||||
elif fm_type == "synthesis":
|
||||
type_tag = "concept"
|
||||
|
||||
tags = ["wiki", type_tag]
|
||||
# 仅 concept 类型把 subtype 转为 concept/{subtype} 标签
|
||||
# (entity/person 用 people 标签 + [[People]] category,不加 concept/person)
|
||||
if subtype and fm_type == "concept":
|
||||
tags.append(f"concept/{subtype}")
|
||||
for t in _as_list(fm.get("tags")):
|
||||
t = str(t).strip()
|
||||
if t and t not in tags:
|
||||
tags.append(t)
|
||||
|
||||
sources = _as_list(fm.get("sources"))
|
||||
src0 = str(sources[0]) if sources else ""
|
||||
if ("/" in src0 or "\\" in src0) and not src0.startswith("http"):
|
||||
src_val = "[[" + Path(src0).stem + "]]"
|
||||
elif src0:
|
||||
src_val = src0
|
||||
else:
|
||||
src_val = ""
|
||||
|
||||
return [
|
||||
("categories", categories),
|
||||
("tags", tags),
|
||||
("created", str(fm.get("created", ""))),
|
||||
("source", src_val),
|
||||
("type", fm_type),
|
||||
("aliases", [str(title)]),
|
||||
]
|
||||
|
||||
|
||||
def _yaml_quote(val):
|
||||
"""如果值包含 YAML 特殊字符(如 [[ wikilink 的方括号),加双引号。"""
|
||||
s = str(val)
|
||||
if not s:
|
||||
return '""'
|
||||
if '[' in s or ']' in s or '{' in s or '}' in s or ':' in s or '#' in s:
|
||||
return f'"{s}"'
|
||||
return s
|
||||
|
||||
|
||||
def dump_frontmatter(pairs):
|
||||
out = ["---"]
|
||||
for k, v in pairs:
|
||||
if isinstance(v, list):
|
||||
out.append(f"{k}:")
|
||||
for item in v:
|
||||
out.append(f" - {_yaml_quote(item)}")
|
||||
else:
|
||||
out.append(f"{k}: {_yaml_quote(v)}")
|
||||
out.append("---")
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 同步主流程
|
||||
# ---------------------------------------------------------------------------
|
||||
def target_name(stem):
|
||||
"""home-wiki 文件名(stem) → kepano 目标文件名。"""
|
||||
return NAME_MAP.get(stem, stem)
|
||||
|
||||
|
||||
def collect_sources():
|
||||
"""收集所有待同步源文件(排除 index.md)。返回 [(src_path, stem)]。"""
|
||||
items = []
|
||||
for d in SRC_DIRS:
|
||||
sdir = SRC_ROOT / d
|
||||
if not sdir.is_dir():
|
||||
continue
|
||||
for p in sorted(sdir.glob("*.md")):
|
||||
if p.stem.lower() == "index":
|
||||
continue
|
||||
items.append((p, p.stem))
|
||||
return items
|
||||
|
||||
|
||||
def build_page(src_path, stem):
|
||||
"""读取源文件,生成 kepano 页面内容 (frontmatter_str + body)。"""
|
||||
text = src_path.read_text(encoding="utf-8")
|
||||
fm, body = parse_doc(text)
|
||||
pairs = convert_frontmatter(fm, stem)
|
||||
body_clean = strip_openclaw(body)
|
||||
# 确保正文与 frontmatter 间有空行
|
||||
return dump_frontmatter(pairs) + "\n" + body_clean.lstrip("\n")
|
||||
|
||||
|
||||
def _norm_full(text):
|
||||
"""规范化整篇(frontmatter+正文)用于对比:去行尾空白、去空行。"""
|
||||
return "\n".join(ln.rstrip() for ln in text.splitlines() if ln.strip())
|
||||
|
||||
|
||||
def get_page_meta():
|
||||
"""返回 {target_stem: {"type":..., "src_dir":..., "src_stem":...}}。"""
|
||||
meta = {}
|
||||
for src_path, stem in collect_sources():
|
||||
fm, _ = parse_doc(src_path.read_text(encoding="utf-8"))
|
||||
meta[target_name(stem)] = {
|
||||
"type": str(fm.get("type", "concept")),
|
||||
"src_dir": src_path.parent.name,
|
||||
"src_stem": stem,
|
||||
}
|
||||
return meta
|
||||
|
||||
|
||||
def sync(dry_run=False):
|
||||
if not SRC_ROOT.is_dir():
|
||||
sys.stderr.write(f"ERROR: 源目录不存在: {SRC_ROOT}\n")
|
||||
sys.exit(1)
|
||||
if not DST_WIKI.is_dir():
|
||||
sys.stderr.write(f"ERROR: 目标 wiki 目录不存在: {DST_WIKI}\n")
|
||||
sys.exit(1)
|
||||
|
||||
items = collect_sources()
|
||||
created, updated, unchanged = [], [], []
|
||||
errors = []
|
||||
|
||||
for src_path, stem in items:
|
||||
try:
|
||||
dst_path = DST_WIKI / f"{target_name(stem)}.md"
|
||||
new_content = build_page(src_path, stem)
|
||||
if not dst_path.exists():
|
||||
created.append(stem)
|
||||
if not dry_run:
|
||||
dst_path.write_text(new_content, encoding="utf-8")
|
||||
continue
|
||||
old_content = dst_path.read_text(encoding="utf-8")
|
||||
# 对比完整生成内容(frontmatter+正文),确保 frontmatter 损坏也会被修复
|
||||
if _norm_full(new_content) == _norm_full(old_content):
|
||||
unchanged.append(stem)
|
||||
continue
|
||||
updated.append(stem)
|
||||
if not dry_run:
|
||||
dst_path.write_text(new_content, encoding="utf-8")
|
||||
except Exception as e: # noqa
|
||||
errors.append(f"{stem}: {e}")
|
||||
|
||||
return {
|
||||
"total": len(items),
|
||||
"created": created,
|
||||
"updated": updated,
|
||||
"unchanged": unchanged,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 简介提取与缓存(index 自动维护用)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _wikilink_to_text(s):
|
||||
"""将简介里的 wikilink 转为纯文本显示:[[a|b]]→b,[[a]]→a。"""
|
||||
s = re.sub(r"\[\[([^\]|]+?)\|([^\]]+?)\]\]", r"\2", s)
|
||||
s = re.sub(r"\[\[([^\]]+?)\]\]", r"\1", s)
|
||||
return s
|
||||
|
||||
|
||||
def _trunc(s, n=80):
|
||||
s = s.strip()
|
||||
if len(s) > n:
|
||||
s = s[:n].rstrip() + "…"
|
||||
return s
|
||||
|
||||
|
||||
def extract_summary(body, fm):
|
||||
"""从页面提取一句话简介。优先级:description → 顶部首个引用块 → 概述/定义首句。"""
|
||||
if fm.get("description"):
|
||||
return _trunc(_wikilink_to_text(str(fm["description"])))
|
||||
lines = body.splitlines()
|
||||
i, n = 0, len(lines)
|
||||
while i < n and not lines[i].lstrip().startswith(">"):
|
||||
i += 1
|
||||
if i < n:
|
||||
quotes = []
|
||||
while i < n and lines[i].lstrip().startswith(">"):
|
||||
content = lines[i].lstrip()[1:].strip()
|
||||
if content and not re.match(r"^[—\-]+", content):
|
||||
quotes.append(content)
|
||||
i += 1
|
||||
if quotes:
|
||||
return _trunc(_wikilink_to_text(" ".join(quotes)))
|
||||
m = re.search(r"^##\s*(?:概述|定义|简介)\s*\n\s*(.+)$", body, re.M)
|
||||
if m:
|
||||
first = m.group(1).strip().splitlines()[0]
|
||||
if first:
|
||||
return _trunc(_wikilink_to_text(first))
|
||||
return ""
|
||||
|
||||
|
||||
def seed_from_index():
|
||||
"""首次运行:从 index.md 现有跨库表格解析人工简介作为初始缓存。"""
|
||||
cache = {}
|
||||
if not DST_INDEX.exists():
|
||||
return cache
|
||||
text = DST_INDEX.read_text(encoding="utf-8")
|
||||
# 只解析来源列含 [[home-wiki/ 的行,避免误抓其他来源页面
|
||||
pattern = re.compile(
|
||||
r"^\|\s*\[\[([^\]|]+?)(?:\|[^\]]+)?\]\]\s*\|\s*([^|]+?)\s*\|\s*\[\[home-wiki/",
|
||||
re.M,
|
||||
)
|
||||
for m in pattern.finditer(text):
|
||||
stem = m.group(1).strip()
|
||||
summary = m.group(2).strip()
|
||||
if stem and summary:
|
||||
cache[stem] = {"summary": summary}
|
||||
return cache
|
||||
|
||||
|
||||
def load_summaries():
|
||||
if SUMMARIES_CACHE.exists():
|
||||
try:
|
||||
return json.loads(SUMMARIES_CACHE.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
return seed_from_index()
|
||||
|
||||
|
||||
def save_summaries(cache):
|
||||
SUMMARIES_CACHE.parent.mkdir(parents=True, exist_ok=True)
|
||||
SUMMARIES_CACHE.write_text(
|
||||
json.dumps(cache, ensure_ascii=False, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# index.md 自动维护
|
||||
# ---------------------------------------------------------------------------
|
||||
def _build_table(stems, cache, meta, title):
|
||||
"""生成单个分类表格(Markdown 文本)。"""
|
||||
lines = [f"### {title}({len(stems)})", "", "| 页面 | 简介 | 来源 |", "|------|------|------|"]
|
||||
for s in stems:
|
||||
summ = cache.get(s, {}).get("summary", "")
|
||||
src = f"[[home-wiki/{meta[s]['src_dir']}/{s}]]"
|
||||
lines.append(f"| [[{s}]] | {summ} | {src} |")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def update_index(meta):
|
||||
"""用标记块自动维护 index.md 的概念/实体表,并刷新「最后更新」日期。"""
|
||||
cache_existed = SUMMARIES_CACHE.exists()
|
||||
cache = load_summaries()
|
||||
# 为缓存缺失的页面提取简介
|
||||
dirty = False
|
||||
for stem, m in meta.items():
|
||||
if not cache.get(stem, {}).get("summary"):
|
||||
dst = DST_WIKI / f"{stem}.md"
|
||||
if dst.exists():
|
||||
fm, body = parse_doc(dst.read_text(encoding="utf-8"))
|
||||
cache[stem] = {"summary": extract_summary(body, fm), "type": m["type"]}
|
||||
dirty = True
|
||||
# 首次运行(缓存文件不存在)或有新提取,都持久化(JSON 是简介权威来源)
|
||||
if dirty or not cache_existed:
|
||||
save_summaries(cache)
|
||||
|
||||
concepts = sorted(s for s, m in meta.items() if m["type"] == "concept")
|
||||
entities = sorted(s for s, m in meta.items() if m["type"] == "entity")
|
||||
|
||||
block_body = [
|
||||
"## 跨库同步(home-wiki)",
|
||||
"",
|
||||
"> 来自 `D:\\TC_UP\\home-wiki` 的知识内容,聚焦高等教育 AI、系统理论、哲学思想等方向。",
|
||||
"",
|
||||
_build_table(concepts, cache, meta, "概念页"),
|
||||
"",
|
||||
_build_table(entities, cache, meta, "实体页"),
|
||||
]
|
||||
generated = INDEX_BEGIN + "\n" + "\n".join(block_body) + "\n" + INDEX_END
|
||||
|
||||
text = DST_INDEX.read_text(encoding="utf-8")
|
||||
if INDEX_BEGIN in text and INDEX_END in text:
|
||||
new_text = re.sub(
|
||||
re.escape(INDEX_BEGIN) + r".*?" + re.escape(INDEX_END),
|
||||
lambda _: generated,
|
||||
text,
|
||||
flags=re.S,
|
||||
)
|
||||
else:
|
||||
# 首次:替换现有手动「跨库同步」章节(到「综合报告」之前)为标记块
|
||||
new_text = re.sub(
|
||||
r"## 跨库同步(home-wiki).*?(?=### 综合报告)",
|
||||
lambda _: generated + "\n\n",
|
||||
text,
|
||||
flags=re.S,
|
||||
)
|
||||
|
||||
# 刷新「最后更新」日期(pattern 匹配整格含结尾 |,replacement 给完整行,避免管道符累加)
|
||||
today = date.today().isoformat()
|
||||
new_text = re.sub(
|
||||
r"\|\s*最后更新\s*\|\s*[^\n|]*\|",
|
||||
f"| 最后更新 | {today} (home-wiki 跨库同步) |",
|
||||
new_text,
|
||||
)
|
||||
|
||||
if new_text != text:
|
||||
DST_INDEX.write_text(new_text, encoding="utf-8")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# lint 体检
|
||||
# ---------------------------------------------------------------------------
|
||||
def lint():
|
||||
meta = get_page_meta()
|
||||
synced = list(meta.keys())
|
||||
allfiles = {p.stem for p in DST_WIKI.glob("*.md")}
|
||||
|
||||
# aliases 反向映射(MarkItDown 等大小写变体)
|
||||
aliases_map = {}
|
||||
for p in DST_WIKI.glob("*.md"):
|
||||
fm, _ = parse_doc(p.read_text(encoding="utf-8"))
|
||||
for a in _as_list(fm.get("aliases")):
|
||||
aliases_map[str(a).strip()] = p.stem
|
||||
|
||||
# 孤儿:同步页面在 wiki 的入站链接数(排除自身)
|
||||
inbound = {s: 0 for s in synced}
|
||||
for p in DST_WIKI.glob("*.md"):
|
||||
t = p.read_text(encoding="utf-8")
|
||||
for s in synced:
|
||||
if p.stem == s:
|
||||
continue
|
||||
if re.search(rf"\[\[{re.escape(s)}[\]|\]]", t):
|
||||
inbound[s] += 1
|
||||
orphans = [s for s, c in inbound.items() if c == 0]
|
||||
|
||||
# source 空值
|
||||
empty_source = []
|
||||
for s in synced:
|
||||
t = (DST_WIKI / f"{s}.md").read_text(encoding="utf-8")
|
||||
m = re.search(r"^source:[ \t]*(.*)$", t, re.M)
|
||||
if m and not m.group(1).strip():
|
||||
empty_source.append(s)
|
||||
|
||||
# 断链:同步页面引用的 [[x]] 在 wiki 是否存在(排除 raw 来源类、aliases、category)
|
||||
valid = allfiles | set(aliases_map.keys()) | set(synced)
|
||||
skip = {"People", "LLM Wiki", "wikilink"}
|
||||
broken = {}
|
||||
for s in synced:
|
||||
t = (DST_WIKI / f"{s}.md").read_text(encoding="utf-8")
|
||||
for m in re.findall(r"\[\[([^\]|]+?)(?:\|[^\]]+)?\]\]", t):
|
||||
tgt = m.strip()
|
||||
if not tgt or tgt.startswith("http") or "/" in tgt or "\\" in tgt:
|
||||
continue
|
||||
if tgt in valid or tgt in skip:
|
||||
continue
|
||||
broken.setdefault(tgt, []).append(s)
|
||||
|
||||
# index 覆盖检查:同步的概念/实体是否都在 index 跨库章节
|
||||
idx_text = DST_INDEX.read_text(encoding="utf-8") if DST_INDEX.exists() else ""
|
||||
not_in_index = [s for s in synced if meta[s]["type"] in ("concept", "entity")
|
||||
and not re.search(rf"\[\[{re.escape(s)}[\]|\]]", idx_text)]
|
||||
|
||||
print("== home-wiki 同步体检 ==")
|
||||
print(f"同步页面: {len(synced)} | wiki 总文件: {len(allfiles)}")
|
||||
print(f"\n[孤儿] {len(orphans)} 个(0 入站链接)")
|
||||
for s in orphans:
|
||||
print(f" - {s}")
|
||||
print(f"\n[source 空值] {len(empty_source)} 个(违反 wiki 层 source 必填)")
|
||||
for s in empty_source:
|
||||
print(f" - {s}")
|
||||
print(f"\n[断链] {len(broken)} 个目标(同步页面引用、kepano 无对应页)")
|
||||
for tgt in sorted(broken):
|
||||
print(f" - {tgt} ← {', '.join(sorted(broken[tgt]))}")
|
||||
print(f"\n[未入 index] {len(not_in_index)} 个(概念/实体未在跨库章节)")
|
||||
for s in not_in_index:
|
||||
print(f" - {s}")
|
||||
|
||||
issues = len(orphans) + len(empty_source) + len(not_in_index)
|
||||
print(f"\n硬性问题(孤儿+空source+未入index): {issues}")
|
||||
print(f"软性问题(断链,多为待创建概念/raw来源): {len(broken)}")
|
||||
return issues
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 日志
|
||||
# ---------------------------------------------------------------------------
|
||||
def append_log(result):
|
||||
today = date.today().isoformat()
|
||||
lines = [
|
||||
f"\n## [{today}] sync | home-wiki 跨库同步",
|
||||
"",
|
||||
f"**来源**: `{SRC_ROOT}`",
|
||||
"",
|
||||
"**操作**:",
|
||||
f"- 扫描 concepts/entities/syntheses 共 {result['total']} 个页面",
|
||||
]
|
||||
if result["updated"]:
|
||||
lines.append(f"- 更新 {len(result['updated'])} 页: {', '.join(result['updated'])}")
|
||||
if result["created"]:
|
||||
lines.append(f"- 新建 {len(result['created'])} 页: {', '.join(result['created'])}")
|
||||
if result["unchanged"]:
|
||||
lines.append(f"- 未变 {len(result['unchanged'])} 页")
|
||||
if result["errors"]:
|
||||
lines.append(f"- 错误 {len(result['errors'])} 项: {'; '.join(result['errors'])}")
|
||||
lines += ["", f"**涉及页面**: {len(result['updated'])} 更新,{len(result['created'])} 新建", ""]
|
||||
with DST_LOG.open("a", encoding="utf-8") as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="home-wiki → kepano LLM Wiki 同步")
|
||||
ap.add_argument("--dry-run", action="store_true", help="仅诊断,不写入")
|
||||
ap.add_argument("--log", action="store_true", help="同步后追加 wiki/log.md")
|
||||
ap.add_argument("--index", action="store_true", help="同步后自动维护 wiki/index.md 跨库章节")
|
||||
ap.add_argument("--lint", action="store_true", help="仅体检(孤儿/断链/source空值),不同步")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.lint:
|
||||
issues = lint()
|
||||
sys.exit(0 if issues == 0 else 0) # lint 不以问题数为错误码
|
||||
|
||||
res = sync(dry_run=args.dry_run)
|
||||
verb = "诊断" if args.dry_run else "同步"
|
||||
print(f"== home-wiki {verb}报告 ==")
|
||||
print(f"扫描: {res['total']} 页")
|
||||
print(f"需更新: {len(res['updated'])} -> {res['updated']}")
|
||||
print(f"需新建: {len(res['created'])} -> {res['created']}")
|
||||
print(f"未变化: {len(res['unchanged'])} -> {res['unchanged']}")
|
||||
if res["errors"]:
|
||||
print(f"错误: {res['errors']}")
|
||||
sys.exit(2)
|
||||
|
||||
if not args.dry_run:
|
||||
if args.log and (res["updated"] or res["created"]):
|
||||
append_log(res)
|
||||
print("已追加 wiki/log.md")
|
||||
if args.index:
|
||||
meta = get_page_meta()
|
||||
changed = update_index(meta)
|
||||
print(f"wiki/index.md 跨库章节: {'已更新' if changed else '无变化'}")
|
||||
print("完成。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
validate-frontmatter.py — Pre-commit hook: 验证被修改的 wiki 页面 frontmatter
|
||||
|
||||
检查项:
|
||||
- 必须包含 categories(含 [[LLM Wiki]])
|
||||
- 必须包含 tags(含 wiki)
|
||||
- 必须包含 type(合法值列表)
|
||||
- 必须包含 source
|
||||
- 如果 status=superseded,必须包含 superseded_by
|
||||
- relations 的 target 必须指向存在的页面
|
||||
|
||||
用法:
|
||||
python tools/scripts/validate-frontmatter.py # 所有 wiki 页面
|
||||
python tools/scripts/validate-frontmatter.py --files file1.md file2.md # 指定文件
|
||||
python tools/scripts/validate-frontmatter.py --git-hook # 从 git diff 读取
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
WIKI = Path(r"D:\Applications\app\kepano-obsidian-main\wiki")
|
||||
|
||||
VALID_TYPES = [
|
||||
"concept", "entity", "tool", "reference", "place",
|
||||
"institution", "method", "knowledge-card", "synthesis",
|
||||
"index", "log", "research-report", "lesson"
|
||||
]
|
||||
|
||||
|
||||
def validate_one(fp: Path) -> list[str]:
|
||||
"""验证单个文件,返回错误列表"""
|
||||
errors = []
|
||||
content = fp.read_text(encoding="utf-8")
|
||||
m = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not m:
|
||||
return ["No frontmatter found"]
|
||||
if not yaml:
|
||||
return ["PyYAML not installed"]
|
||||
try:
|
||||
front = yaml.safe_load(m.group(1)) or {}
|
||||
except yaml.YAMLError as e:
|
||||
return [f"YAML parse error: {e}"]
|
||||
|
||||
# categories must contain [[LLM Wiki]]
|
||||
cats = front.get("categories", [])
|
||||
if isinstance(cats, str):
|
||||
cats = [cats]
|
||||
if not any(str(c).strip("[]") == "LLM Wiki" for c in cats):
|
||||
errors.append("Missing [[LLM Wiki]] in categories")
|
||||
|
||||
# tags must contain wiki
|
||||
tags = front.get("tags", [])
|
||||
if isinstance(tags, str):
|
||||
tags = [tags]
|
||||
if "wiki" not in tags:
|
||||
errors.append("Missing 'wiki' in tags")
|
||||
|
||||
# type must exist and be valid
|
||||
ptype = front.get("type")
|
||||
if not ptype:
|
||||
errors.append("Missing type")
|
||||
elif isinstance(ptype, str) and ptype not in VALID_TYPES:
|
||||
errors.append(f"Invalid type: '{ptype}' (valid: {', '.join(VALID_TYPES)})")
|
||||
elif isinstance(ptype, list):
|
||||
for t in ptype:
|
||||
if t not in VALID_TYPES:
|
||||
errors.append(f"Invalid type in list: '{t}'")
|
||||
|
||||
# source must exist
|
||||
if not front.get("source"):
|
||||
errors.append("Missing source")
|
||||
|
||||
# status=superseded must have superseded_by
|
||||
if front.get("status") == "superseded" and not front.get("superseded_by"):
|
||||
errors.append("status=superseded but missing superseded_by")
|
||||
|
||||
# relations targets must exist
|
||||
for rel in front.get("relations", []):
|
||||
target = str(rel.get("target", "")).strip("[]")
|
||||
if target and not (WIKI / f"{target}.md").exists():
|
||||
errors.append(f"Relation target [[{target}]] not found")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def get_git_changed_wiki_files() -> list[Path]:
|
||||
"""获取 git 暂存区中被修改的 wiki .md 文件"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--cached", "--name-only", "--diff-filter=ACM"],
|
||||
capture_output=True, text=True, check=True, cwd=WIKI.parent
|
||||
)
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
return []
|
||||
files = []
|
||||
for line in result.stdout.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("wiki/") and line.endswith(".md"):
|
||||
fp = WIKI.parent / line
|
||||
if fp.exists():
|
||||
files.append(fp)
|
||||
return files
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Validate wiki page frontmatter")
|
||||
parser.add_argument("--files", nargs="+", help="Specific files to check")
|
||||
parser.add_argument("--git-hook", action="store_true", help="Read from git diff")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.git_hook:
|
||||
files = get_git_changed_wiki_files()
|
||||
if not files:
|
||||
sys.exit(0)
|
||||
elif args.files:
|
||||
files = [Path(f) if Path(f).is_absolute() else WIKI / f for f in args.files]
|
||||
else:
|
||||
files = sorted(WIKI.glob("*.md"))
|
||||
|
||||
all_errors = {}
|
||||
for fp in files:
|
||||
if fp.name in ("index.md", "log.md"):
|
||||
continue
|
||||
errors = validate_one(fp)
|
||||
if errors:
|
||||
all_errors[fp.stem] = errors
|
||||
|
||||
if all_errors:
|
||||
print(f"=== Frontmatter Validation Errors ({len(all_errors)} files) ===")
|
||||
for name, errs in sorted(all_errors.items()):
|
||||
print(f"\n [[{name}]]:")
|
||||
for e in errs:
|
||||
print(f" ❌ {e}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("✅ All frontmatter valid")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,49 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
每周 Wiki Lint:孤儿 + 断链 + 时效 + 矛盾 + index 同步
|
||||
.DESCRIPTION
|
||||
聚合所有 lint 脚本,输出报告到 tools/data/lint-reports/YYYY-MM-DD.log
|
||||
配合 Windows 定时任务使用:每周日 22:00
|
||||
#>
|
||||
$ErrorActionPreference = "Continue"
|
||||
$env:PYTHONIOENCODING = "utf-8"
|
||||
$vaultRoot = "D:\Applications\app\kepano-obsidian-main"
|
||||
$reportDir = "$vaultRoot\tools\data\lint-reports"
|
||||
$today = Get-Date -Format "yyyy-MM-dd"
|
||||
$logFile = "$reportDir\$today.log"
|
||||
|
||||
# 创建报告目录
|
||||
New-Item -ItemType Directory -Path $reportDir -Force | Out-Null
|
||||
|
||||
"=== Weekly Lint Report — $today ===" | Out-File -FilePath $logFile -Encoding utf8
|
||||
"=" * 40 | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
|
||||
# 1. Orphan detection
|
||||
"`n=== 1. Orphan Pages ===" | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
$orphans = & "$vaultRoot\tools\scripts\wiki-lint-orphan.ps1" 2>&1
|
||||
$orphans | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
|
||||
# 2. Broken links
|
||||
"`n=== 2. Broken Links ===" | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
$broken = & "$vaultRoot\tools\scripts\wiki-lint-broken-v2.ps1" 2>&1
|
||||
$broken | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
|
||||
# 3. Staleness
|
||||
"`n=== 3. Staleness ===" | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
$staleness = python "$vaultRoot\tools\scripts\check-staleness.py" 2>&1
|
||||
$staleness | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
|
||||
# 4. Conflicts
|
||||
"`n=== 4. Conflicts ===" | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
$conflicts = python "$vaultRoot\tools\scripts\detect-conflicts.py" 2>&1
|
||||
$conflicts | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
|
||||
# 5. Frontmatter validation
|
||||
"`n=== 5. Frontmatter Validation ===" | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
$validation = python "$vaultRoot\tools\scripts\validate-frontmatter.py" 2>&1
|
||||
$validation | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
|
||||
# Summary
|
||||
"`n$('=' * 40)" | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
"Report saved: $logFile" | Out-File -FilePath $logFile -Encoding utf8 -Append
|
||||
Write-Output "Weekly lint complete: $logFile"
|
||||
@@ -0,0 +1,59 @@
|
||||
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
|
||||
$wiki = 'D:\Applications\app\kepano-obsidian-main\wiki'
|
||||
$vault = 'D:\Applications\app\kepano-obsidian-main'
|
||||
|
||||
# Get all available pages in vault (excluding special Obsidian files)
|
||||
$allMd = Get-ChildItem -Path $vault -Filter '*.md' -File -Recurse | Where-Object {
|
||||
$_.FullName -notmatch '\\\.git\\' -and
|
||||
$_.FullName -notmatch '\\node_modules\\'
|
||||
}
|
||||
$availablePages = @{}
|
||||
foreach ($f in $allMd) {
|
||||
$bn = $f.BaseName
|
||||
$availablePages[$bn] = $true
|
||||
}
|
||||
|
||||
# Add known non-.md references that are valid
|
||||
$validRefs = @(
|
||||
'Trips.base','Map.base','Places.base','Books.base','Movies.base',
|
||||
'黄鹤楼.base','Templates/Bases/Places.base','Templates/Bases/Map.base'
|
||||
)
|
||||
foreach ($ref in $validRefs) { $availablePages[$ref] = $true }
|
||||
|
||||
# Scan wiki files for broken links
|
||||
$brokenLinks = @()
|
||||
$wikiFiles = Get-ChildItem -Path $wiki -Filter '*.md' -File
|
||||
|
||||
foreach ($f in $wikiFiles) {
|
||||
$content = [System.IO.File]::ReadAllText($f.FullName)
|
||||
$matches = [regex]::Matches($content, '\[\[([^\]|]+)(?:\|[^\]]+)?\]\]')
|
||||
|
||||
foreach ($m in $matches) {
|
||||
$linkTarget = $m.Groups[1].Value.Trim()
|
||||
|
||||
# Skip various patterns
|
||||
if ($linkTarget -match '^http' -or $linkTarget -match '^#') { continue }
|
||||
if ($linkTarget -match '^Attachments/') { continue }
|
||||
if ($linkTarget -match '\.base#') { continue } # .base section refs
|
||||
if ($linkTarget -match '^Templates/') { continue }
|
||||
if ($linkTarget -match '^References/') { continue }
|
||||
if ($linkTarget -match '^Categories/') { continue }
|
||||
if ($linkTarget -match '百度百科/') { continue }
|
||||
if ($linkTarget -match '^raw/') { continue } # raw/ references
|
||||
|
||||
# Check if target exists
|
||||
if (-not $availablePages.ContainsKey($linkTarget)) {
|
||||
$brokenLinks += [PSCustomObject]@{
|
||||
File = $f.Name
|
||||
BrokenLink = $linkTarget
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($brokenLinks.Count -eq 0) {
|
||||
Write-Output "BROKEN_LINKS: 0"
|
||||
} else {
|
||||
Write-Output "BROKEN_LINKS: $($brokenLinks.Count)"
|
||||
$brokenLinks | ForEach-Object { Write-Output "$($_.File) -> [[$($_.BrokenLink)]]" }
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
|
||||
$wiki = 'D:\Applications\app\kepano-obsidian-main\wiki'
|
||||
$vault = 'D:\Applications\app\kepano-obsidian-main'
|
||||
|
||||
# Get all available pages in vault
|
||||
$allMd = Get-ChildItem -Path $vault -Filter '*.md' -File -Recurse
|
||||
$availablePages = @{}
|
||||
foreach ($f in $allMd) {
|
||||
$bn = $f.BaseName
|
||||
$availablePages[$bn] = $true
|
||||
# Also register with Chinese characters variations
|
||||
}
|
||||
|
||||
# Scan wiki files for broken links
|
||||
$brokenLinks = @()
|
||||
$wikiFiles = Get-ChildItem -Path $wiki -Filter '*.md' -File
|
||||
|
||||
foreach ($f in $wikiFiles) {
|
||||
$content = [System.IO.File]::ReadAllText($f.FullName)
|
||||
# Match [[Link]] or [[Link|Display]]
|
||||
$matches = [regex]::Matches($content, '\[\[([^\]|]+)(?:\|[^\]]+)?\]\]')
|
||||
foreach ($m in $matches) {
|
||||
$linkTarget = $m.Groups[1].Value.Trim()
|
||||
# Skip anchors and external links
|
||||
if ($linkTarget -match '^http' -or $linkTarget -match '^#') { continue }
|
||||
|
||||
if (-not $availablePages.ContainsKey($linkTarget)) {
|
||||
$brokenLinks += [PSCustomObject]@{
|
||||
File = $f.Name
|
||||
BrokenLink = $linkTarget
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($brokenLinks.Count -eq 0) {
|
||||
Write-Output "BROKEN_LINKS: 0"
|
||||
} else {
|
||||
Write-Output "BROKEN_LINKS: $($brokenLinks.Count)"
|
||||
$brokenLinks | ForEach-Object { Write-Output "$($_.File) -> [[$($_.BrokenLink)]]" }
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
|
||||
$wiki = 'D:\Applications\app\kepano-obsidian-main\wiki'
|
||||
|
||||
foreach ($f in Get-ChildItem $wiki -Filter '*.md' -File) {
|
||||
$content = [System.IO.File]::ReadAllText($f.FullName)
|
||||
$name = $f.BaseName
|
||||
|
||||
# Check if type is empty
|
||||
if ($content -match '(?m)^type:\s*$') {
|
||||
$newType = 'entity'
|
||||
|
||||
# Check if it's a place
|
||||
if ($content -match 'Places|地点' -or $name -match '^([天地山河湖海关陵墓])' -or $name -match '城$|省$|市$|县$|山$|河$|湖$|海$|关$|陵$|墓$|遗址$|塔$|院$|阁$|庙$') {
|
||||
$newType = 'place'
|
||||
}
|
||||
|
||||
$content = $content -replace '(?m)^type:\s*$', ('type: ' + $newType)
|
||||
[System.IO.File]::WriteAllText($f.FullName, $content)
|
||||
Write-Output "FIXED: $name -> type: $newType"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
|
||||
$wikiPath = 'D:\Applications\app\kepano-obsidian-main\wiki'
|
||||
$vaultPath = 'D:\Applications\app\kepano-obsidian-main'
|
||||
|
||||
$wikiFiles = Get-ChildItem -Path $wikiPath -Filter '*.md' -File
|
||||
$pageNames = $wikiFiles | ForEach-Object { $_.BaseName }
|
||||
|
||||
$allMd = Get-ChildItem -Path $vaultPath -Filter '*.md' -File -Recurse | Where-Object { $_.Length -lt 500KB }
|
||||
$allContent = @{}
|
||||
foreach ($f in $allMd) {
|
||||
try { $allContent[$f.FullName] = [System.IO.File]::ReadAllText($f.FullName) } catch {}
|
||||
}
|
||||
|
||||
$orphans = @()
|
||||
foreach ($pn in $pageNames) {
|
||||
if ($pn -eq 'index' -or $pn -eq 'log') { continue }
|
||||
$linkPattern = '[[' + $pn + ']'
|
||||
$hasInbound = $false
|
||||
foreach ($fp in $allContent.Keys) {
|
||||
$baseF = [System.IO.Path]::GetFileNameWithoutExtension($fp)
|
||||
if ($baseF -eq $pn) { continue }
|
||||
if ($allContent[$fp] -match [regex]::Escape($linkPattern)) {
|
||||
$hasInbound = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
if (-not $hasInbound) { $orphans += $pn }
|
||||
}
|
||||
Write-Output "ORPHAN_COUNT: $($orphans.Count)"
|
||||
$orphans | ForEach-Object { Write-Output $_ }
|
||||
Reference in New Issue
Block a user