hehaiguang1123
a6f05ab2d5
- Phase 0: AGENTS.md cleanup (dedup quotes, renumber sections, merge qmd) - Phase 1: typed relations (manage-relations.py, graph-search.py, check-staleness.py, detect-conflicts.py) - Phase 2: frontmatter validator, weekly lint, knowledge promotion, git hooks - Fix .gitignore to track tools/ and .githooks/ - Fix git remote URL (remove plaintext token) - New wiki pages: 504 pages, 34 raw sources
364 lines
10 KiB
Python
364 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
home-wiki → kepano-obsidian-main 同步转换脚本
|
||
|
||
用途：将 home-wiki 的笔记内容转换为目标仓库格式并同步。
|
||
功能：
|
||
1. Frontmatter 格式转换
|
||
2. OpenClaw 标记清理
|
||
3. Wikilink 路径保持
|
||
4. 批量处理目录
|
||
|
||
用法：
|
||
python sync_homewiki.py concepts # 同步概念页
|
||
python sync_homewiki.py entities # 同步实体页
|
||
python sync_sync_homewiki.py all # 同步全部
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import sys
|
||
import shutil
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
|
||
# 路径配置
|
||
SOURCE_ROOT = Path(r"D:\TC_UP\home-wiki")
|
||
TARGET_ROOT = Path(r"D:\Applications\app\kepano-obsidian-main")
|
||
|
||
# 目录映射
|
||
DIR_MAPPING = {
|
||
"concepts": "wiki",
|
||
"entities": "wiki",
|
||
"syntheses": "wiki",
|
||
}
|
||
|
||
# 标签映射（home-wiki tag → 目标仓库 tag）
|
||
TAG_MAPPING = {
|
||
"complexity": "concept/complexity",
|
||
"systems-theory": "concept/systems-theory",
|
||
"consciousness": "concept/consciousness",
|
||
"physics": "concept/physics",
|
||
"phenomenon": "concept/phenomenon",
|
||
"theory": "concept/theory",
|
||
"technology": "concept/technology",
|
||
"practice": "concept/practice",
|
||
"person": "people",
|
||
"innovation-management": "concept/innovation",
|
||
"methodology": "concept/methodology",
|
||
"higher-ed": "education/higher-ed",
|
||
"AI-education": "ai/education",
|
||
"LLM": "llm",
|
||
"agent": "agent",
|
||
"RAG": "rag",
|
||
"knowledge-graph": "knowledge-graph",
|
||
}
|
||
|
||
|
||
def parse_frontmatter(content):
|
||
"""解析 markdown 文件的 frontmatter，返回 (yaml_text, body)"""
|
||
match = re.match(r'^---\n(.*?)\n---\n(.*)', content, re.DOTALL)
|
||
if not match:
|
||
return None, content
|
||
return match.group(1), match.group(2)
|
||
|
||
|
||
def convert_frontmatter(yaml_text, source_type="concept"):
|
||
"""将 home-wiki frontmatter 转换为目标仓库格式"""
|
||
lines = yaml_text.split('\n')
|
||
converted = []
|
||
|
||
# 提取字段
|
||
tags = []
|
||
sources = []
|
||
created = None
|
||
page_type = source_type
|
||
subtype = None
|
||
title = None
|
||
aliases = []
|
||
|
||
i = 0
|
||
while i < len(lines):
|
||
line = lines[i]
|
||
|
||
# type 字段
|
||
m = re.match(r'^type:\s*(.+)', line)
|
||
if m:
|
||
page_type = m.group(1).strip()
|
||
i += 1
|
||
continue
|
||
|
||
# subtype 字段
|
||
m = re.match(r'^subtype:\s*(.+)', line)
|
||
if m:
|
||
subtype = m.group(1).strip()
|
||
i += 1
|
||
continue
|
||
|
||
# title 字段
|
||
m = re.match(r'^title:\s*(.+)', line)
|
||
if m:
|
||
title = m.group(1).strip().strip('"').strip("'")
|
||
i += 1
|
||
continue
|
||
|
||
# created 字段
|
||
m = re.match(r'^created:\s*(.+)', line)
|
||
if m:
|
||
created = m.group(1).strip()
|
||
i += 1
|
||
continue
|
||
|
||
# tags 字段（行内列表格式）
|
||
m = re.match(r'^tags:\s*\[(.*)\]', line)
|
||
if m:
|
||
tag_str = m.group(1)
|
||
tags = [t.strip().strip('"').strip("'") for t in tag_str.split(',') if t.strip()]
|
||
i += 1
|
||
continue
|
||
|
||
# tags 字段（YAML 列表格式）
|
||
m = re.match(r'^tags:\s*$', line)
|
||
if m:
|
||
i += 1
|
||
while i < len(lines) and lines[i].strip().startswith('-'):
|
||
tag = lines[i].strip().lstrip('-').strip()
|
||
tags.append(tag)
|
||
i += 1
|
||
continue
|
||
|
||
# sources 字段（行内列表格式）
|
||
m = re.match(r'^sources:\s*\[(.*)\]', line)
|
||
if m:
|
||
src_str = m.group(1)
|
||
sources = [s.strip().strip('"').strip("'") for s in src_str.split(',') if s.strip()]
|
||
i += 1
|
||
continue
|
||
|
||
# sources 字段（YAML 列表格式）
|
||
m = re.match(r'^sources:\s*$', line)
|
||
if m:
|
||
i += 1
|
||
while i < len(lines) and lines[i].strip().startswith('-'):
|
||
src = lines[i].strip().lstrip('-').strip().strip('"').strip("'")
|
||
sources.append(src)
|
||
i += 1
|
||
continue
|
||
|
||
# 跳过不需要的字段
|
||
if re.match(r'^(updated|confidence|status|description|name|nameEn|born|field):\s*', line):
|
||
i += 1
|
||
continue
|
||
|
||
# claims 字段（跳过多行）
|
||
if re.match(r'^claims:\s*', line):
|
||
i += 1
|
||
while i < len(lines) and (lines[i].startswith(' ') or lines[i].startswith(' -')):
|
||
i += 1
|
||
continue
|
||
|
||
i += 1
|
||
|
||
# 构建新的 frontmatter
|
||
if not created:
|
||
created = datetime.now().strftime('%Y-%m-%d')
|
||
|
||
new_lines = []
|
||
new_lines.append('---')
|
||
new_lines.append('categories:')
|
||
new_lines.append(' - "[[LLM Wiki]]"')
|
||
|
||
# tags
|
||
new_lines.append('tags:')
|
||
new_lines.append(' - wiki')
|
||
if page_type == 'entity':
|
||
new_lines.append(' - people')
|
||
else:
|
||
new_lines.append(' - concept')
|
||
|
||
if subtype:
|
||
mapped = TAG_MAPPING.get(subtype, subtype)
|
||
new_lines.append(f' - {mapped}')
|
||
|
||
for tag in tags:
|
||
mapped = TAG_MAPPING.get(tag, tag)
|
||
if mapped not in ['wiki', 'concept', 'people'] and mapped not in [l.strip().lstrip('-').strip() for l in new_lines[4:]]:
|
||
new_lines.append(f' - {mapped}')
|
||
|
||
new_lines.append(f'created: {created}')
|
||
|
||
# source 字段
|
||
if sources:
|
||
src = sources[0]
|
||
# 提取文件名（去掉路径）
|
||
src_filename = Path(src).name
|
||
if src.startswith('http'):
|
||
new_lines.append(f'source: "{src}"')
|
||
else:
|
||
src_name = Path(src_filename).stem
|
||
new_lines.append(f'source: "[[{src_name}]]"')
|
||
|
||
new_lines.append(f'type: {page_type}')
|
||
|
||
if title and title != page_type:
|
||
new_lines.append('aliases:')
|
||
new_lines.append(f' - {title}')
|
||
|
||
new_lines.append('---')
|
||
|
||
return '\n'.join(new_lines)
|
||
|
||
|
||
def clean_body(body):
|
||
"""清理正文中的 OpenClaw 标记和无用章节"""
|
||
# 移除 openclaw 注释块（单行）
|
||
body = re.sub(r'<!-- openclaw:[^>]*-->', '', body)
|
||
|
||
# 移除 Related 章节（openclaw 生成的）
|
||
body = re.sub(r'## Related\s*\n<!-- openclaw:wiki:related:start -->.*?<!-- openclaw:wiki:related:end -->',
|
||
'', body, flags=re.DOTALL)
|
||
|
||
# 移除 Notes 章节（openclaw 空标记）
|
||
body = re.sub(r'## Notes\s*\n<!-- openclaw:human:start -->\s*<!-- openclaw:human:end -->\s*',
|
||
'', body)
|
||
|
||
# 移除 Summary 的 openclaw 标记（保留内容）
|
||
body = re.sub(r'<!-- openclaw:wiki:generated:start -->\s*', '', body)
|
||
body = re.sub(r'\s*<!-- openclaw:wiki:generated:end -->', '', body)
|
||
|
||
# 清理多余空行
|
||
body = re.sub(r'\n{4,}', '\n\n\n', body)
|
||
|
||
# 移除末尾的 "最后更新" 行（home-wiki 特有）
|
||
body = re.sub(r'\n\*最后更新[：:][^*]*\*\s*$', '', body)
|
||
|
||
return body.strip() + '\n'
|
||
|
||
|
||
def convert_file(source_path, target_dir, source_type="concept"):
|
||
"""转换单个文件"""
|
||
source_path = Path(source_path)
|
||
target_dir = Path(target_dir)
|
||
|
||
# 跳过 index.md
|
||
if source_path.name == 'index.md':
|
||
return False, f"跳过 index.md"
|
||
|
||
# 读取源文件
|
||
content = source_path.read_text(encoding='utf-8')
|
||
|
||
# 解析 frontmatter
|
||
yaml_text, body = parse_frontmatter(content)
|
||
|
||
if yaml_text is None:
|
||
return False, f"无 frontmatter，跳过"
|
||
|
||
# 转换 frontmatter
|
||
new_yaml = convert_frontmatter(yaml_text, source_type)
|
||
|
||
# 清理正文
|
||
cleaned_body = clean_body(body)
|
||
|
||
# 组合
|
||
new_content = new_yaml + '\n\n' + cleaned_body
|
||
|
||
# 写入目标
|
||
target_path = target_dir / source_path.name
|
||
|
||
# 冲突检查
|
||
if target_path.exists():
|
||
return False, f"目标文件已存在: {target_path.name}"
|
||
|
||
target_path.write_text(new_content, encoding='utf-8')
|
||
return True, f"已同步: {source_path.name}"
|
||
|
||
|
||
def sync_directory(source_subdir, target_subdir, source_type="concept"):
|
||
"""同步整个目录"""
|
||
source_dir = SOURCE_ROOT / source_subdir
|
||
target_dir = TARGET_ROOT / target_subdir
|
||
|
||
if not source_dir.exists():
|
||
print(f"源目录不存在: {source_dir}")
|
||
return
|
||
|
||
print(f"\n{'='*60}")
|
||
print(f"同步: {source_subdir} → {target_subdir}")
|
||
print(f"{'='*60}")
|
||
|
||
success = 0
|
||
skipped = 0
|
||
failed = 0
|
||
|
||
for md_file in sorted(source_dir.glob("*.md")):
|
||
ok, msg = convert_file(md_file, target_dir, source_type)
|
||
if ok:
|
||
print(f" ✅ {msg}")
|
||
success += 1
|
||
else:
|
||
print(f" ⏭️ {msg}")
|
||
skipped += 1
|
||
|
||
print(f"\n汇总: {success} 同步, {skipped} 跳过")
|
||
return success
|
||
|
||
|
||
def sync_raw_articles():
|
||
"""同步 raw/articles/ 的 HTML 文件"""
|
||
source_dir = SOURCE_ROOT / "raw" / "articles"
|
||
target_dir = TARGET_ROOT / "raw" / "homewiki-articles"
|
||
|
||
if not source_dir.exists():
|
||
print(f"源目录不存在: {source_dir}")
|
||
return 0
|
||
|
||
target_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
print(f"\n{'='*60}")
|
||
print(f"同步: raw/articles/ → raw/homewiki-articles/")
|
||
print(f"{'='*60}")
|
||
|
||
count = 0
|
||
for file in source_dir.iterdir():
|
||
if file.is_file():
|
||
target_file = target_dir / file.name
|
||
if not target_file.exists():
|
||
shutil.copy2(file, target_file)
|
||
print(f" ✅ 已复制: {file.name}")
|
||
count += 1
|
||
else:
|
||
print(f" ⏭️ 已存在: {file.name}")
|
||
|
||
print(f"\n汇总: {count} 文件复制")
|
||
return count
|
||
|
||
|
||
def main():
|
||
if len(sys.argv) < 2:
|
||
print("用法: python sync_homewiki.py [concepts|entities|syntheses|raw|all]")
|
||
sys.exit(1)
|
||
|
||
task = sys.argv[1]
|
||
total = 0
|
||
|
||
if task in ('concepts', 'all'):
|
||
total += sync_directory("concepts", "wiki", "concept")
|
||
|
||
if task in ('entities', 'all'):
|
||
total += sync_directory("entities", "wiki", "entity")
|
||
|
||
if task in ('syntheses', 'all'):
|
||
total += sync_directory("syntheses", "wiki", "synthesis")
|
||
|
||
if task in ('raw', 'all'):
|
||
total += sync_raw_articles()
|
||
|
||
print(f"\n{'='*60}")
|
||
print(f"全部完成！共同步 {total} 个文件")
|
||
print(f"{'='*60}")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|