giteahh/llm_wiki
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(vault): backup 2026-07-03 21:18:15

Browse Source
This commit is contained in:
hehaiguang1123
2026-07-03 21:18:15 +08:00
parent 81c7842432
commit 7635b2e377
34 changed files with 3691 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/experiments/wiki-generation-compare/scripts/evaluate.py
+394
View File
@@ -0,0 +1,394 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
评估脚本
"""
import json
import os
from datetime import datetime
import re
print('=== 开始评估 ===')
# 配置 - 从当前工作目录计算
base_dir = os.getcwd()
experiment_dir = os.path.join(base_dir, "tools", "experiments", "wiki-generation-compare")
output_dir = os.path.join(experiment_dir, "output")
gold_standard_file = os.path.join(experiment_dir, "gold-standard.json")
group_a_dir = os.path.join(output_dir, "group-a", "wiki")
group_b_dir = os.path.join(output_dir, "group-b", "wiki")
# 加载金标准
with open(gold_standard_file, 'r', encoding='utf-8') as f:
gold_standard = json.load(f)
# 构建金标准术语表
gold_terms = {}
for item in gold_standard.get('concepts', []):
gold_terms[item['term']] = item
for item in gold_standard.get('methods', []):
gold_terms[item['term']] = item
for item in gold_standard.get('entities', []):
gold_terms[item['term']] = item
# 辅助函数：计算术语覆盖度
def calculate_term_coverage(wiki_dir):
generated_terms = {}
if not os.path.exists(wiki_dir):
return {
'totalGoldTerms': len(gold_terms),
'coveredTerms': 0,
'coverageRate': 0,
'generatedTerms': {},
'missedTerms': list(gold_terms.keys())
}
for root, dirs, files in os.walk(wiki_dir):
for file in files:
if file.endswith('.md'):
term_name = file.replace('.md', '')
file_path = os.path.join(root, file)
generated_terms[term_name] = {
'file': file_path,
'hasDefinition': False,
'hasSourceLink': False,
'hasLineNumber': False
}
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
if '**一句话定义**' in content:
generated_terms[term_name]['hasDefinition'] = True
if '[[raw/呼吸之间_李谨伯' in content:
generated_terms[term_name]['hasSourceLink'] = True
if '[raw:第一编 从身体入手.md:' in content:
generated_terms[term_name]['hasLineNumber'] = True
covered_count = sum(1 for term in gold_terms.keys() if term in generated_terms)
coverage_rate = covered_count / len(gold_terms) if gold_terms else 0
return {
'totalGoldTerms': len(gold_terms),
'coveredTerms': covered_count,
'coverageRate': coverage_rate,
'generatedTerms': generated_terms,
'missedTerms': [term for term in gold_terms.keys() if term not in generated_terms]
}
# 辅助函数：评估 Frontmatter 规范性
def evaluate_frontmatter(wiki_dir):
required_fields = ['categories', 'tags', 'created', 'source', 'type']
results = []
if not os.path.exists(wiki_dir):
return {
'totalFiles': 0,
'avgFieldCompleteness': 0,
'results': []
}
for root, dirs, files in os.walk(wiki_dir):
for file in files:
if file.endswith('.md'):
file_path = os.path.join(root, file)
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
file_result = {
'file': file,
'hasFrontmatter': False,
'requiredFieldsComplete': 0,
'requiredFieldsTotal': len(required_fields),
'hasCategories': False,
'hasTags': False,
'hasCreated': False,
'hasSource': False,
'hasType': False,
'booleanQuotes': True
}
if content.startswith('---'):
file_result['hasFrontmatter'] = True
# 简化的 frontmatter 解析
for field in required_fields:
if f'{field}:' in content:
file_result['has' + field.capitalize()] = True
file_result['requiredFieldsComplete'] += 1
results.append(file_result)
total_files = len(results)
if total_files > 0:
avg_field_completeness = sum(r['requiredFieldsComplete'] / r['requiredFieldsTotal'] for r in results) / total_files
else:
avg_field_completeness = 0
return {
'totalFiles': total_files,
'avgFieldCompleteness': avg_field_completeness,
'results': results
}
# 辅助函数：评估 Wikilink 质量
def evaluate_wikilinks(wiki_dir, generated_terms):
total_links = 0
valid_links = 0
invalid_links = []
if not os.path.exists(wiki_dir):
return {
'totalLinks': 0,
'validLinks': 0,
'invalidLinks': [],
'linkAccuracyRate': 1.0
}
for root, dirs, files in os.walk(wiki_dir):
for file in files:
if file.endswith('.md'):
file_path = os.path.join(root, file)
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
links = re.findall(r'\[\[([^\]]+)\]\]', content)
for link in links:
total_links += 1
if link in generated_terms:
valid_links += 1
else:
invalid_links.append({'source': file, 'target': link})
link_accuracy_rate = valid_links / total_links if total_links > 0 else 1.0
return {
'totalLinks': total_links,
'validLinks': valid_links,
'invalidLinks': invalid_links,
'linkAccuracyRate': link_accuracy_rate
}
# 评估 A 组
print('评估 A 组...')
group_a_coverage = calculate_term_coverage(group_a_dir)
group_a_frontmatter = evaluate_frontmatter(group_a_dir)
group_a_wikilinks = evaluate_wikilinks(group_a_dir, group_a_coverage['generatedTerms'])
# 评估 B 组
print('评估 B 组...')
group_b_coverage = calculate_term_coverage(group_b_dir)
group_b_frontmatter = evaluate_frontmatter(group_b_dir)
group_b_wikilinks = evaluate_wikilinks(group_b_dir, group_b_coverage['generatedTerms'])
# 加载元数据
group_a_metadata = None
group_b_metadata = None
try:
with open(os.path.join(output_dir, "group-a", "metadata.json"), 'r', encoding='utf-8') as f:
group_a_metadata = json.load(f)
except:
pass
try:
with open(os.path.join(output_dir, "group-b", "metadata.json"), 'r', encoding='utf-8') as f:
group_b_metadata = json.load(f)
except:
pass
# 计算得分
def calculate_score(coverage, frontmatter, wikilinks, metadata):
# 内容完整性（40%）
coverage_score = coverage['coverageRate']
if coverage['generatedTerms']:
definition_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasDefinition']) / len(coverage['generatedTerms'])
source_link_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasSourceLink']) / len(coverage['generatedTerms'])
line_number_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasLineNumber']) / len(coverage['generatedTerms'])
else:
definition_quality = 0
source_link_quality = 0
line_number_quality = 0
content_completeness = coverage_score * 0.4 + definition_quality * 0.3 + source_link_quality * 0.2 + line_number_quality * 0.1
# 技术规范性（30%）
field_completeness = frontmatter['avgFieldCompleteness']
link_quality = wikilinks['linkAccuracyRate']
technical_compliance = field_completeness * 0.6 + link_quality * 0.4
# 可维护性（20%）
maintainability = 0.8 if metadata else 0.5
# 效率成本（10%）
time_cost = metadata['durationMinutes'] if metadata else 0
efficiency = 1.0 if time_cost == 0 else (1.0 if time_cost < 10 else 0.8 if time_cost < 20 else 0.6)
total_score = content_completeness * 0.4 + technical_compliance * 0.3 + maintainability * 0.2 + efficiency * 0.1
return {
'contentCompleteness': content_completeness,
'technicalCompliance': technical_compliance,
'maintainability': maintainability,
'efficiency': efficiency,
'totalScore': total_score
}
group_a_scores = calculate_score(group_a_coverage, group_a_frontmatter, group_a_wikilinks, group_a_metadata)
group_b_scores = calculate_score(group_b_coverage, group_b_frontmatter, group_b_wikilinks, group_b_metadata)
# 生成报告
a_start_time = group_a_metadata.get('startTime') if group_a_metadata else 'N/A'
a_end_time = group_a_metadata.get('endTime') if group_a_metadata else 'N/A'
a_duration = group_a_metadata.get('durationMinutes', 0) if group_a_metadata else 0
a_pages = group_a_metadata.get('pagesGenerated', 0) if group_a_metadata else 0
b_start_time = group_b_metadata.get('startTime') if group_b_metadata else 'N/A'
b_end_time = group_b_metadata.get('endTime') if group_b_metadata else 'N/A'
b_duration = group_b_metadata.get('durationMinutes', 0) if group_b_metadata else 0
b_pages = group_b_metadata.get('pagesGenerated', 0) if group_b_metadata else 0
report = f"""# Wiki 生成质量对比实验报告
## 实验概览
- **测试文件**: raw/呼吸之间_李谨伯/第一编 从身体入手.md
- **金标准术语数**: {len(gold_terms)}
- **A 组模式**: Two-Step（分析 + 生成）
- **B 组模式**: Single-Step（直接生成）
- **权重配置**: 40% 内容完整性 + 30% 技术规范性 + 20% 可维护性 + 10% 效率成本
## 执行时间
| 组别 | 模式 | 开始时间 | 结束时间 | 耗时（分钟） | 生成页面数 |
|------|------|----------|----------|-------------|-----------|
| A 组 | Two-Step | {a_start_time} | {a_end_time} | {a_duration} | {a_pages} |
| B 组 | Single-Step | {b_start_time} | {b_end_time} | {b_duration} | {b_pages} |
## 详细得分
### A 组（Two-Step）
| 维度 | 得分 | 权重 | 加权得分 |
|------|------|------|---------|
| 内容完整性 | {round(group_a_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_a_scores['contentCompleteness'] * 0.4 * 100, 1)} |
| 技术规范性 | {round(group_a_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_a_scores['technicalCompliance'] * 0.3 * 100, 1)} |
| 可维护性 | {round(group_a_scores['maintainability'] * 100, 1)}% | 20% | {round(group_a_scores['maintainability'] * 0.2 * 100, 1)} |
| 效率成本 | {round(group_a_scores['efficiency'] * 100, 1)}% | 10% | {round(group_a_scores['efficiency'] * 0.1 * 100, 1)} |
| **总分** | - | **100%** | **{round(group_a_scores['totalScore'] * 100, 1)}** |
### B 组（Single-Step）
| 维度 | 得分 | 权重 | 加权得分 |
|------|------|------|---------|
| 内容完整性 | {round(group_b_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_b_scores['contentCompleteness'] * 0.4 * 100, 1)} |
| 技术规范性 | {round(group_b_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_b_scores['technicalCompliance'] * 0.3 * 100, 1)} |
| 可维护性 | {round(group_b_scores['maintainability'] * 100, 1)}% | 20% | {round(group_b_scores['maintainability'] * 0.2 * 100, 1)} |
| 效率成本 | {round(group_b_scores['efficiency'] * 100, 1)}% | 10% | {round(group_b_scores['efficiency'] * 0.1 * 100, 1)} |
| **总分** | - | **100%** | **{round(group_b_scores['totalScore'] * 100, 1)}** |
## 详细对比
### 1. 术语覆盖度
| 组别 | 金标准术语数 | 覆盖术语数 | 覆盖率 | 未覆盖术语 |
|------|-------------|-----------|--------|-----------|
| A 组 | {group_a_coverage['totalGoldTerms']} | {group_a_coverage['coveredTerms']} | {round(group_a_coverage['coverageRate'] * 100, 1)}% | {len(group_a_coverage['missedTerms'])} 个 |
| B 组 | {group_b_coverage['totalGoldTerms']} | {group_b_coverage['coveredTerms']} | {round(group_b_coverage['coverageRate'] * 100, 1)}% | {len(group_b_coverage['missedTerms'])} 个 |
**A 组未覆盖术语**: {', '.join(group_a_coverage['missedTerms'][:5])}{'...' if len(group_a_coverage['missedTerms']) > 5 else ''}
**B 组未覆盖术语**: {', '.join(group_b_coverage['missedTerms'][:5])}{'...' if len(group_b_coverage['missedTerms']) > 5 else ''}
### 2. Frontmatter 规范性
| 组别 | 总文件数 | 平均字段完整度 |
|------|---------|---------------|
| A 组 | {group_a_frontmatter['totalFiles']} | {round(group_a_frontmatter['avgFieldCompleteness'] * 100, 1)}% |
| B 组 | {group_b_frontmatter['totalFiles']} | {round(group_b_frontmatter['avgFieldCompleteness'] * 100, 1)}% |
### 3. Wikilink 质量
| 组别 | 总链接数 | 有效链接数 | 无效链接数 | 链接准确率 |
|------|---------|-----------|-----------|-----------|
| A 组 | {group_a_wikilinks['totalLinks']} | {group_a_wikilinks['validLinks']} | {len(group_a_wikilinks['invalidLinks'])} | {round(group_a_wikilinks['linkAccuracyRate'] * 100, 1)}% |
| B 组 | {group_b_wikilinks['totalLinks']} | {group_b_wikilinks['validLinks']} | {len(group_b_wikilinks['invalidLinks'])} | {round(group_b_wikilinks['linkAccuracyRate'] * 100, 1)}% |
### 4. 内容质量细节
| 组别 | 有定义 | 有来源链接 | 有行号标注 |
|------|--------|-----------|-----------|
| A 组 | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_a_coverage['generatedTerms'])} |
| B 组 | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_b_coverage['generatedTerms'])} |
## 结论
### 综合评价
"""
if group_a_scores['totalScore'] > group_b_scores['totalScore']:
report += "**A 组（Two-Step 模式）总分更高，推荐使用。**"
elif group_b_scores['totalScore'] > group_a_scores['totalScore']:
report += "**B 组（Single-Step 模式）总分更高，推荐使用。**"
else:
report += "**A 组和 B 组得分相当，可根据其他因素选择。**"
report += f"""
### 各维度对比
"""
if group_a_scores['contentCompleteness'] > group_b_scores['contentCompleteness']:
report += f"- **内容完整性**: A 组优于 B 组（{round(group_a_scores['contentCompleteness'] * 100, 1)}% vs {round(group_b_scores['contentCompleteness'] * 100, 1)}%）"
else:
report += f"- **内容完整性**: B 组优于 A 组（{round(group_b_scores['contentCompleteness'] * 100, 1)}% vs {round(group_a_scores['contentCompleteness'] * 100, 1)}%）"
report += "\n"
if group_a_scores['technicalCompliance'] > group_b_scores['technicalCompliance']:
report += f"- **技术规范性**: A 组优于 B 组（{round(group_a_scores['technicalCompliance'] * 100, 1)}% vs {round(group_b_scores['technicalCompliance'] * 100, 1)}%）"
else:
report += f"- **技术规范性**: B 组优于 A 组（{round(group_b_scores['technicalCompliance'] * 100, 1)}% vs {round(group_a_scores['technicalCompliance'] * 100, 1)}%）"
report += "\n"
if group_a_scores['efficiency'] > group_b_scores['efficiency']:
report += f"- **效率成本**: A 组优于 B 组"
else:
report += f"- **效率成本**: B 组优于 A 组"
report += f"""
## 附录：生成页面列表
### A 组生成页面
{', '.join(group_a_coverage['generatedTerms'].keys())}
### B 组生成页面
{', '.join(group_b_coverage['generatedTerms'].keys())}
---
**报告生成时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
"""
# 保存报告
report_file = os.path.join(output_dir, "report.md")
with open(report_file, 'w', encoding='utf-8') as f:
f.write(report)
print('=== 评估完成 ===')
print(f'报告已保存到 {report_file}')
print('')
print('总分对比:')
a_score_str = str(round(group_a_scores['totalScore'] * 100, 1))
b_score_str = str(round(group_b_scores['totalScore'] * 100, 1))
print('A 组（Two-Step）: ' + a_score_str + ' 分')
print('B 组（Single-Step）: ' + b_score_str + ' 分')