chore(vault): backup 2026-07-03 21:40:41

This commit is contained in:
2026-07-03 21:40:41 +08:00
parent 7635b2e377
commit f21c3a1662
4 changed files with 830 additions and 11 deletions
@@ -0,0 +1,436 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
评估脚本 - 真实 API 结果对比
"""
import json
import os
from datetime import datetime
import re
print('=== 真实实验评估开始 ===')
# 配置
experiment_dir = os.path.join(os.getcwd(), "tools", "experiments", "wiki-generation-compare")
output_dir = os.path.join(experiment_dir, "output")
gold_standard_file = os.path.join(experiment_dir, "gold-standard.json")
group_a_dir = os.path.join(output_dir, "group-a-real", "wiki")
group_b_dir = os.path.join(output_dir, "group-b-real", "wiki")
# 检查输出目录是否存在
if not os.path.exists(group_a_dir):
print(f"[ERROR] A 组输出目录不存在: {group_a_dir}")
print("请先运行真实实验脚本")
exit(1)
if not os.path.exists(group_b_dir):
print(f"[ERROR] B 组输出目录不存在: {group_b_dir}")
print("请先运行真实实验脚本")
exit(1)
# 加载金标准
with open(gold_standard_file, 'r', encoding='utf-8') as f:
gold_standard = json.load(f)
# 构建金标准术语表
gold_terms = {}
for item in gold_standard.get('concepts', []):
gold_terms[item['term']] = item
for item in gold_standard.get('methods', []):
gold_terms[item['term']] = item
for item in gold_standard.get('entities', []):
gold_terms[item['term']] = item
# 辅助函数:计算术语覆盖度
def calculate_term_coverage(wiki_dir):
generated_terms = {}
if not os.path.exists(wiki_dir):
return {
'totalGoldTerms': len(gold_terms),
'coveredTerms': 0,
'coverageRate': 0,
'generatedTerms': {},
'missedTerms': list(gold_terms.keys())
}
for root, dirs, files in os.walk(wiki_dir):
for file in files:
if file.endswith('.md'):
term_name = file.replace('.md', '')
file_path = os.path.join(root, file)
generated_terms[term_name] = {
'file': file_path,
'hasDefinition': False,
'hasSourceLink': False,
'hasLineNumber': False
}
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
if '**一句话定义**' in content:
generated_terms[term_name]['hasDefinition'] = True
if '[[raw/呼吸之间_李谨伯' in content:
generated_terms[term_name]['hasSourceLink'] = True
if '[raw:第一编 从身体入手.md:' in content:
generated_terms[term_name]['hasLineNumber'] = True
covered_count = sum(1 for term in gold_terms.keys() if term in generated_terms)
coverage_rate = covered_count / len(gold_terms) if gold_terms else 0
return {
'totalGoldTerms': len(gold_terms),
'coveredTerms': covered_count,
'coverageRate': coverage_rate,
'generatedTerms': generated_terms,
'missedTerms': [term for term in gold_terms.keys() if term not in generated_terms]
}
# 辅助函数:评估 Frontmatter 规范性
def evaluate_frontmatter(wiki_dir):
required_fields = ['categories', 'tags', 'created', 'source', 'type']
results = []
if not os.path.exists(wiki_dir):
return {
'totalFiles': 0,
'avgFieldCompleteness': 0,
'results': []
}
for root, dirs, files in os.walk(wiki_dir):
for file in files:
if file.endswith('.md'):
file_path = os.path.join(root, file)
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
file_result = {
'file': file,
'hasFrontmatter': False,
'requiredFieldsComplete': 0,
'requiredFieldsTotal': len(required_fields),
'hasCategories': False,
'hasTags': False,
'hasCreated': False,
'hasSource': False,
'hasType': False,
'booleanQuotes': True
}
if content.startswith('---'):
file_result['hasFrontmatter'] = True
for field in required_fields:
if f'{field}:' in content:
file_result['has' + field.capitalize()] = True
file_result['requiredFieldsComplete'] += 1
results.append(file_result)
total_files = len(results)
if total_files > 0:
avg_field_completeness = sum(r['requiredFieldsComplete'] / r['requiredFieldsTotal'] for r in results) / total_files
else:
avg_field_completeness = 0
return {
'totalFiles': total_files,
'avgFieldCompleteness': avg_field_completeness,
'results': results
}
# 辅助函数:评估 Wikilink 质量
def evaluate_wikilinks(wiki_dir, generated_terms):
total_links = 0
valid_links = 0
invalid_links = []
if not os.path.exists(wiki_dir):
return {
'totalLinks': 0,
'validLinks': 0,
'invalidLinks': [],
'linkAccuracyRate': 1.0
}
for root, dirs, files in os.walk(wiki_dir):
for file in files:
if file.endswith('.md'):
file_path = os.path.join(root, file)
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
links = re.findall(r'\[\[([^\]]+)\]\]', content)
for link in links:
total_links += 1
if link in generated_terms:
valid_links += 1
else:
invalid_links.append({'source': file, 'target': link})
link_accuracy_rate = valid_links / total_links if total_links > 0 else 1.0
return {
'totalLinks': total_links,
'validLinks': valid_links,
'invalidLinks': invalid_links,
'linkAccuracyRate': link_accuracy_rate
}
# 评估 A 组
print('评估 A 组(真实 API...')
group_a_coverage = calculate_term_coverage(group_a_dir)
group_a_frontmatter = evaluate_frontmatter(group_a_dir)
group_a_wikilinks = evaluate_wikilinks(group_a_dir, group_a_coverage['generatedTerms'])
# 评估 B 组
print('评估 B 组(真实 API...')
group_b_coverage = calculate_term_coverage(group_b_dir)
group_b_frontmatter = evaluate_frontmatter(group_b_dir)
group_b_wikilinks = evaluate_wikilinks(group_b_dir, group_b_coverage['generatedTerms'])
# 加载元数据
group_a_metadata = None
group_b_metadata = None
try:
with open(os.path.join(output_dir, "group-a-real", "metadata.json"), 'r', encoding='utf-8') as f:
group_a_metadata = json.load(f)
except:
pass
try:
with open(os.path.join(output_dir, "group-b-real", "metadata.json"), 'r', encoding='utf-8') as f:
group_b_metadata = json.load(f)
except:
pass
# 计算得分
def calculate_score(coverage, frontmatter, wikilinks, metadata):
# 内容完整性(40%
coverage_score = coverage['coverageRate']
if coverage['generatedTerms']:
definition_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasDefinition']) / len(coverage['generatedTerms'])
source_link_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasSourceLink']) / len(coverage['generatedTerms'])
line_number_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasLineNumber']) / len(coverage['generatedTerms'])
else:
definition_quality = 0
source_link_quality = 0
line_number_quality = 0
content_completeness = coverage_score * 0.4 + definition_quality * 0.3 + source_link_quality * 0.2 + line_number_quality * 0.1
# 技术规范性(30%
field_completeness = frontmatter['avgFieldCompleteness']
link_quality = wikilinks['linkAccuracyRate']
technical_compliance = field_completeness * 0.6 + link_quality * 0.4
# 可维护性(20%
maintainability = 0.8 if metadata else 0.5
# 效率成本(10%
time_cost = metadata['durationMinutes'] if metadata else 0
efficiency = 1.0 if time_cost == 0 else (1.0 if time_cost < 10 else 0.8 if time_cost < 20 else 0.6)
total_score = content_completeness * 0.4 + technical_compliance * 0.3 + maintainability * 0.2 + efficiency * 0.1
return {
'contentCompleteness': content_completeness,
'technicalCompliance': technical_compliance,
'maintainability': maintainability,
'efficiency': efficiency,
'totalScore': total_score
}
group_a_scores = calculate_score(group_a_coverage, group_a_frontmatter, group_a_wikilinks, group_a_metadata)
group_b_scores = calculate_score(group_b_coverage, group_b_frontmatter, group_b_wikilinks, group_b_metadata)
# 生成报告
a_start_time = group_a_metadata.get('startTime') if group_a_metadata else 'N/A'
a_end_time = group_a_metadata.get('endTime') if group_a_metadata else 'N/A'
a_duration = group_a_metadata.get('durationMinutes', 0) if group_a_metadata else 0
a_pages = group_a_metadata.get('pagesGenerated', 0) if group_a_metadata else 0
a_api_cost = group_a_metadata.get('apiCost', 'N/A') if group_a_metadata else 'N/A'
b_start_time = group_b_metadata.get('startTime') if group_b_metadata else 'N/A'
b_end_time = group_b_metadata.get('endTime') if group_b_metadata else 'N/A'
b_duration = group_b_metadata.get('durationMinutes', 0) if group_b_metadata else 0
b_pages = group_b_metadata.get('pagesGenerated', 0) if group_b_metadata else 0
b_api_cost = group_b_metadata.get('apiCost', 'N/A') if group_b_metadata else 'N/A'
report = f"""# Wiki 生成质量对比实验报告(真实 API 测试)
## 实验概览
- **测试文件**: raw/呼吸之间_李谨伯/第一编 从身体入手.md
- **金标准术语数**: {len(gold_terms)}
- **A 组模式**: Two-Step(分析 + 生成)- 真实 OpenAI API
- **B 组模式**: Single-Step(直接生成)- 真实 OpenAI API
- **API 模型**: gpt-4o-mini
- **权重配置**: 40% 内容完整性 + 30% 技术规范性 + 20% 可维护性 + 10% 效率成本
## 执行时间
| 组别 | 模式 | 开始时间 | 结束时间 | 耗时(分钟) | 生成页面数 | API 成本 |
|------|------|----------|----------|-------------|-----------|---------|
| A 组 | Two-Step | {a_start_time} | {a_end_time} | {a_duration} | {a_pages} | {a_api_cost} |
| B 组 | Single-Step | {b_start_time} | {b_end_time} | {b_duration} | {b_pages} | {b_api_cost} |
## 详细得分
### A 组(Two-Step - 真实 API
| 维度 | 得分 | 权重 | 加权得分 |
|------|------|------|---------|
| 内容完整性 | {round(group_a_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_a_scores['contentCompleteness'] * 0.4 * 100, 1)} |
| 技术规范性 | {round(group_a_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_a_scores['technicalCompliance'] * 0.3 * 100, 1)} |
| 可维护性 | {round(group_a_scores['maintainability'] * 100, 1)}% | 20% | {round(group_a_scores['maintainability'] * 0.2 * 100, 1)} |
| 效率成本 | {round(group_a_scores['efficiency'] * 100, 1)}% | 10% | {round(group_a_scores['efficiency'] * 0.1 * 100, 1)} |
| **总分** | - | **100%** | **{round(group_a_scores['totalScore'] * 100, 1)}** |
### B 组(Single-Step - 真实 API
| 维度 | 得分 | 权重 | 加权得分 |
|------|------|------|---------|
| 内容完整性 | {round(group_b_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_b_scores['contentCompleteness'] * 0.4 * 100, 1)} |
| 技术规范性 | {round(group_b_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_b_scores['technicalCompliance'] * 0.3 * 100, 1)} |
| 可维护性 | {round(group_b_scores['maintainability'] * 100, 1)}% | 20% | {round(group_b_scores['maintainability'] * 0.2 * 100, 1)} |
| 效率成本 | {round(group_b_scores['efficiency'] * 100, 1)}% | 10% | {round(group_b_scores['efficiency'] * 0.1 * 100, 1)} |
| **总分** | - | **100%** | **{round(group_b_scores['totalScore'] * 100, 1)}** |
## 详细对比
### 1. 术语覆盖度
| 组别 | 金标准术语数 | 覆盖术语数 | 覆盖率 | 未覆盖术语数 |
|------|-------------|-----------|--------|-----------|
| A 组 | {group_a_coverage['totalGoldTerms']} | {group_a_coverage['coveredTerms']} | {round(group_a_coverage['coverageRate'] * 100, 1)}% | {len(group_a_coverage['missedTerms'])} |
| B 组 | {group_b_coverage['totalGoldTerms']} | {group_b_coverage['coveredTerms']} | {round(group_b_coverage['coverageRate'] * 100, 1)}% | {len(group_b_coverage['missedTerms'])} |
**A 组未覆盖术语**: {', '.join(group_a_coverage['missedTerms'][:10])}{'...' if len(group_a_coverage['missedTerms']) > 10 else ''}
**B 组未覆盖术语**: {', '.join(group_b_coverage['missedTerms'][:10])}{'...' if len(group_b_coverage['missedTerms']) > 10 else ''}
### 2. Frontmatter 规范性
| 组别 | 总文件数 | 平均字段完整度 |
|------|---------|---------------|
| A 组 | {group_a_frontmatter['totalFiles']} | {round(group_a_frontmatter['avgFieldCompleteness'] * 100, 1)}% |
| B 组 | {group_b_frontmatter['totalFiles']} | {round(group_b_frontmatter['avgFieldCompleteness'] * 100, 1)}% |
### 3. Wikilink 质量
| 组别 | 总链接数 | 有效链接数 | 无效链接数 | 链接准确率 |
|------|---------|-----------|-----------|-----------|
| A 组 | {group_a_wikilinks['totalLinks']} | {group_a_wikilinks['validLinks']} | {len(group_a_wikilinks['invalidLinks'])} | {round(group_a_wikilinks['linkAccuracyRate'] * 100, 1)}% |
| B 组 | {group_b_wikilinks['totalLinks']} | {group_bikilinks['validLinks']} | {len(group_b_wikilinks['invalidLinks'])} | {round(group_b_wikilinks['linkAccuracyRate'] * 100, 1)}% |
### 4. 内容质量细节
| 组别 | 有定义页面 | 有来源链接页面 | 有行号标注页面 |
|------|-----------|---------------|---------------|
| A 组 | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_a_coverage['generatedTerms'])} |
| B 组 | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_b_coverage['generatedTerms'])} |
## 结论
### 综合评价
"""
if group_a_scores['totalScore'] > group_b_scores['totalScore']:
winner = "A 组(Two-Step 模式)"
winner_score = round(group_a_scores['totalScore'] * 100, 1)
reason = f"高出 {round((group_a_scores['totalScore'] - group_b_scores['totalScore']) * 100, 1)}"
elif group_b_scores['totalScore'] > group_a_scores['totalScore']:
winner = "B 组(Single-Step 模式)"
winner_score = round(group_b_scores['totalScore'] * 100, 1)
reason = f"高出 {round((group_b_scores['totalScore'] - group_a_scores['totalScore']) * 100, 1)}"
else:
winner = "A 组和 B 组得分相当"
winner_score = round(group_a_scores['totalScore'] * 100, 1)
reason = "差异 < 0.5 分"
report += f"""**{winner}总分更高 ({winner_score} 分){reason}。**
### 各维度对比
"""
if group_a_scores['contentCompleteness'] > group_b_scores['contentCompleteness']:
report += f"- **内容完整性**: A 组优于 B 组({round(group_a_scores['contentCompleteness'] * 100, 1)}% vs {round(group_b_scores['contentCompleteness'] * 100, 1)}%"
elif group_b_scores['contentCompleteness'] > group_a_scores['contentCompleteness']:
report += f"- **内容完整性**: B 组优于 A 组({round(group_b_scores['contentCompleteness'] * 100, 1)}% vs {round(group_a_scores['contentCompleteness'] * 100, 1)}%"
else:
report += f"- **内容完整性**: A 组和 B 组持平({round(group_a_scores['contentCompleteness'] * 100, 1)}% vs {round(group_b_scores['contentCompleteness'] * 100, 1)}%"
report += "\n"
if group_a_scores['technicalCompliance'] > group_b_scores['technicalCompliance']:
report += f"- **技术规范性**: A 组优于 B 组({round(group_a_scores['technicalCompliance'] * 100, 1)}% vs {round(group_b_scores['technicalCompliance'] * 100, 1)}%"
elif group_b_scores['technicalCompliance'] > group_a_scores['technicalCompliance']:
report += f"- **技术规范性**: B 组优于 A 组({round(group_b_scores['technicalCompliance'] * 100, 1)}% vs {round(group_a_scores['technicalCompliance'] * 100, 1)}%"
else:
report += f"- **技术规范性**: A 组和 B 组持平({round(group_a_scores['technicalCompliance'] * 100, 1)}% vs {round(group_b_scores['technicalCompliance'] * 100, 1)}%"
report += "\n"
if group_a_scores['efficiency'] > group_b_scores['efficiency']:
report += f"- **效率成本**: A 组优于 B 组(耗时 {a_duration} vs {b_duration} 分钟)"
elif group_b_scores['efficiency'] > group_a_scores['efficiency']:
report += f"- **效率成本**: B 组优于 A 组(耗时 {b_duration} vs {a_duration} 分钟)"
else:
report += "- **效率成本**: A 组和 B 组持平"
report += f"""
### 成本效益分析
- **A 组总成本**: {a_api_cost}Analysis + Generation
- **B 组总成本**: {b_api_cost}Generation only
- **成本差异**: A 组需要两次 LLM 调用,成本约为 B 组的 1.5-2 倍
- **质量提升**: {winner} 提供了更好的内容质量,但需要更高的成本
## 实际应用建议
### 何时使用 Two-Step 模式
- 对内容质量要求高的项目
- 术语复杂、需要深度分析的场景
- 有充足预算和时间的情况
### 何时使用 Single-Step 模式
- 对速度和成本敏感的项目
- 简单内容、术语明确的场景
- 预算有限或需要快速迭代的情况
## 附录:生成页面列表
### A 组生成页面(真实 API
{', '.join(group_a_coverage['generatedTerms'].keys())}
### B 组生成页面(真实 API
{', '.join(group_b_coverage['generatedTerms'].keys())}
---
**报告生成时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
**测试环境**: Windows, Python 3.x, OpenAI gpt-4o-mini
"""
# 保存报告
report_file = os.path.join(output_dir, "report-real.md")
with open(report_file, 'w', encoding='utf-8') as f:
f.write(report)
print('=== 评估完成 ===')
print(f'报告已保存到 {report_file}')
print('')
print('总分对比:')
print(f'A 组(Two-Step - 真实 API: {round(group_a_scores['totalScore'] * 100, 1)}')
print(f'B 组(Single-Step - 真实 API: {round(group_b_scores['totalScore'] * 100, 1)}')
print(f'获胜者: {winner}')