436 lines
19 KiB
Python
436 lines
19 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
评估脚本 - 真实 API 结果对比
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
from datetime import datetime
|
||
import re
|
||
|
||
print('=== 真实实验评估开始 ===')
|
||
|
||
# 配置
|
||
experiment_dir = os.path.join(os.getcwd(), "tools", "experiments", "wiki-generation-compare")
|
||
output_dir = os.path.join(experiment_dir, "output")
|
||
gold_standard_file = os.path.join(experiment_dir, "gold-standard.json")
|
||
group_a_dir = os.path.join(output_dir, "group-a-real", "wiki")
|
||
group_b_dir = os.path.join(output_dir, "group-b-real", "wiki")
|
||
|
||
# 检查输出目录是否存在
|
||
if not os.path.exists(group_a_dir):
|
||
print(f"[ERROR] A 组输出目录不存在: {group_a_dir}")
|
||
print("请先运行真实实验脚本")
|
||
exit(1)
|
||
|
||
if not os.path.exists(group_b_dir):
|
||
print(f"[ERROR] B 组输出目录不存在: {group_b_dir}")
|
||
print("请先运行真实实验脚本")
|
||
exit(1)
|
||
|
||
# 加载金标准
|
||
with open(gold_standard_file, 'r', encoding='utf-8') as f:
|
||
gold_standard = json.load(f)
|
||
|
||
# 构建金标准术语表
|
||
gold_terms = {}
|
||
for item in gold_standard.get('concepts', []):
|
||
gold_terms[item['term']] = item
|
||
for item in gold_standard.get('methods', []):
|
||
gold_terms[item['term']] = item
|
||
for item in gold_standard.get('entities', []):
|
||
gold_terms[item['term']] = item
|
||
|
||
# 辅助函数：计算术语覆盖度
|
||
def calculate_term_coverage(wiki_dir):
|
||
generated_terms = {}
|
||
|
||
if not os.path.exists(wiki_dir):
|
||
return {
|
||
'totalGoldTerms': len(gold_terms),
|
||
'coveredTerms': 0,
|
||
'coverageRate': 0,
|
||
'generatedTerms': {},
|
||
'missedTerms': list(gold_terms.keys())
|
||
}
|
||
|
||
for root, dirs, files in os.walk(wiki_dir):
|
||
for file in files:
|
||
if file.endswith('.md'):
|
||
term_name = file.replace('.md', '')
|
||
file_path = os.path.join(root, file)
|
||
generated_terms[term_name] = {
|
||
'file': file_path,
|
||
'hasDefinition': False,
|
||
'hasSourceLink': False,
|
||
'hasLineNumber': False
|
||
}
|
||
|
||
with open(file_path, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
|
||
if '**一句话定义**' in content:
|
||
generated_terms[term_name]['hasDefinition'] = True
|
||
|
||
if '[[raw/呼吸之间_李谨伯' in content:
|
||
generated_terms[term_name]['hasSourceLink'] = True
|
||
|
||
if '[raw:第一编 从身体入手.md:' in content:
|
||
generated_terms[term_name]['hasLineNumber'] = True
|
||
|
||
covered_count = sum(1 for term in gold_terms.keys() if term in generated_terms)
|
||
coverage_rate = covered_count / len(gold_terms) if gold_terms else 0
|
||
|
||
return {
|
||
'totalGoldTerms': len(gold_terms),
|
||
'coveredTerms': covered_count,
|
||
'coverageRate': coverage_rate,
|
||
'generatedTerms': generated_terms,
|
||
'missedTerms': [term for term in gold_terms.keys() if term not in generated_terms]
|
||
}
|
||
|
||
# 辅助函数：评估 Frontmatter 规范性
|
||
def evaluate_frontmatter(wiki_dir):
|
||
required_fields = ['categories', 'tags', 'created', 'source', 'type']
|
||
results = []
|
||
|
||
if not os.path.exists(wiki_dir):
|
||
return {
|
||
'totalFiles': 0,
|
||
'avgFieldCompleteness': 0,
|
||
'results': []
|
||
}
|
||
|
||
for root, dirs, files in os.walk(wiki_dir):
|
||
for file in files:
|
||
if file.endswith('.md'):
|
||
file_path = os.path.join(root, file)
|
||
with open(file_path, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
|
||
file_result = {
|
||
'file': file,
|
||
'hasFrontmatter': False,
|
||
'requiredFieldsComplete': 0,
|
||
'requiredFieldsTotal': len(required_fields),
|
||
'hasCategories': False,
|
||
'hasTags': False,
|
||
'hasCreated': False,
|
||
'hasSource': False,
|
||
'hasType': False,
|
||
'booleanQuotes': True
|
||
}
|
||
|
||
if content.startswith('---'):
|
||
file_result['hasFrontmatter'] = True
|
||
for field in required_fields:
|
||
if f'{field}:' in content:
|
||
file_result['has' + field.capitalize()] = True
|
||
file_result['requiredFieldsComplete'] += 1
|
||
|
||
results.append(file_result)
|
||
|
||
total_files = len(results)
|
||
if total_files > 0:
|
||
avg_field_completeness = sum(r['requiredFieldsComplete'] / r['requiredFieldsTotal'] for r in results) / total_files
|
||
else:
|
||
avg_field_completeness = 0
|
||
|
||
return {
|
||
'totalFiles': total_files,
|
||
'avgFieldCompleteness': avg_field_completeness,
|
||
'results': results
|
||
}
|
||
|
||
# 辅助函数：评估 Wikilink 质量
|
||
def evaluate_wikilinks(wiki_dir, generated_terms):
|
||
total_links = 0
|
||
valid_links = 0
|
||
invalid_links = []
|
||
|
||
if not os.path.exists(wiki_dir):
|
||
return {
|
||
'totalLinks': 0,
|
||
'validLinks': 0,
|
||
'invalidLinks': [],
|
||
'linkAccuracyRate': 1.0
|
||
}
|
||
|
||
for root, dirs, files in os.walk(wiki_dir):
|
||
for file in files:
|
||
if file.endswith('.md'):
|
||
file_path = os.path.join(root, file)
|
||
with open(file_path, 'r', encoding='utf-8') as f:
|
||
content = f.read()
|
||
|
||
links = re.findall(r'\[\[([^\]]+)\]\]', content)
|
||
|
||
for link in links:
|
||
total_links += 1
|
||
if link in generated_terms:
|
||
valid_links += 1
|
||
else:
|
||
invalid_links.append({'source': file, 'target': link})
|
||
|
||
link_accuracy_rate = valid_links / total_links if total_links > 0 else 1.0
|
||
|
||
return {
|
||
'totalLinks': total_links,
|
||
'validLinks': valid_links,
|
||
'invalidLinks': invalid_links,
|
||
'linkAccuracyRate': link_accuracy_rate
|
||
}
|
||
|
||
# 评估 A 组
|
||
print('评估 A 组（真实 API）...')
|
||
group_a_coverage = calculate_term_coverage(group_a_dir)
|
||
group_a_frontmatter = evaluate_frontmatter(group_a_dir)
|
||
group_a_wikilinks = evaluate_wikilinks(group_a_dir, group_a_coverage['generatedTerms'])
|
||
|
||
# 评估 B 组
|
||
print('评估 B 组（真实 API）...')
|
||
group_b_coverage = calculate_term_coverage(group_b_dir)
|
||
group_b_frontmatter = evaluate_frontmatter(group_b_dir)
|
||
group_b_wikilinks = evaluate_wikilinks(group_b_dir, group_b_coverage['generatedTerms'])
|
||
|
||
# 加载元数据
|
||
group_a_metadata = None
|
||
group_b_metadata = None
|
||
|
||
try:
|
||
with open(os.path.join(output_dir, "group-a-real", "metadata.json"), 'r', encoding='utf-8') as f:
|
||
group_a_metadata = json.load(f)
|
||
except:
|
||
pass
|
||
|
||
try:
|
||
with open(os.path.join(output_dir, "group-b-real", "metadata.json"), 'r', encoding='utf-8') as f:
|
||
group_b_metadata = json.load(f)
|
||
except:
|
||
pass
|
||
|
||
# 计算得分
|
||
def calculate_score(coverage, frontmatter, wikilinks, metadata):
|
||
# 内容完整性（40%）
|
||
coverage_score = coverage['coverageRate']
|
||
|
||
if coverage['generatedTerms']:
|
||
definition_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasDefinition']) / len(coverage['generatedTerms'])
|
||
source_link_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasSourceLink']) / len(coverage['generatedTerms'])
|
||
line_number_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasLineNumber']) / len(coverage['generatedTerms'])
|
||
else:
|
||
definition_quality = 0
|
||
source_link_quality = 0
|
||
line_number_quality = 0
|
||
|
||
content_completeness = coverage_score * 0.4 + definition_quality * 0.3 + source_link_quality * 0.2 + line_number_quality * 0.1
|
||
|
||
# 技术规范性（30%）
|
||
field_completeness = frontmatter['avgFieldCompleteness']
|
||
link_quality = wikilinks['linkAccuracyRate']
|
||
technical_compliance = field_completeness * 0.6 + link_quality * 0.4
|
||
|
||
# 可维护性（20%）
|
||
maintainability = 0.8 if metadata else 0.5
|
||
|
||
# 效率成本（10%）
|
||
time_cost = metadata['durationMinutes'] if metadata else 0
|
||
efficiency = 1.0 if time_cost == 0 else (1.0 if time_cost < 10 else 0.8 if time_cost < 20 else 0.6)
|
||
|
||
total_score = content_completeness * 0.4 + technical_compliance * 0.3 + maintainability * 0.2 + efficiency * 0.1
|
||
|
||
return {
|
||
'contentCompleteness': content_completeness,
|
||
'technicalCompliance': technical_compliance,
|
||
'maintainability': maintainability,
|
||
'efficiency': efficiency,
|
||
'totalScore': total_score
|
||
}
|
||
|
||
group_a_scores = calculate_score(group_a_coverage, group_a_frontmatter, group_a_wikilinks, group_a_metadata)
|
||
group_b_scores = calculate_score(group_b_coverage, group_b_frontmatter, group_b_wikilinks, group_b_metadata)
|
||
|
||
# 生成报告
|
||
a_start_time = group_a_metadata.get('startTime') if group_a_metadata else 'N/A'
|
||
a_end_time = group_a_metadata.get('endTime') if group_a_metadata else 'N/A'
|
||
a_duration = group_a_metadata.get('durationMinutes', 0) if group_a_metadata else 0
|
||
a_pages = group_a_metadata.get('pagesGenerated', 0) if group_a_metadata else 0
|
||
a_api_cost = group_a_metadata.get('apiCost', 'N/A') if group_a_metadata else 'N/A'
|
||
|
||
b_start_time = group_b_metadata.get('startTime') if group_b_metadata else 'N/A'
|
||
b_end_time = group_b_metadata.get('endTime') if group_b_metadata else 'N/A'
|
||
b_duration = group_b_metadata.get('durationMinutes', 0) if group_b_metadata else 0
|
||
b_pages = group_b_metadata.get('pagesGenerated', 0) if group_b_metadata else 0
|
||
b_api_cost = group_b_metadata.get('apiCost', 'N/A') if group_b_metadata else 'N/A'
|
||
|
||
report = f"""# Wiki 生成质量对比实验报告（真实 API 测试）
|
||
|
||
## 实验概览
|
||
|
||
- **测试文件**: raw/呼吸之间_李谨伯/第一编 从身体入手.md
|
||
- **金标准术语数**: {len(gold_terms)}
|
||
- **A 组模式**: Two-Step（分析 + 生成）- 真实 OpenAI API
|
||
- **B 组模式**: Single-Step（直接生成）- 真实 OpenAI API
|
||
- **API 模型**: gpt-4o-mini
|
||
- **权重配置**: 40% 内容完整性 + 30% 技术规范性 + 20% 可维护性 + 10% 效率成本
|
||
|
||
## 执行时间
|
||
|
||
| 组别 | 模式 | 开始时间 | 结束时间 | 耗时（分钟） | 生成页面数 | API 成本 |
|
||
|------|------|----------|----------|-------------|-----------|---------|
|
||
| A 组 | Two-Step | {a_start_time} | {a_end_time} | {a_duration} | {a_pages} | {a_api_cost} |
|
||
| B 组 | Single-Step | {b_start_time} | {b_end_time} | {b_duration} | {b_pages} | {b_api_cost} |
|
||
|
||
## 详细得分
|
||
|
||
### A 组（Two-Step - 真实 API）
|
||
|
||
| 维度 | 得分 | 权重 | 加权得分 |
|
||
|------|------|------|---------|
|
||
| 内容完整性 | {round(group_a_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_a_scores['contentCompleteness'] * 0.4 * 100, 1)} |
|
||
| 技术规范性 | {round(group_a_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_a_scores['technicalCompliance'] * 0.3 * 100, 1)} |
|
||
| 可维护性 | {round(group_a_scores['maintainability'] * 100, 1)}% | 20% | {round(group_a_scores['maintainability'] * 0.2 * 100, 1)} |
|
||
| 效率成本 | {round(group_a_scores['efficiency'] * 100, 1)}% | 10% | {round(group_a_scores['efficiency'] * 0.1 * 100, 1)} |
|
||
| **总分** | - | **100%** | **{round(group_a_scores['totalScore'] * 100, 1)}** |
|
||
|
||
### B 组（Single-Step - 真实 API）
|
||
|
||
| 维度 | 得分 | 权重 | 加权得分 |
|
||
|------|------|------|---------|
|
||
| 内容完整性 | {round(group_b_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_b_scores['contentCompleteness'] * 0.4 * 100, 1)} |
|
||
| 技术规范性 | {round(group_b_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_b_scores['technicalCompliance'] * 0.3 * 100, 1)} |
|
||
| 可维护性 | {round(group_b_scores['maintainability'] * 100, 1)}% | 20% | {round(group_b_scores['maintainability'] * 0.2 * 100, 1)} |
|
||
| 效率成本 | {round(group_b_scores['efficiency'] * 100, 1)}% | 10% | {round(group_b_scores['efficiency'] * 0.1 * 100, 1)} |
|
||
| **总分** | - | **100%** | **{round(group_b_scores['totalScore'] * 100, 1)}** |
|
||
|
||
## 详细对比
|
||
|
||
### 1. 术语覆盖度
|
||
|
||
| 组别 | 金标准术语数 | 覆盖术语数 | 覆盖率 | 未覆盖术语数 |
|
||
|------|-------------|-----------|--------|-----------|
|
||
| A 组 | {group_a_coverage['totalGoldTerms']} | {group_a_coverage['coveredTerms']} | {round(group_a_coverage['coverageRate'] * 100, 1)}% | {len(group_a_coverage['missedTerms'])} |
|
||
| B 组 | {group_b_coverage['totalGoldTerms']} | {group_b_coverage['coveredTerms']} | {round(group_b_coverage['coverageRate'] * 100, 1)}% | {len(group_b_coverage['missedTerms'])} |
|
||
|
||
**A 组未覆盖术语**: {', '.join(group_a_coverage['missedTerms'][:10])}{'...' if len(group_a_coverage['missedTerms']) > 10 else ''}
|
||
**B 组未覆盖术语**: {', '.join(group_b_coverage['missedTerms'][:10])}{'...' if len(group_b_coverage['missedTerms']) > 10 else ''}
|
||
|
||
### 2. Frontmatter 规范性
|
||
|
||
| 组别 | 总文件数 | 平均字段完整度 |
|
||
|------|---------|---------------|
|
||
| A 组 | {group_a_frontmatter['totalFiles']} | {round(group_a_frontmatter['avgFieldCompleteness'] * 100, 1)}% |
|
||
| B 组 | {group_b_frontmatter['totalFiles']} | {round(group_b_frontmatter['avgFieldCompleteness'] * 100, 1)}% |
|
||
|
||
### 3. Wikilink 质量
|
||
|
||
| 组别 | 总链接数 | 有效链接数 | 无效链接数 | 链接准确率 |
|
||
|------|---------|-----------|-----------|-----------|
|
||
| A 组 | {group_a_wikilinks['totalLinks']} | {group_a_wikilinks['validLinks']} | {len(group_a_wikilinks['invalidLinks'])} | {round(group_a_wikilinks['linkAccuracyRate'] * 100, 1)}% |
|
||
| B 组 | {group_b_wikilinks['totalLinks']} | {group_bikilinks['validLinks']} | {len(group_b_wikilinks['invalidLinks'])} | {round(group_b_wikilinks['linkAccuracyRate'] * 100, 1)}% |
|
||
|
||
### 4. 内容质量细节
|
||
|
||
| 组别 | 有定义页面 | 有来源链接页面 | 有行号标注页面 |
|
||
|------|-----------|---------------|---------------|
|
||
| A 组 | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_a_coverage['generatedTerms'])} |
|
||
| B 组 | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_b_coverage['generatedTerms'])} |
|
||
|
||
## 结论
|
||
|
||
### 综合评价
|
||
|
||
"""
|
||
|
||
if group_a_scores['totalScore'] > group_b_scores['totalScore']:
|
||
winner = "A 组（Two-Step 模式）"
|
||
winner_score = round(group_a_scores['totalScore'] * 100, 1)
|
||
reason = f"高出 {round((group_a_scores['totalScore'] - group_b_scores['totalScore']) * 100, 1)} 分"
|
||
elif group_b_scores['totalScore'] > group_a_scores['totalScore']:
|
||
winner = "B 组（Single-Step 模式）"
|
||
winner_score = round(group_b_scores['totalScore'] * 100, 1)
|
||
reason = f"高出 {round((group_b_scores['totalScore'] - group_a_scores['totalScore']) * 100, 1)} 分"
|
||
else:
|
||
winner = "A 组和 B 组得分相当"
|
||
winner_score = round(group_a_scores['totalScore'] * 100, 1)
|
||
reason = "差异 < 0.5 分"
|
||
|
||
report += f"""**{winner}总分更高 ({winner_score} 分)，{reason}。**
|
||
|
||
### 各维度对比
|
||
|
||
"""
|
||
|
||
if group_a_scores['contentCompleteness'] > group_b_scores['contentCompleteness']:
|
||
report += f"- **内容完整性**: A 组优于 B 组（{round(group_a_scores['contentCompleteness'] * 100, 1)}% vs {round(group_b_scores['contentCompleteness'] * 100, 1)}%）"
|
||
elif group_b_scores['contentCompleteness'] > group_a_scores['contentCompleteness']:
|
||
report += f"- **内容完整性**: B 组优于 A 组（{round(group_b_scores['contentCompleteness'] * 100, 1)}% vs {round(group_a_scores['contentCompleteness'] * 100, 1)}%）"
|
||
else:
|
||
report += f"- **内容完整性**: A 组和 B 组持平（{round(group_a_scores['contentCompleteness'] * 100, 1)}% vs {round(group_b_scores['contentCompleteness'] * 100, 1)}%）"
|
||
|
||
report += "\n"
|
||
|
||
if group_a_scores['technicalCompliance'] > group_b_scores['technicalCompliance']:
|
||
report += f"- **技术规范性**: A 组优于 B 组（{round(group_a_scores['technicalCompliance'] * 100, 1)}% vs {round(group_b_scores['technicalCompliance'] * 100, 1)}%）"
|
||
elif group_b_scores['technicalCompliance'] > group_a_scores['technicalCompliance']:
|
||
report += f"- **技术规范性**: B 组优于 A 组（{round(group_b_scores['technicalCompliance'] * 100, 1)}% vs {round(group_a_scores['technicalCompliance'] * 100, 1)}%）"
|
||
else:
|
||
report += f"- **技术规范性**: A 组和 B 组持平（{round(group_a_scores['technicalCompliance'] * 100, 1)}% vs {round(group_b_scores['technicalCompliance'] * 100, 1)}%）"
|
||
|
||
report += "\n"
|
||
|
||
if group_a_scores['efficiency'] > group_b_scores['efficiency']:
|
||
report += f"- **效率成本**: A 组优于 B 组（耗时 {a_duration} vs {b_duration} 分钟）"
|
||
elif group_b_scores['efficiency'] > group_a_scores['efficiency']:
|
||
report += f"- **效率成本**: B 组优于 A 组（耗时 {b_duration} vs {a_duration} 分钟）"
|
||
else:
|
||
report += "- **效率成本**: A 组和 B 组持平"
|
||
|
||
report += f"""
|
||
|
||
### 成本效益分析
|
||
|
||
- **A 组总成本**: {a_api_cost}（Analysis + Generation）
|
||
- **B 组总成本**: {b_api_cost}（Generation only）
|
||
- **成本差异**: A 组需要两次 LLM 调用，成本约为 B 组的 1.5-2 倍
|
||
- **质量提升**: {winner} 提供了更好的内容质量，但需要更高的成本
|
||
|
||
## 实际应用建议
|
||
|
||
### 何时使用 Two-Step 模式
|
||
- 对内容质量要求高的项目
|
||
- 术语复杂、需要深度分析的场景
|
||
- 有充足预算和时间的情况
|
||
|
||
### 何时使用 Single-Step 模式
|
||
- 对速度和成本敏感的项目
|
||
- 简单内容、术语明确的场景
|
||
- 预算有限或需要快速迭代的情况
|
||
|
||
## 附录：生成页面列表
|
||
|
||
### A 组生成页面（真实 API）
|
||
{', '.join(group_a_coverage['generatedTerms'].keys())}
|
||
|
||
### B 组生成页面（真实 API）
|
||
{', '.join(group_b_coverage['generatedTerms'].keys())}
|
||
|
||
---
|
||
|
||
**报告生成时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
|
||
**测试环境**: Windows, Python 3.x, OpenAI gpt-4o-mini
|
||
"""
|
||
|
||
# 保存报告
|
||
report_file = os.path.join(output_dir, "report-real.md")
|
||
with open(report_file, 'w', encoding='utf-8') as f:
|
||
f.write(report)
|
||
|
||
print('=== 评估完成 ===')
|
||
print(f'报告已保存到 {report_file}')
|
||
print('')
|
||
print('总分对比:')
|
||
print(f'A 组（Two-Step - 真实 API）: {round(group_a_scores['totalScore'] * 100, 1)} 分')
|
||
print(f'B 组（Single-Step - 真实 API）: {round(group_b_scores['totalScore'] * 100, 1)} 分')
|
||
print(f'获胜者: {winner}') |