#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ 评估脚本 - 真实 API 结果对比 """ import json import os from datetime import datetime import re print('=== 真实实验评估开始 ===') # 配置 experiment_dir = os.path.join(os.getcwd(), "tools", "experiments", "wiki-generation-compare") output_dir = os.path.join(experiment_dir, "output") gold_standard_file = os.path.join(experiment_dir, "gold-standard.json") group_a_dir = os.path.join(output_dir, "group-a-real", "wiki") group_b_dir = os.path.join(output_dir, "group-b-real", "wiki") # 检查输出目录是否存在 if not os.path.exists(group_a_dir): print(f"[ERROR] A 组输出目录不存在: {group_a_dir}") print("请先运行真实实验脚本") exit(1) if not os.path.exists(group_b_dir): print(f"[ERROR] B 组输出目录不存在: {group_b_dir}") print("请先运行真实实验脚本") exit(1) # 加载金标准 with open(gold_standard_file, 'r', encoding='utf-8') as f: gold_standard = json.load(f) # 构建金标准术语表 gold_terms = {} for item in gold_standard.get('concepts', []): gold_terms[item['term']] = item for item in gold_standard.get('methods', []): gold_terms[item['term']] = item for item in gold_standard.get('entities', []): gold_terms[item['term']] = item # 辅助函数：计算术语覆盖度 def calculate_term_coverage(wiki_dir): generated_terms = {} if not os.path.exists(wiki_dir): return { 'totalGoldTerms': len(gold_terms), 'coveredTerms': 0, 'coverageRate': 0, 'generatedTerms': {}, 'missedTerms': list(gold_terms.keys()) } for root, dirs, files in os.walk(wiki_dir): for file in files: if file.endswith('.md'): term_name = file.replace('.md', '') file_path = os.path.join(root, file) generated_terms[term_name] = { 'file': file_path, 'hasDefinition': False, 'hasSourceLink': False, 'hasLineNumber': False } with open(file_path, 'r', encoding='utf-8') as f: content = f.read() if '**一句话定义**' in content: generated_terms[term_name]['hasDefinition'] = True if '[[raw/呼吸之间_李谨伯' in content: generated_terms[term_name]['hasSourceLink'] = True if '[raw:第一编 从身体入手.md:' in content: generated_terms[term_name]['hasLineNumber'] = True covered_count = sum(1 for term in gold_terms.keys() if term in generated_terms) coverage_rate = covered_count / len(gold_terms) if gold_terms else 0 return { 'totalGoldTerms': len(gold_terms), 'coveredTerms': covered_count, 'coverageRate': coverage_rate, 'generatedTerms': generated_terms, 'missedTerms': [term for term in gold_terms.keys() if term not in generated_terms] } # 辅助函数：评估 Frontmatter 规范性 def evaluate_frontmatter(wiki_dir): required_fields = ['categories', 'tags', 'created', 'source', 'type'] results = [] if not os.path.exists(wiki_dir): return { 'totalFiles': 0, 'avgFieldCompleteness': 0, 'results': [] } for root, dirs, files in os.walk(wiki_dir): for file in files: if file.endswith('.md'): file_path = os.path.join(root, file) with open(file_path, 'r', encoding='utf-8') as f: content = f.read() file_result = { 'file': file, 'hasFrontmatter': False, 'requiredFieldsComplete': 0, 'requiredFieldsTotal': len(required_fields), 'hasCategories': False, 'hasTags': False, 'hasCreated': False, 'hasSource': False, 'hasType': False, 'booleanQuotes': True } if content.startswith('---'): file_result['hasFrontmatter'] = True for field in required_fields: if f'{field}:' in content: file_result['has' + field.capitalize()] = True file_result['requiredFieldsComplete'] += 1 results.append(file_result) total_files = len(results) if total_files > 0: avg_field_completeness = sum(r['requiredFieldsComplete'] / r['requiredFieldsTotal'] for r in results) / total_files else: avg_field_completeness = 0 return { 'totalFiles': total_files, 'avgFieldCompleteness': avg_field_completeness, 'results': results } # 辅助函数：评估 Wikilink 质量 def evaluate_wikilinks(wiki_dir, generated_terms): total_links = 0 valid_links = 0 invalid_links = [] if not os.path.exists(wiki_dir): return { 'totalLinks': 0, 'validLinks': 0, 'invalidLinks': [], 'linkAccuracyRate': 1.0 } for root, dirs, files in os.walk(wiki_dir): for file in files: if file.endswith('.md'): file_path = os.path.join(root, file) with open(file_path, 'r', encoding='utf-8') as f: content = f.read() links = re.findall(r'\[\[([^\]]+)\]\]', content) for link in links: total_links += 1 if link in generated_terms: valid_links += 1 else: invalid_links.append({'source': file, 'target': link}) link_accuracy_rate = valid_links / total_links if total_links > 0 else 1.0 return { 'totalLinks': total_links, 'validLinks': valid_links, 'invalidLinks': invalid_links, 'linkAccuracyRate': link_accuracy_rate } # 评估 A 组 print('评估 A 组（真实 API）...') group_a_coverage = calculate_term_coverage(group_a_dir) group_a_frontmatter = evaluate_frontmatter(group_a_dir) group_a_wikilinks = evaluate_wikilinks(group_a_dir, group_a_coverage['generatedTerms']) # 评估 B 组 print('评估 B 组（真实 API）...') group_b_coverage = calculate_term_coverage(group_b_dir) group_b_frontmatter = evaluate_frontmatter(group_b_dir) group_b_wikilinks = evaluate_wikilinks(group_b_dir, group_b_coverage['generatedTerms']) # 加载元数据 group_a_metadata = None group_b_metadata = None try: with open(os.path.join(output_dir, "group-a-real", "metadata.json"), 'r', encoding='utf-8') as f: group_a_metadata = json.load(f) except: pass try: with open(os.path.join(output_dir, "group-b-real", "metadata.json"), 'r', encoding='utf-8') as f: group_b_metadata = json.load(f) except: pass # 计算得分 def calculate_score(coverage, frontmatter, wikilinks, metadata): # 内容完整性（40%） coverage_score = coverage['coverageRate'] if coverage['generatedTerms']: definition_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasDefinition']) / len(coverage['generatedTerms']) source_link_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasSourceLink']) / len(coverage['generatedTerms']) line_number_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasLineNumber']) / len(coverage['generatedTerms']) else: definition_quality = 0 source_link_quality = 0 line_number_quality = 0 content_completeness = coverage_score * 0.4 + definition_quality * 0.3 + source_link_quality * 0.2 + line_number_quality * 0.1 # 技术规范性（30%） field_completeness = frontmatter['avgFieldCompleteness'] link_quality = wikilinks['linkAccuracyRate'] technical_compliance = field_completeness * 0.6 + link_quality * 0.4 # 可维护性（20%） maintainability = 0.8 if metadata else 0.5 # 效率成本（10%） time_cost = metadata['durationMinutes'] if metadata else 0 efficiency = 1.0 if time_cost == 0 else (1.0 if time_cost < 10 else 0.8 if time_cost < 20 else 0.6) total_score = content_completeness * 0.4 + technical_compliance * 0.3 + maintainability * 0.2 + efficiency * 0.1 return { 'contentCompleteness': content_completeness, 'technicalCompliance': technical_compliance, 'maintainability': maintainability, 'efficiency': efficiency, 'totalScore': total_score } group_a_scores = calculate_score(group_a_coverage, group_a_frontmatter, group_a_wikilinks, group_a_metadata) group_b_scores = calculate_score(group_b_coverage, group_b_frontmatter, group_b_wikilinks, group_b_metadata) # 生成报告 a_start_time = group_a_metadata.get('startTime') if group_a_metadata else 'N/A' a_end_time = group_a_metadata.get('endTime') if group_a_metadata else 'N/A' a_duration = group_a_metadata.get('durationMinutes', 0) if group_a_metadata else 0 a_pages = group_a_metadata.get('pagesGenerated', 0) if group_a_metadata else 0 a_api_cost = group_a_metadata.get('apiCost', 'N/A') if group_a_metadata else 'N/A' b_start_time = group_b_metadata.get('startTime') if group_b_metadata else 'N/A' b_end_time = group_b_metadata.get('endTime') if group_b_metadata else 'N/A' b_duration = group_b_metadata.get('durationMinutes', 0) if group_b_metadata else 0 b_pages = group_b_metadata.get('pagesGenerated', 0) if group_b_metadata else 0 b_api_cost = group_b_metadata.get('apiCost', 'N/A') if group_b_metadata else 'N/A' report = f"""# Wiki 生成质量对比实验报告（真实 API 测试） ## 实验概览 - **测试文件**: raw/呼吸之间_李谨伯/第一编 从身体入手.md - **金标准术语数**: {len(gold_terms)} - **A 组模式**: Two-Step（分析 + 生成）- 真实 OpenAI API - **B 组模式**: Single-Step（直接生成）- 真实 OpenAI API - **API 模型**: gpt-4o-mini - **权重配置**: 40% 内容完整性 + 30% 技术规范性 + 20% 可维护性 + 10% 效率成本 ## 执行时间 | 组别 | 模式 | 开始时间 | 结束时间 | 耗时（分钟） | 生成页面数 | API 成本 | |------|------|----------|----------|-------------|-----------|---------| | A 组 | Two-Step | {a_start_time} | {a_end_time} | {a_duration} | {a_pages} | {a_api_cost} | | B 组 | Single-Step | {b_start_time} | {b_end_time} | {b_duration} | {b_pages} | {b_api_cost} | ## 详细得分 ### A 组（Two-Step - 真实 API） | 维度 | 得分 | 权重 | 加权得分 | |------|------|------|---------| | 内容完整性 | {round(group_a_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_a_scores['contentCompleteness'] * 0.4 * 100, 1)} | | 技术规范性 | {round(group_a_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_a_scores['technicalCompliance'] * 0.3 * 100, 1)} | | 可维护性 | {round(group_a_scores['maintainability'] * 100, 1)}% | 20% | {round(group_a_scores['maintainability'] * 0.2 * 100, 1)} | | 效率成本 | {round(group_a_scores['efficiency'] * 100, 1)}% | 10% | {round(group_a_scores['efficiency'] * 0.1 * 100, 1)} | | **总分** | - | **100%** | **{round(group_a_scores['totalScore'] * 100, 1)}** | ### B 组（Single-Step - 真实 API） | 维度 | 得分 | 权重 | 加权得分 | |------|------|------|---------| | 内容完整性 | {round(group_b_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_b_scores['contentCompleteness'] * 0.4 * 100, 1)} | | 技术规范性 | {round(group_b_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_b_scores['technicalCompliance'] * 0.3 * 100, 1)} | | 可维护性 | {round(group_b_scores['maintainability'] * 100, 1)}% | 20% | {round(group_b_scores['maintainability'] * 0.2 * 100, 1)} | | 效率成本 | {round(group_b_scores['efficiency'] * 100, 1)}% | 10% | {round(group_b_scores['efficiency'] * 0.1 * 100, 1)} | | **总分** | - | **100%** | **{round(group_b_scores['totalScore'] * 100, 1)}** | ## 详细对比 ### 1. 术语覆盖度 | 组别 | 金标准术语数 | 覆盖术语数 | 覆盖率 | 未覆盖术语数 | |------|-------------|-----------|--------|-----------| | A 组 | {group_a_coverage['totalGoldTerms']} | {group_a_coverage['coveredTerms']} | {round(group_a_coverage['coverageRate'] * 100, 1)}% | {len(group_a_coverage['missedTerms'])} | | B 组 | {group_b_coverage['totalGoldTerms']} | {group_b_coverage['coveredTerms']} | {round(group_b_coverage['coverageRate'] * 100, 1)}% | {len(group_b_coverage['missedTerms'])} | **A 组未覆盖术语**: {', '.join(group_a_coverage['missedTerms'][:10])}{'...' if len(group_a_coverage['missedTerms']) > 10 else ''} **B 组未覆盖术语**: {', '.join(group_b_coverage['missedTerms'][:10])}{'...' if len(group_b_coverage['missedTerms']) > 10 else ''} ### 2. Frontmatter 规范性 | 组别 | 总文件数 | 平均字段完整度 | |------|---------|---------------| | A 组 | {group_a_frontmatter['totalFiles']} | {round(group_a_frontmatter['avgFieldCompleteness'] * 100, 1)}% | | B 组 | {group_b_frontmatter['totalFiles']} | {round(group_b_frontmatter['avgFieldCompleteness'] * 100, 1)}% | ### 3. Wikilink 质量 | 组别 | 总链接数 | 有效链接数 | 无效链接数 | 链接准确率 | |------|---------|-----------|-----------|-----------| | A 组 | {group_a_wikilinks['totalLinks']} | {group_a_wikilinks['validLinks']} | {len(group_a_wikilinks['invalidLinks'])} | {round(group_a_wikilinks['linkAccuracyRate'] * 100, 1)}% | | B 组 | {group_b_wikilinks['totalLinks']} | {group_bikilinks['validLinks']} | {len(group_b_wikilinks['invalidLinks'])} | {round(group_b_wikilinks['linkAccuracyRate'] * 100, 1)}% | ### 4. 内容质量细节 | 组别 | 有定义页面 | 有来源链接页面 | 有行号标注页面 | |------|-----------|---------------|---------------| | A 组 | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_a_coverage['generatedTerms'])} | | B 组 | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_b_coverage['generatedTerms'])} | ## 结论 ### 综合评价 """ if group_a_scores['totalScore'] > group_b_scores['totalScore']: winner = "A 组（Two-Step 模式）" winner_score = round(group_a_scores['totalScore'] * 100, 1) reason = f"高出 {round((group_a_scores['totalScore'] - group_b_scores['totalScore']) * 100, 1)} 分" elif group_b_scores['totalScore'] > group_a_scores['totalScore']: winner = "B 组（Single-Step 模式）" winner_score = round(group_b_scores['totalScore'] * 100, 1) reason = f"高出 {round((group_b_scores['totalScore'] - group_a_scores['totalScore']) * 100, 1)} 分" else: winner = "A 组和 B 组得分相当" winner_score = round(group_a_scores['totalScore'] * 100, 1) reason = "差异 < 0.5 分" report += f"""**{winner}总分更高 ({winner_score} 分)，{reason}。** ### 各维度对比 """ if group_a_scores['contentCompleteness'] > group_b_scores['contentCompleteness']: report += f"- **内容完整性**: A 组优于 B 组（{round(group_a_scores['contentCompleteness'] * 100, 1)}% vs {round(group_b_scores['contentCompleteness'] * 100, 1)}%）" elif group_b_scores['contentCompleteness'] > group_a_scores['contentCompleteness']: report += f"- **内容完整性**: B 组优于 A 组（{round(group_b_scores['contentCompleteness'] * 100, 1)}% vs {round(group_a_scores['contentCompleteness'] * 100, 1)}%）" else: report += f"- **内容完整性**: A 组和 B 组持平（{round(group_a_scores['contentCompleteness'] * 100, 1)}% vs {round(group_b_scores['contentCompleteness'] * 100, 1)}%）" report += "\n" if group_a_scores['technicalCompliance'] > group_b_scores['technicalCompliance']: report += f"- **技术规范性**: A 组优于 B 组（{round(group_a_scores['technicalCompliance'] * 100, 1)}% vs {round(group_b_scores['technicalCompliance'] * 100, 1)}%）" elif group_b_scores['technicalCompliance'] > group_a_scores['technicalCompliance']: report += f"- **技术规范性**: B 组优于 A 组（{round(group_b_scores['technicalCompliance'] * 100, 1)}% vs {round(group_a_scores['technicalCompliance'] * 100, 1)}%）" else: report += f"- **技术规范性**: A 组和 B 组持平（{round(group_a_scores['technicalCompliance'] * 100, 1)}% vs {round(group_b_scores['technicalCompliance'] * 100, 1)}%）" report += "\n" if group_a_scores['efficiency'] > group_b_scores['efficiency']: report += f"- **效率成本**: A 组优于 B 组（耗时 {a_duration} vs {b_duration} 分钟）" elif group_b_scores['efficiency'] > group_a_scores['efficiency']: report += f"- **效率成本**: B 组优于 A 组（耗时 {b_duration} vs {a_duration} 分钟）" else: report += "- **效率成本**: A 组和 B 组持平" report += f""" ### 成本效益分析 - **A 组总成本**: {a_api_cost}（Analysis + Generation） - **B 组总成本**: {b_api_cost}（Generation only） - **成本差异**: A 组需要两次 LLM 调用，成本约为 B 组的 1.5-2 倍 - **质量提升**: {winner} 提供了更好的内容质量，但需要更高的成本 ## 实际应用建议 ### 何时使用 Two-Step 模式 - 对内容质量要求高的项目 - 术语复杂、需要深度分析的场景 - 有充足预算和时间的情况 ### 何时使用 Single-Step 模式 - 对速度和成本敏感的项目 - 简单内容、术语明确的场景 - 预算有限或需要快速迭代的情况 ## 附录：生成页面列表 ### A 组生成页面（真实 API） {', '.join(group_a_coverage['generatedTerms'].keys())} ### B 组生成页面（真实 API） {', '.join(group_b_coverage['generatedTerms'].keys())} --- **报告生成时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} **测试环境**: Windows, Python 3.x, OpenAI gpt-4o-mini """ # 保存报告 report_file = os.path.join(output_dir, "report-real.md") with open(report_file, 'w', encoding='utf-8') as f: f.write(report) print('=== 评估完成 ===') print(f'报告已保存到 {report_file}') print('') print('总分对比:') print(f'A 组（Two-Step - 真实 API）: {round(group_a_scores['totalScore'] * 100, 1)} 分') print(f'B 组（Single-Step - 真实 API）: {round(group_b_scores['totalScore'] * 100, 1)} 分') print(f'获胜者: {winner}')