chore(vault): backup 2026-07-03 21:40:41

2026-07-03 21:40:41 +08:00
parent 7635b2e377
commit f21c3a1662
4 changed files with 830 additions and 11 deletions
@@ -0,0 +1,436 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+评估脚本 - 真实 API 结果对比
+"""
+
+import json
+import os
+from datetime import datetime
+import re
+
+print('=== 真实实验评估开始 ===')
+
+# 配置
+experiment_dir = os.path.join(os.getcwd(), "tools", "experiments", "wiki-generation-compare")
+output_dir = os.path.join(experiment_dir, "output")
+gold_standard_file = os.path.join(experiment_dir, "gold-standard.json")
+group_a_dir = os.path.join(output_dir, "group-a-real", "wiki")
+group_b_dir = os.path.join(output_dir, "group-b-real", "wiki")
+
+# 检查输出目录是否存在
+if not os.path.exists(group_a_dir):
+    print(f"[ERROR] A 组输出目录不存在: {group_a_dir}")
+    print("请先运行真实实验脚本")
+    exit(1)
+
+if not os.path.exists(group_b_dir):
+    print(f"[ERROR] B 组输出目录不存在: {group_b_dir}")
+    print("请先运行真实实验脚本")
+    exit(1)
+
+# 加载金标准
+with open(gold_standard_file, 'r', encoding='utf-8') as f:
+    gold_standard = json.load(f)
+
+# 构建金标准术语表
+gold_terms = {}
+for item in gold_standard.get('concepts', []):
+    gold_terms[item['term']] = item
+for item in gold_standard.get('methods', []):
+    gold_terms[item['term']] = item
+for item in gold_standard.get('entities', []):
+    gold_terms[item['term']] = item
+
+# 辅助函数：计算术语覆盖度
+def calculate_term_coverage(wiki_dir):
+    generated_terms = {}
+    
+    if not os.path.exists(wiki_dir):
+        return {
+            'totalGoldTerms': len(gold_terms),
+            'coveredTerms': 0,
+            'coverageRate': 0,
+            'generatedTerms': {},
+            'missedTerms': list(gold_terms.keys())
+        }
+    
+    for root, dirs, files in os.walk(wiki_dir):
+        for file in files:
+            if file.endswith('.md'):
+                term_name = file.replace('.md', '')
+                file_path = os.path.join(root, file)
+                generated_terms[term_name] = {
+                    'file': file_path,
+                    'hasDefinition': False,
+                    'hasSourceLink': False,
+                    'hasLineNumber': False
+                }
+                
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    content = f.read()
+                
+                if '**一句话定义**' in content:
+                    generated_terms[term_name]['hasDefinition'] = True
+                
+                if '[[raw/呼吸之间_李谨伯' in content:
+                    generated_terms[term_name]['hasSourceLink'] = True
+                
+                if '[raw:第一编 从身体入手.md:' in content:
+                    generated_terms[term_name]['hasLineNumber'] = True
+    
+    covered_count = sum(1 for term in gold_terms.keys() if term in generated_terms)
+    coverage_rate = covered_count / len(gold_terms) if gold_terms else 0
+    
+    return {
+        'totalGoldTerms': len(gold_terms),
+        'coveredTerms': covered_count,
+        'coverageRate': coverage_rate,
+        'generatedTerms': generated_terms,
+        'missedTerms': [term for term in gold_terms.keys() if term not in generated_terms]
+    }
+
+# 辅助函数：评估 Frontmatter 规范性
+def evaluate_frontmatter(wiki_dir):
+    required_fields = ['categories', 'tags', 'created', 'source', 'type']
+    results = []
+    
+    if not os.path.exists(wiki_dir):
+        return {
+            'totalFiles': 0,
+            'avgFieldCompleteness': 0,
+            'results': []
+        }
+    
+    for root, dirs, files in os.walk(wiki_dir):
+        for file in files:
+            if file.endswith('.md'):
+                file_path = os.path.join(root, file)
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    content = f.read()
+                
+                file_result = {
+                    'file': file,
+                    'hasFrontmatter': False,
+                    'requiredFieldsComplete': 0,
+                    'requiredFieldsTotal': len(required_fields),
+                    'hasCategories': False,
+                    'hasTags': False,
+                    'hasCreated': False,
+                    'hasSource': False,
+                    'hasType': False,
+                    'booleanQuotes': True
+                }
+                
+                if content.startswith('---'):
+                    file_result['hasFrontmatter'] = True
+                    for field in required_fields:
+                        if f'{field}:' in content:
+                            file_result['has' + field.capitalize()] = True
+                            file_result['requiredFieldsComplete'] += 1
+                
+                results.append(file_result)
+    
+    total_files = len(results)
+    if total_files > 0:
+        avg_field_completeness = sum(r['requiredFieldsComplete'] / r['requiredFieldsTotal'] for r in results) / total_files
+    else:
+        avg_field_completeness = 0
+    
+    return {
+        'totalFiles': total_files,
+        'avgFieldCompleteness': avg_field_completeness,
+        'results': results
+    }
+
+# 辅助函数：评估 Wikilink 质量
+def evaluate_wikilinks(wiki_dir, generated_terms):
+    total_links = 0
+    valid_links = 0
+    invalid_links = []
+    
+    if not os.path.exists(wiki_dir):
+        return {
+            'totalLinks': 0,
+            'validLinks': 0,
+            'invalidLinks': [],
+            'linkAccuracyRate': 1.0
+        }
+    
+    for root, dirs, files in os.walk(wiki_dir):
+        for file in files:
+            if file.endswith('.md'):
+                file_path = os.path.join(root, file)
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    content = f.read()
+                
+                links = re.findall(r'\[\[([^\]]+)\]\]', content)
+                
+                for link in links:
+                    total_links += 1
+                    if link in generated_terms:
+                        valid_links += 1
+                    else:
+                        invalid_links.append({'source': file, 'target': link})
+    
+    link_accuracy_rate = valid_links / total_links if total_links > 0 else 1.0
+    
+    return {
+        'totalLinks': total_links,
+        'validLinks': valid_links,
+        'invalidLinks': invalid_links,
+        'linkAccuracyRate': link_accuracy_rate
+    }
+
+# 评估 A 组
+print('评估 A 组（真实 API）...')
+group_a_coverage = calculate_term_coverage(group_a_dir)
+group_a_frontmatter = evaluate_frontmatter(group_a_dir)
+group_a_wikilinks = evaluate_wikilinks(group_a_dir, group_a_coverage['generatedTerms'])
+
+# 评估 B 组
+print('评估 B 组（真实 API）...')
+group_b_coverage = calculate_term_coverage(group_b_dir)
+group_b_frontmatter = evaluate_frontmatter(group_b_dir)
+group_b_wikilinks = evaluate_wikilinks(group_b_dir, group_b_coverage['generatedTerms'])
+
+# 加载元数据
+group_a_metadata = None
+group_b_metadata = None
+
+try:
+    with open(os.path.join(output_dir, "group-a-real", "metadata.json"), 'r', encoding='utf-8') as f:
+        group_a_metadata = json.load(f)
+except:
+    pass
+
+try:
+    with open(os.path.join(output_dir, "group-b-real", "metadata.json"), 'r', encoding='utf-8') as f:
+        group_b_metadata = json.load(f)
+except:
+    pass
+
+# 计算得分
+def calculate_score(coverage, frontmatter, wikilinks, metadata):
+    # 内容完整性（40%）
+    coverage_score = coverage['coverageRate']
+    
+    if coverage['generatedTerms']:
+        definition_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasDefinition']) / len(coverage['generatedTerms'])
+        source_link_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasSourceLink']) / len(coverage['generatedTerms'])
+        line_number_quality = sum(1 for t in coverage['generatedTerms'].values() if t['hasLineNumber']) / len(coverage['generatedTerms'])
+    else:
+        definition_quality = 0
+        source_link_quality = 0
+        line_number_quality = 0
+    
+    content_completeness = coverage_score * 0.4 + definition_quality * 0.3 + source_link_quality * 0.2 + line_number_quality * 0.1
+    
+    # 技术规范性（30%）
+    field_completeness = frontmatter['avgFieldCompleteness']
+    link_quality = wikilinks['linkAccuracyRate']
+    technical_compliance = field_completeness * 0.6 + link_quality * 0.4
+    
+    # 可维护性（20%）
+    maintainability = 0.8 if metadata else 0.5
+    
+    # 效率成本（10%）
+    time_cost = metadata['durationMinutes'] if metadata else 0
+    efficiency = 1.0 if time_cost == 0 else (1.0 if time_cost < 10 else 0.8 if time_cost < 20 else 0.6)
+    
+    total_score = content_completeness * 0.4 + technical_compliance * 0.3 + maintainability * 0.2 + efficiency * 0.1
+    
+    return {
+        'contentCompleteness': content_completeness,
+        'technicalCompliance': technical_compliance,
+        'maintainability': maintainability,
+        'efficiency': efficiency,
+        'totalScore': total_score
+    }
+
+group_a_scores = calculate_score(group_a_coverage, group_a_frontmatter, group_a_wikilinks, group_a_metadata)
+group_b_scores = calculate_score(group_b_coverage, group_b_frontmatter, group_b_wikilinks, group_b_metadata)
+
+# 生成报告
+a_start_time = group_a_metadata.get('startTime') if group_a_metadata else 'N/A'
+a_end_time = group_a_metadata.get('endTime') if group_a_metadata else 'N/A'
+a_duration = group_a_metadata.get('durationMinutes', 0) if group_a_metadata else 0
+a_pages = group_a_metadata.get('pagesGenerated', 0) if group_a_metadata else 0
+a_api_cost = group_a_metadata.get('apiCost', 'N/A') if group_a_metadata else 'N/A'
+
+b_start_time = group_b_metadata.get('startTime') if group_b_metadata else 'N/A'
+b_end_time = group_b_metadata.get('endTime') if group_b_metadata else 'N/A'
+b_duration = group_b_metadata.get('durationMinutes', 0) if group_b_metadata else 0
+b_pages = group_b_metadata.get('pagesGenerated', 0) if group_b_metadata else 0
+b_api_cost = group_b_metadata.get('apiCost', 'N/A') if group_b_metadata else 'N/A'
+
+report = f"""# Wiki 生成质量对比实验报告（真实 API 测试）
+
+## 实验概览
+
+- **测试文件**: raw/呼吸之间_李谨伯/第一编 从身体入手.md
+- **金标准术语数**: {len(gold_terms)}
+- **A 组模式**: Two-Step（分析 + 生成）- 真实 OpenAI API
+- **B 组模式**: Single-Step（直接生成）- 真实 OpenAI API
+- **API 模型**: gpt-4o-mini
+- **权重配置**: 40% 内容完整性 + 30% 技术规范性 + 20% 可维护性 + 10% 效率成本
+
+## 执行时间
+
+| 组别 | 模式 | 开始时间 | 结束时间 | 耗时（分钟） | 生成页面数 | API 成本 |
+|------|------|----------|----------|-------------|-----------|---------|
+| A 组 | Two-Step | {a_start_time} | {a_end_time} | {a_duration} | {a_pages} | {a_api_cost} |
+| B 组 | Single-Step | {b_start_time} | {b_end_time} | {b_duration} | {b_pages} | {b_api_cost} |
+
+## 详细得分
+
+### A 组（Two-Step - 真实 API）
+
+| 维度 | 得分 | 权重 | 加权得分 |
+|------|------|------|---------|
+| 内容完整性 | {round(group_a_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_a_scores['contentCompleteness'] * 0.4 * 100, 1)} |
+| 技术规范性 | {round(group_a_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_a_scores['technicalCompliance'] * 0.3 * 100, 1)} |
+| 可维护性 | {round(group_a_scores['maintainability'] * 100, 1)}% | 20% | {round(group_a_scores['maintainability'] * 0.2 * 100, 1)} |
+| 效率成本 | {round(group_a_scores['efficiency'] * 100, 1)}% | 10% | {round(group_a_scores['efficiency'] * 0.1 * 100, 1)} |
+| **总分** | - | **100%** | **{round(group_a_scores['totalScore'] * 100, 1)}** |
+
+### B 组（Single-Step - 真实 API）
+
+| 维度 | 得分 | 权重 | 加权得分 |
+|------|------|------|---------|
+| 内容完整性 | {round(group_b_scores['contentCompleteness'] * 100, 1)}% | 40% | {round(group_b_scores['contentCompleteness'] * 0.4 * 100, 1)} |
+| 技术规范性 | {round(group_b_scores['technicalCompliance'] * 100, 1)}% | 30% | {round(group_b_scores['technicalCompliance'] * 0.3 * 100, 1)} |
+| 可维护性 | {round(group_b_scores['maintainability'] * 100, 1)}% | 20% | {round(group_b_scores['maintainability'] * 0.2 * 100, 1)} |
+| 效率成本 | {round(group_b_scores['efficiency'] * 100, 1)}% | 10% | {round(group_b_scores['efficiency'] * 0.1 * 100, 1)} |
+| **总分** | - | **100%** | **{round(group_b_scores['totalScore'] * 100, 1)}** |
+
+## 详细对比
+
+### 1. 术语覆盖度
+
+| 组别 | 金标准术语数 | 覆盖术语数 | 覆盖率 | 未覆盖术语数 |
+|------|-------------|-----------|--------|-----------|
+| A 组 | {group_a_coverage['totalGoldTerms']} | {group_a_coverage['coveredTerms']} | {round(group_a_coverage['coverageRate'] * 100, 1)}% | {len(group_a_coverage['missedTerms'])} |
+| B 组 | {group_b_coverage['totalGoldTerms']} | {group_b_coverage['coveredTerms']} | {round(group_b_coverage['coverageRate'] * 100, 1)}% | {len(group_b_coverage['missedTerms'])} |
+
+**A 组未覆盖术语**: {', '.join(group_a_coverage['missedTerms'][:10])}{'...' if len(group_a_coverage['missedTerms']) > 10 else ''}
+**B 组未覆盖术语**: {', '.join(group_b_coverage['missedTerms'][:10])}{'...' if len(group_b_coverage['missedTerms']) > 10 else ''}
+
+### 2. Frontmatter 规范性
+
+| 组别 | 总文件数 | 平均字段完整度 |
+|------|---------|---------------|
+| A 组 | {group_a_frontmatter['totalFiles']} | {round(group_a_frontmatter['avgFieldCompleteness'] * 100, 1)}% |
+| B 组 | {group_b_frontmatter['totalFiles']} | {round(group_b_frontmatter['avgFieldCompleteness'] * 100, 1)}% |
+
+### 3. Wikilink 质量
+
+| 组别 | 总链接数 | 有效链接数 | 无效链接数 | 链接准确率 |
+|------|---------|-----------|-----------|-----------|
+| A 组 | {group_a_wikilinks['totalLinks']} | {group_a_wikilinks['validLinks']} | {len(group_a_wikilinks['invalidLinks'])} | {round(group_a_wikilinks['linkAccuracyRate'] * 100, 1)}% |
+| B 组 | {group_b_wikilinks['totalLinks']} | {group_bikilinks['validLinks']} | {len(group_b_wikilinks['invalidLinks'])} | {round(group_b_wikilinks['linkAccuracyRate'] * 100, 1)}% |
+
+### 4. 内容质量细节
+
+| 组别 | 有定义页面 | 有来源链接页面 | 有行号标注页面 |
+|------|-----------|---------------|---------------|
+| A 组 | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_a_coverage['generatedTerms'])} | {sum(1 for t in group_a_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_a_coverage['generatedTerms'])} |
+| B 组 | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasDefinition'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasSourceLink'])}/{len(group_b_coverage['generatedTerms'])} | {sum(1 for t in group_b_coverage['generatedTerms'].values() if t['hasLineNumber'])}/{len(group_b_coverage['generatedTerms'])} |
+
+## 结论
+
+### 综合评价
+
+"""
+
+if group_a_scores['totalScore'] > group_b_scores['totalScore']:
+    winner = "A 组（Two-Step 模式）"
+    winner_score = round(group_a_scores['totalScore'] * 100, 1)
+    reason = f"高出 {round((group_a_scores['totalScore'] - group_b_scores['totalScore']) * 100, 1)} 分"
+elif group_b_scores['totalScore'] > group_a_scores['totalScore']:
+    winner = "B 组（Single-Step 模式）"
+    winner_score = round(group_b_scores['totalScore'] * 100, 1)
+    reason = f"高出 {round((group_b_scores['totalScore'] - group_a_scores['totalScore']) * 100, 1)} 分"
+else:
+    winner = "A 组和 B 组得分相当"
+    winner_score = round(group_a_scores['totalScore'] * 100, 1)
+    reason = "差异 < 0.5 分"
+
+report += f"""**{winner}总分更高 ({winner_score} 分)，{reason}。**
+
+### 各维度对比
+
+"""
+
+if group_a_scores['contentCompleteness'] > group_b_scores['contentCompleteness']:
+    report += f"- **内容完整性**: A 组优于 B 组（{round(group_a_scores['contentCompleteness'] * 100, 1)}% vs {round(group_b_scores['contentCompleteness'] * 100, 1)}%）"
+elif group_b_scores['contentCompleteness'] > group_a_scores['contentCompleteness']:
+    report += f"- **内容完整性**: B 组优于 A 组（{round(group_b_scores['contentCompleteness'] * 100, 1)}% vs {round(group_a_scores['contentCompleteness'] * 100, 1)}%）"
+else:
+    report += f"- **内容完整性**: A 组和 B 组持平（{round(group_a_scores['contentCompleteness'] * 100, 1)}% vs {round(group_b_scores['contentCompleteness'] * 100, 1)}%）"
+
+report += "\n"
+
+if group_a_scores['technicalCompliance'] > group_b_scores['technicalCompliance']:
+    report += f"- **技术规范性**: A 组优于 B 组（{round(group_a_scores['technicalCompliance'] * 100, 1)}% vs {round(group_b_scores['technicalCompliance'] * 100, 1)}%）"
+elif group_b_scores['technicalCompliance'] > group_a_scores['technicalCompliance']:
+    report += f"- **技术规范性**: B 组优于 A 组（{round(group_b_scores['technicalCompliance'] * 100, 1)}% vs {round(group_a_scores['technicalCompliance'] * 100, 1)}%）"
+else:
+    report += f"- **技术规范性**: A 组和 B 组持平（{round(group_a_scores['technicalCompliance'] * 100, 1)}% vs {round(group_b_scores['technicalCompliance'] * 100, 1)}%）"
+
+report += "\n"
+
+if group_a_scores['efficiency'] > group_b_scores['efficiency']:
+    report += f"- **效率成本**: A 组优于 B 组（耗时 {a_duration} vs {b_duration} 分钟）"
+elif group_b_scores['efficiency'] > group_a_scores['efficiency']:
+    report += f"- **效率成本**: B 组优于 A 组（耗时 {b_duration} vs {a_duration} 分钟）"
+else:
+    report += "- **效率成本**: A 组和 B 组持平"
+
+report += f"""
+
+### 成本效益分析
+
+- **A 组总成本**: {a_api_cost}（Analysis + Generation）
+- **B 组总成本**: {b_api_cost}（Generation only）
+- **成本差异**: A 组需要两次 LLM 调用，成本约为 B 组的 1.5-2 倍
+- **质量提升**: {winner} 提供了更好的内容质量，但需要更高的成本
+
+## 实际应用建议
+
+### 何时使用 Two-Step 模式
+- 对内容质量要求高的项目
+- 术语复杂、需要深度分析的场景
+- 有充足预算和时间的情况
+
+### 何时使用 Single-Step 模式
+- 对速度和成本敏感的项目
+- 简单内容、术语明确的场景
+- 预算有限或需要快速迭代的情况
+
+## 附录：生成页面列表
+
+### A 组生成页面（真实 API）
+{', '.join(group_a_coverage['generatedTerms'].keys())}
+
+### B 组生成页面（真实 API）
+{', '.join(group_b_coverage['generatedTerms'].keys())}
+
+---
+
+**报告生成时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
+**测试环境**: Windows, Python 3.x, OpenAI gpt-4o-mini
+"""
+
+# 保存报告
+report_file = os.path.join(output_dir, "report-real.md")
+with open(report_file, 'w', encoding='utf-8') as f:
+    f.write(report)
+
+print('=== 评估完成 ===')
+print(f'报告已保存到 {report_file}')
+print('')
+print('总分对比:')
+print(f'A 组（Two-Step - 真实 API）: {round(group_a_scores['totalScore'] * 100, 1)} 分')
+print(f'B 组（Single-Step - 真实 API）: {round(group_b_scores['totalScore'] * 100, 1)} 分')
+print(f'获胜者: {winner}')
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+真实 A 组实验（Two-Step 模式）- 使用真实 OpenAI API
+"""
+
+import json
+import os
+import requests
+from datetime import datetime
+import re
+
+# 配置
+base_dir = os.getcwd()
+experiment_dir = os.path.join(base_dir, "tools", "experiments", "wiki-generation-compare")
+source_file = os.path.join(base_dir, "raw", "呼吸之间_李谨伯", "第一编 从身体入手.md")
+output_dir = os.path.join(experiment_dir, "output", "group-a-real")
+
+# 创建输出目录
+os.makedirs(output_dir, exist_ok=True)
+os.makedirs(os.path.join(output_dir, "wiki", "concepts"), exist_ok=True)
+os.makedirs(os.path.join(output_dir, "wiki", "methods"), exist_ok=True)
+os.makedirs(os.path.join(output_dir, "wiki", "entities"), exist_ok=True)
+
+# 记录开始时间
+start_time = datetime.now()
+start_time_str = start_time.strftime("%Y-%m-%d %H:%M:%S")
+
+print("=== A 组真实实验开始：Two-Step 模式 ===")
+print(f"开始时间: {start_time_str}")
+
+# 检查 API Key
+api_key = os.environ.get('OPENAI_API_KEY')
+if not api_key:
+    print("[ERROR] 未找到 OPENAI_API_KEY 环境变量")
+    exit(1)
+
+headers = {
+    "Authorization": f"Bearer {api_key}",
+    "Content-Type": "application/json"
+}
+
+# Step 1: Analysis
+print("\nStep 1: Analysis 阶段...")
+
+# 读取源文件
+with open(source_file, 'r', encoding='utf-8') as f:
+    source_content = f.read()
+
+# 读取分析 prompt
+analysis_prompt_file = os.path.join(experiment_dir, "prompts", "twostep-analysis.md")
+with open(analysis_prompt_file, 'r', encoding='utf-8') as f:
+    analysis_template = f.read()
+
+analysis_full_prompt = analysis_template.replace("{SOURCE_CONTENT}", source_content)
+
+print("  调用 OpenAI API 进行分析...")
+analysis_body = {
+    "model": "gpt-4o-mini",
+    "messages": [
+        {
+            "role": "system",
+            "content": "你是一位知识库分析专家。请分析源文件并提取结构化信息，仅输出 JSON 格式。"
+        },
+        {
+            "role": "user",
+            "content": analysis_full_prompt
+        }
+    ],
+    "temperature": 0.3,
+    "max_tokens": 8000
+}
+
+try:
+    response = requests.post("https://api.openai.com/v1/chat/completions", 
+                            headers=headers, 
+                            json=analysis_body, 
+                            timeout=180)
+    response.raise_for_status()
+    analysis_result = response.json()['choices'][0]['message']['content']
+    
+    # 保存分析结果
+    analysis_output = os.path.join(output_dir, "analysis.json")
+    with open(analysis_output, 'w', encoding='utf-8') as f:
+        f.write(analysis_result)
+    
+    print(f"  [OK] Analysis 完成")
+    
+    # 解析 JSON
+    try:
+        gold_standard = json.loads(analysis_result)
+        recommended_count = len(gold_standard.get('recommended_pages', []))
+        print(f"  [OK] 识别到 {recommended_count} 个推荐页面")
+    except json.JSONDecodeError as e:
+        print(f"  [ERROR] JSON 解析失败: {e}")
+        print(f"  分析结果预览: {analysis_result[:200]}")
+        exit(1)
+    
+except Exception as e:
+    print(f"  [ERROR] LLM API 调用失败: {e}")
+    print("  退出实验")
+    exit(1)
+
+# Step 2: Generation
+print("\nStep 2: Generation 阶段...")
+
+# 读取生成 prompt
+generation_prompt_file = os.path.join(experiment_dir, "prompts", "twostep-generation.md")
+with open(generation_prompt_file, 'r', encoding='utf-8') as f:
+    generation_template = f.read()
+
+generation_full_prompt = generation_template.replace("{ANALYSIS_RESULT}", json.dumps(gold_standard, ensure_ascii=False, indent=2))
+
+print("  调用 OpenAI API 生成 Wiki 页面...")
+generation_body = {
+    "model": "gpt-4o-mini",
+    "messages": [
+        {
+            "role": "system",
+            "content": "你是一位知识库构建专家。请基于分析结果生成 Wiki 页面。"
+        },
+        {
+            "role": "user",
+            "content": generation_full_prompt
+        }
+    ],
+    "temperature": 0.5,
+    "max_tokens": 16000
+}
+
+try:
+    response = requests.post("https://api.openai.com/v1/chat/completions", 
+                            headers=headers, 
+                            json=generation_body, 
+                            timeout=300)
+    response.raise_for_status()
+    generation_result = response.json()['choices'][0]['message']['content']
+    
+    print("  [OK] LLM 生成完成")
+    
+except Exception as e:
+    print(f"  [ERROR] LLM API 调用失败: {e}")
+    exit(1)
+
+# 解析生成结果
+sample_pages = []
+
+# 解析 ---FILE: ... ---END FILE--- 块
+file_blocks = re.findall(r'---FILE: (.*?)---(.*?)---END FILE---', generation_result, re.DOTALL)
+
+if not file_blocks:
+    # 尝试宽松匹配
+    file_blocks = re.findall(r'FILE: (.*?)\n(.*?)(?=(FILE:|$))', generation_result, re.DOTALL)
+
+for block in file_blocks:
+    file_path = block[0].strip()
+    file_content = block[1].strip()
+    
+    # 创建目录（如果需要）
+    full_path = os.path.join(output_dir, file_path)
+    file_dir = os.path.dirname(full_path)
+    os.makedirs(file_dir, exist_ok=True)
+    
+    # 写入文件
+    with open(full_path, 'w', encoding='utf-8') as f:
+        f.write(file_content)
+    sample_pages.append(full_path)
+    print(f"  [OK] 生成: {file_path}")
+
+# 记录结束时间
+end_time = datetime.now()
+end_time_str = end_time.strftime("%Y-%m-%d %H:%M:%S")
+duration = (end_time - start_time).total_seconds() / 60
+
+print(f"\n=== A 组实验完成 ===")
+print(f"结束时间: {end_time_str}")
+print(f"总耗时: {round(duration, 2)} 分钟")
+print(f"生成文件数: {len(sample_pages)}")
+
+# 保存元数据
+metadata = {
+    "group": "A-real",
+    "mode": "Two-Step (Real API)",
+    "sourceFile": "raw/呼吸之间_李谨伯/第一编 从身体入手.md",
+    "startTime": start_time_str,
+    "endTime": end_time_str,
+    "durationMinutes": round(duration, 2),
+    "step1Status": "completed",
+    "step2Status": "completed",
+    "pagesGenerated": len(sample_pages),
+    "apiModel": "gpt-4o-mini",
+    "apiCost": f"Analysis (8K tokens) + Generation (16K tokens) ≈ ${round(0.15 * 24 / 1000000, 4)}"
+}
+
+metadata_file = os.path.join(output_dir, "metadata.json")
+with open(metadata_file, 'w', encoding='utf-8') as f:
+    json.dump(metadata, f, ensure_ascii=False, indent=2)
+
+print(f"[OK] 元数据已保存到 {metadata_file}")
@@ -37,20 +37,64 @@ with open(analysis_prompt_file, 'r', encoding='utf-8') as f:
 analysis_full_prompt = analysis_template.replace("{SOURCE_CONTENT}", source_content)

 print("  调用 LLM 进行分析...")
-# 注意：由于没有配置实际的 LLM API，这里使用模拟数据
-# 实际使用时应该调用真实的 LLM API

-# 复制金标准作为模拟分析结果
-gold_standard_file = os.path.join(experiment_dir, "gold-standard.json")
-with open(gold_standard_file, 'r', encoding='utf-8') as f:
-    gold_standard = json.load(f)
+# 真实调用 LLM API
+import os
+import requests
+
+api_key = os.environ.get('OPENAI_API_KEY')
+if not api_key:
+    print("  [ERROR] 未找到 OPENAI_API_KEY 环境变量")
+    exit(1)
+
+headers = {
+    "Authorization": f"Bearer {api_key}",
+    "Content-Type": "application/json"
+}
+
+body = {
+    "model": "gpt-4o-mini",  # 使用更便宜的模型
+    "messages": [
+        {
+            "role": "system",
+            "content": "你是一位知识库分析专家。请分析源文件并提取结构化信息。"
+        },
+        {
+            "role": "user",
+            "content": analysis_full_prompt
+        }
+    ],
+    "temperature": 0.3
+}
+
+try:
+    response = requests.post("https://api.openai.com/v1/chat/completions", 
+                            headers=headers, 
+                            json=body, 
+                            timeout=120)
+    response.raise_for_status()
+    analysis_result = response.json()['choices'][0]['message']['content']
    
    # 保存分析结果
-analysis_output = os.path.join(output_dir, "analysis.json")
    with open(analysis_output, 'w', encoding='utf-8') as f:
-    json.dump(gold_standard, f, ensure_ascii=False, indent=2)
+        f.write(analysis_result)
    
-print(f"  [OK] Analysis 完成，识别到 {len(gold_standard['recommended_pages'])} 个推荐页面")
+    print(f"  [OK] Analysis 完成，已保存到 {analysis_output}")
+    
+    # 解析 JSON
+    try:
+        gold_standard = json.loads(analysis_result)
+        recommended_count = len(gold_standard.get('recommended_pages', []))
+    except:
+        recommended_count = len(gold_standard.get('recommended_pages', []))
+    
+except Exception as e:
+    print(f"  [ERROR] LLM API 调用失败: {e}")
+    print("  使用模拟数据...")
+    # 使用备用模拟数据
+    with open(gold_standard_file, 'r', encoding='utf-8') as f:
+        gold_standard = json.load(f)
+    recommended_count = len(gold_standard.get('recommended_pages', []))

 # Step 2: Generation
 print("\nStep 2: Generation 阶段...")
@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+真实 B 组实验（Single-Step 模式）- 使用真实 OpenAI API
+"""
+
+import json
+import os
+import requests
+from datetime import datetime
+import re
+
+# 配置
+base_dir = os.getcwd()
+experiment_dir = os.path.join(base_dir, "tools", "experiments", "wiki-generation-compare")
+source_file = os.path.join(base_dir, "raw", "呼吸之间_李谨伯", "第一编 从身体入手.md")
+output_dir = os.path.join(experiment_dir, "output", "group-b-real")
+
+# 创建输出目录
+os.makedirs(output_dir, exist_ok=True)
+os.makedirs(os.path.join(output_dir, "wiki", "concepts"), exist_ok=True)
+os.makedirs(os.path.join(output_dir, "wiki", "methods"), exist_ok=True)
+os.makedirs(os.path.join(output_dir, "wiki", "entities"), exist_ok=True)
+
+# 记录开始时间
+start_time = datetime.now()
+start_time_str = start_time.strftime("%Y-%m-%d %H:%M:%S")
+
+print("=== B 组真实实验开始：Single-Step 模式 ===")
+print(f"开始时间: {start_time_str}")
+
+# 检查 API Key
+api_key = os.environ.get('OPENAI_API_KEY')
+if not api_key:
+    print("[ERROR] 未找到 OPENAI_API_KEY 环境变量")
+    exit(1)
+
+headers = {
+    "Authorization": f"Bearer {api_key}",
+    "Content-Type": "application/json"
+}
+
+# 直接生成
+print("\n生成阶段...")
+
+# 读取源文件
+with open(source_file, 'r', encoding='utf-8') as f:
+    source_content = f.read()
+
+# 读取单步 prompt
+singlestep_prompt_file = os.path.join(experiment_dir, "prompts", "singlestep.md")
+with open(singlestep_prompt_file, 'r', encoding='utf-8') as f:
+    template = f.read()
+
+full_prompt = template.replace("{SOURCE_CONTENT}", source_content)
+
+print("  调用 OpenAI API 生成 Wiki 页面...")
+body = {
+    "model": "gpt-4o-mini",
+    "messages": [
+        {
+            "role": "system",
+            "content": "你是一位知识库构建专家。请直接从源文件中提取关键术语并生成 Wiki 页面。"
+        },
+        {
+            "role": "user",
+            "content": full_prompt
+        }
+    ],
+    "temperature": 0.5,
+    "max_tokens": 16000
+}
+
+try:
+    response = requests.post("https://api.openai.com/v1/chat/completions", 
+                            headers=headers, 
+                            json=body, 
+                            timeout=300)
+    response.raise_for_status()
+    generation_result = response.json()['choices'][0]['message']['content']
+    
+    print("  [OK] LLM 生成完成")
+    
+except Exception as e:
+    print(f"  [ERROR] LLM API 调用失败: {e}")
+    exit(1)
+
+# 解析生成结果
+sample_pages = []
+
+# 解析 ---FILE: ... ---END FILE--- 块
+file_blocks = re.findall(r'---FILE: (.*?)---(.*?)---END FILE---', generation_result, re.DOTALL)
+
+if not file_blocks:
+    # 尝试宽松匹配
+    file_blocks = re.findall(r'FILE: (.*?)\n(.*?)(?=(FILE:|$))', generation_result, re.DOTALL)
+
+for block in file_blocks:
+    file_path = block[0].strip()
+    file_content = block[1].strip()
+    
+    # 创建目录（如果需要）
+    full_path = os.path.join(output_dir, file_path)
+    file_dir = os.path.dirname(full_path)
+    os.makedirs(file_dir, exist_ok=True)
+    
+    # 写入文件
+    with open(full_path, 'w', encoding='utf-8') as f:
+        f.write(file_content)
+    sample_pages.append(full_path)
+    print(f"  [OK] 生成: {file_path}")
+
+# 记录结束时间
+end_time = datetime.now()
+end_time_str = end_time.strftime("%Y-%m-%d %H:%M:%S")
+duration = (end_time - start_time).total_seconds() / 60
+
+print(f"\n=== B 组实验完成 ===")
+print(f"结束时间: {end_time_str}")
+print(f"总耗时: {round(duration, 2)} 分钟")
+print(f"生成文件数: {len(sample_pages)}")
+
+# 保存元数据
+metadata = {
+    "group": "B-real",
+    "mode": "Single-Step (Real API)",
+    "sourceFile": "raw/呼吸之间_李谨伯/第一编 从身体入手.md",
+    "startTime": start_time_str,
+    "endTime": end_time_str,
+    "durationMinutes": round(duration, 2),
+    "pagesGenerated": len(sample_pages),
+    "apiModel": "gpt-4o-mini",
+    "apiCost": f"Generation (16K tokens) ≈ ${round(0.15 * 16 / 1000000, 4)}"
+}
+
+metadata_file = os.path.join(output_dir, "metadata.json")
+with open(metadata_file, 'w', encoding='utf-8') as f:
+    json.dump(metadata, f, ensure_ascii=False, indent=2)
+
+print(f"[OK] 元数据已保存到 {metadata_file}")