Files

466 lines
19 KiB
PowerShell
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# 评估脚本
$ErrorActionPreference = "Stop"
# 配置
$goldStandardFile = "tools/experiments/wiki-generation-compare/gold-standard.json"
$groupADir = "tools/experiments/wiki-generation-compare/output/group-a/wiki"
$groupBDir = "tools/experiments/wiki-generation-compare/output/group-b/wiki"
$reportFile = "tools/experiments/wiki-generation-compare/output/report.md"
Write-Host "=== 开始评估 ===" -ForegroundColor Cyan
# 检查金标准文件
if (!(Test-Path $goldStandardFile)) {
Write-Host "✗ 金标准文件不存在: $goldStandardFile" -ForegroundColor Red
exit 1
}
# 加载金标准
$goldStandard = Get-Content $goldStandardFile -Raw | ConvertFrom-Json
# 定义权重
$weights = @{
contentCompleteness = 0.4
technicalCompliance = 0.3
maintainability = 0.2
efficiency = 0.1
}
# 辅助函数:计算术语覆盖度
function Calculate-TermCoverage {
param (
[string]$wikiDir,
[hashtable]$goldTerms
)
# 获取所有生成的 Wiki 文件
$wikiFiles = Get-ChildItem -Path $wikiDir -Recurse -Filter "*.md" -ErrorAction SilentlyContinue
$generatedTerms = @{}
if ($wikiFiles) {
foreach ($file in $wikiFiles) {
$termName = $file.BaseName
$generatedTerms[$termName] = @{
file = $file.FullName
hasDefinition = $false
hasSourceLink = $false
hasLineNumber = $false
}
# 检查内容
$content = Get-Content $file.FullName -Raw -Encoding UTF8
# 检查是否有定义
if ($content -match "> \*\*一句话定义\*\*") {
$generatedTerms[$termName].hasDefinition = $true
}
# 检查是否有来源链接
if ($content -match "\[\[raw/呼吸之间_李谨伯") {
$generatedTerms[$termName].hasSourceLink = $true
}
# 检查是否有行号标注
if ($content -match "\[raw:.*:\d+\]") {
$generatedTerms[$termName].hasLineNumber = $true
}
}
}
# 计算覆盖度
$coveredCount = 0
$totalTerms = $goldTerms.Count
foreach ($term in $goldTerms.Keys) {
if ($generatedTerms.ContainsKey($term)) {
$coveredCount++
}
}
$coverageRate = if ($totalTerms -gt 0) { $coveredCount / $totalTerms } else { 0 }
@{
totalGoldTerms = $totalTerms
coveredTerms = $coveredCount
coverageRate = $coverageRate
generatedTerms = $generatedTerms
missedTerms = $goldTerms.Keys | Where-Object { -not $generatedTerms.ContainsKey($_) }
}
}
# 辅助函数:评估 Frontmatter 规范性
function Evaluate-Frontmatter {
param (
[string]$wikiDir
)
$wikiFiles = Get-ChildItem -Path $wikiDir -Recurse -Filter "*.md" -ErrorAction SilentlyContinue
$results = @()
$requiredFields = @("categories", "tags", "created", "source", "type")
if ($wikiFiles) {
foreach ($file in $wikiFiles) {
$content = Get-Content $file.FullName -Raw -Encoding UTF8
$fileResult = @{
file = $file.Name
hasFrontmatter = $false
requiredFieldsComplete = 0
requiredFieldsTotal = $requiredFields.Count
hasCategories = $false
hasTags = $false
hasCreated = $false
hasSource = $false
hasType = $false
booleanQuotes = $true
yamlValid = $true
}
# 检查是否有 YAML frontmatter
if ($content -match "^---\n(.*?)\n---") {
$fileResult.hasFrontmatter = $true
$frontmatter = $matches[1]
# 检查必需字段
foreach ($field in $requiredFields) {
if ($frontmatter -match "$field:") {
$fileResult."has$($field -replace '\s', '')" = $true
$fileResult.requiredFieldsComplete++
}
}
# 检查布尔值是否用引号包裹
if ($content -match ":\s*(true|false)\s*$") {
if (!($content -match ':\s*"(true|false)"')) {
$fileResult.booleanQuotes = $false
}
}
}
$results += $fileResult
}
}
# 计算总体得分
$totalFiles = $results.Count
if ($totalFiles -gt 0) {
$avgFieldCompleteness = ($results | ForEach-Object { $_.requiredFieldsComplete / $_.requiredFieldsTotal } | Measure-Object -Average).Average
$booleanComplianceRate = ($results | Where-Object { $_.booleanQuotes } | Measure-Object).Count / $totalFiles
} else {
$avgFieldCompleteness = 0
$booleanComplianceRate = 1.0
}
@{
totalFiles = $totalFiles
avgFieldCompleteness = $avgFieldCompleteness
booleanComplianceRate = $booleanComplianceRate
results = $results
}
}
# 辅助函数:评估 Wikilink 质量
function Evaluate-Wikilinks {
param (
[string]$wikiDir,
[hashtable]$generatedTerms
)
$wikiFiles = Get-ChildItem -Path $wikiDir -Recurse -Filter "*.md" -ErrorAction SilentlyContinue
$totalLinks = 0
$validLinks = 0
$invalidLinks = @()
if ($wikiFiles) {
foreach ($file in $wikiFiles) {
$content = Get-Content $file.FullName -Raw -Encoding UTF8
# 查找所有 wikilinks
$links = [regex]::Matches($content, "\[\[([^\]]+)\]\]")
foreach ($link in $links) {
$totalLinks++
$target = $link.Groups[1].Value
# 检查目标是否存在
if ($generatedTerms.ContainsKey($target)) {
$validLinks++
} else {
$invalidLinks += @{
source = $file.Name
target = $target
}
}
}
}
}
@{
totalLinks = $totalLinks
validLinks = $validLinks
invalidLinks = $invalidLinks
linkAccuracyRate = if ($totalLinks -gt 0) { $validLinks / $totalLinks } else { 1.0 }
}
}
# 构建金标准术语表
$goldTerms = @{}
$goldStandard.concepts | ForEach-Object { $goldTerms[$_.term] = $_ }
$goldStandard.methods | ForEach-Object { $goldTerms[$_.term] = $_ }
$goldStandard.entities | ForEach-Object { $goldTerms[$_.term] = $_ }
# 评估 A 组
Write-Host "`n评估 A 组..." -ForegroundColor Green
$groupATermCoverage = Calculate-TermCoverage -wikiDir $groupADir -goldTerms $goldTerms
$groupAFrontmatter = Evaluate-Frontmatter -wikiDir $groupADir
$groupAWikilinks = Evaluate-Wikilinks -wikiDir $groupADir -generatedTerms $groupATermCoverage.generatedTerms
# 评估 B 组
Write-Host "`n评估 B 组..." -ForegroundColor Green
$groupBTermCoverage = Calculate-TermCoverage -wikiDir $groupBDir -goldTerms $goldTerms
$groupBFrontmatter = Evaluate-Frontmatter -wikiDir $groupBDir
$groupBWikilinks = Evaluate-Wikilinks -wikiDir $groupBDir -generatedTerms $groupBTermCoverage.generatedTerms
# 加载元数据
$groupAMetadata = $null
$groupBMetadata = $null
$metadataFileA = "tools/experiments/wiki-generation-compare/output/group-a/metadata.json"
$metadataFileB = "tools/experiments/wiki-generation-compare/output/group-b/metadata.json"
if (Test-Path $metadataFileA) {
$groupAMetadata = Get-Content $metadataFileA -Raw | ConvertFrom-Json
}
if (Test-Path $metadataFileB) {
$groupBMetadata = Get-Content $metadataFileB -Raw | ConvertFrom-Json
}
# 计算得分
function Calculate-Score {
param (
[hashtable]$termCoverage,
[hashtable]$frontmatter,
[hashtable]$wikilinks,
[hashtable]$metadata,
[hashtable]$weights
)
# 内容完整性(40%
$coverageScore = $termCoverage.coverageRate
$definitionQuality = if ($termCoverage.generatedTerms.Count -gt 0) {
($termCoverage.generatedTerms.Values | Where-Object { $_.hasDefinition } | Measure-Object).Count / $termCoverage.generatedTerms.Count
} else { 0 }
$sourceLinkQuality = if ($termCoverage.generatedTerms.Count -gt 0) {
($termCoverage.generatedTerms.Values | Where-Object { $_.hasSourceLink } | Measure-Object).Count / $termCoverage.generatedTerms.Count
} else { 0 }
$lineNumberQuality = if ($termCoverage.generatedTerms.Count -gt 0) {
($termCoverage.generatedTerms.Values | Where-Object { $_.hasLineNumber } | Measure-Object).Count / $termCoverage.generatedTerms.Count
} else { 0 }
$contentCompleteness = ($coverageScore * 0.4 + $definitionQuality * 0.3 + $sourceLinkQuality * 0.2 + $lineNumberQuality * 0.1)
# 技术规范性(30%
$fieldCompleteness = $frontmatter.avgFieldCompleteness
$booleanCompliance = $frontmatter.booleanComplianceRate
$linkQuality = $wikilinks.linkAccuracyRate
$technicalCompliance = ($fieldCompleteness * 0.4 + $booleanCompliance * 0.3 + $linkQuality * 0.3)
# 可维护性(20%
# 简化处理:如果有元数据就给分
$maintainability = if ($metadata) { 0.8 } else { 0.5 }
# 效率成本(10%
# 基于时间成本,越快越好
$timeCost = if ($metadata) { $metadata.durationMinutes } else { 0 }
$efficiency = if ($timeCost -eq 0) { 0.5 } elseif ($timeCost -lt 10) { 1.0 } elseif ($timeCost -lt 20) { 0.8 } else { 0.6 }
# 总分
$totalScore = $contentCompleteness * $weights.contentCompleteness + `
$technicalCompliance * $weights.technicalCompliance + `
$maintainability * $weights.maintainability + `
$efficiency * $weights.efficiency
@{
contentCompleteness = $contentCompleteness
technicalCompliance = $technicalCompliance
maintainability = $maintainability
efficiency = $efficiency
totalScore = $totalScore
}
}
$groupAScores = Calculate-Score -termCoverage $groupATermCoverage -frontmatter $groupAFrontmatter -wikilinks $groupAWikilinks -metadata $groupAMetadata -weights $weights
$groupBScores = Calculate-Score -termCoverage $groupBTermCoverage -frontmatter $groupBFrontmatter -wikilinks $groupBWikilinks -metadata $groupBMetadata -weights $weights
# 生成报告
$aStartTime = if ($groupAMetadata) { $groupAMetadata.startTime } else { "N/A" }
$aEndTime = if ($groupAMetadata) { $groupAMetadata.endTime } else { "N/A" }
$aDuration = if ($groupAMetadata) { $groupAMetadata.durationMinutes } else { 0 }
$aPages = if ($groupAMetadata) { $groupAMetadata.pagesGenerated } else { $groupATermCoverage.generatedTerms.Count }
$bStartTime = if ($groupBMetadata) { $groupBMetadata.startTime } else { "N/A" }
$bEndTime = if ($groupBMetadata) { $groupBMetadata.endTime } else { "N/A" }
$bDuration = if ($groupBMetadata) { $groupBMetadata.durationMinutes } else { 0 }
$bPages = if ($groupBMetadata) { $groupBMetadata.pagesGenerated } else { $groupBTermCoverage.generatedTerms.Count }
$aMissedTerms = if ($groupATermCoverage.missedTerms) { $groupATermCoverage.missedTerms -join ', ' } else { "" }
$bMissedTerms = if ($groupBTermCoverage.missedTerms) { $groupBTermCoverage.missedTerms -join ', ' } else { "" }
$aInvalidLinks = if ($groupAWikilinks.invalidLinks) { $groupAWikilinks.invalidLinks | ForEach-Object { "$($_.source)$($_.target)" } -join '; ' } else { "" }
$bInvalidLinks = if ($groupBWikilinks.invalidLinks) { $groupBWikilinks.invalidLinks | ForEach-Object { "$($_.source)$($_.target)" } -join '; ' } else { "" }
$report = @"
# Wiki
##
- ****: raw/_/ .md
- ****: $($goldTerms.Count)
- **A **: Two-Step +
- **B **: Single-Step
- ****: 40% + 30% + 20% + 10%
##
| | | | | | |
|------|------|----------|----------|-------------|-----------|
| A | Two-Step | $aStartTime | $aEndTime | $aDuration | $aPages |
| B | Single-Step | $bStartTime | $bEndTime | $bDuration | $bPages |
##
### A Two-Step
| | | | |
|------|------|------|---------|
| | $([math]::Round($groupAScores.contentCompleteness * 100, 1))% | 40% | $([math]::Round($groupAScores.contentCompleteness * 0.4 * 100, 1)) |
| | $([math]::Round($groupAScores.technicalCompliance * 100, 1))% | 30% | $([math]::Round($groupAScores.technicalCompliance * 0.3 * 100, 1)) |
| | $([math]::Round($groupAScores.maintainability * 100, 1))% | 20% | $([math]::Round($groupAScores.maintainability * 0.2 * 100, 1)) |
| | $([math]::Round($groupAScores.efficiency * 100, 1))% | 10% | $([math]::Round($groupAScores.efficiency * 0.1 * 100, 1)) |
| **** | - | **100%** | **$([math]::Round($groupAScores.totalScore * 100, 1))** |
### B Single-Step
| | | | |
|------|------|------|---------|
| | $([math]::Round($groupBScores.contentCompleteness * 100, 1))% | 40% | $([math]::Round($groupBScores.contentCompleteness * 0.4 * 100, 1)) |
| | $([math]::Round($groupBScores.technicalCompliance * 100, 1))% | 30% | $([math]::Round($groupBScores.technicalCompliance * 0.3 * 100, 1)) |
| | $([math]::Round($groupBScores.maintainability * 100, 1))% | 20% | $([math]::Round($groupBScores.maintainability * 0.2 * 100, 1)) |
| | $([math]::Round($groupBScores.efficiency * 100, 1))% | 10% | $([math]::Round($groupBScores.efficiency * 0.1 * 100, 1)) |
| **** | - | **100%** | **$([math]::Round($groupBScores.totalScore * 100, 1))** |
##
### 1.
| | | | | |
|------|-------------|-----------|--------|-----------|
| A | $($groupATermCoverage.totalGoldTerms) | $($groupATermCoverage.coveredTerms) | $([math]::Round($groupATermCoverage.coverageRate * 100, 1))% | $aMissedTerms |
| B | $($groupBTermCoverage.totalGoldTerms) | $($groupBTermCoverage.coveredTerms) | $([math]::Round($groupBTermCoverage.coverageRate * 100, 1))% | $bMissedTerms |
**A **: $aMissedTerms
**B **: $bMissedTerms
### 2. Frontmatter
| | | | |
|------|---------|---------------|-------------|
| A | $($groupAFrontmatter.totalFiles) | $([math]::Round($groupAFrontmatter.avgFieldCompleteness * 100, 1))% | $([math]::Round($groupAFrontmatter.booleanComplianceRate * 100, 1))% |
| B | $($groupBFrontmatter.totalFiles) | $([math]::Round($groupBFrontmatter.avgFieldCompleteness * 100, 1))% | $([math]::Round($groupBFrontmatter.booleanComplianceRate * 100, 1))% |
### 3. Wikilink
| | | | | |
|------|---------|-----------|-----------|-----------|
| A | $($groupAWikilinks.totalLinks) | $($groupAWikilinks.validLinks) | $($groupAWikilinks.invalidLinks.Count) | $([math]::Round($groupAWikilinks.linkAccuracyRate * 100, 1))% |
| B | $($groupBWikilinks.totalLinks) | $($groupBWikilinks.validLinks) | $($groupBWikilinks.invalidLinks.Count) | $([math]::Round($groupBWikilinks.linkAccuracyRate * 100, 1))% |
**A **: $aInvalidLinks
**B **: $bInvalidLinks
### 4.
| | | | |
|------|--------|-----------|-----------|
| A | $(($groupATermCoverage.generatedTerms.Values | Where-Object { $_.hasDefinition } | Measure-Object).Count) / $($groupATermCoverage.generatedTerms.Count) | $(($groupATermCoverage.generatedTerms.Values | Where-Object { $_.hasSourceLink } | Measure-Object).Count) / $($groupATermCoverage.generatedTerms.Count) | $(($groupATermCoverage.generatedTerms.Values | Where-Object { $_.hasLineNumber } | Measure-Object).Count) / $($groupATermCoverage.generatedTerms.Count) |
| B | $(($groupBTermCoverage.generatedTerms.Values | Where-Object { $_.hasDefinition } | Measure-Object).Count) / $($groupBTermCoverage.generatedTerms.Count) | $(($groupBTermCoverage.generatedTerms.Values | Where-Object { $_.hasSourceLink } | Measure-Object).Count) / $($groupBTermCoverage.generatedTerms.Count) | $(($groupBTermCoverage.generatedTerms.Values | Where-Object { $_.hasLineNumber } | Measure-Object).Count) / $($groupBTermCoverage.generatedTerms.Count) |
##
###
$(
if ($groupAScores.totalScore -gt $groupBScores.totalScore) {
"**A 组(Two-Step 模式)总分更高,推荐使用。**"
} elseif ($groupBScores.totalScore -gt $groupAScores.totalScore) {
"**B 组(Single-Step 模式)总分更高,推荐使用。**"
} else {
"**A 组和 B 组得分相当,可根据其他因素选择。**"
}
)
###
$(
if ($groupAScores.contentCompleteness -gt $groupBScores.contentCompleteness) {
"- **内容完整性**:A 组优于 B 组($([math]::Round($groupAScores.contentCompleteness * 100, 1))% vs $([math]::Round($groupBScores.contentCompleteness * 100, 1))%"
} else {
"- **内容完整性**:B 组优于 A 组($([math]::Round($groupBScores.contentCompleteness * 100, 1))% vs $([math]::Round($groupAScores.contentCompleteness * 100, 1))%"
}
)
$(
if ($groupAScores.technicalCompliance -gt $groupBScores.technicalCompliance) {
"- **技术规范性**:A 组优于 B 组($([math]::Round($groupAScores.technicalCompliance * 100, 1))% vs $([math]::Round($groupBScores.technicalCompliance * 100, 1))%"
} else {
"- **技术规范性**:B 组优于 A 组($([math]::Round($groupBScores.technicalCompliance * 100, 1))% vs $([math]::Round($groupAScores.technicalCompliance * 100, 1))%"
}
)
$(
if ($groupAScores.efficiency -gt $groupBScores.efficiency) {
"- **效率成本**:A 组优于 B 组"
} else {
"- **效率成本**:B 组优于 A 组"
}
)
###
1. ****
- A $aMissedTerms
- B $bMissedTerms
2. **Frontmatter **
-
-
3. **Wikilink **
-
-
##
### A
$($groupATermCoverage.generatedTerms.Keys -join ", ")
### B
$($groupBTermCoverage.generatedTerms.Keys -join ", ")
---
****: $(Get-Date -Format "yyyy-MM-dd HH:mm:ss")
"@
# 保存报告
$report | Out-File $reportFile -Encoding UTF8
Write-Host "`n=== 评估完成 ===" -ForegroundColor Cyan
Write-Host "报告已保存到 $reportFile" -ForegroundColor Green
Write-Host "`n总分对比:" -ForegroundColor Yellow
Write-Host "A 组(Two-Step: $([math]::Round($groupAScores.totalScore * 100, 1))" -ForegroundColor Cyan
Write-Host "B 组(Single-Step: $([math]::Round($groupBScores.totalScore * 100, 1))" -ForegroundColor Cyan
# 显示报告摘要
Write-Host "`n" -NoNewline
Get-Content $reportFile -Encoding UTF8 | Select-Object -First 50