Phase 0-2: Schema cleanup, typed relations, event-driven automation
- Phase 0: AGENTS.md cleanup (dedup quotes, renumber sections, merge qmd) - Phase 1: typed relations (manage-relations.py, graph-search.py, check-staleness.py, detect-conflicts.py) - Phase 2: frontmatter validator, weekly lint, knowledge promotion, git hooks - Fix .gitignore to track tools/ and .githooks/ - Fix git remote URL (remove plaintext token) - New wiki pages: 504 pages, 34 raw sources
This commit is contained in:
hehaiguang1123
@@ -0,0 +1,530 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
学术论文批量下载工具 (Academic Paper Downloader)
|
||||
支持HTTP(S)/HTTPS重试、大文件分块下载、进度显示、日志记录
|
||||
|
||||
创建时间：2026-04-14
|
||||
最后更新：2026-04-14
|
||||
"""
|
||||
|
||||
import urllib.request
|
||||
import ssl
|
||||
import os
|
||||
import sys
|
||||
import io
|
||||
import time
|
||||
import json
|
||||
from pathlib import Path
|
||||
from urllib.error import URLError, HTTPError
|
||||
from datetime import datetime
|
||||
|
||||
# Force UTF-8 encoding for console output
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout, encoding="utf-8")
|
||||
|
||||
# =============================================================================
|
||||
# 配置区域
|
||||
# =============================================================================
|
||||
|
||||
# 基础目录
|
||||
BASE_DIR = r"D:\TC_UP\2023card\projects\openclaw\教育AI研究\高等教育AI专题\文献库"
|
||||
LOG_DIR = os.path.join(BASE_DIR, "logs")
|
||||
CONFIG_DIR = os.path.join(BASE_DIR, "configs")
|
||||
os.makedirs(LOG_DIR, exist_ok=True)
|
||||
os.makedirs(CONFIG_DIR, exist_ok=True)
|
||||
|
||||
|
||||
# 下载配置
|
||||
class DownloadConfig:
|
||||
"""下载配置"""
|
||||
|
||||
def __init__(self):
|
||||
self.timeout = 300 # 5分钟
|
||||
self.retry_count = 3
|
||||
self.retry_delay = 5 # 秒
|
||||
self.chunk_size = 8192 # 8KB per chunk
|
||||
self.user_agent = "Mozilla/5.0 (compatible; AcademicPaperDownloader/1.0)"
|
||||
|
||||
def set_timeout(self, seconds):
|
||||
"""设置超时时间"""
|
||||
self.timeout = seconds
|
||||
|
||||
def set_retry(self, count, delay):
|
||||
"""设置重试次数和延迟"""
|
||||
self.retry_count = count
|
||||
self.retry_delay = delay
|
||||
|
||||
|
||||
# SSL上下文
|
||||
SSL_CONTEXT = ssl._create_default_https_context()
|
||||
|
||||
# =============================================================================
|
||||
# 日志系统
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class DownloadLogger:
|
||||
"""下载日志记录器"""
|
||||
|
||||
def __init__(self, log_file):
|
||||
self.log_file = os.path.join(LOG_DIR, log_file)
|
||||
self.records = []
|
||||
self._load_existing_logs()
|
||||
|
||||
def _load_existing_logs(self):
|
||||
"""加载已有日志"""
|
||||
if os.path.exists(self.log_file):
|
||||
try:
|
||||
with open(self.log_file, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if line.strip():
|
||||
try:
|
||||
self.records.append(json.loads(line))
|
||||
except:
|
||||
pass
|
||||
except:
|
||||
pass
|
||||
|
||||
def log(self, paper_name, url, status, size_mb, error=None):
|
||||
"""记录下载日志"""
|
||||
record = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"paper": paper_name,
|
||||
"url": url,
|
||||
"status": status, # 'success' or 'failed'
|
||||
"size_mb": size_mb,
|
||||
"error": error,
|
||||
}
|
||||
self.records.append(record)
|
||||
self._save_log()
|
||||
|
||||
def _save_log(self):
|
||||
"""保存日志到文件"""
|
||||
with open(self.log_file, "w", encoding="utf-8") as f:
|
||||
for record in self.records:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
def get_stats(self):
|
||||
"""获取统计信息"""
|
||||
total = len(self.records)
|
||||
success = len([r for r in self.records if r["status"] == "success"])
|
||||
failed = len([r for r in self.records if r["status"] == "failed"])
|
||||
total_size = sum(
|
||||
[r.get("size_mb", 0) for r in self.records if r["status"] == "success"]
|
||||
)
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"success_rate": f"{(success / total * 100):.1f}%",
|
||||
"total_size_mb": f"{total_size:.2f}",
|
||||
"avg_size_mb": f"{(total_size / success if success > 0 else 0):.2f}",
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 核心下载器
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class PaperDownloader:
|
||||
"""学术论文下载器"""
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.logger = DownloadLogger("paper_downloads.log")
|
||||
|
||||
def download_with_retry(self, url, filepath, paper_name=""):
|
||||
"""带重试机制的下载"""
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
for attempt in range(self.config.retry_count):
|
||||
try:
|
||||
print(f"\n📥 尝试 {attempt + 1}/{self.config.retry_count}: {filename}")
|
||||
return self._download(url, filepath, paper_name)
|
||||
|
||||
except (URLError, HTTPError) as e:
|
||||
print(f"❌ 尝试 {attempt + 1}/{self.config.retry_count} 失败: {e}")
|
||||
|
||||
if attempt < self.config.retry_count - 1:
|
||||
print(f"⏳ {self.config.retry_delay} 秒后重试...")
|
||||
time.sleep(self.config.retry_delay)
|
||||
else:
|
||||
# 最后一次失败，记录日志
|
||||
self.logger.log(
|
||||
paper_name or filename, url, "failed", 0, error=str(e)
|
||||
)
|
||||
return False
|
||||
|
||||
def _download(self, url, filepath, paper_name=""):
|
||||
"""实际下载方法，支持分块下载"""
|
||||
try:
|
||||
print(f"📥 开始下载: {paper_name or os.path.basename(filepath)}")
|
||||
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": self.config.user_agent}
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(
|
||||
req, context=SSL_CONTEXT, timeout=self.config.timeout
|
||||
) as response:
|
||||
total_size = int(response.headers.get("content-length", 0))
|
||||
total_mb = total_size / (1024 * 1024)
|
||||
|
||||
print(f"📊 文件大小: {total_mb:.2f} MB")
|
||||
|
||||
# 创建目录
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
|
||||
downloaded = 0
|
||||
start_time = time.time()
|
||||
|
||||
with open(filepath, "wb") as f:
|
||||
while downloaded < total_size:
|
||||
chunk = response.read(self.config.chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
|
||||
# 进度显示
|
||||
percent = (downloaded / total_size) * 100
|
||||
downloaded_mb = downloaded / (1024 * 1024)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
# 进度条
|
||||
progress_bar = "█" * int(percent / 5)
|
||||
print(
|
||||
f"\r进度: {percent:5.1f}% [{progress_bar:20}] {downloaded_mb:.2f}/{total_mb:.2f}MB {elapsed:.0f}s",
|
||||
end="",
|
||||
)
|
||||
|
||||
print() # 换行
|
||||
|
||||
# 记录成功日志
|
||||
self.logger.log(
|
||||
paper_name or os.path.basename(filepath), url, "success", total_mb
|
||||
)
|
||||
|
||||
print(
|
||||
f"\n✅ 下载成功: {paper_name or os.path.basename(filepath)} ({total_mb:.2f} MB)"
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ 下载失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 配置文件管理
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def load_config(config_file):
|
||||
"""从JSON文件加载配置"""
|
||||
config_path = os.path.join(CONFIG_DIR, config_file)
|
||||
|
||||
if not os.path.exists(config_path):
|
||||
print(f"⚠️ 配置文件不存在: {config_file}")
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
config_data = json.load(f)
|
||||
print(f"✅ 已加载配置: {config_file}")
|
||||
print(f"📋 论文数量: {len(config_data.get('papers', []))}")
|
||||
return config_data
|
||||
except Exception as e:
|
||||
print(f"❌ 加载配置失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def create_sample_config():
|
||||
"""创建示例配置文件（使用英文以避免编码问题）"""
|
||||
sample_config = {
|
||||
"name": "Academic Paper Downloader Configuration Sample",
|
||||
"description": "Configuration file for batch downloading academic papers",
|
||||
"papers": [
|
||||
{
|
||||
"name": "Teaching CS50 with AI (SIGCSE 2024)",
|
||||
"url": "https://cs.harvard.edu/malan/publications/V1fp0567-liu.pdf",
|
||||
"filename": "Teaching_CS50_with_AI_SIGCSE2024.pdf",
|
||||
"enabled": True,
|
||||
},
|
||||
{
|
||||
"name": "Improving AI in CS50 (SIGCSE 2025)",
|
||||
"url": "https://cs.harvard.edu/malan/publications/fp0627-liu.pdf",
|
||||
"filename": "Improving_AI_in_CS50_SIGCSE2025.pdf",
|
||||
"enabled": True,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
sample_path = os.path.join(CONFIG_DIR, "sample_config.json")
|
||||
|
||||
try:
|
||||
with open(sample_path, "w", encoding="utf-8") as f:
|
||||
json.dump(sample_data, f, ensure_ascii=False, indent=2)
|
||||
print("Sample config created successfully: " + sample_path)
|
||||
return sample_path
|
||||
except Exception as e:
|
||||
print(f"❌ 创建示例配置失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 主命令
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def download_from_config(config_file):
|
||||
"""从配置文件批量下载"""
|
||||
config_data = load_config(config_file)
|
||||
|
||||
if not config_data:
|
||||
return
|
||||
|
||||
downloader = PaperDownloader(DownloadConfig())
|
||||
papers = config_data.get("papers", [])
|
||||
enabled_papers = [p for p in papers if p.get("enabled", True)]
|
||||
|
||||
if not enabled_papers:
|
||||
print("⚠️ 没有启用的论文可下载")
|
||||
return
|
||||
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f"准备下载 {len(enabled_papers)} 篇论文...\n")
|
||||
|
||||
success_count = 0
|
||||
failed_count = 0
|
||||
|
||||
for idx, paper in enumerate(enabled_papers, 1):
|
||||
print(
|
||||
f"\n[{'=' * 40}/{len(enabled_papers)}] {paper.get('name', 'Unknown Paper')}"
|
||||
)
|
||||
|
||||
url = paper.get("url")
|
||||
filename = paper.get("filename", f"paper_{idx}.pdf")
|
||||
filepath = os.path.join(BASE_DIR, filename)
|
||||
name = paper.get("name", "Unknown Paper")
|
||||
|
||||
if downloader.download_with_retry(url, filepath, name):
|
||||
success_count += 1
|
||||
else:
|
||||
failed_count += 1
|
||||
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f"✅ 成功: {success_count}")
|
||||
print(f"❌ 失败: {failed_count}")
|
||||
|
||||
# 显示统计
|
||||
stats = downloader.logger.get_stats()
|
||||
print(f"\n📊 下载统计:")
|
||||
print(f"总论文数: {stats['total']}")
|
||||
print(f"成功率: {stats['success_rate']}")
|
||||
print(f"总下载大小: {stats['total_size_mb']} MB")
|
||||
print(f"平均大小: {stats['avg_size_mb']} MB")
|
||||
|
||||
|
||||
def download_single(url, filename=None):
|
||||
"""单篇论文下载"""
|
||||
if not filename:
|
||||
# 从URL提取文件名
|
||||
filename = url.split("/")[-1]
|
||||
if not filename.endswith(".pdf"):
|
||||
filename = filename + ".pdf"
|
||||
|
||||
filepath = os.path.join(BASE_DIR, filename)
|
||||
paper_name = os.path.splitext(filename)[0]
|
||||
|
||||
downloader = PaperDownloader(DownloadConfig())
|
||||
downloader.download_with_retry(url, filepath, paper_name)
|
||||
|
||||
# 显示统计
|
||||
stats = downloader.logger.get_stats()
|
||||
print(f"\n📊 当前下载统计:")
|
||||
print(f"总下载次数: {stats['total']}")
|
||||
print(f"成功率: {stats['success_rate']}")
|
||||
print(f"总下载大小: {stats['total_size_mb']} MB")
|
||||
|
||||
|
||||
def show_logs():
|
||||
"""显示下载日志"""
|
||||
stats = DownloadLogger("paper_downloads.log").get_stats()
|
||||
|
||||
print(f"\n📊 下载日志统计:")
|
||||
print(f"{'=' * 40}")
|
||||
print(f"总下载次数: {stats['total']}")
|
||||
print(f"成功次数: {stats['success']}")
|
||||
print(f"失败次数: {stats['failed']}")
|
||||
print(f"成功率: {stats['success_rate']}")
|
||||
print(f"总下载大小: {stats['total_size_mb']} MB")
|
||||
print(f"平均大小: {stats['avg_size_mb']} MB")
|
||||
print(f"日志文件: {os.path.join(LOG_DIR, 'paper_downloads.log')}")
|
||||
|
||||
|
||||
def clear_logs():
|
||||
"""清除日志"""
|
||||
log_file = os.path.join(LOG_DIR, "paper_downloads.log")
|
||||
|
||||
try:
|
||||
if os.path.exists(log_file):
|
||||
os.remove(log_file)
|
||||
print(f"✅ 日志已清除: {log_file}")
|
||||
else:
|
||||
print("⚠️ 日志文件不存在")
|
||||
except Exception as e:
|
||||
print(f"❌ 清除日志失败: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# 主程序
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def print_help():
|
||||
"""显示帮助信息"""
|
||||
help_text = """
|
||||
学术论文批量下载工具 (Academic Paper Downloader)
|
||||
|
||||
使用方法:
|
||||
python paper_downloader.py <command> [options]
|
||||
|
||||
命令:
|
||||
download <url> 下载单篇论文
|
||||
config <config.json> 从配置文件批量下载
|
||||
logs 显示下载日志统计
|
||||
clear 清除下载日志
|
||||
create-sample 创建示例配置文件
|
||||
help 显示此帮助信息
|
||||
|
||||
选项:
|
||||
--timeout <seconds> 设置超时时间（默认: 300秒）
|
||||
--retry <count> 设置重试次数（默认: 3次）
|
||||
--chunk-size <bytes> 设置分块大小（默认: 8192字节）
|
||||
--no-progress 不显示进度条
|
||||
|
||||
示例:
|
||||
# 下载单篇论文
|
||||
python paper_downloader.py download https://cs.harvard.edu/malan/publications/V1fp0567-liu.pdf
|
||||
|
||||
# 从配置文件批量下载
|
||||
python paper_downloader.py config my_papers.json
|
||||
|
||||
# 创建示例配置文件
|
||||
python paper_downloader.py create-sample
|
||||
|
||||
# 查看下载日志
|
||||
python paper_downloader.py logs
|
||||
|
||||
# 清除下载日志
|
||||
python paper_downloader.py clear
|
||||
|
||||
配置文件格式:
|
||||
{
|
||||
"name": "论文集合名称",
|
||||
"papers": [
|
||||
{
|
||||
"name": "论文标题",
|
||||
"url": "论文URL",
|
||||
"filename": "保存的文件名.pdf",
|
||||
"enabled": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
作者: Sisyphus
|
||||
版本: 1.0.0
|
||||
更新: 2026-04-14
|
||||
"""
|
||||
print(help_text)
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
if len(sys.argv) < 2:
|
||||
print_help()
|
||||
return
|
||||
|
||||
command = sys.argv[1].lower()
|
||||
|
||||
# 解析选项
|
||||
options = {"timeout": 300, "retry": 3, "chunk_size": 8192, "no_progress": False}
|
||||
|
||||
args = sys.argv[2:]
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--timeout":
|
||||
if i + 1 < len(args):
|
||||
options["timeout"] = int(args[i + 1])
|
||||
i += 2
|
||||
elif args[i] == "--retry":
|
||||
if i + 1 < len(args):
|
||||
options["retry"] = int(args[i + 1])
|
||||
i += 2
|
||||
elif args[i] == "--chunk-size":
|
||||
if i + 1 < len(args):
|
||||
options["chunk_size"] = int(args[i + 1])
|
||||
i += 2
|
||||
elif args[i] == "--no-progress":
|
||||
options["no_progress"] = True
|
||||
i += 1
|
||||
else:
|
||||
print(f"⚠️ 未知选项: {args[i]}")
|
||||
i += 1
|
||||
|
||||
# 应用配置
|
||||
config = DownloadConfig()
|
||||
if options["timeout"]:
|
||||
config.set_timeout(options["timeout"])
|
||||
if options["retry"]:
|
||||
config.set_retry(
|
||||
options["retry"], options.get("retry_delay", config.retry_delay)
|
||||
)
|
||||
if options["chunk_size"]:
|
||||
config.chunk_size = options["chunk_size"]
|
||||
if options["no_progress"]:
|
||||
config.user_agent = (
|
||||
"Mozilla/5.0 (compatible; AcademicPaperDownloader/1.0) (No Progress)"
|
||||
)
|
||||
|
||||
# 执行命令
|
||||
try:
|
||||
if command == "download":
|
||||
if len(args) > i:
|
||||
download_single(args[i])
|
||||
else:
|
||||
print("❌ 请提供论文URL")
|
||||
|
||||
elif command == "config":
|
||||
if len(args) > i:
|
||||
download_from_config(args[i])
|
||||
else:
|
||||
print("❌ 请提供配置文件路径")
|
||||
|
||||
elif command == "logs":
|
||||
show_logs()
|
||||
|
||||
elif command == "clear":
|
||||
clear_logs()
|
||||
|
||||
elif command == "create-sample":
|
||||
create_sample_config()
|
||||
|
||||
elif command == "help":
|
||||
print_help()
|
||||
|
||||
else:
|
||||
print(f"❌ 未知命令: {command}")
|
||||
print_help()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ 下载已中断")
|
||||
except Exception as e:
|
||||
print(f"\n❌ 发生错误: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user