scholaraio/config.yaml at main · ZimoLiao/scholaraio · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
# ScholarAIO configuration 配置文件

paths:
  # papers_dir: data/libraries/papers # Optional override; when omitted/legacy-default, ScholarAIO uses data/libraries/papers and auto-falls back to existing data/papers
  state_root: data/state # Durable internal state root 持久内部状态根目录
  cache_root: data/cache # Rebuildable cache root 可重建缓存根目录
  runtime_root: data/runtime # Ephemeral runtime root 临时运行产物根目录
  # index_db: data/state/search/index.db # Optional override; when omitted ScholarAIO uses state_root/search/index.db and auto-falls back to existing data/index.db

# LLM backend LLM 后端配置
# Put api_key in config.local.yaml (gitignored) or use env vars:
# 敏感信息（api_key）放 config.local.yaml（不进 git），或用环境变量：
#   SCHOLARAIO_LLM_API_KEY (通用) 或按 backend 自动查找对应环境变量：
#   openai-compat: DEEPSEEK_API_KEY / OPENAI_API_KEY
#   anthropic:     ANTHROPIC_API_KEY
#   google:        GOOGLE_API_KEY / GEMINI_API_KEY
llm:
  backend: openai-compat # openai-compat | anthropic | google
  model: deepseek-chat
  base_url: https://api.deepseek.com
  api_key: null # → config.local.yaml
  timeout: 30 # Normal LLM call timeout (sec) 普通调用超时（秒）
  timeout_toc: 120 # enrich-toc timeout enrich-toc 超时
  timeout_clean: 90 # validate_and_clean timeout 结论清洗超时
  concurrency: 32 # Max concurrent LLM calls for enrich (toc/l3) enrich 并发数

# --- Backend presets 后端预设示例 ---
# DeepSeek (default 默认):
#   backend: openai-compat
#   model: deepseek-chat
#   base_url: https://api.deepseek.com
#
# OpenAI:
#   backend: openai-compat
#   model: gpt-4o
#   base_url: https://api.openai.com
#
# Anthropic Claude:
#   backend: anthropic
#   model: claude-sonnet-4-20250514
#   base_url: https://api.anthropic.com
#
# Google Gemini:
#   backend: google
#   model: gemini-2.5-flash
#   base_url: https://generativelanguage.googleapis.com
#
# Local Ollama:
#   backend: openai-compat
#   model: qwen2.5:14b
#   base_url: http://localhost:11434

# Ingest pipeline 数据入库管道
ingest:
  extractor: robust # auto | regex | llm | robust
  mineru_endpoint: http://localhost:8000 # Local MinerU API 本地 MinerU API
  mineru_cloud_url: https://mineru.net/api/v4 # `mineru-open-api --base-url` override 私有部署覆盖地址
  mineru_api_key: null # → config.local.yaml or env MINERU_TOKEN / MINERU_API_KEY
  mineru_backend_local: pipeline # local-only backend; keep default unless you self-host MinerU 本地专用，非本地部署一般不要改
  mineru_model_version_cloud: pipeline # `mineru-open-api extract --model`: pipeline | vlm（PDF 默认用 pipeline；只在你明确需要时改成 vlm）
  mineru_lang: ch # 中文或中英混排文档保持 ch；纯英文再改 en
  mineru_parse_method: auto # auto | txt | ocr；`mineru-open-api` 只有 ocr 会映射为 --ocr
  mineru_enable_formula: true # Enable formula parsing; only effective for pipeline/vlm models 仅对 pipeline/vlm 有效
  mineru_enable_table: true # Enable table parsing; only effective for pipeline/vlm models 仅对 pipeline/vlm 有效
  abstract_llm_mode: verify # off | fallback | verify
  contact_email: null # Crossref polite pool email → config.local.yaml
  s2_api_key: null # Semantic Scholar API key → config.local.yaml or env S2_API_KEY
  mineru_batch_size: 20 # Cloud batch size per request (1-200, official limit) 云端每批文件数上限
  chunk_page_limit: 100 # Auto-split PDFs exceeding this page count 超长 PDF 自动切分页数阈值
  pdf_preferred_parser: mineru # Preferred PDF parser: mineru | docling | pymupdf；默认优先 MinerU
  pdf_fallback_order: [auto] # 完全自定义降级链；可写 [pymupdf] 或 [docling,pymupdf]，auto=自动检测本机可用解析器
  pdf_fallback_auto_detect: true # 是否启用自动检测（建议开启）

# Optional rendered web extraction for URL ingestion.
# Only use this when host-native reading cannot provide ingestion-ready rendered content.
webextract:
  transport: mcp
  mcp_url: http://127.0.0.1:8766/mcp
  api_key: null # optional bearer token -> config.local.yaml or env WEBEXTRACT_API_KEY
  mcp_tool: fetch_url

# Paper2Any external extension Paper2Any 外部增强组件
# ScholarAIO talks to a lightweight MCP sidecar. The upstream OpenDCAI/Paper2Any
# checkout stays outside tracked source, by default under data/runtime/extensions/paper2any/Paper2Any.
paper2any:
  transport: mcp
  mcp_url: http://127.0.0.1:8770/mcp
  root: null # default: data/runtime/extensions/paper2any/Paper2Any
  base_url: http://127.0.0.1:8000 # optional upstream Paper2Any FastAPI backend
  api_key: null # optional MCP bearer token -> config.local.yaml
  backend_api_key: null # required for upstream /api routes -> config.local.yaml

# Semantic embeddings 语义向量
embed:
  provider: local # local | openai-compat | none
  model: Qwen/Qwen3-Embedding-0.6B
  cache_dir: ~/.cache/modelscope/hub/models
  device: auto # auto | cpu | cuda
  top_k: 10 # vsearch default results vsearch 默认返回条数
  source: modelscope # modelscope (China) | huggingface (international) 模型下载源
  hf_endpoint: null # optional HuggingFace mirror endpoint (env SCHOLARAIO_HF_ENDPOINT > HF_ENDPOINT)
  api_base: null # OpenAI-compatible endpoint, e.g. https://api.openai.com/v1
  api_key: null # -> config.local.yaml or env SCHOLARAIO_EMBED_API_KEY
  api_timeout: 30
  batch_size: 64
  max_retries: 3

search:
  top_k: 20 # FTS5 search default results 关键词搜索默认返回条数

# Logging & metrics 日志与指标
logging:
  level: INFO
  file: data/scholaraio.log
  max_bytes: 10000000 # 10 MB, rotate on overflow 超出轮转
  backup_count: 3
  # metrics_db: data/state/metrics/metrics.db # Optional override; omitted by default so existing data/metrics.db can still be discovered automatically

# Topic modeling 主题建模
topics:
  min_topic_size: 5 # HDBSCAN min cluster size 最小聚类大小
  nr_topics: 0 # 0=auto, -1=no merge, N=target count 目标主题数
  # model_dir: data/state/topics # Optional override; omitted by default so existing data/topic_model/ can still be discovered automatically

# Translation 翻译
translate:
  auto_translate: false # Auto-translate non-target-language papers during pipeline ingest 是否在入库时自动翻译
  target_lang: zh # Target language code 目标语言代码
  chunk_size: 4000 # Max chars per translation chunk 分块翻译最大字符数
  concurrency: 20 # Total translation concurrency budget; single-paper uses it for chunk parallelism, batch mode splits it across papers 总翻译并发预算

# Rsync backup 远程增量备份
backup:
  source_dir: data # Local directory to sync 本地备份源目录（默认整个 data）
  rsync_bin: rsync # rsync executable 可执行文件名或绝对路径
  ssh_bin: ssh # ssh executable 可执行文件名或绝对路径；backup run 会强制追加 -o BatchMode=yes
  targets: {}
  # targets:
  #   lab:
  #     host: backup.example.com
  #     user: alice
  #     path: /srv/scholaraio
  #     port: 22
  #     identity_file: ~/.ssh/id_ed25519 # Prefer placing secrets/host-specific overrides in config.local.yaml
  #     password: "" # Optional fallback for password-only hosts; keep it only in config.local.yaml
  #     mode: default # default | append | append-verify
  #     compress: true
  #     enabled: true
  #     exclude: ["*.tmp", "metrics.db"]
  # Prepare SSH keys or a config.local.yaml password override, plus known_hosts; backup run is non-interactive.

# Static published-paper site generation 发布站点生成
publish:
  site_output_dir: null # Optional default for `scholaraio publish-site --out-dir`
  published_dir: published # Local audited archive root; git-ignored by this repo

# OpenAlex API key (optional, enables higher rate limits)
# openalex:
#   api_key: null  # -> config.local.yaml or env OPENALEX_API_KEY

# Zotero integration Zotero 集成
zotero:
  library_type: user # user | group