Files
SakuraHub/scripts/append_review_to_docx.py
T
2026-11-16 20:45:22 +08:00

556 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Append "两项核心改进" markdown content to the main docx document.
"""
import sys, os, re
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from docx import Document
from docx.shared import Pt, Inches, RGBColor
from docx.oxml.ns import qn
DOCX_PATH = "docs/Sakura_Downloader_完整项目文档.docx"
MD_PATH = "docs/三项核心改进-分页规则_搜索健康分_书架保存解耦.md"
doc = Document(DOCX_PATH)
# ---- helpers ----
from docx.oxml import OxmlElement
def page_break():
p = doc.add_paragraph()
run = p.add_run()
br = OxmlElement('w:br')
br.set(qn('w:type'), 'page')
run._element.append(br)
def heading(text, level=1):
return doc.add_heading(text, level=level)
def para(text):
return doc.add_paragraph(text)
def bold_para(text):
p = doc.add_paragraph()
run = p.add_run(text)
run.bold = True
return p
def code_block(code_text):
for line in code_text.split('\n'):
p = doc.add_paragraph()
p.paragraph_format.space_before = Pt(0)
p.paragraph_format.space_after = Pt(0)
p.paragraph_format.left_indent = Inches(0.3)
run = p.add_run(line if line else ' ')
run.font.name = 'Consolas'
run.font.size = Pt(8.5)
run.font.color.rgb = RGBColor(0x33, 0x33, 0x33)
try:
run._element.rPr.rFonts.set(qn('w:eastAsia'), 'Consolas')
except Exception:
pass
def make_table(headers, rows):
table = doc.add_table(rows=1, cols=len(headers), style='Table Grid')
hdr_cells = table.rows[0].cells
for i, h in enumerate(headers):
hdr_cells[i].text = h
for p in hdr_cells[i].paragraphs:
for run in p.runs:
run.bold = True
for row_data in rows:
row = table.add_row()
for i, txt in enumerate(row_data):
row.cells[i].text = str(txt)
return table
# ============================================================
# APPEND NEW APPENDIX
# ============================================================
page_break()
heading('附录 B — 两项核心改进:TOC 分页规则 · 搜索健康分', level=1)
para('')
para('日期:2026-06-22(v2,整合 Claude / ChatGPT / Gemini 三方 review)')
para('范围:仅移动端 Dart 侧(sakura_mobile/lib/src/core/)')
para('原则:不改 Python 插件,不改 UI 层,只动 core 层。')
para('')
# ── Part 0 ──
heading('〇、前置声明:图片解析问题是独立维度', level=2)
para('ChatGPT 指出一个关键事实——当前用户最痛的是"搜得到、看不了",而不是"搜得慢"。图片解析失败的根因是 @js 规则执行器缺失(newImgs、eval(result.match(...)) 等 JS 表达式无法执行),这套方案没有直接解决它。')
para('真正解决"看不了"需要:LegadoRuleExecutor V2 + QuickJS,见《Legado规则引擎重构方案》。')
para('本文档解决的是另外两个独立的、也必须解决的痛点:')
para(' • 目录只拿到前 50 章 → 改进 1')
para(' • 搜索等 20 秒才出结果 → 改进 2')
para(' • 打开章节图片为空 → 需要 JS 引擎(本文档范围外)')
para('实施时不冲突,可以并行推进。但 TOC 分页规则中有 <js> / @js: 语法,如果涉及 java.get() / java.put() / java.ajax(),需要等 JS 引擎到位后才能完整支持。详见 §1.7。')
para('')
# ── Part 1 ──
heading('一、改进 1:TOC 章节目录分页规则完整支持', level=2)
heading('1.1 问题', level=3)
para('fetchToc() 只拉第一页目录。很多书源第一页只给 50~100 章,剩余的靠分页目录规则指向下一页。不支持分页规则 → 用户看到的目录永远不全。')
heading('1.2 兼容的字段名(解析时一次性确定)', level=3)
para('书源 JSON 中,下一页 URL 的字段名可能是以下任意一种。在 fromJson 阶段遍历一次并存入 _resolvedTocNextPageRule,避免每次访问 getter 都遍历。')
code_block("""static const List<String> _tocNextPageCandidates = [
'tocNextPageRule', // Legado 标准名
'nextTocUrl', // 常见别名
'nextUrl', // 通用别名
'nextPage', // 页码式
'tocNextPage', // 变体
];
String? _resolvedTocNextPageRule;
String? get tocNextPageRule => _resolvedTocNextPageRule;
// 在 fromJson 中调用一次:
void _resolveTocNextPageRule(Map<String, dynamic> json) {
for (final name in _tocNextPageCandidates) {
final v = json[name];
if (v is String && v.trim().isNotEmpty) {
_resolvedTocNextPageRule = v.trim();
return;
}
}
_resolvedTocNextPageRule = null;
}""")
heading('1.3 分页规则语法全集', level=3)
make_table(
['规则类型', '语法示例', '说明'],
[
['JS 表达式(简单)', '<js>nextUrl</js>', '简单 JS 可直接 eval'],
['JS 表达式(复杂)', '<js>java.get("baseUrl")</js>', '需要 JS 引擎(见 §1.7)'],
['JS 缩写', '@js:nextUrl', '同上'],
['模板拼接', 'https://host/list?page={{page}}', '{{page}} 从 1 自增'],
['偏移模板', '{{page+1}} / {{page-1}}', '页号偏移'],
['CSS 选择器', 'text.下一页@href', '从当前页 HTML 提取'],
['class 选择器', 'class.next@href', '同上'],
['id 选择器', 'id.pagenext@href', '同上'],
['XPath', '//a[contains(text(),\'下一页\')]/@href', 'XPath 提取'],
['字符串切片', '!0:-1', '对当前 URL 切片'],
['offset 翻页', 'offset=50', '按 offset 递增'],
['纯文本 URL', 'https://...', '直接跳转'],
['相对路径', '/list/2.html', '基于当前 URL 拼接'],
]
)
para('')
heading('1.4 实现 — TocPaginationResolver', level=3)
code_block("""class TocPaginationResolver {
final LegadoRuleExecutor _ruleExecutor;
final HttpClient _client;
TocPaginationResolver(this._ruleExecutor, this._client);
/// 拉取全部分页章节,合并去重。
/// 返回 Stream 以便 UI 逐页展示进度。
Stream<List<ChapterInfo>> fetchAllChapters({
required String firstPageUrl,
required String? tocNextPageRule,
required String tocRule,
required Map<String, String>? headers,
int maxPages = 20,
}) async* {
final all = <ChapterInfo>[];
final seenPageUrls = <String>{};
final seenChapterUrls = <String>{};
String? url = firstPageUrl;
int page = 0;
while (url != null && page < maxPages) {
if (!seenPageUrls.add(url)) break;
final html = await _client.get(url, headers: headers);
final batch = _ruleExecutor.selectList(html, url, rule: tocRule);
int added = 0;
for (final ch in batch) {
if (seenChapterUrls.add(ch.url)) {
all.add(ch);
added++;
}
}
if (batch.isNotEmpty) yield List.of(batch);
if (added == 0) break;
url = _resolveNextPage(
currentUrl: url,
currentHtml: html,
rule: tocNextPageRule,
pageIndex: page,
);
page++;
}
}
String? _resolveNextPage({
required String currentUrl,
required String currentHtml,
required String? rule,
required int pageIndex,
}) {
if (rule == null || rule.trim().isEmpty) return null;
final r = rule.trim();
// 1) JS: <js>...</js> 或 @js:...
if (r.startsWith('<js>') || r.startsWith('@js:')) {
final code = r
.replaceFirst(RegExp(r'^<js>'), '')
.replaceFirst(RegExp(r'</js>$'), '')
.replaceFirst(RegExp(r'^@js:'), '');
return _ruleExecutor.evalJs(code, context: {
'result': currentHtml,
'baseUrl': currentUrl,
'page': pageIndex,
});
}
// 2) 切片
if (r.startsWith('!')) return _applySlice(currentUrl, r.substring(1));
// 3) CSS 选择器 / XPath
if (_isSelector(r)) return _ruleExecutor.select(currentHtml, currentUrl, rule: r);
// 4) 模板 {{page}} / {{page+1}} / {{page-1}}
if (r.contains('{{page')) return _renderPageTemplate(r, pageIndex);
// 5) offset / start 参数
if (RegExp(r'[?&](offset|start)=').hasMatch(r)) return _renderOffsetTemplate(r, pageIndex);
// 6) 绝对 URL
if (r.startsWith('http://') || r.startsWith('https://')) return _renderPageTemplate(r, pageIndex);
// 7) 相对路径
return Uri.parse(currentUrl).resolve(r).toString();
}
bool _isSelector(String s) =>
s.startsWith('text.') || s.startsWith('class.') ||
s.startsWith('id.') || s.startsWith('@') || s.startsWith('//');
// 模板渲染(Dart 语法修正:用 .toString() 而非字符串插值)
String _renderPageTemplate(String tpl, int page) {
return tpl
.replaceAll('{{page+1}}', (page + 1).toString())
.replaceAll('{{page-1}}', (page - 1).toString())
.replaceAll('{{page}}', page.toString());
}
// offset 修正:initial + initial * page,而非 initial * page
String _renderOffsetTemplate(String tpl, int page) {
return tpl
.replaceAllMapped(RegExp(r'offset=(\\d+)'), (m) {
final initial = int.parse(m.group(1)!);
return 'offset=${initial + initial * page}';
})
.replaceAllMapped(RegExp(r'start=(\\d+)'), (m) {
final initial = int.parse(m.group(1)!);
return 'start=${initial + initial * page}';
});
}
String? _applySlice(String url, String expr) {
final parts = expr.split(':');
if (parts.length != 2) return null;
final a = int.tryParse(parts[0]) ?? 0;
final b = int.tryParse(parts[1]);
final start = a < 0 ? url.length + a : a;
if (b != null) {
final end = b < 0 ? url.length + b : b;
return url.substring(start, end);
}
return url.substring(start);
}
}""")
para('')
heading('1.5 集成到 LegadoEngine.fetchToc', level=3)
code_block("""Future<List<ChapterInfo>> fetchToc(BookSource source, String tocUrl) async {
final rule = source.tocRule;
if (rule == null || rule.isEmpty) return [];
final resolver = TocPaginationResolver(_ruleExecutor, _client);
final all = <ChapterInfo>[];
await for (final batch in resolver.fetchAllChapters(
firstPageUrl: tocUrl,
tocNextPageRule: source.tocNextPageRule,
tocRule: rule,
headers: _buildHeaders(source),
maxPages: 20,
)) {
all.addAll(batch);
}
return all;
}""")
para('')
heading('1.6 死循环防护(三层)', level=3)
para(' 1. maxPages = 20 — 单书最多翻 20 页目录')
para(' 2. seenPageUrls — 同一 URL 不重复请求')
para(' 3. added == 0 终止 — 当前页无新增章节则停止,防止环状分页')
heading('1.7 JS 规则的兼容边界', level=3)
para('TOC 分页规则中有 <js> / @js: 语法,分为两档:')
make_table(
['复杂度', '示例', '当前能否跑'],
[
['简单 JS', 'nextUrl', '✅ 正则/字符串拼接能跑'],
['中等 JS', 'JSON.parse(result).nextPage', '✅ 如有 JSON 解析'],
['复杂 JS', 'java.get("baseUrl")', '❌ 需要 LegadoJsRuntime'],
['复杂 JS', 'java.ajax(url)', '❌ 需要完整 QuickJS'],
]
)
para('简单 JS 分页规则(约占 60%)可立即支持;复杂 JS 需要等 JS 引擎到位后补上。届时只需在 _resolveNextPage 的 JS 分支里替换 eval 实现即可,代码结构不需要改。')
para('')
# ── Part 2 ──
heading('二、改进 2:搜索源健康分 + 短超时 + 滑动窗口并发', level=2)
heading('2.1 问题', level=3)
para('当前 search() 对所有启用源并发请求,每个源 timeout 20~30 秒。10 个源一批,坏源多时整批拖到 20+ 秒。两个根因:')
para(' 1. 坏源不降权 — 每次都等它超时')
para(' 2. Future.wait 批量阻塞 — 一个慢源卡住整批')
heading('2.2 源健康分模型(SourceHealth)', level=3)
para('核心特性:')
para(' • 成功率 60% + 平均延迟 25% + 新鲜度 15%,满分 100')
para(' • 连续失败 ≥5 → 熔断,但每 1 小时自动释放一次探测机会(熔断恢复)')
para(' • 分数在 getter 中动态计算衰减(超过 7 天无活动折半),不依赖 save 时机')
para(' • 衰减不修改原始数据,只在读取时乘以系数')
code_block("""class SourceHealth {
final String sourceId;
int successCount = 0, failCount = 0, timeoutCount = 0;
int totalLatencyMs = 0, _consecutiveFails = 0;
DateTime? lastSuccessAt, lastFailAt;
// 熔断 + 自动恢复:连续失败 >=5 且距上次失败 <1h -> 禁用
bool get isUsable {
if (_consecutiveFails >= 5) {
if (lastFailAt != null &&
DateTime.now().difference(lastFailAt!).inHours >= 1) return true;
return false;
}
if (failCount >= 5 && successCount == 0) return false;
return true;
}
// 健康分 0~100(动态衰减,不依赖 save 时机)
double get score {
final latest = lastSuccessAt ?? lastFailAt;
final decay = latest != null &&
DateTime.now().difference(latest).inDays >= 7 ? 0.5 : 1.0;
final total = successCount + failCount + timeoutCount;
if (total == 0) return 50.0 * decay;
double s = 60.0 * (successCount / total.toDouble());
if (successCount > 0 && totalLatencyMs > 0) {
final avg = totalLatencyMs / successCount;
if (avg < 500) s += 25.0;
else if (avg < 2000) s += 25.0 * (1.0 - (avg - 500) / 1500);
}
if (lastSuccessAt != null) {
final h = DateTime.now().difference(lastSuccessAt!).inHours;
if (h < 1) s += 15.0;
else if (h < 24) s += 15.0 * (1.0 - h / 24);
}
return (s * decay).clamp(0.0, 100.0);
}
void recordSuccess(int latencyMs) { successCount++; totalLatencyMs += latencyMs; lastSuccessAt = DateTime.now(); _consecutiveFails = 0; }
void recordFailure() { failCount++; lastFailAt = DateTime.now(); _consecutiveFails++; }
void recordTimeout() { timeoutCount++; lastFailAt = DateTime.now(); _consecutiveFails++; }
}""")
para('')
heading('2.3 健康分持久化(防抖 + 延迟写盘)', level=3)
para('SourceHealthStore 关键设计:')
para(' • recordSuccess/recordFailure/recordTimeout 只改内存 Map,不触发 IO')
para(' • save() 启动 2 秒 Timer,同一轮搜索的多次调用被合并(防抖)')
para(' • flush() 立即写盘(仅在 app 退出时调用,取消 Timer 后直接写)')
para(' • 衰减计算在 SourceHealth.score getter 中动态完成,save 时不修改原始数据')
heading('2.4 分阶段搜索调度器(滑动窗口 + 取消机制)', level=3)
para('核心改进:')
para(' • 滑动窗口 _pool 替代 Future.wait — 最大并发 6,谁先完成谁先返回,快源不阻塞')
para(' • _searchToken 版本号 — 用户连续搜索 A→B→C 时旧结果自动丢弃')
para(' • 慢源结果走同一个 Stream yield(不引入 EventBus),保持数据流单一源')
para(' • 整轮搜索结束后统一调用一次 health.save()')
code_block("""class SearchScheduler {
static const fastTimeout = Duration(seconds: 5);
static const slowTimeout = Duration(seconds: 8);
static const maxConcurrent = 6;
int _searchToken = 0; // 取消旧搜索用
Stream<SearchPhase> search(String keyword, List<BookSource> allSources) async* {
final token = ++_searchToken;
final usable = allSources.where((s) => _health.isUsable(s.id)).toList()
..sort((a, b) => _health.score(b.id).compareTo(_health.score(a.id)));
if (usable.isEmpty) { yield SearchPhase(name: 'error', message: '没有可用的搜索源'); return; }
// 分三档
final fast = usable.where((s) => _health.score(s.id) >= 70).toList();
final normal = usable.where((s) { final sc = _health.score(s.id); return sc >= 40 && sc < 70; }).toList();
final slow = usable.where((s) => _health.score(s.id) < 40).toList();
final results = <SearchItem>[], doneIds = <String>{}, seen = <String>{};
// 阶段 1:快速源(5s 超时,滑动窗口)
await for (final r in _pool(fast, keyword, timeout: fastTimeout, token: token))
{ doneIds.add(r.sourceId); _merge(results, r.items, seen); }
yield SearchPhase(name: 'partial', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length);
// 阶段 2:普通源(8s 超时)
await for (final r in _pool(normal, keyword, timeout: slowTimeout, token: token))
{ doneIds.add(r.sourceId); _merge(results, r.items, seen); }
yield SearchPhase(name: 'partial', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length);
// 阶段 3:慢源后台继续,走同一个 Stream
if (slow.isNotEmpty) {
yield SearchPhase(name: 'pending_slow', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length,
message: '${slow.length} 个慢源正在后台搜索...');
await for (final r in _pool(slow, keyword, timeout: slowTimeout, token: token)) {
if (token != _searchToken) return; // 用户发起新搜索,丢弃旧结果
doneIds.add(r.sourceId); _merge(results, r.items, seen);
yield SearchPhase(name: 'slow_append', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length);
}
}
yield SearchPhase(name: 'complete', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length);
unawaited(_health.save()); // 整轮搜索结束统一写盘
}
// 滑动窗口并发(替代 Future.wait)
Stream<_SourceResult> _pool(List<BookSource> sources, String keyword, {required Duration timeout, required int token}) async* {
final pending = sources.toList();
final running = <Future<_SourceResult?>>{};
while (pending.isNotEmpty || running.isNotEmpty) {
while (running.length < maxConcurrent && pending.isNotEmpty)
running.add(_searchOne(pending.removeAt(0), keyword, timeout, token: token));
if (running.isEmpty) break;
final done = await Future.any(running.map((f) => f.then((v) => (f, v))));
running.remove(done.$1);
if (done.$2 != null) yield done.$2!;
}
}
Future<_SourceResult?> _searchOne(BookSource source, String keyword, Duration timeout, {required int token}) async {
if (token != _searchToken) return null;
final sw = Stopwatch()..start();
try {
final items = await _engine.search(source, keyword).timeout(timeout);
if (token != _searchToken) return null;
sw.stop();
_health.recordSuccess(source.id, sw.elapsedMilliseconds);
return _SourceResult(source.id, items);
} on TimeoutException {
_health.recordTimeout(source.id);
return null;
} catch (_) {
_health.recordFailure(source.id);
return null;
}
}
}
class _SourceResult { final String sourceId; final List<SearchItem> items; _SourceResult(this.sourceId, this.items); }
class SearchPhase {
final String name; // searching | partial | pending_slow | slow_append | complete | error
final List<SearchItem> items;
final int doneCount, totalCount;
final String? message;
SearchPhase({required this.name, this.items = const [], this.doneCount = 0, this.totalCount = 0, this.message});
}""")
para('')
heading('2.5 UI 增量渲染时序', level=3)
para('t=0s → 显示"正在搜索 N 个源..."')
para('t=0~5s → 快速源逐个完成 → 滑动窗口下结果立刻追加到列表')
para('t=5~8s → 普通源逐个完成 → 继续追加')
para('t=8s → "慢源正在后台搜索..." → 用户已看到前两档结果')
para('t=8~15s → 慢源逐个完成 → 静默追加到列表末尾')
para('t=15s → "搜索完成"')
heading('2.6 取消机制', level=3)
para('用户连续搜 A→B→C 时:_searchToken 递增版本号。_searchOne 开头检查 token != _searchToken 则 return null;阶段 3 yield 前检查则 return 丢弃旧搜索的后台任务。旧搜索的未完成请求仍会走完网络超时,但结果不会污染新搜索的 UI。')
para('')
# ── Part 3 ──
heading('三、两改进的协作关系', level=2)
para('用户搜索一本书 → SearchScheduler(按健康分排序、滑动窗口、增量推 UI)→ 用户点击结果 → fetchBookInfo → fetchToc(TocPaginationResolver 拉全章节,支持分页)→ 用户阅读某章 → fetchChapterContent(需要 JS 引擎,本文档范围外)。')
para('改进 1 保证"搜到书后章节不断尾",改进 2 保证"搜索本身不卡"。')
para('')
# ── Part 4 ──
heading('四、实施顺序', level=2)
para('第 1 步:SourceHealth + SourceHealthStore(纯数据层,不影响现有流程,可与图片解析的 JS 引擎并行推进,先跑几天积累健康数据)')
para('第 2 步:SearchScheduler 滑动窗口 + 短超时 + Stream(替换现有 search() 并发逻辑,UI 适配 Stream<SearchPhase>)')
para('第 3 步:TocPaginationResolver(简单 JS/CSS/模板分页立即生效约 60% 源;复杂 JS 分页需要等 JS 引擎,代码结构无需改动)')
para('第 4 步:联调 + 回归')
para('')
para('与图片解析的关系:第 1~2 步与 LegadoRuleExecutor V2 + QuickJS 可并行推进、互不冲突。')
para('')
# ── Part 5 ──
heading('五、风险控制', level=2)
make_table(
['风险', '应对'],
[
['TOC 分页死循环', 'maxPages=20 + seenPageUrls 去重 + added==0 终止'],
['复杂 JS 分页规则暂时跑不通', '简单规则(~60%)立即支持;复杂规则骨架预留,换 JS 引擎即可'],
['健康分误杀快源(网站临时维护)', '连续失败 ≥5 才熔断 + 每 1h 自动释放探测机会 + 手动可重置'],
['搜索时 SharedPreferences 频繁 I/O', '内存改、整轮结束统一写盘 + 2s 防抖'],
['滑动窗口仍有慢源卡槽位', '超时兜底(5s/8s),超时后释放槽位'],
['连续快速搜索旧结果污染 UI', '_searchToken 版本号,旧结果自动丢弃'],
['新源全部 50 分', '第一轮搜索后就有真实分,冷启动无影响'],
['Future.wait 批阻塞', '已替换为滑动窗口 _pool,先完成先返回'],
]
)
para('')
para('')
# ── Part 6 ──
heading('六、Review 整合清单', level=2)
para('以下为三方 review 指出的问题和本方案的对应处理:')
make_table(
['Reviewer', '指出的问题', 'v2 的处理'],
[
['Claude', 'offset 计算 initial * page 有误', '改为 initial + initial * page'],
['Claude', '阶段 3 后台任务无取消机制', '加入 _searchToken 版本号取消'],
['Claude', '衰减在 save() 里做不可靠', '衰减改为 score getter 动态计算'],
['Claude', '字段名每次访问都遍历', '改为 fromJson 时一次性解析到字段'],
['Claude', '第 3 步依赖 Legado 重构前置', '明确标注依赖,与 JS 引擎可并行'],
['ChatGPT', '图片解析才是 P0,优先级需修正', '§0 前置声明 + §4 与图片解析并行'],
['ChatGPT', '<js> 规则含 java.xxx 无法执行', '§1.7 明确兼容边界,骨架预留'],
['Gemini', 'Future.wait 批阻塞快源', '改为滑动窗口 _pool'],
['Gemini', '熔断无自动恢复', 'isUsable 加 1h 自动释放探测'],
['Gemini', 'save() 高频 I/O', '防抖 2s + 整轮结束统一写'],
['Gemini', '慢源用 EventBus 引入第二数据源', '慢源走同一个 Stream yield'],
['Gemini', '\'${page + 1}\' 字符串字面量错误', '改为 (page + 1).toString()'],
]
)
# ---- Save ----
doc.save(DOCX_PATH)
print('Done! Document updated successfully.')
print(f'Total paragraphs now: {len(doc.paragraphs)}')
print(f'Total tables now: {len(doc.tables)}')