""" Append "两项核心改进" markdown content to the main docx document. """ import sys, os, re sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from docx import Document from docx.shared import Pt, Inches, RGBColor from docx.oxml.ns import qn DOCX_PATH = "docs/Sakura_Downloader_完整项目文档.docx" MD_PATH = "docs/三项核心改进-分页规则_搜索健康分_书架保存解耦.md" doc = Document(DOCX_PATH) # ---- helpers ---- from docx.oxml import OxmlElement def page_break(): p = doc.add_paragraph() run = p.add_run() br = OxmlElement('w:br') br.set(qn('w:type'), 'page') run._element.append(br) def heading(text, level=1): return doc.add_heading(text, level=level) def para(text): return doc.add_paragraph(text) def bold_para(text): p = doc.add_paragraph() run = p.add_run(text) run.bold = True return p def code_block(code_text): for line in code_text.split('\n'): p = doc.add_paragraph() p.paragraph_format.space_before = Pt(0) p.paragraph_format.space_after = Pt(0) p.paragraph_format.left_indent = Inches(0.3) run = p.add_run(line if line else ' ') run.font.name = 'Consolas' run.font.size = Pt(8.5) run.font.color.rgb = RGBColor(0x33, 0x33, 0x33) try: run._element.rPr.rFonts.set(qn('w:eastAsia'), 'Consolas') except Exception: pass def make_table(headers, rows): table = doc.add_table(rows=1, cols=len(headers), style='Table Grid') hdr_cells = table.rows[0].cells for i, h in enumerate(headers): hdr_cells[i].text = h for p in hdr_cells[i].paragraphs: for run in p.runs: run.bold = True for row_data in rows: row = table.add_row() for i, txt in enumerate(row_data): row.cells[i].text = str(txt) return table # ============================================================ # APPEND NEW APPENDIX # ============================================================ page_break() heading('附录 B — 两项核心改进:TOC 分页规则 · 搜索健康分', level=1) para('') para('日期:2026-06-22(v2,整合 Claude / ChatGPT / Gemini 三方 review)') para('范围:仅移动端 Dart 侧(sakura_mobile/lib/src/core/)') para('原则:不改 Python 插件,不改 UI 层,只动 core 层。') para('') # ── Part 0 ── heading('〇、前置声明:图片解析问题是独立维度', level=2) para('ChatGPT 指出一个关键事实——当前用户最痛的是"搜得到、看不了",而不是"搜得慢"。图片解析失败的根因是 @js 规则执行器缺失(newImgs、eval(result.match(...)) 等 JS 表达式无法执行),这套方案没有直接解决它。') para('真正解决"看不了"需要:LegadoRuleExecutor V2 + QuickJS,见《Legado规则引擎重构方案》。') para('本文档解决的是另外两个独立的、也必须解决的痛点:') para(' • 目录只拿到前 50 章 → 改进 1') para(' • 搜索等 20 秒才出结果 → 改进 2') para(' • 打开章节图片为空 → 需要 JS 引擎(本文档范围外)') para('实施时不冲突,可以并行推进。但 TOC 分页规则中有 / @js: 语法,如果涉及 java.get() / java.put() / java.ajax(),需要等 JS 引擎到位后才能完整支持。详见 §1.7。') para('') # ── Part 1 ── heading('一、改进 1:TOC 章节目录分页规则完整支持', level=2) heading('1.1 问题', level=3) para('fetchToc() 只拉第一页目录。很多书源第一页只给 50~100 章,剩余的靠分页目录规则指向下一页。不支持分页规则 → 用户看到的目录永远不全。') heading('1.2 兼容的字段名(解析时一次性确定)', level=3) para('书源 JSON 中,下一页 URL 的字段名可能是以下任意一种。在 fromJson 阶段遍历一次并存入 _resolvedTocNextPageRule,避免每次访问 getter 都遍历。') code_block("""static const List _tocNextPageCandidates = [ 'tocNextPageRule', // Legado 标准名 'nextTocUrl', // 常见别名 'nextUrl', // 通用别名 'nextPage', // 页码式 'tocNextPage', // 变体 ]; String? _resolvedTocNextPageRule; String? get tocNextPageRule => _resolvedTocNextPageRule; // 在 fromJson 中调用一次: void _resolveTocNextPageRule(Map json) { for (final name in _tocNextPageCandidates) { final v = json[name]; if (v is String && v.trim().isNotEmpty) { _resolvedTocNextPageRule = v.trim(); return; } } _resolvedTocNextPageRule = null; }""") heading('1.3 分页规则语法全集', level=3) make_table( ['规则类型', '语法示例', '说明'], [ ['JS 表达式(简单)', 'nextUrl', '简单 JS 可直接 eval'], ['JS 表达式(复杂)', 'java.get("baseUrl")', '需要 JS 引擎(见 §1.7)'], ['JS 缩写', '@js:nextUrl', '同上'], ['模板拼接', 'https://host/list?page={{page}}', '{{page}} 从 1 自增'], ['偏移模板', '{{page+1}} / {{page-1}}', '页号偏移'], ['CSS 选择器', 'text.下一页@href', '从当前页 HTML 提取'], ['class 选择器', 'class.next@href', '同上'], ['id 选择器', 'id.pagenext@href', '同上'], ['XPath', '//a[contains(text(),\'下一页\')]/@href', 'XPath 提取'], ['字符串切片', '!0:-1', '对当前 URL 切片'], ['offset 翻页', 'offset=50', '按 offset 递增'], ['纯文本 URL', 'https://...', '直接跳转'], ['相对路径', '/list/2.html', '基于当前 URL 拼接'], ] ) para('') heading('1.4 实现 — TocPaginationResolver', level=3) code_block("""class TocPaginationResolver { final LegadoRuleExecutor _ruleExecutor; final HttpClient _client; TocPaginationResolver(this._ruleExecutor, this._client); /// 拉取全部分页章节,合并去重。 /// 返回 Stream 以便 UI 逐页展示进度。 Stream> fetchAllChapters({ required String firstPageUrl, required String? tocNextPageRule, required String tocRule, required Map? headers, int maxPages = 20, }) async* { final all = []; final seenPageUrls = {}; final seenChapterUrls = {}; String? url = firstPageUrl; int page = 0; while (url != null && page < maxPages) { if (!seenPageUrls.add(url)) break; final html = await _client.get(url, headers: headers); final batch = _ruleExecutor.selectList(html, url, rule: tocRule); int added = 0; for (final ch in batch) { if (seenChapterUrls.add(ch.url)) { all.add(ch); added++; } } if (batch.isNotEmpty) yield List.of(batch); if (added == 0) break; url = _resolveNextPage( currentUrl: url, currentHtml: html, rule: tocNextPageRule, pageIndex: page, ); page++; } } String? _resolveNextPage({ required String currentUrl, required String currentHtml, required String? rule, required int pageIndex, }) { if (rule == null || rule.trim().isEmpty) return null; final r = rule.trim(); // 1) JS: ... 或 @js:... if (r.startsWith('') || r.startsWith('@js:')) { final code = r .replaceFirst(RegExp(r'^'), '') .replaceFirst(RegExp(r'$'), '') .replaceFirst(RegExp(r'^@js:'), ''); return _ruleExecutor.evalJs(code, context: { 'result': currentHtml, 'baseUrl': currentUrl, 'page': pageIndex, }); } // 2) 切片 if (r.startsWith('!')) return _applySlice(currentUrl, r.substring(1)); // 3) CSS 选择器 / XPath if (_isSelector(r)) return _ruleExecutor.select(currentHtml, currentUrl, rule: r); // 4) 模板 {{page}} / {{page+1}} / {{page-1}} if (r.contains('{{page')) return _renderPageTemplate(r, pageIndex); // 5) offset / start 参数 if (RegExp(r'[?&](offset|start)=').hasMatch(r)) return _renderOffsetTemplate(r, pageIndex); // 6) 绝对 URL if (r.startsWith('http://') || r.startsWith('https://')) return _renderPageTemplate(r, pageIndex); // 7) 相对路径 return Uri.parse(currentUrl).resolve(r).toString(); } bool _isSelector(String s) => s.startsWith('text.') || s.startsWith('class.') || s.startsWith('id.') || s.startsWith('@') || s.startsWith('//'); // 模板渲染(Dart 语法修正:用 .toString() 而非字符串插值) String _renderPageTemplate(String tpl, int page) { return tpl .replaceAll('{{page+1}}', (page + 1).toString()) .replaceAll('{{page-1}}', (page - 1).toString()) .replaceAll('{{page}}', page.toString()); } // offset 修正:initial + initial * page,而非 initial * page String _renderOffsetTemplate(String tpl, int page) { return tpl .replaceAllMapped(RegExp(r'offset=(\\d+)'), (m) { final initial = int.parse(m.group(1)!); return 'offset=${initial + initial * page}'; }) .replaceAllMapped(RegExp(r'start=(\\d+)'), (m) { final initial = int.parse(m.group(1)!); return 'start=${initial + initial * page}'; }); } String? _applySlice(String url, String expr) { final parts = expr.split(':'); if (parts.length != 2) return null; final a = int.tryParse(parts[0]) ?? 0; final b = int.tryParse(parts[1]); final start = a < 0 ? url.length + a : a; if (b != null) { final end = b < 0 ? url.length + b : b; return url.substring(start, end); } return url.substring(start); } }""") para('') heading('1.5 集成到 LegadoEngine.fetchToc', level=3) code_block("""Future> fetchToc(BookSource source, String tocUrl) async { final rule = source.tocRule; if (rule == null || rule.isEmpty) return []; final resolver = TocPaginationResolver(_ruleExecutor, _client); final all = []; await for (final batch in resolver.fetchAllChapters( firstPageUrl: tocUrl, tocNextPageRule: source.tocNextPageRule, tocRule: rule, headers: _buildHeaders(source), maxPages: 20, )) { all.addAll(batch); } return all; }""") para('') heading('1.6 死循环防护(三层)', level=3) para(' 1. maxPages = 20 — 单书最多翻 20 页目录') para(' 2. seenPageUrls — 同一 URL 不重复请求') para(' 3. added == 0 终止 — 当前页无新增章节则停止,防止环状分页') heading('1.7 JS 规则的兼容边界', level=3) para('TOC 分页规则中有 / @js: 语法,分为两档:') make_table( ['复杂度', '示例', '当前能否跑'], [ ['简单 JS', 'nextUrl', '✅ 正则/字符串拼接能跑'], ['中等 JS', 'JSON.parse(result).nextPage', '✅ 如有 JSON 解析'], ['复杂 JS', 'java.get("baseUrl")', '❌ 需要 LegadoJsRuntime'], ['复杂 JS', 'java.ajax(url)', '❌ 需要完整 QuickJS'], ] ) para('简单 JS 分页规则(约占 60%)可立即支持;复杂 JS 需要等 JS 引擎到位后补上。届时只需在 _resolveNextPage 的 JS 分支里替换 eval 实现即可,代码结构不需要改。') para('') # ── Part 2 ── heading('二、改进 2:搜索源健康分 + 短超时 + 滑动窗口并发', level=2) heading('2.1 问题', level=3) para('当前 search() 对所有启用源并发请求,每个源 timeout 20~30 秒。10 个源一批,坏源多时整批拖到 20+ 秒。两个根因:') para(' 1. 坏源不降权 — 每次都等它超时') para(' 2. Future.wait 批量阻塞 — 一个慢源卡住整批') heading('2.2 源健康分模型(SourceHealth)', level=3) para('核心特性:') para(' • 成功率 60% + 平均延迟 25% + 新鲜度 15%,满分 100') para(' • 连续失败 ≥5 → 熔断,但每 1 小时自动释放一次探测机会(熔断恢复)') para(' • 分数在 getter 中动态计算衰减(超过 7 天无活动折半),不依赖 save 时机') para(' • 衰减不修改原始数据,只在读取时乘以系数') code_block("""class SourceHealth { final String sourceId; int successCount = 0, failCount = 0, timeoutCount = 0; int totalLatencyMs = 0, _consecutiveFails = 0; DateTime? lastSuccessAt, lastFailAt; // 熔断 + 自动恢复:连续失败 >=5 且距上次失败 <1h -> 禁用 bool get isUsable { if (_consecutiveFails >= 5) { if (lastFailAt != null && DateTime.now().difference(lastFailAt!).inHours >= 1) return true; return false; } if (failCount >= 5 && successCount == 0) return false; return true; } // 健康分 0~100(动态衰减,不依赖 save 时机) double get score { final latest = lastSuccessAt ?? lastFailAt; final decay = latest != null && DateTime.now().difference(latest).inDays >= 7 ? 0.5 : 1.0; final total = successCount + failCount + timeoutCount; if (total == 0) return 50.0 * decay; double s = 60.0 * (successCount / total.toDouble()); if (successCount > 0 && totalLatencyMs > 0) { final avg = totalLatencyMs / successCount; if (avg < 500) s += 25.0; else if (avg < 2000) s += 25.0 * (1.0 - (avg - 500) / 1500); } if (lastSuccessAt != null) { final h = DateTime.now().difference(lastSuccessAt!).inHours; if (h < 1) s += 15.0; else if (h < 24) s += 15.0 * (1.0 - h / 24); } return (s * decay).clamp(0.0, 100.0); } void recordSuccess(int latencyMs) { successCount++; totalLatencyMs += latencyMs; lastSuccessAt = DateTime.now(); _consecutiveFails = 0; } void recordFailure() { failCount++; lastFailAt = DateTime.now(); _consecutiveFails++; } void recordTimeout() { timeoutCount++; lastFailAt = DateTime.now(); _consecutiveFails++; } }""") para('') heading('2.3 健康分持久化(防抖 + 延迟写盘)', level=3) para('SourceHealthStore 关键设计:') para(' • recordSuccess/recordFailure/recordTimeout 只改内存 Map,不触发 IO') para(' • save() 启动 2 秒 Timer,同一轮搜索的多次调用被合并(防抖)') para(' • flush() 立即写盘(仅在 app 退出时调用,取消 Timer 后直接写)') para(' • 衰减计算在 SourceHealth.score getter 中动态完成,save 时不修改原始数据') heading('2.4 分阶段搜索调度器(滑动窗口 + 取消机制)', level=3) para('核心改进:') para(' • 滑动窗口 _pool 替代 Future.wait — 最大并发 6,谁先完成谁先返回,快源不阻塞') para(' • _searchToken 版本号 — 用户连续搜索 A→B→C 时旧结果自动丢弃') para(' • 慢源结果走同一个 Stream yield(不引入 EventBus),保持数据流单一源') para(' • 整轮搜索结束后统一调用一次 health.save()') code_block("""class SearchScheduler { static const fastTimeout = Duration(seconds: 5); static const slowTimeout = Duration(seconds: 8); static const maxConcurrent = 6; int _searchToken = 0; // 取消旧搜索用 Stream search(String keyword, List allSources) async* { final token = ++_searchToken; final usable = allSources.where((s) => _health.isUsable(s.id)).toList() ..sort((a, b) => _health.score(b.id).compareTo(_health.score(a.id))); if (usable.isEmpty) { yield SearchPhase(name: 'error', message: '没有可用的搜索源'); return; } // 分三档 final fast = usable.where((s) => _health.score(s.id) >= 70).toList(); final normal = usable.where((s) { final sc = _health.score(s.id); return sc >= 40 && sc < 70; }).toList(); final slow = usable.where((s) => _health.score(s.id) < 40).toList(); final results = [], doneIds = {}, seen = {}; // 阶段 1:快速源(5s 超时,滑动窗口) await for (final r in _pool(fast, keyword, timeout: fastTimeout, token: token)) { doneIds.add(r.sourceId); _merge(results, r.items, seen); } yield SearchPhase(name: 'partial', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length); // 阶段 2:普通源(8s 超时) await for (final r in _pool(normal, keyword, timeout: slowTimeout, token: token)) { doneIds.add(r.sourceId); _merge(results, r.items, seen); } yield SearchPhase(name: 'partial', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length); // 阶段 3:慢源后台继续,走同一个 Stream if (slow.isNotEmpty) { yield SearchPhase(name: 'pending_slow', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length, message: '${slow.length} 个慢源正在后台搜索...'); await for (final r in _pool(slow, keyword, timeout: slowTimeout, token: token)) { if (token != _searchToken) return; // 用户发起新搜索,丢弃旧结果 doneIds.add(r.sourceId); _merge(results, r.items, seen); yield SearchPhase(name: 'slow_append', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length); } } yield SearchPhase(name: 'complete', items: List.of(results), doneCount: doneIds.length, totalCount: usable.length); unawaited(_health.save()); // 整轮搜索结束统一写盘 } // 滑动窗口并发(替代 Future.wait) Stream<_SourceResult> _pool(List sources, String keyword, {required Duration timeout, required int token}) async* { final pending = sources.toList(); final running = >{}; while (pending.isNotEmpty || running.isNotEmpty) { while (running.length < maxConcurrent && pending.isNotEmpty) running.add(_searchOne(pending.removeAt(0), keyword, timeout, token: token)); if (running.isEmpty) break; final done = await Future.any(running.map((f) => f.then((v) => (f, v)))); running.remove(done.$1); if (done.$2 != null) yield done.$2!; } } Future<_SourceResult?> _searchOne(BookSource source, String keyword, Duration timeout, {required int token}) async { if (token != _searchToken) return null; final sw = Stopwatch()..start(); try { final items = await _engine.search(source, keyword).timeout(timeout); if (token != _searchToken) return null; sw.stop(); _health.recordSuccess(source.id, sw.elapsedMilliseconds); return _SourceResult(source.id, items); } on TimeoutException { _health.recordTimeout(source.id); return null; } catch (_) { _health.recordFailure(source.id); return null; } } } class _SourceResult { final String sourceId; final List items; _SourceResult(this.sourceId, this.items); } class SearchPhase { final String name; // searching | partial | pending_slow | slow_append | complete | error final List items; final int doneCount, totalCount; final String? message; SearchPhase({required this.name, this.items = const [], this.doneCount = 0, this.totalCount = 0, this.message}); }""") para('') heading('2.5 UI 增量渲染时序', level=3) para('t=0s → 显示"正在搜索 N 个源..."') para('t=0~5s → 快速源逐个完成 → 滑动窗口下结果立刻追加到列表') para('t=5~8s → 普通源逐个完成 → 继续追加') para('t=8s → "慢源正在后台搜索..." → 用户已看到前两档结果') para('t=8~15s → 慢源逐个完成 → 静默追加到列表末尾') para('t=15s → "搜索完成"') heading('2.6 取消机制', level=3) para('用户连续搜 A→B→C 时:_searchToken 递增版本号。_searchOne 开头检查 token != _searchToken 则 return null;阶段 3 yield 前检查则 return 丢弃旧搜索的后台任务。旧搜索的未完成请求仍会走完网络超时,但结果不会污染新搜索的 UI。') para('') # ── Part 3 ── heading('三、两改进的协作关系', level=2) para('用户搜索一本书 → SearchScheduler(按健康分排序、滑动窗口、增量推 UI)→ 用户点击结果 → fetchBookInfo → fetchToc(TocPaginationResolver 拉全章节,支持分页)→ 用户阅读某章 → fetchChapterContent(需要 JS 引擎,本文档范围外)。') para('改进 1 保证"搜到书后章节不断尾",改进 2 保证"搜索本身不卡"。') para('') # ── Part 4 ── heading('四、实施顺序', level=2) para('第 1 步:SourceHealth + SourceHealthStore(纯数据层,不影响现有流程,可与图片解析的 JS 引擎并行推进,先跑几天积累健康数据)') para('第 2 步:SearchScheduler 滑动窗口 + 短超时 + Stream(替换现有 search() 并发逻辑,UI 适配 Stream)') para('第 3 步:TocPaginationResolver(简单 JS/CSS/模板分页立即生效约 60% 源;复杂 JS 分页需要等 JS 引擎,代码结构无需改动)') para('第 4 步:联调 + 回归') para('') para('与图片解析的关系:第 1~2 步与 LegadoRuleExecutor V2 + QuickJS 可并行推进、互不冲突。') para('') # ── Part 5 ── heading('五、风险控制', level=2) make_table( ['风险', '应对'], [ ['TOC 分页死循环', 'maxPages=20 + seenPageUrls 去重 + added==0 终止'], ['复杂 JS 分页规则暂时跑不通', '简单规则(~60%)立即支持;复杂规则骨架预留,换 JS 引擎即可'], ['健康分误杀快源(网站临时维护)', '连续失败 ≥5 才熔断 + 每 1h 自动释放探测机会 + 手动可重置'], ['搜索时 SharedPreferences 频繁 I/O', '内存改、整轮结束统一写盘 + 2s 防抖'], ['滑动窗口仍有慢源卡槽位', '超时兜底(5s/8s),超时后释放槽位'], ['连续快速搜索旧结果污染 UI', '_searchToken 版本号,旧结果自动丢弃'], ['新源全部 50 分', '第一轮搜索后就有真实分,冷启动无影响'], ['Future.wait 批阻塞', '已替换为滑动窗口 _pool,先完成先返回'], ] ) para('') para('') # ── Part 6 ── heading('六、Review 整合清单', level=2) para('以下为三方 review 指出的问题和本方案的对应处理:') make_table( ['Reviewer', '指出的问题', 'v2 的处理'], [ ['Claude', 'offset 计算 initial * page 有误', '改为 initial + initial * page'], ['Claude', '阶段 3 后台任务无取消机制', '加入 _searchToken 版本号取消'], ['Claude', '衰减在 save() 里做不可靠', '衰减改为 score getter 动态计算'], ['Claude', '字段名每次访问都遍历', '改为 fromJson 时一次性解析到字段'], ['Claude', '第 3 步依赖 Legado 重构前置', '明确标注依赖,与 JS 引擎可并行'], ['ChatGPT', '图片解析才是 P0,优先级需修正', '§0 前置声明 + §4 与图片解析并行'], ['ChatGPT', ' 规则含 java.xxx 无法执行', '§1.7 明确兼容边界,骨架预留'], ['Gemini', 'Future.wait 批阻塞快源', '改为滑动窗口 _pool'], ['Gemini', '熔断无自动恢复', 'isUsable 加 1h 自动释放探测'], ['Gemini', 'save() 高频 I/O', '防抖 2s + 整轮结束统一写'], ['Gemini', '慢源用 EventBus 引入第二数据源', '慢源走同一个 Stream yield'], ['Gemini', '\'${page + 1}\' 字符串字面量错误', '改为 (page + 1).toString()'], ] ) # ---- Save ---- doc.save(DOCX_PATH) print('Done! Document updated successfully.') print(f'Total paragraphs now: {len(doc.paragraphs)}') print(f'Total tables now: {len(doc.tables)}')