from __future__ import annotations import ipaddress import json import os import re import socket import time from collections import OrderedDict from dataclasses import dataclass, field from html import unescape from typing import Any, Iterable from urllib.parse import quote, urljoin, urlparse import requests from bs4 import BeautifulSoup class SearchEngineError(RuntimeError): """在线搜索规则引擎的基础异常,不包含 Cookie/Token 等敏感信息。""" @dataclass(frozen=True) class SearchCandidate: source_name: str source_url: str title: str url: str resource_type: str score: float author: str | None = None cover: str | None = None matched_rule: str | None = None @dataclass(frozen=True) class ParsedChapter: name: str url: str | None = None content: str | None = None images: tuple[str, ...] = () @dataclass(frozen=True) class ParsedNovel: title: str chapters: tuple[ParsedChapter, ...] cover: str | None = None def as_dict(self) -> dict[str, Any]: return { "title": self.title, "chapters": [ {"name": chapter.name, "content": chapter.content or ""} for chapter in self.chapters ], "cover": self.cover or "", } @dataclass(frozen=True) class ParsedComic: title: str chapters: tuple[ParsedChapter, ...] cover: str | None = None def as_dict(self) -> dict[str, Any]: return { "title": self.title, "chapters": [ {"name": chapter.name, "images": list(chapter.images)} for chapter in self.chapters ], "cover": self.cover or "", } @dataclass class RuleConfig: rule_id: str name: str base_url: str resource_type: str = "all" search_url: str = "" method: str = "GET" enabled: bool = True headers: dict[str, str] = field(default_factory=dict) cookies: dict[str, str] = field(default_factory=dict) rule_search: dict[str, Any] = field(default_factory=dict) success_count: int = 0 failure_count: int = 0 average_latency_ms: float = 1000.0 ad_density: float = 0.0 @dataclass(frozen=True) class DebugStep: rule_id: str stage: str status: str detail: str data_length: int = 0 image_count: int = 0 class RuleManager: """规则配置管理器,支持动态添加、禁用和站点健康评分。""" def __init__(self, rules: Iterable[RuleConfig] | None = None) -> None: self._rules: OrderedDict[str, RuleConfig] = OrderedDict() for rule in rules or (): self.add_rule(rule) def add_rule(self, rule: RuleConfig) -> None: self._rules[rule.rule_id] = rule def disable_rule(self, rule_id: str) -> None: if rule_id in self._rules: self._rules[rule_id].enabled = False def enable_rule(self, rule_id: str) -> None: if rule_id in self._rules: self._rules[rule_id].enabled = True def get_rules(self, resource_type: str = "all") -> list[RuleConfig]: allowed = {resource_type, "all"} return [ rule for rule in self._rules.values() if rule.enabled and (resource_type == "all" or rule.resource_type in allowed) ] def update_health(self, rule_id: str, *, ok: bool, latency_ms: float) -> None: rule = self._rules.get(rule_id) if rule is None: return if ok: rule.success_count += 1 else: rule.failure_count += 1 rule.average_latency_ms = (rule.average_latency_ms * 0.8) + (latency_ms * 0.2) def trust_score(self, rule_id: str) -> float: rule = self._rules.get(rule_id) if rule is None: return 0.3 total = rule.success_count + rule.failure_count success_rate = rule.success_count / total if total else 0.65 latency_score = max(0.0, min(1.0, 1.0 - (rule.average_latency_ms / 10_000.0))) ad_penalty = max(0.0, min(0.4, rule.ad_density * 0.4)) return max(0.0, min(1.0, success_rate * 0.6 + latency_score * 0.4 - ad_penalty)) class LruTextCache: """缓存页面文本,降低重复解析同一站点结构的网络开销。""" def __init__(self, max_size: int = 128, ttl_seconds: int = 600) -> None: self.max_size = max_size self.ttl_seconds = ttl_seconds self._rows: OrderedDict[str, tuple[float, str]] = OrderedDict() def get(self, key: str) -> str | None: row = self._rows.get(key) if row is None: return None created, value = row if time.time() - created > self.ttl_seconds: self._rows.pop(key, None) return None self._rows.move_to_end(key) return value def set(self, key: str, value: str) -> None: self._rows[key] = (time.time(), value) self._rows.move_to_end(key) while len(self._rows) > self.max_size: self._rows.popitem(last=False) class OnlineSearchEngine: def __init__( self, rule_manager: RuleManager | None = None, *, timeout: float = 10.0, session: requests.Session | None = None, cache: LruTextCache | None = None, ) -> None: self.rule_manager = rule_manager or RuleManager() self.timeout = timeout self.session = session or requests.Session() self.cache = cache or LruTextCache() self.debug_steps: list[DebugStep] = [] def search_resource( self, query: str, resource_type: str = "all", *, limit: int = 50, ) -> list[SearchCandidate]: query = normalize_text(query) if not query: return [] rows: list[SearchCandidate] = [] for rule in self.rule_manager.get_rules(resource_type): started = time.perf_counter() try: rows.extend(self._search_rule(rule, query, limit=limit)) latency_ms = (time.perf_counter() - started) * 1000 self.rule_manager.update_health(rule.rule_id, ok=True, latency_ms=latency_ms) except Exception as exc: latency_ms = (time.perf_counter() - started) * 1000 self.rule_manager.update_health(rule.rule_id, ok=False, latency_ms=latency_ms) self._debug(rule.rule_id, "search", "failed", _safe_error(exc)) return dedupe_candidates(rows)[:limit] def search_legado_source( self, source: dict[str, Any], keyword: str, *, mode: str, limit: int, ) -> list[SearchCandidate]: rule = legado_source_to_rule(source, mode) if not rule.base_url or not rule.search_url: return [] return self._search_rule(rule, normalize_text(keyword), limit=limit) def parse_novel(self, url: str) -> ParsedNovel: html = self._get_text(url, url, rule_id="generic_novel") soup = BeautifulSoup(html, "lxml") title = extract_page_title(soup) cover = maybe_url(url, first_meta_content(soup, "og:image")) or first_image_url(soup, url) chapters = extract_chapter_links(soup, url, prefer_images=False) current_content = extract_main_text(soup) if not chapters: chapters = [ParsedChapter(name=title or "正文", url=url, content=current_content)] else: rows = [ParsedChapter(name=title or "当前章节", url=url, content=current_content)] if current_content else [] rows.extend(chapter for chapter in chapters if chapter.url != url) chapters = rows self._debug("generic_novel", "parse", "ok", title, len(html), 0) return ParsedNovel(title=title or "未命名小说", chapters=tuple(chapters), cover=cover) def parse_comic(self, url: str) -> ParsedComic: html = self._get_text(url, url, rule_id="generic_comic") soup = BeautifulSoup(html, "lxml") title = extract_page_title(soup) cover = maybe_url(url, first_meta_content(soup, "og:image")) or first_image_url(soup, url) images = extract_comic_images(soup, url) chapters = extract_chapter_links(soup, url, prefer_images=True) if images: chapters = [ParsedChapter(name=title or "第1话", url=url, images=tuple(images))] elif chapters: chapters = [ParsedChapter(name=chapter.name, url=chapter.url) for chapter in chapters] else: chapters = [] self._debug("generic_comic", "parse", "ok", title, len(html), len(images)) return ParsedComic(title=title or "未命名漫画", chapters=tuple(chapters), cover=cover) def _search_rule(self, rule: RuleConfig, keyword: str, *, limit: int) -> list[SearchCandidate]: final_url, data = build_search_request(rule.search_url, keyword, rule.base_url) assert_public_url(final_url) html = self._request_text(rule, final_url, data=data) self._debug(rule.rule_id, "fetch", "ok", final_url, len(html), 0) rows = self._parse_search_payload(rule, html, final_url, keyword, limit) if not rows: rows = fallback_search_cards( html, final_url, rule, keyword, limit, trust=self.rule_manager.trust_score(rule.rule_id), ) self._debug(rule.rule_id, "search", "ok", f"{len(rows)} candidates", len(html), 0) return rows def _parse_search_payload( self, rule: RuleConfig, body: str, base_url: str, keyword: str, limit: int, ) -> list[SearchCandidate]: rule_search = rule.rule_search or {} list_rule = first_string(rule_search, "bookList", "comicList", "list", "items", "data", "results") if not list_rule: return [] fragments = select_list(body, list_rule, base_url) rows: list[SearchCandidate] = [] trust = self.rule_manager.trust_score(rule.rule_id) for fragment in fragments: title = select_one(fragment, first_string(rule_search, "name", "bookName", "comicName", "title"), base_url) detail = select_one(fragment, first_string(rule_search, "bookUrl", "comicUrl", "detailUrl", "url", "href"), base_url) if not title or not detail: continue score = title_match_score(title, keyword) if score < 0.38: continue rows.append( SearchCandidate( source_name=rule.name, source_url=rule.base_url, title=clean_text(title), author=empty_to_none( select_one( fragment, first_string(rule_search, "author", "bookAuthor", "comicAuthor", "writer"), base_url, ) ), cover=maybe_url( base_url, select_one( fragment, first_string(rule_search, "coverUrl", "cover", "bookCover", "comicCover", "img", "image"), base_url, ), ), url=resolve_resource_url(base_url, detail), resource_type=rule.resource_type, score=(score * 0.75) + (trust * 0.25), matched_rule=rule.rule_id, ) ) if len(rows) >= limit: break return sorted(rows, key=lambda item: item.score, reverse=True) def _request_text(self, rule: RuleConfig, url: str, *, data: dict[str, str] | None = None) -> str: cache_key = f"{rule.method}:{url}:{json.dumps(data or {}, sort_keys=True, ensure_ascii=False)}" cached = self.cache.get(cache_key) if cached is not None: return cached headers = build_headers(rule.base_url, rule.headers) cookies = rule.cookies or {} method = rule.method.upper() response = self.session.request( method, url, data=data if method == "POST" else None, headers=headers, cookies=cookies, timeout=self.timeout, allow_redirects=True, ) if response.status_code == 404: raise SearchEngineError("源站搜索接口不存在,可能源已失效") if response.status_code in {403, 444}: raise SearchEngineError("源站拒绝访问,可能需要请求头、Cookie 或反爬处理") response.raise_for_status() text = decode_response_text(response) self.cache.set(cache_key, text) return text def _get_text(self, url: str, referer: str, *, rule_id: str) -> str: assert_public_url(url) rule = RuleConfig(rule_id=rule_id, name=rule_id, base_url=referer, search_url=url) return self._request_text(rule, url) def _debug( self, rule_id: str, stage: str, status: str, detail: str, data_length: int = 0, image_count: int = 0, ) -> None: step = DebugStep(rule_id, stage, status, detail[:240], data_length, image_count) self.debug_steps.append(step) if os.getenv("SAKURA_ONLINE_SEARCH_DEBUG") == "1": print( f"[online-search] rule={step.rule_id} stage={step.stage} " f"status={step.status} len={step.data_length} images={step.image_count} detail={step.detail}" ) DEFAULT_RULE_MANAGER = RuleManager() DEFAULT_ENGINE = OnlineSearchEngine(DEFAULT_RULE_MANAGER) def search_resource(query: str, resource_type: str = "all") -> list[dict[str, Any]]: return [ { "title": item.title, "url": item.url, "source_name": item.source_name, "source_url": item.source_url, "resource_type": item.resource_type, "score": item.score, "author": item.author, "cover": item.cover, } for item in DEFAULT_ENGINE.search_resource(query, resource_type) ] def parse_novel(url: str) -> dict[str, Any]: return DEFAULT_ENGINE.parse_novel(url).as_dict() def parse_comic(url: str) -> dict[str, Any]: return DEFAULT_ENGINE.parse_comic(url).as_dict() def legado_source_to_rule(source: dict[str, Any], mode: str) -> RuleConfig: rule_search = source.get("ruleSearch") if isinstance(source.get("ruleSearch"), dict) else {} base_url = string_value(source.get("bookSourceUrl") or source.get("sourceUrl") or source.get("url")) or "" search_url = string_value(source.get("searchUrl") or rule_search.get("url") or rule_search.get("searchUrl")) or "" headers = source.get("headers") or source.get("header") or {} cookies = source.get("cookies") or {} if isinstance(source.get("cookie"), str): cookies = parse_cookie_header(source["cookie"]) return RuleConfig( rule_id=string_value(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or base_url, name=string_value(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or "未命名源", base_url=base_url, resource_type=mode, search_url=search_url, method=(string_value(source.get("searchMethod") or source.get("method")) or "GET").upper(), headers={str(key): str(value) for key, value in headers.items()} if isinstance(headers, dict) else {}, cookies={str(key): str(value) for key, value in cookies.items()} if isinstance(cookies, dict) else {}, rule_search=rule_search, ) def build_search_request(template: str, keyword: str, base_url: str) -> tuple[str, dict[str, str] | None]: raw = build_search_url(template, keyword) if "," in raw and raw.lstrip().upper().startswith("POST"): _, payload = raw.split(",", 1) url, _, body = payload.partition("@") return resolve_resource_url(base_url, url), parse_form_body(body) if raw.lstrip().upper().startswith("POST "): payload = raw.split(None, 1)[1] url, _, body = payload.partition("@") return resolve_resource_url(base_url, url), parse_form_body(body) return resolve_resource_url(base_url, raw), None def build_search_url(template: str, keyword: str) -> str: encoded = quote(keyword, safe="") value = template.strip() for token in ("{{key}}", "{{keyword}}", "{key}", "{keyword}", "", "", "searchKey"): value = value.replace(token, encoded) value = value.replace("%s", encoded) return value.replace("{{page}}", "1").replace("{page}", "1").replace("", "1") def resolve_resource_url(base_url: str, raw_url: str) -> str: url = (raw_url or "").strip() if not url: raise SearchEngineError("Empty resource URL") if url.startswith(("http://", "https://")): return url if url.startswith("//"): scheme = urlparse(base_url).scheme or "https" return f"{scheme}:{url}" return urljoin(base_url, url) def assert_public_url(url: str) -> None: parsed = urlparse(url) if parsed.scheme not in {"http", "https"} or not parsed.hostname: raise SearchEngineError("Only http/https source URLs are allowed") infos = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM) for item in infos: ip = ipaddress.ip_address(item[4][0]) if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved: raise SearchEngineError("Private network source URLs are not allowed") def build_headers(referer: str, extra: dict[str, str] | None = None) -> dict[str, str]: headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36" ), "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", "Referer": referer, } for key, value in (extra or {}).items(): if key.lower() in {"authorization"}: continue headers[key] = value return headers def decode_response_text(response: requests.Response) -> str: content_type = response.headers.get("content-type", "") candidates = [ response.encoding, charset_from_content_type(content_type), "utf-8", "gb18030", "gbk", "gb2312", ] raw = response.content[:4_000_000] for candidate in candidates: if not candidate: continue try: return raw.decode(candidate, errors="replace") except LookupError: continue return raw.decode("utf-8", errors="replace") def select_list(body: str, rule: str | None, base_url: str) -> list[str]: if not rule: return [] for part in split_rule(rule, "||"): part = strip_replace(part) if not part: continue if part.lower().startswith("regex:"): pattern = part.split(":", 1)[1] matches = re.findall(pattern, body, re.I | re.S) rows = [clean_text(item[0] if isinstance(item, tuple) else item) for item in matches] if rows: return [row for row in rows if row] if part.startswith("$.") or part.startswith("$["): decoded = decode_json_like(body) rows = json_path(decoded, part) if decoded is not None else [] if rows: return [json.dumps(row, ensure_ascii=False) if isinstance(row, (dict, list)) else str(row) for row in rows] rows = soup_rule_list(body, part, base_url) if rows: return rows return [] def select_one(fragment: str, rule: str | None, base_url: str) -> str: rows = select_list(fragment, rule, base_url) return rows[0] if rows else "" def soup_rule_list(body: str, rule: str, base_url: str) -> list[str]: tokens = [token.strip() for token in rule.split("@") if token.strip()] if not tokens: return [] attr = tokens[-1] if tokens[-1].lower() in {"text", "html", "href", "src", "data-src", "data-original", "alt", "title"} else "" selector_tokens = tokens[:-1] if attr else tokens selector = " ".join(rule_token_to_css(token) for token in selector_tokens if rule_token_to_css(token)) if not selector: return [] soup = BeautifulSoup(body, "lxml") rows: list[str] = [] for element in soup.select(selector): if attr == "text": rows.append(clean_text(element.get_text(" "))) elif attr == "html": rows.append(str(element)) elif attr: value = element.get(attr) if value: rows.append(resolve_resource_url(base_url, value) if attr in {"href", "src", "data-src", "data-original"} else clean_text(value)) else: rows.append(str(element)) return [row for row in rows if row] def rule_token_to_css(token: str) -> str: if token.startswith("class."): return "." + ".".join(part for part in token[6:].split(".") if part) if token.startswith("id."): return "#" + token[3:] if token.startswith("tag."): return token[4:] if token.startswith(".") or token.startswith("#") or token in {"a", "li", "div", "img", "ul", "ol"}: return token return token if re.match(r"^[a-zA-Z0-9_.# >:-]+$", token) else "" def fallback_search_cards( body: str, base_url: str, rule: RuleConfig, keyword: str, limit: int, *, trust: float, ) -> list[SearchCandidate]: decoded = decode_json_like(body) rows: list[SearchCandidate] = [] if decoded is not None: rows = fallback_json_candidates(decoded, base_url, rule, keyword, trust) if not rows: rows = fallback_html_candidates(body, base_url, rule, keyword, trust) return sorted(rows, key=lambda item: item.score, reverse=True)[:limit] def fallback_json_candidates( decoded: Any, base_url: str, rule: RuleConfig, keyword: str, trust: float, ) -> list[SearchCandidate]: rows: list[SearchCandidate] = [] for node in walk_dicts(decoded): title = first_value(node, "title", "name", "bookName", "book_name", "novelName", "comicName", "comic_name") if not title: continue score = title_match_score(title, keyword) if score < 0.38: continue detail = first_value(node, "url", "href", "link", "bookUrl", "book_url", "comicUrl", "comic_url", "detailUrl", "detail_url", "path") if not detail: detail = infer_detail_path( node, rule.resource_type, first_value(node, "id", "book_id", "bookId", "comic_id", "comicId", "bid", "cid"), ) if not detail: continue rows.append( SearchCandidate( source_name=rule.name, source_url=rule.base_url, title=title, author=empty_to_none(first_value(node, "author", "bookAuthor", "writer", "nickname", "userName", "user_name")), cover=maybe_url(base_url, first_value(node, "cover", "coverUrl", "cover_url", "img", "image", "pic", "pic_url")), url=resolve_resource_url(base_url, detail), resource_type=rule.resource_type, score=(score * 0.75) + (trust * 0.25), matched_rule=f"{rule.rule_id}:json-fallback", ) ) return rows def fallback_html_candidates(body: str, base_url: str, rule: RuleConfig, keyword: str, trust: float) -> list[SearchCandidate]: soup = BeautifulSoup(body, "lxml") rows: list[SearchCandidate] = [] for anchor in soup.select("a[href]"): href = anchor.get("href") or "" text = clean_text(anchor.get("title") or anchor.get("alt") or anchor.get_text(" ")) image = first_image_url(anchor, base_url) if not text and image: text = clean_text(anchor.find("img").get("alt", "") if anchor.find("img") else "") score = title_match_score(text, keyword) if not href or score < 0.38: continue if is_noise_link(text, href): continue rows.append( SearchCandidate( source_name=rule.name, source_url=rule.base_url, title=text, author=None, cover=image, url=resolve_resource_url(base_url, href), resource_type=rule.resource_type, score=(score * 0.75) + (trust * 0.25), matched_rule=f"{rule.rule_id}:html-fallback", ) ) return rows def extract_main_text(soup: BeautifulSoup) -> str: for selector in ("#content", ".content", ".chapter-content", ".read-content", "article", "main"): element = soup.select_one(selector) if element: text = normalize_paragraphs(element.get_text("\n")) if len(text) > 80: return text best = "" for element in soup.find_all(["article", "main", "div", "section"]): text = normalize_paragraphs(element.get_text("\n")) if text_density(text) > text_density(best): best = text return best def extract_chapter_links(soup: BeautifulSoup, base_url: str, *, prefer_images: bool) -> list[ParsedChapter]: chapters: list[ParsedChapter] = [] patterns = ( r"(第\s*\d+\s*[章节话回卷]|chapter\s*\d+|下一章|上一章|目录)" if not prefer_images else r"(第\s*\d+\s*[话回卷]|chapter\s*\d+|episode\s*\d+|下一话|上一话|目录)" ) for anchor in soup.select("a[href]"): name = clean_text(anchor.get_text(" ") or anchor.get("title", "")) href = anchor.get("href") or "" if not name or not href: continue if re.search(patterns, name, re.I) or re.search(r"(chapter|read|episode|manga|comic)", href, re.I): chapters.append(ParsedChapter(name=name[:120], url=resolve_resource_url(base_url, href))) return dedupe_chapters(chapters) def extract_comic_images(soup: BeautifulSoup, base_url: str) -> list[str]: rows: list[str] = [] for image in soup.select("img"): value = "" for attr in ("data-src", "data-original", "data-lazy-src", "data-url", "src", "data-srcset", "srcset"): raw = image.get(attr) if raw: value = split_srcset(raw)[0] if attr.endswith("srcset") else raw break if not value: continue url = resolve_resource_url(base_url, value) if is_valid_comic_image(url, image): rows.append(url) for match in re.finditer(r"https?://[^\"']+\.(?:jpg|jpeg|png|webp|avif)(?:\?[^\"']*)?", str(soup), re.I): url = match.group(0) if is_valid_comic_image(url, None): rows.append(url) for raw in extract_script_image_candidates(str(soup)): try: url = resolve_resource_url(base_url, raw) except SearchEngineError: continue if is_valid_comic_image(url, None): rows.append(url) return dedupe_strings(rows) def split_srcset(value: str) -> list[str]: urls: list[str] = [] for part in value.split(","): url = part.strip().split()[0] if part.strip() else "" if url: urls.append(url) return urls def extract_script_image_candidates(html: str) -> list[str]: text = html.replace("\\/", "/") rows: list[str] = [] for match in re.finditer(r'''["']((?:(?:https?:)?//|/)[^"']+\.(?:jpg|jpeg|png|webp|avif)(?:\?[^"']*)?)["']''', text, re.I): rows.append(match.group(1)) for assignment in re.finditer( r"(?:var|let|const)?\s*(?:chapterImages|imageList|imgList|pageImgs|pics|photosr|newImgs|img_data)\s*=\s*(\[[\s\S]*?\])\s*;?", text, re.I, ): try: decoded = json.loads(assignment.group(1)) except Exception: continue collect_image_strings(decoded, rows) return dedupe_strings(rows) def collect_image_strings(node: Any, rows: list[str]) -> None: if isinstance(node, str): text = node.strip().replace("\\/", "/") if re.search(r"\.(jpg|jpeg|png|webp|avif)(?:[?#].*)?$", text, re.I): rows.append(text) elif isinstance(node, dict): for value in node.values(): collect_image_strings(value, rows) elif isinstance(node, list): for value in node: collect_image_strings(value, rows) def is_valid_comic_image(url: str, image: Any | None) -> bool: lower = url.lower() if not re.search(r"\.(jpg|jpeg|png|webp|avif)(?:[?#].*)?$", lower): return False if any(marker in lower for marker in ("logo", "avatar", "icon", "blank", "placeholder", "loading")): return False if image is not None: width = int_or_zero(image.get("width")) height = int_or_zero(image.get("height")) if width and height and (width < 100 or height < 100): return False return True def infer_detail_path(node: dict[str, Any], resource_type: str, raw_id: str) -> str: value = clean_text(raw_id) if not value or not re.match(r"^[\w-]+$", value): return "" if resource_type == "manga": return first_value(node, "path") or f"/comic/{value}/" return first_value(node, "path") or f"/book/{value}/" def title_match_score(title: str, keyword: str) -> float: title_norm = strip_noise_words(normalize_text(title)).lower() keyword_norm = strip_noise_words(normalize_text(keyword)).lower() if not title_norm or not keyword_norm: return 0.0 if title_norm == keyword_norm: return 1.0 contains = 0.78 if keyword_norm in title_norm else 0.0 tokens = [token for token in re.split(r"\s+", keyword_norm) if token] coverage = sum(1 for token in tokens if token in title_norm) / len(tokens) if tokens else 0.0 distance = levenshtein(title_norm[:80], keyword_norm[:80]) max_len = max(len(title_norm[:80]), len(keyword_norm[:80]), 1) fuzzy = 1.0 - min(distance / max_len, 1.0) return max(contains, coverage * 0.82, fuzzy * 0.72) def levenshtein(left: str, right: str) -> int: if left == right: return 0 if len(left) < len(right): left, right = right, left previous = list(range(len(right) + 1)) for i, left_char in enumerate(left, 1): current = [i] for j, right_char in enumerate(right, 1): insert = current[j - 1] + 1 delete = previous[j] + 1 replace = previous[j - 1] + (left_char != right_char) current.append(min(insert, delete, replace)) previous = current return previous[-1] def strip_noise_words(value: str) -> str: return re.sub(r"(最新章节|免费阅读|全文阅读|漫画全集|在线阅读|无弹窗|笔趣阁|手机版)", " ", value) def decode_json_like(body: str) -> Any: text = body.lstrip("\ufeff \t\r\n") match = re.match(r"^[a-zA-Z_$][\w$]*\s*\((.*)\)\s*;?\s*$", text, re.S) if match: text = match.group(1) try: return json.loads(text) except Exception: return None def json_path(decoded: Any, rule: str) -> list[Any]: if decoded is None: return [] path = re.sub(r"^\$\.?", "", rule.strip()) if path in {"", "*", "[*]"}: return decoded if isinstance(decoded, list) else [decoded] current = [decoded] for segment in [part for part in path.split(".") if part]: next_values: list[Any] = [] star = segment.endswith("[*]") key = segment[:-3] if star else segment for value in current: child = value.get(key) if isinstance(value, dict) and key else value if star and isinstance(child, list): next_values.extend(child) elif child is not None: next_values.append(child) current = next_values return current def extract_page_title(soup: BeautifulSoup) -> str: for selector in ("h1", ".book-title", ".comic-title", ".title"): element = soup.select_one(selector) if element: text = clean_text(element.get_text(" ")) if text: return text return clean_text(first_meta_content(soup, "og:title") or (soup.title.get_text(" ") if soup.title else "")) def first_meta_content(soup: BeautifulSoup, name: str) -> str | None: for selector in (f'meta[property="{name}"]', f'meta[name="{name}"]'): element = soup.select_one(selector) if element and element.get("content"): return clean_text(element["content"]) return None def first_image_url(node: Any, base_url: str) -> str | None: image = node.find("img") if hasattr(node, "find") else None if not image: return None for attr in ("data-src", "data-original", "data-lazy-src", "src"): value = image.get(attr) if value: return resolve_resource_url(base_url, value) return None def text_density(text: str) -> float: normalized = normalize_paragraphs(text) if not normalized: return 0.0 punctuation = len(re.findall(r"[,。!?,.!?]", normalized)) return len(normalized) + punctuation * 8 def fallback_decode_raw(raw: bytes, content_type: str = "") -> str: for candidate in (charset_from_content_type(content_type), "utf-8", "gb18030", "gbk", "gb2312"): if not candidate: continue try: return raw.decode(candidate, errors="replace") except LookupError: continue return raw.decode("utf-8", errors="replace") def parse_cookie_header(value: str) -> dict[str, str]: cookies: dict[str, str] = {} for part in value.split(";"): key, _, raw = part.partition("=") key = key.strip() if key: cookies[key] = raw.strip() return cookies def parse_form_body(value: str) -> dict[str, str] | None: if not value: return None rows: dict[str, str] = {} for part in value.split("&"): key, _, raw = part.partition("=") if key: rows[key] = raw return rows or None def dedupe_candidates(rows: list[SearchCandidate]) -> list[SearchCandidate]: seen: set[str] = set() output: list[SearchCandidate] = [] for row in sorted(rows, key=lambda item: item.score, reverse=True): key = row.url if key in seen: continue seen.add(key) output.append(row) return output def dedupe_chapters(rows: list[ParsedChapter]) -> list[ParsedChapter]: seen: set[str] = set() output: list[ParsedChapter] = [] for row in rows: key = row.url or row.name if key in seen: continue seen.add(key) output.append(row) return output def dedupe_strings(rows: list[str]) -> list[str]: seen: set[str] = set() output: list[str] = [] for row in rows: if row in seen: continue seen.add(row) output.append(row) return output def walk_dicts(node: Any): if isinstance(node, dict): yield node for value in node.values(): yield from walk_dicts(value) elif isinstance(node, list): for item in node: yield from walk_dicts(item) def first_value(node: dict[str, Any], *keys: str) -> str: for key in keys: value = node.get(key) if value is not None: text = clean_text(str(value)) if text and text.lower() != "null": return text return "" def first_string(mapping: dict[str, Any], *keys: str) -> str | None: for key in keys: value = string_value(mapping.get(key)) if value: return value return None def source_type(source: dict[str, Any]) -> int: value = source.get("bookSourceType", source.get("sourceType", source.get("type", 0))) try: numeric = int(value) except Exception: numeric = 0 text = json.dumps(source, ensure_ascii=False).lower() manga_markers = ("comiclist", "comicname", "comicurl", "chapterimages", "manga", "manhua", "漫画") if numeric == 0 and any(marker in text for marker in manga_markers): return 2 return numeric def split_rule(rule: str, sep: str) -> list[str]: return [item.strip() for item in rule.split(sep) if item.strip()] def strip_replace(rule: str) -> str: return re.split(r"##|###", rule, maxsplit=1)[0].strip() def string_value(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text or None def empty_to_none(value: str | None) -> str | None: text = clean_text(value or "") return text or None def maybe_url(base_url: str, value: str | None) -> str | None: text = clean_text(value or "") if not text or text.lower() == "null": return None return resolve_resource_url(base_url, text) def clean_text(value: str) -> str: return unescape(re.sub(r"\s+", " ", value or "")).strip() def normalize_text(value: str) -> str: return clean_text(value).replace("\u3000", " ") def normalize_paragraphs(value: str) -> str: lines = [clean_text(line) for line in re.split(r"[\r\n]+", value or "")] return "\n".join(line for line in lines if line) def charset_from_content_type(value: str) -> str | None: match = re.search(r"charset=([^;]+)", value or "", re.I) return match.group(1).strip("\"'") if match else None def int_or_zero(value: Any) -> int: try: return int(str(value)) except Exception: return 0 def is_noise_link(title: str, href: str) -> bool: text = f"{title} {href}".lower() return any(marker in text for marker in ("javascript:", "login", "register", "广告", "排行", "history")) def _safe_error(exc: Exception) -> str: text = str(exc) text = re.sub(r"(token|cookie|authorization)=([^&\s]+)", r"\1=***", text, flags=re.I) return text[:240]