Files
2026-11-16 20:45:22 +08:00

1067 lines
38 KiB
Python

from __future__ import annotations
import ipaddress
import json
import os
import re
import socket
import time
from collections import OrderedDict
from dataclasses import dataclass, field
from html import unescape
from typing import Any, Iterable
from urllib.parse import quote, urljoin, urlparse
import requests
from bs4 import BeautifulSoup
class SearchEngineError(RuntimeError):
"""在线搜索规则引擎的基础异常,不包含 Cookie/Token 等敏感信息。"""
@dataclass(frozen=True)
class SearchCandidate:
source_name: str
source_url: str
title: str
url: str
resource_type: str
score: float
author: str | None = None
cover: str | None = None
matched_rule: str | None = None
@dataclass(frozen=True)
class ParsedChapter:
name: str
url: str | None = None
content: str | None = None
images: tuple[str, ...] = ()
@dataclass(frozen=True)
class ParsedNovel:
title: str
chapters: tuple[ParsedChapter, ...]
cover: str | None = None
def as_dict(self) -> dict[str, Any]:
return {
"title": self.title,
"chapters": [
{"name": chapter.name, "content": chapter.content or ""}
for chapter in self.chapters
],
"cover": self.cover or "",
}
@dataclass(frozen=True)
class ParsedComic:
title: str
chapters: tuple[ParsedChapter, ...]
cover: str | None = None
def as_dict(self) -> dict[str, Any]:
return {
"title": self.title,
"chapters": [
{"name": chapter.name, "images": list(chapter.images)}
for chapter in self.chapters
],
"cover": self.cover or "",
}
@dataclass
class RuleConfig:
rule_id: str
name: str
base_url: str
resource_type: str = "all"
search_url: str = ""
method: str = "GET"
enabled: bool = True
headers: dict[str, str] = field(default_factory=dict)
cookies: dict[str, str] = field(default_factory=dict)
rule_search: dict[str, Any] = field(default_factory=dict)
success_count: int = 0
failure_count: int = 0
average_latency_ms: float = 1000.0
ad_density: float = 0.0
@dataclass(frozen=True)
class DebugStep:
rule_id: str
stage: str
status: str
detail: str
data_length: int = 0
image_count: int = 0
class RuleManager:
"""规则配置管理器,支持动态添加、禁用和站点健康评分。"""
def __init__(self, rules: Iterable[RuleConfig] | None = None) -> None:
self._rules: OrderedDict[str, RuleConfig] = OrderedDict()
for rule in rules or ():
self.add_rule(rule)
def add_rule(self, rule: RuleConfig) -> None:
self._rules[rule.rule_id] = rule
def disable_rule(self, rule_id: str) -> None:
if rule_id in self._rules:
self._rules[rule_id].enabled = False
def enable_rule(self, rule_id: str) -> None:
if rule_id in self._rules:
self._rules[rule_id].enabled = True
def get_rules(self, resource_type: str = "all") -> list[RuleConfig]:
allowed = {resource_type, "all"}
return [
rule
for rule in self._rules.values()
if rule.enabled and (resource_type == "all" or rule.resource_type in allowed)
]
def update_health(self, rule_id: str, *, ok: bool, latency_ms: float) -> None:
rule = self._rules.get(rule_id)
if rule is None:
return
if ok:
rule.success_count += 1
else:
rule.failure_count += 1
rule.average_latency_ms = (rule.average_latency_ms * 0.8) + (latency_ms * 0.2)
def trust_score(self, rule_id: str) -> float:
rule = self._rules.get(rule_id)
if rule is None:
return 0.3
total = rule.success_count + rule.failure_count
success_rate = rule.success_count / total if total else 0.65
latency_score = max(0.0, min(1.0, 1.0 - (rule.average_latency_ms / 10_000.0)))
ad_penalty = max(0.0, min(0.4, rule.ad_density * 0.4))
return max(0.0, min(1.0, success_rate * 0.6 + latency_score * 0.4 - ad_penalty))
class LruTextCache:
"""缓存页面文本,降低重复解析同一站点结构的网络开销。"""
def __init__(self, max_size: int = 128, ttl_seconds: int = 600) -> None:
self.max_size = max_size
self.ttl_seconds = ttl_seconds
self._rows: OrderedDict[str, tuple[float, str]] = OrderedDict()
def get(self, key: str) -> str | None:
row = self._rows.get(key)
if row is None:
return None
created, value = row
if time.time() - created > self.ttl_seconds:
self._rows.pop(key, None)
return None
self._rows.move_to_end(key)
return value
def set(self, key: str, value: str) -> None:
self._rows[key] = (time.time(), value)
self._rows.move_to_end(key)
while len(self._rows) > self.max_size:
self._rows.popitem(last=False)
class OnlineSearchEngine:
def __init__(
self,
rule_manager: RuleManager | None = None,
*,
timeout: float = 10.0,
session: requests.Session | None = None,
cache: LruTextCache | None = None,
) -> None:
self.rule_manager = rule_manager or RuleManager()
self.timeout = timeout
self.session = session or requests.Session()
self.cache = cache or LruTextCache()
self.debug_steps: list[DebugStep] = []
def search_resource(
self,
query: str,
resource_type: str = "all",
*,
limit: int = 50,
) -> list[SearchCandidate]:
query = normalize_text(query)
if not query:
return []
rows: list[SearchCandidate] = []
for rule in self.rule_manager.get_rules(resource_type):
started = time.perf_counter()
try:
rows.extend(self._search_rule(rule, query, limit=limit))
latency_ms = (time.perf_counter() - started) * 1000
self.rule_manager.update_health(rule.rule_id, ok=True, latency_ms=latency_ms)
except Exception as exc:
latency_ms = (time.perf_counter() - started) * 1000
self.rule_manager.update_health(rule.rule_id, ok=False, latency_ms=latency_ms)
self._debug(rule.rule_id, "search", "failed", _safe_error(exc))
return dedupe_candidates(rows)[:limit]
def search_legado_source(
self,
source: dict[str, Any],
keyword: str,
*,
mode: str,
limit: int,
) -> list[SearchCandidate]:
rule = legado_source_to_rule(source, mode)
if not rule.base_url or not rule.search_url:
return []
return self._search_rule(rule, normalize_text(keyword), limit=limit)
def parse_novel(self, url: str) -> ParsedNovel:
html = self._get_text(url, url, rule_id="generic_novel")
soup = BeautifulSoup(html, "lxml")
title = extract_page_title(soup)
cover = maybe_url(url, first_meta_content(soup, "og:image")) or first_image_url(soup, url)
chapters = extract_chapter_links(soup, url, prefer_images=False)
current_content = extract_main_text(soup)
if not chapters:
chapters = [ParsedChapter(name=title or "正文", url=url, content=current_content)]
else:
rows = [ParsedChapter(name=title or "当前章节", url=url, content=current_content)] if current_content else []
rows.extend(chapter for chapter in chapters if chapter.url != url)
chapters = rows
self._debug("generic_novel", "parse", "ok", title, len(html), 0)
return ParsedNovel(title=title or "未命名小说", chapters=tuple(chapters), cover=cover)
def parse_comic(self, url: str) -> ParsedComic:
html = self._get_text(url, url, rule_id="generic_comic")
soup = BeautifulSoup(html, "lxml")
title = extract_page_title(soup)
cover = maybe_url(url, first_meta_content(soup, "og:image")) or first_image_url(soup, url)
images = extract_comic_images(soup, url)
chapters = extract_chapter_links(soup, url, prefer_images=True)
if images:
chapters = [ParsedChapter(name=title or "第1话", url=url, images=tuple(images))]
elif chapters:
chapters = [ParsedChapter(name=chapter.name, url=chapter.url) for chapter in chapters]
else:
chapters = []
self._debug("generic_comic", "parse", "ok", title, len(html), len(images))
return ParsedComic(title=title or "未命名漫画", chapters=tuple(chapters), cover=cover)
def _search_rule(self, rule: RuleConfig, keyword: str, *, limit: int) -> list[SearchCandidate]:
final_url, data = build_search_request(rule.search_url, keyword, rule.base_url)
assert_public_url(final_url)
html = self._request_text(rule, final_url, data=data)
self._debug(rule.rule_id, "fetch", "ok", final_url, len(html), 0)
rows = self._parse_search_payload(rule, html, final_url, keyword, limit)
if not rows:
rows = fallback_search_cards(
html,
final_url,
rule,
keyword,
limit,
trust=self.rule_manager.trust_score(rule.rule_id),
)
self._debug(rule.rule_id, "search", "ok", f"{len(rows)} candidates", len(html), 0)
return rows
def _parse_search_payload(
self,
rule: RuleConfig,
body: str,
base_url: str,
keyword: str,
limit: int,
) -> list[SearchCandidate]:
rule_search = rule.rule_search or {}
list_rule = first_string(rule_search, "bookList", "comicList", "list", "items", "data", "results")
if not list_rule:
return []
fragments = select_list(body, list_rule, base_url)
rows: list[SearchCandidate] = []
trust = self.rule_manager.trust_score(rule.rule_id)
for fragment in fragments:
title = select_one(fragment, first_string(rule_search, "name", "bookName", "comicName", "title"), base_url)
detail = select_one(fragment, first_string(rule_search, "bookUrl", "comicUrl", "detailUrl", "url", "href"), base_url)
if not title or not detail:
continue
score = title_match_score(title, keyword)
if score < 0.38:
continue
rows.append(
SearchCandidate(
source_name=rule.name,
source_url=rule.base_url,
title=clean_text(title),
author=empty_to_none(
select_one(
fragment,
first_string(rule_search, "author", "bookAuthor", "comicAuthor", "writer"),
base_url,
)
),
cover=maybe_url(
base_url,
select_one(
fragment,
first_string(rule_search, "coverUrl", "cover", "bookCover", "comicCover", "img", "image"),
base_url,
),
),
url=resolve_resource_url(base_url, detail),
resource_type=rule.resource_type,
score=(score * 0.75) + (trust * 0.25),
matched_rule=rule.rule_id,
)
)
if len(rows) >= limit:
break
return sorted(rows, key=lambda item: item.score, reverse=True)
def _request_text(self, rule: RuleConfig, url: str, *, data: dict[str, str] | None = None) -> str:
cache_key = f"{rule.method}:{url}:{json.dumps(data or {}, sort_keys=True, ensure_ascii=False)}"
cached = self.cache.get(cache_key)
if cached is not None:
return cached
headers = build_headers(rule.base_url, rule.headers)
cookies = rule.cookies or {}
method = rule.method.upper()
response = self.session.request(
method,
url,
data=data if method == "POST" else None,
headers=headers,
cookies=cookies,
timeout=self.timeout,
allow_redirects=True,
)
if response.status_code == 404:
raise SearchEngineError("源站搜索接口不存在,可能源已失效")
if response.status_code in {403, 444}:
raise SearchEngineError("源站拒绝访问,可能需要请求头、Cookie 或反爬处理")
response.raise_for_status()
text = decode_response_text(response)
self.cache.set(cache_key, text)
return text
def _get_text(self, url: str, referer: str, *, rule_id: str) -> str:
assert_public_url(url)
rule = RuleConfig(rule_id=rule_id, name=rule_id, base_url=referer, search_url=url)
return self._request_text(rule, url)
def _debug(
self,
rule_id: str,
stage: str,
status: str,
detail: str,
data_length: int = 0,
image_count: int = 0,
) -> None:
step = DebugStep(rule_id, stage, status, detail[:240], data_length, image_count)
self.debug_steps.append(step)
if os.getenv("SAKURA_ONLINE_SEARCH_DEBUG") == "1":
print(
f"[online-search] rule={step.rule_id} stage={step.stage} "
f"status={step.status} len={step.data_length} images={step.image_count} detail={step.detail}"
)
DEFAULT_RULE_MANAGER = RuleManager()
DEFAULT_ENGINE = OnlineSearchEngine(DEFAULT_RULE_MANAGER)
def search_resource(query: str, resource_type: str = "all") -> list[dict[str, Any]]:
return [
{
"title": item.title,
"url": item.url,
"source_name": item.source_name,
"source_url": item.source_url,
"resource_type": item.resource_type,
"score": item.score,
"author": item.author,
"cover": item.cover,
}
for item in DEFAULT_ENGINE.search_resource(query, resource_type)
]
def parse_novel(url: str) -> dict[str, Any]:
return DEFAULT_ENGINE.parse_novel(url).as_dict()
def parse_comic(url: str) -> dict[str, Any]:
return DEFAULT_ENGINE.parse_comic(url).as_dict()
def legado_source_to_rule(source: dict[str, Any], mode: str) -> RuleConfig:
rule_search = source.get("ruleSearch") if isinstance(source.get("ruleSearch"), dict) else {}
base_url = string_value(source.get("bookSourceUrl") or source.get("sourceUrl") or source.get("url")) or ""
search_url = string_value(source.get("searchUrl") or rule_search.get("url") or rule_search.get("searchUrl")) or ""
headers = source.get("headers") or source.get("header") or {}
cookies = source.get("cookies") or {}
if isinstance(source.get("cookie"), str):
cookies = parse_cookie_header(source["cookie"])
return RuleConfig(
rule_id=string_value(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or base_url,
name=string_value(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or "未命名源",
base_url=base_url,
resource_type=mode,
search_url=search_url,
method=(string_value(source.get("searchMethod") or source.get("method")) or "GET").upper(),
headers={str(key): str(value) for key, value in headers.items()} if isinstance(headers, dict) else {},
cookies={str(key): str(value) for key, value in cookies.items()} if isinstance(cookies, dict) else {},
rule_search=rule_search,
)
def build_search_request(template: str, keyword: str, base_url: str) -> tuple[str, dict[str, str] | None]:
raw = build_search_url(template, keyword)
if "," in raw and raw.lstrip().upper().startswith("POST"):
_, payload = raw.split(",", 1)
url, _, body = payload.partition("@")
return resolve_resource_url(base_url, url), parse_form_body(body)
if raw.lstrip().upper().startswith("POST "):
payload = raw.split(None, 1)[1]
url, _, body = payload.partition("@")
return resolve_resource_url(base_url, url), parse_form_body(body)
return resolve_resource_url(base_url, raw), None
def build_search_url(template: str, keyword: str) -> str:
encoded = quote(keyword, safe="")
value = template.strip()
for token in ("{{key}}", "{{keyword}}", "{key}", "{keyword}", "<key>", "<keyword>", "searchKey"):
value = value.replace(token, encoded)
value = value.replace("%s", encoded)
return value.replace("{{page}}", "1").replace("{page}", "1").replace("<page>", "1")
def resolve_resource_url(base_url: str, raw_url: str) -> str:
url = (raw_url or "").strip()
if not url:
raise SearchEngineError("Empty resource URL")
if url.startswith(("http://", "https://")):
return url
if url.startswith("//"):
scheme = urlparse(base_url).scheme or "https"
return f"{scheme}:{url}"
return urljoin(base_url, url)
def assert_public_url(url: str) -> None:
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise SearchEngineError("Only http/https source URLs are allowed")
infos = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM)
for item in infos:
ip = ipaddress.ip_address(item[4][0])
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved:
raise SearchEngineError("Private network source URLs are not allowed")
def build_headers(referer: str, extra: dict[str, str] | None = None) -> dict[str, str]:
headers = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36"
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Referer": referer,
}
for key, value in (extra or {}).items():
if key.lower() in {"authorization"}:
continue
headers[key] = value
return headers
def decode_response_text(response: requests.Response) -> str:
content_type = response.headers.get("content-type", "")
candidates = [
response.encoding,
charset_from_content_type(content_type),
"utf-8",
"gb18030",
"gbk",
"gb2312",
]
raw = response.content[:4_000_000]
for candidate in candidates:
if not candidate:
continue
try:
return raw.decode(candidate, errors="replace")
except LookupError:
continue
return raw.decode("utf-8", errors="replace")
def select_list(body: str, rule: str | None, base_url: str) -> list[str]:
if not rule:
return []
for part in split_rule(rule, "||"):
part = strip_replace(part)
if not part:
continue
if part.lower().startswith("regex:"):
pattern = part.split(":", 1)[1]
matches = re.findall(pattern, body, re.I | re.S)
rows = [clean_text(item[0] if isinstance(item, tuple) else item) for item in matches]
if rows:
return [row for row in rows if row]
if part.startswith("$.") or part.startswith("$["):
decoded = decode_json_like(body)
rows = json_path(decoded, part) if decoded is not None else []
if rows:
return [json.dumps(row, ensure_ascii=False) if isinstance(row, (dict, list)) else str(row) for row in rows]
rows = soup_rule_list(body, part, base_url)
if rows:
return rows
return []
def select_one(fragment: str, rule: str | None, base_url: str) -> str:
rows = select_list(fragment, rule, base_url)
return rows[0] if rows else ""
def soup_rule_list(body: str, rule: str, base_url: str) -> list[str]:
tokens = [token.strip() for token in rule.split("@") if token.strip()]
if not tokens:
return []
attr = tokens[-1] if tokens[-1].lower() in {"text", "html", "href", "src", "data-src", "data-original", "alt", "title"} else ""
selector_tokens = tokens[:-1] if attr else tokens
selector = " ".join(rule_token_to_css(token) for token in selector_tokens if rule_token_to_css(token))
if not selector:
return []
soup = BeautifulSoup(body, "lxml")
rows: list[str] = []
for element in soup.select(selector):
if attr == "text":
rows.append(clean_text(element.get_text(" ")))
elif attr == "html":
rows.append(str(element))
elif attr:
value = element.get(attr)
if value:
rows.append(resolve_resource_url(base_url, value) if attr in {"href", "src", "data-src", "data-original"} else clean_text(value))
else:
rows.append(str(element))
return [row for row in rows if row]
def rule_token_to_css(token: str) -> str:
if token.startswith("class."):
return "." + ".".join(part for part in token[6:].split(".") if part)
if token.startswith("id."):
return "#" + token[3:]
if token.startswith("tag."):
return token[4:]
if token.startswith(".") or token.startswith("#") or token in {"a", "li", "div", "img", "ul", "ol"}:
return token
return token if re.match(r"^[a-zA-Z0-9_.# >:-]+$", token) else ""
def fallback_search_cards(
body: str,
base_url: str,
rule: RuleConfig,
keyword: str,
limit: int,
*,
trust: float,
) -> list[SearchCandidate]:
decoded = decode_json_like(body)
rows: list[SearchCandidate] = []
if decoded is not None:
rows = fallback_json_candidates(decoded, base_url, rule, keyword, trust)
if not rows:
rows = fallback_html_candidates(body, base_url, rule, keyword, trust)
return sorted(rows, key=lambda item: item.score, reverse=True)[:limit]
def fallback_json_candidates(
decoded: Any,
base_url: str,
rule: RuleConfig,
keyword: str,
trust: float,
) -> list[SearchCandidate]:
rows: list[SearchCandidate] = []
for node in walk_dicts(decoded):
title = first_value(node, "title", "name", "bookName", "book_name", "novelName", "comicName", "comic_name")
if not title:
continue
score = title_match_score(title, keyword)
if score < 0.38:
continue
detail = first_value(node, "url", "href", "link", "bookUrl", "book_url", "comicUrl", "comic_url", "detailUrl", "detail_url", "path")
if not detail:
detail = infer_detail_path(
node,
rule.resource_type,
first_value(node, "id", "book_id", "bookId", "comic_id", "comicId", "bid", "cid"),
)
if not detail:
continue
rows.append(
SearchCandidate(
source_name=rule.name,
source_url=rule.base_url,
title=title,
author=empty_to_none(first_value(node, "author", "bookAuthor", "writer", "nickname", "userName", "user_name")),
cover=maybe_url(base_url, first_value(node, "cover", "coverUrl", "cover_url", "img", "image", "pic", "pic_url")),
url=resolve_resource_url(base_url, detail),
resource_type=rule.resource_type,
score=(score * 0.75) + (trust * 0.25),
matched_rule=f"{rule.rule_id}:json-fallback",
)
)
return rows
def fallback_html_candidates(body: str, base_url: str, rule: RuleConfig, keyword: str, trust: float) -> list[SearchCandidate]:
soup = BeautifulSoup(body, "lxml")
rows: list[SearchCandidate] = []
for anchor in soup.select("a[href]"):
href = anchor.get("href") or ""
text = clean_text(anchor.get("title") or anchor.get("alt") or anchor.get_text(" "))
image = first_image_url(anchor, base_url)
if not text and image:
text = clean_text(anchor.find("img").get("alt", "") if anchor.find("img") else "")
score = title_match_score(text, keyword)
if not href or score < 0.38:
continue
if is_noise_link(text, href):
continue
rows.append(
SearchCandidate(
source_name=rule.name,
source_url=rule.base_url,
title=text,
author=None,
cover=image,
url=resolve_resource_url(base_url, href),
resource_type=rule.resource_type,
score=(score * 0.75) + (trust * 0.25),
matched_rule=f"{rule.rule_id}:html-fallback",
)
)
return rows
def extract_main_text(soup: BeautifulSoup) -> str:
for selector in ("#content", ".content", ".chapter-content", ".read-content", "article", "main"):
element = soup.select_one(selector)
if element:
text = normalize_paragraphs(element.get_text("\n"))
if len(text) > 80:
return text
best = ""
for element in soup.find_all(["article", "main", "div", "section"]):
text = normalize_paragraphs(element.get_text("\n"))
if text_density(text) > text_density(best):
best = text
return best
def extract_chapter_links(soup: BeautifulSoup, base_url: str, *, prefer_images: bool) -> list[ParsedChapter]:
chapters: list[ParsedChapter] = []
patterns = (
r"(第\s*\d+\s*[章节话回卷]|chapter\s*\d+|下一章|上一章|目录)"
if not prefer_images
else r"(第\s*\d+\s*[话回卷]|chapter\s*\d+|episode\s*\d+|下一话|上一话|目录)"
)
for anchor in soup.select("a[href]"):
name = clean_text(anchor.get_text(" ") or anchor.get("title", ""))
href = anchor.get("href") or ""
if not name or not href:
continue
if re.search(patterns, name, re.I) or re.search(r"(chapter|read|episode|manga|comic)", href, re.I):
chapters.append(ParsedChapter(name=name[:120], url=resolve_resource_url(base_url, href)))
return dedupe_chapters(chapters)
def extract_comic_images(soup: BeautifulSoup, base_url: str) -> list[str]:
rows: list[str] = []
for image in soup.select("img"):
value = ""
for attr in ("data-src", "data-original", "data-lazy-src", "data-url", "src", "data-srcset", "srcset"):
raw = image.get(attr)
if raw:
value = split_srcset(raw)[0] if attr.endswith("srcset") else raw
break
if not value:
continue
url = resolve_resource_url(base_url, value)
if is_valid_comic_image(url, image):
rows.append(url)
for match in re.finditer(r"https?://[^\"']+\.(?:jpg|jpeg|png|webp|avif)(?:\?[^\"']*)?", str(soup), re.I):
url = match.group(0)
if is_valid_comic_image(url, None):
rows.append(url)
for raw in extract_script_image_candidates(str(soup)):
try:
url = resolve_resource_url(base_url, raw)
except SearchEngineError:
continue
if is_valid_comic_image(url, None):
rows.append(url)
return dedupe_strings(rows)
def split_srcset(value: str) -> list[str]:
urls: list[str] = []
for part in value.split(","):
url = part.strip().split()[0] if part.strip() else ""
if url:
urls.append(url)
return urls
def extract_script_image_candidates(html: str) -> list[str]:
text = html.replace("\\/", "/")
rows: list[str] = []
for match in re.finditer(r'''["']((?:(?:https?:)?//|/)[^"']+\.(?:jpg|jpeg|png|webp|avif)(?:\?[^"']*)?)["']''', text, re.I):
rows.append(match.group(1))
for assignment in re.finditer(
r"(?:var|let|const)?\s*(?:chapterImages|imageList|imgList|pageImgs|pics|photosr|newImgs|img_data)\s*=\s*(\[[\s\S]*?\])\s*;?",
text,
re.I,
):
try:
decoded = json.loads(assignment.group(1))
except Exception:
continue
collect_image_strings(decoded, rows)
return dedupe_strings(rows)
def collect_image_strings(node: Any, rows: list[str]) -> None:
if isinstance(node, str):
text = node.strip().replace("\\/", "/")
if re.search(r"\.(jpg|jpeg|png|webp|avif)(?:[?#].*)?$", text, re.I):
rows.append(text)
elif isinstance(node, dict):
for value in node.values():
collect_image_strings(value, rows)
elif isinstance(node, list):
for value in node:
collect_image_strings(value, rows)
def is_valid_comic_image(url: str, image: Any | None) -> bool:
lower = url.lower()
if not re.search(r"\.(jpg|jpeg|png|webp|avif)(?:[?#].*)?$", lower):
return False
if any(marker in lower for marker in ("logo", "avatar", "icon", "blank", "placeholder", "loading")):
return False
if image is not None:
width = int_or_zero(image.get("width"))
height = int_or_zero(image.get("height"))
if width and height and (width < 100 or height < 100):
return False
return True
def infer_detail_path(node: dict[str, Any], resource_type: str, raw_id: str) -> str:
value = clean_text(raw_id)
if not value or not re.match(r"^[\w-]+$", value):
return ""
if resource_type == "manga":
return first_value(node, "path") or f"/comic/{value}/"
return first_value(node, "path") or f"/book/{value}/"
def title_match_score(title: str, keyword: str) -> float:
title_norm = strip_noise_words(normalize_text(title)).lower()
keyword_norm = strip_noise_words(normalize_text(keyword)).lower()
if not title_norm or not keyword_norm:
return 0.0
if title_norm == keyword_norm:
return 1.0
contains = 0.78 if keyword_norm in title_norm else 0.0
tokens = [token for token in re.split(r"\s+", keyword_norm) if token]
coverage = sum(1 for token in tokens if token in title_norm) / len(tokens) if tokens else 0.0
distance = levenshtein(title_norm[:80], keyword_norm[:80])
max_len = max(len(title_norm[:80]), len(keyword_norm[:80]), 1)
fuzzy = 1.0 - min(distance / max_len, 1.0)
return max(contains, coverage * 0.82, fuzzy * 0.72)
def levenshtein(left: str, right: str) -> int:
if left == right:
return 0
if len(left) < len(right):
left, right = right, left
previous = list(range(len(right) + 1))
for i, left_char in enumerate(left, 1):
current = [i]
for j, right_char in enumerate(right, 1):
insert = current[j - 1] + 1
delete = previous[j] + 1
replace = previous[j - 1] + (left_char != right_char)
current.append(min(insert, delete, replace))
previous = current
return previous[-1]
def strip_noise_words(value: str) -> str:
return re.sub(r"(最新章节|免费阅读|全文阅读|漫画全集|在线阅读|无弹窗|笔趣阁|手机版)", " ", value)
def decode_json_like(body: str) -> Any:
text = body.lstrip("\ufeff \t\r\n")
match = re.match(r"^[a-zA-Z_$][\w$]*\s*\((.*)\)\s*;?\s*$", text, re.S)
if match:
text = match.group(1)
try:
return json.loads(text)
except Exception:
return None
def json_path(decoded: Any, rule: str) -> list[Any]:
if decoded is None:
return []
path = re.sub(r"^\$\.?", "", rule.strip())
if path in {"", "*", "[*]"}:
return decoded if isinstance(decoded, list) else [decoded]
current = [decoded]
for segment in [part for part in path.split(".") if part]:
next_values: list[Any] = []
star = segment.endswith("[*]")
key = segment[:-3] if star else segment
for value in current:
child = value.get(key) if isinstance(value, dict) and key else value
if star and isinstance(child, list):
next_values.extend(child)
elif child is not None:
next_values.append(child)
current = next_values
return current
def extract_page_title(soup: BeautifulSoup) -> str:
for selector in ("h1", ".book-title", ".comic-title", ".title"):
element = soup.select_one(selector)
if element:
text = clean_text(element.get_text(" "))
if text:
return text
return clean_text(first_meta_content(soup, "og:title") or (soup.title.get_text(" ") if soup.title else ""))
def first_meta_content(soup: BeautifulSoup, name: str) -> str | None:
for selector in (f'meta[property="{name}"]', f'meta[name="{name}"]'):
element = soup.select_one(selector)
if element and element.get("content"):
return clean_text(element["content"])
return None
def first_image_url(node: Any, base_url: str) -> str | None:
image = node.find("img") if hasattr(node, "find") else None
if not image:
return None
for attr in ("data-src", "data-original", "data-lazy-src", "src"):
value = image.get(attr)
if value:
return resolve_resource_url(base_url, value)
return None
def text_density(text: str) -> float:
normalized = normalize_paragraphs(text)
if not normalized:
return 0.0
punctuation = len(re.findall(r"[,。!?,.!?]", normalized))
return len(normalized) + punctuation * 8
def fallback_decode_raw(raw: bytes, content_type: str = "") -> str:
for candidate in (charset_from_content_type(content_type), "utf-8", "gb18030", "gbk", "gb2312"):
if not candidate:
continue
try:
return raw.decode(candidate, errors="replace")
except LookupError:
continue
return raw.decode("utf-8", errors="replace")
def parse_cookie_header(value: str) -> dict[str, str]:
cookies: dict[str, str] = {}
for part in value.split(";"):
key, _, raw = part.partition("=")
key = key.strip()
if key:
cookies[key] = raw.strip()
return cookies
def parse_form_body(value: str) -> dict[str, str] | None:
if not value:
return None
rows: dict[str, str] = {}
for part in value.split("&"):
key, _, raw = part.partition("=")
if key:
rows[key] = raw
return rows or None
def dedupe_candidates(rows: list[SearchCandidate]) -> list[SearchCandidate]:
seen: set[str] = set()
output: list[SearchCandidate] = []
for row in sorted(rows, key=lambda item: item.score, reverse=True):
key = row.url
if key in seen:
continue
seen.add(key)
output.append(row)
return output
def dedupe_chapters(rows: list[ParsedChapter]) -> list[ParsedChapter]:
seen: set[str] = set()
output: list[ParsedChapter] = []
for row in rows:
key = row.url or row.name
if key in seen:
continue
seen.add(key)
output.append(row)
return output
def dedupe_strings(rows: list[str]) -> list[str]:
seen: set[str] = set()
output: list[str] = []
for row in rows:
if row in seen:
continue
seen.add(row)
output.append(row)
return output
def walk_dicts(node: Any):
if isinstance(node, dict):
yield node
for value in node.values():
yield from walk_dicts(value)
elif isinstance(node, list):
for item in node:
yield from walk_dicts(item)
def first_value(node: dict[str, Any], *keys: str) -> str:
for key in keys:
value = node.get(key)
if value is not None:
text = clean_text(str(value))
if text and text.lower() != "null":
return text
return ""
def first_string(mapping: dict[str, Any], *keys: str) -> str | None:
for key in keys:
value = string_value(mapping.get(key))
if value:
return value
return None
def source_type(source: dict[str, Any]) -> int:
value = source.get("bookSourceType", source.get("sourceType", source.get("type", 0)))
try:
numeric = int(value)
except Exception:
numeric = 0
text = json.dumps(source, ensure_ascii=False).lower()
manga_markers = ("comiclist", "comicname", "comicurl", "chapterimages", "manga", "manhua", "漫画")
if numeric == 0 and any(marker in text for marker in manga_markers):
return 2
return numeric
def split_rule(rule: str, sep: str) -> list[str]:
return [item.strip() for item in rule.split(sep) if item.strip()]
def strip_replace(rule: str) -> str:
return re.split(r"##|###", rule, maxsplit=1)[0].strip()
def string_value(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def empty_to_none(value: str | None) -> str | None:
text = clean_text(value or "")
return text or None
def maybe_url(base_url: str, value: str | None) -> str | None:
text = clean_text(value or "")
if not text or text.lower() == "null":
return None
return resolve_resource_url(base_url, text)
def clean_text(value: str) -> str:
return unescape(re.sub(r"\s+", " ", value or "")).strip()
def normalize_text(value: str) -> str:
return clean_text(value).replace("\u3000", " ")
def normalize_paragraphs(value: str) -> str:
lines = [clean_text(line) for line in re.split(r"[\r\n]+", value or "")]
return "\n".join(line for line in lines if line)
def charset_from_content_type(value: str) -> str | None:
match = re.search(r"charset=([^;]+)", value or "", re.I)
return match.group(1).strip("\"'") if match else None
def int_or_zero(value: Any) -> int:
try:
return int(str(value))
except Exception:
return 0
def is_noise_link(title: str, href: str) -> bool:
text = f"{title} {href}".lower()
return any(marker in text for marker in ("javascript:", "login", "register", "广告", "排行", "history"))
def _safe_error(exc: Exception) -> str:
text = str(exc)
text = re.sub(r"(token|cookie|authorization)=([^&\s]+)", r"\1=***", text, flags=re.I)
return text[:240]