1067 lines
38 KiB
Python
1067 lines
38 KiB
Python
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import re
|
|
import socket
|
|
import time
|
|
from collections import OrderedDict
|
|
from dataclasses import dataclass, field
|
|
from html import unescape
|
|
from typing import Any, Iterable
|
|
from urllib.parse import quote, urljoin, urlparse
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
|
class SearchEngineError(RuntimeError):
|
|
"""在线搜索规则引擎的基础异常,不包含 Cookie/Token 等敏感信息。"""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SearchCandidate:
|
|
source_name: str
|
|
source_url: str
|
|
title: str
|
|
url: str
|
|
resource_type: str
|
|
score: float
|
|
author: str | None = None
|
|
cover: str | None = None
|
|
matched_rule: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ParsedChapter:
|
|
name: str
|
|
url: str | None = None
|
|
content: str | None = None
|
|
images: tuple[str, ...] = ()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ParsedNovel:
|
|
title: str
|
|
chapters: tuple[ParsedChapter, ...]
|
|
cover: str | None = None
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"title": self.title,
|
|
"chapters": [
|
|
{"name": chapter.name, "content": chapter.content or ""}
|
|
for chapter in self.chapters
|
|
],
|
|
"cover": self.cover or "",
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ParsedComic:
|
|
title: str
|
|
chapters: tuple[ParsedChapter, ...]
|
|
cover: str | None = None
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"title": self.title,
|
|
"chapters": [
|
|
{"name": chapter.name, "images": list(chapter.images)}
|
|
for chapter in self.chapters
|
|
],
|
|
"cover": self.cover or "",
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class RuleConfig:
|
|
rule_id: str
|
|
name: str
|
|
base_url: str
|
|
resource_type: str = "all"
|
|
search_url: str = ""
|
|
method: str = "GET"
|
|
enabled: bool = True
|
|
headers: dict[str, str] = field(default_factory=dict)
|
|
cookies: dict[str, str] = field(default_factory=dict)
|
|
rule_search: dict[str, Any] = field(default_factory=dict)
|
|
success_count: int = 0
|
|
failure_count: int = 0
|
|
average_latency_ms: float = 1000.0
|
|
ad_density: float = 0.0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DebugStep:
|
|
rule_id: str
|
|
stage: str
|
|
status: str
|
|
detail: str
|
|
data_length: int = 0
|
|
image_count: int = 0
|
|
|
|
|
|
class RuleManager:
|
|
"""规则配置管理器,支持动态添加、禁用和站点健康评分。"""
|
|
|
|
def __init__(self, rules: Iterable[RuleConfig] | None = None) -> None:
|
|
self._rules: OrderedDict[str, RuleConfig] = OrderedDict()
|
|
for rule in rules or ():
|
|
self.add_rule(rule)
|
|
|
|
def add_rule(self, rule: RuleConfig) -> None:
|
|
self._rules[rule.rule_id] = rule
|
|
|
|
def disable_rule(self, rule_id: str) -> None:
|
|
if rule_id in self._rules:
|
|
self._rules[rule_id].enabled = False
|
|
|
|
def enable_rule(self, rule_id: str) -> None:
|
|
if rule_id in self._rules:
|
|
self._rules[rule_id].enabled = True
|
|
|
|
def get_rules(self, resource_type: str = "all") -> list[RuleConfig]:
|
|
allowed = {resource_type, "all"}
|
|
return [
|
|
rule
|
|
for rule in self._rules.values()
|
|
if rule.enabled and (resource_type == "all" or rule.resource_type in allowed)
|
|
]
|
|
|
|
def update_health(self, rule_id: str, *, ok: bool, latency_ms: float) -> None:
|
|
rule = self._rules.get(rule_id)
|
|
if rule is None:
|
|
return
|
|
if ok:
|
|
rule.success_count += 1
|
|
else:
|
|
rule.failure_count += 1
|
|
rule.average_latency_ms = (rule.average_latency_ms * 0.8) + (latency_ms * 0.2)
|
|
|
|
def trust_score(self, rule_id: str) -> float:
|
|
rule = self._rules.get(rule_id)
|
|
if rule is None:
|
|
return 0.3
|
|
total = rule.success_count + rule.failure_count
|
|
success_rate = rule.success_count / total if total else 0.65
|
|
latency_score = max(0.0, min(1.0, 1.0 - (rule.average_latency_ms / 10_000.0)))
|
|
ad_penalty = max(0.0, min(0.4, rule.ad_density * 0.4))
|
|
return max(0.0, min(1.0, success_rate * 0.6 + latency_score * 0.4 - ad_penalty))
|
|
|
|
|
|
class LruTextCache:
|
|
"""缓存页面文本,降低重复解析同一站点结构的网络开销。"""
|
|
|
|
def __init__(self, max_size: int = 128, ttl_seconds: int = 600) -> None:
|
|
self.max_size = max_size
|
|
self.ttl_seconds = ttl_seconds
|
|
self._rows: OrderedDict[str, tuple[float, str]] = OrderedDict()
|
|
|
|
def get(self, key: str) -> str | None:
|
|
row = self._rows.get(key)
|
|
if row is None:
|
|
return None
|
|
created, value = row
|
|
if time.time() - created > self.ttl_seconds:
|
|
self._rows.pop(key, None)
|
|
return None
|
|
self._rows.move_to_end(key)
|
|
return value
|
|
|
|
def set(self, key: str, value: str) -> None:
|
|
self._rows[key] = (time.time(), value)
|
|
self._rows.move_to_end(key)
|
|
while len(self._rows) > self.max_size:
|
|
self._rows.popitem(last=False)
|
|
|
|
|
|
class OnlineSearchEngine:
|
|
def __init__(
|
|
self,
|
|
rule_manager: RuleManager | None = None,
|
|
*,
|
|
timeout: float = 10.0,
|
|
session: requests.Session | None = None,
|
|
cache: LruTextCache | None = None,
|
|
) -> None:
|
|
self.rule_manager = rule_manager or RuleManager()
|
|
self.timeout = timeout
|
|
self.session = session or requests.Session()
|
|
self.cache = cache or LruTextCache()
|
|
self.debug_steps: list[DebugStep] = []
|
|
|
|
def search_resource(
|
|
self,
|
|
query: str,
|
|
resource_type: str = "all",
|
|
*,
|
|
limit: int = 50,
|
|
) -> list[SearchCandidate]:
|
|
query = normalize_text(query)
|
|
if not query:
|
|
return []
|
|
rows: list[SearchCandidate] = []
|
|
for rule in self.rule_manager.get_rules(resource_type):
|
|
started = time.perf_counter()
|
|
try:
|
|
rows.extend(self._search_rule(rule, query, limit=limit))
|
|
latency_ms = (time.perf_counter() - started) * 1000
|
|
self.rule_manager.update_health(rule.rule_id, ok=True, latency_ms=latency_ms)
|
|
except Exception as exc:
|
|
latency_ms = (time.perf_counter() - started) * 1000
|
|
self.rule_manager.update_health(rule.rule_id, ok=False, latency_ms=latency_ms)
|
|
self._debug(rule.rule_id, "search", "failed", _safe_error(exc))
|
|
return dedupe_candidates(rows)[:limit]
|
|
|
|
def search_legado_source(
|
|
self,
|
|
source: dict[str, Any],
|
|
keyword: str,
|
|
*,
|
|
mode: str,
|
|
limit: int,
|
|
) -> list[SearchCandidate]:
|
|
rule = legado_source_to_rule(source, mode)
|
|
if not rule.base_url or not rule.search_url:
|
|
return []
|
|
return self._search_rule(rule, normalize_text(keyword), limit=limit)
|
|
|
|
def parse_novel(self, url: str) -> ParsedNovel:
|
|
html = self._get_text(url, url, rule_id="generic_novel")
|
|
soup = BeautifulSoup(html, "lxml")
|
|
title = extract_page_title(soup)
|
|
cover = maybe_url(url, first_meta_content(soup, "og:image")) or first_image_url(soup, url)
|
|
chapters = extract_chapter_links(soup, url, prefer_images=False)
|
|
current_content = extract_main_text(soup)
|
|
if not chapters:
|
|
chapters = [ParsedChapter(name=title or "正文", url=url, content=current_content)]
|
|
else:
|
|
rows = [ParsedChapter(name=title or "当前章节", url=url, content=current_content)] if current_content else []
|
|
rows.extend(chapter for chapter in chapters if chapter.url != url)
|
|
chapters = rows
|
|
self._debug("generic_novel", "parse", "ok", title, len(html), 0)
|
|
return ParsedNovel(title=title or "未命名小说", chapters=tuple(chapters), cover=cover)
|
|
|
|
def parse_comic(self, url: str) -> ParsedComic:
|
|
html = self._get_text(url, url, rule_id="generic_comic")
|
|
soup = BeautifulSoup(html, "lxml")
|
|
title = extract_page_title(soup)
|
|
cover = maybe_url(url, first_meta_content(soup, "og:image")) or first_image_url(soup, url)
|
|
images = extract_comic_images(soup, url)
|
|
chapters = extract_chapter_links(soup, url, prefer_images=True)
|
|
if images:
|
|
chapters = [ParsedChapter(name=title or "第1话", url=url, images=tuple(images))]
|
|
elif chapters:
|
|
chapters = [ParsedChapter(name=chapter.name, url=chapter.url) for chapter in chapters]
|
|
else:
|
|
chapters = []
|
|
self._debug("generic_comic", "parse", "ok", title, len(html), len(images))
|
|
return ParsedComic(title=title or "未命名漫画", chapters=tuple(chapters), cover=cover)
|
|
|
|
def _search_rule(self, rule: RuleConfig, keyword: str, *, limit: int) -> list[SearchCandidate]:
|
|
final_url, data = build_search_request(rule.search_url, keyword, rule.base_url)
|
|
assert_public_url(final_url)
|
|
html = self._request_text(rule, final_url, data=data)
|
|
self._debug(rule.rule_id, "fetch", "ok", final_url, len(html), 0)
|
|
rows = self._parse_search_payload(rule, html, final_url, keyword, limit)
|
|
if not rows:
|
|
rows = fallback_search_cards(
|
|
html,
|
|
final_url,
|
|
rule,
|
|
keyword,
|
|
limit,
|
|
trust=self.rule_manager.trust_score(rule.rule_id),
|
|
)
|
|
self._debug(rule.rule_id, "search", "ok", f"{len(rows)} candidates", len(html), 0)
|
|
return rows
|
|
|
|
def _parse_search_payload(
|
|
self,
|
|
rule: RuleConfig,
|
|
body: str,
|
|
base_url: str,
|
|
keyword: str,
|
|
limit: int,
|
|
) -> list[SearchCandidate]:
|
|
rule_search = rule.rule_search or {}
|
|
list_rule = first_string(rule_search, "bookList", "comicList", "list", "items", "data", "results")
|
|
if not list_rule:
|
|
return []
|
|
fragments = select_list(body, list_rule, base_url)
|
|
rows: list[SearchCandidate] = []
|
|
trust = self.rule_manager.trust_score(rule.rule_id)
|
|
for fragment in fragments:
|
|
title = select_one(fragment, first_string(rule_search, "name", "bookName", "comicName", "title"), base_url)
|
|
detail = select_one(fragment, first_string(rule_search, "bookUrl", "comicUrl", "detailUrl", "url", "href"), base_url)
|
|
if not title or not detail:
|
|
continue
|
|
score = title_match_score(title, keyword)
|
|
if score < 0.38:
|
|
continue
|
|
rows.append(
|
|
SearchCandidate(
|
|
source_name=rule.name,
|
|
source_url=rule.base_url,
|
|
title=clean_text(title),
|
|
author=empty_to_none(
|
|
select_one(
|
|
fragment,
|
|
first_string(rule_search, "author", "bookAuthor", "comicAuthor", "writer"),
|
|
base_url,
|
|
)
|
|
),
|
|
cover=maybe_url(
|
|
base_url,
|
|
select_one(
|
|
fragment,
|
|
first_string(rule_search, "coverUrl", "cover", "bookCover", "comicCover", "img", "image"),
|
|
base_url,
|
|
),
|
|
),
|
|
url=resolve_resource_url(base_url, detail),
|
|
resource_type=rule.resource_type,
|
|
score=(score * 0.75) + (trust * 0.25),
|
|
matched_rule=rule.rule_id,
|
|
)
|
|
)
|
|
if len(rows) >= limit:
|
|
break
|
|
return sorted(rows, key=lambda item: item.score, reverse=True)
|
|
|
|
def _request_text(self, rule: RuleConfig, url: str, *, data: dict[str, str] | None = None) -> str:
|
|
cache_key = f"{rule.method}:{url}:{json.dumps(data or {}, sort_keys=True, ensure_ascii=False)}"
|
|
cached = self.cache.get(cache_key)
|
|
if cached is not None:
|
|
return cached
|
|
headers = build_headers(rule.base_url, rule.headers)
|
|
cookies = rule.cookies or {}
|
|
method = rule.method.upper()
|
|
response = self.session.request(
|
|
method,
|
|
url,
|
|
data=data if method == "POST" else None,
|
|
headers=headers,
|
|
cookies=cookies,
|
|
timeout=self.timeout,
|
|
allow_redirects=True,
|
|
)
|
|
if response.status_code == 404:
|
|
raise SearchEngineError("源站搜索接口不存在,可能源已失效")
|
|
if response.status_code in {403, 444}:
|
|
raise SearchEngineError("源站拒绝访问,可能需要请求头、Cookie 或反爬处理")
|
|
response.raise_for_status()
|
|
text = decode_response_text(response)
|
|
self.cache.set(cache_key, text)
|
|
return text
|
|
|
|
def _get_text(self, url: str, referer: str, *, rule_id: str) -> str:
|
|
assert_public_url(url)
|
|
rule = RuleConfig(rule_id=rule_id, name=rule_id, base_url=referer, search_url=url)
|
|
return self._request_text(rule, url)
|
|
|
|
def _debug(
|
|
self,
|
|
rule_id: str,
|
|
stage: str,
|
|
status: str,
|
|
detail: str,
|
|
data_length: int = 0,
|
|
image_count: int = 0,
|
|
) -> None:
|
|
step = DebugStep(rule_id, stage, status, detail[:240], data_length, image_count)
|
|
self.debug_steps.append(step)
|
|
if os.getenv("SAKURA_ONLINE_SEARCH_DEBUG") == "1":
|
|
print(
|
|
f"[online-search] rule={step.rule_id} stage={step.stage} "
|
|
f"status={step.status} len={step.data_length} images={step.image_count} detail={step.detail}"
|
|
)
|
|
|
|
|
|
DEFAULT_RULE_MANAGER = RuleManager()
|
|
DEFAULT_ENGINE = OnlineSearchEngine(DEFAULT_RULE_MANAGER)
|
|
|
|
|
|
def search_resource(query: str, resource_type: str = "all") -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"title": item.title,
|
|
"url": item.url,
|
|
"source_name": item.source_name,
|
|
"source_url": item.source_url,
|
|
"resource_type": item.resource_type,
|
|
"score": item.score,
|
|
"author": item.author,
|
|
"cover": item.cover,
|
|
}
|
|
for item in DEFAULT_ENGINE.search_resource(query, resource_type)
|
|
]
|
|
|
|
|
|
def parse_novel(url: str) -> dict[str, Any]:
|
|
return DEFAULT_ENGINE.parse_novel(url).as_dict()
|
|
|
|
|
|
def parse_comic(url: str) -> dict[str, Any]:
|
|
return DEFAULT_ENGINE.parse_comic(url).as_dict()
|
|
|
|
|
|
def legado_source_to_rule(source: dict[str, Any], mode: str) -> RuleConfig:
|
|
rule_search = source.get("ruleSearch") if isinstance(source.get("ruleSearch"), dict) else {}
|
|
base_url = string_value(source.get("bookSourceUrl") or source.get("sourceUrl") or source.get("url")) or ""
|
|
search_url = string_value(source.get("searchUrl") or rule_search.get("url") or rule_search.get("searchUrl")) or ""
|
|
headers = source.get("headers") or source.get("header") or {}
|
|
cookies = source.get("cookies") or {}
|
|
if isinstance(source.get("cookie"), str):
|
|
cookies = parse_cookie_header(source["cookie"])
|
|
return RuleConfig(
|
|
rule_id=string_value(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or base_url,
|
|
name=string_value(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or "未命名源",
|
|
base_url=base_url,
|
|
resource_type=mode,
|
|
search_url=search_url,
|
|
method=(string_value(source.get("searchMethod") or source.get("method")) or "GET").upper(),
|
|
headers={str(key): str(value) for key, value in headers.items()} if isinstance(headers, dict) else {},
|
|
cookies={str(key): str(value) for key, value in cookies.items()} if isinstance(cookies, dict) else {},
|
|
rule_search=rule_search,
|
|
)
|
|
|
|
|
|
def build_search_request(template: str, keyword: str, base_url: str) -> tuple[str, dict[str, str] | None]:
|
|
raw = build_search_url(template, keyword)
|
|
if "," in raw and raw.lstrip().upper().startswith("POST"):
|
|
_, payload = raw.split(",", 1)
|
|
url, _, body = payload.partition("@")
|
|
return resolve_resource_url(base_url, url), parse_form_body(body)
|
|
if raw.lstrip().upper().startswith("POST "):
|
|
payload = raw.split(None, 1)[1]
|
|
url, _, body = payload.partition("@")
|
|
return resolve_resource_url(base_url, url), parse_form_body(body)
|
|
return resolve_resource_url(base_url, raw), None
|
|
|
|
|
|
def build_search_url(template: str, keyword: str) -> str:
|
|
encoded = quote(keyword, safe="")
|
|
value = template.strip()
|
|
for token in ("{{key}}", "{{keyword}}", "{key}", "{keyword}", "<key>", "<keyword>", "searchKey"):
|
|
value = value.replace(token, encoded)
|
|
value = value.replace("%s", encoded)
|
|
return value.replace("{{page}}", "1").replace("{page}", "1").replace("<page>", "1")
|
|
|
|
|
|
def resolve_resource_url(base_url: str, raw_url: str) -> str:
|
|
url = (raw_url or "").strip()
|
|
if not url:
|
|
raise SearchEngineError("Empty resource URL")
|
|
if url.startswith(("http://", "https://")):
|
|
return url
|
|
if url.startswith("//"):
|
|
scheme = urlparse(base_url).scheme or "https"
|
|
return f"{scheme}:{url}"
|
|
return urljoin(base_url, url)
|
|
|
|
|
|
def assert_public_url(url: str) -> None:
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
raise SearchEngineError("Only http/https source URLs are allowed")
|
|
infos = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM)
|
|
for item in infos:
|
|
ip = ipaddress.ip_address(item[4][0])
|
|
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved:
|
|
raise SearchEngineError("Private network source URLs are not allowed")
|
|
|
|
|
|
def build_headers(referer: str, extra: dict[str, str] | None = None) -> dict[str, str]:
|
|
headers = {
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36"
|
|
),
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7",
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
"Referer": referer,
|
|
}
|
|
for key, value in (extra or {}).items():
|
|
if key.lower() in {"authorization"}:
|
|
continue
|
|
headers[key] = value
|
|
return headers
|
|
|
|
|
|
def decode_response_text(response: requests.Response) -> str:
|
|
content_type = response.headers.get("content-type", "")
|
|
candidates = [
|
|
response.encoding,
|
|
charset_from_content_type(content_type),
|
|
"utf-8",
|
|
"gb18030",
|
|
"gbk",
|
|
"gb2312",
|
|
]
|
|
raw = response.content[:4_000_000]
|
|
for candidate in candidates:
|
|
if not candidate:
|
|
continue
|
|
try:
|
|
return raw.decode(candidate, errors="replace")
|
|
except LookupError:
|
|
continue
|
|
return raw.decode("utf-8", errors="replace")
|
|
|
|
|
|
def select_list(body: str, rule: str | None, base_url: str) -> list[str]:
|
|
if not rule:
|
|
return []
|
|
for part in split_rule(rule, "||"):
|
|
part = strip_replace(part)
|
|
if not part:
|
|
continue
|
|
if part.lower().startswith("regex:"):
|
|
pattern = part.split(":", 1)[1]
|
|
matches = re.findall(pattern, body, re.I | re.S)
|
|
rows = [clean_text(item[0] if isinstance(item, tuple) else item) for item in matches]
|
|
if rows:
|
|
return [row for row in rows if row]
|
|
if part.startswith("$.") or part.startswith("$["):
|
|
decoded = decode_json_like(body)
|
|
rows = json_path(decoded, part) if decoded is not None else []
|
|
if rows:
|
|
return [json.dumps(row, ensure_ascii=False) if isinstance(row, (dict, list)) else str(row) for row in rows]
|
|
rows = soup_rule_list(body, part, base_url)
|
|
if rows:
|
|
return rows
|
|
return []
|
|
|
|
|
|
def select_one(fragment: str, rule: str | None, base_url: str) -> str:
|
|
rows = select_list(fragment, rule, base_url)
|
|
return rows[0] if rows else ""
|
|
|
|
|
|
def soup_rule_list(body: str, rule: str, base_url: str) -> list[str]:
|
|
tokens = [token.strip() for token in rule.split("@") if token.strip()]
|
|
if not tokens:
|
|
return []
|
|
attr = tokens[-1] if tokens[-1].lower() in {"text", "html", "href", "src", "data-src", "data-original", "alt", "title"} else ""
|
|
selector_tokens = tokens[:-1] if attr else tokens
|
|
selector = " ".join(rule_token_to_css(token) for token in selector_tokens if rule_token_to_css(token))
|
|
if not selector:
|
|
return []
|
|
soup = BeautifulSoup(body, "lxml")
|
|
rows: list[str] = []
|
|
for element in soup.select(selector):
|
|
if attr == "text":
|
|
rows.append(clean_text(element.get_text(" ")))
|
|
elif attr == "html":
|
|
rows.append(str(element))
|
|
elif attr:
|
|
value = element.get(attr)
|
|
if value:
|
|
rows.append(resolve_resource_url(base_url, value) if attr in {"href", "src", "data-src", "data-original"} else clean_text(value))
|
|
else:
|
|
rows.append(str(element))
|
|
return [row for row in rows if row]
|
|
|
|
|
|
def rule_token_to_css(token: str) -> str:
|
|
if token.startswith("class."):
|
|
return "." + ".".join(part for part in token[6:].split(".") if part)
|
|
if token.startswith("id."):
|
|
return "#" + token[3:]
|
|
if token.startswith("tag."):
|
|
return token[4:]
|
|
if token.startswith(".") or token.startswith("#") or token in {"a", "li", "div", "img", "ul", "ol"}:
|
|
return token
|
|
return token if re.match(r"^[a-zA-Z0-9_.# >:-]+$", token) else ""
|
|
|
|
|
|
def fallback_search_cards(
|
|
body: str,
|
|
base_url: str,
|
|
rule: RuleConfig,
|
|
keyword: str,
|
|
limit: int,
|
|
*,
|
|
trust: float,
|
|
) -> list[SearchCandidate]:
|
|
decoded = decode_json_like(body)
|
|
rows: list[SearchCandidate] = []
|
|
if decoded is not None:
|
|
rows = fallback_json_candidates(decoded, base_url, rule, keyword, trust)
|
|
if not rows:
|
|
rows = fallback_html_candidates(body, base_url, rule, keyword, trust)
|
|
return sorted(rows, key=lambda item: item.score, reverse=True)[:limit]
|
|
|
|
|
|
def fallback_json_candidates(
|
|
decoded: Any,
|
|
base_url: str,
|
|
rule: RuleConfig,
|
|
keyword: str,
|
|
trust: float,
|
|
) -> list[SearchCandidate]:
|
|
rows: list[SearchCandidate] = []
|
|
for node in walk_dicts(decoded):
|
|
title = first_value(node, "title", "name", "bookName", "book_name", "novelName", "comicName", "comic_name")
|
|
if not title:
|
|
continue
|
|
score = title_match_score(title, keyword)
|
|
if score < 0.38:
|
|
continue
|
|
detail = first_value(node, "url", "href", "link", "bookUrl", "book_url", "comicUrl", "comic_url", "detailUrl", "detail_url", "path")
|
|
if not detail:
|
|
detail = infer_detail_path(
|
|
node,
|
|
rule.resource_type,
|
|
first_value(node, "id", "book_id", "bookId", "comic_id", "comicId", "bid", "cid"),
|
|
)
|
|
if not detail:
|
|
continue
|
|
rows.append(
|
|
SearchCandidate(
|
|
source_name=rule.name,
|
|
source_url=rule.base_url,
|
|
title=title,
|
|
author=empty_to_none(first_value(node, "author", "bookAuthor", "writer", "nickname", "userName", "user_name")),
|
|
cover=maybe_url(base_url, first_value(node, "cover", "coverUrl", "cover_url", "img", "image", "pic", "pic_url")),
|
|
url=resolve_resource_url(base_url, detail),
|
|
resource_type=rule.resource_type,
|
|
score=(score * 0.75) + (trust * 0.25),
|
|
matched_rule=f"{rule.rule_id}:json-fallback",
|
|
)
|
|
)
|
|
return rows
|
|
|
|
|
|
def fallback_html_candidates(body: str, base_url: str, rule: RuleConfig, keyword: str, trust: float) -> list[SearchCandidate]:
|
|
soup = BeautifulSoup(body, "lxml")
|
|
rows: list[SearchCandidate] = []
|
|
for anchor in soup.select("a[href]"):
|
|
href = anchor.get("href") or ""
|
|
text = clean_text(anchor.get("title") or anchor.get("alt") or anchor.get_text(" "))
|
|
image = first_image_url(anchor, base_url)
|
|
if not text and image:
|
|
text = clean_text(anchor.find("img").get("alt", "") if anchor.find("img") else "")
|
|
score = title_match_score(text, keyword)
|
|
if not href or score < 0.38:
|
|
continue
|
|
if is_noise_link(text, href):
|
|
continue
|
|
rows.append(
|
|
SearchCandidate(
|
|
source_name=rule.name,
|
|
source_url=rule.base_url,
|
|
title=text,
|
|
author=None,
|
|
cover=image,
|
|
url=resolve_resource_url(base_url, href),
|
|
resource_type=rule.resource_type,
|
|
score=(score * 0.75) + (trust * 0.25),
|
|
matched_rule=f"{rule.rule_id}:html-fallback",
|
|
)
|
|
)
|
|
return rows
|
|
|
|
|
|
def extract_main_text(soup: BeautifulSoup) -> str:
|
|
for selector in ("#content", ".content", ".chapter-content", ".read-content", "article", "main"):
|
|
element = soup.select_one(selector)
|
|
if element:
|
|
text = normalize_paragraphs(element.get_text("\n"))
|
|
if len(text) > 80:
|
|
return text
|
|
best = ""
|
|
for element in soup.find_all(["article", "main", "div", "section"]):
|
|
text = normalize_paragraphs(element.get_text("\n"))
|
|
if text_density(text) > text_density(best):
|
|
best = text
|
|
return best
|
|
|
|
|
|
def extract_chapter_links(soup: BeautifulSoup, base_url: str, *, prefer_images: bool) -> list[ParsedChapter]:
|
|
chapters: list[ParsedChapter] = []
|
|
patterns = (
|
|
r"(第\s*\d+\s*[章节话回卷]|chapter\s*\d+|下一章|上一章|目录)"
|
|
if not prefer_images
|
|
else r"(第\s*\d+\s*[话回卷]|chapter\s*\d+|episode\s*\d+|下一话|上一话|目录)"
|
|
)
|
|
for anchor in soup.select("a[href]"):
|
|
name = clean_text(anchor.get_text(" ") or anchor.get("title", ""))
|
|
href = anchor.get("href") or ""
|
|
if not name or not href:
|
|
continue
|
|
if re.search(patterns, name, re.I) or re.search(r"(chapter|read|episode|manga|comic)", href, re.I):
|
|
chapters.append(ParsedChapter(name=name[:120], url=resolve_resource_url(base_url, href)))
|
|
return dedupe_chapters(chapters)
|
|
|
|
|
|
def extract_comic_images(soup: BeautifulSoup, base_url: str) -> list[str]:
|
|
rows: list[str] = []
|
|
for image in soup.select("img"):
|
|
value = ""
|
|
for attr in ("data-src", "data-original", "data-lazy-src", "data-url", "src", "data-srcset", "srcset"):
|
|
raw = image.get(attr)
|
|
if raw:
|
|
value = split_srcset(raw)[0] if attr.endswith("srcset") else raw
|
|
break
|
|
if not value:
|
|
continue
|
|
url = resolve_resource_url(base_url, value)
|
|
if is_valid_comic_image(url, image):
|
|
rows.append(url)
|
|
for match in re.finditer(r"https?://[^\"']+\.(?:jpg|jpeg|png|webp|avif)(?:\?[^\"']*)?", str(soup), re.I):
|
|
url = match.group(0)
|
|
if is_valid_comic_image(url, None):
|
|
rows.append(url)
|
|
for raw in extract_script_image_candidates(str(soup)):
|
|
try:
|
|
url = resolve_resource_url(base_url, raw)
|
|
except SearchEngineError:
|
|
continue
|
|
if is_valid_comic_image(url, None):
|
|
rows.append(url)
|
|
return dedupe_strings(rows)
|
|
|
|
|
|
def split_srcset(value: str) -> list[str]:
|
|
urls: list[str] = []
|
|
for part in value.split(","):
|
|
url = part.strip().split()[0] if part.strip() else ""
|
|
if url:
|
|
urls.append(url)
|
|
return urls
|
|
|
|
|
|
def extract_script_image_candidates(html: str) -> list[str]:
|
|
text = html.replace("\\/", "/")
|
|
rows: list[str] = []
|
|
for match in re.finditer(r'''["']((?:(?:https?:)?//|/)[^"']+\.(?:jpg|jpeg|png|webp|avif)(?:\?[^"']*)?)["']''', text, re.I):
|
|
rows.append(match.group(1))
|
|
for assignment in re.finditer(
|
|
r"(?:var|let|const)?\s*(?:chapterImages|imageList|imgList|pageImgs|pics|photosr|newImgs|img_data)\s*=\s*(\[[\s\S]*?\])\s*;?",
|
|
text,
|
|
re.I,
|
|
):
|
|
try:
|
|
decoded = json.loads(assignment.group(1))
|
|
except Exception:
|
|
continue
|
|
collect_image_strings(decoded, rows)
|
|
return dedupe_strings(rows)
|
|
|
|
|
|
def collect_image_strings(node: Any, rows: list[str]) -> None:
|
|
if isinstance(node, str):
|
|
text = node.strip().replace("\\/", "/")
|
|
if re.search(r"\.(jpg|jpeg|png|webp|avif)(?:[?#].*)?$", text, re.I):
|
|
rows.append(text)
|
|
elif isinstance(node, dict):
|
|
for value in node.values():
|
|
collect_image_strings(value, rows)
|
|
elif isinstance(node, list):
|
|
for value in node:
|
|
collect_image_strings(value, rows)
|
|
|
|
|
|
def is_valid_comic_image(url: str, image: Any | None) -> bool:
|
|
lower = url.lower()
|
|
if not re.search(r"\.(jpg|jpeg|png|webp|avif)(?:[?#].*)?$", lower):
|
|
return False
|
|
if any(marker in lower for marker in ("logo", "avatar", "icon", "blank", "placeholder", "loading")):
|
|
return False
|
|
if image is not None:
|
|
width = int_or_zero(image.get("width"))
|
|
height = int_or_zero(image.get("height"))
|
|
if width and height and (width < 100 or height < 100):
|
|
return False
|
|
return True
|
|
|
|
|
|
def infer_detail_path(node: dict[str, Any], resource_type: str, raw_id: str) -> str:
|
|
value = clean_text(raw_id)
|
|
if not value or not re.match(r"^[\w-]+$", value):
|
|
return ""
|
|
if resource_type == "manga":
|
|
return first_value(node, "path") or f"/comic/{value}/"
|
|
return first_value(node, "path") or f"/book/{value}/"
|
|
|
|
|
|
def title_match_score(title: str, keyword: str) -> float:
|
|
title_norm = strip_noise_words(normalize_text(title)).lower()
|
|
keyword_norm = strip_noise_words(normalize_text(keyword)).lower()
|
|
if not title_norm or not keyword_norm:
|
|
return 0.0
|
|
if title_norm == keyword_norm:
|
|
return 1.0
|
|
contains = 0.78 if keyword_norm in title_norm else 0.0
|
|
tokens = [token for token in re.split(r"\s+", keyword_norm) if token]
|
|
coverage = sum(1 for token in tokens if token in title_norm) / len(tokens) if tokens else 0.0
|
|
distance = levenshtein(title_norm[:80], keyword_norm[:80])
|
|
max_len = max(len(title_norm[:80]), len(keyword_norm[:80]), 1)
|
|
fuzzy = 1.0 - min(distance / max_len, 1.0)
|
|
return max(contains, coverage * 0.82, fuzzy * 0.72)
|
|
|
|
|
|
def levenshtein(left: str, right: str) -> int:
|
|
if left == right:
|
|
return 0
|
|
if len(left) < len(right):
|
|
left, right = right, left
|
|
previous = list(range(len(right) + 1))
|
|
for i, left_char in enumerate(left, 1):
|
|
current = [i]
|
|
for j, right_char in enumerate(right, 1):
|
|
insert = current[j - 1] + 1
|
|
delete = previous[j] + 1
|
|
replace = previous[j - 1] + (left_char != right_char)
|
|
current.append(min(insert, delete, replace))
|
|
previous = current
|
|
return previous[-1]
|
|
|
|
|
|
def strip_noise_words(value: str) -> str:
|
|
return re.sub(r"(最新章节|免费阅读|全文阅读|漫画全集|在线阅读|无弹窗|笔趣阁|手机版)", " ", value)
|
|
|
|
|
|
def decode_json_like(body: str) -> Any:
|
|
text = body.lstrip("\ufeff \t\r\n")
|
|
match = re.match(r"^[a-zA-Z_$][\w$]*\s*\((.*)\)\s*;?\s*$", text, re.S)
|
|
if match:
|
|
text = match.group(1)
|
|
try:
|
|
return json.loads(text)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def json_path(decoded: Any, rule: str) -> list[Any]:
|
|
if decoded is None:
|
|
return []
|
|
path = re.sub(r"^\$\.?", "", rule.strip())
|
|
if path in {"", "*", "[*]"}:
|
|
return decoded if isinstance(decoded, list) else [decoded]
|
|
current = [decoded]
|
|
for segment in [part for part in path.split(".") if part]:
|
|
next_values: list[Any] = []
|
|
star = segment.endswith("[*]")
|
|
key = segment[:-3] if star else segment
|
|
for value in current:
|
|
child = value.get(key) if isinstance(value, dict) and key else value
|
|
if star and isinstance(child, list):
|
|
next_values.extend(child)
|
|
elif child is not None:
|
|
next_values.append(child)
|
|
current = next_values
|
|
return current
|
|
|
|
|
|
def extract_page_title(soup: BeautifulSoup) -> str:
|
|
for selector in ("h1", ".book-title", ".comic-title", ".title"):
|
|
element = soup.select_one(selector)
|
|
if element:
|
|
text = clean_text(element.get_text(" "))
|
|
if text:
|
|
return text
|
|
return clean_text(first_meta_content(soup, "og:title") or (soup.title.get_text(" ") if soup.title else ""))
|
|
|
|
|
|
def first_meta_content(soup: BeautifulSoup, name: str) -> str | None:
|
|
for selector in (f'meta[property="{name}"]', f'meta[name="{name}"]'):
|
|
element = soup.select_one(selector)
|
|
if element and element.get("content"):
|
|
return clean_text(element["content"])
|
|
return None
|
|
|
|
|
|
def first_image_url(node: Any, base_url: str) -> str | None:
|
|
image = node.find("img") if hasattr(node, "find") else None
|
|
if not image:
|
|
return None
|
|
for attr in ("data-src", "data-original", "data-lazy-src", "src"):
|
|
value = image.get(attr)
|
|
if value:
|
|
return resolve_resource_url(base_url, value)
|
|
return None
|
|
|
|
|
|
def text_density(text: str) -> float:
|
|
normalized = normalize_paragraphs(text)
|
|
if not normalized:
|
|
return 0.0
|
|
punctuation = len(re.findall(r"[,。!?,.!?]", normalized))
|
|
return len(normalized) + punctuation * 8
|
|
|
|
|
|
def fallback_decode_raw(raw: bytes, content_type: str = "") -> str:
|
|
for candidate in (charset_from_content_type(content_type), "utf-8", "gb18030", "gbk", "gb2312"):
|
|
if not candidate:
|
|
continue
|
|
try:
|
|
return raw.decode(candidate, errors="replace")
|
|
except LookupError:
|
|
continue
|
|
return raw.decode("utf-8", errors="replace")
|
|
|
|
|
|
def parse_cookie_header(value: str) -> dict[str, str]:
|
|
cookies: dict[str, str] = {}
|
|
for part in value.split(";"):
|
|
key, _, raw = part.partition("=")
|
|
key = key.strip()
|
|
if key:
|
|
cookies[key] = raw.strip()
|
|
return cookies
|
|
|
|
|
|
def parse_form_body(value: str) -> dict[str, str] | None:
|
|
if not value:
|
|
return None
|
|
rows: dict[str, str] = {}
|
|
for part in value.split("&"):
|
|
key, _, raw = part.partition("=")
|
|
if key:
|
|
rows[key] = raw
|
|
return rows or None
|
|
|
|
|
|
def dedupe_candidates(rows: list[SearchCandidate]) -> list[SearchCandidate]:
|
|
seen: set[str] = set()
|
|
output: list[SearchCandidate] = []
|
|
for row in sorted(rows, key=lambda item: item.score, reverse=True):
|
|
key = row.url
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
output.append(row)
|
|
return output
|
|
|
|
|
|
def dedupe_chapters(rows: list[ParsedChapter]) -> list[ParsedChapter]:
|
|
seen: set[str] = set()
|
|
output: list[ParsedChapter] = []
|
|
for row in rows:
|
|
key = row.url or row.name
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
output.append(row)
|
|
return output
|
|
|
|
|
|
def dedupe_strings(rows: list[str]) -> list[str]:
|
|
seen: set[str] = set()
|
|
output: list[str] = []
|
|
for row in rows:
|
|
if row in seen:
|
|
continue
|
|
seen.add(row)
|
|
output.append(row)
|
|
return output
|
|
|
|
|
|
def walk_dicts(node: Any):
|
|
if isinstance(node, dict):
|
|
yield node
|
|
for value in node.values():
|
|
yield from walk_dicts(value)
|
|
elif isinstance(node, list):
|
|
for item in node:
|
|
yield from walk_dicts(item)
|
|
|
|
|
|
def first_value(node: dict[str, Any], *keys: str) -> str:
|
|
for key in keys:
|
|
value = node.get(key)
|
|
if value is not None:
|
|
text = clean_text(str(value))
|
|
if text and text.lower() != "null":
|
|
return text
|
|
return ""
|
|
|
|
|
|
def first_string(mapping: dict[str, Any], *keys: str) -> str | None:
|
|
for key in keys:
|
|
value = string_value(mapping.get(key))
|
|
if value:
|
|
return value
|
|
return None
|
|
|
|
|
|
def source_type(source: dict[str, Any]) -> int:
|
|
value = source.get("bookSourceType", source.get("sourceType", source.get("type", 0)))
|
|
try:
|
|
numeric = int(value)
|
|
except Exception:
|
|
numeric = 0
|
|
text = json.dumps(source, ensure_ascii=False).lower()
|
|
manga_markers = ("comiclist", "comicname", "comicurl", "chapterimages", "manga", "manhua", "漫画")
|
|
if numeric == 0 and any(marker in text for marker in manga_markers):
|
|
return 2
|
|
return numeric
|
|
|
|
|
|
def split_rule(rule: str, sep: str) -> list[str]:
|
|
return [item.strip() for item in rule.split(sep) if item.strip()]
|
|
|
|
|
|
def strip_replace(rule: str) -> str:
|
|
return re.split(r"##|###", rule, maxsplit=1)[0].strip()
|
|
|
|
|
|
def string_value(value: Any) -> str | None:
|
|
if value is None:
|
|
return None
|
|
text = str(value).strip()
|
|
return text or None
|
|
|
|
|
|
def empty_to_none(value: str | None) -> str | None:
|
|
text = clean_text(value or "")
|
|
return text or None
|
|
|
|
|
|
def maybe_url(base_url: str, value: str | None) -> str | None:
|
|
text = clean_text(value or "")
|
|
if not text or text.lower() == "null":
|
|
return None
|
|
return resolve_resource_url(base_url, text)
|
|
|
|
|
|
def clean_text(value: str) -> str:
|
|
return unescape(re.sub(r"\s+", " ", value or "")).strip()
|
|
|
|
|
|
def normalize_text(value: str) -> str:
|
|
return clean_text(value).replace("\u3000", " ")
|
|
|
|
|
|
def normalize_paragraphs(value: str) -> str:
|
|
lines = [clean_text(line) for line in re.split(r"[\r\n]+", value or "")]
|
|
return "\n".join(line for line in lines if line)
|
|
|
|
|
|
def charset_from_content_type(value: str) -> str | None:
|
|
match = re.search(r"charset=([^;]+)", value or "", re.I)
|
|
return match.group(1).strip("\"'") if match else None
|
|
|
|
|
|
def int_or_zero(value: Any) -> int:
|
|
try:
|
|
return int(str(value))
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def is_noise_link(title: str, href: str) -> bool:
|
|
text = f"{title} {href}".lower()
|
|
return any(marker in text for marker in ("javascript:", "login", "register", "广告", "排行", "history"))
|
|
|
|
|
|
def _safe_error(exc: Exception) -> str:
|
|
text = str(exc)
|
|
text = re.sub(r"(token|cookie|authorization)=([^&\s]+)", r"\1=***", text, flags=re.I)
|
|
return text[:240]
|