Files
SakuraHub/sakura_cloud/services/online_search.py
T
2026-11-16 20:45:22 +08:00

538 lines
19 KiB
Python

from __future__ import annotations
import ipaddress
import json
import re
import socket
from dataclasses import dataclass
from html import unescape
from html.parser import HTMLParser
from typing import Any
from urllib.parse import quote, urljoin, urlparse
from urllib.request import Request, urlopen
from sakura_cloud.services import online_search_engine
from sakura_cloud.services.online_search_engine import (
OnlineSearchEngine,
SearchEngineError,
assert_public_url,
build_search_url,
parse_comic,
parse_novel,
resolve_resource_url,
search_resource,
source_type,
)
class OnlineSearchError(RuntimeError):
pass
@dataclass(frozen=True)
class OnlineSearchItem:
source_name: str
source_url: str
title: str
author: str | None
cover_url: str | None
detail_url: str
kind: str
def parse_source_payload(payload: Any) -> list[dict[str, Any]]:
if isinstance(payload, str):
payload = json.loads(payload)
if isinstance(payload, list):
rows = payload
elif isinstance(payload, dict):
for key in ("sources", "bookSources", "data", "list", "items", "results"):
value = payload.get(key)
if isinstance(value, list):
rows = value
break
else:
rows = [payload]
else:
rows = []
return [dict(item) for item in rows if isinstance(item, dict)]
def search_sources(
sources: list[dict[str, Any]],
keyword: str,
*,
mode: str,
limit_sources: int = 30,
per_source_limit: int = 20,
) -> list[OnlineSearchItem]:
results: list[OnlineSearchItem] = []
expected_type = 2 if mode == "manga" else 0
engine = OnlineSearchEngine(timeout=10.0)
for source in sources[:limit_sources]:
if _source_type(source) != expected_type:
continue
try:
online_search_engine.assert_public_url = _assert_public_url
engine._request_text = ( # type: ignore[method-assign]
lambda rule, url, data=None, _source=source: _fetch_text(url, rule.base_url, _source)
)
candidates = engine.search_legado_source(source, keyword, mode=mode, limit=per_source_limit)
results.extend(
OnlineSearchItem(
source_name=item.source_name,
source_url=item.source_url,
title=item.title,
author=item.author,
cover_url=item.cover,
detail_url=item.url,
kind=mode,
)
for item in candidates
)
except SearchEngineError:
continue
except Exception:
continue
return results
def _search_one(source: dict[str, Any], keyword: str, *, mode: str, limit: int) -> list[OnlineSearchItem]:
source_name = _string(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or "未命名源"
source_url = _string(source.get("bookSourceUrl") or source.get("sourceUrl") or source.get("url")) or ""
search_url = _string(source.get("searchUrl"))
rule_search = source.get("ruleSearch") if isinstance(source.get("ruleSearch"), dict) else {}
if not search_url:
search_url = _string(rule_search.get("url") or rule_search.get("searchUrl")) or ""
if not source_url or not search_url:
return []
final_url = _resolve_url(source_url, _build_search_url(search_url, keyword))
_assert_public_url(final_url)
body = _fetch_text(final_url, source_url, source)
list_rule = _first_string(
rule_search,
"bookList",
"comicList",
"list",
"items",
"data",
"results",
)
if list_rule:
parsed = _parse_by_rules(body, final_url, source, source_name, source_url, rule_search, mode, limit)
if parsed:
return parsed
if mode == "manga":
return _fallback_manga(body, final_url, source_name, source_url, keyword, limit)
return _fallback_novel(body, final_url, source_name, source_url, keyword, limit)
def _parse_by_rules(
body: str,
base_url: str,
source: dict[str, Any],
source_name: str,
source_url: str,
rule_search: dict[str, Any],
mode: str,
limit: int,
) -> list[OnlineSearchItem]:
fragments = _select_list(body, _first_string(rule_search, "bookList", "comicList", "list", "items", "data", "results"))
results: list[OnlineSearchItem] = []
seen: set[str] = set()
for fragment in fragments:
title = _select_one(fragment, _first_string(rule_search, "name", "bookName", "comicName", "title"))
detail = _select_one(fragment, _first_string(rule_search, "bookUrl", "comicUrl", "detailUrl", "url", "href"))
if not title or not detail:
continue
detail_url = _resolve_url(base_url, detail)
if detail_url in seen:
continue
seen.add(detail_url)
results.append(
OnlineSearchItem(
source_name=source_name,
source_url=source_url,
title=title,
author=_select_one(fragment, _first_string(rule_search, "author", "bookAuthor", "comicAuthor", "writer")) or None,
cover_url=_maybe_url(base_url, _select_one(fragment, _first_string(rule_search, "coverUrl", "cover", "bookCover", "comicCover", "img", "image"))),
detail_url=detail_url,
kind=mode,
)
)
if len(results) >= limit:
break
return results
def _fallback_manga(
body: str,
base_url: str,
source_name: str,
source_url: str,
keyword: str,
limit: int,
) -> list[OnlineSearchItem]:
decoded = _decode_json_like(body)
results: list[OnlineSearchItem] = []
seen: set[str] = set()
if decoded is not None:
for node in _walk_dicts(decoded):
title = _first_value(node, "title", "name", "comicName", "comic_name", "topic_title", "topicTitle", "bookName")
if not _matches(title, keyword):
continue
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "comicUrl", "comic_url", "detailUrl", "detail_url", "path")
if not detail:
detail = _first_value(node, "id", "book_id", "comic_id", "topic_id")
if not detail:
continue
detail_url = _resolve_url(base_url, detail)
if detail_url in seen:
continue
seen.add(detail_url)
results.append(
OnlineSearchItem(
source_name=source_name,
source_url=source_url,
title=title,
author=_first_value(node, "author", "writer", "nickname", "user_name", "userName") or None,
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "image", "imageUrl", "image_url", "pic", "pic_url", "vertical_image_url")),
detail_url=detail_url,
kind="manga",
)
)
if len(results) >= limit:
return results
if results:
return results
return _fallback_html_cards(body, base_url, source_name, source_url, keyword, "manga", limit)
def _fallback_novel(
body: str,
base_url: str,
source_name: str,
source_url: str,
keyword: str,
limit: int,
) -> list[OnlineSearchItem]:
decoded = _decode_json_like(body)
results: list[OnlineSearchItem] = []
if decoded is not None:
for node in _walk_dicts(decoded):
title = _first_value(node, "title", "name", "bookName", "book_name", "novelName")
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "detailUrl", "detail_url")
if not _matches(title, keyword) or not detail:
continue
results.append(
OnlineSearchItem(
source_name=source_name,
source_url=source_url,
title=title,
author=_first_value(node, "author", "bookAuthor", "writer") or None,
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "img", "image")),
detail_url=_resolve_url(base_url, detail),
kind="novel",
)
)
if len(results) >= limit:
return results
return results or _fallback_html_cards(body, base_url, source_name, source_url, keyword, "novel", limit)
def _fallback_html_cards(
body: str,
base_url: str,
source_name: str,
source_url: str,
keyword: str,
kind: str,
limit: int,
) -> list[OnlineSearchItem]:
parser = _AnchorParser()
parser.feed(body)
results: list[OnlineSearchItem] = []
seen: set[str] = set()
for anchor in parser.anchors:
title = _clean(anchor.get("title") or anchor.get("alt") or anchor.get("text") or "")
href = _clean(anchor.get("href") or "")
if not href or not _matches(title, keyword):
continue
detail_url = _resolve_url(base_url, href)
if detail_url in seen:
continue
seen.add(detail_url)
results.append(
OnlineSearchItem(
source_name=source_name,
source_url=source_url,
title=title,
author=None,
cover_url=_maybe_url(base_url, anchor.get("img") or ""),
detail_url=detail_url,
kind=kind,
)
)
if len(results) >= limit:
break
return results
def _select_list(body: str, rule: str | None) -> list[str]:
if not rule:
return []
for part in _split_rule(rule, "||"):
if part.lower().startswith("regex:"):
pattern = _strip_replace(part.split(":", 1)[1])
matches = re.findall(pattern, body, re.I | re.S)
output = [_clean(item[0] if isinstance(item, tuple) else item) for item in matches]
if output:
return [item for item in output if item]
if part.startswith("$.") or part.startswith("$["):
decoded = _decode_json_like(body)
values = _json_path(decoded, part) if decoded is not None else []
output = [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in values]
if output:
return output
matches = _html_rule_list(body, part)
if matches:
return matches
return []
def _select_one(fragment: str, rule: str | None) -> str:
values = _select_list(fragment, rule)
return values[0] if values else ""
def _html_rule_list(body: str, rule: str) -> list[str]:
rule = _strip_replace(rule.strip())
if not rule:
return []
tokens = [item.strip() for item in rule.split("@") if item.strip()]
selector = tokens[0] if tokens else rule
attr = tokens[-1] if tokens and tokens[-1] not in {selector, "text", "html"} else ""
if selector.startswith("class."):
classes = [item for item in selector[6:].split() if item]
class_checks = "".join(rf'(?=[^"\']*{re.escape(item)})' for item in classes[:3])
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*class=["\']{class_checks}[^"\']*["\'][^>]*>.*?</(?P=tag)>'
elif selector.startswith("id."):
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*id=["\']{re.escape(selector[3:])}["\'][^>]*>.*?</(?P=tag)>'
elif selector.startswith("tag."):
tag = re.escape(selector[4:])
pattern = rf'<{tag}\b[^>]*>.*?</{tag}>'
elif selector.startswith("$.") or selector.startswith("$["):
decoded = _decode_json_like(body)
return [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in _json_path(decoded, selector)]
else:
pattern = rf'<a\b[^>]*>.*?</a>' if "href" in tokens else ""
if not pattern:
return []
blocks = [m.group(0) for m in re.finditer(pattern, body, re.I | re.S)]
if attr and attr not in {"text", "html"}:
return [_clean(m.group(1)) for block in blocks for m in [re.search(rf'{re.escape(attr)}=["\']([^"\']+)["\']', block, re.I)] if m]
if "text" in tokens:
return [_html_text(block) for block in blocks]
return blocks
def _json_path(decoded: Any, rule: str) -> list[Any]:
if decoded is None:
return []
path = re.sub(r"^\$\.?", "", rule.strip())
if path in {"", "*", "[*]"}:
return decoded if isinstance(decoded, list) else [decoded]
current = [decoded]
for segment in [item for item in path.split(".") if item]:
next_values: list[Any] = []
star = segment.endswith("[*]")
key = segment[:-3] if star else segment
for value in current:
child = value.get(key) if isinstance(value, dict) and key else value
if star and isinstance(child, list):
next_values.extend(child)
elif child is not None:
next_values.append(child)
current = next_values
return current
class _AnchorParser(HTMLParser):
def __init__(self) -> None:
super().__init__()
self.anchors: list[dict[str, str]] = []
self._current: dict[str, str] | None = None
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attr = {key: value or "" for key, value in attrs}
if tag.lower() == "a" and attr.get("href"):
self._current = {"href": attr.get("href", ""), "title": attr.get("title", ""), "text": ""}
elif tag.lower() == "img" and self._current is not None:
for key in ("data-original", "data-src", "src"):
if attr.get(key):
self._current["img"] = attr[key]
break
if attr.get("alt"):
self._current["alt"] = attr["alt"]
def handle_data(self, data: str) -> None:
if self._current is not None:
self._current["text"] = f'{self._current.get("text", "")} {data}'
def handle_endtag(self, tag: str) -> None:
if tag.lower() == "a" and self._current is not None:
self.anchors.append(self._current)
self._current = None
def _fetch_text(url: str, referer: str, source: dict[str, Any]) -> str:
headers = {
"User-Agent": _string(source.get("httpUserAgent")) or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7",
"Accept-Language": "zh-CN,zh;q=0.9",
"Referer": referer,
}
extra = source.get("headers") or source.get("header")
if isinstance(extra, dict):
for key, value in extra.items():
if str(key).lower() in {"cookie", "authorization"}:
continue
headers[str(key)] = str(value)
with urlopen(Request(url, headers=headers), timeout=12) as response:
raw = response.read(2_000_000)
content_type = response.headers.get("content-type", "")
charset = _string(source.get("charset") or source.get("encoding")) or _charset_from_content_type(content_type) or "utf-8"
for candidate in (charset, "utf-8", "gb18030", "gbk", "gb2312"):
try:
return raw.decode(candidate, errors="replace")
except LookupError:
continue
return raw.decode("utf-8", errors="replace")
def _assert_public_url(url: str) -> None:
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise OnlineSearchError("Only http/https source URLs are allowed")
infos = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM)
for item in infos:
ip = ipaddress.ip_address(item[4][0])
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved:
raise OnlineSearchError("Private network source URLs are not allowed")
def _build_search_url(template: str, keyword: str) -> str:
encoded = quote(keyword, safe="")
value = template.strip()
for token in ("{{key}}", "{{keyword}}", "{key}", "{keyword}", "<key>", "<keyword>", "searchKey"):
value = value.replace(token, encoded)
value = value.replace("%s", encoded, 1)
return value.replace("{{page}}", "1").replace("{page}", "1").replace("<page>", "1")
def _resolve_url(base: str, value: str) -> str:
return urljoin(base, value.strip())
def _maybe_url(base: str, value: str | None) -> str | None:
text = _clean(value or "")
if not text or text.lower() == "null":
return None
return _resolve_url(base, text)
def _decode_json_like(body: str) -> Any:
text = body.lstrip("\ufeff \t\r\n")
match = re.match(r"^[a-zA-Z_$][\w$]*\s*\((.*)\)\s*;?\s*$", text, re.S)
if match:
text = match.group(1)
try:
return json.loads(text)
except Exception:
return None
def _walk_dicts(node: Any):
if isinstance(node, dict):
yield node
for value in node.values():
yield from _walk_dicts(value)
elif isinstance(node, list):
for item in node:
yield from _walk_dicts(item)
def _first_value(node: dict[str, Any], *keys: str) -> str:
for key in keys:
value = node.get(key)
if value is not None:
text = _clean(str(value))
if text and text.lower() != "null":
return text
return ""
def _first_string(mapping: dict[str, Any], *keys: str) -> str | None:
for key in keys:
value = _string(mapping.get(key))
if value:
return value
return None
def _source_type(source: dict[str, Any]) -> int:
value = source.get("bookSourceType", source.get("sourceType", source.get("type", 0)))
try:
numeric = int(value)
except Exception:
numeric = 0
text = json.dumps(source, ensure_ascii=False).lower()
manga_markers = (
"comiclist",
"comicname",
"comicurl",
"imageStyle".lower(),
"chapterImages".lower(),
"manga",
"manhua",
"漫画",
)
if numeric == 0 and any(marker in text for marker in manga_markers):
return 2
return numeric
def _split_rule(rule: str, sep: str) -> list[str]:
return [item.strip() for item in rule.split(sep) if item.strip()]
def _strip_replace(rule: str) -> str:
return re.split(r"##|###", rule, maxsplit=1)[0].strip()
def _string(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def _matches(title: str, keyword: str) -> bool:
title = _clean(title).lower()
keyword = _clean(keyword).lower()
return bool(title) and (not keyword or keyword in title)
def _clean(value: str) -> str:
return unescape(re.sub(r"\s+", " ", value or "")).strip()
def _html_text(value: str) -> str:
return _clean(re.sub(r"<[^>]+>", " ", value))
def _charset_from_content_type(value: str) -> str | None:
match = re.search(r"charset=([^;]+)", value or "", re.I)
return match.group(1).strip("\"'") if match else None