538 lines
19 KiB
Python
538 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
import json
|
|
import re
|
|
import socket
|
|
from dataclasses import dataclass
|
|
from html import unescape
|
|
from html.parser import HTMLParser
|
|
from typing import Any
|
|
from urllib.parse import quote, urljoin, urlparse
|
|
from urllib.request import Request, urlopen
|
|
|
|
from sakura_cloud.services import online_search_engine
|
|
from sakura_cloud.services.online_search_engine import (
|
|
OnlineSearchEngine,
|
|
SearchEngineError,
|
|
assert_public_url,
|
|
build_search_url,
|
|
parse_comic,
|
|
parse_novel,
|
|
resolve_resource_url,
|
|
search_resource,
|
|
source_type,
|
|
)
|
|
|
|
|
|
class OnlineSearchError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class OnlineSearchItem:
|
|
source_name: str
|
|
source_url: str
|
|
title: str
|
|
author: str | None
|
|
cover_url: str | None
|
|
detail_url: str
|
|
kind: str
|
|
|
|
|
|
def parse_source_payload(payload: Any) -> list[dict[str, Any]]:
|
|
if isinstance(payload, str):
|
|
payload = json.loads(payload)
|
|
if isinstance(payload, list):
|
|
rows = payload
|
|
elif isinstance(payload, dict):
|
|
for key in ("sources", "bookSources", "data", "list", "items", "results"):
|
|
value = payload.get(key)
|
|
if isinstance(value, list):
|
|
rows = value
|
|
break
|
|
else:
|
|
rows = [payload]
|
|
else:
|
|
rows = []
|
|
return [dict(item) for item in rows if isinstance(item, dict)]
|
|
|
|
|
|
def search_sources(
|
|
sources: list[dict[str, Any]],
|
|
keyword: str,
|
|
*,
|
|
mode: str,
|
|
limit_sources: int = 30,
|
|
per_source_limit: int = 20,
|
|
) -> list[OnlineSearchItem]:
|
|
results: list[OnlineSearchItem] = []
|
|
expected_type = 2 if mode == "manga" else 0
|
|
engine = OnlineSearchEngine(timeout=10.0)
|
|
for source in sources[:limit_sources]:
|
|
if _source_type(source) != expected_type:
|
|
continue
|
|
try:
|
|
online_search_engine.assert_public_url = _assert_public_url
|
|
engine._request_text = ( # type: ignore[method-assign]
|
|
lambda rule, url, data=None, _source=source: _fetch_text(url, rule.base_url, _source)
|
|
)
|
|
candidates = engine.search_legado_source(source, keyword, mode=mode, limit=per_source_limit)
|
|
results.extend(
|
|
OnlineSearchItem(
|
|
source_name=item.source_name,
|
|
source_url=item.source_url,
|
|
title=item.title,
|
|
author=item.author,
|
|
cover_url=item.cover,
|
|
detail_url=item.url,
|
|
kind=mode,
|
|
)
|
|
for item in candidates
|
|
)
|
|
except SearchEngineError:
|
|
continue
|
|
except Exception:
|
|
continue
|
|
return results
|
|
|
|
|
|
def _search_one(source: dict[str, Any], keyword: str, *, mode: str, limit: int) -> list[OnlineSearchItem]:
|
|
source_name = _string(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or "未命名源"
|
|
source_url = _string(source.get("bookSourceUrl") or source.get("sourceUrl") or source.get("url")) or ""
|
|
search_url = _string(source.get("searchUrl"))
|
|
rule_search = source.get("ruleSearch") if isinstance(source.get("ruleSearch"), dict) else {}
|
|
if not search_url:
|
|
search_url = _string(rule_search.get("url") or rule_search.get("searchUrl")) or ""
|
|
if not source_url or not search_url:
|
|
return []
|
|
|
|
final_url = _resolve_url(source_url, _build_search_url(search_url, keyword))
|
|
_assert_public_url(final_url)
|
|
body = _fetch_text(final_url, source_url, source)
|
|
list_rule = _first_string(
|
|
rule_search,
|
|
"bookList",
|
|
"comicList",
|
|
"list",
|
|
"items",
|
|
"data",
|
|
"results",
|
|
)
|
|
if list_rule:
|
|
parsed = _parse_by_rules(body, final_url, source, source_name, source_url, rule_search, mode, limit)
|
|
if parsed:
|
|
return parsed
|
|
if mode == "manga":
|
|
return _fallback_manga(body, final_url, source_name, source_url, keyword, limit)
|
|
return _fallback_novel(body, final_url, source_name, source_url, keyword, limit)
|
|
|
|
|
|
def _parse_by_rules(
|
|
body: str,
|
|
base_url: str,
|
|
source: dict[str, Any],
|
|
source_name: str,
|
|
source_url: str,
|
|
rule_search: dict[str, Any],
|
|
mode: str,
|
|
limit: int,
|
|
) -> list[OnlineSearchItem]:
|
|
fragments = _select_list(body, _first_string(rule_search, "bookList", "comicList", "list", "items", "data", "results"))
|
|
results: list[OnlineSearchItem] = []
|
|
seen: set[str] = set()
|
|
for fragment in fragments:
|
|
title = _select_one(fragment, _first_string(rule_search, "name", "bookName", "comicName", "title"))
|
|
detail = _select_one(fragment, _first_string(rule_search, "bookUrl", "comicUrl", "detailUrl", "url", "href"))
|
|
if not title or not detail:
|
|
continue
|
|
detail_url = _resolve_url(base_url, detail)
|
|
if detail_url in seen:
|
|
continue
|
|
seen.add(detail_url)
|
|
results.append(
|
|
OnlineSearchItem(
|
|
source_name=source_name,
|
|
source_url=source_url,
|
|
title=title,
|
|
author=_select_one(fragment, _first_string(rule_search, "author", "bookAuthor", "comicAuthor", "writer")) or None,
|
|
cover_url=_maybe_url(base_url, _select_one(fragment, _first_string(rule_search, "coverUrl", "cover", "bookCover", "comicCover", "img", "image"))),
|
|
detail_url=detail_url,
|
|
kind=mode,
|
|
)
|
|
)
|
|
if len(results) >= limit:
|
|
break
|
|
return results
|
|
|
|
|
|
def _fallback_manga(
|
|
body: str,
|
|
base_url: str,
|
|
source_name: str,
|
|
source_url: str,
|
|
keyword: str,
|
|
limit: int,
|
|
) -> list[OnlineSearchItem]:
|
|
decoded = _decode_json_like(body)
|
|
results: list[OnlineSearchItem] = []
|
|
seen: set[str] = set()
|
|
if decoded is not None:
|
|
for node in _walk_dicts(decoded):
|
|
title = _first_value(node, "title", "name", "comicName", "comic_name", "topic_title", "topicTitle", "bookName")
|
|
if not _matches(title, keyword):
|
|
continue
|
|
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "comicUrl", "comic_url", "detailUrl", "detail_url", "path")
|
|
if not detail:
|
|
detail = _first_value(node, "id", "book_id", "comic_id", "topic_id")
|
|
if not detail:
|
|
continue
|
|
detail_url = _resolve_url(base_url, detail)
|
|
if detail_url in seen:
|
|
continue
|
|
seen.add(detail_url)
|
|
results.append(
|
|
OnlineSearchItem(
|
|
source_name=source_name,
|
|
source_url=source_url,
|
|
title=title,
|
|
author=_first_value(node, "author", "writer", "nickname", "user_name", "userName") or None,
|
|
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "image", "imageUrl", "image_url", "pic", "pic_url", "vertical_image_url")),
|
|
detail_url=detail_url,
|
|
kind="manga",
|
|
)
|
|
)
|
|
if len(results) >= limit:
|
|
return results
|
|
if results:
|
|
return results
|
|
return _fallback_html_cards(body, base_url, source_name, source_url, keyword, "manga", limit)
|
|
|
|
|
|
def _fallback_novel(
|
|
body: str,
|
|
base_url: str,
|
|
source_name: str,
|
|
source_url: str,
|
|
keyword: str,
|
|
limit: int,
|
|
) -> list[OnlineSearchItem]:
|
|
decoded = _decode_json_like(body)
|
|
results: list[OnlineSearchItem] = []
|
|
if decoded is not None:
|
|
for node in _walk_dicts(decoded):
|
|
title = _first_value(node, "title", "name", "bookName", "book_name", "novelName")
|
|
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "detailUrl", "detail_url")
|
|
if not _matches(title, keyword) or not detail:
|
|
continue
|
|
results.append(
|
|
OnlineSearchItem(
|
|
source_name=source_name,
|
|
source_url=source_url,
|
|
title=title,
|
|
author=_first_value(node, "author", "bookAuthor", "writer") or None,
|
|
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "img", "image")),
|
|
detail_url=_resolve_url(base_url, detail),
|
|
kind="novel",
|
|
)
|
|
)
|
|
if len(results) >= limit:
|
|
return results
|
|
return results or _fallback_html_cards(body, base_url, source_name, source_url, keyword, "novel", limit)
|
|
|
|
|
|
def _fallback_html_cards(
|
|
body: str,
|
|
base_url: str,
|
|
source_name: str,
|
|
source_url: str,
|
|
keyword: str,
|
|
kind: str,
|
|
limit: int,
|
|
) -> list[OnlineSearchItem]:
|
|
parser = _AnchorParser()
|
|
parser.feed(body)
|
|
results: list[OnlineSearchItem] = []
|
|
seen: set[str] = set()
|
|
for anchor in parser.anchors:
|
|
title = _clean(anchor.get("title") or anchor.get("alt") or anchor.get("text") or "")
|
|
href = _clean(anchor.get("href") or "")
|
|
if not href or not _matches(title, keyword):
|
|
continue
|
|
detail_url = _resolve_url(base_url, href)
|
|
if detail_url in seen:
|
|
continue
|
|
seen.add(detail_url)
|
|
results.append(
|
|
OnlineSearchItem(
|
|
source_name=source_name,
|
|
source_url=source_url,
|
|
title=title,
|
|
author=None,
|
|
cover_url=_maybe_url(base_url, anchor.get("img") or ""),
|
|
detail_url=detail_url,
|
|
kind=kind,
|
|
)
|
|
)
|
|
if len(results) >= limit:
|
|
break
|
|
return results
|
|
|
|
|
|
def _select_list(body: str, rule: str | None) -> list[str]:
|
|
if not rule:
|
|
return []
|
|
for part in _split_rule(rule, "||"):
|
|
if part.lower().startswith("regex:"):
|
|
pattern = _strip_replace(part.split(":", 1)[1])
|
|
matches = re.findall(pattern, body, re.I | re.S)
|
|
output = [_clean(item[0] if isinstance(item, tuple) else item) for item in matches]
|
|
if output:
|
|
return [item for item in output if item]
|
|
if part.startswith("$.") or part.startswith("$["):
|
|
decoded = _decode_json_like(body)
|
|
values = _json_path(decoded, part) if decoded is not None else []
|
|
output = [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in values]
|
|
if output:
|
|
return output
|
|
matches = _html_rule_list(body, part)
|
|
if matches:
|
|
return matches
|
|
return []
|
|
|
|
|
|
def _select_one(fragment: str, rule: str | None) -> str:
|
|
values = _select_list(fragment, rule)
|
|
return values[0] if values else ""
|
|
|
|
|
|
def _html_rule_list(body: str, rule: str) -> list[str]:
|
|
rule = _strip_replace(rule.strip())
|
|
if not rule:
|
|
return []
|
|
tokens = [item.strip() for item in rule.split("@") if item.strip()]
|
|
selector = tokens[0] if tokens else rule
|
|
attr = tokens[-1] if tokens and tokens[-1] not in {selector, "text", "html"} else ""
|
|
if selector.startswith("class."):
|
|
classes = [item for item in selector[6:].split() if item]
|
|
class_checks = "".join(rf'(?=[^"\']*{re.escape(item)})' for item in classes[:3])
|
|
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*class=["\']{class_checks}[^"\']*["\'][^>]*>.*?</(?P=tag)>'
|
|
elif selector.startswith("id."):
|
|
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*id=["\']{re.escape(selector[3:])}["\'][^>]*>.*?</(?P=tag)>'
|
|
elif selector.startswith("tag."):
|
|
tag = re.escape(selector[4:])
|
|
pattern = rf'<{tag}\b[^>]*>.*?</{tag}>'
|
|
elif selector.startswith("$.") or selector.startswith("$["):
|
|
decoded = _decode_json_like(body)
|
|
return [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in _json_path(decoded, selector)]
|
|
else:
|
|
pattern = rf'<a\b[^>]*>.*?</a>' if "href" in tokens else ""
|
|
if not pattern:
|
|
return []
|
|
blocks = [m.group(0) for m in re.finditer(pattern, body, re.I | re.S)]
|
|
if attr and attr not in {"text", "html"}:
|
|
return [_clean(m.group(1)) for block in blocks for m in [re.search(rf'{re.escape(attr)}=["\']([^"\']+)["\']', block, re.I)] if m]
|
|
if "text" in tokens:
|
|
return [_html_text(block) for block in blocks]
|
|
return blocks
|
|
|
|
|
|
def _json_path(decoded: Any, rule: str) -> list[Any]:
|
|
if decoded is None:
|
|
return []
|
|
path = re.sub(r"^\$\.?", "", rule.strip())
|
|
if path in {"", "*", "[*]"}:
|
|
return decoded if isinstance(decoded, list) else [decoded]
|
|
current = [decoded]
|
|
for segment in [item for item in path.split(".") if item]:
|
|
next_values: list[Any] = []
|
|
star = segment.endswith("[*]")
|
|
key = segment[:-3] if star else segment
|
|
for value in current:
|
|
child = value.get(key) if isinstance(value, dict) and key else value
|
|
if star and isinstance(child, list):
|
|
next_values.extend(child)
|
|
elif child is not None:
|
|
next_values.append(child)
|
|
current = next_values
|
|
return current
|
|
|
|
|
|
class _AnchorParser(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
self.anchors: list[dict[str, str]] = []
|
|
self._current: dict[str, str] | None = None
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
attr = {key: value or "" for key, value in attrs}
|
|
if tag.lower() == "a" and attr.get("href"):
|
|
self._current = {"href": attr.get("href", ""), "title": attr.get("title", ""), "text": ""}
|
|
elif tag.lower() == "img" and self._current is not None:
|
|
for key in ("data-original", "data-src", "src"):
|
|
if attr.get(key):
|
|
self._current["img"] = attr[key]
|
|
break
|
|
if attr.get("alt"):
|
|
self._current["alt"] = attr["alt"]
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if self._current is not None:
|
|
self._current["text"] = f'{self._current.get("text", "")} {data}'
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag.lower() == "a" and self._current is not None:
|
|
self.anchors.append(self._current)
|
|
self._current = None
|
|
|
|
|
|
def _fetch_text(url: str, referer: str, source: dict[str, Any]) -> str:
|
|
headers = {
|
|
"User-Agent": _string(source.get("httpUserAgent")) or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36",
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7",
|
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
|
"Referer": referer,
|
|
}
|
|
extra = source.get("headers") or source.get("header")
|
|
if isinstance(extra, dict):
|
|
for key, value in extra.items():
|
|
if str(key).lower() in {"cookie", "authorization"}:
|
|
continue
|
|
headers[str(key)] = str(value)
|
|
with urlopen(Request(url, headers=headers), timeout=12) as response:
|
|
raw = response.read(2_000_000)
|
|
content_type = response.headers.get("content-type", "")
|
|
charset = _string(source.get("charset") or source.get("encoding")) or _charset_from_content_type(content_type) or "utf-8"
|
|
for candidate in (charset, "utf-8", "gb18030", "gbk", "gb2312"):
|
|
try:
|
|
return raw.decode(candidate, errors="replace")
|
|
except LookupError:
|
|
continue
|
|
return raw.decode("utf-8", errors="replace")
|
|
|
|
|
|
def _assert_public_url(url: str) -> None:
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
raise OnlineSearchError("Only http/https source URLs are allowed")
|
|
infos = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM)
|
|
for item in infos:
|
|
ip = ipaddress.ip_address(item[4][0])
|
|
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved:
|
|
raise OnlineSearchError("Private network source URLs are not allowed")
|
|
|
|
|
|
def _build_search_url(template: str, keyword: str) -> str:
|
|
encoded = quote(keyword, safe="")
|
|
value = template.strip()
|
|
for token in ("{{key}}", "{{keyword}}", "{key}", "{keyword}", "<key>", "<keyword>", "searchKey"):
|
|
value = value.replace(token, encoded)
|
|
value = value.replace("%s", encoded, 1)
|
|
return value.replace("{{page}}", "1").replace("{page}", "1").replace("<page>", "1")
|
|
|
|
|
|
def _resolve_url(base: str, value: str) -> str:
|
|
return urljoin(base, value.strip())
|
|
|
|
|
|
def _maybe_url(base: str, value: str | None) -> str | None:
|
|
text = _clean(value or "")
|
|
if not text or text.lower() == "null":
|
|
return None
|
|
return _resolve_url(base, text)
|
|
|
|
|
|
def _decode_json_like(body: str) -> Any:
|
|
text = body.lstrip("\ufeff \t\r\n")
|
|
match = re.match(r"^[a-zA-Z_$][\w$]*\s*\((.*)\)\s*;?\s*$", text, re.S)
|
|
if match:
|
|
text = match.group(1)
|
|
try:
|
|
return json.loads(text)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _walk_dicts(node: Any):
|
|
if isinstance(node, dict):
|
|
yield node
|
|
for value in node.values():
|
|
yield from _walk_dicts(value)
|
|
elif isinstance(node, list):
|
|
for item in node:
|
|
yield from _walk_dicts(item)
|
|
|
|
|
|
def _first_value(node: dict[str, Any], *keys: str) -> str:
|
|
for key in keys:
|
|
value = node.get(key)
|
|
if value is not None:
|
|
text = _clean(str(value))
|
|
if text and text.lower() != "null":
|
|
return text
|
|
return ""
|
|
|
|
|
|
def _first_string(mapping: dict[str, Any], *keys: str) -> str | None:
|
|
for key in keys:
|
|
value = _string(mapping.get(key))
|
|
if value:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _source_type(source: dict[str, Any]) -> int:
|
|
value = source.get("bookSourceType", source.get("sourceType", source.get("type", 0)))
|
|
try:
|
|
numeric = int(value)
|
|
except Exception:
|
|
numeric = 0
|
|
text = json.dumps(source, ensure_ascii=False).lower()
|
|
manga_markers = (
|
|
"comiclist",
|
|
"comicname",
|
|
"comicurl",
|
|
"imageStyle".lower(),
|
|
"chapterImages".lower(),
|
|
"manga",
|
|
"manhua",
|
|
"漫画",
|
|
)
|
|
if numeric == 0 and any(marker in text for marker in manga_markers):
|
|
return 2
|
|
return numeric
|
|
|
|
|
|
def _split_rule(rule: str, sep: str) -> list[str]:
|
|
return [item.strip() for item in rule.split(sep) if item.strip()]
|
|
|
|
|
|
def _strip_replace(rule: str) -> str:
|
|
return re.split(r"##|###", rule, maxsplit=1)[0].strip()
|
|
|
|
|
|
def _string(value: Any) -> str | None:
|
|
if value is None:
|
|
return None
|
|
text = str(value).strip()
|
|
return text or None
|
|
|
|
|
|
def _matches(title: str, keyword: str) -> bool:
|
|
title = _clean(title).lower()
|
|
keyword = _clean(keyword).lower()
|
|
return bool(title) and (not keyword or keyword in title)
|
|
|
|
|
|
def _clean(value: str) -> str:
|
|
return unescape(re.sub(r"\s+", " ", value or "")).strip()
|
|
|
|
|
|
def _html_text(value: str) -> str:
|
|
return _clean(re.sub(r"<[^>]+>", " ", value))
|
|
|
|
|
|
def _charset_from_content_type(value: str) -> str | None:
|
|
match = re.search(r"charset=([^;]+)", value or "", re.I)
|
|
return match.group(1).strip("\"'") if match else None
|