from __future__ import annotations import ipaddress import json import re import socket from dataclasses import dataclass from html import unescape from html.parser import HTMLParser from typing import Any from urllib.parse import quote, urljoin, urlparse from urllib.request import Request, urlopen from sakura_cloud.services import online_search_engine from sakura_cloud.services.online_search_engine import ( OnlineSearchEngine, SearchEngineError, assert_public_url, build_search_url, parse_comic, parse_novel, resolve_resource_url, search_resource, source_type, ) class OnlineSearchError(RuntimeError): pass @dataclass(frozen=True) class OnlineSearchItem: source_name: str source_url: str title: str author: str | None cover_url: str | None detail_url: str kind: str def parse_source_payload(payload: Any) -> list[dict[str, Any]]: if isinstance(payload, str): payload = json.loads(payload) if isinstance(payload, list): rows = payload elif isinstance(payload, dict): for key in ("sources", "bookSources", "data", "list", "items", "results"): value = payload.get(key) if isinstance(value, list): rows = value break else: rows = [payload] else: rows = [] return [dict(item) for item in rows if isinstance(item, dict)] def search_sources( sources: list[dict[str, Any]], keyword: str, *, mode: str, limit_sources: int = 30, per_source_limit: int = 20, ) -> list[OnlineSearchItem]: results: list[OnlineSearchItem] = [] expected_type = 2 if mode == "manga" else 0 engine = OnlineSearchEngine(timeout=10.0) for source in sources[:limit_sources]: if _source_type(source) != expected_type: continue try: online_search_engine.assert_public_url = _assert_public_url engine._request_text = ( # type: ignore[method-assign] lambda rule, url, data=None, _source=source: _fetch_text(url, rule.base_url, _source) ) candidates = engine.search_legado_source(source, keyword, mode=mode, limit=per_source_limit) results.extend( OnlineSearchItem( source_name=item.source_name, source_url=item.source_url, title=item.title, author=item.author, cover_url=item.cover, detail_url=item.url, kind=mode, ) for item in candidates ) except SearchEngineError: continue except Exception: continue return results def _search_one(source: dict[str, Any], keyword: str, *, mode: str, limit: int) -> list[OnlineSearchItem]: source_name = _string(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or "未命名源" source_url = _string(source.get("bookSourceUrl") or source.get("sourceUrl") or source.get("url")) or "" search_url = _string(source.get("searchUrl")) rule_search = source.get("ruleSearch") if isinstance(source.get("ruleSearch"), dict) else {} if not search_url: search_url = _string(rule_search.get("url") or rule_search.get("searchUrl")) or "" if not source_url or not search_url: return [] final_url = _resolve_url(source_url, _build_search_url(search_url, keyword)) _assert_public_url(final_url) body = _fetch_text(final_url, source_url, source) list_rule = _first_string( rule_search, "bookList", "comicList", "list", "items", "data", "results", ) if list_rule: parsed = _parse_by_rules(body, final_url, source, source_name, source_url, rule_search, mode, limit) if parsed: return parsed if mode == "manga": return _fallback_manga(body, final_url, source_name, source_url, keyword, limit) return _fallback_novel(body, final_url, source_name, source_url, keyword, limit) def _parse_by_rules( body: str, base_url: str, source: dict[str, Any], source_name: str, source_url: str, rule_search: dict[str, Any], mode: str, limit: int, ) -> list[OnlineSearchItem]: fragments = _select_list(body, _first_string(rule_search, "bookList", "comicList", "list", "items", "data", "results")) results: list[OnlineSearchItem] = [] seen: set[str] = set() for fragment in fragments: title = _select_one(fragment, _first_string(rule_search, "name", "bookName", "comicName", "title")) detail = _select_one(fragment, _first_string(rule_search, "bookUrl", "comicUrl", "detailUrl", "url", "href")) if not title or not detail: continue detail_url = _resolve_url(base_url, detail) if detail_url in seen: continue seen.add(detail_url) results.append( OnlineSearchItem( source_name=source_name, source_url=source_url, title=title, author=_select_one(fragment, _first_string(rule_search, "author", "bookAuthor", "comicAuthor", "writer")) or None, cover_url=_maybe_url(base_url, _select_one(fragment, _first_string(rule_search, "coverUrl", "cover", "bookCover", "comicCover", "img", "image"))), detail_url=detail_url, kind=mode, ) ) if len(results) >= limit: break return results def _fallback_manga( body: str, base_url: str, source_name: str, source_url: str, keyword: str, limit: int, ) -> list[OnlineSearchItem]: decoded = _decode_json_like(body) results: list[OnlineSearchItem] = [] seen: set[str] = set() if decoded is not None: for node in _walk_dicts(decoded): title = _first_value(node, "title", "name", "comicName", "comic_name", "topic_title", "topicTitle", "bookName") if not _matches(title, keyword): continue detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "comicUrl", "comic_url", "detailUrl", "detail_url", "path") if not detail: detail = _first_value(node, "id", "book_id", "comic_id", "topic_id") if not detail: continue detail_url = _resolve_url(base_url, detail) if detail_url in seen: continue seen.add(detail_url) results.append( OnlineSearchItem( source_name=source_name, source_url=source_url, title=title, author=_first_value(node, "author", "writer", "nickname", "user_name", "userName") or None, cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "image", "imageUrl", "image_url", "pic", "pic_url", "vertical_image_url")), detail_url=detail_url, kind="manga", ) ) if len(results) >= limit: return results if results: return results return _fallback_html_cards(body, base_url, source_name, source_url, keyword, "manga", limit) def _fallback_novel( body: str, base_url: str, source_name: str, source_url: str, keyword: str, limit: int, ) -> list[OnlineSearchItem]: decoded = _decode_json_like(body) results: list[OnlineSearchItem] = [] if decoded is not None: for node in _walk_dicts(decoded): title = _first_value(node, "title", "name", "bookName", "book_name", "novelName") detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "detailUrl", "detail_url") if not _matches(title, keyword) or not detail: continue results.append( OnlineSearchItem( source_name=source_name, source_url=source_url, title=title, author=_first_value(node, "author", "bookAuthor", "writer") or None, cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "img", "image")), detail_url=_resolve_url(base_url, detail), kind="novel", ) ) if len(results) >= limit: return results return results or _fallback_html_cards(body, base_url, source_name, source_url, keyword, "novel", limit) def _fallback_html_cards( body: str, base_url: str, source_name: str, source_url: str, keyword: str, kind: str, limit: int, ) -> list[OnlineSearchItem]: parser = _AnchorParser() parser.feed(body) results: list[OnlineSearchItem] = [] seen: set[str] = set() for anchor in parser.anchors: title = _clean(anchor.get("title") or anchor.get("alt") or anchor.get("text") or "") href = _clean(anchor.get("href") or "") if not href or not _matches(title, keyword): continue detail_url = _resolve_url(base_url, href) if detail_url in seen: continue seen.add(detail_url) results.append( OnlineSearchItem( source_name=source_name, source_url=source_url, title=title, author=None, cover_url=_maybe_url(base_url, anchor.get("img") or ""), detail_url=detail_url, kind=kind, ) ) if len(results) >= limit: break return results def _select_list(body: str, rule: str | None) -> list[str]: if not rule: return [] for part in _split_rule(rule, "||"): if part.lower().startswith("regex:"): pattern = _strip_replace(part.split(":", 1)[1]) matches = re.findall(pattern, body, re.I | re.S) output = [_clean(item[0] if isinstance(item, tuple) else item) for item in matches] if output: return [item for item in output if item] if part.startswith("$.") or part.startswith("$["): decoded = _decode_json_like(body) values = _json_path(decoded, part) if decoded is not None else [] output = [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in values] if output: return output matches = _html_rule_list(body, part) if matches: return matches return [] def _select_one(fragment: str, rule: str | None) -> str: values = _select_list(fragment, rule) return values[0] if values else "" def _html_rule_list(body: str, rule: str) -> list[str]: rule = _strip_replace(rule.strip()) if not rule: return [] tokens = [item.strip() for item in rule.split("@") if item.strip()] selector = tokens[0] if tokens else rule attr = tokens[-1] if tokens and tokens[-1] not in {selector, "text", "html"} else "" if selector.startswith("class."): classes = [item for item in selector[6:].split() if item] class_checks = "".join(rf'(?=[^"\']*{re.escape(item)})' for item in classes[:3]) pattern = rf'<(?P[a-zA-Z0-9]+)\b[^>]*class=["\']{class_checks}[^"\']*["\'][^>]*>.*?' elif selector.startswith("id."): pattern = rf'<(?P[a-zA-Z0-9]+)\b[^>]*id=["\']{re.escape(selector[3:])}["\'][^>]*>.*?' elif selector.startswith("tag."): tag = re.escape(selector[4:]) pattern = rf'<{tag}\b[^>]*>.*?' elif selector.startswith("$.") or selector.startswith("$["): decoded = _decode_json_like(body) return [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in _json_path(decoded, selector)] else: pattern = rf']*>.*?' if "href" in tokens else "" if not pattern: return [] blocks = [m.group(0) for m in re.finditer(pattern, body, re.I | re.S)] if attr and attr not in {"text", "html"}: return [_clean(m.group(1)) for block in blocks for m in [re.search(rf'{re.escape(attr)}=["\']([^"\']+)["\']', block, re.I)] if m] if "text" in tokens: return [_html_text(block) for block in blocks] return blocks def _json_path(decoded: Any, rule: str) -> list[Any]: if decoded is None: return [] path = re.sub(r"^\$\.?", "", rule.strip()) if path in {"", "*", "[*]"}: return decoded if isinstance(decoded, list) else [decoded] current = [decoded] for segment in [item for item in path.split(".") if item]: next_values: list[Any] = [] star = segment.endswith("[*]") key = segment[:-3] if star else segment for value in current: child = value.get(key) if isinstance(value, dict) and key else value if star and isinstance(child, list): next_values.extend(child) elif child is not None: next_values.append(child) current = next_values return current class _AnchorParser(HTMLParser): def __init__(self) -> None: super().__init__() self.anchors: list[dict[str, str]] = [] self._current: dict[str, str] | None = None def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: attr = {key: value or "" for key, value in attrs} if tag.lower() == "a" and attr.get("href"): self._current = {"href": attr.get("href", ""), "title": attr.get("title", ""), "text": ""} elif tag.lower() == "img" and self._current is not None: for key in ("data-original", "data-src", "src"): if attr.get(key): self._current["img"] = attr[key] break if attr.get("alt"): self._current["alt"] = attr["alt"] def handle_data(self, data: str) -> None: if self._current is not None: self._current["text"] = f'{self._current.get("text", "")} {data}' def handle_endtag(self, tag: str) -> None: if tag.lower() == "a" and self._current is not None: self.anchors.append(self._current) self._current = None def _fetch_text(url: str, referer: str, source: dict[str, Any]) -> str: headers = { "User-Agent": _string(source.get("httpUserAgent")) or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7", "Accept-Language": "zh-CN,zh;q=0.9", "Referer": referer, } extra = source.get("headers") or source.get("header") if isinstance(extra, dict): for key, value in extra.items(): if str(key).lower() in {"cookie", "authorization"}: continue headers[str(key)] = str(value) with urlopen(Request(url, headers=headers), timeout=12) as response: raw = response.read(2_000_000) content_type = response.headers.get("content-type", "") charset = _string(source.get("charset") or source.get("encoding")) or _charset_from_content_type(content_type) or "utf-8" for candidate in (charset, "utf-8", "gb18030", "gbk", "gb2312"): try: return raw.decode(candidate, errors="replace") except LookupError: continue return raw.decode("utf-8", errors="replace") def _assert_public_url(url: str) -> None: parsed = urlparse(url) if parsed.scheme not in {"http", "https"} or not parsed.hostname: raise OnlineSearchError("Only http/https source URLs are allowed") infos = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM) for item in infos: ip = ipaddress.ip_address(item[4][0]) if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved: raise OnlineSearchError("Private network source URLs are not allowed") def _build_search_url(template: str, keyword: str) -> str: encoded = quote(keyword, safe="") value = template.strip() for token in ("{{key}}", "{{keyword}}", "{key}", "{keyword}", "", "", "searchKey"): value = value.replace(token, encoded) value = value.replace("%s", encoded, 1) return value.replace("{{page}}", "1").replace("{page}", "1").replace("", "1") def _resolve_url(base: str, value: str) -> str: return urljoin(base, value.strip()) def _maybe_url(base: str, value: str | None) -> str | None: text = _clean(value or "") if not text or text.lower() == "null": return None return _resolve_url(base, text) def _decode_json_like(body: str) -> Any: text = body.lstrip("\ufeff \t\r\n") match = re.match(r"^[a-zA-Z_$][\w$]*\s*\((.*)\)\s*;?\s*$", text, re.S) if match: text = match.group(1) try: return json.loads(text) except Exception: return None def _walk_dicts(node: Any): if isinstance(node, dict): yield node for value in node.values(): yield from _walk_dicts(value) elif isinstance(node, list): for item in node: yield from _walk_dicts(item) def _first_value(node: dict[str, Any], *keys: str) -> str: for key in keys: value = node.get(key) if value is not None: text = _clean(str(value)) if text and text.lower() != "null": return text return "" def _first_string(mapping: dict[str, Any], *keys: str) -> str | None: for key in keys: value = _string(mapping.get(key)) if value: return value return None def _source_type(source: dict[str, Any]) -> int: value = source.get("bookSourceType", source.get("sourceType", source.get("type", 0))) try: numeric = int(value) except Exception: numeric = 0 text = json.dumps(source, ensure_ascii=False).lower() manga_markers = ( "comiclist", "comicname", "comicurl", "imageStyle".lower(), "chapterImages".lower(), "manga", "manhua", "漫画", ) if numeric == 0 and any(marker in text for marker in manga_markers): return 2 return numeric def _split_rule(rule: str, sep: str) -> list[str]: return [item.strip() for item in rule.split(sep) if item.strip()] def _strip_replace(rule: str) -> str: return re.split(r"##|###", rule, maxsplit=1)[0].strip() def _string(value: Any) -> str | None: if value is None: return None text = str(value).strip() return text or None def _matches(title: str, keyword: str) -> bool: title = _clean(title).lower() keyword = _clean(keyword).lower() return bool(title) and (not keyword or keyword in title) def _clean(value: str) -> str: return unescape(re.sub(r"\s+", " ", value or "")).strip() def _html_text(value: str) -> str: return _clean(re.sub(r"<[^>]+>", " ", value)) def _charset_from_content_type(value: str) -> str | None: match = re.search(r"charset=([^;]+)", value or "", re.I) return match.group(1).strip("\"'") if match else None