commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,537 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from html import unescape
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urljoin, urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from sakura_cloud.services import online_search_engine
|
||||
from sakura_cloud.services.online_search_engine import (
|
||||
OnlineSearchEngine,
|
||||
SearchEngineError,
|
||||
assert_public_url,
|
||||
build_search_url,
|
||||
parse_comic,
|
||||
parse_novel,
|
||||
resolve_resource_url,
|
||||
search_resource,
|
||||
source_type,
|
||||
)
|
||||
|
||||
|
||||
class OnlineSearchError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OnlineSearchItem:
|
||||
source_name: str
|
||||
source_url: str
|
||||
title: str
|
||||
author: str | None
|
||||
cover_url: str | None
|
||||
detail_url: str
|
||||
kind: str
|
||||
|
||||
|
||||
def parse_source_payload(payload: Any) -> list[dict[str, Any]]:
|
||||
if isinstance(payload, str):
|
||||
payload = json.loads(payload)
|
||||
if isinstance(payload, list):
|
||||
rows = payload
|
||||
elif isinstance(payload, dict):
|
||||
for key in ("sources", "bookSources", "data", "list", "items", "results"):
|
||||
value = payload.get(key)
|
||||
if isinstance(value, list):
|
||||
rows = value
|
||||
break
|
||||
else:
|
||||
rows = [payload]
|
||||
else:
|
||||
rows = []
|
||||
return [dict(item) for item in rows if isinstance(item, dict)]
|
||||
|
||||
|
||||
def search_sources(
|
||||
sources: list[dict[str, Any]],
|
||||
keyword: str,
|
||||
*,
|
||||
mode: str,
|
||||
limit_sources: int = 30,
|
||||
per_source_limit: int = 20,
|
||||
) -> list[OnlineSearchItem]:
|
||||
results: list[OnlineSearchItem] = []
|
||||
expected_type = 2 if mode == "manga" else 0
|
||||
engine = OnlineSearchEngine(timeout=10.0)
|
||||
for source in sources[:limit_sources]:
|
||||
if _source_type(source) != expected_type:
|
||||
continue
|
||||
try:
|
||||
online_search_engine.assert_public_url = _assert_public_url
|
||||
engine._request_text = ( # type: ignore[method-assign]
|
||||
lambda rule, url, data=None, _source=source: _fetch_text(url, rule.base_url, _source)
|
||||
)
|
||||
candidates = engine.search_legado_source(source, keyword, mode=mode, limit=per_source_limit)
|
||||
results.extend(
|
||||
OnlineSearchItem(
|
||||
source_name=item.source_name,
|
||||
source_url=item.source_url,
|
||||
title=item.title,
|
||||
author=item.author,
|
||||
cover_url=item.cover,
|
||||
detail_url=item.url,
|
||||
kind=mode,
|
||||
)
|
||||
for item in candidates
|
||||
)
|
||||
except SearchEngineError:
|
||||
continue
|
||||
except Exception:
|
||||
continue
|
||||
return results
|
||||
|
||||
|
||||
def _search_one(source: dict[str, Any], keyword: str, *, mode: str, limit: int) -> list[OnlineSearchItem]:
|
||||
source_name = _string(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or "未命名源"
|
||||
source_url = _string(source.get("bookSourceUrl") or source.get("sourceUrl") or source.get("url")) or ""
|
||||
search_url = _string(source.get("searchUrl"))
|
||||
rule_search = source.get("ruleSearch") if isinstance(source.get("ruleSearch"), dict) else {}
|
||||
if not search_url:
|
||||
search_url = _string(rule_search.get("url") or rule_search.get("searchUrl")) or ""
|
||||
if not source_url or not search_url:
|
||||
return []
|
||||
|
||||
final_url = _resolve_url(source_url, _build_search_url(search_url, keyword))
|
||||
_assert_public_url(final_url)
|
||||
body = _fetch_text(final_url, source_url, source)
|
||||
list_rule = _first_string(
|
||||
rule_search,
|
||||
"bookList",
|
||||
"comicList",
|
||||
"list",
|
||||
"items",
|
||||
"data",
|
||||
"results",
|
||||
)
|
||||
if list_rule:
|
||||
parsed = _parse_by_rules(body, final_url, source, source_name, source_url, rule_search, mode, limit)
|
||||
if parsed:
|
||||
return parsed
|
||||
if mode == "manga":
|
||||
return _fallback_manga(body, final_url, source_name, source_url, keyword, limit)
|
||||
return _fallback_novel(body, final_url, source_name, source_url, keyword, limit)
|
||||
|
||||
|
||||
def _parse_by_rules(
|
||||
body: str,
|
||||
base_url: str,
|
||||
source: dict[str, Any],
|
||||
source_name: str,
|
||||
source_url: str,
|
||||
rule_search: dict[str, Any],
|
||||
mode: str,
|
||||
limit: int,
|
||||
) -> list[OnlineSearchItem]:
|
||||
fragments = _select_list(body, _first_string(rule_search, "bookList", "comicList", "list", "items", "data", "results"))
|
||||
results: list[OnlineSearchItem] = []
|
||||
seen: set[str] = set()
|
||||
for fragment in fragments:
|
||||
title = _select_one(fragment, _first_string(rule_search, "name", "bookName", "comicName", "title"))
|
||||
detail = _select_one(fragment, _first_string(rule_search, "bookUrl", "comicUrl", "detailUrl", "url", "href"))
|
||||
if not title or not detail:
|
||||
continue
|
||||
detail_url = _resolve_url(base_url, detail)
|
||||
if detail_url in seen:
|
||||
continue
|
||||
seen.add(detail_url)
|
||||
results.append(
|
||||
OnlineSearchItem(
|
||||
source_name=source_name,
|
||||
source_url=source_url,
|
||||
title=title,
|
||||
author=_select_one(fragment, _first_string(rule_search, "author", "bookAuthor", "comicAuthor", "writer")) or None,
|
||||
cover_url=_maybe_url(base_url, _select_one(fragment, _first_string(rule_search, "coverUrl", "cover", "bookCover", "comicCover", "img", "image"))),
|
||||
detail_url=detail_url,
|
||||
kind=mode,
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def _fallback_manga(
|
||||
body: str,
|
||||
base_url: str,
|
||||
source_name: str,
|
||||
source_url: str,
|
||||
keyword: str,
|
||||
limit: int,
|
||||
) -> list[OnlineSearchItem]:
|
||||
decoded = _decode_json_like(body)
|
||||
results: list[OnlineSearchItem] = []
|
||||
seen: set[str] = set()
|
||||
if decoded is not None:
|
||||
for node in _walk_dicts(decoded):
|
||||
title = _first_value(node, "title", "name", "comicName", "comic_name", "topic_title", "topicTitle", "bookName")
|
||||
if not _matches(title, keyword):
|
||||
continue
|
||||
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "comicUrl", "comic_url", "detailUrl", "detail_url", "path")
|
||||
if not detail:
|
||||
detail = _first_value(node, "id", "book_id", "comic_id", "topic_id")
|
||||
if not detail:
|
||||
continue
|
||||
detail_url = _resolve_url(base_url, detail)
|
||||
if detail_url in seen:
|
||||
continue
|
||||
seen.add(detail_url)
|
||||
results.append(
|
||||
OnlineSearchItem(
|
||||
source_name=source_name,
|
||||
source_url=source_url,
|
||||
title=title,
|
||||
author=_first_value(node, "author", "writer", "nickname", "user_name", "userName") or None,
|
||||
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "image", "imageUrl", "image_url", "pic", "pic_url", "vertical_image_url")),
|
||||
detail_url=detail_url,
|
||||
kind="manga",
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
return results
|
||||
if results:
|
||||
return results
|
||||
return _fallback_html_cards(body, base_url, source_name, source_url, keyword, "manga", limit)
|
||||
|
||||
|
||||
def _fallback_novel(
|
||||
body: str,
|
||||
base_url: str,
|
||||
source_name: str,
|
||||
source_url: str,
|
||||
keyword: str,
|
||||
limit: int,
|
||||
) -> list[OnlineSearchItem]:
|
||||
decoded = _decode_json_like(body)
|
||||
results: list[OnlineSearchItem] = []
|
||||
if decoded is not None:
|
||||
for node in _walk_dicts(decoded):
|
||||
title = _first_value(node, "title", "name", "bookName", "book_name", "novelName")
|
||||
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "detailUrl", "detail_url")
|
||||
if not _matches(title, keyword) or not detail:
|
||||
continue
|
||||
results.append(
|
||||
OnlineSearchItem(
|
||||
source_name=source_name,
|
||||
source_url=source_url,
|
||||
title=title,
|
||||
author=_first_value(node, "author", "bookAuthor", "writer") or None,
|
||||
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "img", "image")),
|
||||
detail_url=_resolve_url(base_url, detail),
|
||||
kind="novel",
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
return results
|
||||
return results or _fallback_html_cards(body, base_url, source_name, source_url, keyword, "novel", limit)
|
||||
|
||||
|
||||
def _fallback_html_cards(
|
||||
body: str,
|
||||
base_url: str,
|
||||
source_name: str,
|
||||
source_url: str,
|
||||
keyword: str,
|
||||
kind: str,
|
||||
limit: int,
|
||||
) -> list[OnlineSearchItem]:
|
||||
parser = _AnchorParser()
|
||||
parser.feed(body)
|
||||
results: list[OnlineSearchItem] = []
|
||||
seen: set[str] = set()
|
||||
for anchor in parser.anchors:
|
||||
title = _clean(anchor.get("title") or anchor.get("alt") or anchor.get("text") or "")
|
||||
href = _clean(anchor.get("href") or "")
|
||||
if not href or not _matches(title, keyword):
|
||||
continue
|
||||
detail_url = _resolve_url(base_url, href)
|
||||
if detail_url in seen:
|
||||
continue
|
||||
seen.add(detail_url)
|
||||
results.append(
|
||||
OnlineSearchItem(
|
||||
source_name=source_name,
|
||||
source_url=source_url,
|
||||
title=title,
|
||||
author=None,
|
||||
cover_url=_maybe_url(base_url, anchor.get("img") or ""),
|
||||
detail_url=detail_url,
|
||||
kind=kind,
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def _select_list(body: str, rule: str | None) -> list[str]:
|
||||
if not rule:
|
||||
return []
|
||||
for part in _split_rule(rule, "||"):
|
||||
if part.lower().startswith("regex:"):
|
||||
pattern = _strip_replace(part.split(":", 1)[1])
|
||||
matches = re.findall(pattern, body, re.I | re.S)
|
||||
output = [_clean(item[0] if isinstance(item, tuple) else item) for item in matches]
|
||||
if output:
|
||||
return [item for item in output if item]
|
||||
if part.startswith("$.") or part.startswith("$["):
|
||||
decoded = _decode_json_like(body)
|
||||
values = _json_path(decoded, part) if decoded is not None else []
|
||||
output = [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in values]
|
||||
if output:
|
||||
return output
|
||||
matches = _html_rule_list(body, part)
|
||||
if matches:
|
||||
return matches
|
||||
return []
|
||||
|
||||
|
||||
def _select_one(fragment: str, rule: str | None) -> str:
|
||||
values = _select_list(fragment, rule)
|
||||
return values[0] if values else ""
|
||||
|
||||
|
||||
def _html_rule_list(body: str, rule: str) -> list[str]:
|
||||
rule = _strip_replace(rule.strip())
|
||||
if not rule:
|
||||
return []
|
||||
tokens = [item.strip() for item in rule.split("@") if item.strip()]
|
||||
selector = tokens[0] if tokens else rule
|
||||
attr = tokens[-1] if tokens and tokens[-1] not in {selector, "text", "html"} else ""
|
||||
if selector.startswith("class."):
|
||||
classes = [item for item in selector[6:].split() if item]
|
||||
class_checks = "".join(rf'(?=[^"\']*{re.escape(item)})' for item in classes[:3])
|
||||
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*class=["\']{class_checks}[^"\']*["\'][^>]*>.*?</(?P=tag)>'
|
||||
elif selector.startswith("id."):
|
||||
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*id=["\']{re.escape(selector[3:])}["\'][^>]*>.*?</(?P=tag)>'
|
||||
elif selector.startswith("tag."):
|
||||
tag = re.escape(selector[4:])
|
||||
pattern = rf'<{tag}\b[^>]*>.*?</{tag}>'
|
||||
elif selector.startswith("$.") or selector.startswith("$["):
|
||||
decoded = _decode_json_like(body)
|
||||
return [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in _json_path(decoded, selector)]
|
||||
else:
|
||||
pattern = rf'<a\b[^>]*>.*?</a>' if "href" in tokens else ""
|
||||
if not pattern:
|
||||
return []
|
||||
blocks = [m.group(0) for m in re.finditer(pattern, body, re.I | re.S)]
|
||||
if attr and attr not in {"text", "html"}:
|
||||
return [_clean(m.group(1)) for block in blocks for m in [re.search(rf'{re.escape(attr)}=["\']([^"\']+)["\']', block, re.I)] if m]
|
||||
if "text" in tokens:
|
||||
return [_html_text(block) for block in blocks]
|
||||
return blocks
|
||||
|
||||
|
||||
def _json_path(decoded: Any, rule: str) -> list[Any]:
|
||||
if decoded is None:
|
||||
return []
|
||||
path = re.sub(r"^\$\.?", "", rule.strip())
|
||||
if path in {"", "*", "[*]"}:
|
||||
return decoded if isinstance(decoded, list) else [decoded]
|
||||
current = [decoded]
|
||||
for segment in [item for item in path.split(".") if item]:
|
||||
next_values: list[Any] = []
|
||||
star = segment.endswith("[*]")
|
||||
key = segment[:-3] if star else segment
|
||||
for value in current:
|
||||
child = value.get(key) if isinstance(value, dict) and key else value
|
||||
if star and isinstance(child, list):
|
||||
next_values.extend(child)
|
||||
elif child is not None:
|
||||
next_values.append(child)
|
||||
current = next_values
|
||||
return current
|
||||
|
||||
|
||||
class _AnchorParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.anchors: list[dict[str, str]] = []
|
||||
self._current: dict[str, str] | None = None
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
attr = {key: value or "" for key, value in attrs}
|
||||
if tag.lower() == "a" and attr.get("href"):
|
||||
self._current = {"href": attr.get("href", ""), "title": attr.get("title", ""), "text": ""}
|
||||
elif tag.lower() == "img" and self._current is not None:
|
||||
for key in ("data-original", "data-src", "src"):
|
||||
if attr.get(key):
|
||||
self._current["img"] = attr[key]
|
||||
break
|
||||
if attr.get("alt"):
|
||||
self._current["alt"] = attr["alt"]
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._current is not None:
|
||||
self._current["text"] = f'{self._current.get("text", "")} {data}'
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag.lower() == "a" and self._current is not None:
|
||||
self.anchors.append(self._current)
|
||||
self._current = None
|
||||
|
||||
|
||||
def _fetch_text(url: str, referer: str, source: dict[str, Any]) -> str:
|
||||
headers = {
|
||||
"User-Agent": _string(source.get("httpUserAgent")) or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
"Referer": referer,
|
||||
}
|
||||
extra = source.get("headers") or source.get("header")
|
||||
if isinstance(extra, dict):
|
||||
for key, value in extra.items():
|
||||
if str(key).lower() in {"cookie", "authorization"}:
|
||||
continue
|
||||
headers[str(key)] = str(value)
|
||||
with urlopen(Request(url, headers=headers), timeout=12) as response:
|
||||
raw = response.read(2_000_000)
|
||||
content_type = response.headers.get("content-type", "")
|
||||
charset = _string(source.get("charset") or source.get("encoding")) or _charset_from_content_type(content_type) or "utf-8"
|
||||
for candidate in (charset, "utf-8", "gb18030", "gbk", "gb2312"):
|
||||
try:
|
||||
return raw.decode(candidate, errors="replace")
|
||||
except LookupError:
|
||||
continue
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _assert_public_url(url: str) -> None:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise OnlineSearchError("Only http/https source URLs are allowed")
|
||||
infos = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM)
|
||||
for item in infos:
|
||||
ip = ipaddress.ip_address(item[4][0])
|
||||
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved:
|
||||
raise OnlineSearchError("Private network source URLs are not allowed")
|
||||
|
||||
|
||||
def _build_search_url(template: str, keyword: str) -> str:
|
||||
encoded = quote(keyword, safe="")
|
||||
value = template.strip()
|
||||
for token in ("{{key}}", "{{keyword}}", "{key}", "{keyword}", "<key>", "<keyword>", "searchKey"):
|
||||
value = value.replace(token, encoded)
|
||||
value = value.replace("%s", encoded, 1)
|
||||
return value.replace("{{page}}", "1").replace("{page}", "1").replace("<page>", "1")
|
||||
|
||||
|
||||
def _resolve_url(base: str, value: str) -> str:
|
||||
return urljoin(base, value.strip())
|
||||
|
||||
|
||||
def _maybe_url(base: str, value: str | None) -> str | None:
|
||||
text = _clean(value or "")
|
||||
if not text or text.lower() == "null":
|
||||
return None
|
||||
return _resolve_url(base, text)
|
||||
|
||||
|
||||
def _decode_json_like(body: str) -> Any:
|
||||
text = body.lstrip("\ufeff \t\r\n")
|
||||
match = re.match(r"^[a-zA-Z_$][\w$]*\s*\((.*)\)\s*;?\s*$", text, re.S)
|
||||
if match:
|
||||
text = match.group(1)
|
||||
try:
|
||||
return json.loads(text)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _walk_dicts(node: Any):
|
||||
if isinstance(node, dict):
|
||||
yield node
|
||||
for value in node.values():
|
||||
yield from _walk_dicts(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
yield from _walk_dicts(item)
|
||||
|
||||
|
||||
def _first_value(node: dict[str, Any], *keys: str) -> str:
|
||||
for key in keys:
|
||||
value = node.get(key)
|
||||
if value is not None:
|
||||
text = _clean(str(value))
|
||||
if text and text.lower() != "null":
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _first_string(mapping: dict[str, Any], *keys: str) -> str | None:
|
||||
for key in keys:
|
||||
value = _string(mapping.get(key))
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _source_type(source: dict[str, Any]) -> int:
|
||||
value = source.get("bookSourceType", source.get("sourceType", source.get("type", 0)))
|
||||
try:
|
||||
numeric = int(value)
|
||||
except Exception:
|
||||
numeric = 0
|
||||
text = json.dumps(source, ensure_ascii=False).lower()
|
||||
manga_markers = (
|
||||
"comiclist",
|
||||
"comicname",
|
||||
"comicurl",
|
||||
"imageStyle".lower(),
|
||||
"chapterImages".lower(),
|
||||
"manga",
|
||||
"manhua",
|
||||
"漫画",
|
||||
)
|
||||
if numeric == 0 and any(marker in text for marker in manga_markers):
|
||||
return 2
|
||||
return numeric
|
||||
|
||||
|
||||
def _split_rule(rule: str, sep: str) -> list[str]:
|
||||
return [item.strip() for item in rule.split(sep) if item.strip()]
|
||||
|
||||
|
||||
def _strip_replace(rule: str) -> str:
|
||||
return re.split(r"##|###", rule, maxsplit=1)[0].strip()
|
||||
|
||||
|
||||
def _string(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _matches(title: str, keyword: str) -> bool:
|
||||
title = _clean(title).lower()
|
||||
keyword = _clean(keyword).lower()
|
||||
return bool(title) and (not keyword or keyword in title)
|
||||
|
||||
|
||||
def _clean(value: str) -> str:
|
||||
return unescape(re.sub(r"\s+", " ", value or "")).strip()
|
||||
|
||||
|
||||
def _html_text(value: str) -> str:
|
||||
return _clean(re.sub(r"<[^>]+>", " ", value))
|
||||
|
||||
|
||||
def _charset_from_content_type(value: str) -> str | None:
|
||||
match = re.search(r"charset=([^;]+)", value or "", re.I)
|
||||
return match.group(1).strip("\"'") if match else None
|
||||
Reference in new issue
Block a user