1134 lines
43 KiB
Python
1134 lines
43 KiB
Python
"""
|
||
Biquge novel plugin backed by the reader site's public API.
|
||
|
||
Primary flow:
|
||
- search books via https://apibi.cc/api/search
|
||
- resolve books via https://apibi.cc/api/book + /api/booklist
|
||
- fetch chapter text via https://apibi.cc/api/chapter
|
||
|
||
The plugin still keeps limited HTML / Playwright fallbacks for legacy pages.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import atexit
|
||
import base64
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import threading
|
||
from dataclasses import dataclass
|
||
from html import unescape
|
||
from pathlib import Path
|
||
from typing import Any
|
||
from urllib.parse import parse_qs, quote, urljoin, urlparse
|
||
|
||
import requests
|
||
|
||
try:
|
||
from cryptography.hazmat.primitives import padding
|
||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||
|
||
CRYPTOGRAPHY_AVAILABLE = True
|
||
except Exception:
|
||
padding = None
|
||
Cipher = None
|
||
algorithms = None
|
||
modes = None
|
||
CRYPTOGRAPHY_AVAILABLE = False
|
||
|
||
from pixiv_app.core.downloader import sanitize_filename, save_text_file
|
||
from pixiv_app.core.playwright_runtime import (
|
||
configure_playwright_runtime_env,
|
||
explain_missing_browser,
|
||
preferred_chromium_executable,
|
||
)
|
||
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
|
||
|
||
try:
|
||
from playwright.sync_api import sync_playwright
|
||
|
||
PLAYWRIGHT_AVAILABLE = True
|
||
except Exception:
|
||
sync_playwright = None
|
||
PLAYWRIGHT_AVAILABLE = False
|
||
|
||
|
||
LEGACY_BASE_URL = "https://www.bqgui.cc"
|
||
READER_BASE_URL = "https://www.bqg551.cc"
|
||
API_BASE_URL = "https://apibi.cc"
|
||
TOKEN_SECRET = os.environ.get("SAKURA_BIQUGE_TOKEN_SECRET", "book@token.html")
|
||
USER_AGENT = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||
)
|
||
TOKEN_MD5 = hashlib.md5(TOKEN_SECRET.encode("utf-8")).hexdigest()
|
||
TOKEN_IV = TOKEN_MD5[:16].encode("utf-8")
|
||
TOKEN_KEY = TOKEN_MD5[16:].encode("utf-8")
|
||
SORT_SLUGS = {
|
||
"xuanhuan",
|
||
"wuxia",
|
||
"dushi",
|
||
"lishi",
|
||
"wangyou",
|
||
"kehuan",
|
||
"mm",
|
||
"top",
|
||
"finish",
|
||
}
|
||
|
||
_BOOK_PATH_RE = re.compile(r"/book/(?P<book_id>\d+)/?$", re.I)
|
||
_CHAPTER_PATH_RE = re.compile(
|
||
r"/book/(?P<book_id>\d+)/(?P<chapter_id>\d+)(?:_\d+)?\.html(?:[?#].*)?$",
|
||
re.I,
|
||
)
|
||
_HASH_BOOK_RE = re.compile(r"^book/(?P<book_id>\d+)/?$", re.I)
|
||
_HASH_CHAPTER_RE = re.compile(
|
||
r"^book/(?P<book_id>\d+)/(?P<chapter_id>\d+)(?:_\d+)?\.html$",
|
||
re.I,
|
||
)
|
||
_BOOK_LINK_RE = re.compile(
|
||
r"""<a[^>]+href\s*=\s*["'](?P<href>/book/\d+/)["'][^>]*>(?P<title>.*?)</a>""",
|
||
re.I | re.S,
|
||
)
|
||
_CHAPTER_LINK_RE = re.compile(
|
||
r"""<a[^>]+href\s*=\s*["'](?P<href>/book/\d+/\d+\.html)["'][^>]*>(?P<title>.*?)</a>""",
|
||
re.I | re.S,
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class BookCandidate:
|
||
title: str
|
||
url: str
|
||
author: str = ""
|
||
latest_chapter: str = ""
|
||
|
||
|
||
@dataclass
|
||
class TargetIds:
|
||
book_id: str = ""
|
||
chapter_id: str = ""
|
||
|
||
|
||
def _clean_html_text(raw: str) -> str:
|
||
text = re.sub(r"<br\s*/?>", "\n", raw, flags=re.I)
|
||
text = re.sub(r"</p\s*>", "\n", text, flags=re.I)
|
||
text = re.sub(r"<[^>]+>", "", text)
|
||
text = unescape(text)
|
||
text = text.replace("\r", "")
|
||
text = text.replace("\u3000", " ")
|
||
lines = [line.strip().strip('"').strip("'") for line in text.splitlines()]
|
||
return "\n".join(line for line in lines if line)
|
||
|
||
|
||
def _normalize_space(value: str) -> str:
|
||
return re.sub(r"\s+", " ", _clean_html_text(value)).strip()
|
||
|
||
|
||
class BiqugePlugin(BasePlugin):
|
||
name = "笔趣阁"
|
||
domain = "www.bqg551.cc"
|
||
version = "3.0.0"
|
||
|
||
def __init__(self) -> None:
|
||
self.session = requests.Session()
|
||
self.session.headers.update(self.get_headers())
|
||
self._playwright = None
|
||
self._browser = None
|
||
self._page = None
|
||
self._browser_lock = threading.Lock()
|
||
atexit.register(self._close_browser)
|
||
|
||
def validate(self) -> bool:
|
||
return True
|
||
|
||
def can_handle(self, url: str) -> bool:
|
||
lowered = url.strip().lower()
|
||
return any(host in lowered for host in ("bqgui.cc", "bqg551.cc", "apibi.cc"))
|
||
|
||
def get_headers(self) -> dict[str, str]:
|
||
return {
|
||
"User-Agent": USER_AGENT,
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||
"Referer": READER_BASE_URL + "/",
|
||
}
|
||
|
||
def resolve_target(self, target: str, *, limit: int = 20) -> dict[str, Any]:
|
||
raw = target.strip()
|
||
if not raw:
|
||
raise PluginParseError("请输入书名、作者、书籍链接或章节链接。")
|
||
|
||
normalized = self._normalize_target(raw)
|
||
if normalized:
|
||
route = self._parse_reader_route(normalized)
|
||
if route["kind"] == "search":
|
||
keyword = route["keyword"]
|
||
books = self._search_books(keyword, limit=limit)
|
||
if not books:
|
||
raise PluginParseError(f"没有搜索到与“{keyword}”相关的书籍。")
|
||
return {
|
||
"mode": "books",
|
||
"title": f"搜索结果:{keyword}",
|
||
"books": [candidate.__dict__ for candidate in books],
|
||
}
|
||
if route["kind"] == "sort":
|
||
books = self._reader_sort_books(str(route["slug"]), limit=limit)
|
||
if not books:
|
||
raise PluginParseError("该列表页没有解析到书籍。")
|
||
return {
|
||
"mode": "books",
|
||
"title": f"列表结果:{route['slug']}",
|
||
"books": [candidate.__dict__ for candidate in books],
|
||
}
|
||
if route["kind"] == "index":
|
||
books = self._reader_index_books(limit=limit)
|
||
if not books:
|
||
raise PluginParseError("首页推荐列表为空。")
|
||
return {
|
||
"mode": "books",
|
||
"title": "首页推荐",
|
||
"books": [candidate.__dict__ for candidate in books],
|
||
}
|
||
|
||
ids = self._extract_ids_from_url(normalized)
|
||
if ids.book_id and ids.chapter_id:
|
||
chapter = self._chapter_resource_from_ids(ids.book_id, ids.chapter_id, source_url=normalized)
|
||
return {
|
||
"mode": "chapters",
|
||
"book_title": str(chapter.metadata.get("book_title") or ""),
|
||
"resources": [chapter],
|
||
}
|
||
if ids.book_id:
|
||
return self._resolve_book_id(ids.book_id)
|
||
return self._resolve_listing_url(normalized, limit=limit)
|
||
|
||
if raw.isdigit():
|
||
return self._resolve_book_id(raw)
|
||
|
||
books = self._search_books(raw, limit=limit)
|
||
if not books:
|
||
raise PluginParseError(f"没有搜索到与“{raw}”相关的书籍。")
|
||
return {
|
||
"mode": "books",
|
||
"title": f"搜索结果:{raw}",
|
||
"books": [candidate.__dict__ for candidate in books],
|
||
}
|
||
|
||
def parse(self, url: str) -> list[Resource]:
|
||
outcome = self.resolve_target(url, limit=20)
|
||
if outcome.get("mode") == "chapters":
|
||
return list(outcome.get("resources") or [])
|
||
books = list(outcome.get("books") or [])
|
||
if not books:
|
||
raise PluginParseError("未解析到可下载内容。")
|
||
selected = self.resolve_target(str(books[0].get("url") or ""), limit=20)
|
||
resources = list(selected.get("resources") or [])
|
||
if not resources:
|
||
raise PluginParseError("选中的书籍没有章节内容。")
|
||
return resources
|
||
|
||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||
if str(resource.metadata.get("kind") or "") != "novel_chapter":
|
||
raise PluginParseError("不支持的笔趣阁资源类型。")
|
||
|
||
content = str(resource.metadata.get("content") or "")
|
||
if not content.strip():
|
||
chapter_data = self._parse_chapter_page(resource.url)
|
||
content = chapter_data["content"]
|
||
resource.metadata.update(chapter_data)
|
||
|
||
book_title = str(resource.metadata.get("book_title") or "未命名小说")
|
||
chapter_title = str(resource.metadata.get("chapter_title") or resource.title or "未命名章节")
|
||
chapter_order = int(resource.metadata.get("chapter_order") or 0)
|
||
|
||
folder = save_path / "novels" / sanitize_filename(book_title, "biquge_book")
|
||
filename = (
|
||
f"{chapter_order:04d}_{sanitize_filename(chapter_title, 'chapter')}.txt"
|
||
if chapter_order
|
||
else f"{sanitize_filename(chapter_title, 'chapter')}.txt"
|
||
)
|
||
final_path = folder / filename
|
||
text = "\n".join(
|
||
[
|
||
f"书名: {book_title}",
|
||
f"章节: {chapter_title}",
|
||
f"作者: {resource.author or '未知'}",
|
||
f"链接: {resource.url}",
|
||
"",
|
||
content.strip(),
|
||
]
|
||
).strip()
|
||
save_text_file(final_path, text)
|
||
resource.metadata["word_count"] = len(content)
|
||
return [final_path]
|
||
|
||
def _normalize_target(self, target: str) -> str:
|
||
value = target.strip()
|
||
lowered = value.lower()
|
||
if value.startswith("//"):
|
||
return "https:" + value
|
||
if value.startswith("#/"):
|
||
return READER_BASE_URL + "/" + value
|
||
if value.startswith("/#/"):
|
||
return READER_BASE_URL + value
|
||
if value.startswith("/book/"):
|
||
return urljoin(LEGACY_BASE_URL, value)
|
||
if value.startswith("/"):
|
||
return urljoin(LEGACY_BASE_URL, value)
|
||
if lowered.startswith(("http://", "https://")):
|
||
return value
|
||
if lowered.startswith(("www.bqgui.cc", "m.bqgui.cc", "www.bqg551.cc", "bqg551.cc", "apibi.cc")):
|
||
return "https://" + value
|
||
return ""
|
||
|
||
def _parse_reader_route(self, url: str) -> dict[str, str]:
|
||
parsed = urlparse(url)
|
||
fragment = parsed.fragment.lstrip("/")
|
||
if not fragment:
|
||
return {"kind": "none"}
|
||
|
||
fragment_path, _, fragment_query = fragment.partition("?")
|
||
cleaned_path = fragment_path.strip("/")
|
||
if not cleaned_path:
|
||
return {"kind": "index"}
|
||
|
||
if cleaned_path == "search":
|
||
keyword = parse_qs(fragment_query).get("q", [""])[0].strip()
|
||
return {"kind": "search", "keyword": keyword}
|
||
|
||
if cleaned_path in SORT_SLUGS:
|
||
return {"kind": "sort", "slug": cleaned_path}
|
||
|
||
return {"kind": "none"}
|
||
|
||
def _extract_ids_from_url(self, url: str) -> TargetIds:
|
||
parsed = urlparse(url)
|
||
|
||
for source in (parsed.path, parsed.fragment.partition("?")[0]):
|
||
if not source:
|
||
continue
|
||
normalized = source if source.startswith("/") else f"/{source}"
|
||
match = _CHAPTER_PATH_RE.search(normalized)
|
||
if match:
|
||
return TargetIds(book_id=match.group("book_id"), chapter_id=match.group("chapter_id"))
|
||
match = _BOOK_PATH_RE.search(normalized)
|
||
if match:
|
||
return TargetIds(book_id=match.group("book_id"))
|
||
|
||
cleaned = source.strip("/")
|
||
match = _HASH_CHAPTER_RE.match(cleaned)
|
||
if match:
|
||
return TargetIds(book_id=match.group("book_id"), chapter_id=match.group("chapter_id"))
|
||
match = _HASH_BOOK_RE.match(cleaned)
|
||
if match:
|
||
return TargetIds(book_id=match.group("book_id"))
|
||
|
||
return TargetIds()
|
||
|
||
def _resolve_book_id(self, book_id: str) -> dict[str, Any]:
|
||
book_data = self._book_data_from_api(book_id)
|
||
chapters = self._chapter_resources_from_book_data(book_data)
|
||
if not chapters:
|
||
raise PluginParseError(f"书籍 {book_id} 没有解析到章节目录。")
|
||
return {
|
||
"mode": "chapters",
|
||
"book_title": str(chapters[0].metadata.get("book_title") or ""),
|
||
"resources": chapters,
|
||
}
|
||
|
||
def _resolve_listing_url(self, url: str, *, limit: int) -> dict[str, Any]:
|
||
html = self._fetch_html(url)
|
||
chapters = self._extract_chapter_resources(html, url)
|
||
if chapters:
|
||
return {
|
||
"mode": "chapters",
|
||
"book_title": str(chapters[0].metadata.get("book_title") or ""),
|
||
"resources": chapters,
|
||
}
|
||
|
||
books = self._extract_book_candidates(html, url, limit=limit)
|
||
if books:
|
||
return {
|
||
"mode": "books",
|
||
"title": "请选择要抓取的书籍",
|
||
"books": [candidate.__dict__ for candidate in books],
|
||
}
|
||
raise PluginParseError("该页面没有解析到书籍或章节列表。")
|
||
|
||
def _fetch_html(self, url: str, *, params: dict[str, str] | None = None) -> str:
|
||
response = self.session.get(url, params=params, timeout=25)
|
||
response.raise_for_status()
|
||
if not response.encoding or response.encoding.lower() in {"iso-8859-1", "ascii"}:
|
||
response.encoding = response.apparent_encoding or "utf-8"
|
||
return response.text
|
||
|
||
def _search_books(self, keyword: str, *, limit: int) -> list[BookCandidate]:
|
||
books = self._search_books_api(keyword, limit=limit)
|
||
if books:
|
||
return books
|
||
|
||
attempts = [
|
||
{"url": f"{LEGACY_BASE_URL}/s", "params": {"q": keyword}},
|
||
{"url": f"{LEGACY_BASE_URL}/search", "params": {"q": keyword}},
|
||
]
|
||
for attempt in attempts:
|
||
try:
|
||
html = self._fetch_html(str(attempt["url"]), params=attempt.get("params"))
|
||
except Exception:
|
||
continue
|
||
books = self._extract_book_candidates(html, str(attempt["url"]), limit=limit, keyword=keyword)
|
||
if books:
|
||
return books
|
||
return []
|
||
|
||
def _search_books_api(self, keyword: str, *, limit: int) -> list[BookCandidate]:
|
||
try:
|
||
payload = self._api_get_json(
|
||
"/api/search",
|
||
params={"q": keyword},
|
||
referer=f"{READER_BASE_URL}/#/search?q={quote(keyword)}",
|
||
)
|
||
except Exception:
|
||
return []
|
||
|
||
rows = list(payload.get("data") or []) if isinstance(payload, dict) else []
|
||
ranked: list[tuple[int, BookCandidate]] = []
|
||
seen: set[str] = set()
|
||
keyword_parts = self._keyword_parts(keyword)
|
||
normalized_keyword = _normalize_space(keyword).lower()
|
||
|
||
for row in rows:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
book_id = str(row.get("id") or row.get("dirid") or "").strip()
|
||
title = _normalize_space(str(row.get("title") or ""))
|
||
if not book_id or not title:
|
||
continue
|
||
href = self._reader_book_url(book_id)
|
||
if href in seen:
|
||
continue
|
||
seen.add(href)
|
||
|
||
author = _normalize_space(str(row.get("author") or ""))
|
||
latest = _normalize_space(str(row.get("lastchapter") or ""))
|
||
candidate = BookCandidate(title=title, url=href, author=author, latest_chapter=latest)
|
||
score = self._candidate_score(candidate, keyword_parts)
|
||
title_lower = title.lower()
|
||
author_lower = author.lower()
|
||
if normalized_keyword and title_lower == normalized_keyword:
|
||
score += 1000
|
||
elif normalized_keyword and title_lower.startswith(normalized_keyword):
|
||
score += 200
|
||
elif normalized_keyword and normalized_keyword in title_lower:
|
||
score += 120
|
||
elif normalized_keyword and normalized_keyword in author_lower:
|
||
score += 60
|
||
ranked.append((score, candidate))
|
||
|
||
ranked.sort(key=lambda item: (-item[0], len(item[1].title), item[1].title))
|
||
return [item[1] for item in ranked[:limit]]
|
||
|
||
def _reader_index_books(self, *, limit: int) -> list[BookCandidate]:
|
||
payload = self._api_get_json("/api/index", params={"sort": "index"}, referer=READER_BASE_URL + "/#/")
|
||
if not isinstance(payload, dict):
|
||
return []
|
||
rows: list[dict[str, Any]] = []
|
||
for value in payload.values():
|
||
if isinstance(value, list):
|
||
rows.extend(item for item in value if isinstance(item, dict))
|
||
return self._book_candidates_from_api_rows(rows, limit=limit)
|
||
|
||
def _reader_sort_books(self, slug: str, *, limit: int) -> list[BookCandidate]:
|
||
payload = self._api_get_json(
|
||
"/api/sort",
|
||
params={"sort": slug},
|
||
referer=f"{READER_BASE_URL}/#/{slug}",
|
||
)
|
||
rows = list(payload.get("data") or []) if isinstance(payload, dict) else []
|
||
return self._book_candidates_from_api_rows(rows, limit=limit)
|
||
|
||
def _book_candidates_from_api_rows(self, rows: list[dict[str, Any]], *, limit: int) -> list[BookCandidate]:
|
||
candidates: list[BookCandidate] = []
|
||
seen: set[str] = set()
|
||
for row in rows:
|
||
book_id = str(row.get("id") or row.get("dirid") or "").strip()
|
||
title = _normalize_space(str(row.get("title") or ""))
|
||
if not book_id or not title:
|
||
continue
|
||
href = self._reader_book_url(book_id)
|
||
if href in seen:
|
||
continue
|
||
seen.add(href)
|
||
candidates.append(
|
||
BookCandidate(
|
||
title=title,
|
||
url=href,
|
||
author=_normalize_space(str(row.get("author") or "")),
|
||
latest_chapter=_normalize_space(str(row.get("lastchapter") or "")),
|
||
)
|
||
)
|
||
if len(candidates) >= limit:
|
||
break
|
||
return candidates
|
||
|
||
def _keyword_parts(self, keyword: str) -> list[str]:
|
||
normalized = _normalize_space(keyword).lower()
|
||
if not normalized:
|
||
return []
|
||
parts = [part for part in re.split(r"[\s/|,,、]+", normalized) if part]
|
||
return sorted(set(parts), key=len, reverse=True)
|
||
|
||
def _candidate_score(self, candidate: BookCandidate, keyword_parts: list[str]) -> int:
|
||
if not keyword_parts:
|
||
return 0
|
||
title = candidate.title.lower()
|
||
author = candidate.author.lower()
|
||
score = -1
|
||
for part in keyword_parts:
|
||
if title == part:
|
||
score = max(score, 200)
|
||
elif part in title:
|
||
score = max(score, 120 + min(40, len(part) * 4))
|
||
if author == part:
|
||
score = max(score, 100)
|
||
elif part in author:
|
||
score = max(score, 70 + min(20, len(part) * 2))
|
||
return score
|
||
|
||
def _extract_book_candidates(
|
||
self,
|
||
html: str,
|
||
base_url: str,
|
||
*,
|
||
limit: int,
|
||
keyword: str = "",
|
||
) -> list[BookCandidate]:
|
||
ranked: list[tuple[int, BookCandidate]] = []
|
||
seen: set[str] = set()
|
||
search_mode = "/s" in base_url.lower() or "search" in base_url.lower()
|
||
keyword_parts = self._keyword_parts(keyword)
|
||
|
||
for match in _BOOK_LINK_RE.finditer(html):
|
||
href = urljoin(base_url, match.group("href"))
|
||
if href in seen:
|
||
continue
|
||
seen.add(href)
|
||
title = _normalize_space(match.group("title"))
|
||
if not title or len(title) > 120:
|
||
continue
|
||
author = self._extract_author_near_anchor(html, match.start())
|
||
latest = self._extract_latest_chapter_near_anchor(html, match.start())
|
||
candidate = BookCandidate(title=title, url=href, author=author, latest_chapter=latest)
|
||
score = self._candidate_score(candidate, keyword_parts)
|
||
if search_mode and keyword_parts and score < 0:
|
||
continue
|
||
ranked.append((score, candidate))
|
||
|
||
ranked.sort(key=lambda item: (-item[0], len(item[1].title), item[1].title))
|
||
return [item[1] for item in ranked[:limit]]
|
||
|
||
def _extract_author_near_anchor(self, html: str, index: int) -> str:
|
||
snippet = html[max(0, index - 260) : min(len(html), index + 260)]
|
||
patterns = (
|
||
r"作者[::]\s*</[^>]+>\s*([^<]{1,40})",
|
||
r"作者[::]\s*([^<\s]{1,40})",
|
||
r"</a>\s*([^<]{1,40})\s*</",
|
||
)
|
||
for pattern in patterns:
|
||
match = re.search(pattern, snippet, re.I)
|
||
if match:
|
||
author = _normalize_space(match.group(1))
|
||
if author and author != "最新章节":
|
||
return author
|
||
return ""
|
||
|
||
def _extract_latest_chapter_near_anchor(self, html: str, index: int) -> str:
|
||
snippet = html[max(0, index - 260) : min(len(html), index + 360)]
|
||
match = re.search(r"最新章(?:节|節)?[::]?\s*([^<]{1,80})", snippet, re.I)
|
||
return _normalize_space(match.group(1)) if match else ""
|
||
|
||
def _book_data_from_api(self, book_id: str) -> dict[str, Any]:
|
||
referer = self._reader_book_url(book_id)
|
||
try:
|
||
book_data = self._api_get_json("/api/book", payload={"id": int(book_id)}, referer=referer)
|
||
except Exception:
|
||
book_data = self._api_get_json(
|
||
"/api/book",
|
||
params={"id": book_id},
|
||
host=READER_BASE_URL,
|
||
referer=referer,
|
||
)
|
||
|
||
book_list_id = str(
|
||
(book_data or {}).get("dirid")
|
||
or (book_data or {}).get("id")
|
||
or book_id
|
||
).strip()
|
||
|
||
try:
|
||
book_list = self._api_get_json("/api/booklist", payload={"id": int(book_list_id)}, referer=referer)
|
||
except Exception:
|
||
book_list = self._api_get_json(
|
||
"/api/booklist",
|
||
params={"id": book_list_id},
|
||
host=READER_BASE_URL,
|
||
referer=referer,
|
||
)
|
||
|
||
if not isinstance(book_data, dict):
|
||
raise PluginParseError(f"书籍 {book_id} 信息返回异常。")
|
||
if not isinstance(book_list, dict):
|
||
raise PluginParseError(f"书籍 {book_id} 目录返回异常。")
|
||
book_data["_chapter_list"] = list(book_list.get("list") or [])
|
||
return book_data
|
||
|
||
def _chapter_resources_from_book_data(self, book_data: dict[str, Any]) -> list[Resource]:
|
||
book_id = str(book_data.get("id") or book_data.get("dirid") or "").strip()
|
||
if not book_id:
|
||
raise PluginParseError("书籍目录缺少书籍 ID。")
|
||
|
||
book_title = _normalize_space(str(book_data.get("title") or ""))
|
||
author = _normalize_space(str(book_data.get("author") or ""))
|
||
description = _normalize_space(str(book_data.get("intro") or ""))
|
||
chapter_list = list(book_data.get("_chapter_list") or [])
|
||
resources: list[Resource] = []
|
||
|
||
for order, raw_title in enumerate(chapter_list, start=1):
|
||
chapter_title = _normalize_space(str(raw_title or ""))
|
||
if not chapter_title:
|
||
continue
|
||
chapter_id = str(order)
|
||
source_id = f"{book_id}:{chapter_id}"
|
||
resources.append(
|
||
Resource(
|
||
id=source_id,
|
||
url=self._reader_chapter_url(book_id, chapter_id),
|
||
title=chapter_title,
|
||
author=author,
|
||
author_id="",
|
||
files=[],
|
||
metadata={
|
||
"kind": "novel_chapter",
|
||
"resource_type": "novel",
|
||
"content_type": "novel",
|
||
"source_platform": "biquge",
|
||
"source_id": source_id,
|
||
"book_id": book_id,
|
||
"book_title": book_title,
|
||
"series_id": book_id,
|
||
"series_title": book_title,
|
||
"chapter_title": chapter_title,
|
||
"chapter_order": self._extract_maybe_numeric_order(chapter_title, fallback=order),
|
||
"description": description,
|
||
"last_update": str(book_data.get("lastupdate") or ""),
|
||
"book_status": str(book_data.get("full") or ""),
|
||
},
|
||
)
|
||
)
|
||
|
||
return resources
|
||
|
||
def _chapter_resource_from_ids(self, book_id: str, chapter_id: str, *, source_url: str = "") -> Resource:
|
||
data = self._chapter_data_from_api(book_id, chapter_id, source_url=source_url)
|
||
return Resource(
|
||
id=str(data["source_id"]),
|
||
url=source_url or self._reader_chapter_url(book_id, chapter_id),
|
||
title=str(data["chapter_title"]),
|
||
author=str(data.get("author") or ""),
|
||
author_id="",
|
||
files=[],
|
||
metadata={
|
||
"kind": "novel_chapter",
|
||
"resource_type": "novel",
|
||
"content_type": "novel",
|
||
"source_platform": "biquge",
|
||
"source_id": str(data["source_id"]),
|
||
"book_id": str(data.get("book_id") or ""),
|
||
"book_title": str(data.get("book_title") or ""),
|
||
"series_id": str(data.get("book_id") or ""),
|
||
"series_title": str(data.get("book_title") or ""),
|
||
"chapter_title": str(data["chapter_title"]),
|
||
"chapter_order": int(data.get("chapter_order") or 1),
|
||
"description": str(data.get("description") or ""),
|
||
"content": str(data.get("content") or ""),
|
||
"word_count": int(data.get("word_count") or 0),
|
||
},
|
||
)
|
||
|
||
def _chapter_data_from_api(self, book_id: str, chapter_id: str, *, source_url: str = "") -> dict[str, Any]:
|
||
referer = source_url or self._reader_chapter_url(book_id, chapter_id)
|
||
payload = {"id": int(book_id), "chapterid": int(chapter_id)}
|
||
data = self._api_get_json("/api/chapter", payload=payload, referer=referer)
|
||
if not isinstance(data, dict):
|
||
raise PluginParseError(f"章节 {book_id}/{chapter_id} 返回异常。")
|
||
|
||
chapter_title = _normalize_space(str(data.get("chaptername") or f"第 {chapter_id} 章"))
|
||
book_title = _normalize_space(str(data.get("title") or ""))
|
||
author = _normalize_space(str(data.get("author") or ""))
|
||
content = self._clean_api_content(str(data.get("txt") or ""))
|
||
if not content:
|
||
raise PluginParseError(f"章节《{chapter_title or chapter_id}》正文为空。")
|
||
|
||
order_value = str(data.get("chapterid") or chapter_id)
|
||
chapter_order = self._extract_maybe_numeric_order(
|
||
chapter_title,
|
||
fallback=int(order_value) if order_value.isdigit() else 1,
|
||
)
|
||
source_id = f"{book_id}:{chapter_id}"
|
||
return {
|
||
"source_id": source_id,
|
||
"book_id": str(book_id),
|
||
"book_title": book_title,
|
||
"chapter_title": chapter_title,
|
||
"chapter_order": chapter_order,
|
||
"author": author,
|
||
"description": "",
|
||
"content": content,
|
||
"word_count": len(content),
|
||
}
|
||
|
||
def _clean_api_content(self, raw: str) -> str:
|
||
text = raw.replace("\r", "")
|
||
lines: list[str] = []
|
||
for raw_line in text.splitlines():
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
if "请收藏本站" in line:
|
||
break
|
||
lines.append(line)
|
||
return "\n\n".join(lines).strip()
|
||
|
||
def _parse_chapter_page(self, url: str) -> dict[str, Any]:
|
||
ids = self._extract_ids_from_url(url)
|
||
if ids.book_id and ids.chapter_id:
|
||
try:
|
||
return self._chapter_data_from_api(ids.book_id, ids.chapter_id, source_url=url)
|
||
except Exception:
|
||
pass
|
||
|
||
html = self._fetch_html(url)
|
||
if self._looks_like_verify_page(html):
|
||
return self._parse_chapter_page_with_playwright(url)
|
||
|
||
chapter_title = self._extract_h1(html) or self._extract_title_tag(html) or "未命名章节"
|
||
book_title = self._extract_book_title(html)
|
||
author = self._extract_author(html)
|
||
description = self._extract_description(html)
|
||
book_id = ids.book_id or self._extract_book_id(url, html)
|
||
chapter_id = ids.chapter_id or "1"
|
||
chapter_order = self._extract_maybe_numeric_order(
|
||
chapter_title,
|
||
fallback=int(chapter_id) if chapter_id.isdigit() else 1,
|
||
)
|
||
content = self._extract_chapter_content(html)
|
||
if (
|
||
not content
|
||
or chapter_title.strip() == "加载中……"
|
||
or book_title.strip() == "加载中……"
|
||
or "function getCookie" in content
|
||
or "/userverify" in content
|
||
):
|
||
return self._parse_chapter_page_with_playwright(
|
||
url,
|
||
book_title=book_title,
|
||
author=author,
|
||
description=description,
|
||
book_id=book_id,
|
||
chapter_id=chapter_id,
|
||
chapter_title=chapter_title,
|
||
chapter_order=chapter_order,
|
||
)
|
||
|
||
source_id = f"{book_id}:{chapter_id}" if book_id else chapter_id
|
||
return {
|
||
"source_id": source_id,
|
||
"book_id": book_id,
|
||
"book_title": book_title,
|
||
"chapter_title": chapter_title,
|
||
"chapter_order": chapter_order,
|
||
"author": author,
|
||
"description": description,
|
||
"content": content,
|
||
"word_count": len(content),
|
||
}
|
||
|
||
def _looks_like_verify_page(self, html: str) -> bool:
|
||
lowered = html.lower()
|
||
return (
|
||
"/userverify" in lowered
|
||
or "/user/verify.html" in lowered
|
||
or "<title>加载中" in html
|
||
or "function getcookie" in lowered
|
||
or "location.href = \"/userverify" in lowered
|
||
)
|
||
|
||
def _parse_chapter_page_with_playwright(
|
||
self,
|
||
url: str,
|
||
*,
|
||
book_title: str = "",
|
||
author: str = "",
|
||
description: str = "",
|
||
book_id: str = "",
|
||
chapter_id: str = "",
|
||
chapter_title: str = "",
|
||
chapter_order: int = 1,
|
||
) -> dict[str, Any]:
|
||
if not PLAYWRIGHT_AVAILABLE or sync_playwright is None:
|
||
raise PluginParseError("当前章节触发了站点校验,请安装 Playwright Chromium 后再抓取。")
|
||
|
||
try:
|
||
page = self._ensure_browser_page()
|
||
page.goto(url, wait_until="domcontentloaded", timeout=120000)
|
||
page.wait_for_selector("#title", timeout=30000)
|
||
page.wait_for_selector("#chaptercontent", timeout=30000)
|
||
page.wait_for_function(
|
||
"""
|
||
() => {
|
||
const title = document.querySelector('#title')?.innerText?.trim() || '';
|
||
const text = document.querySelector('#chaptercontent')?.innerText?.trim() || '';
|
||
return title.length > 0 && text.length > 40 && !text.includes('加载中');
|
||
}
|
||
""",
|
||
timeout=45000,
|
||
)
|
||
page.wait_for_timeout(800)
|
||
except Exception as exc:
|
||
raise PluginParseError(f"Playwright 解析章节失败:{exc}") from exc
|
||
|
||
resolved_chapter_title = page.locator("#title").inner_text().strip() or chapter_title or "未命名章节"
|
||
resolved_content = self._clean_browser_content(page.locator("#chaptercontent").inner_text())
|
||
if not resolved_content:
|
||
raise PluginParseError(f"章节《{resolved_chapter_title}》正文为空。")
|
||
|
||
resolved_book_title = book_title
|
||
try:
|
||
if page.locator("#pathname a").count() > 0:
|
||
resolved_book_title = page.locator("#pathname a").first.inner_text().strip() or resolved_book_title
|
||
except Exception:
|
||
pass
|
||
if not resolved_book_title:
|
||
resolved_book_title = self._extract_book_title(page.content())
|
||
|
||
resolved_author = author or self._extract_author(page.content())
|
||
resolved_ids = self._extract_ids_from_url(page.url)
|
||
resolved_book_id = resolved_ids.book_id or book_id
|
||
resolved_chapter_id = resolved_ids.chapter_id or chapter_id
|
||
|
||
resolved_chapter_order = chapter_order
|
||
if not resolved_chapter_order or resolved_chapter_order <= 0:
|
||
resolved_chapter_order = self._extract_maybe_numeric_order(
|
||
resolved_chapter_title,
|
||
fallback=int(resolved_chapter_id) if resolved_chapter_id.isdigit() else 1,
|
||
)
|
||
|
||
source_id = (
|
||
f"{resolved_book_id}:{resolved_chapter_id}"
|
||
if resolved_book_id and resolved_chapter_id
|
||
else (resolved_chapter_id or resolved_book_id or chapter_id or "1")
|
||
)
|
||
return {
|
||
"source_id": source_id,
|
||
"book_id": resolved_book_id,
|
||
"book_title": resolved_book_title,
|
||
"chapter_title": resolved_chapter_title,
|
||
"chapter_order": resolved_chapter_order,
|
||
"author": resolved_author,
|
||
"description": description,
|
||
"content": resolved_content,
|
||
"word_count": len(resolved_content),
|
||
}
|
||
|
||
def _ensure_browser_page(self):
|
||
with self._browser_lock:
|
||
if self._page is not None and not self._page.is_closed():
|
||
return self._page
|
||
|
||
if self._browser is None:
|
||
try:
|
||
configure_playwright_runtime_env()
|
||
self._playwright = sync_playwright().start()
|
||
launch_kw: dict[str, Any] = {"headless": True}
|
||
executable = preferred_chromium_executable()
|
||
if executable is not None:
|
||
launch_kw["executable_path"] = str(executable)
|
||
self._browser = self._playwright.chromium.launch(**launch_kw)
|
||
except Exception as exc:
|
||
self._close_browser()
|
||
raise PluginParseError(
|
||
explain_missing_browser(exc, feature_name="笔趣阁章节解析")
|
||
) from exc
|
||
|
||
self._page = self._browser.new_page()
|
||
self._page.set_extra_http_headers({"User-Agent": USER_AGENT})
|
||
return self._page
|
||
|
||
def _close_browser(self) -> None:
|
||
with self._browser_lock:
|
||
page = self._page
|
||
browser = self._browser
|
||
playwright = self._playwright
|
||
self._page = None
|
||
self._browser = None
|
||
self._playwright = None
|
||
|
||
try:
|
||
if page is not None and not page.is_closed():
|
||
page.close()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if browser is not None:
|
||
browser.close()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
if playwright is not None:
|
||
playwright.stop()
|
||
except Exception:
|
||
pass
|
||
|
||
def _clean_browser_content(self, raw: str) -> str:
|
||
text = raw.replace("\r", "")
|
||
lines: list[str] = []
|
||
for raw_line in text.splitlines():
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
if "请收藏本站" in line:
|
||
break
|
||
if any(marker in line for marker in ("点此报错", "加入书签", "上一章", "下一章", "目录")):
|
||
continue
|
||
lines.append(line)
|
||
return "\n\n".join(lines).strip()
|
||
|
||
def _extract_chapter_content(self, html: str) -> str:
|
||
area_match = re.search(
|
||
r'<div[^>]+id="chaptercontent"[^>]*>(?P<body>.*?)</div>',
|
||
html,
|
||
re.I | re.S,
|
||
)
|
||
if area_match is None:
|
||
area_match = re.search(
|
||
r'<div[^>]+(?:id|class)="[^"]*(?:chaptercontent|content|readcontent|article|txt)[^"]*"[^>]*>(?P<body>.*?)</div>',
|
||
html,
|
||
re.I | re.S,
|
||
)
|
||
area = area_match.group("body") if area_match else html
|
||
text = _clean_html_text(area)
|
||
lines: list[str] = []
|
||
for raw_line in text.splitlines():
|
||
line = raw_line.strip()
|
||
if not line:
|
||
continue
|
||
if any(marker in line for marker in ("最新网址", "上一章", "下一章", "加入书签", "返回目录")):
|
||
continue
|
||
lines.append(line)
|
||
return "\n\n".join(lines).strip()
|
||
|
||
def _extract_chapter_resources(self, html: str, page_url: str) -> list[Resource]:
|
||
area = html
|
||
marker = re.search(r"(章节目录|章节列表|目录)", html, re.I)
|
||
if marker:
|
||
area = html[marker.start() :]
|
||
|
||
book_title = self._extract_book_title(html)
|
||
author = self._extract_author(html)
|
||
description = self._extract_description(html)
|
||
book_id = self._extract_book_id(page_url, html)
|
||
seen: set[str] = set()
|
||
resources: list[Resource] = []
|
||
|
||
for order, match in enumerate(_CHAPTER_LINK_RE.finditer(area), start=1):
|
||
href = urljoin(page_url, match.group("href"))
|
||
if href in seen:
|
||
continue
|
||
seen.add(href)
|
||
chapter_title = _normalize_space(match.group("title"))
|
||
if not chapter_title or chapter_title in {"上一章", "下一章", "返回目录"}:
|
||
continue
|
||
chapter_match = _CHAPTER_PATH_RE.search(urlparse(href).path)
|
||
chapter_id = chapter_match.group("chapter_id") if chapter_match else str(order)
|
||
source_id = f"{book_id}:{chapter_id}" if book_id else chapter_id
|
||
resources.append(
|
||
Resource(
|
||
id=source_id,
|
||
url=href,
|
||
title=chapter_title,
|
||
author=author,
|
||
author_id="",
|
||
files=[],
|
||
metadata={
|
||
"kind": "novel_chapter",
|
||
"resource_type": "novel",
|
||
"content_type": "novel",
|
||
"source_platform": "biquge",
|
||
"source_id": source_id,
|
||
"book_id": book_id,
|
||
"book_title": book_title,
|
||
"series_id": book_id,
|
||
"series_title": book_title,
|
||
"chapter_title": chapter_title,
|
||
"chapter_order": self._extract_maybe_numeric_order(chapter_title, fallback=order),
|
||
"description": description,
|
||
},
|
||
)
|
||
)
|
||
|
||
return resources
|
||
|
||
def _extract_book_title(self, html: str) -> str:
|
||
patterns = (
|
||
r'<meta[^>]+property="og:novel:book_name"[^>]+content="([^"]+)"',
|
||
r'<meta[^>]+property="og:title"[^>]+content="([^"]+)"',
|
||
r'<h1[^>]*>(.*?)</h1>',
|
||
r'<title>(.*?)</title>',
|
||
)
|
||
for pattern in patterns:
|
||
match = re.search(pattern, html, re.I | re.S)
|
||
if match:
|
||
text = _normalize_space(match.group(1))
|
||
if text:
|
||
return re.sub(r"\s*最新章节.*$", "", text, flags=re.I).strip(" -")
|
||
match = re.search(r'<a[^>]+href="/book/\d+/"[^>]*>(.*?)</a>', html, re.I | re.S)
|
||
if match:
|
||
return _normalize_space(match.group(1))
|
||
return ""
|
||
|
||
def _extract_author(self, html: str) -> str:
|
||
patterns = (
|
||
r"作者[::]\s*</span>\s*([^<]{1,40})",
|
||
r"作者[::]\s*([^<\s]{1,40})",
|
||
r'<meta[^>]+property="og:novel:author"[^>]+content="([^"]+)"',
|
||
)
|
||
for pattern in patterns:
|
||
match = re.search(pattern, html, re.I)
|
||
if match:
|
||
return _normalize_space(match.group(1))
|
||
return ""
|
||
|
||
def _extract_description(self, html: str) -> str:
|
||
patterns = (
|
||
r'<meta[^>]+name="description"[^>]+content="([^"]+)"',
|
||
r'<meta[^>]+property="og:description"[^>]+content="([^"]+)"',
|
||
r'<div[^>]+class="intro"[^>]*>(.*?)</div>',
|
||
)
|
||
for pattern in patterns:
|
||
match = re.search(pattern, html, re.I | re.S)
|
||
if match:
|
||
return _normalize_space(match.group(1))
|
||
return ""
|
||
|
||
def _extract_h1(self, html: str) -> str:
|
||
match = re.search(r"<h1[^>]*>(.*?)</h1>", html, re.I | re.S)
|
||
return _normalize_space(match.group(1)) if match else ""
|
||
|
||
def _extract_title_tag(self, html: str) -> str:
|
||
match = re.search(r"<title>(.*?)</title>", html, re.I | re.S)
|
||
return _normalize_space(match.group(1)) if match else ""
|
||
|
||
def _extract_book_id(self, url: str, html: str) -> str:
|
||
ids = self._extract_ids_from_url(url)
|
||
if ids.book_id:
|
||
return ids.book_id
|
||
match = _BOOK_PATH_RE.search(html)
|
||
return match.group("book_id") if match else ""
|
||
|
||
def _extract_maybe_numeric_order(self, chapter_title: str, *, fallback: int) -> int:
|
||
match = re.search(r"第\s*([0-9]+|[零一二三四五六七八九十百千两]+)\s*(?:章|话|卷|回|节|篇)", chapter_title)
|
||
if match:
|
||
value = self._cn_number_to_int(match.group(1))
|
||
if value > 0:
|
||
return value
|
||
return fallback
|
||
|
||
def _cn_number_to_int(self, text: str) -> int:
|
||
if text.isdigit():
|
||
return int(text)
|
||
|
||
digits = {
|
||
"零": 0,
|
||
"一": 1,
|
||
"二": 2,
|
||
"两": 2,
|
||
"三": 3,
|
||
"四": 4,
|
||
"五": 5,
|
||
"六": 6,
|
||
"七": 7,
|
||
"八": 8,
|
||
"九": 9,
|
||
}
|
||
units = {"十": 10, "百": 100, "千": 1000}
|
||
total = 0
|
||
current = 0
|
||
|
||
for char in text:
|
||
if char in digits:
|
||
current = digits[char]
|
||
elif char in units:
|
||
if current == 0:
|
||
current = 1
|
||
total += current * units[char]
|
||
current = 0
|
||
|
||
return total + current
|
||
|
||
def _reader_book_url(self, book_id: str) -> str:
|
||
return f"{READER_BASE_URL}/#/book/{book_id}/"
|
||
|
||
def _reader_chapter_url(self, book_id: str, chapter_id: str) -> str:
|
||
return f"{READER_BASE_URL}/#/book/{book_id}/{chapter_id}.html"
|
||
|
||
def _api_headers(self, referer: str = "") -> dict[str, str]:
|
||
return {
|
||
"User-Agent": USER_AGENT,
|
||
"Accept": "application/json, text/javascript, */*; q=0.01",
|
||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||
"X-Requested-With": "XMLHttpRequest",
|
||
"Referer": referer or (READER_BASE_URL + "/"),
|
||
}
|
||
|
||
def _api_get_json(
|
||
self,
|
||
endpoint: str,
|
||
*,
|
||
params: dict[str, Any] | None = None,
|
||
payload: dict[str, Any] | None = None,
|
||
host: str = API_BASE_URL,
|
||
referer: str = "",
|
||
) -> Any:
|
||
request_params = dict(params or {})
|
||
if payload is not None:
|
||
request_params["token"] = self._encode_token(payload)
|
||
|
||
url = host.rstrip("/") + endpoint
|
||
response = self.session.get(
|
||
url,
|
||
params=request_params,
|
||
timeout=25,
|
||
headers=self._api_headers(referer),
|
||
)
|
||
response.raise_for_status()
|
||
return response.json()
|
||
|
||
def _encode_token(self, payload: dict[str, Any]) -> str:
|
||
if not CRYPTOGRAPHY_AVAILABLE or padding is None or Cipher is None or algorithms is None or modes is None:
|
||
raise PluginParseError(
|
||
"当前环境缺少 cryptography,无法生成 apibi 访问 token。"
|
||
)
|
||
|
||
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||
padder = padding.PKCS7(128).padder()
|
||
padded = padder.update(raw) + padder.finalize()
|
||
encryptor = Cipher(algorithms.AES(TOKEN_KEY), modes.CBC(TOKEN_IV)).encryptor()
|
||
encrypted = encryptor.update(padded) + encryptor.finalize()
|
||
return base64.b64encode(encrypted).decode("ascii")
|
||
|
||
|
||
plugin_class = BiqugePlugin
|