Files
SakuraHub/plugins/biquge/plugin.py
T

1133 lines
43 KiB
Python
Raw Normal View History

2026-11-16 20:45:22 +08:00
"""
Biquge novel plugin backed by the reader site's public API.
Primary flow:
- search books via https://apibi.cc/api/search
- resolve books via https://apibi.cc/api/book + /api/booklist
- fetch chapter text via https://apibi.cc/api/chapter
The plugin still keeps limited HTML / Playwright fallbacks for legacy pages.
"""
from __future__ import annotations
import atexit
import base64
import hashlib
import json
import os
import re
import threading
from dataclasses import dataclass
from html import unescape
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, quote, urljoin, urlparse
import requests
try:
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
CRYPTOGRAPHY_AVAILABLE = True
except Exception:
padding = None
Cipher = None
algorithms = None
modes = None
CRYPTOGRAPHY_AVAILABLE = False
from pixiv_app.core.downloader import sanitize_filename, save_text_file
from pixiv_app.core.playwright_runtime import (
configure_playwright_runtime_env,
explain_missing_browser,
preferred_chromium_executable,
)
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
try:
from playwright.sync_api import sync_playwright
PLAYWRIGHT_AVAILABLE = True
except Exception:
sync_playwright = None
PLAYWRIGHT_AVAILABLE = False
LEGACY_BASE_URL = "https://www.bqgui.cc"
READER_BASE_URL = "https://www.bqg551.cc"
API_BASE_URL = "https://apibi.cc"
TOKEN_SECRET = os.environ.get("SAKURA_BIQUGE_TOKEN_SECRET", "book@token.html")
USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
)
TOKEN_MD5 = hashlib.md5(TOKEN_SECRET.encode("utf-8")).hexdigest()
TOKEN_IV = TOKEN_MD5[:16].encode("utf-8")
TOKEN_KEY = TOKEN_MD5[16:].encode("utf-8")
SORT_SLUGS = {
"xuanhuan",
"wuxia",
"dushi",
"lishi",
"wangyou",
"kehuan",
"mm",
"top",
"finish",
}
_BOOK_PATH_RE = re.compile(r"/book/(?P<book_id>\d+)/?$", re.I)
_CHAPTER_PATH_RE = re.compile(
r"/book/(?P<book_id>\d+)/(?P<chapter_id>\d+)(?:_\d+)?\.html(?:[?#].*)?$",
re.I,
)
_HASH_BOOK_RE = re.compile(r"^book/(?P<book_id>\d+)/?$", re.I)
_HASH_CHAPTER_RE = re.compile(
r"^book/(?P<book_id>\d+)/(?P<chapter_id>\d+)(?:_\d+)?\.html$",
re.I,
)
_BOOK_LINK_RE = re.compile(
r"""<a[^>]+href\s*=\s*["'](?P<href>/book/\d+/)["'][^>]*>(?P<title>.*?)</a>""",
re.I | re.S,
)
_CHAPTER_LINK_RE = re.compile(
r"""<a[^>]+href\s*=\s*["'](?P<href>/book/\d+/\d+\.html)["'][^>]*>(?P<title>.*?)</a>""",
re.I | re.S,
)
@dataclass
class BookCandidate:
title: str
url: str
author: str = ""
latest_chapter: str = ""
@dataclass
class TargetIds:
book_id: str = ""
chapter_id: str = ""
def _clean_html_text(raw: str) -> str:
text = re.sub(r"<br\s*/?>", "\n", raw, flags=re.I)
text = re.sub(r"</p\s*>", "\n", text, flags=re.I)
text = re.sub(r"<[^>]+>", "", text)
text = unescape(text)
text = text.replace("\r", "")
text = text.replace("\u3000", " ")
lines = [line.strip().strip('"').strip("'") for line in text.splitlines()]
return "\n".join(line for line in lines if line)
def _normalize_space(value: str) -> str:
return re.sub(r"\s+", " ", _clean_html_text(value)).strip()
class BiqugePlugin(BasePlugin):
name = "笔趣阁"
domain = "www.bqg551.cc"
version = "3.0.0"
def __init__(self) -> None:
self.session = requests.Session()
self.session.headers.update(self.get_headers())
self._playwright = None
self._browser = None
self._page = None
self._browser_lock = threading.Lock()
atexit.register(self._close_browser)
def validate(self) -> bool:
return True
def can_handle(self, url: str) -> bool:
lowered = url.strip().lower()
return any(host in lowered for host in ("bqgui.cc", "bqg551.cc", "apibi.cc"))
def get_headers(self) -> dict[str, str]:
return {
"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9",
"Referer": READER_BASE_URL + "/",
}
def resolve_target(self, target: str, *, limit: int = 20) -> dict[str, Any]:
raw = target.strip()
if not raw:
raise PluginParseError("请输入书名、作者、书籍链接或章节链接。")
normalized = self._normalize_target(raw)
if normalized:
route = self._parse_reader_route(normalized)
if route["kind"] == "search":
keyword = route["keyword"]
books = self._search_books(keyword, limit=limit)
if not books:
raise PluginParseError(f"没有搜索到与“{keyword}”相关的书籍。")
return {
"mode": "books",
"title": f"搜索结果:{keyword}",
"books": [candidate.__dict__ for candidate in books],
}
if route["kind"] == "sort":
books = self._reader_sort_books(str(route["slug"]), limit=limit)
if not books:
raise PluginParseError("该列表页没有解析到书籍。")
return {
"mode": "books",
"title": f"列表结果:{route['slug']}",
"books": [candidate.__dict__ for candidate in books],
}
if route["kind"] == "index":
books = self._reader_index_books(limit=limit)
if not books:
raise PluginParseError("首页推荐列表为空。")
return {
"mode": "books",
"title": "首页推荐",
"books": [candidate.__dict__ for candidate in books],
}
ids = self._extract_ids_from_url(normalized)
if ids.book_id and ids.chapter_id:
chapter = self._chapter_resource_from_ids(ids.book_id, ids.chapter_id, source_url=normalized)
return {
"mode": "chapters",
"book_title": str(chapter.metadata.get("book_title") or ""),
"resources": [chapter],
}
if ids.book_id:
return self._resolve_book_id(ids.book_id)
return self._resolve_listing_url(normalized, limit=limit)
if raw.isdigit():
return self._resolve_book_id(raw)
books = self._search_books(raw, limit=limit)
if not books:
raise PluginParseError(f"没有搜索到与“{raw}”相关的书籍。")
return {
"mode": "books",
"title": f"搜索结果:{raw}",
"books": [candidate.__dict__ for candidate in books],
}
def parse(self, url: str) -> list[Resource]:
outcome = self.resolve_target(url, limit=20)
if outcome.get("mode") == "chapters":
return list(outcome.get("resources") or [])
books = list(outcome.get("books") or [])
if not books:
raise PluginParseError("未解析到可下载内容。")
selected = self.resolve_target(str(books[0].get("url") or ""), limit=20)
resources = list(selected.get("resources") or [])
if not resources:
raise PluginParseError("选中的书籍没有章节内容。")
return resources
def download(self, resource: Resource, save_path: Path) -> list[Path]:
if str(resource.metadata.get("kind") or "") != "novel_chapter":
raise PluginParseError("不支持的笔趣阁资源类型。")
content = str(resource.metadata.get("content") or "")
if not content.strip():
chapter_data = self._parse_chapter_page(resource.url)
content = chapter_data["content"]
resource.metadata.update(chapter_data)
book_title = str(resource.metadata.get("book_title") or "未命名小说")
chapter_title = str(resource.metadata.get("chapter_title") or resource.title or "未命名章节")
chapter_order = int(resource.metadata.get("chapter_order") or 0)
folder = save_path / "novels" / sanitize_filename(book_title, "biquge_book")
filename = (
f"{chapter_order:04d}_{sanitize_filename(chapter_title, 'chapter')}.txt"
if chapter_order
else f"{sanitize_filename(chapter_title, 'chapter')}.txt"
)
final_path = folder / filename
text = "\n".join(
[
f"书名: {book_title}",
f"章节: {chapter_title}",
f"作者: {resource.author or '未知'}",
f"链接: {resource.url}",
"",
content.strip(),
]
).strip()
save_text_file(final_path, text)
resource.metadata["word_count"] = len(content)
return [final_path]
def _normalize_target(self, target: str) -> str:
value = target.strip()
lowered = value.lower()
if value.startswith("//"):
return "https:" + value
if value.startswith("#/"):
return READER_BASE_URL + "/" + value
if value.startswith("/#/"):
return READER_BASE_URL + value
if value.startswith("/book/"):
return urljoin(LEGACY_BASE_URL, value)
if value.startswith("/"):
return urljoin(LEGACY_BASE_URL, value)
if lowered.startswith(("http://", "https://")):
return value
if lowered.startswith(("www.bqgui.cc", "m.bqgui.cc", "www.bqg551.cc", "bqg551.cc", "apibi.cc")):
return "https://" + value
return ""
def _parse_reader_route(self, url: str) -> dict[str, str]:
parsed = urlparse(url)
fragment = parsed.fragment.lstrip("/")
if not fragment:
return {"kind": "none"}
fragment_path, _, fragment_query = fragment.partition("?")
cleaned_path = fragment_path.strip("/")
if not cleaned_path:
return {"kind": "index"}
if cleaned_path == "search":
keyword = parse_qs(fragment_query).get("q", [""])[0].strip()
return {"kind": "search", "keyword": keyword}
if cleaned_path in SORT_SLUGS:
return {"kind": "sort", "slug": cleaned_path}
return {"kind": "none"}
def _extract_ids_from_url(self, url: str) -> TargetIds:
parsed = urlparse(url)
for source in (parsed.path, parsed.fragment.partition("?")[0]):
if not source:
continue
normalized = source if source.startswith("/") else f"/{source}"
match = _CHAPTER_PATH_RE.search(normalized)
if match:
return TargetIds(book_id=match.group("book_id"), chapter_id=match.group("chapter_id"))
match = _BOOK_PATH_RE.search(normalized)
if match:
return TargetIds(book_id=match.group("book_id"))
cleaned = source.strip("/")
match = _HASH_CHAPTER_RE.match(cleaned)
if match:
return TargetIds(book_id=match.group("book_id"), chapter_id=match.group("chapter_id"))
match = _HASH_BOOK_RE.match(cleaned)
if match:
return TargetIds(book_id=match.group("book_id"))
return TargetIds()
def _resolve_book_id(self, book_id: str) -> dict[str, Any]:
book_data = self._book_data_from_api(book_id)
chapters = self._chapter_resources_from_book_data(book_data)
if not chapters:
raise PluginParseError(f"书籍 {book_id} 没有解析到章节目录。")
return {
"mode": "chapters",
"book_title": str(chapters[0].metadata.get("book_title") or ""),
"resources": chapters,
}
def _resolve_listing_url(self, url: str, *, limit: int) -> dict[str, Any]:
html = self._fetch_html(url)
chapters = self._extract_chapter_resources(html, url)
if chapters:
return {
"mode": "chapters",
"book_title": str(chapters[0].metadata.get("book_title") or ""),
"resources": chapters,
}
books = self._extract_book_candidates(html, url, limit=limit)
if books:
return {
"mode": "books",
"title": "请选择要抓取的书籍",
"books": [candidate.__dict__ for candidate in books],
}
raise PluginParseError("该页面没有解析到书籍或章节列表。")
def _fetch_html(self, url: str, *, params: dict[str, str] | None = None) -> str:
response = self.session.get(url, params=params, timeout=25)
response.raise_for_status()
if not response.encoding or response.encoding.lower() in {"iso-8859-1", "ascii"}:
response.encoding = response.apparent_encoding or "utf-8"
return response.text
def _search_books(self, keyword: str, *, limit: int) -> list[BookCandidate]:
books = self._search_books_api(keyword, limit=limit)
if books:
return books
attempts = [
{"url": f"{LEGACY_BASE_URL}/s", "params": {"q": keyword}},
{"url": f"{LEGACY_BASE_URL}/search", "params": {"q": keyword}},
]
for attempt in attempts:
try:
html = self._fetch_html(str(attempt["url"]), params=attempt.get("params"))
except Exception:
continue
books = self._extract_book_candidates(html, str(attempt["url"]), limit=limit, keyword=keyword)
if books:
return books
return []
def _search_books_api(self, keyword: str, *, limit: int) -> list[BookCandidate]:
try:
payload = self._api_get_json(
"/api/search",
params={"q": keyword},
referer=f"{READER_BASE_URL}/#/search?q={quote(keyword)}",
)
except Exception:
return []
rows = list(payload.get("data") or []) if isinstance(payload, dict) else []
ranked: list[tuple[int, BookCandidate]] = []
seen: set[str] = set()
keyword_parts = self._keyword_parts(keyword)
normalized_keyword = _normalize_space(keyword).lower()
for row in rows:
if not isinstance(row, dict):
continue
book_id = str(row.get("id") or row.get("dirid") or "").strip()
title = _normalize_space(str(row.get("title") or ""))
if not book_id or not title:
continue
href = self._reader_book_url(book_id)
if href in seen:
continue
seen.add(href)
author = _normalize_space(str(row.get("author") or ""))
latest = _normalize_space(str(row.get("lastchapter") or ""))
candidate = BookCandidate(title=title, url=href, author=author, latest_chapter=latest)
score = self._candidate_score(candidate, keyword_parts)
title_lower = title.lower()
author_lower = author.lower()
if normalized_keyword and title_lower == normalized_keyword:
score += 1000
elif normalized_keyword and title_lower.startswith(normalized_keyword):
score += 200
elif normalized_keyword and normalized_keyword in title_lower:
score += 120
elif normalized_keyword and normalized_keyword in author_lower:
score += 60
ranked.append((score, candidate))
ranked.sort(key=lambda item: (-item[0], len(item[1].title), item[1].title))
return [item[1] for item in ranked[:limit]]
def _reader_index_books(self, *, limit: int) -> list[BookCandidate]:
payload = self._api_get_json("/api/index", params={"sort": "index"}, referer=READER_BASE_URL + "/#/")
if not isinstance(payload, dict):
return []
rows: list[dict[str, Any]] = []
for value in payload.values():
if isinstance(value, list):
rows.extend(item for item in value if isinstance(item, dict))
return self._book_candidates_from_api_rows(rows, limit=limit)
def _reader_sort_books(self, slug: str, *, limit: int) -> list[BookCandidate]:
payload = self._api_get_json(
"/api/sort",
params={"sort": slug},
referer=f"{READER_BASE_URL}/#/{slug}",
)
rows = list(payload.get("data") or []) if isinstance(payload, dict) else []
return self._book_candidates_from_api_rows(rows, limit=limit)
def _book_candidates_from_api_rows(self, rows: list[dict[str, Any]], *, limit: int) -> list[BookCandidate]:
candidates: list[BookCandidate] = []
seen: set[str] = set()
for row in rows:
book_id = str(row.get("id") or row.get("dirid") or "").strip()
title = _normalize_space(str(row.get("title") or ""))
if not book_id or not title:
continue
href = self._reader_book_url(book_id)
if href in seen:
continue
seen.add(href)
candidates.append(
BookCandidate(
title=title,
url=href,
author=_normalize_space(str(row.get("author") or "")),
latest_chapter=_normalize_space(str(row.get("lastchapter") or "")),
)
)
if len(candidates) >= limit:
break
return candidates
def _keyword_parts(self, keyword: str) -> list[str]:
normalized = _normalize_space(keyword).lower()
if not normalized:
return []
parts = [part for part in re.split(r"[\s/|,,、]+", normalized) if part]
return sorted(set(parts), key=len, reverse=True)
def _candidate_score(self, candidate: BookCandidate, keyword_parts: list[str]) -> int:
if not keyword_parts:
return 0
title = candidate.title.lower()
author = candidate.author.lower()
score = -1
for part in keyword_parts:
if title == part:
score = max(score, 200)
elif part in title:
score = max(score, 120 + min(40, len(part) * 4))
if author == part:
score = max(score, 100)
elif part in author:
score = max(score, 70 + min(20, len(part) * 2))
return score
def _extract_book_candidates(
self,
html: str,
base_url: str,
*,
limit: int,
keyword: str = "",
) -> list[BookCandidate]:
ranked: list[tuple[int, BookCandidate]] = []
seen: set[str] = set()
search_mode = "/s" in base_url.lower() or "search" in base_url.lower()
keyword_parts = self._keyword_parts(keyword)
for match in _BOOK_LINK_RE.finditer(html):
href = urljoin(base_url, match.group("href"))
if href in seen:
continue
seen.add(href)
title = _normalize_space(match.group("title"))
if not title or len(title) > 120:
continue
author = self._extract_author_near_anchor(html, match.start())
latest = self._extract_latest_chapter_near_anchor(html, match.start())
candidate = BookCandidate(title=title, url=href, author=author, latest_chapter=latest)
score = self._candidate_score(candidate, keyword_parts)
if search_mode and keyword_parts and score < 0:
continue
ranked.append((score, candidate))
ranked.sort(key=lambda item: (-item[0], len(item[1].title), item[1].title))
return [item[1] for item in ranked[:limit]]
def _extract_author_near_anchor(self, html: str, index: int) -> str:
snippet = html[max(0, index - 260) : min(len(html), index + 260)]
patterns = (
r"作者[::]\s*</[^>]+>\s*([^<]{1,40})",
r"作者[::]\s*([^<\s]{1,40})",
r"</a>\s*([^<]{1,40})\s*</",
)
for pattern in patterns:
match = re.search(pattern, snippet, re.I)
if match:
author = _normalize_space(match.group(1))
if author and author != "最新章节":
return author
return ""
def _extract_latest_chapter_near_anchor(self, html: str, index: int) -> str:
snippet = html[max(0, index - 260) : min(len(html), index + 360)]
match = re.search(r"最新章(?:节|節)?[::]?\s*([^<]{1,80})", snippet, re.I)
return _normalize_space(match.group(1)) if match else ""
def _book_data_from_api(self, book_id: str) -> dict[str, Any]:
referer = self._reader_book_url(book_id)
try:
book_data = self._api_get_json("/api/book", payload={"id": int(book_id)}, referer=referer)
except Exception:
book_data = self._api_get_json(
"/api/book",
params={"id": book_id},
host=READER_BASE_URL,
referer=referer,
)
book_list_id = str(
(book_data or {}).get("dirid")
or (book_data or {}).get("id")
or book_id
).strip()
try:
book_list = self._api_get_json("/api/booklist", payload={"id": int(book_list_id)}, referer=referer)
except Exception:
book_list = self._api_get_json(
"/api/booklist",
params={"id": book_list_id},
host=READER_BASE_URL,
referer=referer,
)
if not isinstance(book_data, dict):
raise PluginParseError(f"书籍 {book_id} 信息返回异常。")
if not isinstance(book_list, dict):
raise PluginParseError(f"书籍 {book_id} 目录返回异常。")
book_data["_chapter_list"] = list(book_list.get("list") or [])
return book_data
def _chapter_resources_from_book_data(self, book_data: dict[str, Any]) -> list[Resource]:
book_id = str(book_data.get("id") or book_data.get("dirid") or "").strip()
if not book_id:
raise PluginParseError("书籍目录缺少书籍 ID。")
book_title = _normalize_space(str(book_data.get("title") or ""))
author = _normalize_space(str(book_data.get("author") or ""))
description = _normalize_space(str(book_data.get("intro") or ""))
chapter_list = list(book_data.get("_chapter_list") or [])
resources: list[Resource] = []
for order, raw_title in enumerate(chapter_list, start=1):
chapter_title = _normalize_space(str(raw_title or ""))
if not chapter_title:
continue
chapter_id = str(order)
source_id = f"{book_id}:{chapter_id}"
resources.append(
Resource(
id=source_id,
url=self._reader_chapter_url(book_id, chapter_id),
title=chapter_title,
author=author,
author_id="",
files=[],
metadata={
"kind": "novel_chapter",
"resource_type": "novel",
"content_type": "novel",
"source_platform": "biquge",
"source_id": source_id,
"book_id": book_id,
"book_title": book_title,
"series_id": book_id,
"series_title": book_title,
"chapter_title": chapter_title,
"chapter_order": self._extract_maybe_numeric_order(chapter_title, fallback=order),
"description": description,
"last_update": str(book_data.get("lastupdate") or ""),
"book_status": str(book_data.get("full") or ""),
},
)
)
return resources
def _chapter_resource_from_ids(self, book_id: str, chapter_id: str, *, source_url: str = "") -> Resource:
data = self._chapter_data_from_api(book_id, chapter_id, source_url=source_url)
return Resource(
id=str(data["source_id"]),
url=source_url or self._reader_chapter_url(book_id, chapter_id),
title=str(data["chapter_title"]),
author=str(data.get("author") or ""),
author_id="",
files=[],
metadata={
"kind": "novel_chapter",
"resource_type": "novel",
"content_type": "novel",
"source_platform": "biquge",
"source_id": str(data["source_id"]),
"book_id": str(data.get("book_id") or ""),
"book_title": str(data.get("book_title") or ""),
"series_id": str(data.get("book_id") or ""),
"series_title": str(data.get("book_title") or ""),
"chapter_title": str(data["chapter_title"]),
"chapter_order": int(data.get("chapter_order") or 1),
"description": str(data.get("description") or ""),
"content": str(data.get("content") or ""),
"word_count": int(data.get("word_count") or 0),
},
)
def _chapter_data_from_api(self, book_id: str, chapter_id: str, *, source_url: str = "") -> dict[str, Any]:
referer = source_url or self._reader_chapter_url(book_id, chapter_id)
payload = {"id": int(book_id), "chapterid": int(chapter_id)}
data = self._api_get_json("/api/chapter", payload=payload, referer=referer)
if not isinstance(data, dict):
raise PluginParseError(f"章节 {book_id}/{chapter_id} 返回异常。")
chapter_title = _normalize_space(str(data.get("chaptername") or f"第 {chapter_id} 章"))
book_title = _normalize_space(str(data.get("title") or ""))
author = _normalize_space(str(data.get("author") or ""))
content = self._clean_api_content(str(data.get("txt") or ""))
if not content:
raise PluginParseError(f"章节《{chapter_title or chapter_id}》正文为空。")
order_value = str(data.get("chapterid") or chapter_id)
chapter_order = self._extract_maybe_numeric_order(
chapter_title,
fallback=int(order_value) if order_value.isdigit() else 1,
)
source_id = f"{book_id}:{chapter_id}"
return {
"source_id": source_id,
"book_id": str(book_id),
"book_title": book_title,
"chapter_title": chapter_title,
"chapter_order": chapter_order,
"author": author,
"description": "",
"content": content,
"word_count": len(content),
}
def _clean_api_content(self, raw: str) -> str:
text = raw.replace("\r", "")
lines: list[str] = []
for raw_line in text.splitlines():
line = raw_line.strip()
if not line:
continue
if "请收藏本站" in line:
break
lines.append(line)
return "\n\n".join(lines).strip()
def _parse_chapter_page(self, url: str) -> dict[str, Any]:
ids = self._extract_ids_from_url(url)
if ids.book_id and ids.chapter_id:
try:
return self._chapter_data_from_api(ids.book_id, ids.chapter_id, source_url=url)
except Exception:
pass
html = self._fetch_html(url)
if self._looks_like_verify_page(html):
return self._parse_chapter_page_with_playwright(url)
chapter_title = self._extract_h1(html) or self._extract_title_tag(html) or "未命名章节"
book_title = self._extract_book_title(html)
author = self._extract_author(html)
description = self._extract_description(html)
book_id = ids.book_id or self._extract_book_id(url, html)
chapter_id = ids.chapter_id or "1"
chapter_order = self._extract_maybe_numeric_order(
chapter_title,
fallback=int(chapter_id) if chapter_id.isdigit() else 1,
)
content = self._extract_chapter_content(html)
if (
not content
or chapter_title.strip() == "加载中……"
or book_title.strip() == "加载中……"
or "function getCookie" in content
or "/userverify" in content
):
return self._parse_chapter_page_with_playwright(
url,
book_title=book_title,
author=author,
description=description,
book_id=book_id,
chapter_id=chapter_id,
chapter_title=chapter_title,
chapter_order=chapter_order,
)
source_id = f"{book_id}:{chapter_id}" if book_id else chapter_id
return {
"source_id": source_id,
"book_id": book_id,
"book_title": book_title,
"chapter_title": chapter_title,
"chapter_order": chapter_order,
"author": author,
"description": description,
"content": content,
"word_count": len(content),
}
def _looks_like_verify_page(self, html: str) -> bool:
lowered = html.lower()
return (
"/userverify" in lowered
or "/user/verify.html" in lowered
or "<title>加载中" in html
or "function getcookie" in lowered
or "location.href = \"/userverify" in lowered
)
def _parse_chapter_page_with_playwright(
self,
url: str,
*,
book_title: str = "",
author: str = "",
description: str = "",
book_id: str = "",
chapter_id: str = "",
chapter_title: str = "",
chapter_order: int = 1,
) -> dict[str, Any]:
if not PLAYWRIGHT_AVAILABLE or sync_playwright is None:
raise PluginParseError("当前章节触发了站点校验,请安装 Playwright Chromium 后再抓取。")
try:
page = self._ensure_browser_page()
page.goto(url, wait_until="domcontentloaded", timeout=120000)
page.wait_for_selector("#title", timeout=30000)
page.wait_for_selector("#chaptercontent", timeout=30000)
page.wait_for_function(
"""
() => {
const title = document.querySelector('#title')?.innerText?.trim() || '';
const text = document.querySelector('#chaptercontent')?.innerText?.trim() || '';
return title.length > 0 && text.length > 40 && !text.includes('加载中');
}
""",
timeout=45000,
)
page.wait_for_timeout(800)
except Exception as exc:
raise PluginParseError(f"Playwright 解析章节失败:{exc}") from exc
resolved_chapter_title = page.locator("#title").inner_text().strip() or chapter_title or "未命名章节"
resolved_content = self._clean_browser_content(page.locator("#chaptercontent").inner_text())
if not resolved_content:
raise PluginParseError(f"章节《{resolved_chapter_title}》正文为空。")
resolved_book_title = book_title
try:
if page.locator("#pathname a").count() > 0:
resolved_book_title = page.locator("#pathname a").first.inner_text().strip() or resolved_book_title
except Exception:
pass
if not resolved_book_title:
resolved_book_title = self._extract_book_title(page.content())
resolved_author = author or self._extract_author(page.content())
resolved_ids = self._extract_ids_from_url(page.url)
resolved_book_id = resolved_ids.book_id or book_id
resolved_chapter_id = resolved_ids.chapter_id or chapter_id
resolved_chapter_order = chapter_order
if not resolved_chapter_order or resolved_chapter_order <= 0:
resolved_chapter_order = self._extract_maybe_numeric_order(
resolved_chapter_title,
fallback=int(resolved_chapter_id) if resolved_chapter_id.isdigit() else 1,
)
source_id = (
f"{resolved_book_id}:{resolved_chapter_id}"
if resolved_book_id and resolved_chapter_id
else (resolved_chapter_id or resolved_book_id or chapter_id or "1")
)
return {
"source_id": source_id,
"book_id": resolved_book_id,
"book_title": resolved_book_title,
"chapter_title": resolved_chapter_title,
"chapter_order": resolved_chapter_order,
"author": resolved_author,
"description": description,
"content": resolved_content,
"word_count": len(resolved_content),
}
def _ensure_browser_page(self):
with self._browser_lock:
if self._page is not None and not self._page.is_closed():
return self._page
if self._browser is None:
try:
configure_playwright_runtime_env()
self._playwright = sync_playwright().start()
launch_kw: dict[str, Any] = {"headless": True}
executable = preferred_chromium_executable()
if executable is not None:
launch_kw["executable_path"] = str(executable)
self._browser = self._playwright.chromium.launch(**launch_kw)
except Exception as exc:
self._close_browser()
raise PluginParseError(
explain_missing_browser(exc, feature_name="笔趣阁章节解析")
) from exc
self._page = self._browser.new_page()
self._page.set_extra_http_headers({"User-Agent": USER_AGENT})
return self._page
def _close_browser(self) -> None:
with self._browser_lock:
page = self._page
browser = self._browser
playwright = self._playwright
self._page = None
self._browser = None
self._playwright = None
try:
if page is not None and not page.is_closed():
page.close()
except Exception:
pass
try:
if browser is not None:
browser.close()
except Exception:
pass
try:
if playwright is not None:
playwright.stop()
except Exception:
pass
def _clean_browser_content(self, raw: str) -> str:
text = raw.replace("\r", "")
lines: list[str] = []
for raw_line in text.splitlines():
line = raw_line.strip()
if not line:
continue
if "请收藏本站" in line:
break
if any(marker in line for marker in ("点此报错", "加入书签", "上一章", "下一章", "目录")):
continue
lines.append(line)
return "\n\n".join(lines).strip()
def _extract_chapter_content(self, html: str) -> str:
area_match = re.search(
r'<div[^>]+id="chaptercontent"[^>]*>(?P<body>.*?)</div>',
html,
re.I | re.S,
)
if area_match is None:
area_match = re.search(
r'<div[^>]+(?:id|class)="[^"]*(?:chaptercontent|content|readcontent|article|txt)[^"]*"[^>]*>(?P<body>.*?)</div>',
html,
re.I | re.S,
)
area = area_match.group("body") if area_match else html
text = _clean_html_text(area)
lines: list[str] = []
for raw_line in text.splitlines():
line = raw_line.strip()
if not line:
continue
if any(marker in line for marker in ("最新网址", "上一章", "下一章", "加入书签", "返回目录")):
continue
lines.append(line)
return "\n\n".join(lines).strip()
def _extract_chapter_resources(self, html: str, page_url: str) -> list[Resource]:
area = html
marker = re.search(r"(章节目录|章节列表|目录)", html, re.I)
if marker:
area = html[marker.start() :]
book_title = self._extract_book_title(html)
author = self._extract_author(html)
description = self._extract_description(html)
book_id = self._extract_book_id(page_url, html)
seen: set[str] = set()
resources: list[Resource] = []
for order, match in enumerate(_CHAPTER_LINK_RE.finditer(area), start=1):
href = urljoin(page_url, match.group("href"))
if href in seen:
continue
seen.add(href)
chapter_title = _normalize_space(match.group("title"))
if not chapter_title or chapter_title in {"上一章", "下一章", "返回目录"}:
continue
chapter_match = _CHAPTER_PATH_RE.search(urlparse(href).path)
chapter_id = chapter_match.group("chapter_id") if chapter_match else str(order)
source_id = f"{book_id}:{chapter_id}" if book_id else chapter_id
resources.append(
Resource(
id=source_id,
url=href,
title=chapter_title,
author=author,
author_id="",
files=[],
metadata={
"kind": "novel_chapter",
"resource_type": "novel",
"content_type": "novel",
"source_platform": "biquge",
"source_id": source_id,
"book_id": book_id,
"book_title": book_title,
"series_id": book_id,
"series_title": book_title,
"chapter_title": chapter_title,
"chapter_order": self._extract_maybe_numeric_order(chapter_title, fallback=order),
"description": description,
},
)
)
return resources
def _extract_book_title(self, html: str) -> str:
patterns = (
r'<meta[^>]+property="og:novel:book_name"[^>]+content="([^"]+)"',
r'<meta[^>]+property="og:title"[^>]+content="([^"]+)"',
r'<h1[^>]*>(.*?)</h1>',
r'<title>(.*?)</title>',
)
for pattern in patterns:
match = re.search(pattern, html, re.I | re.S)
if match:
text = _normalize_space(match.group(1))
if text:
return re.sub(r"\s*最新章节.*$", "", text, flags=re.I).strip(" -")
match = re.search(r'<a[^>]+href="/book/\d+/"[^>]*>(.*?)</a>', html, re.I | re.S)
if match:
return _normalize_space(match.group(1))
return ""
def _extract_author(self, html: str) -> str:
patterns = (
r"作者[::]\s*</span>\s*([^<]{1,40})",
r"作者[::]\s*([^<\s]{1,40})",
r'<meta[^>]+property="og:novel:author"[^>]+content="([^"]+)"',
)
for pattern in patterns:
match = re.search(pattern, html, re.I)
if match:
return _normalize_space(match.group(1))
return ""
def _extract_description(self, html: str) -> str:
patterns = (
r'<meta[^>]+name="description"[^>]+content="([^"]+)"',
r'<meta[^>]+property="og:description"[^>]+content="([^"]+)"',
r'<div[^>]+class="intro"[^>]*>(.*?)</div>',
)
for pattern in patterns:
match = re.search(pattern, html, re.I | re.S)
if match:
return _normalize_space(match.group(1))
return ""
def _extract_h1(self, html: str) -> str:
match = re.search(r"<h1[^>]*>(.*?)</h1>", html, re.I | re.S)
return _normalize_space(match.group(1)) if match else ""
def _extract_title_tag(self, html: str) -> str:
match = re.search(r"<title>(.*?)</title>", html, re.I | re.S)
return _normalize_space(match.group(1)) if match else ""
def _extract_book_id(self, url: str, html: str) -> str:
ids = self._extract_ids_from_url(url)
if ids.book_id:
return ids.book_id
match = _BOOK_PATH_RE.search(html)
return match.group("book_id") if match else ""
def _extract_maybe_numeric_order(self, chapter_title: str, *, fallback: int) -> int:
match = re.search(r"第\s*([0-9]+|[零一二三四五六七八九十百千两]+)\s*(?:章|话|卷|回|节|篇)", chapter_title)
if match:
value = self._cn_number_to_int(match.group(1))
if value > 0:
return value
return fallback
def _cn_number_to_int(self, text: str) -> int:
if text.isdigit():
return int(text)
digits = {
"零": 0,
"一": 1,
"二": 2,
"两": 2,
"三": 3,
"四": 4,
"五": 5,
"六": 6,
"七": 7,
"八": 8,
"九": 9,
}
units = {"十": 10, "百": 100, "千": 1000}
total = 0
current = 0
for char in text:
if char in digits:
current = digits[char]
elif char in units:
if current == 0:
current = 1
total += current * units[char]
current = 0
return total + current
def _reader_book_url(self, book_id: str) -> str:
return f"{READER_BASE_URL}/#/book/{book_id}/"
def _reader_chapter_url(self, book_id: str, chapter_id: str) -> str:
return f"{READER_BASE_URL}/#/book/{book_id}/{chapter_id}.html"
def _api_headers(self, referer: str = "") -> dict[str, str]:
return {
"User-Agent": USER_AGENT,
"Accept": "application/json, text/javascript, */*; q=0.01",
"Accept-Language": "zh-CN,zh;q=0.9",
"X-Requested-With": "XMLHttpRequest",
"Referer": referer or (READER_BASE_URL + "/"),
}
def _api_get_json(
self,
endpoint: str,
*,
params: dict[str, Any] | None = None,
payload: dict[str, Any] | None = None,
host: str = API_BASE_URL,
referer: str = "",
) -> Any:
request_params = dict(params or {})
if payload is not None:
request_params["token"] = self._encode_token(payload)
url = host.rstrip("/") + endpoint
response = self.session.get(
url,
params=request_params,
timeout=25,
headers=self._api_headers(referer),
)
response.raise_for_status()
return response.json()
def _encode_token(self, payload: dict[str, Any]) -> str:
if not CRYPTOGRAPHY_AVAILABLE or padding is None or Cipher is None or algorithms is None or modes is None:
raise PluginParseError(
"当前环境缺少 cryptography,无法生成 apibi 访问 token。"
)
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
padder = padding.PKCS7(128).padder()
padded = padder.update(raw) + padder.finalize()
encryptor = Cipher(algorithms.AES(TOKEN_KEY), modes.CBC(TOKEN_IV)).encryptor()
encrypted = encryptor.update(padded) + encryptor.finalize()
return base64.b64encode(encrypted).decode("ascii")
plugin_class = BiqugePlugin