1007 lines
36 KiB
Python
1007 lines
36 KiB
Python
from __future__ import annotations
|
|||
|
|
|
||
|
|
import concurrent.futures
|
||
|
|
import html
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import threading
|
||
|
|
import time
|
||
|
|
from dataclasses import dataclass
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Callable, Optional
|
||
|
|
from urllib.parse import quote, urlparse
|
||
|
|
|
||
|
|
import requests
|
||
|
|
|
||
|
|
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, DownloadFileRecord, PixivLibrary
|
||
|
|
from pixiv_app.core.proxy_pool import ProxyPool
|
||
|
|
|
||
|
|
|
||
|
|
USER_AGENT = (
|
||
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
|
|
"Chrome/123.0.0.0 Safari/537.36"
|
||
|
|
)
|
||
|
|
DEFAULT_TIMEOUT = 30
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class DownloadResult:
|
||
|
|
work_id: int
|
||
|
|
total_pages: int
|
||
|
|
downloaded_pages: int
|
||
|
|
skipped_pages: int
|
||
|
|
failed_pages: int
|
||
|
|
ok: bool
|
||
|
|
message: str = ""
|
||
|
|
|
||
|
|
|
||
|
|
class PixivRequestError(RuntimeError):
|
||
|
|
def __init__(self, message: str, status_code: Optional[int] = None) -> None:
|
||
|
|
super().__init__(message)
|
||
|
|
self.status_code = status_code
|
||
|
|
|
||
|
|
|
||
|
|
def sanitize_filename(name: str, fallback: str) -> str:
|
||
|
|
cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1F]+', "_", str(name)).strip().rstrip(".")
|
||
|
|
return cleaned or fallback
|
||
|
|
|
||
|
|
|
||
|
|
def build_headers(cookie: str = "", referer: Optional[str] = None) -> dict[str, str]:
|
||
|
|
headers = {
|
||
|
|
"User-Agent": USER_AGENT,
|
||
|
|
"Accept": "application/json, text/plain, */*",
|
||
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
|
|
"Origin": "https://www.pixiv.net",
|
||
|
|
}
|
||
|
|
headers["Referer"] = referer or "https://www.pixiv.net/"
|
||
|
|
if cookie:
|
||
|
|
headers["Cookie"] = cookie
|
||
|
|
return headers
|
||
|
|
|
||
|
|
|
||
|
|
def create_session(cookie: str = "") -> requests.Session:
|
||
|
|
session = requests.Session()
|
||
|
|
session.headers.update(build_headers(cookie=cookie))
|
||
|
|
return session
|
||
|
|
|
||
|
|
|
||
|
|
def ensure_success(response: requests.Response) -> None:
|
||
|
|
response.raise_for_status()
|
||
|
|
|
||
|
|
|
||
|
|
def parse_pixiv_target(raw_value: str) -> tuple[str, int]:
|
||
|
|
text = raw_value.strip()
|
||
|
|
if not text:
|
||
|
|
raise ValueError("请输入 Pixiv 作者 ID、作品 ID、小说 ID,或对应链接。")
|
||
|
|
|
||
|
|
patterns = (
|
||
|
|
(r"/users/(\d+)", "user"),
|
||
|
|
(r"/artworks/(\d+)", "illust"),
|
||
|
|
(r"[?&]illust_id=(\d+)", "illust"),
|
||
|
|
(r"/novel/series/(\d+)", "novel_series"),
|
||
|
|
(r"/novel/show\.php\?id=(\d+)", "novel"),
|
||
|
|
(r"/novel/show\.php.*?[?&]id=(\d+)", "novel"),
|
||
|
|
(r"/novel/(\d+)", "novel"),
|
||
|
|
)
|
||
|
|
for pattern, target_kind in patterns:
|
||
|
|
match = re.search(pattern, text, flags=re.IGNORECASE)
|
||
|
|
if match:
|
||
|
|
return target_kind, int(match.group(1))
|
||
|
|
|
||
|
|
if text.isdigit():
|
||
|
|
return "unknown", int(text)
|
||
|
|
|
||
|
|
raise ValueError("无法识别输入内容,请填写 Pixiv 作者 ID、作品 ID、小说 ID,或完整链接。")
|
||
|
|
|
||
|
|
|
||
|
|
def extract_work_ids(payload: dict) -> list[int]:
|
||
|
|
body = payload.get("body", {})
|
||
|
|
work_ids: set[int] = set()
|
||
|
|
for section in ("illusts", "manga"):
|
||
|
|
section_data = body.get(section, {})
|
||
|
|
if not isinstance(section_data, dict):
|
||
|
|
continue
|
||
|
|
for key in section_data.keys():
|
||
|
|
if str(key).isdigit():
|
||
|
|
work_ids.add(int(key))
|
||
|
|
return sorted(work_ids, reverse=True)
|
||
|
|
|
||
|
|
|
||
|
|
def _pick_proxy(proxy_pool: Optional[ProxyPool]) -> tuple[Optional[dict[str, str]], object]:
|
||
|
|
if not proxy_pool:
|
||
|
|
return None, None
|
||
|
|
proxy = proxy_pool.get_proxy(rotate=True)
|
||
|
|
if not proxy:
|
||
|
|
return None, None
|
||
|
|
return {"http": proxy.proxy_url, "https": proxy.proxy_url}, proxy
|
||
|
|
|
||
|
|
|
||
|
|
def request_json(
|
||
|
|
session: requests.Session,
|
||
|
|
url: str,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
referer: Optional[str] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
context: str = "Pixiv 请求",
|
||
|
|
) -> dict:
|
||
|
|
last_error: Optional[Exception] = None
|
||
|
|
for _ in range(3):
|
||
|
|
proxies, proxy_obj = _pick_proxy(proxy_pool)
|
||
|
|
try:
|
||
|
|
response = session.get(
|
||
|
|
url,
|
||
|
|
headers=build_headers(cookie=cookie, referer=referer),
|
||
|
|
timeout=DEFAULT_TIMEOUT,
|
||
|
|
proxies=proxies,
|
||
|
|
)
|
||
|
|
if response.status_code == 404:
|
||
|
|
raise PixivRequestError(f"{context}不存在,或当前账号无权访问。", status_code=404)
|
||
|
|
ensure_success(response)
|
||
|
|
payload = response.json()
|
||
|
|
if isinstance(payload, dict) and payload.get("error"):
|
||
|
|
message = str(payload.get("message") or "返回了错误响应")
|
||
|
|
raise PixivRequestError(f"{context}失败: {message}", status_code=response.status_code)
|
||
|
|
if not isinstance(payload, dict):
|
||
|
|
raise PixivRequestError(f"{context}返回格式异常。", status_code=response.status_code)
|
||
|
|
return payload
|
||
|
|
except PixivRequestError as exc:
|
||
|
|
last_error = exc
|
||
|
|
if proxy_pool and proxy_obj:
|
||
|
|
proxy_pool.mark_failed(proxy_obj)
|
||
|
|
if exc.status_code == 404:
|
||
|
|
break
|
||
|
|
time.sleep(0.5)
|
||
|
|
except Exception as exc:
|
||
|
|
last_error = exc
|
||
|
|
if proxy_pool and proxy_obj:
|
||
|
|
proxy_pool.mark_failed(proxy_obj)
|
||
|
|
time.sleep(0.5)
|
||
|
|
|
||
|
|
if last_error:
|
||
|
|
if isinstance(last_error, PixivRequestError):
|
||
|
|
raise last_error
|
||
|
|
raise PixivRequestError(f"{context}失败: {last_error}")
|
||
|
|
raise PixivRequestError(f"{context}失败。")
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_illust_details(
|
||
|
|
illust_id: int,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
) -> dict:
|
||
|
|
close_session = session is None
|
||
|
|
session = session or create_session(cookie)
|
||
|
|
try:
|
||
|
|
payload = request_json(
|
||
|
|
session,
|
||
|
|
f"https://www.pixiv.net/ajax/illust/{illust_id}?lang=zh",
|
||
|
|
cookie=cookie,
|
||
|
|
referer=f"https://www.pixiv.net/artworks/{illust_id}",
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
context=f"作品 {illust_id} 信息请求",
|
||
|
|
)
|
||
|
|
return payload.get("body", {}) if isinstance(payload.get("body"), dict) else {}
|
||
|
|
finally:
|
||
|
|
if close_session:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_user_profile(
|
||
|
|
user_id: int,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
) -> dict:
|
||
|
|
close_session = session is None
|
||
|
|
session = session or create_session(cookie)
|
||
|
|
try:
|
||
|
|
payload = request_json(
|
||
|
|
session,
|
||
|
|
f"https://www.pixiv.net/ajax/user/{int(user_id)}?full=1&lang=zh",
|
||
|
|
cookie=cookie,
|
||
|
|
referer=f"https://www.pixiv.net/users/{int(user_id)}",
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
context=f"作者 {int(user_id)} 信息请求",
|
||
|
|
)
|
||
|
|
body = payload.get("body", {})
|
||
|
|
return body if isinstance(body, dict) else {}
|
||
|
|
finally:
|
||
|
|
if close_session:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def resolve_user_id_from_illust(
|
||
|
|
illust_id: int,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
) -> int:
|
||
|
|
body = fetch_illust_details(
|
||
|
|
illust_id,
|
||
|
|
cookie=cookie,
|
||
|
|
session=session,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
)
|
||
|
|
user_id = body.get("userId")
|
||
|
|
if str(user_id).isdigit():
|
||
|
|
return int(user_id)
|
||
|
|
raise PixivRequestError(f"作品 {illust_id} 未返回有效作者 ID。")
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_user_work_ids(
|
||
|
|
user_id: int,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
input_kind: str = "user",
|
||
|
|
) -> tuple[int, list[int]]:
|
||
|
|
close_session = session is None
|
||
|
|
session = session or create_session(cookie)
|
||
|
|
try:
|
||
|
|
def load_user_works(resolved_user_id: int) -> list[int]:
|
||
|
|
payload = request_json(
|
||
|
|
session,
|
||
|
|
f"https://www.pixiv.net/ajax/user/{resolved_user_id}/profile/all?sensitiveFilterMode=userSetting&lang=zh",
|
||
|
|
cookie=cookie,
|
||
|
|
referer=f"https://www.pixiv.net/users/{resolved_user_id}",
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
context=f"作者 {resolved_user_id} 作品列表请求",
|
||
|
|
)
|
||
|
|
return extract_work_ids(payload)
|
||
|
|
|
||
|
|
if input_kind == "illust":
|
||
|
|
resolved_user_id = resolve_user_id_from_illust(
|
||
|
|
user_id,
|
||
|
|
cookie=cookie,
|
||
|
|
session=session,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
)
|
||
|
|
return resolved_user_id, load_user_works(resolved_user_id)
|
||
|
|
|
||
|
|
try:
|
||
|
|
return user_id, load_user_works(user_id)
|
||
|
|
except PixivRequestError as exc:
|
||
|
|
if input_kind != "unknown" or exc.status_code != 404:
|
||
|
|
raise
|
||
|
|
|
||
|
|
try:
|
||
|
|
resolved_user_id = resolve_user_id_from_illust(
|
||
|
|
user_id,
|
||
|
|
cookie=cookie,
|
||
|
|
session=session,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
)
|
||
|
|
return resolved_user_id, load_user_works(resolved_user_id)
|
||
|
|
except PixivRequestError as exc:
|
||
|
|
if exc.status_code == 404:
|
||
|
|
raise PixivRequestError("输入的 ID 既不是可访问的作者,也不是可访问的作品。", status_code=404) from exc
|
||
|
|
raise
|
||
|
|
finally:
|
||
|
|
if close_session:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_illust_pages(
|
||
|
|
illust_id: int,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
) -> list[str]:
|
||
|
|
close_session = session is None
|
||
|
|
session = session or create_session(cookie)
|
||
|
|
referer = f"https://www.pixiv.net/artworks/{illust_id}"
|
||
|
|
try:
|
||
|
|
payload = request_json(
|
||
|
|
session,
|
||
|
|
f"https://www.pixiv.net/ajax/illust/{illust_id}/pages?lang=zh",
|
||
|
|
cookie=cookie,
|
||
|
|
referer=referer,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
context=f"作品 {illust_id} 页面请求",
|
||
|
|
)
|
||
|
|
body = payload.get("body", [])
|
||
|
|
if isinstance(body, list):
|
||
|
|
urls = [page.get("urls", {}).get("original") for page in body if isinstance(page, dict)]
|
||
|
|
urls = [url for url in urls if url]
|
||
|
|
if urls:
|
||
|
|
return urls
|
||
|
|
|
||
|
|
detail = fetch_illust_details(
|
||
|
|
illust_id,
|
||
|
|
cookie=cookie,
|
||
|
|
session=session,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
)
|
||
|
|
original_url = detail.get("urls", {}).get("original") if isinstance(detail.get("urls"), dict) else None
|
||
|
|
return [original_url] if original_url else []
|
||
|
|
finally:
|
||
|
|
if close_session:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_novel_details(
|
||
|
|
novel_id: int,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
) -> dict:
|
||
|
|
close_session = session is None
|
||
|
|
session = session or create_session(cookie)
|
||
|
|
try:
|
||
|
|
payload = request_json(
|
||
|
|
session,
|
||
|
|
f"https://www.pixiv.net/ajax/novel/{novel_id}?lang=zh",
|
||
|
|
cookie=cookie,
|
||
|
|
referer=f"https://www.pixiv.net/novel/show.php?id={novel_id}",
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
context=f"小说 {novel_id} 信息请求",
|
||
|
|
)
|
||
|
|
return payload.get("body", {}) if isinstance(payload.get("body"), dict) else {}
|
||
|
|
finally:
|
||
|
|
if close_session:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_novel_text(
|
||
|
|
novel_id: int,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
) -> tuple[dict, str]:
|
||
|
|
close_session = session is None
|
||
|
|
session = session or create_session(cookie)
|
||
|
|
try:
|
||
|
|
detail = fetch_novel_details(
|
||
|
|
novel_id,
|
||
|
|
cookie=cookie,
|
||
|
|
session=session,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
)
|
||
|
|
content = detail.get("content")
|
||
|
|
if isinstance(content, str) and content.strip():
|
||
|
|
return detail, content
|
||
|
|
|
||
|
|
referer = f"https://www.pixiv.net/novel/show.php?id={novel_id}"
|
||
|
|
content_body: Optional[dict] = None
|
||
|
|
for url in (
|
||
|
|
f"https://www.pixiv.net/ajax/novel/text/{novel_id}?lang=zh",
|
||
|
|
f"https://www.pixiv.net/ajax/novel/{novel_id}/text?lang=zh",
|
||
|
|
):
|
||
|
|
try:
|
||
|
|
payload = request_json(
|
||
|
|
session,
|
||
|
|
url,
|
||
|
|
cookie=cookie,
|
||
|
|
referer=referer,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
context=f"小说 {novel_id} 正文请求",
|
||
|
|
)
|
||
|
|
body = payload.get("body", {})
|
||
|
|
if isinstance(body, dict):
|
||
|
|
content_body = body
|
||
|
|
break
|
||
|
|
except PixivRequestError as exc:
|
||
|
|
if exc.status_code != 404:
|
||
|
|
raise
|
||
|
|
if content_body is None:
|
||
|
|
raise PixivRequestError(f"未获取到小说 {novel_id} 正文。")
|
||
|
|
|
||
|
|
content = content_body.get("content")
|
||
|
|
if not isinstance(content, str) or not content.strip():
|
||
|
|
raise PixivRequestError(f"小说 {novel_id} 正文为空。")
|
||
|
|
return detail, content
|
||
|
|
finally:
|
||
|
|
if close_session:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_novel_series_ids(
|
||
|
|
series_id: int,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
) -> tuple[str | None, list[int]]:
|
||
|
|
close_session = session is None
|
||
|
|
session = session or create_session(cookie)
|
||
|
|
try:
|
||
|
|
series_title: str | None = None
|
||
|
|
deduped: list[int] = []
|
||
|
|
seen: set[int] = set()
|
||
|
|
|
||
|
|
def add_ids(values: list[int]) -> None:
|
||
|
|
for novel_id in values:
|
||
|
|
if novel_id == series_id or novel_id in seen:
|
||
|
|
continue
|
||
|
|
seen.add(novel_id)
|
||
|
|
deduped.append(novel_id)
|
||
|
|
|
||
|
|
def collect_from_json(value: object) -> list[int]:
|
||
|
|
ids: list[int] = []
|
||
|
|
if isinstance(value, dict):
|
||
|
|
for key, item in value.items():
|
||
|
|
if key in {"novelId", "novel_id", "workId"} and str(item).isdigit():
|
||
|
|
ids.append(int(item))
|
||
|
|
elif (
|
||
|
|
key == "id"
|
||
|
|
and str(item).isdigit()
|
||
|
|
and any(marker in value for marker in ("title", "textCount", "characterCount", "createDate", "url"))
|
||
|
|
):
|
||
|
|
ids.append(int(item))
|
||
|
|
ids.extend(collect_from_json(item))
|
||
|
|
elif isinstance(value, list):
|
||
|
|
for item in value:
|
||
|
|
ids.extend(collect_from_json(item))
|
||
|
|
return ids
|
||
|
|
|
||
|
|
def pick_title(value: object) -> str | None:
|
||
|
|
if isinstance(value, dict):
|
||
|
|
for key in ("title", "seriesTitle", "name"):
|
||
|
|
item = value.get(key)
|
||
|
|
if isinstance(item, str) and item.strip():
|
||
|
|
return item.strip()
|
||
|
|
for item in value.values():
|
||
|
|
found = pick_title(item)
|
||
|
|
if found:
|
||
|
|
return found
|
||
|
|
elif isinstance(value, list):
|
||
|
|
for item in value:
|
||
|
|
found = pick_title(item)
|
||
|
|
if found:
|
||
|
|
return found
|
||
|
|
return None
|
||
|
|
|
||
|
|
for url in (
|
||
|
|
f"https://www.pixiv.net/ajax/novel/series/{series_id}?lang=zh",
|
||
|
|
f"https://www.pixiv.net/ajax/novel/series_content/{series_id}?limit=100&last_order=0&order_by=asc&lang=zh",
|
||
|
|
):
|
||
|
|
try:
|
||
|
|
payload = request_json(
|
||
|
|
session,
|
||
|
|
url,
|
||
|
|
cookie=cookie,
|
||
|
|
referer=f"https://www.pixiv.net/novel/series/{series_id}",
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
context=f"小说合集 {series_id} 列表请求",
|
||
|
|
)
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
body = payload.get("body", payload)
|
||
|
|
series_title = series_title or pick_title(body)
|
||
|
|
add_ids(collect_from_json(body))
|
||
|
|
if deduped:
|
||
|
|
return series_title, deduped
|
||
|
|
|
||
|
|
for page in range(1, 21):
|
||
|
|
before = len(deduped)
|
||
|
|
for url in (
|
||
|
|
f"https://www.pixiv.net/ajax/novel/series/{series_id}?p={page}&lang=zh",
|
||
|
|
(
|
||
|
|
f"https://www.pixiv.net/ajax/novel/series_content/{series_id}"
|
||
|
|
f"?limit=30&last_order={(page - 1) * 30}&order_by=asc&lang=zh"
|
||
|
|
),
|
||
|
|
):
|
||
|
|
try:
|
||
|
|
payload = request_json(
|
||
|
|
session,
|
||
|
|
url,
|
||
|
|
cookie=cookie,
|
||
|
|
referer=f"https://www.pixiv.net/novel/series/{series_id}",
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
context=f"小说合集 {series_id} 第 {page} 页请求",
|
||
|
|
)
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
body = payload.get("body", payload)
|
||
|
|
series_title = series_title or pick_title(body)
|
||
|
|
add_ids(collect_from_json(body))
|
||
|
|
if len(deduped) == before:
|
||
|
|
break
|
||
|
|
|
||
|
|
url = f"https://www.pixiv.net/novel/series/{series_id}"
|
||
|
|
proxies, _proxy_obj = _pick_proxy(proxy_pool)
|
||
|
|
response = session.get(
|
||
|
|
url,
|
||
|
|
headers=build_headers(cookie=cookie, referer=url),
|
||
|
|
timeout=DEFAULT_TIMEOUT,
|
||
|
|
proxies=proxies,
|
||
|
|
)
|
||
|
|
ensure_success(response)
|
||
|
|
page_html = response.text
|
||
|
|
title_match = re.search(r'<meta\s+property="og:title"\s+content="([^"]+)"', page_html, flags=re.IGNORECASE)
|
||
|
|
if title_match and not series_title:
|
||
|
|
series_title = title_match.group(1).strip()
|
||
|
|
add_ids([int(item) for item in re.findall(r"/novel/show\.php\?id=(\d+)", page_html)])
|
||
|
|
add_ids([int(item) for item in re.findall(r'"novelId"\s*:\s*"?(\d+)"?', page_html)])
|
||
|
|
|
||
|
|
preload_matches = re.findall(
|
||
|
|
r'id="(?:meta-preload-data|__NEXT_DATA__)"[^>]*(?:content="([^"]+)"|>(.*?)</script>)',
|
||
|
|
page_html,
|
||
|
|
flags=re.IGNORECASE | re.DOTALL,
|
||
|
|
)
|
||
|
|
for match in preload_matches:
|
||
|
|
raw_json = next((part for part in match if part), "")
|
||
|
|
if not raw_json:
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
preload = json.loads(html.unescape(raw_json))
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
series_title = series_title or pick_title(preload)
|
||
|
|
add_ids(collect_from_json(preload))
|
||
|
|
|
||
|
|
if not deduped:
|
||
|
|
raise PixivRequestError(f"小说合集 {series_id} 未解析到章节,请确认 Cookie 权限或稍后重试。")
|
||
|
|
return series_title, deduped
|
||
|
|
finally:
|
||
|
|
if close_session:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def search_illust_ids_by_keyword(
|
||
|
|
keyword: str,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
limit: int = 30,
|
||
|
|
) -> list[int]:
|
||
|
|
cleaned_keyword = keyword.strip()
|
||
|
|
if not cleaned_keyword:
|
||
|
|
raise ValueError("关键词不能为空。")
|
||
|
|
|
||
|
|
close_session = session is None
|
||
|
|
session = session or create_session(cookie)
|
||
|
|
encoded = quote(cleaned_keyword, safe="")
|
||
|
|
work_ids: list[int] = []
|
||
|
|
seen: set[int] = set()
|
||
|
|
page = 1
|
||
|
|
try:
|
||
|
|
while len(work_ids) < limit and page <= max(3, limit):
|
||
|
|
payload = request_json(
|
||
|
|
session,
|
||
|
|
(
|
||
|
|
f"https://www.pixiv.net/ajax/search/artworks/{encoded}"
|
||
|
|
f"?word={encoded}&order=date_d&mode=all&p={page}&s_mode=s_tag_full&type=all&lang=zh"
|
||
|
|
),
|
||
|
|
cookie=cookie,
|
||
|
|
referer=f"https://www.pixiv.net/tags/{encoded}/artworks",
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
context=f"关键词“{cleaned_keyword}”搜索请求",
|
||
|
|
)
|
||
|
|
body = payload.get("body", {})
|
||
|
|
illust_manga = body.get("illustManga", {}) if isinstance(body, dict) else {}
|
||
|
|
items = illust_manga.get("data", []) if isinstance(illust_manga, dict) else []
|
||
|
|
if not isinstance(items, list) or not items:
|
||
|
|
break
|
||
|
|
|
||
|
|
added_this_page = 0
|
||
|
|
for item in items:
|
||
|
|
item_id = item.get("id") if isinstance(item, dict) else None
|
||
|
|
if str(item_id).isdigit():
|
||
|
|
illust_id = int(item_id)
|
||
|
|
if illust_id not in seen:
|
||
|
|
seen.add(illust_id)
|
||
|
|
work_ids.append(illust_id)
|
||
|
|
added_this_page += 1
|
||
|
|
if len(work_ids) >= limit:
|
||
|
|
break
|
||
|
|
if added_this_page == 0:
|
||
|
|
break
|
||
|
|
page += 1
|
||
|
|
return work_ids[:limit]
|
||
|
|
finally:
|
||
|
|
if close_session:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def is_file_complete(path: Path) -> bool:
|
||
|
|
return path.exists() and path.is_file() and path.stat().st_size > 1024
|
||
|
|
|
||
|
|
|
||
|
|
def download_binary(
|
||
|
|
file_url: str,
|
||
|
|
save_dir: Path,
|
||
|
|
cookie: str = "",
|
||
|
|
session: Optional[requests.Session] = None,
|
||
|
|
timeout: int = DEFAULT_TIMEOUT,
|
||
|
|
retry_count: int = 3,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
) -> tuple[bool, str]:
|
||
|
|
close_session = session is None
|
||
|
|
session = session or create_session(cookie)
|
||
|
|
save_dir.mkdir(parents=True, exist_ok=True)
|
||
|
|
|
||
|
|
filename = os.path.basename(urlparse(file_url).path)
|
||
|
|
save_path = save_dir / filename
|
||
|
|
if is_file_complete(save_path):
|
||
|
|
return True, f"跳过已存在文件: {filename}"
|
||
|
|
temp_path = save_path.with_suffix(save_path.suffix + ".part")
|
||
|
|
|
||
|
|
try:
|
||
|
|
for attempt in range(1, retry_count + 1):
|
||
|
|
proxies, proxy_obj = _pick_proxy(proxy_pool)
|
||
|
|
try:
|
||
|
|
response = session.get(
|
||
|
|
file_url,
|
||
|
|
headers=build_headers(cookie=cookie, referer="https://www.pixiv.net/"),
|
||
|
|
timeout=timeout,
|
||
|
|
stream=True,
|
||
|
|
proxies=proxies,
|
||
|
|
)
|
||
|
|
ensure_success(response)
|
||
|
|
with open(temp_path, "wb") as file_handle:
|
||
|
|
for chunk in response.iter_content(chunk_size=1024 * 256):
|
||
|
|
if chunk:
|
||
|
|
file_handle.write(chunk)
|
||
|
|
if temp_path.stat().st_size <= 1024:
|
||
|
|
raise IOError("文件过小,可能不是有效图片。")
|
||
|
|
temp_path.replace(save_path)
|
||
|
|
return True, f"下载成功: {filename}"
|
||
|
|
except Exception as exc:
|
||
|
|
if proxy_pool and proxy_obj:
|
||
|
|
proxy_pool.mark_failed(proxy_obj)
|
||
|
|
if temp_path.exists():
|
||
|
|
temp_path.unlink()
|
||
|
|
if attempt == retry_count:
|
||
|
|
return False, f"下载失败 {filename}: {exc}"
|
||
|
|
time.sleep(min(0.8 * attempt, 2.0))
|
||
|
|
finally:
|
||
|
|
if close_session:
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def save_text_file(save_path: Path, content: str) -> tuple[bool, str]:
|
||
|
|
save_path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
if save_path.exists() and save_path.stat().st_size > 32:
|
||
|
|
return True, f"跳过已存在文件: {save_path.name}"
|
||
|
|
temp_path = save_path.with_suffix(save_path.suffix + ".part")
|
||
|
|
with open(temp_path, "w", encoding="utf-8-sig") as file_handle:
|
||
|
|
file_handle.write(content)
|
||
|
|
temp_path.replace(save_path)
|
||
|
|
return True, f"保存成功: {save_path.name}"
|
||
|
|
|
||
|
|
|
||
|
|
def download_illust(
|
||
|
|
illust_id: int,
|
||
|
|
user_id: int,
|
||
|
|
cookie: str = "",
|
||
|
|
save_root: str | Path = "Sakura_Downloads",
|
||
|
|
save_subdir: Optional[str] = None,
|
||
|
|
request_delay: float = 0.0,
|
||
|
|
stop_event: Optional[threading.Event] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
library_db_path: str | Path | None = DEFAULT_LIBRARY_DB,
|
||
|
|
) -> DownloadResult:
|
||
|
|
if stop_event and stop_event.is_set():
|
||
|
|
return DownloadResult(illust_id, 0, 0, 0, 0, False, "任务已停止")
|
||
|
|
if request_delay > 0:
|
||
|
|
time.sleep(request_delay)
|
||
|
|
|
||
|
|
session = create_session(cookie)
|
||
|
|
subdir = save_subdir or str(user_id)
|
||
|
|
save_dir = Path(save_root) / subdir
|
||
|
|
library: PixivLibrary | None = PixivLibrary(library_db_path) if library_db_path else None
|
||
|
|
file_records: list[DownloadFileRecord] = []
|
||
|
|
artwork_type = "illust"
|
||
|
|
try:
|
||
|
|
detail = fetch_illust_details(illust_id, cookie=cookie, session=session, proxy_pool=proxy_pool)
|
||
|
|
artwork_type = "manga" if int(detail.get("pageCount") or 0) > 1 or int(detail.get("illustType") or 0) == 1 else "illust"
|
||
|
|
if library is not None:
|
||
|
|
library.ensure_artwork_stub(illust_id, artwork_type=artwork_type)
|
||
|
|
library.upsert_artwork(detail, artwork_type=artwork_type)
|
||
|
|
resolved_user_id = detail.get("userId")
|
||
|
|
if user_id == 0 and str(resolved_user_id).isdigit():
|
||
|
|
user_id = int(resolved_user_id)
|
||
|
|
page_urls = fetch_illust_pages(illust_id, cookie=cookie, session=session, proxy_pool=proxy_pool)
|
||
|
|
if not page_urls:
|
||
|
|
result = DownloadResult(illust_id, 0, 0, 0, 1, False, "未获取到原图地址")
|
||
|
|
if library is not None:
|
||
|
|
library.ensure_artwork_stub(illust_id, artwork_type=artwork_type)
|
||
|
|
library.record_download_history(
|
||
|
|
artwork_id=illust_id,
|
||
|
|
status="failed",
|
||
|
|
downloaded_pages=0,
|
||
|
|
skipped_pages=0,
|
||
|
|
failed_pages=1,
|
||
|
|
message=result.message,
|
||
|
|
)
|
||
|
|
return result
|
||
|
|
|
||
|
|
downloaded_pages, skipped_pages, failed_pages = 0, 0, 0
|
||
|
|
for page_index, page_url in enumerate(page_urls):
|
||
|
|
if stop_event and stop_event.is_set():
|
||
|
|
break
|
||
|
|
filename = os.path.basename(urlparse(page_url).path)
|
||
|
|
target_path = save_dir / filename
|
||
|
|
if is_file_complete(target_path):
|
||
|
|
skipped_pages += 1
|
||
|
|
file_records.append(
|
||
|
|
DownloadFileRecord(
|
||
|
|
page_index=page_index,
|
||
|
|
file_path=str(target_path.resolve()),
|
||
|
|
source_url=page_url,
|
||
|
|
status="skipped",
|
||
|
|
size_bytes=target_path.stat().st_size,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
continue
|
||
|
|
success, _ = download_binary(
|
||
|
|
page_url,
|
||
|
|
save_dir=save_dir,
|
||
|
|
cookie=cookie,
|
||
|
|
session=session,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
)
|
||
|
|
if success:
|
||
|
|
downloaded_pages += 1
|
||
|
|
file_records.append(
|
||
|
|
DownloadFileRecord(
|
||
|
|
page_index=page_index,
|
||
|
|
file_path=str(target_path.resolve()),
|
||
|
|
source_url=page_url,
|
||
|
|
status="downloaded",
|
||
|
|
size_bytes=target_path.stat().st_size if target_path.exists() else 0,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
failed_pages += 1
|
||
|
|
file_records.append(
|
||
|
|
DownloadFileRecord(
|
||
|
|
page_index=page_index,
|
||
|
|
file_path=str(target_path.resolve()),
|
||
|
|
source_url=page_url,
|
||
|
|
status="failed",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
total_pages = len(page_urls)
|
||
|
|
ok = failed_pages == 0 and total_pages > 0
|
||
|
|
message = f"作品 {illust_id}: 共 {total_pages} 页, 新下 {downloaded_pages}, 跳过 {skipped_pages}, 失败 {failed_pages}"
|
||
|
|
if library is not None:
|
||
|
|
library.ensure_artwork_stub(illust_id, artwork_type=artwork_type)
|
||
|
|
library.record_files(illust_id, file_records)
|
||
|
|
library.record_download_history(
|
||
|
|
artwork_id=illust_id,
|
||
|
|
status="success" if ok else "failed",
|
||
|
|
downloaded_pages=downloaded_pages,
|
||
|
|
skipped_pages=skipped_pages,
|
||
|
|
failed_pages=failed_pages,
|
||
|
|
message=message,
|
||
|
|
)
|
||
|
|
return DownloadResult(illust_id, total_pages, downloaded_pages, skipped_pages, failed_pages, ok, message)
|
||
|
|
except Exception as exc:
|
||
|
|
message = f"作品 {illust_id} 处理失败: {exc}"
|
||
|
|
if library is not None:
|
||
|
|
library.ensure_artwork_stub(illust_id, artwork_type="illust")
|
||
|
|
library.record_download_history(
|
||
|
|
artwork_id=illust_id,
|
||
|
|
status="failed",
|
||
|
|
downloaded_pages=0,
|
||
|
|
skipped_pages=0,
|
||
|
|
failed_pages=1,
|
||
|
|
message=message,
|
||
|
|
)
|
||
|
|
return DownloadResult(illust_id, 0, 0, 0, 1, False, message)
|
||
|
|
finally:
|
||
|
|
if library is not None:
|
||
|
|
library.close()
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def download_novel(
|
||
|
|
novel_id: int,
|
||
|
|
cookie: str = "",
|
||
|
|
save_root: str | Path = "Sakura_Downloads",
|
||
|
|
save_subdir: str = "novels",
|
||
|
|
request_delay: float = 0.0,
|
||
|
|
stop_event: Optional[threading.Event] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
library_db_path: str | Path | None = DEFAULT_LIBRARY_DB,
|
||
|
|
metadata_overrides: Optional[dict[str, object]] = None,
|
||
|
|
) -> DownloadResult:
|
||
|
|
if stop_event and stop_event.is_set():
|
||
|
|
return DownloadResult(novel_id, 0, 0, 0, 0, False, "任务已停止")
|
||
|
|
if request_delay > 0:
|
||
|
|
time.sleep(request_delay)
|
||
|
|
|
||
|
|
session = create_session(cookie)
|
||
|
|
library: PixivLibrary | None = PixivLibrary(library_db_path) if library_db_path else None
|
||
|
|
library_artwork_id = novel_id
|
||
|
|
try:
|
||
|
|
detail, content = fetch_novel_text(
|
||
|
|
novel_id,
|
||
|
|
cookie=cookie,
|
||
|
|
session=session,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
)
|
||
|
|
if metadata_overrides:
|
||
|
|
detail.update(metadata_overrides)
|
||
|
|
detail.setdefault("id", novel_id)
|
||
|
|
detail.setdefault("novelId", novel_id)
|
||
|
|
if library is not None:
|
||
|
|
library_artwork_id = library.ensure_artwork_stub(novel_id, artwork_type="novel")
|
||
|
|
library.upsert_artwork(detail, artwork_type="novel")
|
||
|
|
title = sanitize_filename(detail.get("title", ""), f"novel_{novel_id}")
|
||
|
|
user_name = sanitize_filename(detail.get("userName", ""), "unknown_author")
|
||
|
|
user_id = detail.get("userId")
|
||
|
|
author_dir = f"{user_name}_{user_id}" if str(user_id).isdigit() else user_name
|
||
|
|
save_dir = Path(save_root) / save_subdir / sanitize_filename(author_dir, f"novel_{novel_id}")
|
||
|
|
save_path = save_dir / f"{title}_{novel_id}.txt"
|
||
|
|
text = "\n".join(
|
||
|
|
[
|
||
|
|
f"标题: {detail.get('title', title)}",
|
||
|
|
f"作者: {detail.get('userName', '')}",
|
||
|
|
f"小说ID: {novel_id}",
|
||
|
|
"",
|
||
|
|
content,
|
||
|
|
]
|
||
|
|
)
|
||
|
|
_, message = save_text_file(save_path, text)
|
||
|
|
result_message = f"小说 {novel_id}: {message}"
|
||
|
|
status = "skipped" if "跳过" in message else "downloaded"
|
||
|
|
downloaded_pages = 1 if status == "downloaded" else 0
|
||
|
|
skipped_pages = 1 if status == "skipped" else 0
|
||
|
|
if library is not None:
|
||
|
|
library.record_files(
|
||
|
|
library_artwork_id,
|
||
|
|
[
|
||
|
|
DownloadFileRecord(
|
||
|
|
page_index=0,
|
||
|
|
file_path=str(save_path.resolve()),
|
||
|
|
source_url=f"https://www.pixiv.net/novel/show.php?id={novel_id}",
|
||
|
|
status=status,
|
||
|
|
size_bytes=save_path.stat().st_size if save_path.exists() else 0,
|
||
|
|
)
|
||
|
|
],
|
||
|
|
)
|
||
|
|
library.record_download_history(
|
||
|
|
artwork_id=library_artwork_id,
|
||
|
|
status="success",
|
||
|
|
downloaded_pages=downloaded_pages,
|
||
|
|
skipped_pages=skipped_pages,
|
||
|
|
failed_pages=0,
|
||
|
|
message=result_message,
|
||
|
|
)
|
||
|
|
return DownloadResult(novel_id, 1, downloaded_pages, skipped_pages, 0, True, result_message)
|
||
|
|
except Exception as exc:
|
||
|
|
message = f"小说 {novel_id} 下载失败: {exc}"
|
||
|
|
if library is not None:
|
||
|
|
library_artwork_id = library.ensure_artwork_stub(novel_id, artwork_type="novel")
|
||
|
|
library.record_download_history(
|
||
|
|
artwork_id=library_artwork_id,
|
||
|
|
status="failed",
|
||
|
|
downloaded_pages=0,
|
||
|
|
skipped_pages=0,
|
||
|
|
failed_pages=1,
|
||
|
|
message=message,
|
||
|
|
)
|
||
|
|
return DownloadResult(novel_id, 1, 0, 0, 1, False, message)
|
||
|
|
finally:
|
||
|
|
if library is not None:
|
||
|
|
library.close()
|
||
|
|
session.close()
|
||
|
|
|
||
|
|
|
||
|
|
def resolve_workers(user_workers: int, proxy_mode: bool) -> int:
|
||
|
|
if not proxy_mode:
|
||
|
|
return max(1, user_workers)
|
||
|
|
return max(3, min(user_workers, 6))
|
||
|
|
|
||
|
|
|
||
|
|
def _download_illust_batch(
|
||
|
|
work_ids: list[int],
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
max_workers: int = 6,
|
||
|
|
request_delay: float = 0.0,
|
||
|
|
save_root: str | Path = "Sakura_Downloads",
|
||
|
|
save_subdir: Optional[str] = None,
|
||
|
|
stop_event: Optional[threading.Event] = None,
|
||
|
|
progress_callback: Optional[Callable[[DownloadResult, int, int], None]] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
user_id_hint: int = 0,
|
||
|
|
) -> list[DownloadResult]:
|
||
|
|
worker_count = resolve_workers(max_workers, proxy_mode=proxy_pool is not None)
|
||
|
|
results: list[DownloadResult] = []
|
||
|
|
if not work_ids:
|
||
|
|
return results
|
||
|
|
|
||
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=worker_count) as executor:
|
||
|
|
futures = {
|
||
|
|
executor.submit(
|
||
|
|
download_illust,
|
||
|
|
illust_id=work_id,
|
||
|
|
user_id=user_id_hint,
|
||
|
|
cookie=cookie,
|
||
|
|
save_root=save_root,
|
||
|
|
save_subdir=save_subdir,
|
||
|
|
request_delay=request_delay,
|
||
|
|
stop_event=stop_event,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
): work_id
|
||
|
|
for work_id in work_ids
|
||
|
|
}
|
||
|
|
completed, total = 0, len(futures)
|
||
|
|
for future in concurrent.futures.as_completed(futures):
|
||
|
|
result = future.result()
|
||
|
|
results.append(result)
|
||
|
|
completed += 1
|
||
|
|
if progress_callback:
|
||
|
|
progress_callback(result, completed, total)
|
||
|
|
if stop_event and stop_event.is_set():
|
||
|
|
break
|
||
|
|
|
||
|
|
results.sort(key=lambda item: item.work_id, reverse=True)
|
||
|
|
return results
|
||
|
|
|
||
|
|
|
||
|
|
def download_user_works(
|
||
|
|
user_id: int,
|
||
|
|
cookie: str = "",
|
||
|
|
max_workers: int = 6,
|
||
|
|
request_delay: float = 0.0,
|
||
|
|
save_root: str | Path = "Sakura_Downloads",
|
||
|
|
stop_event: Optional[threading.Event] = None,
|
||
|
|
progress_callback: Optional[Callable[[DownloadResult, int, int], None]] = None,
|
||
|
|
work_ids: Optional[list[int]] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
) -> tuple[list[int], list[DownloadResult]]:
|
||
|
|
if work_ids is None:
|
||
|
|
_, work_ids = fetch_user_work_ids(user_id=user_id, cookie=cookie, proxy_pool=proxy_pool)
|
||
|
|
results = _download_illust_batch(
|
||
|
|
work_ids,
|
||
|
|
cookie=cookie,
|
||
|
|
max_workers=max_workers,
|
||
|
|
request_delay=request_delay,
|
||
|
|
save_root=save_root,
|
||
|
|
save_subdir=str(user_id),
|
||
|
|
stop_event=stop_event,
|
||
|
|
progress_callback=progress_callback,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
user_id_hint=user_id,
|
||
|
|
)
|
||
|
|
return work_ids, results
|
||
|
|
|
||
|
|
|
||
|
|
def download_keyword_works(
|
||
|
|
keyword: str,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
max_workers: int = 6,
|
||
|
|
request_delay: float = 0.0,
|
||
|
|
save_root: str | Path = "Sakura_Downloads",
|
||
|
|
stop_event: Optional[threading.Event] = None,
|
||
|
|
progress_callback: Optional[Callable[[DownloadResult, int, int], None]] = None,
|
||
|
|
proxy_pool: Optional[ProxyPool] = None,
|
||
|
|
limit: int = 30,
|
||
|
|
work_ids: Optional[list[int]] = None,
|
||
|
|
) -> tuple[list[int], list[DownloadResult]]:
|
||
|
|
if work_ids is None:
|
||
|
|
work_ids = search_illust_ids_by_keyword(
|
||
|
|
keyword,
|
||
|
|
cookie=cookie,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
limit=limit,
|
||
|
|
)
|
||
|
|
results = _download_illust_batch(
|
||
|
|
work_ids,
|
||
|
|
cookie=cookie,
|
||
|
|
max_workers=max_workers,
|
||
|
|
request_delay=request_delay,
|
||
|
|
save_root=save_root,
|
||
|
|
save_subdir=str(Path("keywords") / sanitize_filename(keyword, "keyword")),
|
||
|
|
stop_event=stop_event,
|
||
|
|
progress_callback=progress_callback,
|
||
|
|
proxy_pool=proxy_pool,
|
||
|
|
)
|
||
|
|
return work_ids, results
|