commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Bundled AO3 (Archive of Our Own) plugin package."""
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "AO3",
|
||||
"version": "1.0.0",
|
||||
"domain": "archiveofourown.org",
|
||||
"author": "SakuraHub",
|
||||
"description": "AO3 fanfiction archive novel plugin. Supports work URL parsing, metadata extraction, EPUB/HTML download, cookie-based auth, and update checking.",
|
||||
"requires_auth": false,
|
||||
"min_platform_version": "1.0.0",
|
||||
"tags": ["novel", "fanfiction", "ao3"],
|
||||
"dependencies": []
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "笔趣阁",
|
||||
"version": "3.0.0",
|
||||
"domain": "www.bqg551.cc",
|
||||
"author": "Sakura Downloader",
|
||||
"description": "基于 bqg551 阅读站公开 API 的小说搜索、目录解析、章节抓取与 TXT 导出。",
|
||||
"requires_auth": false,
|
||||
"tags": ["novel", "bqg551", "biquge", "bookshelf"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "长佩文学",
|
||||
"version": "1.0.0",
|
||||
"domain": "www.gongzicp.com",
|
||||
"author": "Sakura Downloader",
|
||||
"description": "长佩文学(gongzicp.com)小说搜索、目录解析、章节抓取与 TXT 导出。",
|
||||
"requires_auth": true,
|
||||
"tags": ["novel", "gongzicp", "changpei", "bookshelf"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
@@ -0,0 +1,514 @@
|
||||
"""
|
||||
Changpei (长佩文学 / gongzicp.com) novel plugin.
|
||||
|
||||
Uses the public webapi endpoints on https://www.gongzicp.com/webapi/.
|
||||
Chapter bodies are AES-CBC encrypted; decryption matches the site's CryptoJS helper.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from html import unescape
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from cryptography.hazmat.primitives import padding as crypto_padding
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
|
||||
CRYPTOGRAPHY_AVAILABLE = True
|
||||
except Exception:
|
||||
crypto_padding = None
|
||||
Cipher = None
|
||||
algorithms = None
|
||||
modes = None
|
||||
CRYPTOGRAPHY_AVAILABLE = False
|
||||
|
||||
from pixiv_app.core.downloader import sanitize_filename, save_text_file
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
|
||||
|
||||
BASE_URL = "https://www.gongzicp.com"
|
||||
WEBAPI_BASE = f"{BASE_URL}/webapi"
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
_NOVEL_PATH_RE = re.compile(r"/novel-(?P<nid>\d+)\.html", re.I)
|
||||
_READ_PATH_RE = re.compile(r"/read-(?P<nid>\d+)-(?P<cid>\d+)\.html", re.I)
|
||||
_SINGLE_READ_RE = re.compile(r"/read-(?P<cid>\d+)\.html", re.I)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BookCandidate:
|
||||
title: str
|
||||
url: str
|
||||
author: str = ""
|
||||
latest_chapter: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TargetIds:
|
||||
novel_id: str = ""
|
||||
chapter_id: str = ""
|
||||
|
||||
|
||||
def _js_to_string(value: int, base: int) -> str:
|
||||
digits = "0123456789abcdefghijklmnopqrstuvwxyz"
|
||||
chars: list[str] = []
|
||||
while value:
|
||||
chars.append(digits[value % base])
|
||||
value //= base
|
||||
return "".join(reversed(chars)) or "0"
|
||||
|
||||
|
||||
def _clean_html_text(raw: str) -> str:
|
||||
text = re.sub(r"<br\s*/?>", "\n", raw, flags=re.I)
|
||||
text = re.sub(r"</p\s*>", "\n", text, flags=re.I)
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
text = unescape(text)
|
||||
text = text.replace("\r", "")
|
||||
lines = [line.strip() for line in text.splitlines()]
|
||||
return "\n".join(line for line in lines if line)
|
||||
|
||||
|
||||
def _normalize_space(value: str) -> str:
|
||||
return re.sub(r"\s+", " ", _clean_html_text(value)).strip()
|
||||
|
||||
|
||||
class _ChapterDecryptor:
|
||||
"""AES-CBC decryptor matching index.eef719ce.js class XE."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
if not CRYPTOGRAPHY_AVAILABLE:
|
||||
raise PluginParseError("缺少 cryptography 依赖,无法解密长佩章节正文。")
|
||||
suffix = _js_to_string(int("165455", 14), 32)
|
||||
key_seed = "iGzsYn" + suffix
|
||||
iv_str = f"$h$b3!{key_seed}"
|
||||
decoded = base64.b64decode("dTBMUnJidSRFbg==").decode("utf-8")
|
||||
key_str = decoded + _js_to_string(int("4d5a6c8", 14), 36) + "A"
|
||||
self._key = key_str.encode("utf-8")
|
||||
self._iv = iv_str.encode("utf-8")
|
||||
|
||||
def decrypt(self, encrypted_b64: str) -> str:
|
||||
if not encrypted_b64:
|
||||
return ""
|
||||
raw = base64.b64decode(encrypted_b64)
|
||||
cipher = Cipher(algorithms.AES(self._key), modes.CBC(self._iv))
|
||||
decryptor = cipher.decryptor()
|
||||
padded = decryptor.update(raw) + decryptor.finalize()
|
||||
unpadder = crypto_padding.PKCS7(128).unpadder()
|
||||
plain = unpadder.update(padded) + unpadder.finalize()
|
||||
return plain.decode("utf-8")
|
||||
|
||||
|
||||
class ChangpeiPlugin(BasePlugin):
|
||||
name = "长佩文学"
|
||||
domain = "www.gongzicp.com"
|
||||
version = "1.0.0"
|
||||
|
||||
def __init__(self, cookie: str = "") -> None:
|
||||
self.cookie = cookie
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(self.get_headers())
|
||||
self._chapter_decryptor: _ChapterDecryptor | None = None
|
||||
|
||||
def set_auth(self, cookie: str) -> None:
|
||||
"""设置登录 Cookie,用于访问需订阅/登录的章节。"""
|
||||
self.cookie = cookie or ""
|
||||
self.session.headers.update(self.get_headers())
|
||||
|
||||
def validate(self) -> bool:
|
||||
return CRYPTOGRAPHY_AVAILABLE
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
lowered = url.strip().lower()
|
||||
if any(host in lowered for host in ("gongzicp.com", "gongzicp.cn")):
|
||||
return True
|
||||
if lowered.isdigit():
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
"Referer": BASE_URL + "/",
|
||||
}
|
||||
if self.cookie:
|
||||
headers["Cookie"] = self.cookie
|
||||
return headers
|
||||
|
||||
def resolve_target(self, target: str, *, limit: int = 20) -> dict[str, Any]:
|
||||
raw = target.strip()
|
||||
if not raw:
|
||||
raise PluginParseError("请输入书名、作者、书籍链接或章节链接。")
|
||||
|
||||
normalized = self._normalize_target(raw)
|
||||
if normalized:
|
||||
ids = self._extract_ids_from_url(normalized)
|
||||
if ids.novel_id and ids.chapter_id:
|
||||
chapter = self._chapter_resource_from_ids(ids.novel_id, ids.chapter_id, source_url=normalized)
|
||||
return {
|
||||
"mode": "chapters",
|
||||
"book_title": str(chapter.metadata.get("book_title") or ""),
|
||||
"resources": [chapter],
|
||||
}
|
||||
if ids.novel_id:
|
||||
return self._resolve_novel_id(ids.novel_id)
|
||||
if ids.chapter_id and not ids.novel_id:
|
||||
raise PluginParseError("章节链接缺少书籍 ID,请使用 read-{书籍ID}-{章节ID}.html 格式。")
|
||||
|
||||
if raw.isdigit():
|
||||
return self._resolve_novel_id(raw)
|
||||
|
||||
books = self._search_books(raw, limit=limit)
|
||||
if not books:
|
||||
raise PluginParseError(f"没有搜索到与“{raw}”相关的书籍。")
|
||||
return {
|
||||
"mode": "books",
|
||||
"title": f"搜索结果:{raw}",
|
||||
"books": [candidate.__dict__ for candidate in books],
|
||||
}
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
outcome = self.resolve_target(url, limit=20)
|
||||
if outcome.get("mode") == "chapters":
|
||||
return list(outcome.get("resources") or [])
|
||||
books = list(outcome.get("books") or [])
|
||||
if not books:
|
||||
raise PluginParseError("未解析到可下载内容。")
|
||||
selected = self.resolve_target(str(books[0].get("url") or ""), limit=20)
|
||||
resources = list(selected.get("resources") or [])
|
||||
if not resources:
|
||||
raise PluginParseError("选中的书籍没有章节内容。")
|
||||
return resources
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
if str(resource.metadata.get("kind") or "") != "novel_chapter":
|
||||
raise PluginParseError("不支持的长佩资源类型。")
|
||||
|
||||
content = str(resource.metadata.get("content") or "")
|
||||
if not content.strip():
|
||||
nid = str(resource.metadata.get("novel_id") or resource.metadata.get("book_id") or "")
|
||||
cid = str(resource.metadata.get("chapter_id") or "")
|
||||
if nid and cid:
|
||||
preserved_title = str(resource.metadata.get("book_title") or "")
|
||||
preserved_author = str(resource.author or resource.metadata.get("author") or "")
|
||||
chapter_data = self._chapter_data_from_api(nid, cid)
|
||||
content = str(chapter_data.get("content") or "")
|
||||
resource.metadata.update(chapter_data)
|
||||
if preserved_title and not str(resource.metadata.get("book_title") or "").strip():
|
||||
resource.metadata["book_title"] = preserved_title
|
||||
if preserved_author and not str(resource.author or "").strip():
|
||||
resource.author = preserved_author
|
||||
|
||||
book_title = str(resource.metadata.get("book_title") or "未命名小说")
|
||||
chapter_title = str(resource.metadata.get("chapter_title") or resource.title or "未命名章节")
|
||||
chapter_order = int(resource.metadata.get("chapter_order") or 0)
|
||||
|
||||
folder = save_path / "novels" / sanitize_filename(book_title, "changpei_book")
|
||||
filename = (
|
||||
f"{chapter_order:04d}_{sanitize_filename(chapter_title, 'chapter')}.txt"
|
||||
if chapter_order
|
||||
else f"{sanitize_filename(chapter_title, 'chapter')}.txt"
|
||||
)
|
||||
final_path = folder / filename
|
||||
text = "\n".join(
|
||||
[
|
||||
f"书名: {book_title}",
|
||||
f"章节: {chapter_title}",
|
||||
f"作者: {resource.author or '未知'}",
|
||||
f"链接: {resource.url}",
|
||||
"",
|
||||
content.strip(),
|
||||
]
|
||||
).strip()
|
||||
save_text_file(final_path, text)
|
||||
resource.metadata["word_count"] = len(content)
|
||||
return [final_path]
|
||||
|
||||
def _get_decryptor(self) -> _ChapterDecryptor:
|
||||
if self._chapter_decryptor is None:
|
||||
self._chapter_decryptor = _ChapterDecryptor()
|
||||
return self._chapter_decryptor
|
||||
|
||||
def _normalize_target(self, target: str) -> str:
|
||||
value = target.strip()
|
||||
lowered = value.lower()
|
||||
if lowered.startswith(("http://", "https://", "//")):
|
||||
if value.startswith("//"):
|
||||
return "https:" + value
|
||||
return value
|
||||
if lowered.startswith(("www.gongzicp.com", "gongzicp.com")):
|
||||
return "https://" + value.lstrip("/")
|
||||
if _NOVEL_PATH_RE.search(value) or _READ_PATH_RE.search(value) or _SINGLE_READ_RE.search(value):
|
||||
return urljoin(BASE_URL, value.lstrip("/"))
|
||||
return ""
|
||||
|
||||
def _extract_ids_from_url(self, url: str) -> TargetIds:
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path or ""
|
||||
match = _READ_PATH_RE.search(path)
|
||||
if match:
|
||||
return TargetIds(novel_id=match.group("nid"), chapter_id=match.group("cid"))
|
||||
match = _NOVEL_PATH_RE.search(path)
|
||||
if match:
|
||||
return TargetIds(novel_id=match.group("nid"))
|
||||
match = _SINGLE_READ_RE.search(path)
|
||||
if match:
|
||||
return TargetIds(chapter_id=match.group("cid"))
|
||||
return TargetIds()
|
||||
|
||||
def _webapi_get(self, path: str, *, params: dict[str, Any] | None = None, referer: str = "") -> dict[str, Any]:
|
||||
headers = dict(self.get_headers())
|
||||
if referer:
|
||||
headers["Referer"] = referer
|
||||
response = self.session.get(
|
||||
f"{WEBAPI_BASE}{path}",
|
||||
params=params or {},
|
||||
headers=headers,
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
code = payload.get("code")
|
||||
if code not in (200, "200"):
|
||||
message = str(payload.get("msg") or "请求失败")
|
||||
raise PluginParseError(f"长佩接口错误:{message}")
|
||||
data = payload.get("data")
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
if data is None:
|
||||
return {}
|
||||
return {"value": data}
|
||||
|
||||
def _novel_url(self, novel_id: str) -> str:
|
||||
return f"{BASE_URL}/novel-{novel_id}.html"
|
||||
|
||||
def _chapter_url(self, novel_id: str, chapter_id: str) -> str:
|
||||
return f"{BASE_URL}/read-{novel_id}-{chapter_id}.html"
|
||||
|
||||
def _search_books(self, keyword: str, *, limit: int) -> list[BookCandidate]:
|
||||
page = 1
|
||||
books: list[BookCandidate] = []
|
||||
while len(books) < limit:
|
||||
data = self._webapi_get(
|
||||
"/search/novels",
|
||||
params={"k": keyword, "page": page},
|
||||
referer=f"{BASE_URL}/novel/searchNovel",
|
||||
)
|
||||
items = list(data.get("list") or [])
|
||||
if not items:
|
||||
break
|
||||
for item in items:
|
||||
novel_id = str(item.get("novel_id") or "").strip()
|
||||
title = _normalize_space(str(item.get("novel_name") or ""))
|
||||
if not novel_id or not title:
|
||||
continue
|
||||
author = _normalize_space(str(item.get("novel_author") or item.get("author_nickname") or ""))
|
||||
books.append(
|
||||
BookCandidate(
|
||||
title=title,
|
||||
url=self._novel_url(novel_id),
|
||||
author=author,
|
||||
latest_chapter=_normalize_space(str(item.get("novel_uptime") or "")),
|
||||
)
|
||||
)
|
||||
if len(books) >= limit:
|
||||
break
|
||||
if len(items) < 10:
|
||||
break
|
||||
page += 1
|
||||
return books
|
||||
|
||||
def _novel_info(self, novel_id: str) -> dict[str, Any]:
|
||||
referer = self._novel_url(novel_id)
|
||||
return self._webapi_get("/novel/novelInfo", params={"id": novel_id}, referer=referer)
|
||||
|
||||
def _chapter_list(self, novel_id: str) -> list[dict[str, Any]]:
|
||||
referer = self._novel_url(novel_id)
|
||||
data = self._webapi_get("/novel/chapterGetList", params={"nid": novel_id}, referer=referer)
|
||||
return list(data.get("list") or [])
|
||||
|
||||
def _decrypt_chapter_content(self, encrypted: str) -> str:
|
||||
try:
|
||||
return self._get_decryptor().decrypt(encrypted)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def _chapter_data_from_api(self, novel_id: str, chapter_id: str) -> dict[str, Any]:
|
||||
referer = self._chapter_url(novel_id, chapter_id)
|
||||
payload = self._webapi_get(
|
||||
"/novel/chapterGetInfo",
|
||||
params={"nid": novel_id, "cid": chapter_id},
|
||||
referer=referer,
|
||||
)
|
||||
chapter_info = dict(payload.get("chapterInfo") or {})
|
||||
novel_info = dict(payload.get("novelInfo") or {})
|
||||
|
||||
locked = bool(chapter_info.get("lock")) or bool(chapter_info.get("chapter_ispay"))
|
||||
status = int(chapter_info.get("status") or 0)
|
||||
encrypted = str(chapter_info.get("content") or "")
|
||||
content = ""
|
||||
if not locked and status == 1 and encrypted:
|
||||
content = self._decrypt_chapter_content(encrypted)
|
||||
|
||||
chapter_title = _normalize_space(str(chapter_info.get("name") or f"章节 {chapter_id}"))
|
||||
book_title = _normalize_space(str(novel_info.get("name") or novel_info.get("novel_name") or ""))
|
||||
author = _normalize_space(str(novel_info.get("author") or novel_info.get("author_nickname") or ""))
|
||||
order = int(chapter_info.get("order") or 0)
|
||||
source_id = f"{novel_id}:{chapter_id}"
|
||||
|
||||
if locked:
|
||||
content = "[章节需订阅或已锁定,无法抓取正文]"
|
||||
|
||||
return {
|
||||
"source_id": source_id,
|
||||
"novel_id": novel_id,
|
||||
"book_id": novel_id,
|
||||
"book_title": book_title,
|
||||
"chapter_id": chapter_id,
|
||||
"chapter_title": chapter_title,
|
||||
"chapter_order": order,
|
||||
"author": author,
|
||||
"content": content,
|
||||
"word_count": len(content),
|
||||
"locked": locked,
|
||||
"description": _clean_html_text(str(novel_info.get("novel_info") or novel_info.get("novel_desc") or "")),
|
||||
}
|
||||
|
||||
def _flatten_chapter_items(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
result: list[dict[str, Any]] = []
|
||||
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
if str(item.get("type") or "") == "item":
|
||||
result.append(item)
|
||||
|
||||
for key in ("children", "list", "chapterList", "chapters"):
|
||||
child_items = item.get(key)
|
||||
if isinstance(child_items, list):
|
||||
result.extend(self._flatten_chapter_items(child_items))
|
||||
|
||||
return result
|
||||
|
||||
def _chapter_resources_from_novel(self, novel_id: str, novel_info: dict[str, Any]) -> list[Resource]:
|
||||
book_title = _normalize_space(str(novel_info.get("novel_name") or ""))
|
||||
author = _normalize_space(str(novel_info.get("author_nickname") or ""))
|
||||
description = _clean_html_text(str(novel_info.get("novel_info") or novel_info.get("novel_desc") or ""))
|
||||
chapters = self._flatten_chapter_items(self._chapter_list(novel_id))
|
||||
resources: list[Resource] = []
|
||||
|
||||
for index, item in enumerate(chapters, start=1):
|
||||
chapter_id = str(item.get("id") or "").strip()
|
||||
chapter_title = _normalize_space(str(item.get("name") or ""))
|
||||
if not chapter_id or not chapter_title:
|
||||
continue
|
||||
locked = bool(item.get("lock")) or bool(item.get("pay"))
|
||||
order = int(item.get("order") or index)
|
||||
source_id = f"{novel_id}:{chapter_id}"
|
||||
resources.append(
|
||||
Resource(
|
||||
id=source_id,
|
||||
url=self._chapter_url(novel_id, chapter_id),
|
||||
title=chapter_title,
|
||||
author=author,
|
||||
author_id=str(novel_info.get("author_id") or ""),
|
||||
files=[],
|
||||
metadata={
|
||||
"kind": "novel_chapter",
|
||||
"resource_type": "novel",
|
||||
"content_type": "novel",
|
||||
"source_platform": "changpei",
|
||||
"source_id": source_id,
|
||||
"novel_id": novel_id,
|
||||
"book_id": novel_id,
|
||||
"book_title": book_title,
|
||||
"series_id": novel_id,
|
||||
"series_title": book_title,
|
||||
"chapter_id": chapter_id,
|
||||
"chapter_title": chapter_title,
|
||||
"chapter_order": order,
|
||||
"description": description,
|
||||
"locked": locked,
|
||||
"last_update": str(novel_info.get("novel_uptime") or ""),
|
||||
"book_status": str(novel_info.get("novel_process") or ""),
|
||||
},
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
def _chapter_resource_from_ids(self, novel_id: str, chapter_id: str, *, source_url: str = "") -> Resource:
|
||||
data = self._chapter_data_from_api(novel_id, chapter_id)
|
||||
return Resource(
|
||||
id=str(data["source_id"]),
|
||||
url=source_url or self._chapter_url(novel_id, chapter_id),
|
||||
title=str(data["chapter_title"]),
|
||||
author=str(data.get("author") or ""),
|
||||
author_id="",
|
||||
files=[],
|
||||
metadata={
|
||||
"kind": "novel_chapter",
|
||||
"resource_type": "novel",
|
||||
"content_type": "novel",
|
||||
"source_platform": "changpei",
|
||||
"source_id": str(data["source_id"]),
|
||||
"novel_id": novel_id,
|
||||
"book_id": novel_id,
|
||||
"book_title": str(data.get("book_title") or ""),
|
||||
"series_id": novel_id,
|
||||
"series_title": str(data.get("book_title") or ""),
|
||||
"chapter_id": chapter_id,
|
||||
"chapter_title": str(data["chapter_title"]),
|
||||
"chapter_order": int(data.get("chapter_order") or 1),
|
||||
"description": str(data.get("description") or ""),
|
||||
"content": str(data.get("content") or ""),
|
||||
"word_count": int(data.get("word_count") or 0),
|
||||
"locked": bool(data.get("locked")),
|
||||
},
|
||||
)
|
||||
|
||||
def _resolve_novel_id(self, novel_id: str) -> dict[str, Any]:
|
||||
novel_info = self._novel_info(novel_id)
|
||||
resources = self._chapter_resources_from_novel(novel_id, novel_info)
|
||||
if not resources:
|
||||
raise PluginParseError(f"书籍 {novel_id} 没有解析到章节目录。")
|
||||
book_title = str(resources[0].metadata.get("book_title") or "")
|
||||
return {
|
||||
"mode": "chapters",
|
||||
"book_title": book_title,
|
||||
"resources": resources,
|
||||
}
|
||||
|
||||
# ---- cookie config (matches Pixiv / AO3 pattern) ----
|
||||
|
||||
def get_config_schema(self) -> dict[str, Any]:
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"cookie": {
|
||||
"type": "string",
|
||||
"title": "长佩 Cookie",
|
||||
"description": "粘贴长佩 (gongzicp.com) 登录后的 Cookie 文本,用于访问已购章节。",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def load_config(self) -> dict[str, Any]:
|
||||
return {"cookie": self.cookie}
|
||||
|
||||
def save_config(self, data: dict[str, Any]) -> None:
|
||||
self.cookie = str(data.get("cookie") or "")
|
||||
self.session.headers.update(self.get_headers())
|
||||
|
||||
|
||||
plugin_class = ChangpeiPlugin
|
||||
@@ -0,0 +1 @@
|
||||
"""Douyin plugin package."""
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "Douyin",
|
||||
"version": "1.0.0",
|
||||
"domain": "douyin.com",
|
||||
"author": "Sakura Downloader",
|
||||
"description": "Douyin video and image-note parser/downloader for public share URLs.",
|
||||
"requires_auth": false,
|
||||
"tags": ["social", "video", "image"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
"""Douyin plugin.
|
||||
|
||||
Supports public Douyin share, video, note, and modal links. Cookie support is
|
||||
optional and reads from runtime/douyin_cookies.json when present.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from pixiv_app.core.cookie_import import cookies_to_header, parse_cookie_text
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginDownloadError, PluginParseError, Resource
|
||||
|
||||
|
||||
_AWEME_ID_RE = re.compile(
|
||||
r"(?:aweme_id=|modal_id=|/video/|/note/|/share/video/|/share/note/)(\d{6,})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_JSON_SCRIPT_RE = re.compile(
|
||||
r'<script[^>]+id=["\'](?P<id>RENDER_DATA|__UNIVERSAL_DATA_FOR_REHYDRATION__)["\'][^>]*>'
|
||||
r"(?P<body>.*?)</script>",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_JSON_URL_RE = re.compile(r"https?:\\?/\\?/[^\"'<>\\]+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class DouyinPluginConfig:
|
||||
timeout: float = 30.0
|
||||
max_redirects: int = 8
|
||||
cookie_file: str = ""
|
||||
|
||||
|
||||
class DouyinPlugin(BasePlugin):
|
||||
domain = "douyin.com"
|
||||
name = "Douyin"
|
||||
version = "1.0.0"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.config = DouyinPluginConfig(cookie_file=str(runtime_path("douyin_cookies.json")))
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
lower = (url or "").strip().lower()
|
||||
return any(
|
||||
token in lower
|
||||
for token in (
|
||||
"douyin.com",
|
||||
"v.douyin.com",
|
||||
"iesdouyin.com",
|
||||
"amemv.com",
|
||||
)
|
||||
)
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Referer": "https://www.douyin.com/",
|
||||
}
|
||||
cookie_header = self._cookie_header()
|
||||
if cookie_header:
|
||||
headers["Cookie"] = cookie_header
|
||||
return headers
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
target = _extract_first_douyin_url(url)
|
||||
if not target:
|
||||
raise PluginParseError("No valid Douyin URL found.")
|
||||
|
||||
final_url, body = self._fetch_page(target)
|
||||
aweme_id = _extract_aweme_id(final_url) or _extract_aweme_id(body) or _extract_aweme_id(target)
|
||||
payloads = _extract_json_payloads(body)
|
||||
focus_nodes = _find_nodes_for_aweme(payloads, aweme_id) if aweme_id else []
|
||||
media = _extract_media(focus_nodes or payloads)
|
||||
if not media:
|
||||
media = _extract_media_from_raw_html(body)
|
||||
|
||||
files = _dedupe_files(media)
|
||||
if not files:
|
||||
raise PluginParseError(
|
||||
"No Douyin video or image media was found. Login, captcha, or page changes may be required."
|
||||
)
|
||||
|
||||
title = _first_text(focus_nodes or payloads, ("desc", "title", "caption")) or "Douyin"
|
||||
author = _first_text(focus_nodes or payloads, ("nickname", "author_name", "unique_id")) or ""
|
||||
author_id = _first_text(focus_nodes or payloads, ("uid", "sec_uid", "short_id", "unique_id")) or ""
|
||||
created_at = _first_text(focus_nodes or payloads, ("create_time", "createTime"))
|
||||
|
||||
return [
|
||||
Resource(
|
||||
id=aweme_id or str(abs(hash(final_url))),
|
||||
url=final_url,
|
||||
title=title,
|
||||
author=author,
|
||||
author_id=author_id,
|
||||
files=files,
|
||||
metadata={
|
||||
"source_platform": "douyin",
|
||||
"aweme_id": aweme_id or "",
|
||||
"kind": "douyin",
|
||||
"original_url": target,
|
||||
},
|
||||
thumbnail=next(
|
||||
(item.get("thumbnail") or item.get("url") for item in files if item.get("type") == "image"),
|
||||
None,
|
||||
),
|
||||
created_at=str(created_at) if created_at else None,
|
||||
)
|
||||
]
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
downloaded: list[Path] = []
|
||||
with httpx.Client(timeout=max(30.0, self.config.timeout), follow_redirects=True, headers=self.get_headers()) as client:
|
||||
for index, item in enumerate(resource.files, start=1):
|
||||
media_url = str(item.get("url") or "").strip()
|
||||
if not media_url:
|
||||
continue
|
||||
suffix = _guess_suffix(media_url, str(item.get("type") or ""))
|
||||
target = save_path / f"{_safe_filename(resource.id)}_{index:02d}{suffix}"
|
||||
if target.exists() and target.stat().st_size > 512:
|
||||
downloaded.append(target)
|
||||
continue
|
||||
try:
|
||||
response = client.get(media_url, headers={**self.get_headers(), "Referer": resource.url})
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
raise PluginDownloadError(f"Douyin media download failed: {exc}") from exc
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if "text/html" in content_type:
|
||||
raise PluginDownloadError("Douyin media download returned HTML. Login or captcha may be required.")
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size <= 512:
|
||||
target.unlink(missing_ok=True)
|
||||
raise PluginDownloadError("Douyin media download was too small and likely failed.")
|
||||
downloaded.append(target)
|
||||
return downloaded
|
||||
|
||||
def _fetch_page(self, url: str) -> tuple[str, str]:
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=self.config.timeout,
|
||||
follow_redirects=True,
|
||||
max_redirects=self.config.max_redirects,
|
||||
headers=self.get_headers(),
|
||||
) as client:
|
||||
response = client.get(url)
|
||||
response.raise_for_status()
|
||||
return str(response.url), response.text
|
||||
except Exception as exc:
|
||||
raise PluginParseError(f"Douyin page load failed: {exc}") from exc
|
||||
|
||||
def _cookie_header(self) -> str:
|
||||
cookie_file = Path(self.config.cookie_file or "")
|
||||
if not cookie_file.is_file():
|
||||
return ""
|
||||
try:
|
||||
cookies = parse_cookie_text(cookie_file.read_text(encoding="utf-8"))
|
||||
return cookies_to_header(cookies, ("douyin.com", "iesdouyin.com", "amemv.com"))
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_first_douyin_url(text: str) -> str:
|
||||
value = (text or "").strip()
|
||||
match = re.search(r"https?://[^\s\"'<>]*?(?:douyin|iesdouyin|amemv)\.com/[^\s\"'<>]*", value, re.I)
|
||||
if match:
|
||||
return match.group(0).rstrip(",.;)")
|
||||
if "douyin.com" in value.lower() and not value.startswith(("http://", "https://")):
|
||||
return "https://" + value.lstrip("/")
|
||||
return value if value.startswith(("http://", "https://")) else ""
|
||||
|
||||
|
||||
def _extract_aweme_id(text: str) -> str:
|
||||
match = _AWEME_ID_RE.search(text or "")
|
||||
if match:
|
||||
return match.group(1)
|
||||
try:
|
||||
query = parse_qs(urlparse(text).query)
|
||||
except Exception:
|
||||
return ""
|
||||
for key in ("aweme_id", "modal_id", "item_id"):
|
||||
values = query.get(key)
|
||||
if values and str(values[0]).isdigit():
|
||||
return str(values[0])
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_json_payloads(page: str) -> list[Any]:
|
||||
payloads: list[Any] = []
|
||||
for match in _JSON_SCRIPT_RE.finditer(page or ""):
|
||||
raw = html.unescape(match.group("body") or "").strip()
|
||||
if match.group("id") == "RENDER_DATA":
|
||||
raw = unquote(raw)
|
||||
try:
|
||||
payloads.append(json.loads(raw))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for pattern in (
|
||||
r"window\.__INITIAL_STATE__\s*=\s*({.*?})\s*</script>",
|
||||
r"window\._ROUTER_DATA\s*=\s*({.*?})\s*</script>",
|
||||
):
|
||||
for match in re.finditer(pattern, page or "", re.DOTALL):
|
||||
try:
|
||||
payloads.append(json.loads(match.group(1)))
|
||||
except Exception:
|
||||
continue
|
||||
return payloads
|
||||
|
||||
|
||||
def _find_nodes_for_aweme(payloads: list[Any], aweme_id: str) -> list[Any]:
|
||||
if not aweme_id:
|
||||
return []
|
||||
matches: list[Any] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
values = {
|
||||
str(node.get(key) or "")
|
||||
for key in ("aweme_id", "awemeId", "id", "item_id", "itemId", "modal_id", "modalId")
|
||||
}
|
||||
if aweme_id in values:
|
||||
matches.append(node)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
for payload in payloads:
|
||||
walk(payload)
|
||||
return matches
|
||||
|
||||
|
||||
def _extract_media(payloads: list[Any]) -> list[dict[str, str]]:
|
||||
files: list[dict[str, str]] = []
|
||||
|
||||
def add_video(node: dict[str, Any]) -> None:
|
||||
for key in ("play_addr", "download_addr", "playApi", "video", "videoModel"):
|
||||
urls = _urls_from_node(node.get(key))
|
||||
if urls:
|
||||
files.append({"type": "video", "url": _normalize_url(urls[0])})
|
||||
return
|
||||
|
||||
def add_images(node: dict[str, Any]) -> None:
|
||||
for key in ("images", "image_post_info", "image", "imageAlbum"):
|
||||
for image_url in _urls_from_node(node.get(key)):
|
||||
files.append({"type": "image", "url": _normalize_url(image_url)})
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
add_images(node)
|
||||
if any(key in node for key in ("play_addr", "download_addr", "video", "playApi", "videoModel")):
|
||||
add_video(node)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
for payload in payloads:
|
||||
walk(payload)
|
||||
return files
|
||||
|
||||
|
||||
def _extract_media_from_raw_html(page: str) -> list[dict[str, str]]:
|
||||
files: list[dict[str, str]] = []
|
||||
for raw in _JSON_URL_RE.findall(page or ""):
|
||||
url = _normalize_url(raw)
|
||||
low = url.lower()
|
||||
if any(token in low for token in ("douyin", "byteimg", "pstatp", "douyinvod", "amemv", "douyinpic")):
|
||||
if any(token in low for token in (".mp4", "playwm", "play/?", "video_id=")):
|
||||
files.append({"type": "video", "url": url})
|
||||
elif any(token in low for token in (".jpeg", ".jpg", ".png", ".webp", "image")):
|
||||
files.append({"type": "image", "url": url})
|
||||
return files
|
||||
|
||||
|
||||
def _urls_from_node(node: Any) -> list[str]:
|
||||
if node is None:
|
||||
return []
|
||||
if isinstance(node, str):
|
||||
value = _normalize_url(node)
|
||||
return [value] if value.startswith(("http://", "https://", "//")) else []
|
||||
if isinstance(node, list):
|
||||
urls: list[str] = []
|
||||
for item in node:
|
||||
urls.extend(_urls_from_node(item))
|
||||
return urls
|
||||
if isinstance(node, dict):
|
||||
urls: list[str] = []
|
||||
for key in (
|
||||
"url_list",
|
||||
"urlList",
|
||||
"urls",
|
||||
"uri",
|
||||
"url",
|
||||
"src",
|
||||
"download_url",
|
||||
"downloadUrl",
|
||||
"play_addr",
|
||||
"playAddr",
|
||||
"origin_cover",
|
||||
"cover",
|
||||
):
|
||||
urls.extend(_urls_from_node(node.get(key)))
|
||||
return urls
|
||||
return []
|
||||
|
||||
|
||||
def _dedupe_files(files: list[dict[str, str]]) -> list[dict[str, str]]:
|
||||
video: dict[str, str] | None = None
|
||||
images: list[dict[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for item in files:
|
||||
url = _normalize_url(item.get("url", ""))
|
||||
if not url or url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
media_type = item.get("type", "image")
|
||||
row = {"type": media_type, "url": url}
|
||||
if media_type == "video":
|
||||
if video is None:
|
||||
video = row
|
||||
continue
|
||||
images.append(row)
|
||||
return ([video] if video else []) + images
|
||||
|
||||
|
||||
def _first_text(payloads: list[Any], keys: tuple[str, ...]) -> str:
|
||||
result = ""
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
nonlocal result
|
||||
if result:
|
||||
return
|
||||
if isinstance(node, dict):
|
||||
for key in keys:
|
||||
value = node.get(key)
|
||||
if isinstance(value, (str, int)) and str(value).strip():
|
||||
result = str(value).strip()
|
||||
return
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
for payload in payloads:
|
||||
walk(payload)
|
||||
return result
|
||||
|
||||
|
||||
def _normalize_url(url: str) -> str:
|
||||
value = html.unescape(str(url or "")).strip().replace("\\/", "/")
|
||||
if value.startswith("//"):
|
||||
value = "https:" + value
|
||||
return value
|
||||
|
||||
|
||||
def _guess_suffix(url: str, media_type: str) -> str:
|
||||
path = urlparse(url).path.lower()
|
||||
suffix = Path(path).suffix
|
||||
if suffix in {".jpg", ".jpeg", ".png", ".webp", ".gif", ".mp4", ".mov", ".m4v"}:
|
||||
return suffix
|
||||
return ".mp4" if media_type == "video" else ".jpg"
|
||||
|
||||
|
||||
def _safe_filename(value: str) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9_.-]+", "_", value or "douyin").strip("._") or "douyin"
|
||||
|
||||
|
||||
plugin_class = DouyinPlugin
|
||||
@@ -0,0 +1 @@
|
||||
"""Bundled Pixiv plugin package."""
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "Pixiv",
|
||||
"version": "2.0.0",
|
||||
"domain": "pixiv.net",
|
||||
"author": "Sakura Downloader",
|
||||
"description": "Pixiv illustrations, novels, user batches and tag search via existing core modules.",
|
||||
"requires_auth": true,
|
||||
"tags": ["pixiv", "illustration"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
"""
|
||||
Pixiv site plugin — wraps existing pixiv_app downloader helpers (no rewrite).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from pixiv_app.core.downloader import (
|
||||
download_binary,
|
||||
download_novel,
|
||||
fetch_illust_details,
|
||||
fetch_illust_pages,
|
||||
fetch_novel_series_ids,
|
||||
fetch_user_work_ids,
|
||||
parse_pixiv_target,
|
||||
search_illust_ids_by_keyword,
|
||||
)
|
||||
from pixiv_app.core.plugin.base import (
|
||||
BasePlugin,
|
||||
PluginParseError,
|
||||
Resource,
|
||||
)
|
||||
|
||||
try:
|
||||
from pixiv_app.core.fingerprint import default_profile
|
||||
except Exception: # pragma: no cover
|
||||
default_profile = None
|
||||
|
||||
|
||||
_TAG_PATH_RE = re.compile(r"/tags/([^/]+)/?", re.IGNORECASE)
|
||||
|
||||
|
||||
class PixivPlugin(BasePlugin):
|
||||
domain = "pixiv.net"
|
||||
name = "Pixiv"
|
||||
version = "2.0.0"
|
||||
|
||||
def __init__(self, cookie: str = "", proxy_pool: Any = None) -> None:
|
||||
self.cookie = cookie
|
||||
self.proxy_pool = proxy_pool
|
||||
|
||||
def set_auth(self, cookie: str) -> None:
|
||||
self.cookie = cookie or ""
|
||||
|
||||
def set_proxy_pool(self, proxy_pool: Any) -> None:
|
||||
self.proxy_pool = proxy_pool
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
text = url.strip()
|
||||
if not text:
|
||||
return False
|
||||
if text.isdigit():
|
||||
return True
|
||||
candidate = text if "://" in text else f"https://{text.lstrip('/')}"
|
||||
try:
|
||||
hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".")
|
||||
except ValueError:
|
||||
return False
|
||||
return any(hostname == domain or hostname.endswith(f".{domain}") for domain in ("pixiv.net", "pximg.net"))
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
from pixiv_app.core.downloader import build_headers
|
||||
|
||||
headers = build_headers(cookie=self.cookie)
|
||||
if default_profile:
|
||||
headers.update(default_profile("pixiv").headers())
|
||||
return headers
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
raw = url.strip()
|
||||
if not raw:
|
||||
raise PluginParseError("Empty URL")
|
||||
|
||||
if _TAG_PATH_RE.search(raw):
|
||||
m = _TAG_PATH_RE.search(raw)
|
||||
assert m is not None
|
||||
tag = unquote(m.group(1))
|
||||
return self._parse_tag(tag)
|
||||
|
||||
try:
|
||||
kind, num_id = parse_pixiv_target(raw)
|
||||
except ValueError as exc:
|
||||
raise PluginParseError(str(exc)) from exc
|
||||
|
||||
if kind == "novel":
|
||||
return [self._resource_novel_stub(num_id)]
|
||||
|
||||
if kind == "novel_series":
|
||||
return self._parse_novel_series(num_id)
|
||||
|
||||
if kind == "user":
|
||||
return self._parse_user(num_id, input_kind="user")
|
||||
|
||||
if kind == "illust":
|
||||
return [self._parse_illust(num_id)]
|
||||
|
||||
if kind == "unknown":
|
||||
return [self._parse_illust(num_id)]
|
||||
|
||||
raise PluginParseError(f"Unsupported Pixiv target kind: {kind}")
|
||||
|
||||
def _parse_illust(self, illust_id: int) -> Resource:
|
||||
detail = fetch_illust_details(illust_id, cookie=self.cookie, proxy_pool=self.proxy_pool)
|
||||
page_urls = fetch_illust_pages(illust_id, cookie=self.cookie, proxy_pool=self.proxy_pool)
|
||||
files = [{"url": u, "page": i} for i, u in enumerate(page_urls) if u]
|
||||
thumb = page_urls[0] if page_urls else None
|
||||
return Resource(
|
||||
id=str(illust_id),
|
||||
url=f"https://www.pixiv.net/artworks/{illust_id}",
|
||||
title=str(detail.get("title") or ""),
|
||||
author=str(detail.get("userName") or ""),
|
||||
author_id=str(detail.get("userId") or ""),
|
||||
files=files,
|
||||
metadata={**detail, "kind": "illust", "illust_id": illust_id},
|
||||
thumbnail=thumb,
|
||||
created_at=str(detail.get("createDate") or detail.get("uploadDate") or "") or None,
|
||||
)
|
||||
|
||||
def _parse_user(self, seed_id: int, *, input_kind: str) -> list[Resource]:
|
||||
resolved_uid, work_ids = fetch_user_work_ids(
|
||||
seed_id,
|
||||
cookie=self.cookie,
|
||||
proxy_pool=self.proxy_pool,
|
||||
input_kind=input_kind,
|
||||
)
|
||||
out: list[Resource] = []
|
||||
for wid in work_ids:
|
||||
out.append(
|
||||
Resource(
|
||||
id=str(wid),
|
||||
url=f"https://www.pixiv.net/artworks/{wid}",
|
||||
title="",
|
||||
author="",
|
||||
author_id=str(resolved_uid),
|
||||
files=[],
|
||||
metadata={
|
||||
"kind": "illust_lazy",
|
||||
"illust_id": wid,
|
||||
"resolved_user_id": resolved_uid,
|
||||
},
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
def _parse_tag(self, tag: str) -> list[Resource]:
|
||||
ids = search_illust_ids_by_keyword(
|
||||
tag,
|
||||
cookie=self.cookie,
|
||||
proxy_pool=self.proxy_pool,
|
||||
limit=50,
|
||||
)
|
||||
resources: list[Resource] = []
|
||||
for iid in ids:
|
||||
resources.append(
|
||||
Resource(
|
||||
id=str(iid),
|
||||
url=f"https://www.pixiv.net/artworks/{iid}",
|
||||
title="",
|
||||
author="",
|
||||
author_id="",
|
||||
files=[],
|
||||
metadata={"kind": "illust_lazy", "illust_id": iid, "tag": tag},
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
def _parse_novel_series(self, series_id: int) -> list[Resource]:
|
||||
series_title, novel_ids = fetch_novel_series_ids(
|
||||
series_id,
|
||||
cookie=self.cookie,
|
||||
proxy_pool=self.proxy_pool,
|
||||
)
|
||||
resources: list[Resource] = []
|
||||
for chapter_order, novel_id in enumerate(novel_ids, start=1):
|
||||
resource = self._resource_novel_stub(novel_id)
|
||||
resource.metadata.update(
|
||||
{
|
||||
"series_id": str(series_id),
|
||||
"series_title": series_title,
|
||||
"chapter_order": chapter_order,
|
||||
}
|
||||
)
|
||||
resources.append(resource)
|
||||
return resources
|
||||
|
||||
def _resource_novel_stub(self, novel_id: int) -> Resource:
|
||||
return Resource(
|
||||
id=str(novel_id),
|
||||
url=f"https://www.pixiv.net/novel/show.php?id={novel_id}",
|
||||
title="",
|
||||
author="",
|
||||
author_id="",
|
||||
files=[],
|
||||
metadata={"kind": "novel", "novel_id": novel_id},
|
||||
)
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
kind = resource.metadata.get("kind")
|
||||
|
||||
if kind == "novel":
|
||||
novel_id = int(resource.metadata.get("novel_id") or resource.id)
|
||||
download_novel(
|
||||
novel_id,
|
||||
cookie=self.cookie,
|
||||
save_root=save_path,
|
||||
save_subdir="novels",
|
||||
proxy_pool=self.proxy_pool,
|
||||
library_db_path=None,
|
||||
metadata_overrides={
|
||||
"seriesId": resource.metadata.get("series_id"),
|
||||
"seriesTitle": resource.metadata.get("series_title"),
|
||||
"chapterOrder": resource.metadata.get("chapter_order"),
|
||||
},
|
||||
)
|
||||
matches = list(save_path.rglob(f"*{novel_id}.txt"))
|
||||
return matches
|
||||
|
||||
illust_id = int(resource.metadata.get("illust_id") or resource.id)
|
||||
|
||||
if kind == "illust_lazy" or not resource.files:
|
||||
detail = fetch_illust_details(illust_id, cookie=self.cookie, proxy_pool=self.proxy_pool)
|
||||
page_urls = fetch_illust_pages(illust_id, cookie=self.cookie, proxy_pool=self.proxy_pool)
|
||||
resource.files = [{"url": u, "page": i} for i, u in enumerate(page_urls) if u]
|
||||
resource.metadata.update(detail)
|
||||
|
||||
downloaded: list[Path] = []
|
||||
subdir = resource.metadata.get("save_subdir")
|
||||
target_dir = save_path / subdir if subdir else save_path
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for item in resource.files:
|
||||
file_url = item.get("url")
|
||||
if not file_url:
|
||||
continue
|
||||
ok, _msg = download_binary(
|
||||
file_url,
|
||||
save_dir=target_dir,
|
||||
cookie=self.cookie,
|
||||
proxy_pool=self.proxy_pool,
|
||||
)
|
||||
if ok:
|
||||
fname = file_url.rsplit("/", 1)[-1].split("?")[0]
|
||||
downloaded.append(target_dir / fname)
|
||||
|
||||
return downloaded
|
||||
|
||||
|
||||
plugin_class = PixivPlugin
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "Taobao",
|
||||
"version": "1.0.0",
|
||||
"domain": "taobao.com",
|
||||
"author": "Sakura Downloader",
|
||||
"description": "Taobao and Tmall product image/video parser with optional local Cookie support.",
|
||||
"requires_auth": false,
|
||||
"tags": ["shopping", "image", "video"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
"""Taobao and Tmall product parser/downloader.
|
||||
|
||||
The plugin parses public product pages and optionally reuses a locally saved
|
||||
Taobao cookie. Tests mock every HTTP request; no live site is contacted there.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from pixiv_app.core.cookie_import import cookies_to_header, parse_cookie_text
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginDownloadError, PluginParseError, Resource
|
||||
|
||||
|
||||
_ALLOWED_DOMAINS = ("taobao.com", "tmall.com", "tb.cn")
|
||||
_TITLE_KEYS = {"title", "itemtitle", "item_title", "subject", "goodsname", "goods_name"}
|
||||
_AUTHOR_KEYS = {"shopname", "shop_name", "sellername", "seller_name", "nick", "nickname"}
|
||||
_ID_KEYS = {"itemid", "item_id", "nid", "auctionid", "auction_id", "goodsid", "goods_id"}
|
||||
_PRICE_KEYS = {"price", "promotionprice", "promotion_price", "saleprice", "sale_price"}
|
||||
_IMAGE_KEYS = {
|
||||
"images",
|
||||
"image",
|
||||
"pics",
|
||||
"picurl",
|
||||
"pic_url",
|
||||
"imageurl",
|
||||
"image_url",
|
||||
"itemimages",
|
||||
"item_images",
|
||||
"detailimages",
|
||||
"detail_images",
|
||||
"mainimage",
|
||||
"main_image",
|
||||
}
|
||||
_VIDEO_KEYS = {"video", "videos", "videourl", "video_url", "videoid", "video_id"}
|
||||
_URL_RE = re.compile(r"(?:https?:)?(?:\\?/){2}[^\s\"'<>]+", re.IGNORECASE)
|
||||
_JSON_SCRIPT_RE = re.compile(
|
||||
r"<script[^>]*(?:type=[\"']application/(?:ld\+)?json[\"']|id=[\"']__NEXT_DATA__[\"'])[^>]*>"
|
||||
r"(?P<body>.*?)</script>",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_META_RE = re.compile(
|
||||
r"<meta[^>]+(?:property|name)=[\"'](?P<name>og:title|og:image|description|og:description)[\"']"
|
||||
r"[^>]+content=[\"'](?P<content>.*?)[\"'][^>]*>",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_IMG_RE = re.compile(
|
||||
r"<img[^>]+(?:data-src|data-original|data-ks-lazyload|src)=[\"'](?P<url>[^\"']+)[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaobaoPluginConfig:
|
||||
timeout: float = 25.0
|
||||
max_redirects: int = 8
|
||||
cookie_file: str = ""
|
||||
max_media: int = 100
|
||||
|
||||
|
||||
class TaobaoPlugin(BasePlugin):
|
||||
domain = "taobao.com"
|
||||
name = "Taobao"
|
||||
version = "1.0.0"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.config = TaobaoPluginConfig(cookie_file=str(runtime_path("taobao_cookies.txt")))
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
target = _extract_first_product_url(url)
|
||||
if not target:
|
||||
return False
|
||||
try:
|
||||
host = (urlparse(target).hostname or "").lower()
|
||||
except ValueError:
|
||||
return False
|
||||
return _allowed_host(host)
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/json;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.7",
|
||||
"Referer": "https://www.taobao.com/",
|
||||
}
|
||||
cookie = self._cookie_header()
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
return headers
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
target = _extract_first_product_url(url)
|
||||
if not target or not self.can_handle(target):
|
||||
raise PluginParseError("未识别到有效的淘宝或天猫商品链接。")
|
||||
|
||||
final_url, page = self._fetch_page(target)
|
||||
if not self.can_handle(final_url):
|
||||
raise PluginParseError("淘宝短链接跳转到了不受支持的站点。")
|
||||
if _looks_like_challenge(page, final_url):
|
||||
raise PluginParseError("淘宝要求登录或安全验证,请导入有效 Cookie 后重试。")
|
||||
|
||||
payloads = _extract_json_payloads(page)
|
||||
extracted = _extract_structured_fields(payloads)
|
||||
meta = _extract_meta(page)
|
||||
product_id = _extract_item_id(final_url) or extracted["id"] or _extract_item_id(target)
|
||||
title = extracted["title"] or meta.get("og:title", "")
|
||||
title = _clean_title(title) or f"淘宝商品 {product_id or ''}".strip()
|
||||
author = extracted["author"]
|
||||
description = meta.get("og:description") or meta.get("description") or ""
|
||||
|
||||
files = list(extracted["files"])
|
||||
if meta.get("og:image"):
|
||||
files.insert(0, {"type": "image", "url": _normalize_url(meta["og:image"])})
|
||||
for match in _IMG_RE.finditer(page):
|
||||
files.append({"type": "image", "url": _normalize_url(match.group("url"))})
|
||||
files.extend(_extract_media_from_html(page))
|
||||
files = _dedupe_media(files, limit=self.config.max_media)
|
||||
if not files:
|
||||
raise PluginParseError("商品页面已打开,但没有解析到商品图片或视频,页面结构可能已更新。")
|
||||
|
||||
thumbnail = next((item["url"] for item in files if item["type"] == "image"), None)
|
||||
return [
|
||||
Resource(
|
||||
id=product_id or str(abs(hash(final_url))),
|
||||
url=final_url,
|
||||
title=title,
|
||||
author=author,
|
||||
author_id="",
|
||||
files=files,
|
||||
metadata={
|
||||
"source_platform": "taobao",
|
||||
"kind": "product",
|
||||
"product_id": product_id,
|
||||
"price": extracted["price"],
|
||||
"description": description,
|
||||
"original_url": target,
|
||||
},
|
||||
thumbnail=thumbnail,
|
||||
)
|
||||
]
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
downloaded: list[Path] = []
|
||||
headers = self.get_headers()
|
||||
with httpx.Client(timeout=max(30.0, self.config.timeout), follow_redirects=True, headers=headers) as client:
|
||||
for index, item in enumerate(resource.files, start=1):
|
||||
media_url = _normalize_url(str(item.get("url") or ""))
|
||||
if not media_url:
|
||||
continue
|
||||
media_type = str(item.get("type") or "image")
|
||||
suffix = _guess_suffix(media_url, media_type)
|
||||
target = save_path / f"{_safe_filename(resource.id)}_{index:03d}{suffix}"
|
||||
if target.is_file() and target.stat().st_size > 256:
|
||||
downloaded.append(target)
|
||||
continue
|
||||
try:
|
||||
response = client.get(media_url, headers={**headers, "Referer": resource.url})
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
raise PluginDownloadError(f"淘宝媒体下载失败: {exc}") from exc
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if "text/html" in content_type or "application/json" in content_type:
|
||||
raise PluginDownloadError("淘宝媒体地址返回了验证页面,请更新 Cookie 后重试。")
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size <= 256:
|
||||
target.unlink(missing_ok=True)
|
||||
raise PluginDownloadError("下载内容过小,可能是失效链接或占位图。")
|
||||
downloaded.append(target)
|
||||
return downloaded
|
||||
|
||||
def _fetch_page(self, url: str) -> tuple[str, str]:
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=self.config.timeout,
|
||||
follow_redirects=True,
|
||||
max_redirects=self.config.max_redirects,
|
||||
headers=self.get_headers(),
|
||||
) as client:
|
||||
response = client.get(url)
|
||||
response.raise_for_status()
|
||||
return str(response.url), response.text
|
||||
except Exception as exc:
|
||||
raise PluginParseError(f"淘宝商品页面加载失败: {exc}") from exc
|
||||
|
||||
def _cookie_header(self) -> str:
|
||||
path = Path(self.config.cookie_file or "")
|
||||
if not path.is_file():
|
||||
return ""
|
||||
try:
|
||||
cookies = parse_cookie_text(path.read_text(encoding="utf-8"))
|
||||
return cookies_to_header(cookies, _ALLOWED_DOMAINS)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _allowed_host(host: str) -> bool:
|
||||
return any(host == domain or host.endswith(f".{domain}") for domain in _ALLOWED_DOMAINS)
|
||||
|
||||
|
||||
def _extract_first_product_url(text: str) -> str:
|
||||
value = html.unescape((text or "").strip())
|
||||
match = re.search(r"https?://[^\s\"'<>]+", value, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(0).rstrip(",.;)")
|
||||
if any(domain in value.lower() for domain in _ALLOWED_DOMAINS):
|
||||
return "https://" + value.lstrip("/")
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_item_id(url: str) -> str:
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
query = parse_qs(parsed.query)
|
||||
except ValueError:
|
||||
return ""
|
||||
for key in ("id", "itemId", "item_id", "auctionId"):
|
||||
value = (query.get(key) or [""])[0]
|
||||
match = re.search(r"\d{5,}", str(value))
|
||||
if match:
|
||||
return match.group(0)
|
||||
for pattern in (r"/item/(\d{5,})", r"/i(\d{5,})\.htm", r"item_id[=/](\d{5,})"):
|
||||
match = re.search(pattern, parsed.path, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_json_payloads(page: str) -> list[Any]:
|
||||
payloads: list[Any] = []
|
||||
for match in _JSON_SCRIPT_RE.finditer(page or ""):
|
||||
raw = html.unescape(match.group("body") or "").strip()
|
||||
try:
|
||||
payloads.append(json.loads(raw))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
for pattern in (
|
||||
r"window\.__INITIAL_STATE__\s*=\s*({.*?})\s*;?\s*</script>",
|
||||
r"window\.__INIT_DATA__\s*=\s*({.*?})\s*;?\s*</script>",
|
||||
):
|
||||
for match in re.finditer(pattern, page or "", re.IGNORECASE | re.DOTALL):
|
||||
try:
|
||||
payloads.append(json.loads(match.group(1)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return payloads
|
||||
|
||||
|
||||
def _extract_structured_fields(payloads: list[Any]) -> dict[str, Any]:
|
||||
titles: list[str] = []
|
||||
authors: list[str] = []
|
||||
ids: list[str] = []
|
||||
prices: list[str] = []
|
||||
files: list[dict[str, str]] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
normalized = str(key).replace("-", "_").lower()
|
||||
compact = normalized.replace("_", "")
|
||||
values = _text_values(value)
|
||||
if normalized in _TITLE_KEYS or compact in _TITLE_KEYS:
|
||||
titles.extend(values)
|
||||
if normalized in _AUTHOR_KEYS or compact in _AUTHOR_KEYS:
|
||||
authors.extend(values)
|
||||
if normalized in _ID_KEYS or compact in _ID_KEYS:
|
||||
ids.extend(values)
|
||||
if normalized in _PRICE_KEYS or compact in _PRICE_KEYS:
|
||||
prices.extend(values)
|
||||
if normalized in _IMAGE_KEYS or compact in _IMAGE_KEYS:
|
||||
files.extend({"type": "image", "url": url} for url in _urls_from_node(value))
|
||||
if normalized in _VIDEO_KEYS or compact in _VIDEO_KEYS:
|
||||
files.extend({"type": "video", "url": url} for url in _urls_from_node(value))
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
for payload in payloads:
|
||||
walk(payload)
|
||||
product_id = next((match.group(0) for value in ids if (match := re.search(r"\d{5,}", value))), "")
|
||||
return {
|
||||
"title": _best_text(titles),
|
||||
"author": _best_text(authors),
|
||||
"id": product_id,
|
||||
"price": next((value for value in prices if value.strip()), ""),
|
||||
"files": files,
|
||||
}
|
||||
|
||||
|
||||
def _extract_meta(page: str) -> dict[str, str]:
|
||||
values: dict[str, str] = {}
|
||||
for match in _META_RE.finditer(page or ""):
|
||||
values[match.group("name").lower()] = html.unescape(match.group("content")).strip()
|
||||
return values
|
||||
|
||||
|
||||
def _extract_media_from_html(page: str) -> list[dict[str, str]]:
|
||||
files: list[dict[str, str]] = []
|
||||
for raw in _URL_RE.findall(page or ""):
|
||||
url = _normalize_url(raw)
|
||||
lower = url.lower().split("?", 1)[0]
|
||||
if any(lower.endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp", ".avif")):
|
||||
files.append({"type": "image", "url": url})
|
||||
elif any(lower.endswith(ext) for ext in (".mp4", ".m3u8", ".webm")):
|
||||
files.append({"type": "video", "url": url})
|
||||
return files
|
||||
|
||||
|
||||
def _urls_from_node(node: Any) -> list[str]:
|
||||
if isinstance(node, str):
|
||||
value = _normalize_url(node)
|
||||
return [value] if value.startswith(("http://", "https://")) else []
|
||||
if isinstance(node, list):
|
||||
return [url for item in node for url in _urls_from_node(item)]
|
||||
if isinstance(node, dict):
|
||||
urls: list[str] = []
|
||||
for key, value in node.items():
|
||||
if str(key).lower() in {"url", "src", "urls", "url_list", "url_list_v2", "image", "video"}:
|
||||
urls.extend(_urls_from_node(value))
|
||||
elif isinstance(value, (dict, list)):
|
||||
urls.extend(_urls_from_node(value))
|
||||
return urls
|
||||
return []
|
||||
|
||||
|
||||
def _text_values(node: Any) -> list[str]:
|
||||
if isinstance(node, (str, int, float)):
|
||||
return [str(node).strip()]
|
||||
if isinstance(node, dict):
|
||||
return [str(value).strip() for key, value in node.items() if str(key).lower() in {"text", "value", "name"}]
|
||||
return []
|
||||
|
||||
|
||||
def _best_text(values: list[str]) -> str:
|
||||
cleaned = [html.unescape(value).strip() for value in values if value and len(value.strip()) > 1]
|
||||
return max(cleaned, key=len, default="")
|
||||
|
||||
|
||||
def _clean_title(value: str) -> str:
|
||||
title = re.sub(r"\s*[-_–—]\s*(淘宝网|天猫|tmall|taobao).*$", "", value or "", flags=re.IGNORECASE)
|
||||
return re.sub(r"\s+", " ", title).strip()
|
||||
|
||||
|
||||
def _normalize_url(value: str) -> str:
|
||||
url = html.unescape((value or "").strip()).replace("\\u002F", "/").replace("\\/", "/")
|
||||
if url.startswith("//"):
|
||||
return "https:" + url
|
||||
return url
|
||||
|
||||
|
||||
def _dedupe_media(files: list[dict[str, str]], *, limit: int) -> list[dict[str, str]]:
|
||||
result: list[dict[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for item in files:
|
||||
url = _normalize_url(str(item.get("url") or ""))
|
||||
if not url.startswith(("http://", "https://")) or url in seen:
|
||||
continue
|
||||
lower = url.lower()
|
||||
if any(token in lower for token in ("sprite", "icon", "logo", "avatar", "qrcode")):
|
||||
continue
|
||||
seen.add(url)
|
||||
result.append({"type": "video" if item.get("type") == "video" else "image", "url": url})
|
||||
if len(result) >= max(1, limit):
|
||||
break
|
||||
return result
|
||||
|
||||
|
||||
def _looks_like_challenge(page: str, url: str) -> bool:
|
||||
sample = f"{url}\n{page[:20000]}".lower()
|
||||
markers = ("login.taobao.com", "punish", "captcha", "滑动验证", "安全验证", "请登录")
|
||||
return any(marker in sample for marker in markers)
|
||||
|
||||
|
||||
def _guess_suffix(url: str, media_type: str) -> str:
|
||||
suffix = Path(urlparse(url).path).suffix.lower()
|
||||
allowed = {".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif", ".mp4", ".webm", ".m3u8"}
|
||||
if suffix in allowed:
|
||||
return suffix
|
||||
return ".mp4" if media_type == "video" else ".jpg"
|
||||
|
||||
|
||||
def _safe_filename(value: str) -> str:
|
||||
return re.sub(r"[^0-9A-Za-z._-]+", "_", value or "taobao")[:80] or "taobao"
|
||||
|
||||
|
||||
plugin_class = TaobaoPlugin
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""X (Twitter) Playwright plugin."""
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Shared Playwright launch options for X — reduce automation fingerprint; prefer system Chrome."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from playwright.async_api import BrowserContext, Playwright
|
||||
|
||||
from pixiv_app.core.playwright_runtime import configure_playwright_runtime_env, preferred_chromium_executable
|
||||
|
||||
try:
|
||||
from pixiv_app.core.fingerprint import default_profile
|
||||
except Exception: # pragma: no cover - plugin can run standalone
|
||||
default_profile = None
|
||||
|
||||
try:
|
||||
from pixiv_app.core.browser_proxy_pool import parse_proxy_url
|
||||
except Exception: # pragma: no cover
|
||||
parse_proxy_url = None
|
||||
|
||||
|
||||
async def launch_x_persistent_context(
|
||||
playwright: Playwright,
|
||||
profile_dir: Path,
|
||||
*,
|
||||
headless: bool,
|
||||
) -> BrowserContext:
|
||||
"""
|
||||
Launch persistent Chromium with flags that reduce obvious automation signals.
|
||||
|
||||
Order: ``X_PLAYWRIGHT_CHANNEL`` env (if set), then system Chrome, then bundled Chromium.
|
||||
Google OAuth often still blocks automation browsers; users should use X email/phone login
|
||||
or import cookies from a normal browser session.
|
||||
"""
|
||||
configure_playwright_runtime_env()
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
browser_profile = default_profile("x") if default_profile else None
|
||||
|
||||
base_kw: dict = {
|
||||
"user_data_dir": str(profile_dir),
|
||||
"headless": headless,
|
||||
"viewport": browser_profile.viewport if browser_profile else {"width": 1280, "height": 800},
|
||||
"locale": browser_profile.locale if browser_profile else "zh-CN",
|
||||
"timezone_id": browser_profile.timezone_id if browser_profile else "Asia/Shanghai",
|
||||
"screen": browser_profile.viewport if browser_profile else {"width": 1280, "height": 800},
|
||||
"extra_http_headers": {
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
},
|
||||
"ignore_default_args": ["--enable-automation"],
|
||||
"args": [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--window-size=1280,800",
|
||||
],
|
||||
"user_agent": browser_profile.user_agent
|
||||
if browser_profile
|
||||
else (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
if browser_profile:
|
||||
base_kw["device_scale_factor"] = browser_profile.device_scale_factor
|
||||
base_kw["is_mobile"] = browser_profile.is_mobile
|
||||
base_kw["has_touch"] = browser_profile.has_touch
|
||||
|
||||
proxy_value = os.environ.get("X_PROXY_URL", os.environ.get("PIXIV_BROWSER_PROXY", "")).strip()
|
||||
if proxy_value and parse_proxy_url:
|
||||
parsed_proxy = parse_proxy_url(proxy_value)
|
||||
if parsed_proxy is not None:
|
||||
base_kw["proxy"] = parsed_proxy.to_playwright_proxy()
|
||||
|
||||
env_ch = os.environ.get("X_PLAYWRIGHT_CHANNEL", "").strip()
|
||||
local_executable = preferred_chromium_executable()
|
||||
seen: set[str] = set()
|
||||
attempts: list[tuple[str, dict]] = []
|
||||
for label, extra in (
|
||||
("env_channel", {"channel": env_ch} if env_ch else {}),
|
||||
("installed_chromium", {"executable_path": str(local_executable)} if local_executable else {}),
|
||||
("system_chrome", {"channel": "chrome"}),
|
||||
("default", {}),
|
||||
):
|
||||
key = f"{label}:{extra!r}"
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
attempts.append((label, extra))
|
||||
|
||||
last_err: Exception | None = None
|
||||
for _, extra in attempts:
|
||||
kw = dict(base_kw)
|
||||
kw.update(extra)
|
||||
try:
|
||||
ctx = await playwright.chromium.launch_persistent_context(**kw)
|
||||
await _apply_stealth_scripts(ctx)
|
||||
return ctx
|
||||
except Exception as exc:
|
||||
last_err = exc
|
||||
continue
|
||||
|
||||
assert last_err is not None
|
||||
raise last_err
|
||||
|
||||
|
||||
async def _apply_stealth_scripts(ctx: BrowserContext) -> None:
|
||||
"""Best-effort scripts; cannot guarantee OAuth providers will accept the browser."""
|
||||
await ctx.add_init_script(
|
||||
"""
|
||||
try {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
} catch (e) {}
|
||||
"""
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "X",
|
||||
"version": "1.0.0",
|
||||
"domain": "twitter.com",
|
||||
"author": "Sakura Downloader",
|
||||
"description": "X (Twitter) media via Playwright + httpx. Session in runtime/x_profile.",
|
||||
"requires_auth": true,
|
||||
"tags": ["social", "media", "playwright"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
@@ -0,0 +1,946 @@
|
||||
"""Playwright helpers for the X plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from playwright.async_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError, async_playwright
|
||||
|
||||
from .browser_opts import launch_x_persistent_context
|
||||
|
||||
from pixiv_app.core.plugin.base import PluginParseError
|
||||
|
||||
try:
|
||||
from pixiv_app.core.fingerprint import default_profile
|
||||
except Exception: # pragma: no cover
|
||||
default_profile = None
|
||||
|
||||
_GUEST_WALL_HINT = (
|
||||
"X 需要先登录。\n"
|
||||
"请在“插件管理”里点击“X 登录”完成登录,或把 Cookie 导出为 JSON 保存到 "
|
||||
"runtime/x_cookies.json 后重试。"
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class XPlaywrightClient:
|
||||
def __init__(self, config: Any) -> None:
|
||||
self.config = config
|
||||
self._pw: Any = None
|
||||
self.context: BrowserContext | None = None
|
||||
self._response_store: dict[int, list[dict[str, Any]]] = {}
|
||||
self._response_tasks: dict[int, set[asyncio.Task[Any]]] = {}
|
||||
|
||||
async def start(self) -> None:
|
||||
if self.context is not None:
|
||||
return
|
||||
self._pw = await async_playwright().start()
|
||||
profile = Path(self.config.profile_dir)
|
||||
profile.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.context = await launch_x_persistent_context(
|
||||
self._pw,
|
||||
profile,
|
||||
headless=self.config.headless,
|
||||
)
|
||||
|
||||
cookies = self._prepare_cookie_items(self._load_cookie_items())
|
||||
if cookies:
|
||||
loaded = await self._add_cookies_tolerant(cookies)
|
||||
if loaded <= 0:
|
||||
raise PluginParseError(
|
||||
"X Cookie 注入失败:请重新导出 x.com cookies.txt,确认包含 auth_token 和 ct0 后再导入。"
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.context is not None:
|
||||
await self.context.close()
|
||||
self.context = None
|
||||
self._response_tasks.clear()
|
||||
self._response_store.clear()
|
||||
if self._pw is not None:
|
||||
await self._pw.stop()
|
||||
self._pw = None
|
||||
|
||||
async def new_page_goto(self, url: str, *, wait_until: str = "domcontentloaded") -> Page:
|
||||
assert self.context is not None
|
||||
page = await self.context.new_page()
|
||||
self._attach_response_collector(page)
|
||||
timeout = getattr(self.config, "wait_timeout", 30000)
|
||||
try:
|
||||
await page.goto(url, wait_until=wait_until, timeout=timeout)
|
||||
except PlaywrightTimeoutError as exc:
|
||||
if page.url and page.url != "about:blank":
|
||||
await asyncio.sleep(2.0)
|
||||
return page
|
||||
try:
|
||||
await page.goto(url, wait_until="commit", timeout=min(timeout, 15000))
|
||||
except PlaywrightTimeoutError as retry_exc:
|
||||
raise PluginParseError(
|
||||
"X 页面加载超时,可能是网络较慢、访问过快触发风控,或登录会话已失效。"
|
||||
) from retry_exc
|
||||
except Exception as retry_exc:
|
||||
raise PluginParseError(f"X 页面加载失败: {retry_exc}") from retry_exc
|
||||
if page.url == "about:blank":
|
||||
raise PluginParseError("X 页面没有成功打开,请稍后重试。") from exc
|
||||
await self.human_pause(2.0, 4.5)
|
||||
await self.move_mouse_softly(page)
|
||||
return page
|
||||
|
||||
async def human_pause(self, low: float | None = None, high: float | None = None) -> None:
|
||||
low = float(low if low is not None else getattr(self.config, "min_action_delay", 3.0))
|
||||
high = float(high if high is not None else getattr(self.config, "max_action_delay", 8.0))
|
||||
if high < low:
|
||||
high = low
|
||||
await asyncio.sleep(random.uniform(low, high))
|
||||
|
||||
async def move_mouse_softly(self, page: Page) -> None:
|
||||
try:
|
||||
viewport = page.viewport_size or {"width": 1280, "height": 800}
|
||||
width = int(viewport.get("width", 1280))
|
||||
height = int(viewport.get("height", 800))
|
||||
for _ in range(random.randint(1, 3)):
|
||||
x = random.randint(max(12, width // 8), max(20, width - 24))
|
||||
y = random.randint(max(12, height // 8), max(20, height - 24))
|
||||
await page.mouse.move(x, y, steps=random.randint(8, 18))
|
||||
await asyncio.sleep(random.uniform(0.2, 0.7))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def human_scroll_page(self, page: Page) -> float:
|
||||
await self.move_mouse_softly(page)
|
||||
viewport = page.viewport_size or {"height": 800}
|
||||
height = int(viewport.get("height", 800))
|
||||
delta = random.randint(max(260, height // 2), max(420, int(height * 1.2)))
|
||||
if random.random() < 0.12:
|
||||
delta = -random.randint(120, 280)
|
||||
await page.mouse.wheel(0, delta)
|
||||
if random.random() < 0.07:
|
||||
await asyncio.sleep(random.uniform(30.0, 90.0))
|
||||
await self.human_pause()
|
||||
return float(await page.evaluate("document.body.scrollHeight"))
|
||||
|
||||
async def raise_if_guest_wall(self, page: Page) -> None:
|
||||
await asyncio.sleep(0.45)
|
||||
try:
|
||||
url = page.url or ""
|
||||
if "/i/flow/login" in url or "/i/flow/signup" in url:
|
||||
raise PluginParseError(_GUEST_WALL_HINT)
|
||||
except PluginParseError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
wall = page.get_by_text(
|
||||
re.compile(
|
||||
r"注册 X|Sign up for X|登录|Sign in|Phone, email, or username",
|
||||
re.I,
|
||||
)
|
||||
).first
|
||||
if await wall.is_visible(timeout=2200):
|
||||
raise PluginParseError(_GUEST_WALL_HINT)
|
||||
except PluginParseError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
dialog = page.locator('[role="dialog"]').first
|
||||
if await dialog.is_visible(timeout=600):
|
||||
inner = (await dialog.inner_text())[:800]
|
||||
if ("注册" in inner or "Sign up" in inner) and ("Google" in inner or "Apple" in inner or "Next" in inner):
|
||||
raise PluginParseError(_GUEST_WALL_HINT)
|
||||
except PluginParseError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def wait_for_main_content(self, page: Page) -> None:
|
||||
timeout = min(15000, getattr(self.config, "wait_timeout", 30000))
|
||||
for sel in ('article[data-testid="tweet"]', "article", '[data-testid="tweetPhoto"]'):
|
||||
try:
|
||||
await page.wait_for_selector(sel, timeout=timeout)
|
||||
return
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
async def wait_for_media(self, page: Page) -> bool:
|
||||
timeout = 10000
|
||||
for sel in (
|
||||
'[data-testid="tweetPhoto"]',
|
||||
'img[src*="pbs.twimg.com/media"]',
|
||||
"video",
|
||||
"video source",
|
||||
):
|
||||
try:
|
||||
await page.wait_for_selector(sel, timeout=timeout)
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
async def extract_media_urls(self, page: Page) -> list[str]:
|
||||
urls: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
imgs = await page.query_selector_all('img[src*="pbs.twimg.com/media"]')
|
||||
for img in imgs:
|
||||
src = await img.get_attribute("src")
|
||||
if src and "profile_images" not in src:
|
||||
orig = self._original_tw_image_url(src)
|
||||
if orig and orig not in seen:
|
||||
seen.add(orig)
|
||||
urls.append(orig)
|
||||
|
||||
for sel in ('[data-testid="tweetPhoto"] img', "article img"):
|
||||
elements = await page.query_selector_all(sel)
|
||||
for el in elements:
|
||||
src = await el.get_attribute("src")
|
||||
if src and "pbs.twimg.com/media" in src:
|
||||
orig = self._original_tw_image_url(src)
|
||||
if orig and orig not in seen:
|
||||
seen.add(orig)
|
||||
urls.append(orig)
|
||||
|
||||
try:
|
||||
raw_video_urls = await page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const values = [];
|
||||
for (const el of document.querySelectorAll('video, video source')) {
|
||||
const candidates = [
|
||||
el.currentSrc,
|
||||
el.src,
|
||||
el.getAttribute?.('src'),
|
||||
el.getAttribute?.('data-src'),
|
||||
el.getAttribute?.('poster'),
|
||||
];
|
||||
for (const value of candidates) {
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
values.push(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
"""
|
||||
)
|
||||
except Exception:
|
||||
raw_video_urls = []
|
||||
|
||||
for src in raw_video_urls:
|
||||
normalized = self._normalize_media_url(str(src))
|
||||
if normalized and self._is_video_url(normalized) and normalized not in seen:
|
||||
seen.add(normalized)
|
||||
urls.append(normalized)
|
||||
|
||||
try:
|
||||
html = await page.content()
|
||||
except Exception:
|
||||
html = ""
|
||||
|
||||
for image_url in self._extract_image_urls_from_html(html):
|
||||
if image_url not in seen:
|
||||
seen.add(image_url)
|
||||
urls.append(image_url)
|
||||
|
||||
for pattern in (
|
||||
r"https?://video\.twimg\.com/[^\s\"'<>\\]+",
|
||||
r"https:\\/\\/video\.twimg\.com\\/[^\s\"'<>]+",
|
||||
):
|
||||
for match in re.findall(pattern, html):
|
||||
normalized = self._normalize_media_url(match)
|
||||
if normalized and self._is_video_url(normalized) and normalized not in seen:
|
||||
seen.add(normalized)
|
||||
urls.append(normalized)
|
||||
|
||||
return urls
|
||||
|
||||
async def extract_media_urls_from_api_responses(self, page: Page) -> list[str]:
|
||||
await self.wait_for_response_capture(page)
|
||||
payloads = self._response_store.get(id(page), [])
|
||||
if not payloads:
|
||||
return []
|
||||
|
||||
urls: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(url: str | None) -> None:
|
||||
if not url:
|
||||
return
|
||||
normalized = self._normalize_media_url(str(url))
|
||||
if not normalized:
|
||||
return
|
||||
if "pbs.twimg.com/media" in normalized:
|
||||
normalized = self._original_tw_image_url(normalized)
|
||||
if normalized not in seen:
|
||||
seen.add(normalized)
|
||||
urls.append(normalized)
|
||||
|
||||
for payload in payloads:
|
||||
data = payload.get("json")
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
for media_item in self._find_media_entities(data):
|
||||
media_url = media_item.get("media_url_https") or media_item.get("media_url")
|
||||
media_type = str(media_item.get("type") or "").lower()
|
||||
if media_url and media_type in {"photo", "animated_gif", "video", ""}:
|
||||
add(str(media_url))
|
||||
|
||||
video_info = media_item.get("video_info")
|
||||
if isinstance(video_info, dict):
|
||||
best_video = self._pick_best_video_variant(video_info)
|
||||
add(best_video)
|
||||
|
||||
return urls
|
||||
|
||||
async def extract_media_records_from_api_responses(self, page: Page) -> list[dict[str, Any]]:
|
||||
await self.wait_for_response_capture(page)
|
||||
payloads = self._response_store.get(id(page), [])
|
||||
if not payloads:
|
||||
return []
|
||||
|
||||
records: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
|
||||
def add_record(kind: str, url: str | None, *, thumbnail: str | None = None) -> None:
|
||||
if not url:
|
||||
return
|
||||
normalized = self._normalize_media_url(str(url))
|
||||
if not normalized:
|
||||
return
|
||||
key = (kind, normalized)
|
||||
if key in seen:
|
||||
return
|
||||
seen.add(key)
|
||||
records.append(
|
||||
{
|
||||
"type": kind,
|
||||
"url": normalized,
|
||||
"thumbnail": self._normalize_media_url(thumbnail) if thumbnail else None,
|
||||
}
|
||||
)
|
||||
|
||||
for payload in payloads:
|
||||
data = payload.get("json")
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
for media_item in self._find_media_entities(data):
|
||||
media_type = str(media_item.get("type") or "").lower()
|
||||
thumb_url = media_item.get("media_url_https") or media_item.get("media_url")
|
||||
if media_type == "photo":
|
||||
add_record("image", thumb_url)
|
||||
continue
|
||||
|
||||
video_info = media_item.get("video_info")
|
||||
if isinstance(video_info, dict):
|
||||
best_video = self._pick_best_video_variant(video_info)
|
||||
if best_video:
|
||||
add_record("video", best_video, thumbnail=thumb_url)
|
||||
continue
|
||||
|
||||
return records
|
||||
|
||||
async def wait_for_response_capture(self, page: Page, timeout: float = 2.0) -> None:
|
||||
page_id = id(page)
|
||||
tasks = self._response_tasks.get(page_id)
|
||||
if not tasks:
|
||||
return
|
||||
|
||||
deadline = asyncio.get_running_loop().time() + timeout
|
||||
while tasks and asyncio.get_running_loop().time() < deadline:
|
||||
pending = [task for task in list(tasks) if not task.done()]
|
||||
if not pending:
|
||||
break
|
||||
_, pending = await asyncio.wait(pending, timeout=0.2)
|
||||
tasks = self._response_tasks.get(page_id)
|
||||
if not tasks:
|
||||
break
|
||||
|
||||
async def scroll_and_collect_media(self, page: Page, max_scrolls: int = 12, max_items: int = 40) -> list[str]:
|
||||
collected: list[str] = []
|
||||
seen: set[str] = set()
|
||||
last_height = 0.0
|
||||
for _ in range(max_scrolls):
|
||||
batch: list[str] = []
|
||||
for item in await self.extract_media_records_from_api_responses(page):
|
||||
url = item.get("url")
|
||||
if isinstance(url, str) and url:
|
||||
batch.append(url)
|
||||
batch.extend(await self.extract_media_urls(page))
|
||||
for url in batch:
|
||||
if url not in seen:
|
||||
seen.add(url)
|
||||
collected.append(url)
|
||||
if len(collected) >= max_items:
|
||||
return collected
|
||||
try:
|
||||
height = await self.human_scroll_page(page)
|
||||
except Exception:
|
||||
break
|
||||
if height == last_height:
|
||||
break
|
||||
last_height = float(height)
|
||||
return collected
|
||||
|
||||
async def scroll_and_collect_tweet_urls(self, page: Page, limit: int = 40) -> list[str]:
|
||||
found: list[str] = []
|
||||
seen: set[str] = set()
|
||||
stale_rounds = 0
|
||||
max_rounds = 25
|
||||
|
||||
for _ in range(max_rounds):
|
||||
if len(found) >= limit:
|
||||
break
|
||||
before = len(found)
|
||||
links = await page.query_selector_all('a[href*="/status/"]')
|
||||
for link in links:
|
||||
href = await link.get_attribute("href")
|
||||
if not href or "/status/" not in href:
|
||||
continue
|
||||
if href.startswith("/"):
|
||||
full = f"https://x.com{href}"
|
||||
elif href.startswith("http"):
|
||||
full = href
|
||||
else:
|
||||
continue
|
||||
full = re.sub(r"\?.*$", "", full.split("#")[0])
|
||||
if full not in seen:
|
||||
seen.add(full)
|
||||
found.append(full)
|
||||
if len(found) >= limit:
|
||||
break
|
||||
|
||||
if len(found) == before:
|
||||
stale_rounds += 1
|
||||
if stale_rounds >= 5:
|
||||
break
|
||||
else:
|
||||
stale_rounds = 0
|
||||
|
||||
try:
|
||||
await self.human_scroll_page(page)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
return found[:limit]
|
||||
|
||||
async def get_author_info(self, page: Page) -> dict[str, str]:
|
||||
info = {"name": "", "username": ""}
|
||||
try:
|
||||
block = await page.query_selector('[data-testid="User-Name"]')
|
||||
if block:
|
||||
text = await block.inner_text()
|
||||
lines = [line.strip() for line in text.split("\n") if line.strip()]
|
||||
if lines:
|
||||
info["name"] = lines[0]
|
||||
for line in lines:
|
||||
if line.startswith("@"):
|
||||
info["username"] = line.lstrip("@")
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return info
|
||||
|
||||
async def get_user_display_name(self, page: Page, fallback: str) -> dict[str, str]:
|
||||
info = {"name": fallback, "username": fallback}
|
||||
try:
|
||||
block = await page.query_selector('[data-testid="UserName"]')
|
||||
if block:
|
||||
text = await block.inner_text()
|
||||
lines = [line.strip() for line in text.split("\n") if line.strip()]
|
||||
if lines:
|
||||
info["name"] = lines[0]
|
||||
except Exception:
|
||||
pass
|
||||
return info
|
||||
|
||||
async def get_tweet_text(self, page: Page) -> str:
|
||||
try:
|
||||
el = await page.query_selector('[data-testid="tweetText"]')
|
||||
if el:
|
||||
return (await el.inner_text()).strip()
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
async def get_tweet_date(self, page: Page) -> str:
|
||||
try:
|
||||
time_el = await page.query_selector("article time")
|
||||
if time_el:
|
||||
return (await time_el.get_attribute("datetime")) or ""
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
async def download_image(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
name = url.split("/")[-1].split("?")[0]
|
||||
if not any(name.lower().endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp")):
|
||||
name += ".jpg"
|
||||
target = save_path / name
|
||||
if target.exists() and target.stat().st_size > 256:
|
||||
return target
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
|
||||
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if response.status_code == 200 and "text/html" not in content_type:
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size > 256:
|
||||
return target
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
async def download_video(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:
|
||||
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
|
||||
if response.status_code != 200:
|
||||
return None
|
||||
|
||||
final_url = str(response.url)
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
body_text = response.text if self._looks_like_hls(final_url, content_type, "") or "text" in content_type else ""
|
||||
if "text/html" in content_type and not self._looks_like_hls(final_url, content_type, body_text):
|
||||
return None
|
||||
|
||||
if self._looks_like_hls(final_url, content_type, body_text):
|
||||
return await self._download_hls_playlist(
|
||||
client,
|
||||
playlist_url=final_url,
|
||||
playlist_text=body_text,
|
||||
save_path=save_path,
|
||||
referer=referer,
|
||||
)
|
||||
|
||||
target = self._video_target_path(final_url, save_path)
|
||||
if target.exists() and target.stat().st_size > 1024:
|
||||
return target
|
||||
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size <= 1024:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
return target
|
||||
|
||||
async def _download_hls_playlist(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
playlist_url: str,
|
||||
playlist_text: str,
|
||||
save_path: Path,
|
||||
referer: str | None,
|
||||
) -> Path | None:
|
||||
selected_url, selected_text = await self._resolve_hls_playlist(
|
||||
client,
|
||||
playlist_url=playlist_url,
|
||||
playlist_text=playlist_text,
|
||||
referer=referer,
|
||||
)
|
||||
if not selected_text:
|
||||
return None
|
||||
|
||||
segment_urls, init_url = self._parse_hls_segments(selected_url, selected_text)
|
||||
if not segment_urls:
|
||||
return None
|
||||
|
||||
target = self._video_target_path(selected_url, save_path, segment_urls=segment_urls, init_url=init_url)
|
||||
if target.exists() and target.stat().st_size > 1024:
|
||||
return target
|
||||
|
||||
with target.open("wb") as fh:
|
||||
if init_url:
|
||||
init_resp = await client.get(init_url, headers=self._media_request_headers(init_url, referer=referer))
|
||||
if init_resp.status_code != 200:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
fh.write(init_resp.content)
|
||||
|
||||
for segment_url in segment_urls:
|
||||
seg_resp = await client.get(
|
||||
segment_url,
|
||||
headers=self._media_request_headers(segment_url, referer=referer),
|
||||
)
|
||||
if seg_resp.status_code != 200:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
fh.write(seg_resp.content)
|
||||
|
||||
if target.stat().st_size <= 1024:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
return target
|
||||
|
||||
async def _resolve_hls_playlist(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
playlist_url: str,
|
||||
playlist_text: str,
|
||||
referer: str | None,
|
||||
) -> tuple[str, str]:
|
||||
if "#EXT-X-STREAM-INF" not in playlist_text:
|
||||
return playlist_url, playlist_text
|
||||
|
||||
variants = self._parse_hls_variants(playlist_url, playlist_text)
|
||||
for variant_url in variants:
|
||||
resp = await client.get(variant_url, headers=self._media_request_headers(variant_url, referer=referer))
|
||||
if resp.status_code == 200:
|
||||
return str(resp.url), resp.text
|
||||
return playlist_url, playlist_text
|
||||
|
||||
def _parse_hls_variants(self, playlist_url: str, playlist_text: str) -> list[str]:
|
||||
variants: list[tuple[int, str]] = []
|
||||
pending_bandwidth = 0
|
||||
|
||||
for raw_line in playlist_text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#EXT-X-STREAM-INF"):
|
||||
match = re.search(r"BANDWIDTH=(\d+)", line)
|
||||
pending_bandwidth = int(match.group(1)) if match else 0
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
variants.append((pending_bandwidth, urljoin(playlist_url, line)))
|
||||
pending_bandwidth = 0
|
||||
|
||||
variants.sort(key=lambda item: item[0], reverse=True)
|
||||
return [url for _, url in variants]
|
||||
|
||||
def _parse_hls_segments(self, playlist_url: str, playlist_text: str) -> tuple[list[str], str | None]:
|
||||
segments: list[str] = []
|
||||
init_url: str | None = None
|
||||
|
||||
for raw_line in playlist_text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#EXT-X-MAP:"):
|
||||
match = re.search(r'URI="([^"]+)"', line)
|
||||
if match:
|
||||
init_url = urljoin(playlist_url, match.group(1))
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
segments.append(urljoin(playlist_url, line))
|
||||
|
||||
return segments, init_url
|
||||
|
||||
def _media_request_headers(self, url: str, *, referer: str | None = None) -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": self._ua(),
|
||||
"Accept": "*/*",
|
||||
"Referer": referer or "https://x.com/",
|
||||
"Origin": "https://x.com",
|
||||
}
|
||||
cookie_header = self._cookie_header(url)
|
||||
if cookie_header:
|
||||
headers["Cookie"] = cookie_header
|
||||
csrf_token = self._cookie_value("ct0")
|
||||
if csrf_token:
|
||||
headers["x-csrf-token"] = csrf_token
|
||||
return headers
|
||||
|
||||
def _cookie_header(self, url: str) -> str:
|
||||
pairs: list[str] = []
|
||||
for item in self._load_cookie_items():
|
||||
name = str(item.get("name") or "").strip()
|
||||
value = str(item.get("value") or "")
|
||||
if name:
|
||||
pairs.append(f"{name}={value}")
|
||||
return "; ".join(pairs)
|
||||
|
||||
def _cookie_value(self, name: str) -> str:
|
||||
for item in self._load_cookie_items():
|
||||
if str(item.get("name") or "") == name:
|
||||
return str(item.get("value") or "")
|
||||
return ""
|
||||
|
||||
def _load_cookie_items(self) -> list[dict[str, Any]]:
|
||||
cookie_path = Path(getattr(self.config, "cookie_file", "") or "")
|
||||
if not cookie_path.is_file():
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(cookie_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return []
|
||||
return payload if isinstance(payload, list) else []
|
||||
|
||||
async def _add_cookies_tolerant(self, cookies: list[dict[str, Any]]) -> int:
|
||||
assert self.context is not None
|
||||
try:
|
||||
await self.context.add_cookies(cookies)
|
||||
return len(cookies)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
loaded = 0
|
||||
for cookie in cookies:
|
||||
try:
|
||||
await self.context.add_cookies([cookie])
|
||||
loaded += 1
|
||||
except Exception:
|
||||
continue
|
||||
return loaded
|
||||
|
||||
@staticmethod
|
||||
def _prepare_cookie_items(cookies: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
prepared_by_key: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
for cookie in cookies:
|
||||
name = str(cookie.get("name") or "").strip()
|
||||
domain = str(cookie.get("domain") or "").strip()
|
||||
if not name or not domain:
|
||||
continue
|
||||
|
||||
item: dict[str, Any] = {
|
||||
"name": name,
|
||||
"value": str(cookie.get("value") or ""),
|
||||
"domain": domain,
|
||||
"path": str(cookie.get("path") or "/"),
|
||||
"secure": bool(cookie.get("secure", False)),
|
||||
"httpOnly": bool(cookie.get("httpOnly", False)),
|
||||
}
|
||||
|
||||
expires = cookie.get("expires")
|
||||
if expires is not None:
|
||||
try:
|
||||
expires_value = int(float(expires))
|
||||
if expires_value > 0:
|
||||
item["expires"] = expires_value
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
same_site = str(cookie.get("sameSite") or "").strip()
|
||||
if same_site in {"Strict", "Lax", "None"}:
|
||||
if same_site == "None" and not item["secure"]:
|
||||
item["sameSite"] = "Lax"
|
||||
else:
|
||||
item["sameSite"] = same_site
|
||||
|
||||
key = (domain.lower(), str(item["path"]), name)
|
||||
prepared_by_key[key] = item
|
||||
return list(prepared_by_key.values())
|
||||
|
||||
def _video_target_path(
|
||||
self,
|
||||
url: str,
|
||||
save_path: Path,
|
||||
*,
|
||||
segment_urls: list[str] | None = None,
|
||||
init_url: str | None = None,
|
||||
) -> Path:
|
||||
parsed = urlparse(url)
|
||||
name = Path(parsed.path).name or "x_video"
|
||||
|
||||
if name.endswith(".m3u8") or not Path(name).suffix:
|
||||
if segment_urls:
|
||||
seg_name = Path(urlparse(segment_urls[0]).path).name
|
||||
if seg_name:
|
||||
name = seg_name
|
||||
if init_url:
|
||||
init_name = Path(urlparse(init_url).path).name
|
||||
if init_name:
|
||||
name = init_name
|
||||
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix in {".m3u8", ".m4s", ".mpd", ""}:
|
||||
name = f"{Path(name).stem or 'x_video'}.mp4"
|
||||
return save_path / name
|
||||
|
||||
def _looks_like_hls(self, url: str, content_type: str, body: str) -> bool:
|
||||
low_url = url.lower()
|
||||
if ".m3u8" in low_url:
|
||||
return True
|
||||
if "mpegurl" in content_type or "vnd.apple.mpegurl" in content_type:
|
||||
return True
|
||||
return "#EXTM3U" in body[:256]
|
||||
|
||||
def _attach_response_collector(self, page: Page) -> None:
|
||||
page_id = id(page)
|
||||
bucket: list[dict[str, Any]] = []
|
||||
self._response_store[page_id] = bucket
|
||||
self._response_tasks[page_id] = set()
|
||||
|
||||
def handle_response(response: Any) -> None:
|
||||
task = asyncio.create_task(self._capture_response(page, response))
|
||||
task_bucket = self._response_tasks.setdefault(page_id, set())
|
||||
task_bucket.add(task)
|
||||
|
||||
def cleanup(done_task: asyncio.Task[Any]) -> None:
|
||||
current_bucket = self._response_tasks.get(page_id)
|
||||
if current_bucket is not None:
|
||||
current_bucket.discard(done_task)
|
||||
|
||||
task.add_done_callback(cleanup)
|
||||
|
||||
page.on("response", handle_response)
|
||||
|
||||
async def _capture_response(self, page: Page, response: Any) -> None:
|
||||
url = str(getattr(response, "url", "") or "")
|
||||
if not self._should_capture_response(url):
|
||||
return
|
||||
|
||||
try:
|
||||
headers = await response.all_headers()
|
||||
except Exception:
|
||||
headers = {}
|
||||
content_type = str(headers.get("content-type", "")).lower()
|
||||
if "json" not in content_type and "javascript" not in content_type and "text/plain" not in content_type:
|
||||
return
|
||||
|
||||
try:
|
||||
text = await response.text()
|
||||
except Exception:
|
||||
return
|
||||
if not text or len(text) < 2:
|
||||
return
|
||||
|
||||
parsed_json: Any = None
|
||||
try:
|
||||
parsed_json = json.loads(text)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
self._response_store.setdefault(id(page), []).append(
|
||||
{
|
||||
"url": url,
|
||||
"status": getattr(response, "status", None),
|
||||
"json": parsed_json,
|
||||
}
|
||||
)
|
||||
|
||||
def _should_capture_response(self, url: str) -> bool:
|
||||
low = url.lower()
|
||||
return any(
|
||||
token in low
|
||||
for token in (
|
||||
"/i/api/graphql/",
|
||||
"/graphql/",
|
||||
"tweetdetail",
|
||||
"tweetresultbyrestid",
|
||||
"conversation",
|
||||
"adaptive.json",
|
||||
)
|
||||
)
|
||||
|
||||
def _find_media_entities(self, payload: Any) -> list[dict[str, Any]]:
|
||||
media_entities: list[dict[str, Any]] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
if "media_url_https" in node or "video_info" in node:
|
||||
media_entities.append(node)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
return
|
||||
if isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
return media_entities
|
||||
|
||||
def _pick_best_video_variant(self, video_info: dict[str, Any]) -> str | None:
|
||||
variants = video_info.get("variants")
|
||||
if not isinstance(variants, list):
|
||||
return None
|
||||
|
||||
best_mp4: tuple[int, str] | None = None
|
||||
best_hls: str | None = None
|
||||
for item in variants:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
url = item.get("url")
|
||||
if not isinstance(url, str) or not url:
|
||||
continue
|
||||
low_url = url.lower()
|
||||
content_type = str(item.get("content_type") or "").lower()
|
||||
try:
|
||||
bitrate = int(item.get("bitrate") or 0)
|
||||
except (TypeError, ValueError):
|
||||
bitrate = 0
|
||||
if "mp4" in content_type or ".mp4" in low_url:
|
||||
if best_mp4 is None or bitrate > best_mp4[0]:
|
||||
best_mp4 = (bitrate, url)
|
||||
elif "mpegurl" in content_type or ".m3u8" in low_url:
|
||||
best_hls = url
|
||||
|
||||
if best_mp4 is not None:
|
||||
return best_mp4[1]
|
||||
return best_hls
|
||||
|
||||
def _extract_image_urls_from_html(self, html: str) -> list[str]:
|
||||
if not html:
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
urls: list[str] = []
|
||||
patterns = (
|
||||
r"https?://pbs\.twimg\.com/media/[^\s\"'<>\\]+",
|
||||
r"https:\\/\\/pbs\.twimg\.com\\/media\\/[^\s\"'<>]+",
|
||||
)
|
||||
for pattern in patterns:
|
||||
for match in re.findall(pattern, html):
|
||||
normalized = self._normalize_media_url(match)
|
||||
if "pbs.twimg.com/media" not in normalized:
|
||||
continue
|
||||
orig = self._original_tw_image_url(normalized)
|
||||
if orig not in seen:
|
||||
seen.add(orig)
|
||||
urls.append(orig)
|
||||
return urls
|
||||
|
||||
def _ua(self) -> str:
|
||||
if default_profile:
|
||||
return default_profile("x").user_agent
|
||||
return (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _original_tw_image_url(url: str) -> str:
|
||||
if "pbs.twimg.com/media" not in url:
|
||||
return url
|
||||
base = url.split("?")[0]
|
||||
return f"{base}?format=jpg&name=orig" if "format=" not in url else re.sub(
|
||||
r"name=\w+",
|
||||
"name=orig",
|
||||
url,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_media_url(url: str) -> str:
|
||||
value = url.strip().replace("\\/", "/")
|
||||
if value.startswith("//"):
|
||||
value = f"https:{value}"
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _is_video_url(url: str) -> bool:
|
||||
low = url.lower()
|
||||
return any(
|
||||
token in low
|
||||
for token in (
|
||||
".mp4",
|
||||
".m3u8",
|
||||
"video.twimg.com",
|
||||
"/amplify_video/",
|
||||
"/ext_tw_video/",
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,504 @@
|
||||
"""
|
||||
X (Twitter) plugin.
|
||||
|
||||
Uses Playwright for rendering and httpx for media downloads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import queue
|
||||
import random
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .playwright_client import XPlaywrightClient
|
||||
from .x_login import run_interactive_login as x_login_flow
|
||||
from .x_urls import (
|
||||
extract_search_query,
|
||||
extract_tweet_id,
|
||||
extract_username_and_kind,
|
||||
is_search_url,
|
||||
is_status_url,
|
||||
normalize_url,
|
||||
)
|
||||
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
|
||||
|
||||
|
||||
@dataclass
|
||||
class XPluginConfig:
|
||||
cookie_file: str = ""
|
||||
profile_dir: str = ""
|
||||
headless: bool = False
|
||||
safety_mode: bool = True
|
||||
wait_timeout: int = 45000
|
||||
timeline_tweet_limit: int = 8
|
||||
media_scroll_rounds: int = 5
|
||||
max_media_items: int = 40
|
||||
parse_cooldown_seconds: float = 60.0
|
||||
min_action_delay: float = 4.0
|
||||
max_action_delay: float = 11.0
|
||||
download_delay_min: float = 2.0
|
||||
download_delay_max: float = 5.0
|
||||
batch_pause_min: float = 12.0
|
||||
batch_pause_max: float = 28.0
|
||||
batch_pause_after_min: int = 2
|
||||
batch_pause_after_max: int = 4
|
||||
max_parse_per_hour: int = 24
|
||||
max_download_files_per_hour: int = 60
|
||||
max_consecutive_failures: int = 3
|
||||
failure_cooldown_seconds: float = 1800.0
|
||||
sync_parse_timeout_seconds: float = 120.0
|
||||
|
||||
|
||||
class XPlugin(BasePlugin):
|
||||
name = "X"
|
||||
domain = "twitter.com"
|
||||
version = "1.0.0"
|
||||
|
||||
_HANDLE_PATTERNS = (
|
||||
re.compile(r"(?:twitter|x)\.com/\w+/status/\d+", re.I),
|
||||
re.compile(r"(?:twitter|x)\.com/[^/]+/media\b", re.I),
|
||||
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/?$", re.I),
|
||||
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/media/?$", re.I),
|
||||
re.compile(r"(?:twitter|x)\.com/search\?", re.I),
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.config = XPluginConfig(
|
||||
cookie_file=str(runtime_path("x_cookies.json")),
|
||||
profile_dir=str(runtime_path("x_profile")),
|
||||
headless=False,
|
||||
)
|
||||
self._cache_file = runtime_path("x_download_cache.json")
|
||||
self._playwright_lock = threading.Lock()
|
||||
self._last_parse_at = 0.0
|
||||
self._parse_timestamps: list[float] = []
|
||||
self._download_timestamps: list[float] = []
|
||||
self._consecutive_failures = 0
|
||||
self._suspended_until = 0.0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
text = url.strip()
|
||||
if not text:
|
||||
return False
|
||||
candidate = text if "://" in text else f"https://{text.lstrip('/')}"
|
||||
try:
|
||||
hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".")
|
||||
except ValueError:
|
||||
return False
|
||||
if hostname not in {"x.com", "www.x.com", "twitter.com", "www.twitter.com", "mobile.twitter.com"}:
|
||||
return False
|
||||
return any(pattern.search(text) for pattern in self._HANDLE_PATTERNS)
|
||||
|
||||
def validate(self) -> bool:
|
||||
try:
|
||||
import playwright # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
}
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
self._enforce_account_safety("parse", units=1)
|
||||
try:
|
||||
with self._playwright_lock:
|
||||
self._respect_parse_cooldown()
|
||||
try:
|
||||
result = asyncio.run(self._parse_async(normalize_url(url)))
|
||||
self._record_success()
|
||||
self._record_usage(self._parse_timestamps, 1)
|
||||
return result
|
||||
finally:
|
||||
self._last_parse_at = time.monotonic()
|
||||
except PluginParseError:
|
||||
self._record_failure()
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._record_failure()
|
||||
raise PluginParseError(f"X 解析失败: {exc}") from exc
|
||||
|
||||
def parse_for_sync(self, url: str) -> list[Resource]:
|
||||
self._enforce_account_safety("parse", units=1)
|
||||
timeout = max(30.0, float(self.config.sync_parse_timeout_seconds))
|
||||
try:
|
||||
with self._playwright_lock:
|
||||
result = asyncio.run(asyncio.wait_for(self._parse_async(normalize_url(url)), timeout=timeout))
|
||||
self._record_success()
|
||||
self._record_usage(self._parse_timestamps, 1)
|
||||
self._last_parse_at = time.monotonic()
|
||||
return result
|
||||
except asyncio.TimeoutError as exc:
|
||||
self._record_failure()
|
||||
raise PluginParseError(f"X 同步解析超时({int(timeout)} 秒),请稍后重试或检查登录状态。") from exc
|
||||
except PluginParseError:
|
||||
self._record_failure()
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._record_failure()
|
||||
raise PluginParseError(f"X 同步解析失败: {exc}") from exc
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
requested = max(1, len(resource.files))
|
||||
self._enforce_account_safety("download", units=requested)
|
||||
try:
|
||||
with self._playwright_lock:
|
||||
result = asyncio.run(self._download_async(resource, save_path))
|
||||
self._record_success()
|
||||
self._record_usage(self._download_timestamps, max(1, len(result) or requested))
|
||||
return result
|
||||
except Exception as exc:
|
||||
self._record_failure()
|
||||
raise PluginParseError(f"X 下载失败: {exc}") from exc
|
||||
|
||||
def reset_playwright_session(self) -> None:
|
||||
return None
|
||||
|
||||
def _respect_parse_cooldown(self) -> None:
|
||||
elapsed = time.monotonic() - self._last_parse_at
|
||||
remain = float(self.config.parse_cooldown_seconds) - elapsed
|
||||
if 0 < remain < self.config.parse_cooldown_seconds:
|
||||
time.sleep(remain)
|
||||
|
||||
def _enforce_account_safety(self, action: str, *, units: int) -> None:
|
||||
if not self.config.safety_mode:
|
||||
return
|
||||
|
||||
now = time.monotonic()
|
||||
if self._suspended_until > now:
|
||||
remaining = int(self._suspended_until - now)
|
||||
minutes = max(1, remaining // 60)
|
||||
raise PluginParseError(f"X 账号安全模式已暂停操作,请约 {minutes} 分钟后再试。")
|
||||
|
||||
self._trim_usage(now)
|
||||
if action == "parse":
|
||||
if len(self._parse_timestamps) + units > self.config.max_parse_per_hour:
|
||||
raise PluginParseError("X 账号安全模式已触发每小时解析上限,请稍后再试。")
|
||||
return
|
||||
|
||||
if action == "download":
|
||||
if len(self._download_timestamps) + units > self.config.max_download_files_per_hour:
|
||||
raise PluginParseError("X 账号安全模式已触发每小时下载上限,请稍后再试。")
|
||||
|
||||
def _trim_usage(self, now: float | None = None) -> None:
|
||||
current = now if now is not None else time.monotonic()
|
||||
window = 3600.0
|
||||
self._parse_timestamps = [item for item in self._parse_timestamps if current - item < window]
|
||||
self._download_timestamps = [item for item in self._download_timestamps if current - item < window]
|
||||
|
||||
def _record_usage(self, bucket: list[float], units: int) -> None:
|
||||
now = time.monotonic()
|
||||
self._trim_usage(now)
|
||||
for _ in range(max(1, units)):
|
||||
bucket.append(now)
|
||||
|
||||
def _record_success(self) -> None:
|
||||
self._consecutive_failures = 0
|
||||
|
||||
def _record_failure(self) -> None:
|
||||
if not self.config.safety_mode:
|
||||
return
|
||||
self._consecutive_failures += 1
|
||||
if self._consecutive_failures >= self.config.max_consecutive_failures:
|
||||
self._suspended_until = time.monotonic() + float(self.config.failure_cooldown_seconds)
|
||||
self._consecutive_failures = 0
|
||||
|
||||
async def _parse_async(self, url: str) -> list[Resource]:
|
||||
if not self.validate():
|
||||
raise PluginParseError("未安装 Playwright。请先执行: pip install playwright && playwright install chromium")
|
||||
|
||||
client = XPlaywrightClient(self.config)
|
||||
await client.start()
|
||||
try:
|
||||
if is_status_url(url):
|
||||
return await self._parse_single_tweet(client, url)
|
||||
if is_search_url(url):
|
||||
return await self._parse_search_media(client, url)
|
||||
|
||||
user, kind = extract_username_and_kind(url)
|
||||
if user and kind == "media":
|
||||
return await self._parse_user_media(client, url, user)
|
||||
if user and kind == "profile":
|
||||
return await self._parse_user_timeline(client, url, user)
|
||||
|
||||
raise PluginParseError(f"无法识别的 X 链接: {url}")
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
async def _parse_single_tweet(self, client: XPlaywrightClient, url: str) -> list[Resource]:
|
||||
page = await client.new_page_goto(url)
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
await client.wait_for_main_content(page)
|
||||
await client.wait_for_media(page)
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=5000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
tweet_id = extract_tweet_id(url) or ""
|
||||
author = await client.get_author_info(page)
|
||||
media_records = await client.extract_media_records_from_api_responses(page)
|
||||
media_urls = await client.extract_media_urls(page)
|
||||
if not media_urls:
|
||||
try:
|
||||
await page.mouse.wheel(0, 500)
|
||||
await asyncio.sleep(1.2)
|
||||
except Exception:
|
||||
pass
|
||||
media_urls = await client.extract_media_urls(page)
|
||||
if not media_records and not media_urls:
|
||||
media_records = await client.extract_media_records_from_api_responses(page)
|
||||
if not media_urls:
|
||||
media_urls = [item["url"] for item in media_records] or await client.extract_media_urls_from_api_responses(page)
|
||||
|
||||
content = await client.get_tweet_text(page)
|
||||
created = await client.get_tweet_date(page)
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
if not media_records and not media_urls:
|
||||
raise PluginParseError("未在该推文解析到媒体,可能需要重新登录,或该页面结构已变化。")
|
||||
|
||||
resources: list[Resource] = []
|
||||
if media_records:
|
||||
for index, item in enumerate(media_records):
|
||||
file_url = item["url"]
|
||||
thumb = item.get("thumbnail") or file_url
|
||||
resources.append(
|
||||
Resource(
|
||||
id=f"{tweet_id}_{index}",
|
||||
url=url,
|
||||
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
|
||||
author=author.get("name", ""),
|
||||
author_id=author.get("username", ""),
|
||||
files=[{"url": file_url, "type": item.get("type", self._media_type(file_url)), "referer": url}],
|
||||
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
|
||||
thumbnail=thumb,
|
||||
created_at=created or None,
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
for index, media_url in enumerate(media_urls):
|
||||
resources.append(
|
||||
Resource(
|
||||
id=f"{tweet_id}_{index}",
|
||||
url=url,
|
||||
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
|
||||
author=author.get("name", ""),
|
||||
author_id=author.get("username", ""),
|
||||
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
|
||||
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
|
||||
thumbnail=media_urls[0] if index == 0 else None,
|
||||
created_at=created or None,
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
async def _parse_search_media(self, client: XPlaywrightClient, url: str) -> list[Resource]:
|
||||
page = await client.new_page_goto(url)
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
media_urls = await client.scroll_and_collect_media(
|
||||
page,
|
||||
max_scrolls=self.config.media_scroll_rounds,
|
||||
max_items=self.config.max_media_items,
|
||||
)
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
query = extract_search_query(url) or "search"
|
||||
resources: list[Resource] = []
|
||||
for index, media_url in enumerate(media_urls):
|
||||
resources.append(
|
||||
Resource(
|
||||
id=f"search_{abs(hash(query))}_{index}",
|
||||
url=url,
|
||||
title=f"X search - {query}",
|
||||
author=query,
|
||||
author_id="search",
|
||||
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
|
||||
metadata={"source": "search_media", "query": query},
|
||||
thumbnail=media_url,
|
||||
created_at=None,
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
async def _parse_user_media(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
|
||||
page = await client.new_page_goto(url)
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
info = await client.get_user_display_name(page, username)
|
||||
media_urls = await client.scroll_and_collect_media(
|
||||
page,
|
||||
max_scrolls=self.config.media_scroll_rounds,
|
||||
max_items=self.config.max_media_items,
|
||||
)
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
resources: list[Resource] = []
|
||||
for index, media_url in enumerate(media_urls):
|
||||
resources.append(
|
||||
Resource(
|
||||
id=f"{username}_media_{index}",
|
||||
url=url,
|
||||
title=f"{info.get('name', username)} - media",
|
||||
author=info.get("name", username),
|
||||
author_id=username,
|
||||
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
|
||||
metadata={"source": "media_tab", "username": username},
|
||||
thumbnail=media_url,
|
||||
created_at=None,
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
async def _parse_user_timeline(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
|
||||
page = await client.new_page_goto(url)
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
tweet_urls = await client.scroll_and_collect_tweet_urls(
|
||||
page,
|
||||
limit=self.config.timeline_tweet_limit,
|
||||
)
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
all_results: list[Resource] = []
|
||||
for tweet_url in tweet_urls:
|
||||
try:
|
||||
all_results.extend(await self._parse_single_tweet(client, tweet_url))
|
||||
except Exception:
|
||||
continue
|
||||
return all_results
|
||||
|
||||
async def _download_async(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
client = XPlaywrightClient(self.config)
|
||||
results: list[Path] = []
|
||||
subdir = resource.metadata.get("save_subdir")
|
||||
target_base = save_path / subdir if subdir else save_path
|
||||
cache = self._load_download_cache()
|
||||
batch_pause_after = random.randint(self.config.batch_pause_after_min, self.config.batch_pause_after_max)
|
||||
|
||||
try:
|
||||
for index, item in enumerate(resource.files, start=1):
|
||||
file_url = item.get("url")
|
||||
if not file_url:
|
||||
continue
|
||||
|
||||
cached = cache.get(file_url)
|
||||
if cached and Path(cached).is_file():
|
||||
results.append(Path(cached))
|
||||
continue
|
||||
|
||||
kind = item.get("type", "image")
|
||||
referer = item.get("referer") or resource.url
|
||||
if kind == "video":
|
||||
saved = await client.download_video(file_url, target_base, referer=referer)
|
||||
else:
|
||||
saved = await client.download_image(file_url, target_base, referer=referer)
|
||||
|
||||
if saved is None:
|
||||
continue
|
||||
|
||||
cache[file_url] = str(saved)
|
||||
results.append(saved)
|
||||
await asyncio.sleep(
|
||||
random.uniform(self.config.download_delay_min, self.config.download_delay_max)
|
||||
* random.uniform(0.8, 1.5)
|
||||
)
|
||||
|
||||
if index % batch_pause_after == 0:
|
||||
await asyncio.sleep(random.uniform(self.config.batch_pause_min, self.config.batch_pause_max))
|
||||
batch_pause_after = random.randint(
|
||||
self.config.batch_pause_after_min,
|
||||
self.config.batch_pause_after_max,
|
||||
)
|
||||
|
||||
self._save_download_cache(cache)
|
||||
return results
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
def _load_download_cache(self) -> dict[str, str]:
|
||||
try:
|
||||
if self._cache_file.is_file():
|
||||
payload = json.loads(self._cache_file.read_text(encoding="utf-8"))
|
||||
if isinstance(payload, dict):
|
||||
return {str(key): str(value) for key, value in payload.items()}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
def _save_download_cache(self, payload: dict[str, str]) -> None:
|
||||
self._cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._cache_file.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
def run_interactive_login(
|
||||
self,
|
||||
*,
|
||||
proceed: threading.Event | None = None,
|
||||
ready_signal: queue.Queue[bool] | None = None,
|
||||
) -> bool:
|
||||
path = Path(self.config.profile_dir)
|
||||
try:
|
||||
with self._playwright_lock:
|
||||
return asyncio.run(x_login_flow(path, proceed=proceed, ready_signal=ready_signal))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def set_headless(self, headless: bool) -> None:
|
||||
self.config.headless = headless
|
||||
|
||||
@staticmethod
|
||||
def _media_type(url: str) -> str:
|
||||
low = url.lower()
|
||||
if (
|
||||
"video" in low
|
||||
or ".mp4" in low
|
||||
or ".m3u8" in low
|
||||
or "video.twimg.com" in low
|
||||
or "/amplify_video/" in low
|
||||
or "/ext_tw_video/" in low
|
||||
):
|
||||
return "video"
|
||||
return "image"
|
||||
|
||||
def get_config_schema(self) -> dict[str, Any]:
|
||||
return {
|
||||
"headless": {"type": "bool", "label": "无头模式", "default": False},
|
||||
"safety_mode": {"type": "bool", "label": "账号安全模式", "default": True},
|
||||
"timeline_tweet_limit": {"type": "int", "label": "时间线采样条数", "default": 8, "min": 1, "max": 30},
|
||||
"media_scroll_rounds": {"type": "int", "label": "媒体页滚动轮数", "default": 5, "min": 1, "max": 12},
|
||||
"max_media_items": {"type": "int", "label": "单次最多媒体数", "default": 40, "min": 1, "max": 120},
|
||||
"parse_cooldown_seconds": {"type": "float", "label": "解析冷却秒数", "default": 60.0, "min": 10, "max": 600},
|
||||
"min_action_delay": {"type": "float", "label": "最小操作延迟秒数", "default": 4.0, "min": 1, "max": 60},
|
||||
"max_action_delay": {"type": "float", "label": "最大操作延迟秒数", "default": 11.0, "min": 2, "max": 90},
|
||||
"download_delay_min": {"type": "float", "label": "下载后最小停顿秒数", "default": 2.0, "min": 0, "max": 60},
|
||||
"download_delay_max": {"type": "float", "label": "下载后最大停顿秒数", "default": 5.0, "min": 0, "max": 120},
|
||||
"max_parse_per_hour": {"type": "int", "label": "每小时最大解析次数", "default": 24, "min": 1, "max": 120},
|
||||
"max_download_files_per_hour": {"type": "int", "label": "每小时最大下载文件数", "default": 60, "min": 1, "max": 300},
|
||||
"max_consecutive_failures": {"type": "int", "label": "连续失败暂停阈值", "default": 3, "min": 1, "max": 10},
|
||||
"failure_cooldown_seconds": {"type": "float", "label": "连续失败暂停秒数", "default": 1800.0, "min": 60, "max": 86400},
|
||||
}
|
||||
|
||||
|
||||
plugin_class = XPlugin
|
||||
@@ -0,0 +1,8 @@
|
||||
"""DOM selectors for X pages used by Sakura desktop sync."""
|
||||
|
||||
PROFILE_LINK = 'a[data-testid="AppTabBar_Profile_Link"]'
|
||||
USER_CELL = '[data-testid="UserCell"]'
|
||||
USER_NAME = '[data-testid="UserName"]'
|
||||
USER_AVATAR = 'img[src*="profile_images"]'
|
||||
USER_LINK = 'a[href^="/"][role="link"]'
|
||||
FOLLOWING_URL_SUFFIX = "/following"
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Install Python deps and Chromium for the X plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def main() -> None:
|
||||
deps = ["playwright", "httpx"]
|
||||
for dep in deps:
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", dep])
|
||||
subprocess.check_call([sys.executable, "-m", "playwright", "install", "chromium"])
|
||||
print("Done. Restart the app and reload plugins.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Interactive X login via persistent Chromium profile."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import shutil
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from .browser_opts import launch_x_persistent_context
|
||||
from pixiv_app.services.x_login_flow import goto_x_login_entry
|
||||
|
||||
|
||||
async def run_interactive_login(
|
||||
profile_dir: Path,
|
||||
*,
|
||||
proceed: threading.Event | None = None,
|
||||
ready_signal: queue.Queue[bool] | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Open visible Chromium with persistent ``profile_dir``.
|
||||
|
||||
Optionally notify GUI via ``ready_signal`` after the login page loads, then block until
|
||||
``proceed`` is set (e.g. user clicked OK after logging in). If ``proceed`` is None,
|
||||
falls back to terminal ``input()``.
|
||||
"""
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
playwright = await async_playwright().start()
|
||||
context = None
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
context = await launch_x_persistent_context(playwright, profile_dir, headless=False)
|
||||
page = context.pages[0] if context.pages else await context.new_page()
|
||||
await goto_x_login_entry(page, timeout=60_000)
|
||||
|
||||
if ready_signal is not None:
|
||||
ready_signal.put(True)
|
||||
|
||||
if proceed is not None:
|
||||
await loop.run_in_executor(None, proceed.wait)
|
||||
else:
|
||||
await loop.run_in_executor(
|
||||
None,
|
||||
lambda: input("\n在浏览器中完成 X 登录后,回到控制台按 Enter...\n"),
|
||||
)
|
||||
return True
|
||||
finally:
|
||||
if context is not None:
|
||||
await context.close()
|
||||
await playwright.stop()
|
||||
|
||||
|
||||
def clear_profile(profile_dir: Path) -> None:
|
||||
if profile_dir.exists():
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -0,0 +1,57 @@
|
||||
"""URL helpers for X / Twitter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
STATUS_RE = re.compile(
|
||||
r"(?:https?://)?(?:www\.)?(?:twitter|x)\.com/[^/]+/status/(\d+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
PROFILE_OR_MEDIA_RE = re.compile(
|
||||
r"(?:https?://)?(?:www\.)?(?:twitter|x)\.com/([A-Za-z0-9_]{1,15})(?:/(media))?/?$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
u = url.strip()
|
||||
if not u.startswith("http"):
|
||||
u = "https://" + u
|
||||
return u
|
||||
|
||||
|
||||
def extract_tweet_id(url: str) -> str | None:
|
||||
m = STATUS_RE.search(url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def extract_username_and_kind(url: str) -> tuple[str | None, str | None]:
|
||||
u = normalize_url(url)
|
||||
m = PROFILE_OR_MEDIA_RE.match(u)
|
||||
if not m:
|
||||
return None, None
|
||||
user = m.group(1)
|
||||
media_flag = m.group(2)
|
||||
if user.lower() in {"home", "explore", "settings", "messages", "notifications", "i", "intent", "search"}:
|
||||
return None, None
|
||||
if media_flag:
|
||||
return user, "media"
|
||||
return user, "profile"
|
||||
|
||||
|
||||
def is_status_url(url: str) -> bool:
|
||||
return extract_tweet_id(url) is not None
|
||||
|
||||
|
||||
def is_search_url(url: str) -> bool:
|
||||
parsed = urlparse(normalize_url(url))
|
||||
host = parsed.netloc.lower()
|
||||
return host in {"x.com", "www.x.com", "twitter.com", "www.twitter.com"} and parsed.path.rstrip("/") == "/search"
|
||||
|
||||
|
||||
def extract_search_query(url: str) -> str:
|
||||
parsed = urlparse(normalize_url(url))
|
||||
values = parse_qs(parsed.query).get("q") or []
|
||||
return values[0] if values else ""
|
||||
@@ -0,0 +1,2 @@
|
||||
# plugins/xiaohongshu/__init__.py
|
||||
"""小红书 (Xiaohongshu) 爬虫插件 - 换IP + 指纹模拟版本"""
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user