first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""Bundled AO3 (Archive of Our Own) plugin package."""
+11
View File
@@ -0,0 +1,11 @@
{
"name": "AO3",
"version": "1.0.0",
"domain": "archiveofourown.org",
"author": "SakuraHub",
"description": "AO3 fanfiction archive novel plugin. Supports work URL parsing, metadata extraction, EPUB/HTML download, cookie-based auth, and update checking.",
"requires_auth": false,
"min_platform_version": "1.0.0",
"tags": ["novel", "fanfiction", "ao3"],
"dependencies": []
}
File diff suppressed because it is too large. Load diff
+11
View File
@@ -0,0 +1,11 @@
{
"name": "笔趣阁",
"version": "3.0.0",
"domain": "www.bqg551.cc",
"author": "Sakura Downloader",
"description": "基于 bqg551 阅读站公开 API 的小说搜索、目录解析、章节抓取与 TXT 导出。",
"requires_auth": false,
"tags": ["novel", "bqg551", "biquge", "bookshelf"],
"min_platform_version": "1.0.0",
"dependencies": []
}
File diff suppressed because it is too large. Load diff
+11
View File
@@ -0,0 +1,11 @@
{
"name": "长佩文学",
"version": "1.0.0",
"domain": "www.gongzicp.com",
"author": "Sakura Downloader",
"description": "长佩文学(gongzicp.com)小说搜索、目录解析、章节抓取与 TXT 导出。",
"requires_auth": true,
"tags": ["novel", "gongzicp", "changpei", "bookshelf"],
"min_platform_version": "1.0.0",
"dependencies": []
}
+514
View File
@@ -0,0 +1,514 @@
"""
Changpei (长佩文学 / gongzicp.com) novel plugin.
Uses the public webapi endpoints on https://www.gongzicp.com/webapi/.
Chapter bodies are AES-CBC encrypted; decryption matches the site's CryptoJS helper.
"""
from __future__ import annotations
import base64
import re
from dataclasses import dataclass
from html import unescape
from pathlib import Path
from typing import Any
from urllib.parse import urljoin, urlparse
import requests
try:
from cryptography.hazmat.primitives import padding as crypto_padding
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
CRYPTOGRAPHY_AVAILABLE = True
except Exception:
crypto_padding = None
Cipher = None
algorithms = None
modes = None
CRYPTOGRAPHY_AVAILABLE = False
from pixiv_app.core.downloader import sanitize_filename, save_text_file
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
BASE_URL = "https://www.gongzicp.com"
WEBAPI_BASE = f"{BASE_URL}/webapi"
USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
)
_NOVEL_PATH_RE = re.compile(r"/novel-(?P<nid>\d+)\.html", re.I)
_READ_PATH_RE = re.compile(r"/read-(?P<nid>\d+)-(?P<cid>\d+)\.html", re.I)
_SINGLE_READ_RE = re.compile(r"/read-(?P<cid>\d+)\.html", re.I)
@dataclass
class BookCandidate:
title: str
url: str
author: str = ""
latest_chapter: str = ""
@dataclass
class TargetIds:
novel_id: str = ""
chapter_id: str = ""
def _js_to_string(value: int, base: int) -> str:
digits = "0123456789abcdefghijklmnopqrstuvwxyz"
chars: list[str] = []
while value:
chars.append(digits[value % base])
value //= base
return "".join(reversed(chars)) or "0"
def _clean_html_text(raw: str) -> str:
text = re.sub(r"<br\s*/?>", "\n", raw, flags=re.I)
text = re.sub(r"</p\s*>", "\n", text, flags=re.I)
text = re.sub(r"<[^>]+>", "", text)
text = unescape(text)
text = text.replace("\r", "")
lines = [line.strip() for line in text.splitlines()]
return "\n".join(line for line in lines if line)
def _normalize_space(value: str) -> str:
return re.sub(r"\s+", " ", _clean_html_text(value)).strip()
class _ChapterDecryptor:
"""AES-CBC decryptor matching index.eef719ce.js class XE."""
def __init__(self) -> None:
if not CRYPTOGRAPHY_AVAILABLE:
raise PluginParseError("缺少 cryptography 依赖,无法解密长佩章节正文。")
suffix = _js_to_string(int("165455", 14), 32)
key_seed = "iGzsYn" + suffix
iv_str = f"$h$b3!{key_seed}"
decoded = base64.b64decode("dTBMUnJidSRFbg==").decode("utf-8")
key_str = decoded + _js_to_string(int("4d5a6c8", 14), 36) + "A"
self._key = key_str.encode("utf-8")
self._iv = iv_str.encode("utf-8")
def decrypt(self, encrypted_b64: str) -> str:
if not encrypted_b64:
return ""
raw = base64.b64decode(encrypted_b64)
cipher = Cipher(algorithms.AES(self._key), modes.CBC(self._iv))
decryptor = cipher.decryptor()
padded = decryptor.update(raw) + decryptor.finalize()
unpadder = crypto_padding.PKCS7(128).unpadder()
plain = unpadder.update(padded) + unpadder.finalize()
return plain.decode("utf-8")
class ChangpeiPlugin(BasePlugin):
name = "长佩文学"
domain = "www.gongzicp.com"
version = "1.0.0"
def __init__(self, cookie: str = "") -> None:
self.cookie = cookie
self.session = requests.Session()
self.session.headers.update(self.get_headers())
self._chapter_decryptor: _ChapterDecryptor | None = None
def set_auth(self, cookie: str) -> None:
"""设置登录 Cookie,用于访问需订阅/登录的章节。"""
self.cookie = cookie or ""
self.session.headers.update(self.get_headers())
def validate(self) -> bool:
return CRYPTOGRAPHY_AVAILABLE
def can_handle(self, url: str) -> bool:
lowered = url.strip().lower()
if any(host in lowered for host in ("gongzicp.com", "gongzicp.cn")):
return True
if lowered.isdigit():
return True
return False
def get_headers(self) -> dict[str, str]:
headers = {
"User-Agent": USER_AGENT,
"Accept": "application/json, text/plain, */*",
"Accept-Language": "zh-CN,zh;q=0.9",
"Referer": BASE_URL + "/",
}
if self.cookie:
headers["Cookie"] = self.cookie
return headers
def resolve_target(self, target: str, *, limit: int = 20) -> dict[str, Any]:
raw = target.strip()
if not raw:
raise PluginParseError("请输入书名、作者、书籍链接或章节链接。")
normalized = self._normalize_target(raw)
if normalized:
ids = self._extract_ids_from_url(normalized)
if ids.novel_id and ids.chapter_id:
chapter = self._chapter_resource_from_ids(ids.novel_id, ids.chapter_id, source_url=normalized)
return {
"mode": "chapters",
"book_title": str(chapter.metadata.get("book_title") or ""),
"resources": [chapter],
}
if ids.novel_id:
return self._resolve_novel_id(ids.novel_id)
if ids.chapter_id and not ids.novel_id:
raise PluginParseError("章节链接缺少书籍 ID,请使用 read-{书籍ID}-{章节ID}.html 格式。")
if raw.isdigit():
return self._resolve_novel_id(raw)
books = self._search_books(raw, limit=limit)
if not books:
raise PluginParseError(f"没有搜索到与“{raw}”相关的书籍。")
return {
"mode": "books",
"title": f"搜索结果:{raw}",
"books": [candidate.__dict__ for candidate in books],
}
def parse(self, url: str) -> list[Resource]:
outcome = self.resolve_target(url, limit=20)
if outcome.get("mode") == "chapters":
return list(outcome.get("resources") or [])
books = list(outcome.get("books") or [])
if not books:
raise PluginParseError("未解析到可下载内容。")
selected = self.resolve_target(str(books[0].get("url") or ""), limit=20)
resources = list(selected.get("resources") or [])
if not resources:
raise PluginParseError("选中的书籍没有章节内容。")
return resources
def download(self, resource: Resource, save_path: Path) -> list[Path]:
if str(resource.metadata.get("kind") or "") != "novel_chapter":
raise PluginParseError("不支持的长佩资源类型。")
content = str(resource.metadata.get("content") or "")
if not content.strip():
nid = str(resource.metadata.get("novel_id") or resource.metadata.get("book_id") or "")
cid = str(resource.metadata.get("chapter_id") or "")
if nid and cid:
preserved_title = str(resource.metadata.get("book_title") or "")
preserved_author = str(resource.author or resource.metadata.get("author") or "")
chapter_data = self._chapter_data_from_api(nid, cid)
content = str(chapter_data.get("content") or "")
resource.metadata.update(chapter_data)
if preserved_title and not str(resource.metadata.get("book_title") or "").strip():
resource.metadata["book_title"] = preserved_title
if preserved_author and not str(resource.author or "").strip():
resource.author = preserved_author
book_title = str(resource.metadata.get("book_title") or "未命名小说")
chapter_title = str(resource.metadata.get("chapter_title") or resource.title or "未命名章节")
chapter_order = int(resource.metadata.get("chapter_order") or 0)
folder = save_path / "novels" / sanitize_filename(book_title, "changpei_book")
filename = (
f"{chapter_order:04d}_{sanitize_filename(chapter_title, 'chapter')}.txt"
if chapter_order
else f"{sanitize_filename(chapter_title, 'chapter')}.txt"
)
final_path = folder / filename
text = "\n".join(
[
f"书名: {book_title}",
f"章节: {chapter_title}",
f"作者: {resource.author or '未知'}",
f"链接: {resource.url}",
"",
content.strip(),
]
).strip()
save_text_file(final_path, text)
resource.metadata["word_count"] = len(content)
return [final_path]
def _get_decryptor(self) -> _ChapterDecryptor:
if self._chapter_decryptor is None:
self._chapter_decryptor = _ChapterDecryptor()
return self._chapter_decryptor
def _normalize_target(self, target: str) -> str:
value = target.strip()
lowered = value.lower()
if lowered.startswith(("http://", "https://", "//")):
if value.startswith("//"):
return "https:" + value
return value
if lowered.startswith(("www.gongzicp.com", "gongzicp.com")):
return "https://" + value.lstrip("/")
if _NOVEL_PATH_RE.search(value) or _READ_PATH_RE.search(value) or _SINGLE_READ_RE.search(value):
return urljoin(BASE_URL, value.lstrip("/"))
return ""
def _extract_ids_from_url(self, url: str) -> TargetIds:
parsed = urlparse(url)
path = parsed.path or ""
match = _READ_PATH_RE.search(path)
if match:
return TargetIds(novel_id=match.group("nid"), chapter_id=match.group("cid"))
match = _NOVEL_PATH_RE.search(path)
if match:
return TargetIds(novel_id=match.group("nid"))
match = _SINGLE_READ_RE.search(path)
if match:
return TargetIds(chapter_id=match.group("cid"))
return TargetIds()
def _webapi_get(self, path: str, *, params: dict[str, Any] | None = None, referer: str = "") -> dict[str, Any]:
headers = dict(self.get_headers())
if referer:
headers["Referer"] = referer
response = self.session.get(
f"{WEBAPI_BASE}{path}",
params=params or {},
headers=headers,
timeout=30,
)
response.raise_for_status()
payload = response.json()
code = payload.get("code")
if code not in (200, "200"):
message = str(payload.get("msg") or "请求失败")
raise PluginParseError(f"长佩接口错误:{message}")
data = payload.get("data")
if isinstance(data, dict):
return data
if data is None:
return {}
return {"value": data}
def _novel_url(self, novel_id: str) -> str:
return f"{BASE_URL}/novel-{novel_id}.html"
def _chapter_url(self, novel_id: str, chapter_id: str) -> str:
return f"{BASE_URL}/read-{novel_id}-{chapter_id}.html"
def _search_books(self, keyword: str, *, limit: int) -> list[BookCandidate]:
page = 1
books: list[BookCandidate] = []
while len(books) < limit:
data = self._webapi_get(
"/search/novels",
params={"k": keyword, "page": page},
referer=f"{BASE_URL}/novel/searchNovel",
)
items = list(data.get("list") or [])
if not items:
break
for item in items:
novel_id = str(item.get("novel_id") or "").strip()
title = _normalize_space(str(item.get("novel_name") or ""))
if not novel_id or not title:
continue
author = _normalize_space(str(item.get("novel_author") or item.get("author_nickname") or ""))
books.append(
BookCandidate(
title=title,
url=self._novel_url(novel_id),
author=author,
latest_chapter=_normalize_space(str(item.get("novel_uptime") or "")),
)
)
if len(books) >= limit:
break
if len(items) < 10:
break
page += 1
return books
def _novel_info(self, novel_id: str) -> dict[str, Any]:
referer = self._novel_url(novel_id)
return self._webapi_get("/novel/novelInfo", params={"id": novel_id}, referer=referer)
def _chapter_list(self, novel_id: str) -> list[dict[str, Any]]:
referer = self._novel_url(novel_id)
data = self._webapi_get("/novel/chapterGetList", params={"nid": novel_id}, referer=referer)
return list(data.get("list") or [])
def _decrypt_chapter_content(self, encrypted: str) -> str:
try:
return self._get_decryptor().decrypt(encrypted)
except Exception:
return ""
def _chapter_data_from_api(self, novel_id: str, chapter_id: str) -> dict[str, Any]:
referer = self._chapter_url(novel_id, chapter_id)
payload = self._webapi_get(
"/novel/chapterGetInfo",
params={"nid": novel_id, "cid": chapter_id},
referer=referer,
)
chapter_info = dict(payload.get("chapterInfo") or {})
novel_info = dict(payload.get("novelInfo") or {})
locked = bool(chapter_info.get("lock")) or bool(chapter_info.get("chapter_ispay"))
status = int(chapter_info.get("status") or 0)
encrypted = str(chapter_info.get("content") or "")
content = ""
if not locked and status == 1 and encrypted:
content = self._decrypt_chapter_content(encrypted)
chapter_title = _normalize_space(str(chapter_info.get("name") or f"章节 {chapter_id}"))
book_title = _normalize_space(str(novel_info.get("name") or novel_info.get("novel_name") or ""))
author = _normalize_space(str(novel_info.get("author") or novel_info.get("author_nickname") or ""))
order = int(chapter_info.get("order") or 0)
source_id = f"{novel_id}:{chapter_id}"
if locked:
content = "[章节需订阅或已锁定,无法抓取正文]"
return {
"source_id": source_id,
"novel_id": novel_id,
"book_id": novel_id,
"book_title": book_title,
"chapter_id": chapter_id,
"chapter_title": chapter_title,
"chapter_order": order,
"author": author,
"content": content,
"word_count": len(content),
"locked": locked,
"description": _clean_html_text(str(novel_info.get("novel_info") or novel_info.get("novel_desc") or "")),
}
def _flatten_chapter_items(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
result: list[dict[str, Any]] = []
for item in items:
if not isinstance(item, dict):
continue
if str(item.get("type") or "") == "item":
result.append(item)
for key in ("children", "list", "chapterList", "chapters"):
child_items = item.get(key)
if isinstance(child_items, list):
result.extend(self._flatten_chapter_items(child_items))
return result
def _chapter_resources_from_novel(self, novel_id: str, novel_info: dict[str, Any]) -> list[Resource]:
book_title = _normalize_space(str(novel_info.get("novel_name") or ""))
author = _normalize_space(str(novel_info.get("author_nickname") or ""))
description = _clean_html_text(str(novel_info.get("novel_info") or novel_info.get("novel_desc") or ""))
chapters = self._flatten_chapter_items(self._chapter_list(novel_id))
resources: list[Resource] = []
for index, item in enumerate(chapters, start=1):
chapter_id = str(item.get("id") or "").strip()
chapter_title = _normalize_space(str(item.get("name") or ""))
if not chapter_id or not chapter_title:
continue
locked = bool(item.get("lock")) or bool(item.get("pay"))
order = int(item.get("order") or index)
source_id = f"{novel_id}:{chapter_id}"
resources.append(
Resource(
id=source_id,
url=self._chapter_url(novel_id, chapter_id),
title=chapter_title,
author=author,
author_id=str(novel_info.get("author_id") or ""),
files=[],
metadata={
"kind": "novel_chapter",
"resource_type": "novel",
"content_type": "novel",
"source_platform": "changpei",
"source_id": source_id,
"novel_id": novel_id,
"book_id": novel_id,
"book_title": book_title,
"series_id": novel_id,
"series_title": book_title,
"chapter_id": chapter_id,
"chapter_title": chapter_title,
"chapter_order": order,
"description": description,
"locked": locked,
"last_update": str(novel_info.get("novel_uptime") or ""),
"book_status": str(novel_info.get("novel_process") or ""),
},
)
)
return resources
def _chapter_resource_from_ids(self, novel_id: str, chapter_id: str, *, source_url: str = "") -> Resource:
data = self._chapter_data_from_api(novel_id, chapter_id)
return Resource(
id=str(data["source_id"]),
url=source_url or self._chapter_url(novel_id, chapter_id),
title=str(data["chapter_title"]),
author=str(data.get("author") or ""),
author_id="",
files=[],
metadata={
"kind": "novel_chapter",
"resource_type": "novel",
"content_type": "novel",
"source_platform": "changpei",
"source_id": str(data["source_id"]),
"novel_id": novel_id,
"book_id": novel_id,
"book_title": str(data.get("book_title") or ""),
"series_id": novel_id,
"series_title": str(data.get("book_title") or ""),
"chapter_id": chapter_id,
"chapter_title": str(data["chapter_title"]),
"chapter_order": int(data.get("chapter_order") or 1),
"description": str(data.get("description") or ""),
"content": str(data.get("content") or ""),
"word_count": int(data.get("word_count") or 0),
"locked": bool(data.get("locked")),
},
)
def _resolve_novel_id(self, novel_id: str) -> dict[str, Any]:
novel_info = self._novel_info(novel_id)
resources = self._chapter_resources_from_novel(novel_id, novel_info)
if not resources:
raise PluginParseError(f"书籍 {novel_id} 没有解析到章节目录。")
book_title = str(resources[0].metadata.get("book_title") or "")
return {
"mode": "chapters",
"book_title": book_title,
"resources": resources,
}
# ---- cookie config (matches Pixiv / AO3 pattern) ----
def get_config_schema(self) -> dict[str, Any]:
return {
"type": "object",
"properties": {
"cookie": {
"type": "string",
"title": "长佩 Cookie",
"description": "粘贴长佩 (gongzicp.com) 登录后的 Cookie 文本,用于访问已购章节。",
},
},
}
def load_config(self) -> dict[str, Any]:
return {"cookie": self.cookie}
def save_config(self, data: dict[str, Any]) -> None:
self.cookie = str(data.get("cookie") or "")
self.session.headers.update(self.get_headers())
plugin_class = ChangpeiPlugin
+1
View File
@@ -0,0 +1 @@
"""Douyin plugin package."""
+11
View File
@@ -0,0 +1,11 @@
{
"name": "Douyin",
"version": "1.0.0",
"domain": "douyin.com",
"author": "Sakura Downloader",
"description": "Douyin video and image-note parser/downloader for public share URLs.",
"requires_auth": false,
"tags": ["social", "video", "image"],
"min_platform_version": "1.0.0",
"dependencies": []
}
+387
View File
@@ -0,0 +1,387 @@
"""Douyin plugin.
Supports public Douyin share, video, note, and modal links. Cookie support is
optional and reads from runtime/douyin_cookies.json when present.
"""
from __future__ import annotations
import html
import json
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, unquote, urlparse
import httpx
from pixiv_app.core.cookie_import import cookies_to_header, parse_cookie_text
from pixiv_app.core.paths import runtime_path
from pixiv_app.core.plugin.base import BasePlugin, PluginDownloadError, PluginParseError, Resource
_AWEME_ID_RE = re.compile(
r"(?:aweme_id=|modal_id=|/video/|/note/|/share/video/|/share/note/)(\d{6,})",
re.IGNORECASE,
)
_JSON_SCRIPT_RE = re.compile(
r'<script[^>]+id=["\'](?P<id>RENDER_DATA|__UNIVERSAL_DATA_FOR_REHYDRATION__)["\'][^>]*>'
r"(?P<body>.*?)</script>",
re.IGNORECASE | re.DOTALL,
)
_JSON_URL_RE = re.compile(r"https?:\\?/\\?/[^\"'<>\\]+")
@dataclass
class DouyinPluginConfig:
timeout: float = 30.0
max_redirects: int = 8
cookie_file: str = ""
class DouyinPlugin(BasePlugin):
domain = "douyin.com"
name = "Douyin"
version = "1.0.0"
def __init__(self) -> None:
self.config = DouyinPluginConfig(cookie_file=str(runtime_path("douyin_cookies.json")))
def can_handle(self, url: str) -> bool:
lower = (url or "").strip().lower()
return any(
token in lower
for token in (
"douyin.com",
"v.douyin.com",
"iesdouyin.com",
"amemv.com",
)
)
def get_headers(self) -> dict[str, str]:
headers = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Referer": "https://www.douyin.com/",
}
cookie_header = self._cookie_header()
if cookie_header:
headers["Cookie"] = cookie_header
return headers
def parse(self, url: str) -> list[Resource]:
target = _extract_first_douyin_url(url)
if not target:
raise PluginParseError("No valid Douyin URL found.")
final_url, body = self._fetch_page(target)
aweme_id = _extract_aweme_id(final_url) or _extract_aweme_id(body) or _extract_aweme_id(target)
payloads = _extract_json_payloads(body)
focus_nodes = _find_nodes_for_aweme(payloads, aweme_id) if aweme_id else []
media = _extract_media(focus_nodes or payloads)
if not media:
media = _extract_media_from_raw_html(body)
files = _dedupe_files(media)
if not files:
raise PluginParseError(
"No Douyin video or image media was found. Login, captcha, or page changes may be required."
)
title = _first_text(focus_nodes or payloads, ("desc", "title", "caption")) or "Douyin"
author = _first_text(focus_nodes or payloads, ("nickname", "author_name", "unique_id")) or ""
author_id = _first_text(focus_nodes or payloads, ("uid", "sec_uid", "short_id", "unique_id")) or ""
created_at = _first_text(focus_nodes or payloads, ("create_time", "createTime"))
return [
Resource(
id=aweme_id or str(abs(hash(final_url))),
url=final_url,
title=title,
author=author,
author_id=author_id,
files=files,
metadata={
"source_platform": "douyin",
"aweme_id": aweme_id or "",
"kind": "douyin",
"original_url": target,
},
thumbnail=next(
(item.get("thumbnail") or item.get("url") for item in files if item.get("type") == "image"),
None,
),
created_at=str(created_at) if created_at else None,
)
]
def download(self, resource: Resource, save_path: Path) -> list[Path]:
save_path.mkdir(parents=True, exist_ok=True)
downloaded: list[Path] = []
with httpx.Client(timeout=max(30.0, self.config.timeout), follow_redirects=True, headers=self.get_headers()) as client:
for index, item in enumerate(resource.files, start=1):
media_url = str(item.get("url") or "").strip()
if not media_url:
continue
suffix = _guess_suffix(media_url, str(item.get("type") or ""))
target = save_path / f"{_safe_filename(resource.id)}_{index:02d}{suffix}"
if target.exists() and target.stat().st_size > 512:
downloaded.append(target)
continue
try:
response = client.get(media_url, headers={**self.get_headers(), "Referer": resource.url})
response.raise_for_status()
except Exception as exc:
raise PluginDownloadError(f"Douyin media download failed: {exc}") from exc
content_type = response.headers.get("content-type", "").lower()
if "text/html" in content_type:
raise PluginDownloadError("Douyin media download returned HTML. Login or captcha may be required.")
target.write_bytes(response.content)
if target.stat().st_size <= 512:
target.unlink(missing_ok=True)
raise PluginDownloadError("Douyin media download was too small and likely failed.")
downloaded.append(target)
return downloaded
def _fetch_page(self, url: str) -> tuple[str, str]:
try:
with httpx.Client(
timeout=self.config.timeout,
follow_redirects=True,
max_redirects=self.config.max_redirects,
headers=self.get_headers(),
) as client:
response = client.get(url)
response.raise_for_status()
return str(response.url), response.text
except Exception as exc:
raise PluginParseError(f"Douyin page load failed: {exc}") from exc
def _cookie_header(self) -> str:
cookie_file = Path(self.config.cookie_file or "")
if not cookie_file.is_file():
return ""
try:
cookies = parse_cookie_text(cookie_file.read_text(encoding="utf-8"))
return cookies_to_header(cookies, ("douyin.com", "iesdouyin.com", "amemv.com"))
except Exception:
return ""
def _extract_first_douyin_url(text: str) -> str:
value = (text or "").strip()
match = re.search(r"https?://[^\s\"'<>]*?(?:douyin|iesdouyin|amemv)\.com/[^\s\"'<>]*", value, re.I)
if match:
return match.group(0).rstrip(",.;)")
if "douyin.com" in value.lower() and not value.startswith(("http://", "https://")):
return "https://" + value.lstrip("/")
return value if value.startswith(("http://", "https://")) else ""
def _extract_aweme_id(text: str) -> str:
match = _AWEME_ID_RE.search(text or "")
if match:
return match.group(1)
try:
query = parse_qs(urlparse(text).query)
except Exception:
return ""
for key in ("aweme_id", "modal_id", "item_id"):
values = query.get(key)
if values and str(values[0]).isdigit():
return str(values[0])
return ""
def _extract_json_payloads(page: str) -> list[Any]:
payloads: list[Any] = []
for match in _JSON_SCRIPT_RE.finditer(page or ""):
raw = html.unescape(match.group("body") or "").strip()
if match.group("id") == "RENDER_DATA":
raw = unquote(raw)
try:
payloads.append(json.loads(raw))
except Exception:
continue
for pattern in (
r"window\.__INITIAL_STATE__\s*=\s*({.*?})\s*</script>",
r"window\._ROUTER_DATA\s*=\s*({.*?})\s*</script>",
):
for match in re.finditer(pattern, page or "", re.DOTALL):
try:
payloads.append(json.loads(match.group(1)))
except Exception:
continue
return payloads
def _find_nodes_for_aweme(payloads: list[Any], aweme_id: str) -> list[Any]:
if not aweme_id:
return []
matches: list[Any] = []
def walk(node: Any) -> None:
if isinstance(node, dict):
values = {
str(node.get(key) or "")
for key in ("aweme_id", "awemeId", "id", "item_id", "itemId", "modal_id", "modalId")
}
if aweme_id in values:
matches.append(node)
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
for payload in payloads:
walk(payload)
return matches
def _extract_media(payloads: list[Any]) -> list[dict[str, str]]:
files: list[dict[str, str]] = []
def add_video(node: dict[str, Any]) -> None:
for key in ("play_addr", "download_addr", "playApi", "video", "videoModel"):
urls = _urls_from_node(node.get(key))
if urls:
files.append({"type": "video", "url": _normalize_url(urls[0])})
return
def add_images(node: dict[str, Any]) -> None:
for key in ("images", "image_post_info", "image", "imageAlbum"):
for image_url in _urls_from_node(node.get(key)):
files.append({"type": "image", "url": _normalize_url(image_url)})
def walk(node: Any) -> None:
if isinstance(node, dict):
add_images(node)
if any(key in node for key in ("play_addr", "download_addr", "video", "playApi", "videoModel")):
add_video(node)
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
for payload in payloads:
walk(payload)
return files
def _extract_media_from_raw_html(page: str) -> list[dict[str, str]]:
files: list[dict[str, str]] = []
for raw in _JSON_URL_RE.findall(page or ""):
url = _normalize_url(raw)
low = url.lower()
if any(token in low for token in ("douyin", "byteimg", "pstatp", "douyinvod", "amemv", "douyinpic")):
if any(token in low for token in (".mp4", "playwm", "play/?", "video_id=")):
files.append({"type": "video", "url": url})
elif any(token in low for token in (".jpeg", ".jpg", ".png", ".webp", "image")):
files.append({"type": "image", "url": url})
return files
def _urls_from_node(node: Any) -> list[str]:
if node is None:
return []
if isinstance(node, str):
value = _normalize_url(node)
return [value] if value.startswith(("http://", "https://", "//")) else []
if isinstance(node, list):
urls: list[str] = []
for item in node:
urls.extend(_urls_from_node(item))
return urls
if isinstance(node, dict):
urls: list[str] = []
for key in (
"url_list",
"urlList",
"urls",
"uri",
"url",
"src",
"download_url",
"downloadUrl",
"play_addr",
"playAddr",
"origin_cover",
"cover",
):
urls.extend(_urls_from_node(node.get(key)))
return urls
return []
def _dedupe_files(files: list[dict[str, str]]) -> list[dict[str, str]]:
video: dict[str, str] | None = None
images: list[dict[str, str]] = []
seen: set[str] = set()
for item in files:
url = _normalize_url(item.get("url", ""))
if not url or url in seen:
continue
seen.add(url)
media_type = item.get("type", "image")
row = {"type": media_type, "url": url}
if media_type == "video":
if video is None:
video = row
continue
images.append(row)
return ([video] if video else []) + images
def _first_text(payloads: list[Any], keys: tuple[str, ...]) -> str:
result = ""
def walk(node: Any) -> None:
nonlocal result
if result:
return
if isinstance(node, dict):
for key in keys:
value = node.get(key)
if isinstance(value, (str, int)) and str(value).strip():
result = str(value).strip()
return
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
for payload in payloads:
walk(payload)
return result
def _normalize_url(url: str) -> str:
value = html.unescape(str(url or "")).strip().replace("\\/", "/")
if value.startswith("//"):
value = "https:" + value
return value
def _guess_suffix(url: str, media_type: str) -> str:
path = urlparse(url).path.lower()
suffix = Path(path).suffix
if suffix in {".jpg", ".jpeg", ".png", ".webp", ".gif", ".mp4", ".mov", ".m4v"}:
return suffix
return ".mp4" if media_type == "video" else ".jpg"
def _safe_filename(value: str) -> str:
return re.sub(r"[^A-Za-z0-9_.-]+", "_", value or "douyin").strip("._") or "douyin"
plugin_class = DouyinPlugin
+1
View File
@@ -0,0 +1 @@
"""Bundled Pixiv plugin package."""
+11
View File
@@ -0,0 +1,11 @@
{
"name": "Pixiv",
"version": "2.0.0",
"domain": "pixiv.net",
"author": "Sakura Downloader",
"description": "Pixiv illustrations, novels, user batches and tag search via existing core modules.",
"requires_auth": true,
"tags": ["pixiv", "illustration"],
"min_platform_version": "1.0.0",
"dependencies": []
}
+253
View File
@@ -0,0 +1,253 @@
"""
Pixiv site plugin — wraps existing pixiv_app downloader helpers (no rewrite).
"""
from __future__ import annotations
import re
from pathlib import Path
from typing import Any
from urllib.parse import unquote, urlsplit
from pixiv_app.core.downloader import (
download_binary,
download_novel,
fetch_illust_details,
fetch_illust_pages,
fetch_novel_series_ids,
fetch_user_work_ids,
parse_pixiv_target,
search_illust_ids_by_keyword,
)
from pixiv_app.core.plugin.base import (
BasePlugin,
PluginParseError,
Resource,
)
try:
from pixiv_app.core.fingerprint import default_profile
except Exception: # pragma: no cover
default_profile = None
_TAG_PATH_RE = re.compile(r"/tags/([^/]+)/?", re.IGNORECASE)
class PixivPlugin(BasePlugin):
domain = "pixiv.net"
name = "Pixiv"
version = "2.0.0"
def __init__(self, cookie: str = "", proxy_pool: Any = None) -> None:
self.cookie = cookie
self.proxy_pool = proxy_pool
def set_auth(self, cookie: str) -> None:
self.cookie = cookie or ""
def set_proxy_pool(self, proxy_pool: Any) -> None:
self.proxy_pool = proxy_pool
def can_handle(self, url: str) -> bool:
text = url.strip()
if not text:
return False
if text.isdigit():
return True
candidate = text if "://" in text else f"https://{text.lstrip('/')}"
try:
hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".")
except ValueError:
return False
return any(hostname == domain or hostname.endswith(f".{domain}") for domain in ("pixiv.net", "pximg.net"))
def get_headers(self) -> dict[str, str]:
from pixiv_app.core.downloader import build_headers
headers = build_headers(cookie=self.cookie)
if default_profile:
headers.update(default_profile("pixiv").headers())
return headers
def parse(self, url: str) -> list[Resource]:
raw = url.strip()
if not raw:
raise PluginParseError("Empty URL")
if _TAG_PATH_RE.search(raw):
m = _TAG_PATH_RE.search(raw)
assert m is not None
tag = unquote(m.group(1))
return self._parse_tag(tag)
try:
kind, num_id = parse_pixiv_target(raw)
except ValueError as exc:
raise PluginParseError(str(exc)) from exc
if kind == "novel":
return [self._resource_novel_stub(num_id)]
if kind == "novel_series":
return self._parse_novel_series(num_id)
if kind == "user":
return self._parse_user(num_id, input_kind="user")
if kind == "illust":
return [self._parse_illust(num_id)]
if kind == "unknown":
return [self._parse_illust(num_id)]
raise PluginParseError(f"Unsupported Pixiv target kind: {kind}")
def _parse_illust(self, illust_id: int) -> Resource:
detail = fetch_illust_details(illust_id, cookie=self.cookie, proxy_pool=self.proxy_pool)
page_urls = fetch_illust_pages(illust_id, cookie=self.cookie, proxy_pool=self.proxy_pool)
files = [{"url": u, "page": i} for i, u in enumerate(page_urls) if u]
thumb = page_urls[0] if page_urls else None
return Resource(
id=str(illust_id),
url=f"https://www.pixiv.net/artworks/{illust_id}",
title=str(detail.get("title") or ""),
author=str(detail.get("userName") or ""),
author_id=str(detail.get("userId") or ""),
files=files,
metadata={**detail, "kind": "illust", "illust_id": illust_id},
thumbnail=thumb,
created_at=str(detail.get("createDate") or detail.get("uploadDate") or "") or None,
)
def _parse_user(self, seed_id: int, *, input_kind: str) -> list[Resource]:
resolved_uid, work_ids = fetch_user_work_ids(
seed_id,
cookie=self.cookie,
proxy_pool=self.proxy_pool,
input_kind=input_kind,
)
out: list[Resource] = []
for wid in work_ids:
out.append(
Resource(
id=str(wid),
url=f"https://www.pixiv.net/artworks/{wid}",
title="",
author="",
author_id=str(resolved_uid),
files=[],
metadata={
"kind": "illust_lazy",
"illust_id": wid,
"resolved_user_id": resolved_uid,
},
)
)
return out
def _parse_tag(self, tag: str) -> list[Resource]:
ids = search_illust_ids_by_keyword(
tag,
cookie=self.cookie,
proxy_pool=self.proxy_pool,
limit=50,
)
resources: list[Resource] = []
for iid in ids:
resources.append(
Resource(
id=str(iid),
url=f"https://www.pixiv.net/artworks/{iid}",
title="",
author="",
author_id="",
files=[],
metadata={"kind": "illust_lazy", "illust_id": iid, "tag": tag},
)
)
return resources
def _parse_novel_series(self, series_id: int) -> list[Resource]:
series_title, novel_ids = fetch_novel_series_ids(
series_id,
cookie=self.cookie,
proxy_pool=self.proxy_pool,
)
resources: list[Resource] = []
for chapter_order, novel_id in enumerate(novel_ids, start=1):
resource = self._resource_novel_stub(novel_id)
resource.metadata.update(
{
"series_id": str(series_id),
"series_title": series_title,
"chapter_order": chapter_order,
}
)
resources.append(resource)
return resources
def _resource_novel_stub(self, novel_id: int) -> Resource:
return Resource(
id=str(novel_id),
url=f"https://www.pixiv.net/novel/show.php?id={novel_id}",
title="",
author="",
author_id="",
files=[],
metadata={"kind": "novel", "novel_id": novel_id},
)
def download(self, resource: Resource, save_path: Path) -> list[Path]:
save_path.mkdir(parents=True, exist_ok=True)
kind = resource.metadata.get("kind")
if kind == "novel":
novel_id = int(resource.metadata.get("novel_id") or resource.id)
download_novel(
novel_id,
cookie=self.cookie,
save_root=save_path,
save_subdir="novels",
proxy_pool=self.proxy_pool,
library_db_path=None,
metadata_overrides={
"seriesId": resource.metadata.get("series_id"),
"seriesTitle": resource.metadata.get("series_title"),
"chapterOrder": resource.metadata.get("chapter_order"),
},
)
matches = list(save_path.rglob(f"*{novel_id}.txt"))
return matches
illust_id = int(resource.metadata.get("illust_id") or resource.id)
if kind == "illust_lazy" or not resource.files:
detail = fetch_illust_details(illust_id, cookie=self.cookie, proxy_pool=self.proxy_pool)
page_urls = fetch_illust_pages(illust_id, cookie=self.cookie, proxy_pool=self.proxy_pool)
resource.files = [{"url": u, "page": i} for i, u in enumerate(page_urls) if u]
resource.metadata.update(detail)
downloaded: list[Path] = []
subdir = resource.metadata.get("save_subdir")
target_dir = save_path / subdir if subdir else save_path
target_dir.mkdir(parents=True, exist_ok=True)
for item in resource.files:
file_url = item.get("url")
if not file_url:
continue
ok, _msg = download_binary(
file_url,
save_dir=target_dir,
cookie=self.cookie,
proxy_pool=self.proxy_pool,
)
if ok:
fname = file_url.rsplit("/", 1)[-1].split("?")[0]
downloaded.append(target_dir / fname)
return downloaded
plugin_class = PixivPlugin
+11
View File
@@ -0,0 +1,11 @@
{
"name": "Taobao",
"version": "1.0.0",
"domain": "taobao.com",
"author": "Sakura Downloader",
"description": "Taobao and Tmall product image/video parser with optional local Cookie support.",
"requires_auth": false,
"tags": ["shopping", "image", "video"],
"min_platform_version": "1.0.0",
"dependencies": []
}
+400
View File
@@ -0,0 +1,400 @@
"""Taobao and Tmall product parser/downloader.
The plugin parses public product pages and optionally reuses a locally saved
Taobao cookie. Tests mock every HTTP request; no live site is contacted there.
"""
from __future__ import annotations
import html
import json
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, urlparse
import httpx
from pixiv_app.core.cookie_import import cookies_to_header, parse_cookie_text
from pixiv_app.core.paths import runtime_path
from pixiv_app.core.plugin.base import BasePlugin, PluginDownloadError, PluginParseError, Resource
_ALLOWED_DOMAINS = ("taobao.com", "tmall.com", "tb.cn")
_TITLE_KEYS = {"title", "itemtitle", "item_title", "subject", "goodsname", "goods_name"}
_AUTHOR_KEYS = {"shopname", "shop_name", "sellername", "seller_name", "nick", "nickname"}
_ID_KEYS = {"itemid", "item_id", "nid", "auctionid", "auction_id", "goodsid", "goods_id"}
_PRICE_KEYS = {"price", "promotionprice", "promotion_price", "saleprice", "sale_price"}
_IMAGE_KEYS = {
"images",
"image",
"pics",
"picurl",
"pic_url",
"imageurl",
"image_url",
"itemimages",
"item_images",
"detailimages",
"detail_images",
"mainimage",
"main_image",
}
_VIDEO_KEYS = {"video", "videos", "videourl", "video_url", "videoid", "video_id"}
_URL_RE = re.compile(r"(?:https?:)?(?:\\?/){2}[^\s\"'<>]+", re.IGNORECASE)
_JSON_SCRIPT_RE = re.compile(
r"<script[^>]*(?:type=[\"']application/(?:ld\+)?json[\"']|id=[\"']__NEXT_DATA__[\"'])[^>]*>"
r"(?P<body>.*?)</script>",
re.IGNORECASE | re.DOTALL,
)
_META_RE = re.compile(
r"<meta[^>]+(?:property|name)=[\"'](?P<name>og:title|og:image|description|og:description)[\"']"
r"[^>]+content=[\"'](?P<content>.*?)[\"'][^>]*>",
re.IGNORECASE | re.DOTALL,
)
_IMG_RE = re.compile(
r"<img[^>]+(?:data-src|data-original|data-ks-lazyload|src)=[\"'](?P<url>[^\"']+)[\"']",
re.IGNORECASE,
)
@dataclass
class TaobaoPluginConfig:
timeout: float = 25.0
max_redirects: int = 8
cookie_file: str = ""
max_media: int = 100
class TaobaoPlugin(BasePlugin):
domain = "taobao.com"
name = "Taobao"
version = "1.0.0"
def __init__(self) -> None:
self.config = TaobaoPluginConfig(cookie_file=str(runtime_path("taobao_cookies.txt")))
def can_handle(self, url: str) -> bool:
target = _extract_first_product_url(url)
if not target:
return False
try:
host = (urlparse(target).hostname or "").lower()
except ValueError:
return False
return _allowed_host(host)
def get_headers(self) -> dict[str, str]:
headers = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
),
"Accept": "text/html,application/xhtml+xml,application/json;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.7",
"Referer": "https://www.taobao.com/",
}
cookie = self._cookie_header()
if cookie:
headers["Cookie"] = cookie
return headers
def parse(self, url: str) -> list[Resource]:
target = _extract_first_product_url(url)
if not target or not self.can_handle(target):
raise PluginParseError("未识别到有效的淘宝或天猫商品链接。")
final_url, page = self._fetch_page(target)
if not self.can_handle(final_url):
raise PluginParseError("淘宝短链接跳转到了不受支持的站点。")
if _looks_like_challenge(page, final_url):
raise PluginParseError("淘宝要求登录或安全验证,请导入有效 Cookie 后重试。")
payloads = _extract_json_payloads(page)
extracted = _extract_structured_fields(payloads)
meta = _extract_meta(page)
product_id = _extract_item_id(final_url) or extracted["id"] or _extract_item_id(target)
title = extracted["title"] or meta.get("og:title", "")
title = _clean_title(title) or f"淘宝商品 {product_id or ''}".strip()
author = extracted["author"]
description = meta.get("og:description") or meta.get("description") or ""
files = list(extracted["files"])
if meta.get("og:image"):
files.insert(0, {"type": "image", "url": _normalize_url(meta["og:image"])})
for match in _IMG_RE.finditer(page):
files.append({"type": "image", "url": _normalize_url(match.group("url"))})
files.extend(_extract_media_from_html(page))
files = _dedupe_media(files, limit=self.config.max_media)
if not files:
raise PluginParseError("商品页面已打开,但没有解析到商品图片或视频,页面结构可能已更新。")
thumbnail = next((item["url"] for item in files if item["type"] == "image"), None)
return [
Resource(
id=product_id or str(abs(hash(final_url))),
url=final_url,
title=title,
author=author,
author_id="",
files=files,
metadata={
"source_platform": "taobao",
"kind": "product",
"product_id": product_id,
"price": extracted["price"],
"description": description,
"original_url": target,
},
thumbnail=thumbnail,
)
]
def download(self, resource: Resource, save_path: Path) -> list[Path]:
save_path.mkdir(parents=True, exist_ok=True)
downloaded: list[Path] = []
headers = self.get_headers()
with httpx.Client(timeout=max(30.0, self.config.timeout), follow_redirects=True, headers=headers) as client:
for index, item in enumerate(resource.files, start=1):
media_url = _normalize_url(str(item.get("url") or ""))
if not media_url:
continue
media_type = str(item.get("type") or "image")
suffix = _guess_suffix(media_url, media_type)
target = save_path / f"{_safe_filename(resource.id)}_{index:03d}{suffix}"
if target.is_file() and target.stat().st_size > 256:
downloaded.append(target)
continue
try:
response = client.get(media_url, headers={**headers, "Referer": resource.url})
response.raise_for_status()
except Exception as exc:
raise PluginDownloadError(f"淘宝媒体下载失败: {exc}") from exc
content_type = response.headers.get("content-type", "").lower()
if "text/html" in content_type or "application/json" in content_type:
raise PluginDownloadError("淘宝媒体地址返回了验证页面,请更新 Cookie 后重试。")
target.write_bytes(response.content)
if target.stat().st_size <= 256:
target.unlink(missing_ok=True)
raise PluginDownloadError("下载内容过小,可能是失效链接或占位图。")
downloaded.append(target)
return downloaded
def _fetch_page(self, url: str) -> tuple[str, str]:
try:
with httpx.Client(
timeout=self.config.timeout,
follow_redirects=True,
max_redirects=self.config.max_redirects,
headers=self.get_headers(),
) as client:
response = client.get(url)
response.raise_for_status()
return str(response.url), response.text
except Exception as exc:
raise PluginParseError(f"淘宝商品页面加载失败: {exc}") from exc
def _cookie_header(self) -> str:
path = Path(self.config.cookie_file or "")
if not path.is_file():
return ""
try:
cookies = parse_cookie_text(path.read_text(encoding="utf-8"))
return cookies_to_header(cookies, _ALLOWED_DOMAINS)
except Exception:
return ""
def _allowed_host(host: str) -> bool:
return any(host == domain or host.endswith(f".{domain}") for domain in _ALLOWED_DOMAINS)
def _extract_first_product_url(text: str) -> str:
value = html.unescape((text or "").strip())
match = re.search(r"https?://[^\s\"'<>]+", value, re.IGNORECASE)
if match:
return match.group(0).rstrip(",.;)")
if any(domain in value.lower() for domain in _ALLOWED_DOMAINS):
return "https://" + value.lstrip("/")
return ""
def _extract_item_id(url: str) -> str:
try:
parsed = urlparse(url)
query = parse_qs(parsed.query)
except ValueError:
return ""
for key in ("id", "itemId", "item_id", "auctionId"):
value = (query.get(key) or [""])[0]
match = re.search(r"\d{5,}", str(value))
if match:
return match.group(0)
for pattern in (r"/item/(\d{5,})", r"/i(\d{5,})\.htm", r"item_id[=/](\d{5,})"):
match = re.search(pattern, parsed.path, re.IGNORECASE)
if match:
return match.group(1)
return ""
def _extract_json_payloads(page: str) -> list[Any]:
payloads: list[Any] = []
for match in _JSON_SCRIPT_RE.finditer(page or ""):
raw = html.unescape(match.group("body") or "").strip()
try:
payloads.append(json.loads(raw))
except (TypeError, ValueError):
continue
for pattern in (
r"window\.__INITIAL_STATE__\s*=\s*({.*?})\s*;?\s*</script>",
r"window\.__INIT_DATA__\s*=\s*({.*?})\s*;?\s*</script>",
):
for match in re.finditer(pattern, page or "", re.IGNORECASE | re.DOTALL):
try:
payloads.append(json.loads(match.group(1)))
except (TypeError, ValueError):
continue
return payloads
def _extract_structured_fields(payloads: list[Any]) -> dict[str, Any]:
titles: list[str] = []
authors: list[str] = []
ids: list[str] = []
prices: list[str] = []
files: list[dict[str, str]] = []
def walk(node: Any) -> None:
if isinstance(node, dict):
for key, value in node.items():
normalized = str(key).replace("-", "_").lower()
compact = normalized.replace("_", "")
values = _text_values(value)
if normalized in _TITLE_KEYS or compact in _TITLE_KEYS:
titles.extend(values)
if normalized in _AUTHOR_KEYS or compact in _AUTHOR_KEYS:
authors.extend(values)
if normalized in _ID_KEYS or compact in _ID_KEYS:
ids.extend(values)
if normalized in _PRICE_KEYS or compact in _PRICE_KEYS:
prices.extend(values)
if normalized in _IMAGE_KEYS or compact in _IMAGE_KEYS:
files.extend({"type": "image", "url": url} for url in _urls_from_node(value))
if normalized in _VIDEO_KEYS or compact in _VIDEO_KEYS:
files.extend({"type": "video", "url": url} for url in _urls_from_node(value))
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
for payload in payloads:
walk(payload)
product_id = next((match.group(0) for value in ids if (match := re.search(r"\d{5,}", value))), "")
return {
"title": _best_text(titles),
"author": _best_text(authors),
"id": product_id,
"price": next((value for value in prices if value.strip()), ""),
"files": files,
}
def _extract_meta(page: str) -> dict[str, str]:
values: dict[str, str] = {}
for match in _META_RE.finditer(page or ""):
values[match.group("name").lower()] = html.unescape(match.group("content")).strip()
return values
def _extract_media_from_html(page: str) -> list[dict[str, str]]:
files: list[dict[str, str]] = []
for raw in _URL_RE.findall(page or ""):
url = _normalize_url(raw)
lower = url.lower().split("?", 1)[0]
if any(lower.endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp", ".avif")):
files.append({"type": "image", "url": url})
elif any(lower.endswith(ext) for ext in (".mp4", ".m3u8", ".webm")):
files.append({"type": "video", "url": url})
return files
def _urls_from_node(node: Any) -> list[str]:
if isinstance(node, str):
value = _normalize_url(node)
return [value] if value.startswith(("http://", "https://")) else []
if isinstance(node, list):
return [url for item in node for url in _urls_from_node(item)]
if isinstance(node, dict):
urls: list[str] = []
for key, value in node.items():
if str(key).lower() in {"url", "src", "urls", "url_list", "url_list_v2", "image", "video"}:
urls.extend(_urls_from_node(value))
elif isinstance(value, (dict, list)):
urls.extend(_urls_from_node(value))
return urls
return []
def _text_values(node: Any) -> list[str]:
if isinstance(node, (str, int, float)):
return [str(node).strip()]
if isinstance(node, dict):
return [str(value).strip() for key, value in node.items() if str(key).lower() in {"text", "value", "name"}]
return []
def _best_text(values: list[str]) -> str:
cleaned = [html.unescape(value).strip() for value in values if value and len(value.strip()) > 1]
return max(cleaned, key=len, default="")
def _clean_title(value: str) -> str:
title = re.sub(r"\s*[-_–—]\s*(淘宝网|天猫|tmall|taobao).*$", "", value or "", flags=re.IGNORECASE)
return re.sub(r"\s+", " ", title).strip()
def _normalize_url(value: str) -> str:
url = html.unescape((value or "").strip()).replace("\\u002F", "/").replace("\\/", "/")
if url.startswith("//"):
return "https:" + url
return url
def _dedupe_media(files: list[dict[str, str]], *, limit: int) -> list[dict[str, str]]:
result: list[dict[str, str]] = []
seen: set[str] = set()
for item in files:
url = _normalize_url(str(item.get("url") or ""))
if not url.startswith(("http://", "https://")) or url in seen:
continue
lower = url.lower()
if any(token in lower for token in ("sprite", "icon", "logo", "avatar", "qrcode")):
continue
seen.add(url)
result.append({"type": "video" if item.get("type") == "video" else "image", "url": url})
if len(result) >= max(1, limit):
break
return result
def _looks_like_challenge(page: str, url: str) -> bool:
sample = f"{url}\n{page[:20000]}".lower()
markers = ("login.taobao.com", "punish", "captcha", "滑动验证", "安全验证", "请登录")
return any(marker in sample for marker in markers)
def _guess_suffix(url: str, media_type: str) -> str:
suffix = Path(urlparse(url).path).suffix.lower()
allowed = {".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif", ".mp4", ".webm", ".m3u8"}
if suffix in allowed:
return suffix
return ".mp4" if media_type == "video" else ".jpg"
def _safe_filename(value: str) -> str:
return re.sub(r"[^0-9A-Za-z._-]+", "_", value or "taobao")[:80] or "taobao"
plugin_class = TaobaoPlugin
+1
View File
@@ -0,0 +1 @@
"""X (Twitter) Playwright plugin."""
+113
View File
@@ -0,0 +1,113 @@
"""Shared Playwright launch options for X — reduce automation fingerprint; prefer system Chrome."""
from __future__ import annotations
import os
from pathlib import Path
from playwright.async_api import BrowserContext, Playwright
from pixiv_app.core.playwright_runtime import configure_playwright_runtime_env, preferred_chromium_executable
try:
from pixiv_app.core.fingerprint import default_profile
except Exception: # pragma: no cover - plugin can run standalone
default_profile = None
try:
from pixiv_app.core.browser_proxy_pool import parse_proxy_url
except Exception: # pragma: no cover
parse_proxy_url = None
async def launch_x_persistent_context(
playwright: Playwright,
profile_dir: Path,
*,
headless: bool,
) -> BrowserContext:
"""
Launch persistent Chromium with flags that reduce obvious automation signals.
Order: ``X_PLAYWRIGHT_CHANNEL`` env (if set), then system Chrome, then bundled Chromium.
Google OAuth often still blocks automation browsers; users should use X email/phone login
or import cookies from a normal browser session.
"""
configure_playwright_runtime_env()
profile_dir.mkdir(parents=True, exist_ok=True)
browser_profile = default_profile("x") if default_profile else None
base_kw: dict = {
"user_data_dir": str(profile_dir),
"headless": headless,
"viewport": browser_profile.viewport if browser_profile else {"width": 1280, "height": 800},
"locale": browser_profile.locale if browser_profile else "zh-CN",
"timezone_id": browser_profile.timezone_id if browser_profile else "Asia/Shanghai",
"screen": browser_profile.viewport if browser_profile else {"width": 1280, "height": 800},
"extra_http_headers": {
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
},
"ignore_default_args": ["--enable-automation"],
"args": [
"--disable-blink-features=AutomationControlled",
"--window-size=1280,800",
],
"user_agent": browser_profile.user_agent
if browser_profile
else (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
),
}
if browser_profile:
base_kw["device_scale_factor"] = browser_profile.device_scale_factor
base_kw["is_mobile"] = browser_profile.is_mobile
base_kw["has_touch"] = browser_profile.has_touch
proxy_value = os.environ.get("X_PROXY_URL", os.environ.get("PIXIV_BROWSER_PROXY", "")).strip()
if proxy_value and parse_proxy_url:
parsed_proxy = parse_proxy_url(proxy_value)
if parsed_proxy is not None:
base_kw["proxy"] = parsed_proxy.to_playwright_proxy()
env_ch = os.environ.get("X_PLAYWRIGHT_CHANNEL", "").strip()
local_executable = preferred_chromium_executable()
seen: set[str] = set()
attempts: list[tuple[str, dict]] = []
for label, extra in (
("env_channel", {"channel": env_ch} if env_ch else {}),
("installed_chromium", {"executable_path": str(local_executable)} if local_executable else {}),
("system_chrome", {"channel": "chrome"}),
("default", {}),
):
key = f"{label}:{extra!r}"
if key in seen:
continue
seen.add(key)
attempts.append((label, extra))
last_err: Exception | None = None
for _, extra in attempts:
kw = dict(base_kw)
kw.update(extra)
try:
ctx = await playwright.chromium.launch_persistent_context(**kw)
await _apply_stealth_scripts(ctx)
return ctx
except Exception as exc:
last_err = exc
continue
assert last_err is not None
raise last_err
async def _apply_stealth_scripts(ctx: BrowserContext) -> None:
"""Best-effort scripts; cannot guarantee OAuth providers will accept the browser."""
await ctx.add_init_script(
"""
try {
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
} catch (e) {}
"""
)
+11
View File
@@ -0,0 +1,11 @@
{
"name": "X",
"version": "1.0.0",
"domain": "twitter.com",
"author": "Sakura Downloader",
"description": "X (Twitter) media via Playwright + httpx. Session in runtime/x_profile.",
"requires_auth": true,
"tags": ["social", "media", "playwright"],
"min_platform_version": "1.0.0",
"dependencies": []
}
+946
View File
@@ -0,0 +1,946 @@
"""Playwright helpers for the X plugin."""
from __future__ import annotations
import asyncio
import json
import random
import re
from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin, urlparse
import httpx
from playwright.async_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError, async_playwright
from .browser_opts import launch_x_persistent_context
from pixiv_app.core.plugin.base import PluginParseError
try:
from pixiv_app.core.fingerprint import default_profile
except Exception: # pragma: no cover
default_profile = None
_GUEST_WALL_HINT = (
"X 需要先登录。\n"
"请在“插件管理”里点击“X 登录”完成登录,或把 Cookie 导出为 JSON 保存到 "
"runtime/x_cookies.json 后重试。"
)
if TYPE_CHECKING:
pass
class XPlaywrightClient:
def __init__(self, config: Any) -> None:
self.config = config
self._pw: Any = None
self.context: BrowserContext | None = None
self._response_store: dict[int, list[dict[str, Any]]] = {}
self._response_tasks: dict[int, set[asyncio.Task[Any]]] = {}
async def start(self) -> None:
if self.context is not None:
return
self._pw = await async_playwright().start()
profile = Path(self.config.profile_dir)
profile.mkdir(parents=True, exist_ok=True)
self.context = await launch_x_persistent_context(
self._pw,
profile,
headless=self.config.headless,
)
cookies = self._prepare_cookie_items(self._load_cookie_items())
if cookies:
loaded = await self._add_cookies_tolerant(cookies)
if loaded <= 0:
raise PluginParseError(
"X Cookie 注入失败:请重新导出 x.com cookies.txt,确认包含 auth_token 和 ct0 后再导入。"
)
async def close(self) -> None:
if self.context is not None:
await self.context.close()
self.context = None
self._response_tasks.clear()
self._response_store.clear()
if self._pw is not None:
await self._pw.stop()
self._pw = None
async def new_page_goto(self, url: str, *, wait_until: str = "domcontentloaded") -> Page:
assert self.context is not None
page = await self.context.new_page()
self._attach_response_collector(page)
timeout = getattr(self.config, "wait_timeout", 30000)
try:
await page.goto(url, wait_until=wait_until, timeout=timeout)
except PlaywrightTimeoutError as exc:
if page.url and page.url != "about:blank":
await asyncio.sleep(2.0)
return page
try:
await page.goto(url, wait_until="commit", timeout=min(timeout, 15000))
except PlaywrightTimeoutError as retry_exc:
raise PluginParseError(
"X 页面加载超时,可能是网络较慢、访问过快触发风控,或登录会话已失效。"
) from retry_exc
except Exception as retry_exc:
raise PluginParseError(f"X 页面加载失败: {retry_exc}") from retry_exc
if page.url == "about:blank":
raise PluginParseError("X 页面没有成功打开,请稍后重试。") from exc
await self.human_pause(2.0, 4.5)
await self.move_mouse_softly(page)
return page
async def human_pause(self, low: float | None = None, high: float | None = None) -> None:
low = float(low if low is not None else getattr(self.config, "min_action_delay", 3.0))
high = float(high if high is not None else getattr(self.config, "max_action_delay", 8.0))
if high < low:
high = low
await asyncio.sleep(random.uniform(low, high))
async def move_mouse_softly(self, page: Page) -> None:
try:
viewport = page.viewport_size or {"width": 1280, "height": 800}
width = int(viewport.get("width", 1280))
height = int(viewport.get("height", 800))
for _ in range(random.randint(1, 3)):
x = random.randint(max(12, width // 8), max(20, width - 24))
y = random.randint(max(12, height // 8), max(20, height - 24))
await page.mouse.move(x, y, steps=random.randint(8, 18))
await asyncio.sleep(random.uniform(0.2, 0.7))
except Exception:
pass
async def human_scroll_page(self, page: Page) -> float:
await self.move_mouse_softly(page)
viewport = page.viewport_size or {"height": 800}
height = int(viewport.get("height", 800))
delta = random.randint(max(260, height // 2), max(420, int(height * 1.2)))
if random.random() < 0.12:
delta = -random.randint(120, 280)
await page.mouse.wheel(0, delta)
if random.random() < 0.07:
await asyncio.sleep(random.uniform(30.0, 90.0))
await self.human_pause()
return float(await page.evaluate("document.body.scrollHeight"))
async def raise_if_guest_wall(self, page: Page) -> None:
await asyncio.sleep(0.45)
try:
url = page.url or ""
if "/i/flow/login" in url or "/i/flow/signup" in url:
raise PluginParseError(_GUEST_WALL_HINT)
except PluginParseError:
raise
except Exception:
pass
try:
wall = page.get_by_text(
re.compile(
r"注册 X|Sign up for X|登录|Sign in|Phone, email, or username",
re.I,
)
).first
if await wall.is_visible(timeout=2200):
raise PluginParseError(_GUEST_WALL_HINT)
except PluginParseError:
raise
except Exception:
pass
try:
dialog = page.locator('[role="dialog"]').first
if await dialog.is_visible(timeout=600):
inner = (await dialog.inner_text())[:800]
if ("注册" in inner or "Sign up" in inner) and ("Google" in inner or "Apple" in inner or "Next" in inner):
raise PluginParseError(_GUEST_WALL_HINT)
except PluginParseError:
raise
except Exception:
pass
async def wait_for_main_content(self, page: Page) -> None:
timeout = min(15000, getattr(self.config, "wait_timeout", 30000))
for sel in ('article[data-testid="tweet"]', "article", '[data-testid="tweetPhoto"]'):
try:
await page.wait_for_selector(sel, timeout=timeout)
return
except Exception:
continue
async def wait_for_media(self, page: Page) -> bool:
timeout = 10000
for sel in (
'[data-testid="tweetPhoto"]',
'img[src*="pbs.twimg.com/media"]',
"video",
"video source",
):
try:
await page.wait_for_selector(sel, timeout=timeout)
return True
except Exception:
continue
return False
async def extract_media_urls(self, page: Page) -> list[str]:
urls: list[str] = []
seen: set[str] = set()
imgs = await page.query_selector_all('img[src*="pbs.twimg.com/media"]')
for img in imgs:
src = await img.get_attribute("src")
if src and "profile_images" not in src:
orig = self._original_tw_image_url(src)
if orig and orig not in seen:
seen.add(orig)
urls.append(orig)
for sel in ('[data-testid="tweetPhoto"] img', "article img"):
elements = await page.query_selector_all(sel)
for el in elements:
src = await el.get_attribute("src")
if src and "pbs.twimg.com/media" in src:
orig = self._original_tw_image_url(src)
if orig and orig not in seen:
seen.add(orig)
urls.append(orig)
try:
raw_video_urls = await page.evaluate(
"""
() => {
const values = [];
for (const el of document.querySelectorAll('video, video source')) {
const candidates = [
el.currentSrc,
el.src,
el.getAttribute?.('src'),
el.getAttribute?.('data-src'),
el.getAttribute?.('poster'),
];
for (const value of candidates) {
if (typeof value === 'string' && value.trim()) {
values.push(value.trim());
}
}
}
return values;
}
"""
)
except Exception:
raw_video_urls = []
for src in raw_video_urls:
normalized = self._normalize_media_url(str(src))
if normalized and self._is_video_url(normalized) and normalized not in seen:
seen.add(normalized)
urls.append(normalized)
try:
html = await page.content()
except Exception:
html = ""
for image_url in self._extract_image_urls_from_html(html):
if image_url not in seen:
seen.add(image_url)
urls.append(image_url)
for pattern in (
r"https?://video\.twimg\.com/[^\s\"'<>\\]+",
r"https:\\/\\/video\.twimg\.com\\/[^\s\"'<>]+",
):
for match in re.findall(pattern, html):
normalized = self._normalize_media_url(match)
if normalized and self._is_video_url(normalized) and normalized not in seen:
seen.add(normalized)
urls.append(normalized)
return urls
async def extract_media_urls_from_api_responses(self, page: Page) -> list[str]:
await self.wait_for_response_capture(page)
payloads = self._response_store.get(id(page), [])
if not payloads:
return []
urls: list[str] = []
seen: set[str] = set()
def add(url: str | None) -> None:
if not url:
return
normalized = self._normalize_media_url(str(url))
if not normalized:
return
if "pbs.twimg.com/media" in normalized:
normalized = self._original_tw_image_url(normalized)
if normalized not in seen:
seen.add(normalized)
urls.append(normalized)
for payload in payloads:
data = payload.get("json")
if data is None:
continue
for media_item in self._find_media_entities(data):
media_url = media_item.get("media_url_https") or media_item.get("media_url")
media_type = str(media_item.get("type") or "").lower()
if media_url and media_type in {"photo", "animated_gif", "video", ""}:
add(str(media_url))
video_info = media_item.get("video_info")
if isinstance(video_info, dict):
best_video = self._pick_best_video_variant(video_info)
add(best_video)
return urls
async def extract_media_records_from_api_responses(self, page: Page) -> list[dict[str, Any]]:
await self.wait_for_response_capture(page)
payloads = self._response_store.get(id(page), [])
if not payloads:
return []
records: list[dict[str, Any]] = []
seen: set[tuple[str, str]] = set()
def add_record(kind: str, url: str | None, *, thumbnail: str | None = None) -> None:
if not url:
return
normalized = self._normalize_media_url(str(url))
if not normalized:
return
key = (kind, normalized)
if key in seen:
return
seen.add(key)
records.append(
{
"type": kind,
"url": normalized,
"thumbnail": self._normalize_media_url(thumbnail) if thumbnail else None,
}
)
for payload in payloads:
data = payload.get("json")
if data is None:
continue
for media_item in self._find_media_entities(data):
media_type = str(media_item.get("type") or "").lower()
thumb_url = media_item.get("media_url_https") or media_item.get("media_url")
if media_type == "photo":
add_record("image", thumb_url)
continue
video_info = media_item.get("video_info")
if isinstance(video_info, dict):
best_video = self._pick_best_video_variant(video_info)
if best_video:
add_record("video", best_video, thumbnail=thumb_url)
continue
return records
async def wait_for_response_capture(self, page: Page, timeout: float = 2.0) -> None:
page_id = id(page)
tasks = self._response_tasks.get(page_id)
if not tasks:
return
deadline = asyncio.get_running_loop().time() + timeout
while tasks and asyncio.get_running_loop().time() < deadline:
pending = [task for task in list(tasks) if not task.done()]
if not pending:
break
_, pending = await asyncio.wait(pending, timeout=0.2)
tasks = self._response_tasks.get(page_id)
if not tasks:
break
async def scroll_and_collect_media(self, page: Page, max_scrolls: int = 12, max_items: int = 40) -> list[str]:
collected: list[str] = []
seen: set[str] = set()
last_height = 0.0
for _ in range(max_scrolls):
batch: list[str] = []
for item in await self.extract_media_records_from_api_responses(page):
url = item.get("url")
if isinstance(url, str) and url:
batch.append(url)
batch.extend(await self.extract_media_urls(page))
for url in batch:
if url not in seen:
seen.add(url)
collected.append(url)
if len(collected) >= max_items:
return collected
try:
height = await self.human_scroll_page(page)
except Exception:
break
if height == last_height:
break
last_height = float(height)
return collected
async def scroll_and_collect_tweet_urls(self, page: Page, limit: int = 40) -> list[str]:
found: list[str] = []
seen: set[str] = set()
stale_rounds = 0
max_rounds = 25
for _ in range(max_rounds):
if len(found) >= limit:
break
before = len(found)
links = await page.query_selector_all('a[href*="/status/"]')
for link in links:
href = await link.get_attribute("href")
if not href or "/status/" not in href:
continue
if href.startswith("/"):
full = f"https://x.com{href}"
elif href.startswith("http"):
full = href
else:
continue
full = re.sub(r"\?.*$", "", full.split("#")[0])
if full not in seen:
seen.add(full)
found.append(full)
if len(found) >= limit:
break
if len(found) == before:
stale_rounds += 1
if stale_rounds >= 5:
break
else:
stale_rounds = 0
try:
await self.human_scroll_page(page)
except Exception:
break
return found[:limit]
async def get_author_info(self, page: Page) -> dict[str, str]:
info = {"name": "", "username": ""}
try:
block = await page.query_selector('[data-testid="User-Name"]')
if block:
text = await block.inner_text()
lines = [line.strip() for line in text.split("\n") if line.strip()]
if lines:
info["name"] = lines[0]
for line in lines:
if line.startswith("@"):
info["username"] = line.lstrip("@")
break
except Exception:
pass
return info
async def get_user_display_name(self, page: Page, fallback: str) -> dict[str, str]:
info = {"name": fallback, "username": fallback}
try:
block = await page.query_selector('[data-testid="UserName"]')
if block:
text = await block.inner_text()
lines = [line.strip() for line in text.split("\n") if line.strip()]
if lines:
info["name"] = lines[0]
except Exception:
pass
return info
async def get_tweet_text(self, page: Page) -> str:
try:
el = await page.query_selector('[data-testid="tweetText"]')
if el:
return (await el.inner_text()).strip()
except Exception:
pass
return ""
async def get_tweet_date(self, page: Page) -> str:
try:
time_el = await page.query_selector("article time")
if time_el:
return (await time_el.get_attribute("datetime")) or ""
except Exception:
pass
return ""
async def download_image(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
save_path.mkdir(parents=True, exist_ok=True)
name = url.split("/")[-1].split("?")[0]
if not any(name.lower().endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp")):
name += ".jpg"
target = save_path / name
if target.exists() and target.stat().st_size > 256:
return target
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
content_type = response.headers.get("content-type", "").lower()
if response.status_code == 200 and "text/html" not in content_type:
target.write_bytes(response.content)
if target.stat().st_size > 256:
return target
target.unlink(missing_ok=True)
return None
async def download_video(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
save_path.mkdir(parents=True, exist_ok=True)
async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
if response.status_code != 200:
return None
final_url = str(response.url)
content_type = response.headers.get("content-type", "").lower()
body_text = response.text if self._looks_like_hls(final_url, content_type, "") or "text" in content_type else ""
if "text/html" in content_type and not self._looks_like_hls(final_url, content_type, body_text):
return None
if self._looks_like_hls(final_url, content_type, body_text):
return await self._download_hls_playlist(
client,
playlist_url=final_url,
playlist_text=body_text,
save_path=save_path,
referer=referer,
)
target = self._video_target_path(final_url, save_path)
if target.exists() and target.stat().st_size > 1024:
return target
target.write_bytes(response.content)
if target.stat().st_size <= 1024:
target.unlink(missing_ok=True)
return None
return target
async def _download_hls_playlist(
self,
client: httpx.AsyncClient,
*,
playlist_url: str,
playlist_text: str,
save_path: Path,
referer: str | None,
) -> Path | None:
selected_url, selected_text = await self._resolve_hls_playlist(
client,
playlist_url=playlist_url,
playlist_text=playlist_text,
referer=referer,
)
if not selected_text:
return None
segment_urls, init_url = self._parse_hls_segments(selected_url, selected_text)
if not segment_urls:
return None
target = self._video_target_path(selected_url, save_path, segment_urls=segment_urls, init_url=init_url)
if target.exists() and target.stat().st_size > 1024:
return target
with target.open("wb") as fh:
if init_url:
init_resp = await client.get(init_url, headers=self._media_request_headers(init_url, referer=referer))
if init_resp.status_code != 200:
target.unlink(missing_ok=True)
return None
fh.write(init_resp.content)
for segment_url in segment_urls:
seg_resp = await client.get(
segment_url,
headers=self._media_request_headers(segment_url, referer=referer),
)
if seg_resp.status_code != 200:
target.unlink(missing_ok=True)
return None
fh.write(seg_resp.content)
if target.stat().st_size <= 1024:
target.unlink(missing_ok=True)
return None
return target
async def _resolve_hls_playlist(
self,
client: httpx.AsyncClient,
*,
playlist_url: str,
playlist_text: str,
referer: str | None,
) -> tuple[str, str]:
if "#EXT-X-STREAM-INF" not in playlist_text:
return playlist_url, playlist_text
variants = self._parse_hls_variants(playlist_url, playlist_text)
for variant_url in variants:
resp = await client.get(variant_url, headers=self._media_request_headers(variant_url, referer=referer))
if resp.status_code == 200:
return str(resp.url), resp.text
return playlist_url, playlist_text
def _parse_hls_variants(self, playlist_url: str, playlist_text: str) -> list[str]:
variants: list[tuple[int, str]] = []
pending_bandwidth = 0
for raw_line in playlist_text.splitlines():
line = raw_line.strip()
if not line:
continue
if line.startswith("#EXT-X-STREAM-INF"):
match = re.search(r"BANDWIDTH=(\d+)", line)
pending_bandwidth = int(match.group(1)) if match else 0
continue
if line.startswith("#"):
continue
variants.append((pending_bandwidth, urljoin(playlist_url, line)))
pending_bandwidth = 0
variants.sort(key=lambda item: item[0], reverse=True)
return [url for _, url in variants]
def _parse_hls_segments(self, playlist_url: str, playlist_text: str) -> tuple[list[str], str | None]:
segments: list[str] = []
init_url: str | None = None
for raw_line in playlist_text.splitlines():
line = raw_line.strip()
if not line:
continue
if line.startswith("#EXT-X-MAP:"):
match = re.search(r'URI="([^"]+)"', line)
if match:
init_url = urljoin(playlist_url, match.group(1))
continue
if line.startswith("#"):
continue
segments.append(urljoin(playlist_url, line))
return segments, init_url
def _media_request_headers(self, url: str, *, referer: str | None = None) -> dict[str, str]:
headers = {
"User-Agent": self._ua(),
"Accept": "*/*",
"Referer": referer or "https://x.com/",
"Origin": "https://x.com",
}
cookie_header = self._cookie_header(url)
if cookie_header:
headers["Cookie"] = cookie_header
csrf_token = self._cookie_value("ct0")
if csrf_token:
headers["x-csrf-token"] = csrf_token
return headers
def _cookie_header(self, url: str) -> str:
pairs: list[str] = []
for item in self._load_cookie_items():
name = str(item.get("name") or "").strip()
value = str(item.get("value") or "")
if name:
pairs.append(f"{name}={value}")
return "; ".join(pairs)
def _cookie_value(self, name: str) -> str:
for item in self._load_cookie_items():
if str(item.get("name") or "") == name:
return str(item.get("value") or "")
return ""
def _load_cookie_items(self) -> list[dict[str, Any]]:
cookie_path = Path(getattr(self.config, "cookie_file", "") or "")
if not cookie_path.is_file():
return []
try:
payload = json.loads(cookie_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return []
return payload if isinstance(payload, list) else []
async def _add_cookies_tolerant(self, cookies: list[dict[str, Any]]) -> int:
assert self.context is not None
try:
await self.context.add_cookies(cookies)
return len(cookies)
except Exception:
pass
loaded = 0
for cookie in cookies:
try:
await self.context.add_cookies([cookie])
loaded += 1
except Exception:
continue
return loaded
@staticmethod
def _prepare_cookie_items(cookies: list[dict[str, Any]]) -> list[dict[str, Any]]:
prepared_by_key: dict[tuple[str, str, str], dict[str, Any]] = {}
for cookie in cookies:
name = str(cookie.get("name") or "").strip()
domain = str(cookie.get("domain") or "").strip()
if not name or not domain:
continue
item: dict[str, Any] = {
"name": name,
"value": str(cookie.get("value") or ""),
"domain": domain,
"path": str(cookie.get("path") or "/"),
"secure": bool(cookie.get("secure", False)),
"httpOnly": bool(cookie.get("httpOnly", False)),
}
expires = cookie.get("expires")
if expires is not None:
try:
expires_value = int(float(expires))
if expires_value > 0:
item["expires"] = expires_value
except (TypeError, ValueError):
pass
same_site = str(cookie.get("sameSite") or "").strip()
if same_site in {"Strict", "Lax", "None"}:
if same_site == "None" and not item["secure"]:
item["sameSite"] = "Lax"
else:
item["sameSite"] = same_site
key = (domain.lower(), str(item["path"]), name)
prepared_by_key[key] = item
return list(prepared_by_key.values())
def _video_target_path(
self,
url: str,
save_path: Path,
*,
segment_urls: list[str] | None = None,
init_url: str | None = None,
) -> Path:
parsed = urlparse(url)
name = Path(parsed.path).name or "x_video"
if name.endswith(".m3u8") or not Path(name).suffix:
if segment_urls:
seg_name = Path(urlparse(segment_urls[0]).path).name
if seg_name:
name = seg_name
if init_url:
init_name = Path(urlparse(init_url).path).name
if init_name:
name = init_name
suffix = Path(name).suffix.lower()
if suffix in {".m3u8", ".m4s", ".mpd", ""}:
name = f"{Path(name).stem or 'x_video'}.mp4"
return save_path / name
def _looks_like_hls(self, url: str, content_type: str, body: str) -> bool:
low_url = url.lower()
if ".m3u8" in low_url:
return True
if "mpegurl" in content_type or "vnd.apple.mpegurl" in content_type:
return True
return "#EXTM3U" in body[:256]
def _attach_response_collector(self, page: Page) -> None:
page_id = id(page)
bucket: list[dict[str, Any]] = []
self._response_store[page_id] = bucket
self._response_tasks[page_id] = set()
def handle_response(response: Any) -> None:
task = asyncio.create_task(self._capture_response(page, response))
task_bucket = self._response_tasks.setdefault(page_id, set())
task_bucket.add(task)
def cleanup(done_task: asyncio.Task[Any]) -> None:
current_bucket = self._response_tasks.get(page_id)
if current_bucket is not None:
current_bucket.discard(done_task)
task.add_done_callback(cleanup)
page.on("response", handle_response)
async def _capture_response(self, page: Page, response: Any) -> None:
url = str(getattr(response, "url", "") or "")
if not self._should_capture_response(url):
return
try:
headers = await response.all_headers()
except Exception:
headers = {}
content_type = str(headers.get("content-type", "")).lower()
if "json" not in content_type and "javascript" not in content_type and "text/plain" not in content_type:
return
try:
text = await response.text()
except Exception:
return
if not text or len(text) < 2:
return
parsed_json: Any = None
try:
parsed_json = json.loads(text)
except Exception:
return
self._response_store.setdefault(id(page), []).append(
{
"url": url,
"status": getattr(response, "status", None),
"json": parsed_json,
}
)
def _should_capture_response(self, url: str) -> bool:
low = url.lower()
return any(
token in low
for token in (
"/i/api/graphql/",
"/graphql/",
"tweetdetail",
"tweetresultbyrestid",
"conversation",
"adaptive.json",
)
)
def _find_media_entities(self, payload: Any) -> list[dict[str, Any]]:
media_entities: list[dict[str, Any]] = []
def walk(node: Any) -> None:
if isinstance(node, dict):
if "media_url_https" in node or "video_info" in node:
media_entities.append(node)
for value in node.values():
walk(value)
return
if isinstance(node, list):
for item in node:
walk(item)
walk(payload)
return media_entities
def _pick_best_video_variant(self, video_info: dict[str, Any]) -> str | None:
variants = video_info.get("variants")
if not isinstance(variants, list):
return None
best_mp4: tuple[int, str] | None = None
best_hls: str | None = None
for item in variants:
if not isinstance(item, dict):
continue
url = item.get("url")
if not isinstance(url, str) or not url:
continue
low_url = url.lower()
content_type = str(item.get("content_type") or "").lower()
try:
bitrate = int(item.get("bitrate") or 0)
except (TypeError, ValueError):
bitrate = 0
if "mp4" in content_type or ".mp4" in low_url:
if best_mp4 is None or bitrate > best_mp4[0]:
best_mp4 = (bitrate, url)
elif "mpegurl" in content_type or ".m3u8" in low_url:
best_hls = url
if best_mp4 is not None:
return best_mp4[1]
return best_hls
def _extract_image_urls_from_html(self, html: str) -> list[str]:
if not html:
return []
seen: set[str] = set()
urls: list[str] = []
patterns = (
r"https?://pbs\.twimg\.com/media/[^\s\"'<>\\]+",
r"https:\\/\\/pbs\.twimg\.com\\/media\\/[^\s\"'<>]+",
)
for pattern in patterns:
for match in re.findall(pattern, html):
normalized = self._normalize_media_url(match)
if "pbs.twimg.com/media" not in normalized:
continue
orig = self._original_tw_image_url(normalized)
if orig not in seen:
seen.add(orig)
urls.append(orig)
return urls
def _ua(self) -> str:
if default_profile:
return default_profile("x").user_agent
return (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
@staticmethod
def _original_tw_image_url(url: str) -> str:
if "pbs.twimg.com/media" not in url:
return url
base = url.split("?")[0]
return f"{base}?format=jpg&name=orig" if "format=" not in url else re.sub(
r"name=\w+",
"name=orig",
url,
)
@staticmethod
def _normalize_media_url(url: str) -> str:
value = url.strip().replace("\\/", "/")
if value.startswith("//"):
value = f"https:{value}"
return value
@staticmethod
def _is_video_url(url: str) -> bool:
low = url.lower()
return any(
token in low
for token in (
".mp4",
".m3u8",
"video.twimg.com",
"/amplify_video/",
"/ext_tw_video/",
)
)
+504
View File
@@ -0,0 +1,504 @@
"""
X (Twitter) plugin.
Uses Playwright for rendering and httpx for media downloads.
"""
from __future__ import annotations
import asyncio
import json
import queue
import random
import re
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from .playwright_client import XPlaywrightClient
from .x_login import run_interactive_login as x_login_flow
from .x_urls import (
extract_search_query,
extract_tweet_id,
extract_username_and_kind,
is_search_url,
is_status_url,
normalize_url,
)
from pixiv_app.core.paths import runtime_path
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
@dataclass
class XPluginConfig:
cookie_file: str = ""
profile_dir: str = ""
headless: bool = False
safety_mode: bool = True
wait_timeout: int = 45000
timeline_tweet_limit: int = 8
media_scroll_rounds: int = 5
max_media_items: int = 40
parse_cooldown_seconds: float = 60.0
min_action_delay: float = 4.0
max_action_delay: float = 11.0
download_delay_min: float = 2.0
download_delay_max: float = 5.0
batch_pause_min: float = 12.0
batch_pause_max: float = 28.0
batch_pause_after_min: int = 2
batch_pause_after_max: int = 4
max_parse_per_hour: int = 24
max_download_files_per_hour: int = 60
max_consecutive_failures: int = 3
failure_cooldown_seconds: float = 1800.0
sync_parse_timeout_seconds: float = 120.0
class XPlugin(BasePlugin):
name = "X"
domain = "twitter.com"
version = "1.0.0"
_HANDLE_PATTERNS = (
re.compile(r"(?:twitter|x)\.com/\w+/status/\d+", re.I),
re.compile(r"(?:twitter|x)\.com/[^/]+/media\b", re.I),
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/?$", re.I),
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/media/?$", re.I),
re.compile(r"(?:twitter|x)\.com/search\?", re.I),
)
def __init__(self) -> None:
self.config = XPluginConfig(
cookie_file=str(runtime_path("x_cookies.json")),
profile_dir=str(runtime_path("x_profile")),
headless=False,
)
self._cache_file = runtime_path("x_download_cache.json")
self._playwright_lock = threading.Lock()
self._last_parse_at = 0.0
self._parse_timestamps: list[float] = []
self._download_timestamps: list[float] = []
self._consecutive_failures = 0
self._suspended_until = 0.0
def can_handle(self, url: str) -> bool:
text = url.strip()
if not text:
return False
candidate = text if "://" in text else f"https://{text.lstrip('/')}"
try:
hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".")
except ValueError:
return False
if hostname not in {"x.com", "www.x.com", "twitter.com", "www.twitter.com", "mobile.twitter.com"}:
return False
return any(pattern.search(text) for pattern in self._HANDLE_PATTERNS)
def validate(self) -> bool:
try:
import playwright # noqa: F401
except ImportError:
return False
return True
def get_headers(self) -> dict[str, str]:
return {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
}
def parse(self, url: str) -> list[Resource]:
self._enforce_account_safety("parse", units=1)
try:
with self._playwright_lock:
self._respect_parse_cooldown()
try:
result = asyncio.run(self._parse_async(normalize_url(url)))
self._record_success()
self._record_usage(self._parse_timestamps, 1)
return result
finally:
self._last_parse_at = time.monotonic()
except PluginParseError:
self._record_failure()
raise
except Exception as exc:
self._record_failure()
raise PluginParseError(f"X 解析失败: {exc}") from exc
def parse_for_sync(self, url: str) -> list[Resource]:
self._enforce_account_safety("parse", units=1)
timeout = max(30.0, float(self.config.sync_parse_timeout_seconds))
try:
with self._playwright_lock:
result = asyncio.run(asyncio.wait_for(self._parse_async(normalize_url(url)), timeout=timeout))
self._record_success()
self._record_usage(self._parse_timestamps, 1)
self._last_parse_at = time.monotonic()
return result
except asyncio.TimeoutError as exc:
self._record_failure()
raise PluginParseError(f"X 同步解析超时({int(timeout)} 秒),请稍后重试或检查登录状态。") from exc
except PluginParseError:
self._record_failure()
raise
except Exception as exc:
self._record_failure()
raise PluginParseError(f"X 同步解析失败: {exc}") from exc
def download(self, resource: Resource, save_path: Path) -> list[Path]:
requested = max(1, len(resource.files))
self._enforce_account_safety("download", units=requested)
try:
with self._playwright_lock:
result = asyncio.run(self._download_async(resource, save_path))
self._record_success()
self._record_usage(self._download_timestamps, max(1, len(result) or requested))
return result
except Exception as exc:
self._record_failure()
raise PluginParseError(f"X 下载失败: {exc}") from exc
def reset_playwright_session(self) -> None:
return None
def _respect_parse_cooldown(self) -> None:
elapsed = time.monotonic() - self._last_parse_at
remain = float(self.config.parse_cooldown_seconds) - elapsed
if 0 < remain < self.config.parse_cooldown_seconds:
time.sleep(remain)
def _enforce_account_safety(self, action: str, *, units: int) -> None:
if not self.config.safety_mode:
return
now = time.monotonic()
if self._suspended_until > now:
remaining = int(self._suspended_until - now)
minutes = max(1, remaining // 60)
raise PluginParseError(f"X 账号安全模式已暂停操作,请约 {minutes} 分钟后再试。")
self._trim_usage(now)
if action == "parse":
if len(self._parse_timestamps) + units > self.config.max_parse_per_hour:
raise PluginParseError("X 账号安全模式已触发每小时解析上限,请稍后再试。")
return
if action == "download":
if len(self._download_timestamps) + units > self.config.max_download_files_per_hour:
raise PluginParseError("X 账号安全模式已触发每小时下载上限,请稍后再试。")
def _trim_usage(self, now: float | None = None) -> None:
current = now if now is not None else time.monotonic()
window = 3600.0
self._parse_timestamps = [item for item in self._parse_timestamps if current - item < window]
self._download_timestamps = [item for item in self._download_timestamps if current - item < window]
def _record_usage(self, bucket: list[float], units: int) -> None:
now = time.monotonic()
self._trim_usage(now)
for _ in range(max(1, units)):
bucket.append(now)
def _record_success(self) -> None:
self._consecutive_failures = 0
def _record_failure(self) -> None:
if not self.config.safety_mode:
return
self._consecutive_failures += 1
if self._consecutive_failures >= self.config.max_consecutive_failures:
self._suspended_until = time.monotonic() + float(self.config.failure_cooldown_seconds)
self._consecutive_failures = 0
async def _parse_async(self, url: str) -> list[Resource]:
if not self.validate():
raise PluginParseError("未安装 Playwright。请先执行: pip install playwright && playwright install chromium")
client = XPlaywrightClient(self.config)
await client.start()
try:
if is_status_url(url):
return await self._parse_single_tweet(client, url)
if is_search_url(url):
return await self._parse_search_media(client, url)
user, kind = extract_username_and_kind(url)
if user and kind == "media":
return await self._parse_user_media(client, url, user)
if user and kind == "profile":
return await self._parse_user_timeline(client, url, user)
raise PluginParseError(f"无法识别的 X 链接: {url}")
finally:
await client.close()
async def _parse_single_tweet(self, client: XPlaywrightClient, url: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
await client.wait_for_main_content(page)
await client.wait_for_media(page)
try:
await page.wait_for_load_state("networkidle", timeout=5000)
except Exception:
pass
tweet_id = extract_tweet_id(url) or ""
author = await client.get_author_info(page)
media_records = await client.extract_media_records_from_api_responses(page)
media_urls = await client.extract_media_urls(page)
if not media_urls:
try:
await page.mouse.wheel(0, 500)
await asyncio.sleep(1.2)
except Exception:
pass
media_urls = await client.extract_media_urls(page)
if not media_records and not media_urls:
media_records = await client.extract_media_records_from_api_responses(page)
if not media_urls:
media_urls = [item["url"] for item in media_records] or await client.extract_media_urls_from_api_responses(page)
content = await client.get_tweet_text(page)
created = await client.get_tweet_date(page)
finally:
await page.close()
if not media_records and not media_urls:
raise PluginParseError("未在该推文解析到媒体,可能需要重新登录,或该页面结构已变化。")
resources: list[Resource] = []
if media_records:
for index, item in enumerate(media_records):
file_url = item["url"]
thumb = item.get("thumbnail") or file_url
resources.append(
Resource(
id=f"{tweet_id}_{index}",
url=url,
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
author=author.get("name", ""),
author_id=author.get("username", ""),
files=[{"url": file_url, "type": item.get("type", self._media_type(file_url)), "referer": url}],
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
thumbnail=thumb,
created_at=created or None,
)
)
return resources
for index, media_url in enumerate(media_urls):
resources.append(
Resource(
id=f"{tweet_id}_{index}",
url=url,
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
author=author.get("name", ""),
author_id=author.get("username", ""),
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
thumbnail=media_urls[0] if index == 0 else None,
created_at=created or None,
)
)
return resources
async def _parse_search_media(self, client: XPlaywrightClient, url: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
media_urls = await client.scroll_and_collect_media(
page,
max_scrolls=self.config.media_scroll_rounds,
max_items=self.config.max_media_items,
)
finally:
await page.close()
query = extract_search_query(url) or "search"
resources: list[Resource] = []
for index, media_url in enumerate(media_urls):
resources.append(
Resource(
id=f"search_{abs(hash(query))}_{index}",
url=url,
title=f"X search - {query}",
author=query,
author_id="search",
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
metadata={"source": "search_media", "query": query},
thumbnail=media_url,
created_at=None,
)
)
return resources
async def _parse_user_media(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
info = await client.get_user_display_name(page, username)
media_urls = await client.scroll_and_collect_media(
page,
max_scrolls=self.config.media_scroll_rounds,
max_items=self.config.max_media_items,
)
finally:
await page.close()
resources: list[Resource] = []
for index, media_url in enumerate(media_urls):
resources.append(
Resource(
id=f"{username}_media_{index}",
url=url,
title=f"{info.get('name', username)} - media",
author=info.get("name", username),
author_id=username,
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
metadata={"source": "media_tab", "username": username},
thumbnail=media_url,
created_at=None,
)
)
return resources
async def _parse_user_timeline(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
tweet_urls = await client.scroll_and_collect_tweet_urls(
page,
limit=self.config.timeline_tweet_limit,
)
finally:
await page.close()
all_results: list[Resource] = []
for tweet_url in tweet_urls:
try:
all_results.extend(await self._parse_single_tweet(client, tweet_url))
except Exception:
continue
return all_results
async def _download_async(self, resource: Resource, save_path: Path) -> list[Path]:
client = XPlaywrightClient(self.config)
results: list[Path] = []
subdir = resource.metadata.get("save_subdir")
target_base = save_path / subdir if subdir else save_path
cache = self._load_download_cache()
batch_pause_after = random.randint(self.config.batch_pause_after_min, self.config.batch_pause_after_max)
try:
for index, item in enumerate(resource.files, start=1):
file_url = item.get("url")
if not file_url:
continue
cached = cache.get(file_url)
if cached and Path(cached).is_file():
results.append(Path(cached))
continue
kind = item.get("type", "image")
referer = item.get("referer") or resource.url
if kind == "video":
saved = await client.download_video(file_url, target_base, referer=referer)
else:
saved = await client.download_image(file_url, target_base, referer=referer)
if saved is None:
continue
cache[file_url] = str(saved)
results.append(saved)
await asyncio.sleep(
random.uniform(self.config.download_delay_min, self.config.download_delay_max)
* random.uniform(0.8, 1.5)
)
if index % batch_pause_after == 0:
await asyncio.sleep(random.uniform(self.config.batch_pause_min, self.config.batch_pause_max))
batch_pause_after = random.randint(
self.config.batch_pause_after_min,
self.config.batch_pause_after_max,
)
self._save_download_cache(cache)
return results
finally:
await client.close()
def _load_download_cache(self) -> dict[str, str]:
try:
if self._cache_file.is_file():
payload = json.loads(self._cache_file.read_text(encoding="utf-8"))
if isinstance(payload, dict):
return {str(key): str(value) for key, value in payload.items()}
except Exception:
pass
return {}
def _save_download_cache(self, payload: dict[str, str]) -> None:
self._cache_file.parent.mkdir(parents=True, exist_ok=True)
self._cache_file.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def run_interactive_login(
self,
*,
proceed: threading.Event | None = None,
ready_signal: queue.Queue[bool] | None = None,
) -> bool:
path = Path(self.config.profile_dir)
try:
with self._playwright_lock:
return asyncio.run(x_login_flow(path, proceed=proceed, ready_signal=ready_signal))
except Exception:
return False
def set_headless(self, headless: bool) -> None:
self.config.headless = headless
@staticmethod
def _media_type(url: str) -> str:
low = url.lower()
if (
"video" in low
or ".mp4" in low
or ".m3u8" in low
or "video.twimg.com" in low
or "/amplify_video/" in low
or "/ext_tw_video/" in low
):
return "video"
return "image"
def get_config_schema(self) -> dict[str, Any]:
return {
"headless": {"type": "bool", "label": "无头模式", "default": False},
"safety_mode": {"type": "bool", "label": "账号安全模式", "default": True},
"timeline_tweet_limit": {"type": "int", "label": "时间线采样条数", "default": 8, "min": 1, "max": 30},
"media_scroll_rounds": {"type": "int", "label": "媒体页滚动轮数", "default": 5, "min": 1, "max": 12},
"max_media_items": {"type": "int", "label": "单次最多媒体数", "default": 40, "min": 1, "max": 120},
"parse_cooldown_seconds": {"type": "float", "label": "解析冷却秒数", "default": 60.0, "min": 10, "max": 600},
"min_action_delay": {"type": "float", "label": "最小操作延迟秒数", "default": 4.0, "min": 1, "max": 60},
"max_action_delay": {"type": "float", "label": "最大操作延迟秒数", "default": 11.0, "min": 2, "max": 90},
"download_delay_min": {"type": "float", "label": "下载后最小停顿秒数", "default": 2.0, "min": 0, "max": 60},
"download_delay_max": {"type": "float", "label": "下载后最大停顿秒数", "default": 5.0, "min": 0, "max": 120},
"max_parse_per_hour": {"type": "int", "label": "每小时最大解析次数", "default": 24, "min": 1, "max": 120},
"max_download_files_per_hour": {"type": "int", "label": "每小时最大下载文件数", "default": 60, "min": 1, "max": 300},
"max_consecutive_failures": {"type": "int", "label": "连续失败暂停阈值", "default": 3, "min": 1, "max": 10},
"failure_cooldown_seconds": {"type": "float", "label": "连续失败暂停秒数", "default": 1800.0, "min": 60, "max": 86400},
}
plugin_class = XPlugin
+8
View File
@@ -0,0 +1,8 @@
"""DOM selectors for X pages used by Sakura desktop sync."""
PROFILE_LINK = 'a[data-testid="AppTabBar_Profile_Link"]'
USER_CELL = '[data-testid="UserCell"]'
USER_NAME = '[data-testid="UserName"]'
USER_AVATAR = 'img[src*="profile_images"]'
USER_LINK = 'a[href^="/"][role="link"]'
FOLLOWING_URL_SUFFIX = "/following"
+18
View File
@@ -0,0 +1,18 @@
"""Install Python deps and Chromium for the X plugin."""
from __future__ import annotations
import subprocess
import sys
def main() -> None:
deps = ["playwright", "httpx"]
for dep in deps:
subprocess.check_call([sys.executable, "-m", "pip", "install", dep])
subprocess.check_call([sys.executable, "-m", "playwright", "install", "chromium"])
print("Done. Restart the app and reload plugins.")
if __name__ == "__main__":
main()
+59
View File
@@ -0,0 +1,59 @@
"""Interactive X login via persistent Chromium profile."""
from __future__ import annotations
import asyncio
import queue
import shutil
import threading
from pathlib import Path
from playwright.async_api import async_playwright
from .browser_opts import launch_x_persistent_context
from pixiv_app.services.x_login_flow import goto_x_login_entry
async def run_interactive_login(
profile_dir: Path,
*,
proceed: threading.Event | None = None,
ready_signal: queue.Queue[bool] | None = None,
) -> bool:
"""
Open visible Chromium with persistent ``profile_dir``.
Optionally notify GUI via ``ready_signal`` after the login page loads, then block until
``proceed`` is set (e.g. user clicked OK after logging in). If ``proceed`` is None,
falls back to terminal ``input()``.
"""
profile_dir.mkdir(parents=True, exist_ok=True)
playwright = await async_playwright().start()
context = None
loop = asyncio.get_running_loop()
try:
context = await launch_x_persistent_context(playwright, profile_dir, headless=False)
page = context.pages[0] if context.pages else await context.new_page()
await goto_x_login_entry(page, timeout=60_000)
if ready_signal is not None:
ready_signal.put(True)
if proceed is not None:
await loop.run_in_executor(None, proceed.wait)
else:
await loop.run_in_executor(
None,
lambda: input("\n在浏览器中完成 X 登录后,回到控制台按 Enter...\n"),
)
return True
finally:
if context is not None:
await context.close()
await playwright.stop()
def clear_profile(profile_dir: Path) -> None:
if profile_dir.exists():
shutil.rmtree(profile_dir, ignore_errors=True)
profile_dir.mkdir(parents=True, exist_ok=True)
+57
View File
@@ -0,0 +1,57 @@
"""URL helpers for X / Twitter."""
from __future__ import annotations
import re
from urllib.parse import parse_qs, urlparse
STATUS_RE = re.compile(
r"(?:https?://)?(?:www\.)?(?:twitter|x)\.com/[^/]+/status/(\d+)",
re.IGNORECASE,
)
PROFILE_OR_MEDIA_RE = re.compile(
r"(?:https?://)?(?:www\.)?(?:twitter|x)\.com/([A-Za-z0-9_]{1,15})(?:/(media))?/?$",
re.IGNORECASE,
)
def normalize_url(url: str) -> str:
u = url.strip()
if not u.startswith("http"):
u = "https://" + u
return u
def extract_tweet_id(url: str) -> str | None:
m = STATUS_RE.search(url)
return m.group(1) if m else None
def extract_username_and_kind(url: str) -> tuple[str | None, str | None]:
u = normalize_url(url)
m = PROFILE_OR_MEDIA_RE.match(u)
if not m:
return None, None
user = m.group(1)
media_flag = m.group(2)
if user.lower() in {"home", "explore", "settings", "messages", "notifications", "i", "intent", "search"}:
return None, None
if media_flag:
return user, "media"
return user, "profile"
def is_status_url(url: str) -> bool:
return extract_tweet_id(url) is not None
def is_search_url(url: str) -> bool:
parsed = urlparse(normalize_url(url))
host = parsed.netloc.lower()
return host in {"x.com", "www.x.com", "twitter.com", "www.twitter.com"} and parsed.path.rstrip("/") == "/search"
def extract_search_query(url: str) -> str:
parsed = urlparse(normalize_url(url))
values = parse_qs(parsed.query).get("q") or []
return values[0] if values else ""
+2
View File
@@ -0,0 +1,2 @@
# plugins/xiaohongshu/__init__.py
"""小红书 (Xiaohongshu) 爬虫插件 - 换IP + 指纹模拟版本"""
File diff suppressed because it is too large. Load diff