Files
SakuraHub/plugins/x/plugin.py
T

504 lines
20 KiB
Python
Raw Normal View History

2026-11-16 20:45:22 +08:00
"""
X (Twitter) plugin.
Uses Playwright for rendering and httpx for media downloads.
"""
from __future__ import annotations
import asyncio
import json
import queue
import random
import re
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from .playwright_client import XPlaywrightClient
from .x_login import run_interactive_login as x_login_flow
from .x_urls import (
extract_search_query,
extract_tweet_id,
extract_username_and_kind,
is_search_url,
is_status_url,
normalize_url,
)
from pixiv_app.core.paths import runtime_path
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
@dataclass
class XPluginConfig:
cookie_file: str = ""
profile_dir: str = ""
headless: bool = False
safety_mode: bool = True
wait_timeout: int = 45000
timeline_tweet_limit: int = 8
media_scroll_rounds: int = 5
max_media_items: int = 40
parse_cooldown_seconds: float = 60.0
min_action_delay: float = 4.0
max_action_delay: float = 11.0
download_delay_min: float = 2.0
download_delay_max: float = 5.0
batch_pause_min: float = 12.0
batch_pause_max: float = 28.0
batch_pause_after_min: int = 2
batch_pause_after_max: int = 4
max_parse_per_hour: int = 24
max_download_files_per_hour: int = 60
max_consecutive_failures: int = 3
failure_cooldown_seconds: float = 1800.0
sync_parse_timeout_seconds: float = 120.0
class XPlugin(BasePlugin):
name = "X"
domain = "twitter.com"
version = "1.0.0"
_HANDLE_PATTERNS = (
re.compile(r"(?:twitter|x)\.com/\w+/status/\d+", re.I),
re.compile(r"(?:twitter|x)\.com/[^/]+/media\b", re.I),
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/?$", re.I),
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/media/?$", re.I),
re.compile(r"(?:twitter|x)\.com/search\?", re.I),
)
def __init__(self) -> None:
self.config = XPluginConfig(
cookie_file=str(runtime_path("x_cookies.json")),
profile_dir=str(runtime_path("x_profile")),
headless=False,
)
self._cache_file = runtime_path("x_download_cache.json")
self._playwright_lock = threading.Lock()
self._last_parse_at = 0.0
self._parse_timestamps: list[float] = []
self._download_timestamps: list[float] = []
self._consecutive_failures = 0
self._suspended_until = 0.0
def can_handle(self, url: str) -> bool:
text = url.strip()
if not text:
return False
candidate = text if "://" in text else f"https://{text.lstrip('/')}"
try:
hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".")
except ValueError:
return False
if hostname not in {"x.com", "www.x.com", "twitter.com", "www.twitter.com", "mobile.twitter.com"}:
return False
return any(pattern.search(text) for pattern in self._HANDLE_PATTERNS)
def validate(self) -> bool:
try:
import playwright # noqa: F401
except ImportError:
return False
return True
def get_headers(self) -> dict[str, str]:
return {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
}
def parse(self, url: str) -> list[Resource]:
self._enforce_account_safety("parse", units=1)
try:
with self._playwright_lock:
self._respect_parse_cooldown()
try:
result = asyncio.run(self._parse_async(normalize_url(url)))
self._record_success()
self._record_usage(self._parse_timestamps, 1)
return result
finally:
self._last_parse_at = time.monotonic()
except PluginParseError:
self._record_failure()
raise
except Exception as exc:
self._record_failure()
raise PluginParseError(f"X 解析失败: {exc}") from exc
def parse_for_sync(self, url: str) -> list[Resource]:
self._enforce_account_safety("parse", units=1)
timeout = max(30.0, float(self.config.sync_parse_timeout_seconds))
try:
with self._playwright_lock:
result = asyncio.run(asyncio.wait_for(self._parse_async(normalize_url(url)), timeout=timeout))
self._record_success()
self._record_usage(self._parse_timestamps, 1)
self._last_parse_at = time.monotonic()
return result
except asyncio.TimeoutError as exc:
self._record_failure()
raise PluginParseError(f"X 同步解析超时({int(timeout)} 秒),请稍后重试或检查登录状态。") from exc
except PluginParseError:
self._record_failure()
raise
except Exception as exc:
self._record_failure()
raise PluginParseError(f"X 同步解析失败: {exc}") from exc
def download(self, resource: Resource, save_path: Path) -> list[Path]:
requested = max(1, len(resource.files))
self._enforce_account_safety("download", units=requested)
try:
with self._playwright_lock:
result = asyncio.run(self._download_async(resource, save_path))
self._record_success()
self._record_usage(self._download_timestamps, max(1, len(result) or requested))
return result
except Exception as exc:
self._record_failure()
raise PluginParseError(f"X 下载失败: {exc}") from exc
def reset_playwright_session(self) -> None:
return None
def _respect_parse_cooldown(self) -> None:
elapsed = time.monotonic() - self._last_parse_at
remain = float(self.config.parse_cooldown_seconds) - elapsed
if 0 < remain < self.config.parse_cooldown_seconds:
time.sleep(remain)
def _enforce_account_safety(self, action: str, *, units: int) -> None:
if not self.config.safety_mode:
return
now = time.monotonic()
if self._suspended_until > now:
remaining = int(self._suspended_until - now)
minutes = max(1, remaining // 60)
raise PluginParseError(f"X 账号安全模式已暂停操作,请约 {minutes} 分钟后再试。")
self._trim_usage(now)
if action == "parse":
if len(self._parse_timestamps) + units > self.config.max_parse_per_hour:
raise PluginParseError("X 账号安全模式已触发每小时解析上限,请稍后再试。")
return
if action == "download":
if len(self._download_timestamps) + units > self.config.max_download_files_per_hour:
raise PluginParseError("X 账号安全模式已触发每小时下载上限,请稍后再试。")
def _trim_usage(self, now: float | None = None) -> None:
current = now if now is not None else time.monotonic()
window = 3600.0
self._parse_timestamps = [item for item in self._parse_timestamps if current - item < window]
self._download_timestamps = [item for item in self._download_timestamps if current - item < window]
def _record_usage(self, bucket: list[float], units: int) -> None:
now = time.monotonic()
self._trim_usage(now)
for _ in range(max(1, units)):
bucket.append(now)
def _record_success(self) -> None:
self._consecutive_failures = 0
def _record_failure(self) -> None:
if not self.config.safety_mode:
return
self._consecutive_failures += 1
if self._consecutive_failures >= self.config.max_consecutive_failures:
self._suspended_until = time.monotonic() + float(self.config.failure_cooldown_seconds)
self._consecutive_failures = 0
async def _parse_async(self, url: str) -> list[Resource]:
if not self.validate():
raise PluginParseError("未安装 Playwright。请先执行: pip install playwright && playwright install chromium")
client = XPlaywrightClient(self.config)
await client.start()
try:
if is_status_url(url):
return await self._parse_single_tweet(client, url)
if is_search_url(url):
return await self._parse_search_media(client, url)
user, kind = extract_username_and_kind(url)
if user and kind == "media":
return await self._parse_user_media(client, url, user)
if user and kind == "profile":
return await self._parse_user_timeline(client, url, user)
raise PluginParseError(f"无法识别的 X 链接: {url}")
finally:
await client.close()
async def _parse_single_tweet(self, client: XPlaywrightClient, url: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
await client.wait_for_main_content(page)
await client.wait_for_media(page)
try:
await page.wait_for_load_state("networkidle", timeout=5000)
except Exception:
pass
tweet_id = extract_tweet_id(url) or ""
author = await client.get_author_info(page)
media_records = await client.extract_media_records_from_api_responses(page)
media_urls = await client.extract_media_urls(page)
if not media_urls:
try:
await page.mouse.wheel(0, 500)
await asyncio.sleep(1.2)
except Exception:
pass
media_urls = await client.extract_media_urls(page)
if not media_records and not media_urls:
media_records = await client.extract_media_records_from_api_responses(page)
if not media_urls:
media_urls = [item["url"] for item in media_records] or await client.extract_media_urls_from_api_responses(page)
content = await client.get_tweet_text(page)
created = await client.get_tweet_date(page)
finally:
await page.close()
if not media_records and not media_urls:
raise PluginParseError("未在该推文解析到媒体,可能需要重新登录,或该页面结构已变化。")
resources: list[Resource] = []
if media_records:
for index, item in enumerate(media_records):
file_url = item["url"]
thumb = item.get("thumbnail") or file_url
resources.append(
Resource(
id=f"{tweet_id}_{index}",
url=url,
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
author=author.get("name", ""),
author_id=author.get("username", ""),
files=[{"url": file_url, "type": item.get("type", self._media_type(file_url)), "referer": url}],
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
thumbnail=thumb,
created_at=created or None,
)
)
return resources
for index, media_url in enumerate(media_urls):
resources.append(
Resource(
id=f"{tweet_id}_{index}",
url=url,
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
author=author.get("name", ""),
author_id=author.get("username", ""),
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
thumbnail=media_urls[0] if index == 0 else None,
created_at=created or None,
)
)
return resources
async def _parse_search_media(self, client: XPlaywrightClient, url: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
media_urls = await client.scroll_and_collect_media(
page,
max_scrolls=self.config.media_scroll_rounds,
max_items=self.config.max_media_items,
)
finally:
await page.close()
query = extract_search_query(url) or "search"
resources: list[Resource] = []
for index, media_url in enumerate(media_urls):
resources.append(
Resource(
id=f"search_{abs(hash(query))}_{index}",
url=url,
title=f"X search - {query}",
author=query,
author_id="search",
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
metadata={"source": "search_media", "query": query},
thumbnail=media_url,
created_at=None,
)
)
return resources
async def _parse_user_media(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
info = await client.get_user_display_name(page, username)
media_urls = await client.scroll_and_collect_media(
page,
max_scrolls=self.config.media_scroll_rounds,
max_items=self.config.max_media_items,
)
finally:
await page.close()
resources: list[Resource] = []
for index, media_url in enumerate(media_urls):
resources.append(
Resource(
id=f"{username}_media_{index}",
url=url,
title=f"{info.get('name', username)} - media",
author=info.get("name", username),
author_id=username,
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
metadata={"source": "media_tab", "username": username},
thumbnail=media_url,
created_at=None,
)
)
return resources
async def _parse_user_timeline(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
tweet_urls = await client.scroll_and_collect_tweet_urls(
page,
limit=self.config.timeline_tweet_limit,
)
finally:
await page.close()
all_results: list[Resource] = []
for tweet_url in tweet_urls:
try:
all_results.extend(await self._parse_single_tweet(client, tweet_url))
except Exception:
continue
return all_results
async def _download_async(self, resource: Resource, save_path: Path) -> list[Path]:
client = XPlaywrightClient(self.config)
results: list[Path] = []
subdir = resource.metadata.get("save_subdir")
target_base = save_path / subdir if subdir else save_path
cache = self._load_download_cache()
batch_pause_after = random.randint(self.config.batch_pause_after_min, self.config.batch_pause_after_max)
try:
for index, item in enumerate(resource.files, start=1):
file_url = item.get("url")
if not file_url:
continue
cached = cache.get(file_url)
if cached and Path(cached).is_file():
results.append(Path(cached))
continue
kind = item.get("type", "image")
referer = item.get("referer") or resource.url
if kind == "video":
saved = await client.download_video(file_url, target_base, referer=referer)
else:
saved = await client.download_image(file_url, target_base, referer=referer)
if saved is None:
continue
cache[file_url] = str(saved)
results.append(saved)
await asyncio.sleep(
random.uniform(self.config.download_delay_min, self.config.download_delay_max)
* random.uniform(0.8, 1.5)
)
if index % batch_pause_after == 0:
await asyncio.sleep(random.uniform(self.config.batch_pause_min, self.config.batch_pause_max))
batch_pause_after = random.randint(
self.config.batch_pause_after_min,
self.config.batch_pause_after_max,
)
self._save_download_cache(cache)
return results
finally:
await client.close()
def _load_download_cache(self) -> dict[str, str]:
try:
if self._cache_file.is_file():
payload = json.loads(self._cache_file.read_text(encoding="utf-8"))
if isinstance(payload, dict):
return {str(key): str(value) for key, value in payload.items()}
except Exception:
pass
return {}
def _save_download_cache(self, payload: dict[str, str]) -> None:
self._cache_file.parent.mkdir(parents=True, exist_ok=True)
self._cache_file.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def run_interactive_login(
self,
*,
proceed: threading.Event | None = None,
ready_signal: queue.Queue[bool] | None = None,
) -> bool:
path = Path(self.config.profile_dir)
try:
with self._playwright_lock:
return asyncio.run(x_login_flow(path, proceed=proceed, ready_signal=ready_signal))
except Exception:
return False
def set_headless(self, headless: bool) -> None:
self.config.headless = headless
@staticmethod
def _media_type(url: str) -> str:
low = url.lower()
if (
"video" in low
or ".mp4" in low
or ".m3u8" in low
or "video.twimg.com" in low
or "/amplify_video/" in low
or "/ext_tw_video/" in low
):
return "video"
return "image"
def get_config_schema(self) -> dict[str, Any]:
return {
"headless": {"type": "bool", "label": "无头模式", "default": False},
"safety_mode": {"type": "bool", "label": "账号安全模式", "default": True},
"timeline_tweet_limit": {"type": "int", "label": "时间线采样条数", "default": 8, "min": 1, "max": 30},
"media_scroll_rounds": {"type": "int", "label": "媒体页滚动轮数", "default": 5, "min": 1, "max": 12},
"max_media_items": {"type": "int", "label": "单次最多媒体数", "default": 40, "min": 1, "max": 120},
"parse_cooldown_seconds": {"type": "float", "label": "解析冷却秒数", "default": 60.0, "min": 10, "max": 600},
"min_action_delay": {"type": "float", "label": "最小操作延迟秒数", "default": 4.0, "min": 1, "max": 60},
"max_action_delay": {"type": "float", "label": "最大操作延迟秒数", "default": 11.0, "min": 2, "max": 90},
"download_delay_min": {"type": "float", "label": "下载后最小停顿秒数", "default": 2.0, "min": 0, "max": 60},
"download_delay_max": {"type": "float", "label": "下载后最大停顿秒数", "default": 5.0, "min": 0, "max": 120},
"max_parse_per_hour": {"type": "int", "label": "每小时最大解析次数", "default": 24, "min": 1, "max": 120},
"max_download_files_per_hour": {"type": "int", "label": "每小时最大下载文件数", "default": 60, "min": 1, "max": 300},
"max_consecutive_failures": {"type": "int", "label": "连续失败暂停阈值", "default": 3, "min": 1, "max": 10},
"failure_cooldown_seconds": {"type": "float", "label": "连续失败暂停秒数", "default": 1800.0, "min": 60, "max": 86400},
}
plugin_class = XPlugin