""" X (Twitter) plugin. Uses Playwright for rendering and httpx for media downloads. """ from __future__ import annotations import asyncio import json import queue import random import re import threading import time from dataclasses import dataclass from pathlib import Path from typing import Any from urllib.parse import urlsplit from .playwright_client import XPlaywrightClient from .x_login import run_interactive_login as x_login_flow from .x_urls import ( extract_search_query, extract_tweet_id, extract_username_and_kind, is_search_url, is_status_url, normalize_url, ) from pixiv_app.core.paths import runtime_path from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource @dataclass class XPluginConfig: cookie_file: str = "" profile_dir: str = "" headless: bool = False safety_mode: bool = True wait_timeout: int = 45000 timeline_tweet_limit: int = 8 media_scroll_rounds: int = 5 max_media_items: int = 40 parse_cooldown_seconds: float = 60.0 min_action_delay: float = 4.0 max_action_delay: float = 11.0 download_delay_min: float = 2.0 download_delay_max: float = 5.0 batch_pause_min: float = 12.0 batch_pause_max: float = 28.0 batch_pause_after_min: int = 2 batch_pause_after_max: int = 4 max_parse_per_hour: int = 24 max_download_files_per_hour: int = 60 max_consecutive_failures: int = 3 failure_cooldown_seconds: float = 1800.0 sync_parse_timeout_seconds: float = 120.0 class XPlugin(BasePlugin): name = "X" domain = "twitter.com" version = "1.0.0" _HANDLE_PATTERNS = ( re.compile(r"(?:twitter|x)\.com/\w+/status/\d+", re.I), re.compile(r"(?:twitter|x)\.com/[^/]+/media\b", re.I), re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/?$", re.I), re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/media/?$", re.I), re.compile(r"(?:twitter|x)\.com/search\?", re.I), ) def __init__(self) -> None: self.config = XPluginConfig( cookie_file=str(runtime_path("x_cookies.json")), profile_dir=str(runtime_path("x_profile")), headless=False, ) self._cache_file = runtime_path("x_download_cache.json") self._playwright_lock = threading.Lock() self._last_parse_at = 0.0 self._parse_timestamps: list[float] = [] self._download_timestamps: list[float] = [] self._consecutive_failures = 0 self._suspended_until = 0.0 def can_handle(self, url: str) -> bool: text = url.strip() if not text: return False candidate = text if "://" in text else f"https://{text.lstrip('/')}" try: hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".") except ValueError: return False if hostname not in {"x.com", "www.x.com", "twitter.com", "www.twitter.com", "mobile.twitter.com"}: return False return any(pattern.search(text) for pattern in self._HANDLE_PATTERNS) def validate(self) -> bool: try: import playwright # noqa: F401 except ImportError: return False return True def get_headers(self) -> dict[str, str]: return { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ) } def parse(self, url: str) -> list[Resource]: self._enforce_account_safety("parse", units=1) try: with self._playwright_lock: self._respect_parse_cooldown() try: result = asyncio.run(self._parse_async(normalize_url(url))) self._record_success() self._record_usage(self._parse_timestamps, 1) return result finally: self._last_parse_at = time.monotonic() except PluginParseError: self._record_failure() raise except Exception as exc: self._record_failure() raise PluginParseError(f"X 解析失败: {exc}") from exc def parse_for_sync(self, url: str) -> list[Resource]: self._enforce_account_safety("parse", units=1) timeout = max(30.0, float(self.config.sync_parse_timeout_seconds)) try: with self._playwright_lock: result = asyncio.run(asyncio.wait_for(self._parse_async(normalize_url(url)), timeout=timeout)) self._record_success() self._record_usage(self._parse_timestamps, 1) self._last_parse_at = time.monotonic() return result except asyncio.TimeoutError as exc: self._record_failure() raise PluginParseError(f"X 同步解析超时({int(timeout)} 秒),请稍后重试或检查登录状态。") from exc except PluginParseError: self._record_failure() raise except Exception as exc: self._record_failure() raise PluginParseError(f"X 同步解析失败: {exc}") from exc def download(self, resource: Resource, save_path: Path) -> list[Path]: requested = max(1, len(resource.files)) self._enforce_account_safety("download", units=requested) try: with self._playwright_lock: result = asyncio.run(self._download_async(resource, save_path)) self._record_success() self._record_usage(self._download_timestamps, max(1, len(result) or requested)) return result except Exception as exc: self._record_failure() raise PluginParseError(f"X 下载失败: {exc}") from exc def reset_playwright_session(self) -> None: return None def _respect_parse_cooldown(self) -> None: elapsed = time.monotonic() - self._last_parse_at remain = float(self.config.parse_cooldown_seconds) - elapsed if 0 < remain < self.config.parse_cooldown_seconds: time.sleep(remain) def _enforce_account_safety(self, action: str, *, units: int) -> None: if not self.config.safety_mode: return now = time.monotonic() if self._suspended_until > now: remaining = int(self._suspended_until - now) minutes = max(1, remaining // 60) raise PluginParseError(f"X 账号安全模式已暂停操作,请约 {minutes} 分钟后再试。") self._trim_usage(now) if action == "parse": if len(self._parse_timestamps) + units > self.config.max_parse_per_hour: raise PluginParseError("X 账号安全模式已触发每小时解析上限,请稍后再试。") return if action == "download": if len(self._download_timestamps) + units > self.config.max_download_files_per_hour: raise PluginParseError("X 账号安全模式已触发每小时下载上限,请稍后再试。") def _trim_usage(self, now: float | None = None) -> None: current = now if now is not None else time.monotonic() window = 3600.0 self._parse_timestamps = [item for item in self._parse_timestamps if current - item < window] self._download_timestamps = [item for item in self._download_timestamps if current - item < window] def _record_usage(self, bucket: list[float], units: int) -> None: now = time.monotonic() self._trim_usage(now) for _ in range(max(1, units)): bucket.append(now) def _record_success(self) -> None: self._consecutive_failures = 0 def _record_failure(self) -> None: if not self.config.safety_mode: return self._consecutive_failures += 1 if self._consecutive_failures >= self.config.max_consecutive_failures: self._suspended_until = time.monotonic() + float(self.config.failure_cooldown_seconds) self._consecutive_failures = 0 async def _parse_async(self, url: str) -> list[Resource]: if not self.validate(): raise PluginParseError("未安装 Playwright。请先执行: pip install playwright && playwright install chromium") client = XPlaywrightClient(self.config) await client.start() try: if is_status_url(url): return await self._parse_single_tweet(client, url) if is_search_url(url): return await self._parse_search_media(client, url) user, kind = extract_username_and_kind(url) if user and kind == "media": return await self._parse_user_media(client, url, user) if user and kind == "profile": return await self._parse_user_timeline(client, url, user) raise PluginParseError(f"无法识别的 X 链接: {url}") finally: await client.close() async def _parse_single_tweet(self, client: XPlaywrightClient, url: str) -> list[Resource]: page = await client.new_page_goto(url) try: await client.raise_if_guest_wall(page) await client.wait_for_main_content(page) await client.wait_for_media(page) try: await page.wait_for_load_state("networkidle", timeout=5000) except Exception: pass tweet_id = extract_tweet_id(url) or "" author = await client.get_author_info(page) media_records = await client.extract_media_records_from_api_responses(page) media_urls = await client.extract_media_urls(page) if not media_urls: try: await page.mouse.wheel(0, 500) await asyncio.sleep(1.2) except Exception: pass media_urls = await client.extract_media_urls(page) if not media_records and not media_urls: media_records = await client.extract_media_records_from_api_responses(page) if not media_urls: media_urls = [item["url"] for item in media_records] or await client.extract_media_urls_from_api_responses(page) content = await client.get_tweet_text(page) created = await client.get_tweet_date(page) finally: await page.close() if not media_records and not media_urls: raise PluginParseError("未在该推文解析到媒体,可能需要重新登录,或该页面结构已变化。") resources: list[Resource] = [] if media_records: for index, item in enumerate(media_records): file_url = item["url"] thumb = item.get("thumbnail") or file_url resources.append( Resource( id=f"{tweet_id}_{index}", url=url, title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}", author=author.get("name", ""), author_id=author.get("username", ""), files=[{"url": file_url, "type": item.get("type", self._media_type(file_url)), "referer": url}], metadata={"tweet_id": tweet_id, "full_text": content, "author": author}, thumbnail=thumb, created_at=created or None, ) ) return resources for index, media_url in enumerate(media_urls): resources.append( Resource( id=f"{tweet_id}_{index}", url=url, title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}", author=author.get("name", ""), author_id=author.get("username", ""), files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}], metadata={"tweet_id": tweet_id, "full_text": content, "author": author}, thumbnail=media_urls[0] if index == 0 else None, created_at=created or None, ) ) return resources async def _parse_search_media(self, client: XPlaywrightClient, url: str) -> list[Resource]: page = await client.new_page_goto(url) try: await client.raise_if_guest_wall(page) media_urls = await client.scroll_and_collect_media( page, max_scrolls=self.config.media_scroll_rounds, max_items=self.config.max_media_items, ) finally: await page.close() query = extract_search_query(url) or "search" resources: list[Resource] = [] for index, media_url in enumerate(media_urls): resources.append( Resource( id=f"search_{abs(hash(query))}_{index}", url=url, title=f"X search - {query}", author=query, author_id="search", files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}], metadata={"source": "search_media", "query": query}, thumbnail=media_url, created_at=None, ) ) return resources async def _parse_user_media(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]: page = await client.new_page_goto(url) try: await client.raise_if_guest_wall(page) info = await client.get_user_display_name(page, username) media_urls = await client.scroll_and_collect_media( page, max_scrolls=self.config.media_scroll_rounds, max_items=self.config.max_media_items, ) finally: await page.close() resources: list[Resource] = [] for index, media_url in enumerate(media_urls): resources.append( Resource( id=f"{username}_media_{index}", url=url, title=f"{info.get('name', username)} - media", author=info.get("name", username), author_id=username, files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}], metadata={"source": "media_tab", "username": username}, thumbnail=media_url, created_at=None, ) ) return resources async def _parse_user_timeline(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]: page = await client.new_page_goto(url) try: await client.raise_if_guest_wall(page) tweet_urls = await client.scroll_and_collect_tweet_urls( page, limit=self.config.timeline_tweet_limit, ) finally: await page.close() all_results: list[Resource] = [] for tweet_url in tweet_urls: try: all_results.extend(await self._parse_single_tweet(client, tweet_url)) except Exception: continue return all_results async def _download_async(self, resource: Resource, save_path: Path) -> list[Path]: client = XPlaywrightClient(self.config) results: list[Path] = [] subdir = resource.metadata.get("save_subdir") target_base = save_path / subdir if subdir else save_path cache = self._load_download_cache() batch_pause_after = random.randint(self.config.batch_pause_after_min, self.config.batch_pause_after_max) try: for index, item in enumerate(resource.files, start=1): file_url = item.get("url") if not file_url: continue cached = cache.get(file_url) if cached and Path(cached).is_file(): results.append(Path(cached)) continue kind = item.get("type", "image") referer = item.get("referer") or resource.url if kind == "video": saved = await client.download_video(file_url, target_base, referer=referer) else: saved = await client.download_image(file_url, target_base, referer=referer) if saved is None: continue cache[file_url] = str(saved) results.append(saved) await asyncio.sleep( random.uniform(self.config.download_delay_min, self.config.download_delay_max) * random.uniform(0.8, 1.5) ) if index % batch_pause_after == 0: await asyncio.sleep(random.uniform(self.config.batch_pause_min, self.config.batch_pause_max)) batch_pause_after = random.randint( self.config.batch_pause_after_min, self.config.batch_pause_after_max, ) self._save_download_cache(cache) return results finally: await client.close() def _load_download_cache(self) -> dict[str, str]: try: if self._cache_file.is_file(): payload = json.loads(self._cache_file.read_text(encoding="utf-8")) if isinstance(payload, dict): return {str(key): str(value) for key, value in payload.items()} except Exception: pass return {} def _save_download_cache(self, payload: dict[str, str]) -> None: self._cache_file.parent.mkdir(parents=True, exist_ok=True) self._cache_file.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") def run_interactive_login( self, *, proceed: threading.Event | None = None, ready_signal: queue.Queue[bool] | None = None, ) -> bool: path = Path(self.config.profile_dir) try: with self._playwright_lock: return asyncio.run(x_login_flow(path, proceed=proceed, ready_signal=ready_signal)) except Exception: return False def set_headless(self, headless: bool) -> None: self.config.headless = headless @staticmethod def _media_type(url: str) -> str: low = url.lower() if ( "video" in low or ".mp4" in low or ".m3u8" in low or "video.twimg.com" in low or "/amplify_video/" in low or "/ext_tw_video/" in low ): return "video" return "image" def get_config_schema(self) -> dict[str, Any]: return { "headless": {"type": "bool", "label": "无头模式", "default": False}, "safety_mode": {"type": "bool", "label": "账号安全模式", "default": True}, "timeline_tweet_limit": {"type": "int", "label": "时间线采样条数", "default": 8, "min": 1, "max": 30}, "media_scroll_rounds": {"type": "int", "label": "媒体页滚动轮数", "default": 5, "min": 1, "max": 12}, "max_media_items": {"type": "int", "label": "单次最多媒体数", "default": 40, "min": 1, "max": 120}, "parse_cooldown_seconds": {"type": "float", "label": "解析冷却秒数", "default": 60.0, "min": 10, "max": 600}, "min_action_delay": {"type": "float", "label": "最小操作延迟秒数", "default": 4.0, "min": 1, "max": 60}, "max_action_delay": {"type": "float", "label": "最大操作延迟秒数", "default": 11.0, "min": 2, "max": 90}, "download_delay_min": {"type": "float", "label": "下载后最小停顿秒数", "default": 2.0, "min": 0, "max": 60}, "download_delay_max": {"type": "float", "label": "下载后最大停顿秒数", "default": 5.0, "min": 0, "max": 120}, "max_parse_per_hour": {"type": "int", "label": "每小时最大解析次数", "default": 24, "min": 1, "max": 120}, "max_download_files_per_hour": {"type": "int", "label": "每小时最大下载文件数", "default": 60, "min": 1, "max": 300}, "max_consecutive_failures": {"type": "int", "label": "连续失败暂停阈值", "default": 3, "min": 1, "max": 10}, "failure_cooldown_seconds": {"type": "float", "label": "连续失败暂停秒数", "default": 1800.0, "min": 60, "max": 86400}, } plugin_class = XPlugin