"""Playwright helpers for the X plugin.""" from __future__ import annotations import asyncio import json import random import re from pathlib import Path from typing import TYPE_CHECKING, Any from urllib.parse import urljoin, urlparse import httpx from playwright.async_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError, async_playwright from .browser_opts import launch_x_persistent_context from pixiv_app.core.plugin.base import PluginParseError try: from pixiv_app.core.fingerprint import default_profile except Exception: # pragma: no cover default_profile = None _GUEST_WALL_HINT = ( "X 需要先登录。\n" "请在“插件管理”里点击“X 登录”完成登录,或把 Cookie 导出为 JSON 保存到 " "runtime/x_cookies.json 后重试。" ) if TYPE_CHECKING: pass class XPlaywrightClient: def __init__(self, config: Any) -> None: self.config = config self._pw: Any = None self.context: BrowserContext | None = None self._response_store: dict[int, list[dict[str, Any]]] = {} self._response_tasks: dict[int, set[asyncio.Task[Any]]] = {} async def start(self) -> None: if self.context is not None: return self._pw = await async_playwright().start() profile = Path(self.config.profile_dir) profile.mkdir(parents=True, exist_ok=True) self.context = await launch_x_persistent_context( self._pw, profile, headless=self.config.headless, ) cookies = self._prepare_cookie_items(self._load_cookie_items()) if cookies: loaded = await self._add_cookies_tolerant(cookies) if loaded <= 0: raise PluginParseError( "X Cookie 注入失败:请重新导出 x.com cookies.txt,确认包含 auth_token 和 ct0 后再导入。" ) async def close(self) -> None: if self.context is not None: await self.context.close() self.context = None self._response_tasks.clear() self._response_store.clear() if self._pw is not None: await self._pw.stop() self._pw = None async def new_page_goto(self, url: str, *, wait_until: str = "domcontentloaded") -> Page: assert self.context is not None page = await self.context.new_page() self._attach_response_collector(page) timeout = getattr(self.config, "wait_timeout", 30000) try: await page.goto(url, wait_until=wait_until, timeout=timeout) except PlaywrightTimeoutError as exc: if page.url and page.url != "about:blank": await asyncio.sleep(2.0) return page try: await page.goto(url, wait_until="commit", timeout=min(timeout, 15000)) except PlaywrightTimeoutError as retry_exc: raise PluginParseError( "X 页面加载超时,可能是网络较慢、访问过快触发风控,或登录会话已失效。" ) from retry_exc except Exception as retry_exc: raise PluginParseError(f"X 页面加载失败: {retry_exc}") from retry_exc if page.url == "about:blank": raise PluginParseError("X 页面没有成功打开,请稍后重试。") from exc await self.human_pause(2.0, 4.5) await self.move_mouse_softly(page) return page async def human_pause(self, low: float | None = None, high: float | None = None) -> None: low = float(low if low is not None else getattr(self.config, "min_action_delay", 3.0)) high = float(high if high is not None else getattr(self.config, "max_action_delay", 8.0)) if high < low: high = low await asyncio.sleep(random.uniform(low, high)) async def move_mouse_softly(self, page: Page) -> None: try: viewport = page.viewport_size or {"width": 1280, "height": 800} width = int(viewport.get("width", 1280)) height = int(viewport.get("height", 800)) for _ in range(random.randint(1, 3)): x = random.randint(max(12, width // 8), max(20, width - 24)) y = random.randint(max(12, height // 8), max(20, height - 24)) await page.mouse.move(x, y, steps=random.randint(8, 18)) await asyncio.sleep(random.uniform(0.2, 0.7)) except Exception: pass async def human_scroll_page(self, page: Page) -> float: await self.move_mouse_softly(page) viewport = page.viewport_size or {"height": 800} height = int(viewport.get("height", 800)) delta = random.randint(max(260, height // 2), max(420, int(height * 1.2))) if random.random() < 0.12: delta = -random.randint(120, 280) await page.mouse.wheel(0, delta) if random.random() < 0.07: await asyncio.sleep(random.uniform(30.0, 90.0)) await self.human_pause() return float(await page.evaluate("document.body.scrollHeight")) async def raise_if_guest_wall(self, page: Page) -> None: await asyncio.sleep(0.45) try: url = page.url or "" if "/i/flow/login" in url or "/i/flow/signup" in url: raise PluginParseError(_GUEST_WALL_HINT) except PluginParseError: raise except Exception: pass try: wall = page.get_by_text( re.compile( r"注册 X|Sign up for X|登录|Sign in|Phone, email, or username", re.I, ) ).first if await wall.is_visible(timeout=2200): raise PluginParseError(_GUEST_WALL_HINT) except PluginParseError: raise except Exception: pass try: dialog = page.locator('[role="dialog"]').first if await dialog.is_visible(timeout=600): inner = (await dialog.inner_text())[:800] if ("注册" in inner or "Sign up" in inner) and ("Google" in inner or "Apple" in inner or "Next" in inner): raise PluginParseError(_GUEST_WALL_HINT) except PluginParseError: raise except Exception: pass async def wait_for_main_content(self, page: Page) -> None: timeout = min(15000, getattr(self.config, "wait_timeout", 30000)) for sel in ('article[data-testid="tweet"]', "article", '[data-testid="tweetPhoto"]'): try: await page.wait_for_selector(sel, timeout=timeout) return except Exception: continue async def wait_for_media(self, page: Page) -> bool: timeout = 10000 for sel in ( '[data-testid="tweetPhoto"]', 'img[src*="pbs.twimg.com/media"]', "video", "video source", ): try: await page.wait_for_selector(sel, timeout=timeout) return True except Exception: continue return False async def extract_media_urls(self, page: Page) -> list[str]: urls: list[str] = [] seen: set[str] = set() imgs = await page.query_selector_all('img[src*="pbs.twimg.com/media"]') for img in imgs: src = await img.get_attribute("src") if src and "profile_images" not in src: orig = self._original_tw_image_url(src) if orig and orig not in seen: seen.add(orig) urls.append(orig) for sel in ('[data-testid="tweetPhoto"] img', "article img"): elements = await page.query_selector_all(sel) for el in elements: src = await el.get_attribute("src") if src and "pbs.twimg.com/media" in src: orig = self._original_tw_image_url(src) if orig and orig not in seen: seen.add(orig) urls.append(orig) try: raw_video_urls = await page.evaluate( """ () => { const values = []; for (const el of document.querySelectorAll('video, video source')) { const candidates = [ el.currentSrc, el.src, el.getAttribute?.('src'), el.getAttribute?.('data-src'), el.getAttribute?.('poster'), ]; for (const value of candidates) { if (typeof value === 'string' && value.trim()) { values.push(value.trim()); } } } return values; } """ ) except Exception: raw_video_urls = [] for src in raw_video_urls: normalized = self._normalize_media_url(str(src)) if normalized and self._is_video_url(normalized) and normalized not in seen: seen.add(normalized) urls.append(normalized) try: html = await page.content() except Exception: html = "" for image_url in self._extract_image_urls_from_html(html): if image_url not in seen: seen.add(image_url) urls.append(image_url) for pattern in ( r"https?://video\.twimg\.com/[^\s\"'<>\\]+", r"https:\\/\\/video\.twimg\.com\\/[^\s\"'<>]+", ): for match in re.findall(pattern, html): normalized = self._normalize_media_url(match) if normalized and self._is_video_url(normalized) and normalized not in seen: seen.add(normalized) urls.append(normalized) return urls async def extract_media_urls_from_api_responses(self, page: Page) -> list[str]: await self.wait_for_response_capture(page) payloads = self._response_store.get(id(page), []) if not payloads: return [] urls: list[str] = [] seen: set[str] = set() def add(url: str | None) -> None: if not url: return normalized = self._normalize_media_url(str(url)) if not normalized: return if "pbs.twimg.com/media" in normalized: normalized = self._original_tw_image_url(normalized) if normalized not in seen: seen.add(normalized) urls.append(normalized) for payload in payloads: data = payload.get("json") if data is None: continue for media_item in self._find_media_entities(data): media_url = media_item.get("media_url_https") or media_item.get("media_url") media_type = str(media_item.get("type") or "").lower() if media_url and media_type in {"photo", "animated_gif", "video", ""}: add(str(media_url)) video_info = media_item.get("video_info") if isinstance(video_info, dict): best_video = self._pick_best_video_variant(video_info) add(best_video) return urls async def extract_media_records_from_api_responses(self, page: Page) -> list[dict[str, Any]]: await self.wait_for_response_capture(page) payloads = self._response_store.get(id(page), []) if not payloads: return [] records: list[dict[str, Any]] = [] seen: set[tuple[str, str]] = set() def add_record(kind: str, url: str | None, *, thumbnail: str | None = None) -> None: if not url: return normalized = self._normalize_media_url(str(url)) if not normalized: return key = (kind, normalized) if key in seen: return seen.add(key) records.append( { "type": kind, "url": normalized, "thumbnail": self._normalize_media_url(thumbnail) if thumbnail else None, } ) for payload in payloads: data = payload.get("json") if data is None: continue for media_item in self._find_media_entities(data): media_type = str(media_item.get("type") or "").lower() thumb_url = media_item.get("media_url_https") or media_item.get("media_url") if media_type == "photo": add_record("image", thumb_url) continue video_info = media_item.get("video_info") if isinstance(video_info, dict): best_video = self._pick_best_video_variant(video_info) if best_video: add_record("video", best_video, thumbnail=thumb_url) continue return records async def wait_for_response_capture(self, page: Page, timeout: float = 2.0) -> None: page_id = id(page) tasks = self._response_tasks.get(page_id) if not tasks: return deadline = asyncio.get_running_loop().time() + timeout while tasks and asyncio.get_running_loop().time() < deadline: pending = [task for task in list(tasks) if not task.done()] if not pending: break _, pending = await asyncio.wait(pending, timeout=0.2) tasks = self._response_tasks.get(page_id) if not tasks: break async def scroll_and_collect_media(self, page: Page, max_scrolls: int = 12, max_items: int = 40) -> list[str]: collected: list[str] = [] seen: set[str] = set() last_height = 0.0 for _ in range(max_scrolls): batch: list[str] = [] for item in await self.extract_media_records_from_api_responses(page): url = item.get("url") if isinstance(url, str) and url: batch.append(url) batch.extend(await self.extract_media_urls(page)) for url in batch: if url not in seen: seen.add(url) collected.append(url) if len(collected) >= max_items: return collected try: height = await self.human_scroll_page(page) except Exception: break if height == last_height: break last_height = float(height) return collected async def scroll_and_collect_tweet_urls(self, page: Page, limit: int = 40) -> list[str]: found: list[str] = [] seen: set[str] = set() stale_rounds = 0 max_rounds = 25 for _ in range(max_rounds): if len(found) >= limit: break before = len(found) links = await page.query_selector_all('a[href*="/status/"]') for link in links: href = await link.get_attribute("href") if not href or "/status/" not in href: continue if href.startswith("/"): full = f"https://x.com{href}" elif href.startswith("http"): full = href else: continue full = re.sub(r"\?.*$", "", full.split("#")[0]) if full not in seen: seen.add(full) found.append(full) if len(found) >= limit: break if len(found) == before: stale_rounds += 1 if stale_rounds >= 5: break else: stale_rounds = 0 try: await self.human_scroll_page(page) except Exception: break return found[:limit] async def get_author_info(self, page: Page) -> dict[str, str]: info = {"name": "", "username": ""} try: block = await page.query_selector('[data-testid="User-Name"]') if block: text = await block.inner_text() lines = [line.strip() for line in text.split("\n") if line.strip()] if lines: info["name"] = lines[0] for line in lines: if line.startswith("@"): info["username"] = line.lstrip("@") break except Exception: pass return info async def get_user_display_name(self, page: Page, fallback: str) -> dict[str, str]: info = {"name": fallback, "username": fallback} try: block = await page.query_selector('[data-testid="UserName"]') if block: text = await block.inner_text() lines = [line.strip() for line in text.split("\n") if line.strip()] if lines: info["name"] = lines[0] except Exception: pass return info async def get_tweet_text(self, page: Page) -> str: try: el = await page.query_selector('[data-testid="tweetText"]') if el: return (await el.inner_text()).strip() except Exception: pass return "" async def get_tweet_date(self, page: Page) -> str: try: time_el = await page.query_selector("article time") if time_el: return (await time_el.get_attribute("datetime")) or "" except Exception: pass return "" async def download_image(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None: save_path.mkdir(parents=True, exist_ok=True) name = url.split("/")[-1].split("?")[0] if not any(name.lower().endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp")): name += ".jpg" target = save_path / name if target.exists() and target.stat().st_size > 256: return target async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client: response = await client.get(url, headers=self._media_request_headers(url, referer=referer)) content_type = response.headers.get("content-type", "").lower() if response.status_code == 200 and "text/html" not in content_type: target.write_bytes(response.content) if target.stat().st_size > 256: return target target.unlink(missing_ok=True) return None async def download_video(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None: save_path.mkdir(parents=True, exist_ok=True) async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client: response = await client.get(url, headers=self._media_request_headers(url, referer=referer)) if response.status_code != 200: return None final_url = str(response.url) content_type = response.headers.get("content-type", "").lower() body_text = response.text if self._looks_like_hls(final_url, content_type, "") or "text" in content_type else "" if "text/html" in content_type and not self._looks_like_hls(final_url, content_type, body_text): return None if self._looks_like_hls(final_url, content_type, body_text): return await self._download_hls_playlist( client, playlist_url=final_url, playlist_text=body_text, save_path=save_path, referer=referer, ) target = self._video_target_path(final_url, save_path) if target.exists() and target.stat().st_size > 1024: return target target.write_bytes(response.content) if target.stat().st_size <= 1024: target.unlink(missing_ok=True) return None return target async def _download_hls_playlist( self, client: httpx.AsyncClient, *, playlist_url: str, playlist_text: str, save_path: Path, referer: str | None, ) -> Path | None: selected_url, selected_text = await self._resolve_hls_playlist( client, playlist_url=playlist_url, playlist_text=playlist_text, referer=referer, ) if not selected_text: return None segment_urls, init_url = self._parse_hls_segments(selected_url, selected_text) if not segment_urls: return None target = self._video_target_path(selected_url, save_path, segment_urls=segment_urls, init_url=init_url) if target.exists() and target.stat().st_size > 1024: return target with target.open("wb") as fh: if init_url: init_resp = await client.get(init_url, headers=self._media_request_headers(init_url, referer=referer)) if init_resp.status_code != 200: target.unlink(missing_ok=True) return None fh.write(init_resp.content) for segment_url in segment_urls: seg_resp = await client.get( segment_url, headers=self._media_request_headers(segment_url, referer=referer), ) if seg_resp.status_code != 200: target.unlink(missing_ok=True) return None fh.write(seg_resp.content) if target.stat().st_size <= 1024: target.unlink(missing_ok=True) return None return target async def _resolve_hls_playlist( self, client: httpx.AsyncClient, *, playlist_url: str, playlist_text: str, referer: str | None, ) -> tuple[str, str]: if "#EXT-X-STREAM-INF" not in playlist_text: return playlist_url, playlist_text variants = self._parse_hls_variants(playlist_url, playlist_text) for variant_url in variants: resp = await client.get(variant_url, headers=self._media_request_headers(variant_url, referer=referer)) if resp.status_code == 200: return str(resp.url), resp.text return playlist_url, playlist_text def _parse_hls_variants(self, playlist_url: str, playlist_text: str) -> list[str]: variants: list[tuple[int, str]] = [] pending_bandwidth = 0 for raw_line in playlist_text.splitlines(): line = raw_line.strip() if not line: continue if line.startswith("#EXT-X-STREAM-INF"): match = re.search(r"BANDWIDTH=(\d+)", line) pending_bandwidth = int(match.group(1)) if match else 0 continue if line.startswith("#"): continue variants.append((pending_bandwidth, urljoin(playlist_url, line))) pending_bandwidth = 0 variants.sort(key=lambda item: item[0], reverse=True) return [url for _, url in variants] def _parse_hls_segments(self, playlist_url: str, playlist_text: str) -> tuple[list[str], str | None]: segments: list[str] = [] init_url: str | None = None for raw_line in playlist_text.splitlines(): line = raw_line.strip() if not line: continue if line.startswith("#EXT-X-MAP:"): match = re.search(r'URI="([^"]+)"', line) if match: init_url = urljoin(playlist_url, match.group(1)) continue if line.startswith("#"): continue segments.append(urljoin(playlist_url, line)) return segments, init_url def _media_request_headers(self, url: str, *, referer: str | None = None) -> dict[str, str]: headers = { "User-Agent": self._ua(), "Accept": "*/*", "Referer": referer or "https://x.com/", "Origin": "https://x.com", } cookie_header = self._cookie_header(url) if cookie_header: headers["Cookie"] = cookie_header csrf_token = self._cookie_value("ct0") if csrf_token: headers["x-csrf-token"] = csrf_token return headers def _cookie_header(self, url: str) -> str: pairs: list[str] = [] for item in self._load_cookie_items(): name = str(item.get("name") or "").strip() value = str(item.get("value") or "") if name: pairs.append(f"{name}={value}") return "; ".join(pairs) def _cookie_value(self, name: str) -> str: for item in self._load_cookie_items(): if str(item.get("name") or "") == name: return str(item.get("value") or "") return "" def _load_cookie_items(self) -> list[dict[str, Any]]: cookie_path = Path(getattr(self.config, "cookie_file", "") or "") if not cookie_path.is_file(): return [] try: payload = json.loads(cookie_path.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): return [] return payload if isinstance(payload, list) else [] async def _add_cookies_tolerant(self, cookies: list[dict[str, Any]]) -> int: assert self.context is not None try: await self.context.add_cookies(cookies) return len(cookies) except Exception: pass loaded = 0 for cookie in cookies: try: await self.context.add_cookies([cookie]) loaded += 1 except Exception: continue return loaded @staticmethod def _prepare_cookie_items(cookies: list[dict[str, Any]]) -> list[dict[str, Any]]: prepared_by_key: dict[tuple[str, str, str], dict[str, Any]] = {} for cookie in cookies: name = str(cookie.get("name") or "").strip() domain = str(cookie.get("domain") or "").strip() if not name or not domain: continue item: dict[str, Any] = { "name": name, "value": str(cookie.get("value") or ""), "domain": domain, "path": str(cookie.get("path") or "/"), "secure": bool(cookie.get("secure", False)), "httpOnly": bool(cookie.get("httpOnly", False)), } expires = cookie.get("expires") if expires is not None: try: expires_value = int(float(expires)) if expires_value > 0: item["expires"] = expires_value except (TypeError, ValueError): pass same_site = str(cookie.get("sameSite") or "").strip() if same_site in {"Strict", "Lax", "None"}: if same_site == "None" and not item["secure"]: item["sameSite"] = "Lax" else: item["sameSite"] = same_site key = (domain.lower(), str(item["path"]), name) prepared_by_key[key] = item return list(prepared_by_key.values()) def _video_target_path( self, url: str, save_path: Path, *, segment_urls: list[str] | None = None, init_url: str | None = None, ) -> Path: parsed = urlparse(url) name = Path(parsed.path).name or "x_video" if name.endswith(".m3u8") or not Path(name).suffix: if segment_urls: seg_name = Path(urlparse(segment_urls[0]).path).name if seg_name: name = seg_name if init_url: init_name = Path(urlparse(init_url).path).name if init_name: name = init_name suffix = Path(name).suffix.lower() if suffix in {".m3u8", ".m4s", ".mpd", ""}: name = f"{Path(name).stem or 'x_video'}.mp4" return save_path / name def _looks_like_hls(self, url: str, content_type: str, body: str) -> bool: low_url = url.lower() if ".m3u8" in low_url: return True if "mpegurl" in content_type or "vnd.apple.mpegurl" in content_type: return True return "#EXTM3U" in body[:256] def _attach_response_collector(self, page: Page) -> None: page_id = id(page) bucket: list[dict[str, Any]] = [] self._response_store[page_id] = bucket self._response_tasks[page_id] = set() def handle_response(response: Any) -> None: task = asyncio.create_task(self._capture_response(page, response)) task_bucket = self._response_tasks.setdefault(page_id, set()) task_bucket.add(task) def cleanup(done_task: asyncio.Task[Any]) -> None: current_bucket = self._response_tasks.get(page_id) if current_bucket is not None: current_bucket.discard(done_task) task.add_done_callback(cleanup) page.on("response", handle_response) async def _capture_response(self, page: Page, response: Any) -> None: url = str(getattr(response, "url", "") or "") if not self._should_capture_response(url): return try: headers = await response.all_headers() except Exception: headers = {} content_type = str(headers.get("content-type", "")).lower() if "json" not in content_type and "javascript" not in content_type and "text/plain" not in content_type: return try: text = await response.text() except Exception: return if not text or len(text) < 2: return parsed_json: Any = None try: parsed_json = json.loads(text) except Exception: return self._response_store.setdefault(id(page), []).append( { "url": url, "status": getattr(response, "status", None), "json": parsed_json, } ) def _should_capture_response(self, url: str) -> bool: low = url.lower() return any( token in low for token in ( "/i/api/graphql/", "/graphql/", "tweetdetail", "tweetresultbyrestid", "conversation", "adaptive.json", ) ) def _find_media_entities(self, payload: Any) -> list[dict[str, Any]]: media_entities: list[dict[str, Any]] = [] def walk(node: Any) -> None: if isinstance(node, dict): if "media_url_https" in node or "video_info" in node: media_entities.append(node) for value in node.values(): walk(value) return if isinstance(node, list): for item in node: walk(item) walk(payload) return media_entities def _pick_best_video_variant(self, video_info: dict[str, Any]) -> str | None: variants = video_info.get("variants") if not isinstance(variants, list): return None best_mp4: tuple[int, str] | None = None best_hls: str | None = None for item in variants: if not isinstance(item, dict): continue url = item.get("url") if not isinstance(url, str) or not url: continue low_url = url.lower() content_type = str(item.get("content_type") or "").lower() try: bitrate = int(item.get("bitrate") or 0) except (TypeError, ValueError): bitrate = 0 if "mp4" in content_type or ".mp4" in low_url: if best_mp4 is None or bitrate > best_mp4[0]: best_mp4 = (bitrate, url) elif "mpegurl" in content_type or ".m3u8" in low_url: best_hls = url if best_mp4 is not None: return best_mp4[1] return best_hls def _extract_image_urls_from_html(self, html: str) -> list[str]: if not html: return [] seen: set[str] = set() urls: list[str] = [] patterns = ( r"https?://pbs\.twimg\.com/media/[^\s\"'<>\\]+", r"https:\\/\\/pbs\.twimg\.com\\/media\\/[^\s\"'<>]+", ) for pattern in patterns: for match in re.findall(pattern, html): normalized = self._normalize_media_url(match) if "pbs.twimg.com/media" not in normalized: continue orig = self._original_tw_image_url(normalized) if orig not in seen: seen.add(orig) urls.append(orig) return urls def _ua(self) -> str: if default_profile: return default_profile("x").user_agent return ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ) @staticmethod def _original_tw_image_url(url: str) -> str: if "pbs.twimg.com/media" not in url: return url base = url.split("?")[0] return f"{base}?format=jpg&name=orig" if "format=" not in url else re.sub( r"name=\w+", "name=orig", url, ) @staticmethod def _normalize_media_url(url: str) -> str: value = url.strip().replace("\\/", "/") if value.startswith("//"): value = f"https:{value}" return value @staticmethod def _is_video_url(url: str) -> bool: low = url.lower() return any( token in low for token in ( ".mp4", ".m3u8", "video.twimg.com", "/amplify_video/", "/ext_tw_video/", ) )