commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1 @@
|
||||
"""X (Twitter) Playwright plugin."""
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Shared Playwright launch options for X — reduce automation fingerprint; prefer system Chrome."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from playwright.async_api import BrowserContext, Playwright
|
||||
|
||||
from pixiv_app.core.playwright_runtime import configure_playwright_runtime_env, preferred_chromium_executable
|
||||
|
||||
try:
|
||||
from pixiv_app.core.fingerprint import default_profile
|
||||
except Exception: # pragma: no cover - plugin can run standalone
|
||||
default_profile = None
|
||||
|
||||
try:
|
||||
from pixiv_app.core.browser_proxy_pool import parse_proxy_url
|
||||
except Exception: # pragma: no cover
|
||||
parse_proxy_url = None
|
||||
|
||||
|
||||
async def launch_x_persistent_context(
|
||||
playwright: Playwright,
|
||||
profile_dir: Path,
|
||||
*,
|
||||
headless: bool,
|
||||
) -> BrowserContext:
|
||||
"""
|
||||
Launch persistent Chromium with flags that reduce obvious automation signals.
|
||||
|
||||
Order: ``X_PLAYWRIGHT_CHANNEL`` env (if set), then system Chrome, then bundled Chromium.
|
||||
Google OAuth often still blocks automation browsers; users should use X email/phone login
|
||||
or import cookies from a normal browser session.
|
||||
"""
|
||||
configure_playwright_runtime_env()
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
browser_profile = default_profile("x") if default_profile else None
|
||||
|
||||
base_kw: dict = {
|
||||
"user_data_dir": str(profile_dir),
|
||||
"headless": headless,
|
||||
"viewport": browser_profile.viewport if browser_profile else {"width": 1280, "height": 800},
|
||||
"locale": browser_profile.locale if browser_profile else "zh-CN",
|
||||
"timezone_id": browser_profile.timezone_id if browser_profile else "Asia/Shanghai",
|
||||
"screen": browser_profile.viewport if browser_profile else {"width": 1280, "height": 800},
|
||||
"extra_http_headers": {
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
},
|
||||
"ignore_default_args": ["--enable-automation"],
|
||||
"args": [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--window-size=1280,800",
|
||||
],
|
||||
"user_agent": browser_profile.user_agent
|
||||
if browser_profile
|
||||
else (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
if browser_profile:
|
||||
base_kw["device_scale_factor"] = browser_profile.device_scale_factor
|
||||
base_kw["is_mobile"] = browser_profile.is_mobile
|
||||
base_kw["has_touch"] = browser_profile.has_touch
|
||||
|
||||
proxy_value = os.environ.get("X_PROXY_URL", os.environ.get("PIXIV_BROWSER_PROXY", "")).strip()
|
||||
if proxy_value and parse_proxy_url:
|
||||
parsed_proxy = parse_proxy_url(proxy_value)
|
||||
if parsed_proxy is not None:
|
||||
base_kw["proxy"] = parsed_proxy.to_playwright_proxy()
|
||||
|
||||
env_ch = os.environ.get("X_PLAYWRIGHT_CHANNEL", "").strip()
|
||||
local_executable = preferred_chromium_executable()
|
||||
seen: set[str] = set()
|
||||
attempts: list[tuple[str, dict]] = []
|
||||
for label, extra in (
|
||||
("env_channel", {"channel": env_ch} if env_ch else {}),
|
||||
("installed_chromium", {"executable_path": str(local_executable)} if local_executable else {}),
|
||||
("system_chrome", {"channel": "chrome"}),
|
||||
("default", {}),
|
||||
):
|
||||
key = f"{label}:{extra!r}"
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
attempts.append((label, extra))
|
||||
|
||||
last_err: Exception | None = None
|
||||
for _, extra in attempts:
|
||||
kw = dict(base_kw)
|
||||
kw.update(extra)
|
||||
try:
|
||||
ctx = await playwright.chromium.launch_persistent_context(**kw)
|
||||
await _apply_stealth_scripts(ctx)
|
||||
return ctx
|
||||
except Exception as exc:
|
||||
last_err = exc
|
||||
continue
|
||||
|
||||
assert last_err is not None
|
||||
raise last_err
|
||||
|
||||
|
||||
async def _apply_stealth_scripts(ctx: BrowserContext) -> None:
|
||||
"""Best-effort scripts; cannot guarantee OAuth providers will accept the browser."""
|
||||
await ctx.add_init_script(
|
||||
"""
|
||||
try {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
} catch (e) {}
|
||||
"""
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "X",
|
||||
"version": "1.0.0",
|
||||
"domain": "twitter.com",
|
||||
"author": "Sakura Downloader",
|
||||
"description": "X (Twitter) media via Playwright + httpx. Session in runtime/x_profile.",
|
||||
"requires_auth": true,
|
||||
"tags": ["social", "media", "playwright"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
@@ -0,0 +1,946 @@
|
||||
"""Playwright helpers for the X plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from playwright.async_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError, async_playwright
|
||||
|
||||
from .browser_opts import launch_x_persistent_context
|
||||
|
||||
from pixiv_app.core.plugin.base import PluginParseError
|
||||
|
||||
try:
|
||||
from pixiv_app.core.fingerprint import default_profile
|
||||
except Exception: # pragma: no cover
|
||||
default_profile = None
|
||||
|
||||
_GUEST_WALL_HINT = (
|
||||
"X 需要先登录。\n"
|
||||
"请在“插件管理”里点击“X 登录”完成登录,或把 Cookie 导出为 JSON 保存到 "
|
||||
"runtime/x_cookies.json 后重试。"
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class XPlaywrightClient:
|
||||
def __init__(self, config: Any) -> None:
|
||||
self.config = config
|
||||
self._pw: Any = None
|
||||
self.context: BrowserContext | None = None
|
||||
self._response_store: dict[int, list[dict[str, Any]]] = {}
|
||||
self._response_tasks: dict[int, set[asyncio.Task[Any]]] = {}
|
||||
|
||||
async def start(self) -> None:
|
||||
if self.context is not None:
|
||||
return
|
||||
self._pw = await async_playwright().start()
|
||||
profile = Path(self.config.profile_dir)
|
||||
profile.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.context = await launch_x_persistent_context(
|
||||
self._pw,
|
||||
profile,
|
||||
headless=self.config.headless,
|
||||
)
|
||||
|
||||
cookies = self._prepare_cookie_items(self._load_cookie_items())
|
||||
if cookies:
|
||||
loaded = await self._add_cookies_tolerant(cookies)
|
||||
if loaded <= 0:
|
||||
raise PluginParseError(
|
||||
"X Cookie 注入失败:请重新导出 x.com cookies.txt,确认包含 auth_token 和 ct0 后再导入。"
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.context is not None:
|
||||
await self.context.close()
|
||||
self.context = None
|
||||
self._response_tasks.clear()
|
||||
self._response_store.clear()
|
||||
if self._pw is not None:
|
||||
await self._pw.stop()
|
||||
self._pw = None
|
||||
|
||||
async def new_page_goto(self, url: str, *, wait_until: str = "domcontentloaded") -> Page:
|
||||
assert self.context is not None
|
||||
page = await self.context.new_page()
|
||||
self._attach_response_collector(page)
|
||||
timeout = getattr(self.config, "wait_timeout", 30000)
|
||||
try:
|
||||
await page.goto(url, wait_until=wait_until, timeout=timeout)
|
||||
except PlaywrightTimeoutError as exc:
|
||||
if page.url and page.url != "about:blank":
|
||||
await asyncio.sleep(2.0)
|
||||
return page
|
||||
try:
|
||||
await page.goto(url, wait_until="commit", timeout=min(timeout, 15000))
|
||||
except PlaywrightTimeoutError as retry_exc:
|
||||
raise PluginParseError(
|
||||
"X 页面加载超时,可能是网络较慢、访问过快触发风控,或登录会话已失效。"
|
||||
) from retry_exc
|
||||
except Exception as retry_exc:
|
||||
raise PluginParseError(f"X 页面加载失败: {retry_exc}") from retry_exc
|
||||
if page.url == "about:blank":
|
||||
raise PluginParseError("X 页面没有成功打开,请稍后重试。") from exc
|
||||
await self.human_pause(2.0, 4.5)
|
||||
await self.move_mouse_softly(page)
|
||||
return page
|
||||
|
||||
async def human_pause(self, low: float | None = None, high: float | None = None) -> None:
|
||||
low = float(low if low is not None else getattr(self.config, "min_action_delay", 3.0))
|
||||
high = float(high if high is not None else getattr(self.config, "max_action_delay", 8.0))
|
||||
if high < low:
|
||||
high = low
|
||||
await asyncio.sleep(random.uniform(low, high))
|
||||
|
||||
async def move_mouse_softly(self, page: Page) -> None:
|
||||
try:
|
||||
viewport = page.viewport_size or {"width": 1280, "height": 800}
|
||||
width = int(viewport.get("width", 1280))
|
||||
height = int(viewport.get("height", 800))
|
||||
for _ in range(random.randint(1, 3)):
|
||||
x = random.randint(max(12, width // 8), max(20, width - 24))
|
||||
y = random.randint(max(12, height // 8), max(20, height - 24))
|
||||
await page.mouse.move(x, y, steps=random.randint(8, 18))
|
||||
await asyncio.sleep(random.uniform(0.2, 0.7))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def human_scroll_page(self, page: Page) -> float:
|
||||
await self.move_mouse_softly(page)
|
||||
viewport = page.viewport_size or {"height": 800}
|
||||
height = int(viewport.get("height", 800))
|
||||
delta = random.randint(max(260, height // 2), max(420, int(height * 1.2)))
|
||||
if random.random() < 0.12:
|
||||
delta = -random.randint(120, 280)
|
||||
await page.mouse.wheel(0, delta)
|
||||
if random.random() < 0.07:
|
||||
await asyncio.sleep(random.uniform(30.0, 90.0))
|
||||
await self.human_pause()
|
||||
return float(await page.evaluate("document.body.scrollHeight"))
|
||||
|
||||
async def raise_if_guest_wall(self, page: Page) -> None:
|
||||
await asyncio.sleep(0.45)
|
||||
try:
|
||||
url = page.url or ""
|
||||
if "/i/flow/login" in url or "/i/flow/signup" in url:
|
||||
raise PluginParseError(_GUEST_WALL_HINT)
|
||||
except PluginParseError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
wall = page.get_by_text(
|
||||
re.compile(
|
||||
r"注册 X|Sign up for X|登录|Sign in|Phone, email, or username",
|
||||
re.I,
|
||||
)
|
||||
).first
|
||||
if await wall.is_visible(timeout=2200):
|
||||
raise PluginParseError(_GUEST_WALL_HINT)
|
||||
except PluginParseError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
dialog = page.locator('[role="dialog"]').first
|
||||
if await dialog.is_visible(timeout=600):
|
||||
inner = (await dialog.inner_text())[:800]
|
||||
if ("注册" in inner or "Sign up" in inner) and ("Google" in inner or "Apple" in inner or "Next" in inner):
|
||||
raise PluginParseError(_GUEST_WALL_HINT)
|
||||
except PluginParseError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def wait_for_main_content(self, page: Page) -> None:
|
||||
timeout = min(15000, getattr(self.config, "wait_timeout", 30000))
|
||||
for sel in ('article[data-testid="tweet"]', "article", '[data-testid="tweetPhoto"]'):
|
||||
try:
|
||||
await page.wait_for_selector(sel, timeout=timeout)
|
||||
return
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
async def wait_for_media(self, page: Page) -> bool:
|
||||
timeout = 10000
|
||||
for sel in (
|
||||
'[data-testid="tweetPhoto"]',
|
||||
'img[src*="pbs.twimg.com/media"]',
|
||||
"video",
|
||||
"video source",
|
||||
):
|
||||
try:
|
||||
await page.wait_for_selector(sel, timeout=timeout)
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
async def extract_media_urls(self, page: Page) -> list[str]:
|
||||
urls: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
imgs = await page.query_selector_all('img[src*="pbs.twimg.com/media"]')
|
||||
for img in imgs:
|
||||
src = await img.get_attribute("src")
|
||||
if src and "profile_images" not in src:
|
||||
orig = self._original_tw_image_url(src)
|
||||
if orig and orig not in seen:
|
||||
seen.add(orig)
|
||||
urls.append(orig)
|
||||
|
||||
for sel in ('[data-testid="tweetPhoto"] img', "article img"):
|
||||
elements = await page.query_selector_all(sel)
|
||||
for el in elements:
|
||||
src = await el.get_attribute("src")
|
||||
if src and "pbs.twimg.com/media" in src:
|
||||
orig = self._original_tw_image_url(src)
|
||||
if orig and orig not in seen:
|
||||
seen.add(orig)
|
||||
urls.append(orig)
|
||||
|
||||
try:
|
||||
raw_video_urls = await page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const values = [];
|
||||
for (const el of document.querySelectorAll('video, video source')) {
|
||||
const candidates = [
|
||||
el.currentSrc,
|
||||
el.src,
|
||||
el.getAttribute?.('src'),
|
||||
el.getAttribute?.('data-src'),
|
||||
el.getAttribute?.('poster'),
|
||||
];
|
||||
for (const value of candidates) {
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
values.push(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
"""
|
||||
)
|
||||
except Exception:
|
||||
raw_video_urls = []
|
||||
|
||||
for src in raw_video_urls:
|
||||
normalized = self._normalize_media_url(str(src))
|
||||
if normalized and self._is_video_url(normalized) and normalized not in seen:
|
||||
seen.add(normalized)
|
||||
urls.append(normalized)
|
||||
|
||||
try:
|
||||
html = await page.content()
|
||||
except Exception:
|
||||
html = ""
|
||||
|
||||
for image_url in self._extract_image_urls_from_html(html):
|
||||
if image_url not in seen:
|
||||
seen.add(image_url)
|
||||
urls.append(image_url)
|
||||
|
||||
for pattern in (
|
||||
r"https?://video\.twimg\.com/[^\s\"'<>\\]+",
|
||||
r"https:\\/\\/video\.twimg\.com\\/[^\s\"'<>]+",
|
||||
):
|
||||
for match in re.findall(pattern, html):
|
||||
normalized = self._normalize_media_url(match)
|
||||
if normalized and self._is_video_url(normalized) and normalized not in seen:
|
||||
seen.add(normalized)
|
||||
urls.append(normalized)
|
||||
|
||||
return urls
|
||||
|
||||
async def extract_media_urls_from_api_responses(self, page: Page) -> list[str]:
|
||||
await self.wait_for_response_capture(page)
|
||||
payloads = self._response_store.get(id(page), [])
|
||||
if not payloads:
|
||||
return []
|
||||
|
||||
urls: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(url: str | None) -> None:
|
||||
if not url:
|
||||
return
|
||||
normalized = self._normalize_media_url(str(url))
|
||||
if not normalized:
|
||||
return
|
||||
if "pbs.twimg.com/media" in normalized:
|
||||
normalized = self._original_tw_image_url(normalized)
|
||||
if normalized not in seen:
|
||||
seen.add(normalized)
|
||||
urls.append(normalized)
|
||||
|
||||
for payload in payloads:
|
||||
data = payload.get("json")
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
for media_item in self._find_media_entities(data):
|
||||
media_url = media_item.get("media_url_https") or media_item.get("media_url")
|
||||
media_type = str(media_item.get("type") or "").lower()
|
||||
if media_url and media_type in {"photo", "animated_gif", "video", ""}:
|
||||
add(str(media_url))
|
||||
|
||||
video_info = media_item.get("video_info")
|
||||
if isinstance(video_info, dict):
|
||||
best_video = self._pick_best_video_variant(video_info)
|
||||
add(best_video)
|
||||
|
||||
return urls
|
||||
|
||||
async def extract_media_records_from_api_responses(self, page: Page) -> list[dict[str, Any]]:
|
||||
await self.wait_for_response_capture(page)
|
||||
payloads = self._response_store.get(id(page), [])
|
||||
if not payloads:
|
||||
return []
|
||||
|
||||
records: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
|
||||
def add_record(kind: str, url: str | None, *, thumbnail: str | None = None) -> None:
|
||||
if not url:
|
||||
return
|
||||
normalized = self._normalize_media_url(str(url))
|
||||
if not normalized:
|
||||
return
|
||||
key = (kind, normalized)
|
||||
if key in seen:
|
||||
return
|
||||
seen.add(key)
|
||||
records.append(
|
||||
{
|
||||
"type": kind,
|
||||
"url": normalized,
|
||||
"thumbnail": self._normalize_media_url(thumbnail) if thumbnail else None,
|
||||
}
|
||||
)
|
||||
|
||||
for payload in payloads:
|
||||
data = payload.get("json")
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
for media_item in self._find_media_entities(data):
|
||||
media_type = str(media_item.get("type") or "").lower()
|
||||
thumb_url = media_item.get("media_url_https") or media_item.get("media_url")
|
||||
if media_type == "photo":
|
||||
add_record("image", thumb_url)
|
||||
continue
|
||||
|
||||
video_info = media_item.get("video_info")
|
||||
if isinstance(video_info, dict):
|
||||
best_video = self._pick_best_video_variant(video_info)
|
||||
if best_video:
|
||||
add_record("video", best_video, thumbnail=thumb_url)
|
||||
continue
|
||||
|
||||
return records
|
||||
|
||||
async def wait_for_response_capture(self, page: Page, timeout: float = 2.0) -> None:
|
||||
page_id = id(page)
|
||||
tasks = self._response_tasks.get(page_id)
|
||||
if not tasks:
|
||||
return
|
||||
|
||||
deadline = asyncio.get_running_loop().time() + timeout
|
||||
while tasks and asyncio.get_running_loop().time() < deadline:
|
||||
pending = [task for task in list(tasks) if not task.done()]
|
||||
if not pending:
|
||||
break
|
||||
_, pending = await asyncio.wait(pending, timeout=0.2)
|
||||
tasks = self._response_tasks.get(page_id)
|
||||
if not tasks:
|
||||
break
|
||||
|
||||
async def scroll_and_collect_media(self, page: Page, max_scrolls: int = 12, max_items: int = 40) -> list[str]:
|
||||
collected: list[str] = []
|
||||
seen: set[str] = set()
|
||||
last_height = 0.0
|
||||
for _ in range(max_scrolls):
|
||||
batch: list[str] = []
|
||||
for item in await self.extract_media_records_from_api_responses(page):
|
||||
url = item.get("url")
|
||||
if isinstance(url, str) and url:
|
||||
batch.append(url)
|
||||
batch.extend(await self.extract_media_urls(page))
|
||||
for url in batch:
|
||||
if url not in seen:
|
||||
seen.add(url)
|
||||
collected.append(url)
|
||||
if len(collected) >= max_items:
|
||||
return collected
|
||||
try:
|
||||
height = await self.human_scroll_page(page)
|
||||
except Exception:
|
||||
break
|
||||
if height == last_height:
|
||||
break
|
||||
last_height = float(height)
|
||||
return collected
|
||||
|
||||
async def scroll_and_collect_tweet_urls(self, page: Page, limit: int = 40) -> list[str]:
|
||||
found: list[str] = []
|
||||
seen: set[str] = set()
|
||||
stale_rounds = 0
|
||||
max_rounds = 25
|
||||
|
||||
for _ in range(max_rounds):
|
||||
if len(found) >= limit:
|
||||
break
|
||||
before = len(found)
|
||||
links = await page.query_selector_all('a[href*="/status/"]')
|
||||
for link in links:
|
||||
href = await link.get_attribute("href")
|
||||
if not href or "/status/" not in href:
|
||||
continue
|
||||
if href.startswith("/"):
|
||||
full = f"https://x.com{href}"
|
||||
elif href.startswith("http"):
|
||||
full = href
|
||||
else:
|
||||
continue
|
||||
full = re.sub(r"\?.*$", "", full.split("#")[0])
|
||||
if full not in seen:
|
||||
seen.add(full)
|
||||
found.append(full)
|
||||
if len(found) >= limit:
|
||||
break
|
||||
|
||||
if len(found) == before:
|
||||
stale_rounds += 1
|
||||
if stale_rounds >= 5:
|
||||
break
|
||||
else:
|
||||
stale_rounds = 0
|
||||
|
||||
try:
|
||||
await self.human_scroll_page(page)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
return found[:limit]
|
||||
|
||||
async def get_author_info(self, page: Page) -> dict[str, str]:
|
||||
info = {"name": "", "username": ""}
|
||||
try:
|
||||
block = await page.query_selector('[data-testid="User-Name"]')
|
||||
if block:
|
||||
text = await block.inner_text()
|
||||
lines = [line.strip() for line in text.split("\n") if line.strip()]
|
||||
if lines:
|
||||
info["name"] = lines[0]
|
||||
for line in lines:
|
||||
if line.startswith("@"):
|
||||
info["username"] = line.lstrip("@")
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return info
|
||||
|
||||
async def get_user_display_name(self, page: Page, fallback: str) -> dict[str, str]:
|
||||
info = {"name": fallback, "username": fallback}
|
||||
try:
|
||||
block = await page.query_selector('[data-testid="UserName"]')
|
||||
if block:
|
||||
text = await block.inner_text()
|
||||
lines = [line.strip() for line in text.split("\n") if line.strip()]
|
||||
if lines:
|
||||
info["name"] = lines[0]
|
||||
except Exception:
|
||||
pass
|
||||
return info
|
||||
|
||||
async def get_tweet_text(self, page: Page) -> str:
|
||||
try:
|
||||
el = await page.query_selector('[data-testid="tweetText"]')
|
||||
if el:
|
||||
return (await el.inner_text()).strip()
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
async def get_tweet_date(self, page: Page) -> str:
|
||||
try:
|
||||
time_el = await page.query_selector("article time")
|
||||
if time_el:
|
||||
return (await time_el.get_attribute("datetime")) or ""
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
async def download_image(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
name = url.split("/")[-1].split("?")[0]
|
||||
if not any(name.lower().endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp")):
|
||||
name += ".jpg"
|
||||
target = save_path / name
|
||||
if target.exists() and target.stat().st_size > 256:
|
||||
return target
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
|
||||
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if response.status_code == 200 and "text/html" not in content_type:
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size > 256:
|
||||
return target
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
async def download_video(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:
|
||||
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
|
||||
if response.status_code != 200:
|
||||
return None
|
||||
|
||||
final_url = str(response.url)
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
body_text = response.text if self._looks_like_hls(final_url, content_type, "") or "text" in content_type else ""
|
||||
if "text/html" in content_type and not self._looks_like_hls(final_url, content_type, body_text):
|
||||
return None
|
||||
|
||||
if self._looks_like_hls(final_url, content_type, body_text):
|
||||
return await self._download_hls_playlist(
|
||||
client,
|
||||
playlist_url=final_url,
|
||||
playlist_text=body_text,
|
||||
save_path=save_path,
|
||||
referer=referer,
|
||||
)
|
||||
|
||||
target = self._video_target_path(final_url, save_path)
|
||||
if target.exists() and target.stat().st_size > 1024:
|
||||
return target
|
||||
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size <= 1024:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
return target
|
||||
|
||||
async def _download_hls_playlist(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
playlist_url: str,
|
||||
playlist_text: str,
|
||||
save_path: Path,
|
||||
referer: str | None,
|
||||
) -> Path | None:
|
||||
selected_url, selected_text = await self._resolve_hls_playlist(
|
||||
client,
|
||||
playlist_url=playlist_url,
|
||||
playlist_text=playlist_text,
|
||||
referer=referer,
|
||||
)
|
||||
if not selected_text:
|
||||
return None
|
||||
|
||||
segment_urls, init_url = self._parse_hls_segments(selected_url, selected_text)
|
||||
if not segment_urls:
|
||||
return None
|
||||
|
||||
target = self._video_target_path(selected_url, save_path, segment_urls=segment_urls, init_url=init_url)
|
||||
if target.exists() and target.stat().st_size > 1024:
|
||||
return target
|
||||
|
||||
with target.open("wb") as fh:
|
||||
if init_url:
|
||||
init_resp = await client.get(init_url, headers=self._media_request_headers(init_url, referer=referer))
|
||||
if init_resp.status_code != 200:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
fh.write(init_resp.content)
|
||||
|
||||
for segment_url in segment_urls:
|
||||
seg_resp = await client.get(
|
||||
segment_url,
|
||||
headers=self._media_request_headers(segment_url, referer=referer),
|
||||
)
|
||||
if seg_resp.status_code != 200:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
fh.write(seg_resp.content)
|
||||
|
||||
if target.stat().st_size <= 1024:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
return target
|
||||
|
||||
async def _resolve_hls_playlist(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
playlist_url: str,
|
||||
playlist_text: str,
|
||||
referer: str | None,
|
||||
) -> tuple[str, str]:
|
||||
if "#EXT-X-STREAM-INF" not in playlist_text:
|
||||
return playlist_url, playlist_text
|
||||
|
||||
variants = self._parse_hls_variants(playlist_url, playlist_text)
|
||||
for variant_url in variants:
|
||||
resp = await client.get(variant_url, headers=self._media_request_headers(variant_url, referer=referer))
|
||||
if resp.status_code == 200:
|
||||
return str(resp.url), resp.text
|
||||
return playlist_url, playlist_text
|
||||
|
||||
def _parse_hls_variants(self, playlist_url: str, playlist_text: str) -> list[str]:
|
||||
variants: list[tuple[int, str]] = []
|
||||
pending_bandwidth = 0
|
||||
|
||||
for raw_line in playlist_text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#EXT-X-STREAM-INF"):
|
||||
match = re.search(r"BANDWIDTH=(\d+)", line)
|
||||
pending_bandwidth = int(match.group(1)) if match else 0
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
variants.append((pending_bandwidth, urljoin(playlist_url, line)))
|
||||
pending_bandwidth = 0
|
||||
|
||||
variants.sort(key=lambda item: item[0], reverse=True)
|
||||
return [url for _, url in variants]
|
||||
|
||||
def _parse_hls_segments(self, playlist_url: str, playlist_text: str) -> tuple[list[str], str | None]:
|
||||
segments: list[str] = []
|
||||
init_url: str | None = None
|
||||
|
||||
for raw_line in playlist_text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#EXT-X-MAP:"):
|
||||
match = re.search(r'URI="([^"]+)"', line)
|
||||
if match:
|
||||
init_url = urljoin(playlist_url, match.group(1))
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
segments.append(urljoin(playlist_url, line))
|
||||
|
||||
return segments, init_url
|
||||
|
||||
def _media_request_headers(self, url: str, *, referer: str | None = None) -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": self._ua(),
|
||||
"Accept": "*/*",
|
||||
"Referer": referer or "https://x.com/",
|
||||
"Origin": "https://x.com",
|
||||
}
|
||||
cookie_header = self._cookie_header(url)
|
||||
if cookie_header:
|
||||
headers["Cookie"] = cookie_header
|
||||
csrf_token = self._cookie_value("ct0")
|
||||
if csrf_token:
|
||||
headers["x-csrf-token"] = csrf_token
|
||||
return headers
|
||||
|
||||
def _cookie_header(self, url: str) -> str:
|
||||
pairs: list[str] = []
|
||||
for item in self._load_cookie_items():
|
||||
name = str(item.get("name") or "").strip()
|
||||
value = str(item.get("value") or "")
|
||||
if name:
|
||||
pairs.append(f"{name}={value}")
|
||||
return "; ".join(pairs)
|
||||
|
||||
def _cookie_value(self, name: str) -> str:
|
||||
for item in self._load_cookie_items():
|
||||
if str(item.get("name") or "") == name:
|
||||
return str(item.get("value") or "")
|
||||
return ""
|
||||
|
||||
def _load_cookie_items(self) -> list[dict[str, Any]]:
|
||||
cookie_path = Path(getattr(self.config, "cookie_file", "") or "")
|
||||
if not cookie_path.is_file():
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(cookie_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return []
|
||||
return payload if isinstance(payload, list) else []
|
||||
|
||||
async def _add_cookies_tolerant(self, cookies: list[dict[str, Any]]) -> int:
|
||||
assert self.context is not None
|
||||
try:
|
||||
await self.context.add_cookies(cookies)
|
||||
return len(cookies)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
loaded = 0
|
||||
for cookie in cookies:
|
||||
try:
|
||||
await self.context.add_cookies([cookie])
|
||||
loaded += 1
|
||||
except Exception:
|
||||
continue
|
||||
return loaded
|
||||
|
||||
@staticmethod
|
||||
def _prepare_cookie_items(cookies: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
prepared_by_key: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
for cookie in cookies:
|
||||
name = str(cookie.get("name") or "").strip()
|
||||
domain = str(cookie.get("domain") or "").strip()
|
||||
if not name or not domain:
|
||||
continue
|
||||
|
||||
item: dict[str, Any] = {
|
||||
"name": name,
|
||||
"value": str(cookie.get("value") or ""),
|
||||
"domain": domain,
|
||||
"path": str(cookie.get("path") or "/"),
|
||||
"secure": bool(cookie.get("secure", False)),
|
||||
"httpOnly": bool(cookie.get("httpOnly", False)),
|
||||
}
|
||||
|
||||
expires = cookie.get("expires")
|
||||
if expires is not None:
|
||||
try:
|
||||
expires_value = int(float(expires))
|
||||
if expires_value > 0:
|
||||
item["expires"] = expires_value
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
same_site = str(cookie.get("sameSite") or "").strip()
|
||||
if same_site in {"Strict", "Lax", "None"}:
|
||||
if same_site == "None" and not item["secure"]:
|
||||
item["sameSite"] = "Lax"
|
||||
else:
|
||||
item["sameSite"] = same_site
|
||||
|
||||
key = (domain.lower(), str(item["path"]), name)
|
||||
prepared_by_key[key] = item
|
||||
return list(prepared_by_key.values())
|
||||
|
||||
def _video_target_path(
|
||||
self,
|
||||
url: str,
|
||||
save_path: Path,
|
||||
*,
|
||||
segment_urls: list[str] | None = None,
|
||||
init_url: str | None = None,
|
||||
) -> Path:
|
||||
parsed = urlparse(url)
|
||||
name = Path(parsed.path).name or "x_video"
|
||||
|
||||
if name.endswith(".m3u8") or not Path(name).suffix:
|
||||
if segment_urls:
|
||||
seg_name = Path(urlparse(segment_urls[0]).path).name
|
||||
if seg_name:
|
||||
name = seg_name
|
||||
if init_url:
|
||||
init_name = Path(urlparse(init_url).path).name
|
||||
if init_name:
|
||||
name = init_name
|
||||
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix in {".m3u8", ".m4s", ".mpd", ""}:
|
||||
name = f"{Path(name).stem or 'x_video'}.mp4"
|
||||
return save_path / name
|
||||
|
||||
def _looks_like_hls(self, url: str, content_type: str, body: str) -> bool:
|
||||
low_url = url.lower()
|
||||
if ".m3u8" in low_url:
|
||||
return True
|
||||
if "mpegurl" in content_type or "vnd.apple.mpegurl" in content_type:
|
||||
return True
|
||||
return "#EXTM3U" in body[:256]
|
||||
|
||||
def _attach_response_collector(self, page: Page) -> None:
|
||||
page_id = id(page)
|
||||
bucket: list[dict[str, Any]] = []
|
||||
self._response_store[page_id] = bucket
|
||||
self._response_tasks[page_id] = set()
|
||||
|
||||
def handle_response(response: Any) -> None:
|
||||
task = asyncio.create_task(self._capture_response(page, response))
|
||||
task_bucket = self._response_tasks.setdefault(page_id, set())
|
||||
task_bucket.add(task)
|
||||
|
||||
def cleanup(done_task: asyncio.Task[Any]) -> None:
|
||||
current_bucket = self._response_tasks.get(page_id)
|
||||
if current_bucket is not None:
|
||||
current_bucket.discard(done_task)
|
||||
|
||||
task.add_done_callback(cleanup)
|
||||
|
||||
page.on("response", handle_response)
|
||||
|
||||
async def _capture_response(self, page: Page, response: Any) -> None:
|
||||
url = str(getattr(response, "url", "") or "")
|
||||
if not self._should_capture_response(url):
|
||||
return
|
||||
|
||||
try:
|
||||
headers = await response.all_headers()
|
||||
except Exception:
|
||||
headers = {}
|
||||
content_type = str(headers.get("content-type", "")).lower()
|
||||
if "json" not in content_type and "javascript" not in content_type and "text/plain" not in content_type:
|
||||
return
|
||||
|
||||
try:
|
||||
text = await response.text()
|
||||
except Exception:
|
||||
return
|
||||
if not text or len(text) < 2:
|
||||
return
|
||||
|
||||
parsed_json: Any = None
|
||||
try:
|
||||
parsed_json = json.loads(text)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
self._response_store.setdefault(id(page), []).append(
|
||||
{
|
||||
"url": url,
|
||||
"status": getattr(response, "status", None),
|
||||
"json": parsed_json,
|
||||
}
|
||||
)
|
||||
|
||||
def _should_capture_response(self, url: str) -> bool:
|
||||
low = url.lower()
|
||||
return any(
|
||||
token in low
|
||||
for token in (
|
||||
"/i/api/graphql/",
|
||||
"/graphql/",
|
||||
"tweetdetail",
|
||||
"tweetresultbyrestid",
|
||||
"conversation",
|
||||
"adaptive.json",
|
||||
)
|
||||
)
|
||||
|
||||
def _find_media_entities(self, payload: Any) -> list[dict[str, Any]]:
|
||||
media_entities: list[dict[str, Any]] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
if "media_url_https" in node or "video_info" in node:
|
||||
media_entities.append(node)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
return
|
||||
if isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
return media_entities
|
||||
|
||||
def _pick_best_video_variant(self, video_info: dict[str, Any]) -> str | None:
|
||||
variants = video_info.get("variants")
|
||||
if not isinstance(variants, list):
|
||||
return None
|
||||
|
||||
best_mp4: tuple[int, str] | None = None
|
||||
best_hls: str | None = None
|
||||
for item in variants:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
url = item.get("url")
|
||||
if not isinstance(url, str) or not url:
|
||||
continue
|
||||
low_url = url.lower()
|
||||
content_type = str(item.get("content_type") or "").lower()
|
||||
try:
|
||||
bitrate = int(item.get("bitrate") or 0)
|
||||
except (TypeError, ValueError):
|
||||
bitrate = 0
|
||||
if "mp4" in content_type or ".mp4" in low_url:
|
||||
if best_mp4 is None or bitrate > best_mp4[0]:
|
||||
best_mp4 = (bitrate, url)
|
||||
elif "mpegurl" in content_type or ".m3u8" in low_url:
|
||||
best_hls = url
|
||||
|
||||
if best_mp4 is not None:
|
||||
return best_mp4[1]
|
||||
return best_hls
|
||||
|
||||
def _extract_image_urls_from_html(self, html: str) -> list[str]:
|
||||
if not html:
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
urls: list[str] = []
|
||||
patterns = (
|
||||
r"https?://pbs\.twimg\.com/media/[^\s\"'<>\\]+",
|
||||
r"https:\\/\\/pbs\.twimg\.com\\/media\\/[^\s\"'<>]+",
|
||||
)
|
||||
for pattern in patterns:
|
||||
for match in re.findall(pattern, html):
|
||||
normalized = self._normalize_media_url(match)
|
||||
if "pbs.twimg.com/media" not in normalized:
|
||||
continue
|
||||
orig = self._original_tw_image_url(normalized)
|
||||
if orig not in seen:
|
||||
seen.add(orig)
|
||||
urls.append(orig)
|
||||
return urls
|
||||
|
||||
def _ua(self) -> str:
|
||||
if default_profile:
|
||||
return default_profile("x").user_agent
|
||||
return (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _original_tw_image_url(url: str) -> str:
|
||||
if "pbs.twimg.com/media" not in url:
|
||||
return url
|
||||
base = url.split("?")[0]
|
||||
return f"{base}?format=jpg&name=orig" if "format=" not in url else re.sub(
|
||||
r"name=\w+",
|
||||
"name=orig",
|
||||
url,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_media_url(url: str) -> str:
|
||||
value = url.strip().replace("\\/", "/")
|
||||
if value.startswith("//"):
|
||||
value = f"https:{value}"
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _is_video_url(url: str) -> bool:
|
||||
low = url.lower()
|
||||
return any(
|
||||
token in low
|
||||
for token in (
|
||||
".mp4",
|
||||
".m3u8",
|
||||
"video.twimg.com",
|
||||
"/amplify_video/",
|
||||
"/ext_tw_video/",
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,504 @@
|
||||
"""
|
||||
X (Twitter) plugin.
|
||||
|
||||
Uses Playwright for rendering and httpx for media downloads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import queue
|
||||
import random
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .playwright_client import XPlaywrightClient
|
||||
from .x_login import run_interactive_login as x_login_flow
|
||||
from .x_urls import (
|
||||
extract_search_query,
|
||||
extract_tweet_id,
|
||||
extract_username_and_kind,
|
||||
is_search_url,
|
||||
is_status_url,
|
||||
normalize_url,
|
||||
)
|
||||
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
|
||||
|
||||
|
||||
@dataclass
|
||||
class XPluginConfig:
|
||||
cookie_file: str = ""
|
||||
profile_dir: str = ""
|
||||
headless: bool = False
|
||||
safety_mode: bool = True
|
||||
wait_timeout: int = 45000
|
||||
timeline_tweet_limit: int = 8
|
||||
media_scroll_rounds: int = 5
|
||||
max_media_items: int = 40
|
||||
parse_cooldown_seconds: float = 60.0
|
||||
min_action_delay: float = 4.0
|
||||
max_action_delay: float = 11.0
|
||||
download_delay_min: float = 2.0
|
||||
download_delay_max: float = 5.0
|
||||
batch_pause_min: float = 12.0
|
||||
batch_pause_max: float = 28.0
|
||||
batch_pause_after_min: int = 2
|
||||
batch_pause_after_max: int = 4
|
||||
max_parse_per_hour: int = 24
|
||||
max_download_files_per_hour: int = 60
|
||||
max_consecutive_failures: int = 3
|
||||
failure_cooldown_seconds: float = 1800.0
|
||||
sync_parse_timeout_seconds: float = 120.0
|
||||
|
||||
|
||||
class XPlugin(BasePlugin):
|
||||
name = "X"
|
||||
domain = "twitter.com"
|
||||
version = "1.0.0"
|
||||
|
||||
_HANDLE_PATTERNS = (
|
||||
re.compile(r"(?:twitter|x)\.com/\w+/status/\d+", re.I),
|
||||
re.compile(r"(?:twitter|x)\.com/[^/]+/media\b", re.I),
|
||||
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/?$", re.I),
|
||||
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/media/?$", re.I),
|
||||
re.compile(r"(?:twitter|x)\.com/search\?", re.I),
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.config = XPluginConfig(
|
||||
cookie_file=str(runtime_path("x_cookies.json")),
|
||||
profile_dir=str(runtime_path("x_profile")),
|
||||
headless=False,
|
||||
)
|
||||
self._cache_file = runtime_path("x_download_cache.json")
|
||||
self._playwright_lock = threading.Lock()
|
||||
self._last_parse_at = 0.0
|
||||
self._parse_timestamps: list[float] = []
|
||||
self._download_timestamps: list[float] = []
|
||||
self._consecutive_failures = 0
|
||||
self._suspended_until = 0.0
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
text = url.strip()
|
||||
if not text:
|
||||
return False
|
||||
candidate = text if "://" in text else f"https://{text.lstrip('/')}"
|
||||
try:
|
||||
hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".")
|
||||
except ValueError:
|
||||
return False
|
||||
if hostname not in {"x.com", "www.x.com", "twitter.com", "www.twitter.com", "mobile.twitter.com"}:
|
||||
return False
|
||||
return any(pattern.search(text) for pattern in self._HANDLE_PATTERNS)
|
||||
|
||||
def validate(self) -> bool:
|
||||
try:
|
||||
import playwright # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
}
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
self._enforce_account_safety("parse", units=1)
|
||||
try:
|
||||
with self._playwright_lock:
|
||||
self._respect_parse_cooldown()
|
||||
try:
|
||||
result = asyncio.run(self._parse_async(normalize_url(url)))
|
||||
self._record_success()
|
||||
self._record_usage(self._parse_timestamps, 1)
|
||||
return result
|
||||
finally:
|
||||
self._last_parse_at = time.monotonic()
|
||||
except PluginParseError:
|
||||
self._record_failure()
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._record_failure()
|
||||
raise PluginParseError(f"X 解析失败: {exc}") from exc
|
||||
|
||||
def parse_for_sync(self, url: str) -> list[Resource]:
|
||||
self._enforce_account_safety("parse", units=1)
|
||||
timeout = max(30.0, float(self.config.sync_parse_timeout_seconds))
|
||||
try:
|
||||
with self._playwright_lock:
|
||||
result = asyncio.run(asyncio.wait_for(self._parse_async(normalize_url(url)), timeout=timeout))
|
||||
self._record_success()
|
||||
self._record_usage(self._parse_timestamps, 1)
|
||||
self._last_parse_at = time.monotonic()
|
||||
return result
|
||||
except asyncio.TimeoutError as exc:
|
||||
self._record_failure()
|
||||
raise PluginParseError(f"X 同步解析超时({int(timeout)} 秒),请稍后重试或检查登录状态。") from exc
|
||||
except PluginParseError:
|
||||
self._record_failure()
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._record_failure()
|
||||
raise PluginParseError(f"X 同步解析失败: {exc}") from exc
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
requested = max(1, len(resource.files))
|
||||
self._enforce_account_safety("download", units=requested)
|
||||
try:
|
||||
with self._playwright_lock:
|
||||
result = asyncio.run(self._download_async(resource, save_path))
|
||||
self._record_success()
|
||||
self._record_usage(self._download_timestamps, max(1, len(result) or requested))
|
||||
return result
|
||||
except Exception as exc:
|
||||
self._record_failure()
|
||||
raise PluginParseError(f"X 下载失败: {exc}") from exc
|
||||
|
||||
def reset_playwright_session(self) -> None:
|
||||
return None
|
||||
|
||||
def _respect_parse_cooldown(self) -> None:
|
||||
elapsed = time.monotonic() - self._last_parse_at
|
||||
remain = float(self.config.parse_cooldown_seconds) - elapsed
|
||||
if 0 < remain < self.config.parse_cooldown_seconds:
|
||||
time.sleep(remain)
|
||||
|
||||
def _enforce_account_safety(self, action: str, *, units: int) -> None:
|
||||
if not self.config.safety_mode:
|
||||
return
|
||||
|
||||
now = time.monotonic()
|
||||
if self._suspended_until > now:
|
||||
remaining = int(self._suspended_until - now)
|
||||
minutes = max(1, remaining // 60)
|
||||
raise PluginParseError(f"X 账号安全模式已暂停操作,请约 {minutes} 分钟后再试。")
|
||||
|
||||
self._trim_usage(now)
|
||||
if action == "parse":
|
||||
if len(self._parse_timestamps) + units > self.config.max_parse_per_hour:
|
||||
raise PluginParseError("X 账号安全模式已触发每小时解析上限,请稍后再试。")
|
||||
return
|
||||
|
||||
if action == "download":
|
||||
if len(self._download_timestamps) + units > self.config.max_download_files_per_hour:
|
||||
raise PluginParseError("X 账号安全模式已触发每小时下载上限,请稍后再试。")
|
||||
|
||||
def _trim_usage(self, now: float | None = None) -> None:
|
||||
current = now if now is not None else time.monotonic()
|
||||
window = 3600.0
|
||||
self._parse_timestamps = [item for item in self._parse_timestamps if current - item < window]
|
||||
self._download_timestamps = [item for item in self._download_timestamps if current - item < window]
|
||||
|
||||
def _record_usage(self, bucket: list[float], units: int) -> None:
|
||||
now = time.monotonic()
|
||||
self._trim_usage(now)
|
||||
for _ in range(max(1, units)):
|
||||
bucket.append(now)
|
||||
|
||||
def _record_success(self) -> None:
|
||||
self._consecutive_failures = 0
|
||||
|
||||
def _record_failure(self) -> None:
|
||||
if not self.config.safety_mode:
|
||||
return
|
||||
self._consecutive_failures += 1
|
||||
if self._consecutive_failures >= self.config.max_consecutive_failures:
|
||||
self._suspended_until = time.monotonic() + float(self.config.failure_cooldown_seconds)
|
||||
self._consecutive_failures = 0
|
||||
|
||||
async def _parse_async(self, url: str) -> list[Resource]:
|
||||
if not self.validate():
|
||||
raise PluginParseError("未安装 Playwright。请先执行: pip install playwright && playwright install chromium")
|
||||
|
||||
client = XPlaywrightClient(self.config)
|
||||
await client.start()
|
||||
try:
|
||||
if is_status_url(url):
|
||||
return await self._parse_single_tweet(client, url)
|
||||
if is_search_url(url):
|
||||
return await self._parse_search_media(client, url)
|
||||
|
||||
user, kind = extract_username_and_kind(url)
|
||||
if user and kind == "media":
|
||||
return await self._parse_user_media(client, url, user)
|
||||
if user and kind == "profile":
|
||||
return await self._parse_user_timeline(client, url, user)
|
||||
|
||||
raise PluginParseError(f"无法识别的 X 链接: {url}")
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
async def _parse_single_tweet(self, client: XPlaywrightClient, url: str) -> list[Resource]:
|
||||
page = await client.new_page_goto(url)
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
await client.wait_for_main_content(page)
|
||||
await client.wait_for_media(page)
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=5000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
tweet_id = extract_tweet_id(url) or ""
|
||||
author = await client.get_author_info(page)
|
||||
media_records = await client.extract_media_records_from_api_responses(page)
|
||||
media_urls = await client.extract_media_urls(page)
|
||||
if not media_urls:
|
||||
try:
|
||||
await page.mouse.wheel(0, 500)
|
||||
await asyncio.sleep(1.2)
|
||||
except Exception:
|
||||
pass
|
||||
media_urls = await client.extract_media_urls(page)
|
||||
if not media_records and not media_urls:
|
||||
media_records = await client.extract_media_records_from_api_responses(page)
|
||||
if not media_urls:
|
||||
media_urls = [item["url"] for item in media_records] or await client.extract_media_urls_from_api_responses(page)
|
||||
|
||||
content = await client.get_tweet_text(page)
|
||||
created = await client.get_tweet_date(page)
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
if not media_records and not media_urls:
|
||||
raise PluginParseError("未在该推文解析到媒体,可能需要重新登录,或该页面结构已变化。")
|
||||
|
||||
resources: list[Resource] = []
|
||||
if media_records:
|
||||
for index, item in enumerate(media_records):
|
||||
file_url = item["url"]
|
||||
thumb = item.get("thumbnail") or file_url
|
||||
resources.append(
|
||||
Resource(
|
||||
id=f"{tweet_id}_{index}",
|
||||
url=url,
|
||||
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
|
||||
author=author.get("name", ""),
|
||||
author_id=author.get("username", ""),
|
||||
files=[{"url": file_url, "type": item.get("type", self._media_type(file_url)), "referer": url}],
|
||||
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
|
||||
thumbnail=thumb,
|
||||
created_at=created or None,
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
for index, media_url in enumerate(media_urls):
|
||||
resources.append(
|
||||
Resource(
|
||||
id=f"{tweet_id}_{index}",
|
||||
url=url,
|
||||
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
|
||||
author=author.get("name", ""),
|
||||
author_id=author.get("username", ""),
|
||||
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
|
||||
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
|
||||
thumbnail=media_urls[0] if index == 0 else None,
|
||||
created_at=created or None,
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
async def _parse_search_media(self, client: XPlaywrightClient, url: str) -> list[Resource]:
|
||||
page = await client.new_page_goto(url)
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
media_urls = await client.scroll_and_collect_media(
|
||||
page,
|
||||
max_scrolls=self.config.media_scroll_rounds,
|
||||
max_items=self.config.max_media_items,
|
||||
)
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
query = extract_search_query(url) or "search"
|
||||
resources: list[Resource] = []
|
||||
for index, media_url in enumerate(media_urls):
|
||||
resources.append(
|
||||
Resource(
|
||||
id=f"search_{abs(hash(query))}_{index}",
|
||||
url=url,
|
||||
title=f"X search - {query}",
|
||||
author=query,
|
||||
author_id="search",
|
||||
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
|
||||
metadata={"source": "search_media", "query": query},
|
||||
thumbnail=media_url,
|
||||
created_at=None,
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
async def _parse_user_media(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
|
||||
page = await client.new_page_goto(url)
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
info = await client.get_user_display_name(page, username)
|
||||
media_urls = await client.scroll_and_collect_media(
|
||||
page,
|
||||
max_scrolls=self.config.media_scroll_rounds,
|
||||
max_items=self.config.max_media_items,
|
||||
)
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
resources: list[Resource] = []
|
||||
for index, media_url in enumerate(media_urls):
|
||||
resources.append(
|
||||
Resource(
|
||||
id=f"{username}_media_{index}",
|
||||
url=url,
|
||||
title=f"{info.get('name', username)} - media",
|
||||
author=info.get("name", username),
|
||||
author_id=username,
|
||||
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
|
||||
metadata={"source": "media_tab", "username": username},
|
||||
thumbnail=media_url,
|
||||
created_at=None,
|
||||
)
|
||||
)
|
||||
return resources
|
||||
|
||||
async def _parse_user_timeline(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
|
||||
page = await client.new_page_goto(url)
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
tweet_urls = await client.scroll_and_collect_tweet_urls(
|
||||
page,
|
||||
limit=self.config.timeline_tweet_limit,
|
||||
)
|
||||
finally:
|
||||
await page.close()
|
||||
|
||||
all_results: list[Resource] = []
|
||||
for tweet_url in tweet_urls:
|
||||
try:
|
||||
all_results.extend(await self._parse_single_tweet(client, tweet_url))
|
||||
except Exception:
|
||||
continue
|
||||
return all_results
|
||||
|
||||
async def _download_async(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
client = XPlaywrightClient(self.config)
|
||||
results: list[Path] = []
|
||||
subdir = resource.metadata.get("save_subdir")
|
||||
target_base = save_path / subdir if subdir else save_path
|
||||
cache = self._load_download_cache()
|
||||
batch_pause_after = random.randint(self.config.batch_pause_after_min, self.config.batch_pause_after_max)
|
||||
|
||||
try:
|
||||
for index, item in enumerate(resource.files, start=1):
|
||||
file_url = item.get("url")
|
||||
if not file_url:
|
||||
continue
|
||||
|
||||
cached = cache.get(file_url)
|
||||
if cached and Path(cached).is_file():
|
||||
results.append(Path(cached))
|
||||
continue
|
||||
|
||||
kind = item.get("type", "image")
|
||||
referer = item.get("referer") or resource.url
|
||||
if kind == "video":
|
||||
saved = await client.download_video(file_url, target_base, referer=referer)
|
||||
else:
|
||||
saved = await client.download_image(file_url, target_base, referer=referer)
|
||||
|
||||
if saved is None:
|
||||
continue
|
||||
|
||||
cache[file_url] = str(saved)
|
||||
results.append(saved)
|
||||
await asyncio.sleep(
|
||||
random.uniform(self.config.download_delay_min, self.config.download_delay_max)
|
||||
* random.uniform(0.8, 1.5)
|
||||
)
|
||||
|
||||
if index % batch_pause_after == 0:
|
||||
await asyncio.sleep(random.uniform(self.config.batch_pause_min, self.config.batch_pause_max))
|
||||
batch_pause_after = random.randint(
|
||||
self.config.batch_pause_after_min,
|
||||
self.config.batch_pause_after_max,
|
||||
)
|
||||
|
||||
self._save_download_cache(cache)
|
||||
return results
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
def _load_download_cache(self) -> dict[str, str]:
|
||||
try:
|
||||
if self._cache_file.is_file():
|
||||
payload = json.loads(self._cache_file.read_text(encoding="utf-8"))
|
||||
if isinstance(payload, dict):
|
||||
return {str(key): str(value) for key, value in payload.items()}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
def _save_download_cache(self, payload: dict[str, str]) -> None:
|
||||
self._cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._cache_file.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
def run_interactive_login(
|
||||
self,
|
||||
*,
|
||||
proceed: threading.Event | None = None,
|
||||
ready_signal: queue.Queue[bool] | None = None,
|
||||
) -> bool:
|
||||
path = Path(self.config.profile_dir)
|
||||
try:
|
||||
with self._playwright_lock:
|
||||
return asyncio.run(x_login_flow(path, proceed=proceed, ready_signal=ready_signal))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def set_headless(self, headless: bool) -> None:
|
||||
self.config.headless = headless
|
||||
|
||||
@staticmethod
|
||||
def _media_type(url: str) -> str:
|
||||
low = url.lower()
|
||||
if (
|
||||
"video" in low
|
||||
or ".mp4" in low
|
||||
or ".m3u8" in low
|
||||
or "video.twimg.com" in low
|
||||
or "/amplify_video/" in low
|
||||
or "/ext_tw_video/" in low
|
||||
):
|
||||
return "video"
|
||||
return "image"
|
||||
|
||||
def get_config_schema(self) -> dict[str, Any]:
|
||||
return {
|
||||
"headless": {"type": "bool", "label": "无头模式", "default": False},
|
||||
"safety_mode": {"type": "bool", "label": "账号安全模式", "default": True},
|
||||
"timeline_tweet_limit": {"type": "int", "label": "时间线采样条数", "default": 8, "min": 1, "max": 30},
|
||||
"media_scroll_rounds": {"type": "int", "label": "媒体页滚动轮数", "default": 5, "min": 1, "max": 12},
|
||||
"max_media_items": {"type": "int", "label": "单次最多媒体数", "default": 40, "min": 1, "max": 120},
|
||||
"parse_cooldown_seconds": {"type": "float", "label": "解析冷却秒数", "default": 60.0, "min": 10, "max": 600},
|
||||
"min_action_delay": {"type": "float", "label": "最小操作延迟秒数", "default": 4.0, "min": 1, "max": 60},
|
||||
"max_action_delay": {"type": "float", "label": "最大操作延迟秒数", "default": 11.0, "min": 2, "max": 90},
|
||||
"download_delay_min": {"type": "float", "label": "下载后最小停顿秒数", "default": 2.0, "min": 0, "max": 60},
|
||||
"download_delay_max": {"type": "float", "label": "下载后最大停顿秒数", "default": 5.0, "min": 0, "max": 120},
|
||||
"max_parse_per_hour": {"type": "int", "label": "每小时最大解析次数", "default": 24, "min": 1, "max": 120},
|
||||
"max_download_files_per_hour": {"type": "int", "label": "每小时最大下载文件数", "default": 60, "min": 1, "max": 300},
|
||||
"max_consecutive_failures": {"type": "int", "label": "连续失败暂停阈值", "default": 3, "min": 1, "max": 10},
|
||||
"failure_cooldown_seconds": {"type": "float", "label": "连续失败暂停秒数", "default": 1800.0, "min": 60, "max": 86400},
|
||||
}
|
||||
|
||||
|
||||
plugin_class = XPlugin
|
||||
@@ -0,0 +1,8 @@
|
||||
"""DOM selectors for X pages used by Sakura desktop sync."""
|
||||
|
||||
PROFILE_LINK = 'a[data-testid="AppTabBar_Profile_Link"]'
|
||||
USER_CELL = '[data-testid="UserCell"]'
|
||||
USER_NAME = '[data-testid="UserName"]'
|
||||
USER_AVATAR = 'img[src*="profile_images"]'
|
||||
USER_LINK = 'a[href^="/"][role="link"]'
|
||||
FOLLOWING_URL_SUFFIX = "/following"
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Install Python deps and Chromium for the X plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def main() -> None:
|
||||
deps = ["playwright", "httpx"]
|
||||
for dep in deps:
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", dep])
|
||||
subprocess.check_call([sys.executable, "-m", "playwright", "install", "chromium"])
|
||||
print("Done. Restart the app and reload plugins.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Interactive X login via persistent Chromium profile."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import shutil
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from .browser_opts import launch_x_persistent_context
|
||||
from pixiv_app.services.x_login_flow import goto_x_login_entry
|
||||
|
||||
|
||||
async def run_interactive_login(
|
||||
profile_dir: Path,
|
||||
*,
|
||||
proceed: threading.Event | None = None,
|
||||
ready_signal: queue.Queue[bool] | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Open visible Chromium with persistent ``profile_dir``.
|
||||
|
||||
Optionally notify GUI via ``ready_signal`` after the login page loads, then block until
|
||||
``proceed`` is set (e.g. user clicked OK after logging in). If ``proceed`` is None,
|
||||
falls back to terminal ``input()``.
|
||||
"""
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
playwright = await async_playwright().start()
|
||||
context = None
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
context = await launch_x_persistent_context(playwright, profile_dir, headless=False)
|
||||
page = context.pages[0] if context.pages else await context.new_page()
|
||||
await goto_x_login_entry(page, timeout=60_000)
|
||||
|
||||
if ready_signal is not None:
|
||||
ready_signal.put(True)
|
||||
|
||||
if proceed is not None:
|
||||
await loop.run_in_executor(None, proceed.wait)
|
||||
else:
|
||||
await loop.run_in_executor(
|
||||
None,
|
||||
lambda: input("\n在浏览器中完成 X 登录后,回到控制台按 Enter...\n"),
|
||||
)
|
||||
return True
|
||||
finally:
|
||||
if context is not None:
|
||||
await context.close()
|
||||
await playwright.stop()
|
||||
|
||||
|
||||
def clear_profile(profile_dir: Path) -> None:
|
||||
if profile_dir.exists():
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -0,0 +1,57 @@
|
||||
"""URL helpers for X / Twitter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
STATUS_RE = re.compile(
|
||||
r"(?:https?://)?(?:www\.)?(?:twitter|x)\.com/[^/]+/status/(\d+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
PROFILE_OR_MEDIA_RE = re.compile(
|
||||
r"(?:https?://)?(?:www\.)?(?:twitter|x)\.com/([A-Za-z0-9_]{1,15})(?:/(media))?/?$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
u = url.strip()
|
||||
if not u.startswith("http"):
|
||||
u = "https://" + u
|
||||
return u
|
||||
|
||||
|
||||
def extract_tweet_id(url: str) -> str | None:
|
||||
m = STATUS_RE.search(url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def extract_username_and_kind(url: str) -> tuple[str | None, str | None]:
|
||||
u = normalize_url(url)
|
||||
m = PROFILE_OR_MEDIA_RE.match(u)
|
||||
if not m:
|
||||
return None, None
|
||||
user = m.group(1)
|
||||
media_flag = m.group(2)
|
||||
if user.lower() in {"home", "explore", "settings", "messages", "notifications", "i", "intent", "search"}:
|
||||
return None, None
|
||||
if media_flag:
|
||||
return user, "media"
|
||||
return user, "profile"
|
||||
|
||||
|
||||
def is_status_url(url: str) -> bool:
|
||||
return extract_tweet_id(url) is not None
|
||||
|
||||
|
||||
def is_search_url(url: str) -> bool:
|
||||
parsed = urlparse(normalize_url(url))
|
||||
host = parsed.netloc.lower()
|
||||
return host in {"x.com", "www.x.com", "twitter.com", "www.twitter.com"} and parsed.path.rstrip("/") == "/search"
|
||||
|
||||
|
||||
def extract_search_query(url: str) -> str:
|
||||
parsed = urlparse(normalize_url(url))
|
||||
values = parse_qs(parsed.query).get("q") or []
|
||||
return values[0] if values else ""
|
||||
Reference in new issue
Block a user