first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""X (Twitter) Playwright plugin."""
+113
View File
@@ -0,0 +1,113 @@
"""Shared Playwright launch options for X — reduce automation fingerprint; prefer system Chrome."""
from __future__ import annotations
import os
from pathlib import Path
from playwright.async_api import BrowserContext, Playwright
from pixiv_app.core.playwright_runtime import configure_playwright_runtime_env, preferred_chromium_executable
try:
from pixiv_app.core.fingerprint import default_profile
except Exception: # pragma: no cover - plugin can run standalone
default_profile = None
try:
from pixiv_app.core.browser_proxy_pool import parse_proxy_url
except Exception: # pragma: no cover
parse_proxy_url = None
async def launch_x_persistent_context(
playwright: Playwright,
profile_dir: Path,
*,
headless: bool,
) -> BrowserContext:
"""
Launch persistent Chromium with flags that reduce obvious automation signals.
Order: ``X_PLAYWRIGHT_CHANNEL`` env (if set), then system Chrome, then bundled Chromium.
Google OAuth often still blocks automation browsers; users should use X email/phone login
or import cookies from a normal browser session.
"""
configure_playwright_runtime_env()
profile_dir.mkdir(parents=True, exist_ok=True)
browser_profile = default_profile("x") if default_profile else None
base_kw: dict = {
"user_data_dir": str(profile_dir),
"headless": headless,
"viewport": browser_profile.viewport if browser_profile else {"width": 1280, "height": 800},
"locale": browser_profile.locale if browser_profile else "zh-CN",
"timezone_id": browser_profile.timezone_id if browser_profile else "Asia/Shanghai",
"screen": browser_profile.viewport if browser_profile else {"width": 1280, "height": 800},
"extra_http_headers": {
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
},
"ignore_default_args": ["--enable-automation"],
"args": [
"--disable-blink-features=AutomationControlled",
"--window-size=1280,800",
],
"user_agent": browser_profile.user_agent
if browser_profile
else (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
),
}
if browser_profile:
base_kw["device_scale_factor"] = browser_profile.device_scale_factor
base_kw["is_mobile"] = browser_profile.is_mobile
base_kw["has_touch"] = browser_profile.has_touch
proxy_value = os.environ.get("X_PROXY_URL", os.environ.get("PIXIV_BROWSER_PROXY", "")).strip()
if proxy_value and parse_proxy_url:
parsed_proxy = parse_proxy_url(proxy_value)
if parsed_proxy is not None:
base_kw["proxy"] = parsed_proxy.to_playwright_proxy()
env_ch = os.environ.get("X_PLAYWRIGHT_CHANNEL", "").strip()
local_executable = preferred_chromium_executable()
seen: set[str] = set()
attempts: list[tuple[str, dict]] = []
for label, extra in (
("env_channel", {"channel": env_ch} if env_ch else {}),
("installed_chromium", {"executable_path": str(local_executable)} if local_executable else {}),
("system_chrome", {"channel": "chrome"}),
("default", {}),
):
key = f"{label}:{extra!r}"
if key in seen:
continue
seen.add(key)
attempts.append((label, extra))
last_err: Exception | None = None
for _, extra in attempts:
kw = dict(base_kw)
kw.update(extra)
try:
ctx = await playwright.chromium.launch_persistent_context(**kw)
await _apply_stealth_scripts(ctx)
return ctx
except Exception as exc:
last_err = exc
continue
assert last_err is not None
raise last_err
async def _apply_stealth_scripts(ctx: BrowserContext) -> None:
"""Best-effort scripts; cannot guarantee OAuth providers will accept the browser."""
await ctx.add_init_script(
"""
try {
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
} catch (e) {}
"""
)
+11
View File
@@ -0,0 +1,11 @@
{
"name": "X",
"version": "1.0.0",
"domain": "twitter.com",
"author": "Sakura Downloader",
"description": "X (Twitter) media via Playwright + httpx. Session in runtime/x_profile.",
"requires_auth": true,
"tags": ["social", "media", "playwright"],
"min_platform_version": "1.0.0",
"dependencies": []
}
+946
View File
@@ -0,0 +1,946 @@
"""Playwright helpers for the X plugin."""
from __future__ import annotations
import asyncio
import json
import random
import re
from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin, urlparse
import httpx
from playwright.async_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError, async_playwright
from .browser_opts import launch_x_persistent_context
from pixiv_app.core.plugin.base import PluginParseError
try:
from pixiv_app.core.fingerprint import default_profile
except Exception: # pragma: no cover
default_profile = None
_GUEST_WALL_HINT = (
"X 需要先登录。\n"
"请在“插件管理”里点击“X 登录”完成登录,或把 Cookie 导出为 JSON 保存到 "
"runtime/x_cookies.json 后重试。"
)
if TYPE_CHECKING:
pass
class XPlaywrightClient:
def __init__(self, config: Any) -> None:
self.config = config
self._pw: Any = None
self.context: BrowserContext | None = None
self._response_store: dict[int, list[dict[str, Any]]] = {}
self._response_tasks: dict[int, set[asyncio.Task[Any]]] = {}
async def start(self) -> None:
if self.context is not None:
return
self._pw = await async_playwright().start()
profile = Path(self.config.profile_dir)
profile.mkdir(parents=True, exist_ok=True)
self.context = await launch_x_persistent_context(
self._pw,
profile,
headless=self.config.headless,
)
cookies = self._prepare_cookie_items(self._load_cookie_items())
if cookies:
loaded = await self._add_cookies_tolerant(cookies)
if loaded <= 0:
raise PluginParseError(
"X Cookie 注入失败:请重新导出 x.com cookies.txt,确认包含 auth_token 和 ct0 后再导入。"
)
async def close(self) -> None:
if self.context is not None:
await self.context.close()
self.context = None
self._response_tasks.clear()
self._response_store.clear()
if self._pw is not None:
await self._pw.stop()
self._pw = None
async def new_page_goto(self, url: str, *, wait_until: str = "domcontentloaded") -> Page:
assert self.context is not None
page = await self.context.new_page()
self._attach_response_collector(page)
timeout = getattr(self.config, "wait_timeout", 30000)
try:
await page.goto(url, wait_until=wait_until, timeout=timeout)
except PlaywrightTimeoutError as exc:
if page.url and page.url != "about:blank":
await asyncio.sleep(2.0)
return page
try:
await page.goto(url, wait_until="commit", timeout=min(timeout, 15000))
except PlaywrightTimeoutError as retry_exc:
raise PluginParseError(
"X 页面加载超时,可能是网络较慢、访问过快触发风控,或登录会话已失效。"
) from retry_exc
except Exception as retry_exc:
raise PluginParseError(f"X 页面加载失败: {retry_exc}") from retry_exc
if page.url == "about:blank":
raise PluginParseError("X 页面没有成功打开,请稍后重试。") from exc
await self.human_pause(2.0, 4.5)
await self.move_mouse_softly(page)
return page
async def human_pause(self, low: float | None = None, high: float | None = None) -> None:
low = float(low if low is not None else getattr(self.config, "min_action_delay", 3.0))
high = float(high if high is not None else getattr(self.config, "max_action_delay", 8.0))
if high < low:
high = low
await asyncio.sleep(random.uniform(low, high))
async def move_mouse_softly(self, page: Page) -> None:
try:
viewport = page.viewport_size or {"width": 1280, "height": 800}
width = int(viewport.get("width", 1280))
height = int(viewport.get("height", 800))
for _ in range(random.randint(1, 3)):
x = random.randint(max(12, width // 8), max(20, width - 24))
y = random.randint(max(12, height // 8), max(20, height - 24))
await page.mouse.move(x, y, steps=random.randint(8, 18))
await asyncio.sleep(random.uniform(0.2, 0.7))
except Exception:
pass
async def human_scroll_page(self, page: Page) -> float:
await self.move_mouse_softly(page)
viewport = page.viewport_size or {"height": 800}
height = int(viewport.get("height", 800))
delta = random.randint(max(260, height // 2), max(420, int(height * 1.2)))
if random.random() < 0.12:
delta = -random.randint(120, 280)
await page.mouse.wheel(0, delta)
if random.random() < 0.07:
await asyncio.sleep(random.uniform(30.0, 90.0))
await self.human_pause()
return float(await page.evaluate("document.body.scrollHeight"))
async def raise_if_guest_wall(self, page: Page) -> None:
await asyncio.sleep(0.45)
try:
url = page.url or ""
if "/i/flow/login" in url or "/i/flow/signup" in url:
raise PluginParseError(_GUEST_WALL_HINT)
except PluginParseError:
raise
except Exception:
pass
try:
wall = page.get_by_text(
re.compile(
r"注册 X|Sign up for X|登录|Sign in|Phone, email, or username",
re.I,
)
).first
if await wall.is_visible(timeout=2200):
raise PluginParseError(_GUEST_WALL_HINT)
except PluginParseError:
raise
except Exception:
pass
try:
dialog = page.locator('[role="dialog"]').first
if await dialog.is_visible(timeout=600):
inner = (await dialog.inner_text())[:800]
if ("注册" in inner or "Sign up" in inner) and ("Google" in inner or "Apple" in inner or "Next" in inner):
raise PluginParseError(_GUEST_WALL_HINT)
except PluginParseError:
raise
except Exception:
pass
async def wait_for_main_content(self, page: Page) -> None:
timeout = min(15000, getattr(self.config, "wait_timeout", 30000))
for sel in ('article[data-testid="tweet"]', "article", '[data-testid="tweetPhoto"]'):
try:
await page.wait_for_selector(sel, timeout=timeout)
return
except Exception:
continue
async def wait_for_media(self, page: Page) -> bool:
timeout = 10000
for sel in (
'[data-testid="tweetPhoto"]',
'img[src*="pbs.twimg.com/media"]',
"video",
"video source",
):
try:
await page.wait_for_selector(sel, timeout=timeout)
return True
except Exception:
continue
return False
async def extract_media_urls(self, page: Page) -> list[str]:
urls: list[str] = []
seen: set[str] = set()
imgs = await page.query_selector_all('img[src*="pbs.twimg.com/media"]')
for img in imgs:
src = await img.get_attribute("src")
if src and "profile_images" not in src:
orig = self._original_tw_image_url(src)
if orig and orig not in seen:
seen.add(orig)
urls.append(orig)
for sel in ('[data-testid="tweetPhoto"] img', "article img"):
elements = await page.query_selector_all(sel)
for el in elements:
src = await el.get_attribute("src")
if src and "pbs.twimg.com/media" in src:
orig = self._original_tw_image_url(src)
if orig and orig not in seen:
seen.add(orig)
urls.append(orig)
try:
raw_video_urls = await page.evaluate(
"""
() => {
const values = [];
for (const el of document.querySelectorAll('video, video source')) {
const candidates = [
el.currentSrc,
el.src,
el.getAttribute?.('src'),
el.getAttribute?.('data-src'),
el.getAttribute?.('poster'),
];
for (const value of candidates) {
if (typeof value === 'string' && value.trim()) {
values.push(value.trim());
}
}
}
return values;
}
"""
)
except Exception:
raw_video_urls = []
for src in raw_video_urls:
normalized = self._normalize_media_url(str(src))
if normalized and self._is_video_url(normalized) and normalized not in seen:
seen.add(normalized)
urls.append(normalized)
try:
html = await page.content()
except Exception:
html = ""
for image_url in self._extract_image_urls_from_html(html):
if image_url not in seen:
seen.add(image_url)
urls.append(image_url)
for pattern in (
r"https?://video\.twimg\.com/[^\s\"'<>\\]+",
r"https:\\/\\/video\.twimg\.com\\/[^\s\"'<>]+",
):
for match in re.findall(pattern, html):
normalized = self._normalize_media_url(match)
if normalized and self._is_video_url(normalized) and normalized not in seen:
seen.add(normalized)
urls.append(normalized)
return urls
async def extract_media_urls_from_api_responses(self, page: Page) -> list[str]:
await self.wait_for_response_capture(page)
payloads = self._response_store.get(id(page), [])
if not payloads:
return []
urls: list[str] = []
seen: set[str] = set()
def add(url: str | None) -> None:
if not url:
return
normalized = self._normalize_media_url(str(url))
if not normalized:
return
if "pbs.twimg.com/media" in normalized:
normalized = self._original_tw_image_url(normalized)
if normalized not in seen:
seen.add(normalized)
urls.append(normalized)
for payload in payloads:
data = payload.get("json")
if data is None:
continue
for media_item in self._find_media_entities(data):
media_url = media_item.get("media_url_https") or media_item.get("media_url")
media_type = str(media_item.get("type") or "").lower()
if media_url and media_type in {"photo", "animated_gif", "video", ""}:
add(str(media_url))
video_info = media_item.get("video_info")
if isinstance(video_info, dict):
best_video = self._pick_best_video_variant(video_info)
add(best_video)
return urls
async def extract_media_records_from_api_responses(self, page: Page) -> list[dict[str, Any]]:
await self.wait_for_response_capture(page)
payloads = self._response_store.get(id(page), [])
if not payloads:
return []
records: list[dict[str, Any]] = []
seen: set[tuple[str, str]] = set()
def add_record(kind: str, url: str | None, *, thumbnail: str | None = None) -> None:
if not url:
return
normalized = self._normalize_media_url(str(url))
if not normalized:
return
key = (kind, normalized)
if key in seen:
return
seen.add(key)
records.append(
{
"type": kind,
"url": normalized,
"thumbnail": self._normalize_media_url(thumbnail) if thumbnail else None,
}
)
for payload in payloads:
data = payload.get("json")
if data is None:
continue
for media_item in self._find_media_entities(data):
media_type = str(media_item.get("type") or "").lower()
thumb_url = media_item.get("media_url_https") or media_item.get("media_url")
if media_type == "photo":
add_record("image", thumb_url)
continue
video_info = media_item.get("video_info")
if isinstance(video_info, dict):
best_video = self._pick_best_video_variant(video_info)
if best_video:
add_record("video", best_video, thumbnail=thumb_url)
continue
return records
async def wait_for_response_capture(self, page: Page, timeout: float = 2.0) -> None:
page_id = id(page)
tasks = self._response_tasks.get(page_id)
if not tasks:
return
deadline = asyncio.get_running_loop().time() + timeout
while tasks and asyncio.get_running_loop().time() < deadline:
pending = [task for task in list(tasks) if not task.done()]
if not pending:
break
_, pending = await asyncio.wait(pending, timeout=0.2)
tasks = self._response_tasks.get(page_id)
if not tasks:
break
async def scroll_and_collect_media(self, page: Page, max_scrolls: int = 12, max_items: int = 40) -> list[str]:
collected: list[str] = []
seen: set[str] = set()
last_height = 0.0
for _ in range(max_scrolls):
batch: list[str] = []
for item in await self.extract_media_records_from_api_responses(page):
url = item.get("url")
if isinstance(url, str) and url:
batch.append(url)
batch.extend(await self.extract_media_urls(page))
for url in batch:
if url not in seen:
seen.add(url)
collected.append(url)
if len(collected) >= max_items:
return collected
try:
height = await self.human_scroll_page(page)
except Exception:
break
if height == last_height:
break
last_height = float(height)
return collected
async def scroll_and_collect_tweet_urls(self, page: Page, limit: int = 40) -> list[str]:
found: list[str] = []
seen: set[str] = set()
stale_rounds = 0
max_rounds = 25
for _ in range(max_rounds):
if len(found) >= limit:
break
before = len(found)
links = await page.query_selector_all('a[href*="/status/"]')
for link in links:
href = await link.get_attribute("href")
if not href or "/status/" not in href:
continue
if href.startswith("/"):
full = f"https://x.com{href}"
elif href.startswith("http"):
full = href
else:
continue
full = re.sub(r"\?.*$", "", full.split("#")[0])
if full not in seen:
seen.add(full)
found.append(full)
if len(found) >= limit:
break
if len(found) == before:
stale_rounds += 1
if stale_rounds >= 5:
break
else:
stale_rounds = 0
try:
await self.human_scroll_page(page)
except Exception:
break
return found[:limit]
async def get_author_info(self, page: Page) -> dict[str, str]:
info = {"name": "", "username": ""}
try:
block = await page.query_selector('[data-testid="User-Name"]')
if block:
text = await block.inner_text()
lines = [line.strip() for line in text.split("\n") if line.strip()]
if lines:
info["name"] = lines[0]
for line in lines:
if line.startswith("@"):
info["username"] = line.lstrip("@")
break
except Exception:
pass
return info
async def get_user_display_name(self, page: Page, fallback: str) -> dict[str, str]:
info = {"name": fallback, "username": fallback}
try:
block = await page.query_selector('[data-testid="UserName"]')
if block:
text = await block.inner_text()
lines = [line.strip() for line in text.split("\n") if line.strip()]
if lines:
info["name"] = lines[0]
except Exception:
pass
return info
async def get_tweet_text(self, page: Page) -> str:
try:
el = await page.query_selector('[data-testid="tweetText"]')
if el:
return (await el.inner_text()).strip()
except Exception:
pass
return ""
async def get_tweet_date(self, page: Page) -> str:
try:
time_el = await page.query_selector("article time")
if time_el:
return (await time_el.get_attribute("datetime")) or ""
except Exception:
pass
return ""
async def download_image(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
save_path.mkdir(parents=True, exist_ok=True)
name = url.split("/")[-1].split("?")[0]
if not any(name.lower().endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp")):
name += ".jpg"
target = save_path / name
if target.exists() and target.stat().st_size > 256:
return target
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
content_type = response.headers.get("content-type", "").lower()
if response.status_code == 200 and "text/html" not in content_type:
target.write_bytes(response.content)
if target.stat().st_size > 256:
return target
target.unlink(missing_ok=True)
return None
async def download_video(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
save_path.mkdir(parents=True, exist_ok=True)
async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
if response.status_code != 200:
return None
final_url = str(response.url)
content_type = response.headers.get("content-type", "").lower()
body_text = response.text if self._looks_like_hls(final_url, content_type, "") or "text" in content_type else ""
if "text/html" in content_type and not self._looks_like_hls(final_url, content_type, body_text):
return None
if self._looks_like_hls(final_url, content_type, body_text):
return await self._download_hls_playlist(
client,
playlist_url=final_url,
playlist_text=body_text,
save_path=save_path,
referer=referer,
)
target = self._video_target_path(final_url, save_path)
if target.exists() and target.stat().st_size > 1024:
return target
target.write_bytes(response.content)
if target.stat().st_size <= 1024:
target.unlink(missing_ok=True)
return None
return target
async def _download_hls_playlist(
self,
client: httpx.AsyncClient,
*,
playlist_url: str,
playlist_text: str,
save_path: Path,
referer: str | None,
) -> Path | None:
selected_url, selected_text = await self._resolve_hls_playlist(
client,
playlist_url=playlist_url,
playlist_text=playlist_text,
referer=referer,
)
if not selected_text:
return None
segment_urls, init_url = self._parse_hls_segments(selected_url, selected_text)
if not segment_urls:
return None
target = self._video_target_path(selected_url, save_path, segment_urls=segment_urls, init_url=init_url)
if target.exists() and target.stat().st_size > 1024:
return target
with target.open("wb") as fh:
if init_url:
init_resp = await client.get(init_url, headers=self._media_request_headers(init_url, referer=referer))
if init_resp.status_code != 200:
target.unlink(missing_ok=True)
return None
fh.write(init_resp.content)
for segment_url in segment_urls:
seg_resp = await client.get(
segment_url,
headers=self._media_request_headers(segment_url, referer=referer),
)
if seg_resp.status_code != 200:
target.unlink(missing_ok=True)
return None
fh.write(seg_resp.content)
if target.stat().st_size <= 1024:
target.unlink(missing_ok=True)
return None
return target
async def _resolve_hls_playlist(
self,
client: httpx.AsyncClient,
*,
playlist_url: str,
playlist_text: str,
referer: str | None,
) -> tuple[str, str]:
if "#EXT-X-STREAM-INF" not in playlist_text:
return playlist_url, playlist_text
variants = self._parse_hls_variants(playlist_url, playlist_text)
for variant_url in variants:
resp = await client.get(variant_url, headers=self._media_request_headers(variant_url, referer=referer))
if resp.status_code == 200:
return str(resp.url), resp.text
return playlist_url, playlist_text
def _parse_hls_variants(self, playlist_url: str, playlist_text: str) -> list[str]:
variants: list[tuple[int, str]] = []
pending_bandwidth = 0
for raw_line in playlist_text.splitlines():
line = raw_line.strip()
if not line:
continue
if line.startswith("#EXT-X-STREAM-INF"):
match = re.search(r"BANDWIDTH=(\d+)", line)
pending_bandwidth = int(match.group(1)) if match else 0
continue
if line.startswith("#"):
continue
variants.append((pending_bandwidth, urljoin(playlist_url, line)))
pending_bandwidth = 0
variants.sort(key=lambda item: item[0], reverse=True)
return [url for _, url in variants]
def _parse_hls_segments(self, playlist_url: str, playlist_text: str) -> tuple[list[str], str | None]:
segments: list[str] = []
init_url: str | None = None
for raw_line in playlist_text.splitlines():
line = raw_line.strip()
if not line:
continue
if line.startswith("#EXT-X-MAP:"):
match = re.search(r'URI="([^"]+)"', line)
if match:
init_url = urljoin(playlist_url, match.group(1))
continue
if line.startswith("#"):
continue
segments.append(urljoin(playlist_url, line))
return segments, init_url
def _media_request_headers(self, url: str, *, referer: str | None = None) -> dict[str, str]:
headers = {
"User-Agent": self._ua(),
"Accept": "*/*",
"Referer": referer or "https://x.com/",
"Origin": "https://x.com",
}
cookie_header = self._cookie_header(url)
if cookie_header:
headers["Cookie"] = cookie_header
csrf_token = self._cookie_value("ct0")
if csrf_token:
headers["x-csrf-token"] = csrf_token
return headers
def _cookie_header(self, url: str) -> str:
pairs: list[str] = []
for item in self._load_cookie_items():
name = str(item.get("name") or "").strip()
value = str(item.get("value") or "")
if name:
pairs.append(f"{name}={value}")
return "; ".join(pairs)
def _cookie_value(self, name: str) -> str:
for item in self._load_cookie_items():
if str(item.get("name") or "") == name:
return str(item.get("value") or "")
return ""
def _load_cookie_items(self) -> list[dict[str, Any]]:
cookie_path = Path(getattr(self.config, "cookie_file", "") or "")
if not cookie_path.is_file():
return []
try:
payload = json.loads(cookie_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return []
return payload if isinstance(payload, list) else []
async def _add_cookies_tolerant(self, cookies: list[dict[str, Any]]) -> int:
assert self.context is not None
try:
await self.context.add_cookies(cookies)
return len(cookies)
except Exception:
pass
loaded = 0
for cookie in cookies:
try:
await self.context.add_cookies([cookie])
loaded += 1
except Exception:
continue
return loaded
@staticmethod
def _prepare_cookie_items(cookies: list[dict[str, Any]]) -> list[dict[str, Any]]:
prepared_by_key: dict[tuple[str, str, str], dict[str, Any]] = {}
for cookie in cookies:
name = str(cookie.get("name") or "").strip()
domain = str(cookie.get("domain") or "").strip()
if not name or not domain:
continue
item: dict[str, Any] = {
"name": name,
"value": str(cookie.get("value") or ""),
"domain": domain,
"path": str(cookie.get("path") or "/"),
"secure": bool(cookie.get("secure", False)),
"httpOnly": bool(cookie.get("httpOnly", False)),
}
expires = cookie.get("expires")
if expires is not None:
try:
expires_value = int(float(expires))
if expires_value > 0:
item["expires"] = expires_value
except (TypeError, ValueError):
pass
same_site = str(cookie.get("sameSite") or "").strip()
if same_site in {"Strict", "Lax", "None"}:
if same_site == "None" and not item["secure"]:
item["sameSite"] = "Lax"
else:
item["sameSite"] = same_site
key = (domain.lower(), str(item["path"]), name)
prepared_by_key[key] = item
return list(prepared_by_key.values())
def _video_target_path(
self,
url: str,
save_path: Path,
*,
segment_urls: list[str] | None = None,
init_url: str | None = None,
) -> Path:
parsed = urlparse(url)
name = Path(parsed.path).name or "x_video"
if name.endswith(".m3u8") or not Path(name).suffix:
if segment_urls:
seg_name = Path(urlparse(segment_urls[0]).path).name
if seg_name:
name = seg_name
if init_url:
init_name = Path(urlparse(init_url).path).name
if init_name:
name = init_name
suffix = Path(name).suffix.lower()
if suffix in {".m3u8", ".m4s", ".mpd", ""}:
name = f"{Path(name).stem or 'x_video'}.mp4"
return save_path / name
def _looks_like_hls(self, url: str, content_type: str, body: str) -> bool:
low_url = url.lower()
if ".m3u8" in low_url:
return True
if "mpegurl" in content_type or "vnd.apple.mpegurl" in content_type:
return True
return "#EXTM3U" in body[:256]
def _attach_response_collector(self, page: Page) -> None:
page_id = id(page)
bucket: list[dict[str, Any]] = []
self._response_store[page_id] = bucket
self._response_tasks[page_id] = set()
def handle_response(response: Any) -> None:
task = asyncio.create_task(self._capture_response(page, response))
task_bucket = self._response_tasks.setdefault(page_id, set())
task_bucket.add(task)
def cleanup(done_task: asyncio.Task[Any]) -> None:
current_bucket = self._response_tasks.get(page_id)
if current_bucket is not None:
current_bucket.discard(done_task)
task.add_done_callback(cleanup)
page.on("response", handle_response)
async def _capture_response(self, page: Page, response: Any) -> None:
url = str(getattr(response, "url", "") or "")
if not self._should_capture_response(url):
return
try:
headers = await response.all_headers()
except Exception:
headers = {}
content_type = str(headers.get("content-type", "")).lower()
if "json" not in content_type and "javascript" not in content_type and "text/plain" not in content_type:
return
try:
text = await response.text()
except Exception:
return
if not text or len(text) < 2:
return
parsed_json: Any = None
try:
parsed_json = json.loads(text)
except Exception:
return
self._response_store.setdefault(id(page), []).append(
{
"url": url,
"status": getattr(response, "status", None),
"json": parsed_json,
}
)
def _should_capture_response(self, url: str) -> bool:
low = url.lower()
return any(
token in low
for token in (
"/i/api/graphql/",
"/graphql/",
"tweetdetail",
"tweetresultbyrestid",
"conversation",
"adaptive.json",
)
)
def _find_media_entities(self, payload: Any) -> list[dict[str, Any]]:
media_entities: list[dict[str, Any]] = []
def walk(node: Any) -> None:
if isinstance(node, dict):
if "media_url_https" in node or "video_info" in node:
media_entities.append(node)
for value in node.values():
walk(value)
return
if isinstance(node, list):
for item in node:
walk(item)
walk(payload)
return media_entities
def _pick_best_video_variant(self, video_info: dict[str, Any]) -> str | None:
variants = video_info.get("variants")
if not isinstance(variants, list):
return None
best_mp4: tuple[int, str] | None = None
best_hls: str | None = None
for item in variants:
if not isinstance(item, dict):
continue
url = item.get("url")
if not isinstance(url, str) or not url:
continue
low_url = url.lower()
content_type = str(item.get("content_type") or "").lower()
try:
bitrate = int(item.get("bitrate") or 0)
except (TypeError, ValueError):
bitrate = 0
if "mp4" in content_type or ".mp4" in low_url:
if best_mp4 is None or bitrate > best_mp4[0]:
best_mp4 = (bitrate, url)
elif "mpegurl" in content_type or ".m3u8" in low_url:
best_hls = url
if best_mp4 is not None:
return best_mp4[1]
return best_hls
def _extract_image_urls_from_html(self, html: str) -> list[str]:
if not html:
return []
seen: set[str] = set()
urls: list[str] = []
patterns = (
r"https?://pbs\.twimg\.com/media/[^\s\"'<>\\]+",
r"https:\\/\\/pbs\.twimg\.com\\/media\\/[^\s\"'<>]+",
)
for pattern in patterns:
for match in re.findall(pattern, html):
normalized = self._normalize_media_url(match)
if "pbs.twimg.com/media" not in normalized:
continue
orig = self._original_tw_image_url(normalized)
if orig not in seen:
seen.add(orig)
urls.append(orig)
return urls
def _ua(self) -> str:
if default_profile:
return default_profile("x").user_agent
return (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
@staticmethod
def _original_tw_image_url(url: str) -> str:
if "pbs.twimg.com/media" not in url:
return url
base = url.split("?")[0]
return f"{base}?format=jpg&name=orig" if "format=" not in url else re.sub(
r"name=\w+",
"name=orig",
url,
)
@staticmethod
def _normalize_media_url(url: str) -> str:
value = url.strip().replace("\\/", "/")
if value.startswith("//"):
value = f"https:{value}"
return value
@staticmethod
def _is_video_url(url: str) -> bool:
low = url.lower()
return any(
token in low
for token in (
".mp4",
".m3u8",
"video.twimg.com",
"/amplify_video/",
"/ext_tw_video/",
)
)
+504
View File
@@ -0,0 +1,504 @@
"""
X (Twitter) plugin.
Uses Playwright for rendering and httpx for media downloads.
"""
from __future__ import annotations
import asyncio
import json
import queue
import random
import re
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from .playwright_client import XPlaywrightClient
from .x_login import run_interactive_login as x_login_flow
from .x_urls import (
extract_search_query,
extract_tweet_id,
extract_username_and_kind,
is_search_url,
is_status_url,
normalize_url,
)
from pixiv_app.core.paths import runtime_path
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
@dataclass
class XPluginConfig:
cookie_file: str = ""
profile_dir: str = ""
headless: bool = False
safety_mode: bool = True
wait_timeout: int = 45000
timeline_tweet_limit: int = 8
media_scroll_rounds: int = 5
max_media_items: int = 40
parse_cooldown_seconds: float = 60.0
min_action_delay: float = 4.0
max_action_delay: float = 11.0
download_delay_min: float = 2.0
download_delay_max: float = 5.0
batch_pause_min: float = 12.0
batch_pause_max: float = 28.0
batch_pause_after_min: int = 2
batch_pause_after_max: int = 4
max_parse_per_hour: int = 24
max_download_files_per_hour: int = 60
max_consecutive_failures: int = 3
failure_cooldown_seconds: float = 1800.0
sync_parse_timeout_seconds: float = 120.0
class XPlugin(BasePlugin):
name = "X"
domain = "twitter.com"
version = "1.0.0"
_HANDLE_PATTERNS = (
re.compile(r"(?:twitter|x)\.com/\w+/status/\d+", re.I),
re.compile(r"(?:twitter|x)\.com/[^/]+/media\b", re.I),
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/?$", re.I),
re.compile(r"(?:twitter|x)\.com/[A-Za-z0-9_]{1,15}/media/?$", re.I),
re.compile(r"(?:twitter|x)\.com/search\?", re.I),
)
def __init__(self) -> None:
self.config = XPluginConfig(
cookie_file=str(runtime_path("x_cookies.json")),
profile_dir=str(runtime_path("x_profile")),
headless=False,
)
self._cache_file = runtime_path("x_download_cache.json")
self._playwright_lock = threading.Lock()
self._last_parse_at = 0.0
self._parse_timestamps: list[float] = []
self._download_timestamps: list[float] = []
self._consecutive_failures = 0
self._suspended_until = 0.0
def can_handle(self, url: str) -> bool:
text = url.strip()
if not text:
return False
candidate = text if "://" in text else f"https://{text.lstrip('/')}"
try:
hostname = (urlsplit(candidate).hostname or "").lower().rstrip(".")
except ValueError:
return False
if hostname not in {"x.com", "www.x.com", "twitter.com", "www.twitter.com", "mobile.twitter.com"}:
return False
return any(pattern.search(text) for pattern in self._HANDLE_PATTERNS)
def validate(self) -> bool:
try:
import playwright # noqa: F401
except ImportError:
return False
return True
def get_headers(self) -> dict[str, str]:
return {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
}
def parse(self, url: str) -> list[Resource]:
self._enforce_account_safety("parse", units=1)
try:
with self._playwright_lock:
self._respect_parse_cooldown()
try:
result = asyncio.run(self._parse_async(normalize_url(url)))
self._record_success()
self._record_usage(self._parse_timestamps, 1)
return result
finally:
self._last_parse_at = time.monotonic()
except PluginParseError:
self._record_failure()
raise
except Exception as exc:
self._record_failure()
raise PluginParseError(f"X 解析失败: {exc}") from exc
def parse_for_sync(self, url: str) -> list[Resource]:
self._enforce_account_safety("parse", units=1)
timeout = max(30.0, float(self.config.sync_parse_timeout_seconds))
try:
with self._playwright_lock:
result = asyncio.run(asyncio.wait_for(self._parse_async(normalize_url(url)), timeout=timeout))
self._record_success()
self._record_usage(self._parse_timestamps, 1)
self._last_parse_at = time.monotonic()
return result
except asyncio.TimeoutError as exc:
self._record_failure()
raise PluginParseError(f"X 同步解析超时({int(timeout)} 秒),请稍后重试或检查登录状态。") from exc
except PluginParseError:
self._record_failure()
raise
except Exception as exc:
self._record_failure()
raise PluginParseError(f"X 同步解析失败: {exc}") from exc
def download(self, resource: Resource, save_path: Path) -> list[Path]:
requested = max(1, len(resource.files))
self._enforce_account_safety("download", units=requested)
try:
with self._playwright_lock:
result = asyncio.run(self._download_async(resource, save_path))
self._record_success()
self._record_usage(self._download_timestamps, max(1, len(result) or requested))
return result
except Exception as exc:
self._record_failure()
raise PluginParseError(f"X 下载失败: {exc}") from exc
def reset_playwright_session(self) -> None:
return None
def _respect_parse_cooldown(self) -> None:
elapsed = time.monotonic() - self._last_parse_at
remain = float(self.config.parse_cooldown_seconds) - elapsed
if 0 < remain < self.config.parse_cooldown_seconds:
time.sleep(remain)
def _enforce_account_safety(self, action: str, *, units: int) -> None:
if not self.config.safety_mode:
return
now = time.monotonic()
if self._suspended_until > now:
remaining = int(self._suspended_until - now)
minutes = max(1, remaining // 60)
raise PluginParseError(f"X 账号安全模式已暂停操作,请约 {minutes} 分钟后再试。")
self._trim_usage(now)
if action == "parse":
if len(self._parse_timestamps) + units > self.config.max_parse_per_hour:
raise PluginParseError("X 账号安全模式已触发每小时解析上限,请稍后再试。")
return
if action == "download":
if len(self._download_timestamps) + units > self.config.max_download_files_per_hour:
raise PluginParseError("X 账号安全模式已触发每小时下载上限,请稍后再试。")
def _trim_usage(self, now: float | None = None) -> None:
current = now if now is not None else time.monotonic()
window = 3600.0
self._parse_timestamps = [item for item in self._parse_timestamps if current - item < window]
self._download_timestamps = [item for item in self._download_timestamps if current - item < window]
def _record_usage(self, bucket: list[float], units: int) -> None:
now = time.monotonic()
self._trim_usage(now)
for _ in range(max(1, units)):
bucket.append(now)
def _record_success(self) -> None:
self._consecutive_failures = 0
def _record_failure(self) -> None:
if not self.config.safety_mode:
return
self._consecutive_failures += 1
if self._consecutive_failures >= self.config.max_consecutive_failures:
self._suspended_until = time.monotonic() + float(self.config.failure_cooldown_seconds)
self._consecutive_failures = 0
async def _parse_async(self, url: str) -> list[Resource]:
if not self.validate():
raise PluginParseError("未安装 Playwright。请先执行: pip install playwright && playwright install chromium")
client = XPlaywrightClient(self.config)
await client.start()
try:
if is_status_url(url):
return await self._parse_single_tweet(client, url)
if is_search_url(url):
return await self._parse_search_media(client, url)
user, kind = extract_username_and_kind(url)
if user and kind == "media":
return await self._parse_user_media(client, url, user)
if user and kind == "profile":
return await self._parse_user_timeline(client, url, user)
raise PluginParseError(f"无法识别的 X 链接: {url}")
finally:
await client.close()
async def _parse_single_tweet(self, client: XPlaywrightClient, url: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
await client.wait_for_main_content(page)
await client.wait_for_media(page)
try:
await page.wait_for_load_state("networkidle", timeout=5000)
except Exception:
pass
tweet_id = extract_tweet_id(url) or ""
author = await client.get_author_info(page)
media_records = await client.extract_media_records_from_api_responses(page)
media_urls = await client.extract_media_urls(page)
if not media_urls:
try:
await page.mouse.wheel(0, 500)
await asyncio.sleep(1.2)
except Exception:
pass
media_urls = await client.extract_media_urls(page)
if not media_records and not media_urls:
media_records = await client.extract_media_records_from_api_responses(page)
if not media_urls:
media_urls = [item["url"] for item in media_records] or await client.extract_media_urls_from_api_responses(page)
content = await client.get_tweet_text(page)
created = await client.get_tweet_date(page)
finally:
await page.close()
if not media_records and not media_urls:
raise PluginParseError("未在该推文解析到媒体,可能需要重新登录,或该页面结构已变化。")
resources: list[Resource] = []
if media_records:
for index, item in enumerate(media_records):
file_url = item["url"]
thumb = item.get("thumbnail") or file_url
resources.append(
Resource(
id=f"{tweet_id}_{index}",
url=url,
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
author=author.get("name", ""),
author_id=author.get("username", ""),
files=[{"url": file_url, "type": item.get("type", self._media_type(file_url)), "referer": url}],
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
thumbnail=thumb,
created_at=created or None,
)
)
return resources
for index, media_url in enumerate(media_urls):
resources.append(
Resource(
id=f"{tweet_id}_{index}",
url=url,
title=(content[:120] if content else f"tweet_{tweet_id}") or f"tweet_{tweet_id}",
author=author.get("name", ""),
author_id=author.get("username", ""),
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
metadata={"tweet_id": tweet_id, "full_text": content, "author": author},
thumbnail=media_urls[0] if index == 0 else None,
created_at=created or None,
)
)
return resources
async def _parse_search_media(self, client: XPlaywrightClient, url: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
media_urls = await client.scroll_and_collect_media(
page,
max_scrolls=self.config.media_scroll_rounds,
max_items=self.config.max_media_items,
)
finally:
await page.close()
query = extract_search_query(url) or "search"
resources: list[Resource] = []
for index, media_url in enumerate(media_urls):
resources.append(
Resource(
id=f"search_{abs(hash(query))}_{index}",
url=url,
title=f"X search - {query}",
author=query,
author_id="search",
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
metadata={"source": "search_media", "query": query},
thumbnail=media_url,
created_at=None,
)
)
return resources
async def _parse_user_media(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
info = await client.get_user_display_name(page, username)
media_urls = await client.scroll_and_collect_media(
page,
max_scrolls=self.config.media_scroll_rounds,
max_items=self.config.max_media_items,
)
finally:
await page.close()
resources: list[Resource] = []
for index, media_url in enumerate(media_urls):
resources.append(
Resource(
id=f"{username}_media_{index}",
url=url,
title=f"{info.get('name', username)} - media",
author=info.get("name", username),
author_id=username,
files=[{"url": media_url, "type": self._media_type(media_url), "referer": url}],
metadata={"source": "media_tab", "username": username},
thumbnail=media_url,
created_at=None,
)
)
return resources
async def _parse_user_timeline(self, client: XPlaywrightClient, url: str, username: str) -> list[Resource]:
page = await client.new_page_goto(url)
try:
await client.raise_if_guest_wall(page)
tweet_urls = await client.scroll_and_collect_tweet_urls(
page,
limit=self.config.timeline_tweet_limit,
)
finally:
await page.close()
all_results: list[Resource] = []
for tweet_url in tweet_urls:
try:
all_results.extend(await self._parse_single_tweet(client, tweet_url))
except Exception:
continue
return all_results
async def _download_async(self, resource: Resource, save_path: Path) -> list[Path]:
client = XPlaywrightClient(self.config)
results: list[Path] = []
subdir = resource.metadata.get("save_subdir")
target_base = save_path / subdir if subdir else save_path
cache = self._load_download_cache()
batch_pause_after = random.randint(self.config.batch_pause_after_min, self.config.batch_pause_after_max)
try:
for index, item in enumerate(resource.files, start=1):
file_url = item.get("url")
if not file_url:
continue
cached = cache.get(file_url)
if cached and Path(cached).is_file():
results.append(Path(cached))
continue
kind = item.get("type", "image")
referer = item.get("referer") or resource.url
if kind == "video":
saved = await client.download_video(file_url, target_base, referer=referer)
else:
saved = await client.download_image(file_url, target_base, referer=referer)
if saved is None:
continue
cache[file_url] = str(saved)
results.append(saved)
await asyncio.sleep(
random.uniform(self.config.download_delay_min, self.config.download_delay_max)
* random.uniform(0.8, 1.5)
)
if index % batch_pause_after == 0:
await asyncio.sleep(random.uniform(self.config.batch_pause_min, self.config.batch_pause_max))
batch_pause_after = random.randint(
self.config.batch_pause_after_min,
self.config.batch_pause_after_max,
)
self._save_download_cache(cache)
return results
finally:
await client.close()
def _load_download_cache(self) -> dict[str, str]:
try:
if self._cache_file.is_file():
payload = json.loads(self._cache_file.read_text(encoding="utf-8"))
if isinstance(payload, dict):
return {str(key): str(value) for key, value in payload.items()}
except Exception:
pass
return {}
def _save_download_cache(self, payload: dict[str, str]) -> None:
self._cache_file.parent.mkdir(parents=True, exist_ok=True)
self._cache_file.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def run_interactive_login(
self,
*,
proceed: threading.Event | None = None,
ready_signal: queue.Queue[bool] | None = None,
) -> bool:
path = Path(self.config.profile_dir)
try:
with self._playwright_lock:
return asyncio.run(x_login_flow(path, proceed=proceed, ready_signal=ready_signal))
except Exception:
return False
def set_headless(self, headless: bool) -> None:
self.config.headless = headless
@staticmethod
def _media_type(url: str) -> str:
low = url.lower()
if (
"video" in low
or ".mp4" in low
or ".m3u8" in low
or "video.twimg.com" in low
or "/amplify_video/" in low
or "/ext_tw_video/" in low
):
return "video"
return "image"
def get_config_schema(self) -> dict[str, Any]:
return {
"headless": {"type": "bool", "label": "无头模式", "default": False},
"safety_mode": {"type": "bool", "label": "账号安全模式", "default": True},
"timeline_tweet_limit": {"type": "int", "label": "时间线采样条数", "default": 8, "min": 1, "max": 30},
"media_scroll_rounds": {"type": "int", "label": "媒体页滚动轮数", "default": 5, "min": 1, "max": 12},
"max_media_items": {"type": "int", "label": "单次最多媒体数", "default": 40, "min": 1, "max": 120},
"parse_cooldown_seconds": {"type": "float", "label": "解析冷却秒数", "default": 60.0, "min": 10, "max": 600},
"min_action_delay": {"type": "float", "label": "最小操作延迟秒数", "default": 4.0, "min": 1, "max": 60},
"max_action_delay": {"type": "float", "label": "最大操作延迟秒数", "default": 11.0, "min": 2, "max": 90},
"download_delay_min": {"type": "float", "label": "下载后最小停顿秒数", "default": 2.0, "min": 0, "max": 60},
"download_delay_max": {"type": "float", "label": "下载后最大停顿秒数", "default": 5.0, "min": 0, "max": 120},
"max_parse_per_hour": {"type": "int", "label": "每小时最大解析次数", "default": 24, "min": 1, "max": 120},
"max_download_files_per_hour": {"type": "int", "label": "每小时最大下载文件数", "default": 60, "min": 1, "max": 300},
"max_consecutive_failures": {"type": "int", "label": "连续失败暂停阈值", "default": 3, "min": 1, "max": 10},
"failure_cooldown_seconds": {"type": "float", "label": "连续失败暂停秒数", "default": 1800.0, "min": 60, "max": 86400},
}
plugin_class = XPlugin
+8
View File
@@ -0,0 +1,8 @@
"""DOM selectors for X pages used by Sakura desktop sync."""
PROFILE_LINK = 'a[data-testid="AppTabBar_Profile_Link"]'
USER_CELL = '[data-testid="UserCell"]'
USER_NAME = '[data-testid="UserName"]'
USER_AVATAR = 'img[src*="profile_images"]'
USER_LINK = 'a[href^="/"][role="link"]'
FOLLOWING_URL_SUFFIX = "/following"
+18
View File
@@ -0,0 +1,18 @@
"""Install Python deps and Chromium for the X plugin."""
from __future__ import annotations
import subprocess
import sys
def main() -> None:
deps = ["playwright", "httpx"]
for dep in deps:
subprocess.check_call([sys.executable, "-m", "pip", "install", dep])
subprocess.check_call([sys.executable, "-m", "playwright", "install", "chromium"])
print("Done. Restart the app and reload plugins.")
if __name__ == "__main__":
main()
+59
View File
@@ -0,0 +1,59 @@
"""Interactive X login via persistent Chromium profile."""
from __future__ import annotations
import asyncio
import queue
import shutil
import threading
from pathlib import Path
from playwright.async_api import async_playwright
from .browser_opts import launch_x_persistent_context
from pixiv_app.services.x_login_flow import goto_x_login_entry
async def run_interactive_login(
profile_dir: Path,
*,
proceed: threading.Event | None = None,
ready_signal: queue.Queue[bool] | None = None,
) -> bool:
"""
Open visible Chromium with persistent ``profile_dir``.
Optionally notify GUI via ``ready_signal`` after the login page loads, then block until
``proceed`` is set (e.g. user clicked OK after logging in). If ``proceed`` is None,
falls back to terminal ``input()``.
"""
profile_dir.mkdir(parents=True, exist_ok=True)
playwright = await async_playwright().start()
context = None
loop = asyncio.get_running_loop()
try:
context = await launch_x_persistent_context(playwright, profile_dir, headless=False)
page = context.pages[0] if context.pages else await context.new_page()
await goto_x_login_entry(page, timeout=60_000)
if ready_signal is not None:
ready_signal.put(True)
if proceed is not None:
await loop.run_in_executor(None, proceed.wait)
else:
await loop.run_in_executor(
None,
lambda: input("\n在浏览器中完成 X 登录后,回到控制台按 Enter...\n"),
)
return True
finally:
if context is not None:
await context.close()
await playwright.stop()
def clear_profile(profile_dir: Path) -> None:
if profile_dir.exists():
shutil.rmtree(profile_dir, ignore_errors=True)
profile_dir.mkdir(parents=True, exist_ok=True)
+57
View File
@@ -0,0 +1,57 @@
"""URL helpers for X / Twitter."""
from __future__ import annotations
import re
from urllib.parse import parse_qs, urlparse
STATUS_RE = re.compile(
r"(?:https?://)?(?:www\.)?(?:twitter|x)\.com/[^/]+/status/(\d+)",
re.IGNORECASE,
)
PROFILE_OR_MEDIA_RE = re.compile(
r"(?:https?://)?(?:www\.)?(?:twitter|x)\.com/([A-Za-z0-9_]{1,15})(?:/(media))?/?$",
re.IGNORECASE,
)
def normalize_url(url: str) -> str:
u = url.strip()
if not u.startswith("http"):
u = "https://" + u
return u
def extract_tweet_id(url: str) -> str | None:
m = STATUS_RE.search(url)
return m.group(1) if m else None
def extract_username_and_kind(url: str) -> tuple[str | None, str | None]:
u = normalize_url(url)
m = PROFILE_OR_MEDIA_RE.match(u)
if not m:
return None, None
user = m.group(1)
media_flag = m.group(2)
if user.lower() in {"home", "explore", "settings", "messages", "notifications", "i", "intent", "search"}:
return None, None
if media_flag:
return user, "media"
return user, "profile"
def is_status_url(url: str) -> bool:
return extract_tweet_id(url) is not None
def is_search_url(url: str) -> bool:
parsed = urlparse(normalize_url(url))
host = parsed.netloc.lower()
return host in {"x.com", "www.x.com", "twitter.com", "www.twitter.com"} and parsed.path.rstrip("/") == "/search"
def extract_search_query(url: str) -> str:
parsed = urlparse(normalize_url(url))
values = parse_qs(parsed.query).get("q") or []
return values[0] if values else ""