first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+946
View File
@@ -0,0 +1,946 @@
"""Playwright helpers for the X plugin."""
from __future__ import annotations
import asyncio
import json
import random
import re
from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin, urlparse
import httpx
from playwright.async_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError, async_playwright
from .browser_opts import launch_x_persistent_context
from pixiv_app.core.plugin.base import PluginParseError
try:
from pixiv_app.core.fingerprint import default_profile
except Exception: # pragma: no cover
default_profile = None
_GUEST_WALL_HINT = (
"X 需要先登录。\n"
"请在“插件管理”里点击“X 登录”完成登录,或把 Cookie 导出为 JSON 保存到 "
"runtime/x_cookies.json 后重试。"
)
if TYPE_CHECKING:
pass
class XPlaywrightClient:
def __init__(self, config: Any) -> None:
self.config = config
self._pw: Any = None
self.context: BrowserContext | None = None
self._response_store: dict[int, list[dict[str, Any]]] = {}
self._response_tasks: dict[int, set[asyncio.Task[Any]]] = {}
async def start(self) -> None:
if self.context is not None:
return
self._pw = await async_playwright().start()
profile = Path(self.config.profile_dir)
profile.mkdir(parents=True, exist_ok=True)
self.context = await launch_x_persistent_context(
self._pw,
profile,
headless=self.config.headless,
)
cookies = self._prepare_cookie_items(self._load_cookie_items())
if cookies:
loaded = await self._add_cookies_tolerant(cookies)
if loaded <= 0:
raise PluginParseError(
"X Cookie 注入失败:请重新导出 x.com cookies.txt,确认包含 auth_token 和 ct0 后再导入。"
)
async def close(self) -> None:
if self.context is not None:
await self.context.close()
self.context = None
self._response_tasks.clear()
self._response_store.clear()
if self._pw is not None:
await self._pw.stop()
self._pw = None
async def new_page_goto(self, url: str, *, wait_until: str = "domcontentloaded") -> Page:
assert self.context is not None
page = await self.context.new_page()
self._attach_response_collector(page)
timeout = getattr(self.config, "wait_timeout", 30000)
try:
await page.goto(url, wait_until=wait_until, timeout=timeout)
except PlaywrightTimeoutError as exc:
if page.url and page.url != "about:blank":
await asyncio.sleep(2.0)
return page
try:
await page.goto(url, wait_until="commit", timeout=min(timeout, 15000))
except PlaywrightTimeoutError as retry_exc:
raise PluginParseError(
"X 页面加载超时,可能是网络较慢、访问过快触发风控,或登录会话已失效。"
) from retry_exc
except Exception as retry_exc:
raise PluginParseError(f"X 页面加载失败: {retry_exc}") from retry_exc
if page.url == "about:blank":
raise PluginParseError("X 页面没有成功打开,请稍后重试。") from exc
await self.human_pause(2.0, 4.5)
await self.move_mouse_softly(page)
return page
async def human_pause(self, low: float | None = None, high: float | None = None) -> None:
low = float(low if low is not None else getattr(self.config, "min_action_delay", 3.0))
high = float(high if high is not None else getattr(self.config, "max_action_delay", 8.0))
if high < low:
high = low
await asyncio.sleep(random.uniform(low, high))
async def move_mouse_softly(self, page: Page) -> None:
try:
viewport = page.viewport_size or {"width": 1280, "height": 800}
width = int(viewport.get("width", 1280))
height = int(viewport.get("height", 800))
for _ in range(random.randint(1, 3)):
x = random.randint(max(12, width // 8), max(20, width - 24))
y = random.randint(max(12, height // 8), max(20, height - 24))
await page.mouse.move(x, y, steps=random.randint(8, 18))
await asyncio.sleep(random.uniform(0.2, 0.7))
except Exception:
pass
async def human_scroll_page(self, page: Page) -> float:
await self.move_mouse_softly(page)
viewport = page.viewport_size or {"height": 800}
height = int(viewport.get("height", 800))
delta = random.randint(max(260, height // 2), max(420, int(height * 1.2)))
if random.random() < 0.12:
delta = -random.randint(120, 280)
await page.mouse.wheel(0, delta)
if random.random() < 0.07:
await asyncio.sleep(random.uniform(30.0, 90.0))
await self.human_pause()
return float(await page.evaluate("document.body.scrollHeight"))
async def raise_if_guest_wall(self, page: Page) -> None:
await asyncio.sleep(0.45)
try:
url = page.url or ""
if "/i/flow/login" in url or "/i/flow/signup" in url:
raise PluginParseError(_GUEST_WALL_HINT)
except PluginParseError:
raise
except Exception:
pass
try:
wall = page.get_by_text(
re.compile(
r"注册 X|Sign up for X|登录|Sign in|Phone, email, or username",
re.I,
)
).first
if await wall.is_visible(timeout=2200):
raise PluginParseError(_GUEST_WALL_HINT)
except PluginParseError:
raise
except Exception:
pass
try:
dialog = page.locator('[role="dialog"]').first
if await dialog.is_visible(timeout=600):
inner = (await dialog.inner_text())[:800]
if ("注册" in inner or "Sign up" in inner) and ("Google" in inner or "Apple" in inner or "Next" in inner):
raise PluginParseError(_GUEST_WALL_HINT)
except PluginParseError:
raise
except Exception:
pass
async def wait_for_main_content(self, page: Page) -> None:
timeout = min(15000, getattr(self.config, "wait_timeout", 30000))
for sel in ('article[data-testid="tweet"]', "article", '[data-testid="tweetPhoto"]'):
try:
await page.wait_for_selector(sel, timeout=timeout)
return
except Exception:
continue
async def wait_for_media(self, page: Page) -> bool:
timeout = 10000
for sel in (
'[data-testid="tweetPhoto"]',
'img[src*="pbs.twimg.com/media"]',
"video",
"video source",
):
try:
await page.wait_for_selector(sel, timeout=timeout)
return True
except Exception:
continue
return False
async def extract_media_urls(self, page: Page) -> list[str]:
urls: list[str] = []
seen: set[str] = set()
imgs = await page.query_selector_all('img[src*="pbs.twimg.com/media"]')
for img in imgs:
src = await img.get_attribute("src")
if src and "profile_images" not in src:
orig = self._original_tw_image_url(src)
if orig and orig not in seen:
seen.add(orig)
urls.append(orig)
for sel in ('[data-testid="tweetPhoto"] img', "article img"):
elements = await page.query_selector_all(sel)
for el in elements:
src = await el.get_attribute("src")
if src and "pbs.twimg.com/media" in src:
orig = self._original_tw_image_url(src)
if orig and orig not in seen:
seen.add(orig)
urls.append(orig)
try:
raw_video_urls = await page.evaluate(
"""
() => {
const values = [];
for (const el of document.querySelectorAll('video, video source')) {
const candidates = [
el.currentSrc,
el.src,
el.getAttribute?.('src'),
el.getAttribute?.('data-src'),
el.getAttribute?.('poster'),
];
for (const value of candidates) {
if (typeof value === 'string' && value.trim()) {
values.push(value.trim());
}
}
}
return values;
}
"""
)
except Exception:
raw_video_urls = []
for src in raw_video_urls:
normalized = self._normalize_media_url(str(src))
if normalized and self._is_video_url(normalized) and normalized not in seen:
seen.add(normalized)
urls.append(normalized)
try:
html = await page.content()
except Exception:
html = ""
for image_url in self._extract_image_urls_from_html(html):
if image_url not in seen:
seen.add(image_url)
urls.append(image_url)
for pattern in (
r"https?://video\.twimg\.com/[^\s\"'<>\\]+",
r"https:\\/\\/video\.twimg\.com\\/[^\s\"'<>]+",
):
for match in re.findall(pattern, html):
normalized = self._normalize_media_url(match)
if normalized and self._is_video_url(normalized) and normalized not in seen:
seen.add(normalized)
urls.append(normalized)
return urls
async def extract_media_urls_from_api_responses(self, page: Page) -> list[str]:
await self.wait_for_response_capture(page)
payloads = self._response_store.get(id(page), [])
if not payloads:
return []
urls: list[str] = []
seen: set[str] = set()
def add(url: str | None) -> None:
if not url:
return
normalized = self._normalize_media_url(str(url))
if not normalized:
return
if "pbs.twimg.com/media" in normalized:
normalized = self._original_tw_image_url(normalized)
if normalized not in seen:
seen.add(normalized)
urls.append(normalized)
for payload in payloads:
data = payload.get("json")
if data is None:
continue
for media_item in self._find_media_entities(data):
media_url = media_item.get("media_url_https") or media_item.get("media_url")
media_type = str(media_item.get("type") or "").lower()
if media_url and media_type in {"photo", "animated_gif", "video", ""}:
add(str(media_url))
video_info = media_item.get("video_info")
if isinstance(video_info, dict):
best_video = self._pick_best_video_variant(video_info)
add(best_video)
return urls
async def extract_media_records_from_api_responses(self, page: Page) -> list[dict[str, Any]]:
await self.wait_for_response_capture(page)
payloads = self._response_store.get(id(page), [])
if not payloads:
return []
records: list[dict[str, Any]] = []
seen: set[tuple[str, str]] = set()
def add_record(kind: str, url: str | None, *, thumbnail: str | None = None) -> None:
if not url:
return
normalized = self._normalize_media_url(str(url))
if not normalized:
return
key = (kind, normalized)
if key in seen:
return
seen.add(key)
records.append(
{
"type": kind,
"url": normalized,
"thumbnail": self._normalize_media_url(thumbnail) if thumbnail else None,
}
)
for payload in payloads:
data = payload.get("json")
if data is None:
continue
for media_item in self._find_media_entities(data):
media_type = str(media_item.get("type") or "").lower()
thumb_url = media_item.get("media_url_https") or media_item.get("media_url")
if media_type == "photo":
add_record("image", thumb_url)
continue
video_info = media_item.get("video_info")
if isinstance(video_info, dict):
best_video = self._pick_best_video_variant(video_info)
if best_video:
add_record("video", best_video, thumbnail=thumb_url)
continue
return records
async def wait_for_response_capture(self, page: Page, timeout: float = 2.0) -> None:
page_id = id(page)
tasks = self._response_tasks.get(page_id)
if not tasks:
return
deadline = asyncio.get_running_loop().time() + timeout
while tasks and asyncio.get_running_loop().time() < deadline:
pending = [task for task in list(tasks) if not task.done()]
if not pending:
break
_, pending = await asyncio.wait(pending, timeout=0.2)
tasks = self._response_tasks.get(page_id)
if not tasks:
break
async def scroll_and_collect_media(self, page: Page, max_scrolls: int = 12, max_items: int = 40) -> list[str]:
collected: list[str] = []
seen: set[str] = set()
last_height = 0.0
for _ in range(max_scrolls):
batch: list[str] = []
for item in await self.extract_media_records_from_api_responses(page):
url = item.get("url")
if isinstance(url, str) and url:
batch.append(url)
batch.extend(await self.extract_media_urls(page))
for url in batch:
if url not in seen:
seen.add(url)
collected.append(url)
if len(collected) >= max_items:
return collected
try:
height = await self.human_scroll_page(page)
except Exception:
break
if height == last_height:
break
last_height = float(height)
return collected
async def scroll_and_collect_tweet_urls(self, page: Page, limit: int = 40) -> list[str]:
found: list[str] = []
seen: set[str] = set()
stale_rounds = 0
max_rounds = 25
for _ in range(max_rounds):
if len(found) >= limit:
break
before = len(found)
links = await page.query_selector_all('a[href*="/status/"]')
for link in links:
href = await link.get_attribute("href")
if not href or "/status/" not in href:
continue
if href.startswith("/"):
full = f"https://x.com{href}"
elif href.startswith("http"):
full = href
else:
continue
full = re.sub(r"\?.*$", "", full.split("#")[0])
if full not in seen:
seen.add(full)
found.append(full)
if len(found) >= limit:
break
if len(found) == before:
stale_rounds += 1
if stale_rounds >= 5:
break
else:
stale_rounds = 0
try:
await self.human_scroll_page(page)
except Exception:
break
return found[:limit]
async def get_author_info(self, page: Page) -> dict[str, str]:
info = {"name": "", "username": ""}
try:
block = await page.query_selector('[data-testid="User-Name"]')
if block:
text = await block.inner_text()
lines = [line.strip() for line in text.split("\n") if line.strip()]
if lines:
info["name"] = lines[0]
for line in lines:
if line.startswith("@"):
info["username"] = line.lstrip("@")
break
except Exception:
pass
return info
async def get_user_display_name(self, page: Page, fallback: str) -> dict[str, str]:
info = {"name": fallback, "username": fallback}
try:
block = await page.query_selector('[data-testid="UserName"]')
if block:
text = await block.inner_text()
lines = [line.strip() for line in text.split("\n") if line.strip()]
if lines:
info["name"] = lines[0]
except Exception:
pass
return info
async def get_tweet_text(self, page: Page) -> str:
try:
el = await page.query_selector('[data-testid="tweetText"]')
if el:
return (await el.inner_text()).strip()
except Exception:
pass
return ""
async def get_tweet_date(self, page: Page) -> str:
try:
time_el = await page.query_selector("article time")
if time_el:
return (await time_el.get_attribute("datetime")) or ""
except Exception:
pass
return ""
async def download_image(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
save_path.mkdir(parents=True, exist_ok=True)
name = url.split("/")[-1].split("?")[0]
if not any(name.lower().endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp")):
name += ".jpg"
target = save_path / name
if target.exists() and target.stat().st_size > 256:
return target
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
content_type = response.headers.get("content-type", "").lower()
if response.status_code == 200 and "text/html" not in content_type:
target.write_bytes(response.content)
if target.stat().st_size > 256:
return target
target.unlink(missing_ok=True)
return None
async def download_video(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
save_path.mkdir(parents=True, exist_ok=True)
async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
if response.status_code != 200:
return None
final_url = str(response.url)
content_type = response.headers.get("content-type", "").lower()
body_text = response.text if self._looks_like_hls(final_url, content_type, "") or "text" in content_type else ""
if "text/html" in content_type and not self._looks_like_hls(final_url, content_type, body_text):
return None
if self._looks_like_hls(final_url, content_type, body_text):
return await self._download_hls_playlist(
client,
playlist_url=final_url,
playlist_text=body_text,
save_path=save_path,
referer=referer,
)
target = self._video_target_path(final_url, save_path)
if target.exists() and target.stat().st_size > 1024:
return target
target.write_bytes(response.content)
if target.stat().st_size <= 1024:
target.unlink(missing_ok=True)
return None
return target
async def _download_hls_playlist(
self,
client: httpx.AsyncClient,
*,
playlist_url: str,
playlist_text: str,
save_path: Path,
referer: str | None,
) -> Path | None:
selected_url, selected_text = await self._resolve_hls_playlist(
client,
playlist_url=playlist_url,
playlist_text=playlist_text,
referer=referer,
)
if not selected_text:
return None
segment_urls, init_url = self._parse_hls_segments(selected_url, selected_text)
if not segment_urls:
return None
target = self._video_target_path(selected_url, save_path, segment_urls=segment_urls, init_url=init_url)
if target.exists() and target.stat().st_size > 1024:
return target
with target.open("wb") as fh:
if init_url:
init_resp = await client.get(init_url, headers=self._media_request_headers(init_url, referer=referer))
if init_resp.status_code != 200:
target.unlink(missing_ok=True)
return None
fh.write(init_resp.content)
for segment_url in segment_urls:
seg_resp = await client.get(
segment_url,
headers=self._media_request_headers(segment_url, referer=referer),
)
if seg_resp.status_code != 200:
target.unlink(missing_ok=True)
return None
fh.write(seg_resp.content)
if target.stat().st_size <= 1024:
target.unlink(missing_ok=True)
return None
return target
async def _resolve_hls_playlist(
self,
client: httpx.AsyncClient,
*,
playlist_url: str,
playlist_text: str,
referer: str | None,
) -> tuple[str, str]:
if "#EXT-X-STREAM-INF" not in playlist_text:
return playlist_url, playlist_text
variants = self._parse_hls_variants(playlist_url, playlist_text)
for variant_url in variants:
resp = await client.get(variant_url, headers=self._media_request_headers(variant_url, referer=referer))
if resp.status_code == 200:
return str(resp.url), resp.text
return playlist_url, playlist_text
def _parse_hls_variants(self, playlist_url: str, playlist_text: str) -> list[str]:
variants: list[tuple[int, str]] = []
pending_bandwidth = 0
for raw_line in playlist_text.splitlines():
line = raw_line.strip()
if not line:
continue
if line.startswith("#EXT-X-STREAM-INF"):
match = re.search(r"BANDWIDTH=(\d+)", line)
pending_bandwidth = int(match.group(1)) if match else 0
continue
if line.startswith("#"):
continue
variants.append((pending_bandwidth, urljoin(playlist_url, line)))
pending_bandwidth = 0
variants.sort(key=lambda item: item[0], reverse=True)
return [url for _, url in variants]
def _parse_hls_segments(self, playlist_url: str, playlist_text: str) -> tuple[list[str], str | None]:
segments: list[str] = []
init_url: str | None = None
for raw_line in playlist_text.splitlines():
line = raw_line.strip()
if not line:
continue
if line.startswith("#EXT-X-MAP:"):
match = re.search(r'URI="([^"]+)"', line)
if match:
init_url = urljoin(playlist_url, match.group(1))
continue
if line.startswith("#"):
continue
segments.append(urljoin(playlist_url, line))
return segments, init_url
def _media_request_headers(self, url: str, *, referer: str | None = None) -> dict[str, str]:
headers = {
"User-Agent": self._ua(),
"Accept": "*/*",
"Referer": referer or "https://x.com/",
"Origin": "https://x.com",
}
cookie_header = self._cookie_header(url)
if cookie_header:
headers["Cookie"] = cookie_header
csrf_token = self._cookie_value("ct0")
if csrf_token:
headers["x-csrf-token"] = csrf_token
return headers
def _cookie_header(self, url: str) -> str:
pairs: list[str] = []
for item in self._load_cookie_items():
name = str(item.get("name") or "").strip()
value = str(item.get("value") or "")
if name:
pairs.append(f"{name}={value}")
return "; ".join(pairs)
def _cookie_value(self, name: str) -> str:
for item in self._load_cookie_items():
if str(item.get("name") or "") == name:
return str(item.get("value") or "")
return ""
def _load_cookie_items(self) -> list[dict[str, Any]]:
cookie_path = Path(getattr(self.config, "cookie_file", "") or "")
if not cookie_path.is_file():
return []
try:
payload = json.loads(cookie_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return []
return payload if isinstance(payload, list) else []
async def _add_cookies_tolerant(self, cookies: list[dict[str, Any]]) -> int:
assert self.context is not None
try:
await self.context.add_cookies(cookies)
return len(cookies)
except Exception:
pass
loaded = 0
for cookie in cookies:
try:
await self.context.add_cookies([cookie])
loaded += 1
except Exception:
continue
return loaded
@staticmethod
def _prepare_cookie_items(cookies: list[dict[str, Any]]) -> list[dict[str, Any]]:
prepared_by_key: dict[tuple[str, str, str], dict[str, Any]] = {}
for cookie in cookies:
name = str(cookie.get("name") or "").strip()
domain = str(cookie.get("domain") or "").strip()
if not name or not domain:
continue
item: dict[str, Any] = {
"name": name,
"value": str(cookie.get("value") or ""),
"domain": domain,
"path": str(cookie.get("path") or "/"),
"secure": bool(cookie.get("secure", False)),
"httpOnly": bool(cookie.get("httpOnly", False)),
}
expires = cookie.get("expires")
if expires is not None:
try:
expires_value = int(float(expires))
if expires_value > 0:
item["expires"] = expires_value
except (TypeError, ValueError):
pass
same_site = str(cookie.get("sameSite") or "").strip()
if same_site in {"Strict", "Lax", "None"}:
if same_site == "None" and not item["secure"]:
item["sameSite"] = "Lax"
else:
item["sameSite"] = same_site
key = (domain.lower(), str(item["path"]), name)
prepared_by_key[key] = item
return list(prepared_by_key.values())
def _video_target_path(
self,
url: str,
save_path: Path,
*,
segment_urls: list[str] | None = None,
init_url: str | None = None,
) -> Path:
parsed = urlparse(url)
name = Path(parsed.path).name or "x_video"
if name.endswith(".m3u8") or not Path(name).suffix:
if segment_urls:
seg_name = Path(urlparse(segment_urls[0]).path).name
if seg_name:
name = seg_name
if init_url:
init_name = Path(urlparse(init_url).path).name
if init_name:
name = init_name
suffix = Path(name).suffix.lower()
if suffix in {".m3u8", ".m4s", ".mpd", ""}:
name = f"{Path(name).stem or 'x_video'}.mp4"
return save_path / name
def _looks_like_hls(self, url: str, content_type: str, body: str) -> bool:
low_url = url.lower()
if ".m3u8" in low_url:
return True
if "mpegurl" in content_type or "vnd.apple.mpegurl" in content_type:
return True
return "#EXTM3U" in body[:256]
def _attach_response_collector(self, page: Page) -> None:
page_id = id(page)
bucket: list[dict[str, Any]] = []
self._response_store[page_id] = bucket
self._response_tasks[page_id] = set()
def handle_response(response: Any) -> None:
task = asyncio.create_task(self._capture_response(page, response))
task_bucket = self._response_tasks.setdefault(page_id, set())
task_bucket.add(task)
def cleanup(done_task: asyncio.Task[Any]) -> None:
current_bucket = self._response_tasks.get(page_id)
if current_bucket is not None:
current_bucket.discard(done_task)
task.add_done_callback(cleanup)
page.on("response", handle_response)
async def _capture_response(self, page: Page, response: Any) -> None:
url = str(getattr(response, "url", "") or "")
if not self._should_capture_response(url):
return
try:
headers = await response.all_headers()
except Exception:
headers = {}
content_type = str(headers.get("content-type", "")).lower()
if "json" not in content_type and "javascript" not in content_type and "text/plain" not in content_type:
return
try:
text = await response.text()
except Exception:
return
if not text or len(text) < 2:
return
parsed_json: Any = None
try:
parsed_json = json.loads(text)
except Exception:
return
self._response_store.setdefault(id(page), []).append(
{
"url": url,
"status": getattr(response, "status", None),
"json": parsed_json,
}
)
def _should_capture_response(self, url: str) -> bool:
low = url.lower()
return any(
token in low
for token in (
"/i/api/graphql/",
"/graphql/",
"tweetdetail",
"tweetresultbyrestid",
"conversation",
"adaptive.json",
)
)
def _find_media_entities(self, payload: Any) -> list[dict[str, Any]]:
media_entities: list[dict[str, Any]] = []
def walk(node: Any) -> None:
if isinstance(node, dict):
if "media_url_https" in node or "video_info" in node:
media_entities.append(node)
for value in node.values():
walk(value)
return
if isinstance(node, list):
for item in node:
walk(item)
walk(payload)
return media_entities
def _pick_best_video_variant(self, video_info: dict[str, Any]) -> str | None:
variants = video_info.get("variants")
if not isinstance(variants, list):
return None
best_mp4: tuple[int, str] | None = None
best_hls: str | None = None
for item in variants:
if not isinstance(item, dict):
continue
url = item.get("url")
if not isinstance(url, str) or not url:
continue
low_url = url.lower()
content_type = str(item.get("content_type") or "").lower()
try:
bitrate = int(item.get("bitrate") or 0)
except (TypeError, ValueError):
bitrate = 0
if "mp4" in content_type or ".mp4" in low_url:
if best_mp4 is None or bitrate > best_mp4[0]:
best_mp4 = (bitrate, url)
elif "mpegurl" in content_type or ".m3u8" in low_url:
best_hls = url
if best_mp4 is not None:
return best_mp4[1]
return best_hls
def _extract_image_urls_from_html(self, html: str) -> list[str]:
if not html:
return []
seen: set[str] = set()
urls: list[str] = []
patterns = (
r"https?://pbs\.twimg\.com/media/[^\s\"'<>\\]+",
r"https:\\/\\/pbs\.twimg\.com\\/media\\/[^\s\"'<>]+",
)
for pattern in patterns:
for match in re.findall(pattern, html):
normalized = self._normalize_media_url(match)
if "pbs.twimg.com/media" not in normalized:
continue
orig = self._original_tw_image_url(normalized)
if orig not in seen:
seen.add(orig)
urls.append(orig)
return urls
def _ua(self) -> str:
if default_profile:
return default_profile("x").user_agent
return (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
@staticmethod
def _original_tw_image_url(url: str) -> str:
if "pbs.twimg.com/media" not in url:
return url
base = url.split("?")[0]
return f"{base}?format=jpg&name=orig" if "format=" not in url else re.sub(
r"name=\w+",
"name=orig",
url,
)
@staticmethod
def _normalize_media_url(url: str) -> str:
value = url.strip().replace("\\/", "/")
if value.startswith("//"):
value = f"https:{value}"
return value
@staticmethod
def _is_video_url(url: str) -> bool:
low = url.lower()
return any(
token in low
for token in (
".mp4",
".m3u8",
"video.twimg.com",
"/amplify_video/",
"/ext_tw_video/",
)
)