commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,946 @@
|
||||
"""Playwright helpers for the X plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from playwright.async_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError, async_playwright
|
||||
|
||||
from .browser_opts import launch_x_persistent_context
|
||||
|
||||
from pixiv_app.core.plugin.base import PluginParseError
|
||||
|
||||
try:
|
||||
from pixiv_app.core.fingerprint import default_profile
|
||||
except Exception: # pragma: no cover
|
||||
default_profile = None
|
||||
|
||||
_GUEST_WALL_HINT = (
|
||||
"X 需要先登录。\n"
|
||||
"请在“插件管理”里点击“X 登录”完成登录,或把 Cookie 导出为 JSON 保存到 "
|
||||
"runtime/x_cookies.json 后重试。"
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class XPlaywrightClient:
|
||||
def __init__(self, config: Any) -> None:
|
||||
self.config = config
|
||||
self._pw: Any = None
|
||||
self.context: BrowserContext | None = None
|
||||
self._response_store: dict[int, list[dict[str, Any]]] = {}
|
||||
self._response_tasks: dict[int, set[asyncio.Task[Any]]] = {}
|
||||
|
||||
async def start(self) -> None:
|
||||
if self.context is not None:
|
||||
return
|
||||
self._pw = await async_playwright().start()
|
||||
profile = Path(self.config.profile_dir)
|
||||
profile.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self.context = await launch_x_persistent_context(
|
||||
self._pw,
|
||||
profile,
|
||||
headless=self.config.headless,
|
||||
)
|
||||
|
||||
cookies = self._prepare_cookie_items(self._load_cookie_items())
|
||||
if cookies:
|
||||
loaded = await self._add_cookies_tolerant(cookies)
|
||||
if loaded <= 0:
|
||||
raise PluginParseError(
|
||||
"X Cookie 注入失败:请重新导出 x.com cookies.txt,确认包含 auth_token 和 ct0 后再导入。"
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.context is not None:
|
||||
await self.context.close()
|
||||
self.context = None
|
||||
self._response_tasks.clear()
|
||||
self._response_store.clear()
|
||||
if self._pw is not None:
|
||||
await self._pw.stop()
|
||||
self._pw = None
|
||||
|
||||
async def new_page_goto(self, url: str, *, wait_until: str = "domcontentloaded") -> Page:
|
||||
assert self.context is not None
|
||||
page = await self.context.new_page()
|
||||
self._attach_response_collector(page)
|
||||
timeout = getattr(self.config, "wait_timeout", 30000)
|
||||
try:
|
||||
await page.goto(url, wait_until=wait_until, timeout=timeout)
|
||||
except PlaywrightTimeoutError as exc:
|
||||
if page.url and page.url != "about:blank":
|
||||
await asyncio.sleep(2.0)
|
||||
return page
|
||||
try:
|
||||
await page.goto(url, wait_until="commit", timeout=min(timeout, 15000))
|
||||
except PlaywrightTimeoutError as retry_exc:
|
||||
raise PluginParseError(
|
||||
"X 页面加载超时,可能是网络较慢、访问过快触发风控,或登录会话已失效。"
|
||||
) from retry_exc
|
||||
except Exception as retry_exc:
|
||||
raise PluginParseError(f"X 页面加载失败: {retry_exc}") from retry_exc
|
||||
if page.url == "about:blank":
|
||||
raise PluginParseError("X 页面没有成功打开,请稍后重试。") from exc
|
||||
await self.human_pause(2.0, 4.5)
|
||||
await self.move_mouse_softly(page)
|
||||
return page
|
||||
|
||||
async def human_pause(self, low: float | None = None, high: float | None = None) -> None:
|
||||
low = float(low if low is not None else getattr(self.config, "min_action_delay", 3.0))
|
||||
high = float(high if high is not None else getattr(self.config, "max_action_delay", 8.0))
|
||||
if high < low:
|
||||
high = low
|
||||
await asyncio.sleep(random.uniform(low, high))
|
||||
|
||||
async def move_mouse_softly(self, page: Page) -> None:
|
||||
try:
|
||||
viewport = page.viewport_size or {"width": 1280, "height": 800}
|
||||
width = int(viewport.get("width", 1280))
|
||||
height = int(viewport.get("height", 800))
|
||||
for _ in range(random.randint(1, 3)):
|
||||
x = random.randint(max(12, width // 8), max(20, width - 24))
|
||||
y = random.randint(max(12, height // 8), max(20, height - 24))
|
||||
await page.mouse.move(x, y, steps=random.randint(8, 18))
|
||||
await asyncio.sleep(random.uniform(0.2, 0.7))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def human_scroll_page(self, page: Page) -> float:
|
||||
await self.move_mouse_softly(page)
|
||||
viewport = page.viewport_size or {"height": 800}
|
||||
height = int(viewport.get("height", 800))
|
||||
delta = random.randint(max(260, height // 2), max(420, int(height * 1.2)))
|
||||
if random.random() < 0.12:
|
||||
delta = -random.randint(120, 280)
|
||||
await page.mouse.wheel(0, delta)
|
||||
if random.random() < 0.07:
|
||||
await asyncio.sleep(random.uniform(30.0, 90.0))
|
||||
await self.human_pause()
|
||||
return float(await page.evaluate("document.body.scrollHeight"))
|
||||
|
||||
async def raise_if_guest_wall(self, page: Page) -> None:
|
||||
await asyncio.sleep(0.45)
|
||||
try:
|
||||
url = page.url or ""
|
||||
if "/i/flow/login" in url or "/i/flow/signup" in url:
|
||||
raise PluginParseError(_GUEST_WALL_HINT)
|
||||
except PluginParseError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
wall = page.get_by_text(
|
||||
re.compile(
|
||||
r"注册 X|Sign up for X|登录|Sign in|Phone, email, or username",
|
||||
re.I,
|
||||
)
|
||||
).first
|
||||
if await wall.is_visible(timeout=2200):
|
||||
raise PluginParseError(_GUEST_WALL_HINT)
|
||||
except PluginParseError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
dialog = page.locator('[role="dialog"]').first
|
||||
if await dialog.is_visible(timeout=600):
|
||||
inner = (await dialog.inner_text())[:800]
|
||||
if ("注册" in inner or "Sign up" in inner) and ("Google" in inner or "Apple" in inner or "Next" in inner):
|
||||
raise PluginParseError(_GUEST_WALL_HINT)
|
||||
except PluginParseError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def wait_for_main_content(self, page: Page) -> None:
|
||||
timeout = min(15000, getattr(self.config, "wait_timeout", 30000))
|
||||
for sel in ('article[data-testid="tweet"]', "article", '[data-testid="tweetPhoto"]'):
|
||||
try:
|
||||
await page.wait_for_selector(sel, timeout=timeout)
|
||||
return
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
async def wait_for_media(self, page: Page) -> bool:
|
||||
timeout = 10000
|
||||
for sel in (
|
||||
'[data-testid="tweetPhoto"]',
|
||||
'img[src*="pbs.twimg.com/media"]',
|
||||
"video",
|
||||
"video source",
|
||||
):
|
||||
try:
|
||||
await page.wait_for_selector(sel, timeout=timeout)
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
async def extract_media_urls(self, page: Page) -> list[str]:
|
||||
urls: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
imgs = await page.query_selector_all('img[src*="pbs.twimg.com/media"]')
|
||||
for img in imgs:
|
||||
src = await img.get_attribute("src")
|
||||
if src and "profile_images" not in src:
|
||||
orig = self._original_tw_image_url(src)
|
||||
if orig and orig not in seen:
|
||||
seen.add(orig)
|
||||
urls.append(orig)
|
||||
|
||||
for sel in ('[data-testid="tweetPhoto"] img', "article img"):
|
||||
elements = await page.query_selector_all(sel)
|
||||
for el in elements:
|
||||
src = await el.get_attribute("src")
|
||||
if src and "pbs.twimg.com/media" in src:
|
||||
orig = self._original_tw_image_url(src)
|
||||
if orig and orig not in seen:
|
||||
seen.add(orig)
|
||||
urls.append(orig)
|
||||
|
||||
try:
|
||||
raw_video_urls = await page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const values = [];
|
||||
for (const el of document.querySelectorAll('video, video source')) {
|
||||
const candidates = [
|
||||
el.currentSrc,
|
||||
el.src,
|
||||
el.getAttribute?.('src'),
|
||||
el.getAttribute?.('data-src'),
|
||||
el.getAttribute?.('poster'),
|
||||
];
|
||||
for (const value of candidates) {
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
values.push(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
"""
|
||||
)
|
||||
except Exception:
|
||||
raw_video_urls = []
|
||||
|
||||
for src in raw_video_urls:
|
||||
normalized = self._normalize_media_url(str(src))
|
||||
if normalized and self._is_video_url(normalized) and normalized not in seen:
|
||||
seen.add(normalized)
|
||||
urls.append(normalized)
|
||||
|
||||
try:
|
||||
html = await page.content()
|
||||
except Exception:
|
||||
html = ""
|
||||
|
||||
for image_url in self._extract_image_urls_from_html(html):
|
||||
if image_url not in seen:
|
||||
seen.add(image_url)
|
||||
urls.append(image_url)
|
||||
|
||||
for pattern in (
|
||||
r"https?://video\.twimg\.com/[^\s\"'<>\\]+",
|
||||
r"https:\\/\\/video\.twimg\.com\\/[^\s\"'<>]+",
|
||||
):
|
||||
for match in re.findall(pattern, html):
|
||||
normalized = self._normalize_media_url(match)
|
||||
if normalized and self._is_video_url(normalized) and normalized not in seen:
|
||||
seen.add(normalized)
|
||||
urls.append(normalized)
|
||||
|
||||
return urls
|
||||
|
||||
async def extract_media_urls_from_api_responses(self, page: Page) -> list[str]:
|
||||
await self.wait_for_response_capture(page)
|
||||
payloads = self._response_store.get(id(page), [])
|
||||
if not payloads:
|
||||
return []
|
||||
|
||||
urls: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(url: str | None) -> None:
|
||||
if not url:
|
||||
return
|
||||
normalized = self._normalize_media_url(str(url))
|
||||
if not normalized:
|
||||
return
|
||||
if "pbs.twimg.com/media" in normalized:
|
||||
normalized = self._original_tw_image_url(normalized)
|
||||
if normalized not in seen:
|
||||
seen.add(normalized)
|
||||
urls.append(normalized)
|
||||
|
||||
for payload in payloads:
|
||||
data = payload.get("json")
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
for media_item in self._find_media_entities(data):
|
||||
media_url = media_item.get("media_url_https") or media_item.get("media_url")
|
||||
media_type = str(media_item.get("type") or "").lower()
|
||||
if media_url and media_type in {"photo", "animated_gif", "video", ""}:
|
||||
add(str(media_url))
|
||||
|
||||
video_info = media_item.get("video_info")
|
||||
if isinstance(video_info, dict):
|
||||
best_video = self._pick_best_video_variant(video_info)
|
||||
add(best_video)
|
||||
|
||||
return urls
|
||||
|
||||
async def extract_media_records_from_api_responses(self, page: Page) -> list[dict[str, Any]]:
|
||||
await self.wait_for_response_capture(page)
|
||||
payloads = self._response_store.get(id(page), [])
|
||||
if not payloads:
|
||||
return []
|
||||
|
||||
records: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
|
||||
def add_record(kind: str, url: str | None, *, thumbnail: str | None = None) -> None:
|
||||
if not url:
|
||||
return
|
||||
normalized = self._normalize_media_url(str(url))
|
||||
if not normalized:
|
||||
return
|
||||
key = (kind, normalized)
|
||||
if key in seen:
|
||||
return
|
||||
seen.add(key)
|
||||
records.append(
|
||||
{
|
||||
"type": kind,
|
||||
"url": normalized,
|
||||
"thumbnail": self._normalize_media_url(thumbnail) if thumbnail else None,
|
||||
}
|
||||
)
|
||||
|
||||
for payload in payloads:
|
||||
data = payload.get("json")
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
for media_item in self._find_media_entities(data):
|
||||
media_type = str(media_item.get("type") or "").lower()
|
||||
thumb_url = media_item.get("media_url_https") or media_item.get("media_url")
|
||||
if media_type == "photo":
|
||||
add_record("image", thumb_url)
|
||||
continue
|
||||
|
||||
video_info = media_item.get("video_info")
|
||||
if isinstance(video_info, dict):
|
||||
best_video = self._pick_best_video_variant(video_info)
|
||||
if best_video:
|
||||
add_record("video", best_video, thumbnail=thumb_url)
|
||||
continue
|
||||
|
||||
return records
|
||||
|
||||
async def wait_for_response_capture(self, page: Page, timeout: float = 2.0) -> None:
|
||||
page_id = id(page)
|
||||
tasks = self._response_tasks.get(page_id)
|
||||
if not tasks:
|
||||
return
|
||||
|
||||
deadline = asyncio.get_running_loop().time() + timeout
|
||||
while tasks and asyncio.get_running_loop().time() < deadline:
|
||||
pending = [task for task in list(tasks) if not task.done()]
|
||||
if not pending:
|
||||
break
|
||||
_, pending = await asyncio.wait(pending, timeout=0.2)
|
||||
tasks = self._response_tasks.get(page_id)
|
||||
if not tasks:
|
||||
break
|
||||
|
||||
async def scroll_and_collect_media(self, page: Page, max_scrolls: int = 12, max_items: int = 40) -> list[str]:
|
||||
collected: list[str] = []
|
||||
seen: set[str] = set()
|
||||
last_height = 0.0
|
||||
for _ in range(max_scrolls):
|
||||
batch: list[str] = []
|
||||
for item in await self.extract_media_records_from_api_responses(page):
|
||||
url = item.get("url")
|
||||
if isinstance(url, str) and url:
|
||||
batch.append(url)
|
||||
batch.extend(await self.extract_media_urls(page))
|
||||
for url in batch:
|
||||
if url not in seen:
|
||||
seen.add(url)
|
||||
collected.append(url)
|
||||
if len(collected) >= max_items:
|
||||
return collected
|
||||
try:
|
||||
height = await self.human_scroll_page(page)
|
||||
except Exception:
|
||||
break
|
||||
if height == last_height:
|
||||
break
|
||||
last_height = float(height)
|
||||
return collected
|
||||
|
||||
async def scroll_and_collect_tweet_urls(self, page: Page, limit: int = 40) -> list[str]:
|
||||
found: list[str] = []
|
||||
seen: set[str] = set()
|
||||
stale_rounds = 0
|
||||
max_rounds = 25
|
||||
|
||||
for _ in range(max_rounds):
|
||||
if len(found) >= limit:
|
||||
break
|
||||
before = len(found)
|
||||
links = await page.query_selector_all('a[href*="/status/"]')
|
||||
for link in links:
|
||||
href = await link.get_attribute("href")
|
||||
if not href or "/status/" not in href:
|
||||
continue
|
||||
if href.startswith("/"):
|
||||
full = f"https://x.com{href}"
|
||||
elif href.startswith("http"):
|
||||
full = href
|
||||
else:
|
||||
continue
|
||||
full = re.sub(r"\?.*$", "", full.split("#")[0])
|
||||
if full not in seen:
|
||||
seen.add(full)
|
||||
found.append(full)
|
||||
if len(found) >= limit:
|
||||
break
|
||||
|
||||
if len(found) == before:
|
||||
stale_rounds += 1
|
||||
if stale_rounds >= 5:
|
||||
break
|
||||
else:
|
||||
stale_rounds = 0
|
||||
|
||||
try:
|
||||
await self.human_scroll_page(page)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
return found[:limit]
|
||||
|
||||
async def get_author_info(self, page: Page) -> dict[str, str]:
|
||||
info = {"name": "", "username": ""}
|
||||
try:
|
||||
block = await page.query_selector('[data-testid="User-Name"]')
|
||||
if block:
|
||||
text = await block.inner_text()
|
||||
lines = [line.strip() for line in text.split("\n") if line.strip()]
|
||||
if lines:
|
||||
info["name"] = lines[0]
|
||||
for line in lines:
|
||||
if line.startswith("@"):
|
||||
info["username"] = line.lstrip("@")
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return info
|
||||
|
||||
async def get_user_display_name(self, page: Page, fallback: str) -> dict[str, str]:
|
||||
info = {"name": fallback, "username": fallback}
|
||||
try:
|
||||
block = await page.query_selector('[data-testid="UserName"]')
|
||||
if block:
|
||||
text = await block.inner_text()
|
||||
lines = [line.strip() for line in text.split("\n") if line.strip()]
|
||||
if lines:
|
||||
info["name"] = lines[0]
|
||||
except Exception:
|
||||
pass
|
||||
return info
|
||||
|
||||
async def get_tweet_text(self, page: Page) -> str:
|
||||
try:
|
||||
el = await page.query_selector('[data-testid="tweetText"]')
|
||||
if el:
|
||||
return (await el.inner_text()).strip()
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
async def get_tweet_date(self, page: Page) -> str:
|
||||
try:
|
||||
time_el = await page.query_selector("article time")
|
||||
if time_el:
|
||||
return (await time_el.get_attribute("datetime")) or ""
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
async def download_image(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
name = url.split("/")[-1].split("?")[0]
|
||||
if not any(name.lower().endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp")):
|
||||
name += ".jpg"
|
||||
target = save_path / name
|
||||
if target.exists() and target.stat().st_size > 256:
|
||||
return target
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
|
||||
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if response.status_code == 200 and "text/html" not in content_type:
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size > 256:
|
||||
return target
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
|
||||
async def download_video(self, url: str, save_path: Path, *, referer: str | None = None) -> Path | None:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
async with httpx.AsyncClient(timeout=120.0, follow_redirects=True) as client:
|
||||
response = await client.get(url, headers=self._media_request_headers(url, referer=referer))
|
||||
if response.status_code != 200:
|
||||
return None
|
||||
|
||||
final_url = str(response.url)
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
body_text = response.text if self._looks_like_hls(final_url, content_type, "") or "text" in content_type else ""
|
||||
if "text/html" in content_type and not self._looks_like_hls(final_url, content_type, body_text):
|
||||
return None
|
||||
|
||||
if self._looks_like_hls(final_url, content_type, body_text):
|
||||
return await self._download_hls_playlist(
|
||||
client,
|
||||
playlist_url=final_url,
|
||||
playlist_text=body_text,
|
||||
save_path=save_path,
|
||||
referer=referer,
|
||||
)
|
||||
|
||||
target = self._video_target_path(final_url, save_path)
|
||||
if target.exists() and target.stat().st_size > 1024:
|
||||
return target
|
||||
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size <= 1024:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
return target
|
||||
|
||||
async def _download_hls_playlist(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
playlist_url: str,
|
||||
playlist_text: str,
|
||||
save_path: Path,
|
||||
referer: str | None,
|
||||
) -> Path | None:
|
||||
selected_url, selected_text = await self._resolve_hls_playlist(
|
||||
client,
|
||||
playlist_url=playlist_url,
|
||||
playlist_text=playlist_text,
|
||||
referer=referer,
|
||||
)
|
||||
if not selected_text:
|
||||
return None
|
||||
|
||||
segment_urls, init_url = self._parse_hls_segments(selected_url, selected_text)
|
||||
if not segment_urls:
|
||||
return None
|
||||
|
||||
target = self._video_target_path(selected_url, save_path, segment_urls=segment_urls, init_url=init_url)
|
||||
if target.exists() and target.stat().st_size > 1024:
|
||||
return target
|
||||
|
||||
with target.open("wb") as fh:
|
||||
if init_url:
|
||||
init_resp = await client.get(init_url, headers=self._media_request_headers(init_url, referer=referer))
|
||||
if init_resp.status_code != 200:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
fh.write(init_resp.content)
|
||||
|
||||
for segment_url in segment_urls:
|
||||
seg_resp = await client.get(
|
||||
segment_url,
|
||||
headers=self._media_request_headers(segment_url, referer=referer),
|
||||
)
|
||||
if seg_resp.status_code != 200:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
fh.write(seg_resp.content)
|
||||
|
||||
if target.stat().st_size <= 1024:
|
||||
target.unlink(missing_ok=True)
|
||||
return None
|
||||
return target
|
||||
|
||||
async def _resolve_hls_playlist(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
*,
|
||||
playlist_url: str,
|
||||
playlist_text: str,
|
||||
referer: str | None,
|
||||
) -> tuple[str, str]:
|
||||
if "#EXT-X-STREAM-INF" not in playlist_text:
|
||||
return playlist_url, playlist_text
|
||||
|
||||
variants = self._parse_hls_variants(playlist_url, playlist_text)
|
||||
for variant_url in variants:
|
||||
resp = await client.get(variant_url, headers=self._media_request_headers(variant_url, referer=referer))
|
||||
if resp.status_code == 200:
|
||||
return str(resp.url), resp.text
|
||||
return playlist_url, playlist_text
|
||||
|
||||
def _parse_hls_variants(self, playlist_url: str, playlist_text: str) -> list[str]:
|
||||
variants: list[tuple[int, str]] = []
|
||||
pending_bandwidth = 0
|
||||
|
||||
for raw_line in playlist_text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#EXT-X-STREAM-INF"):
|
||||
match = re.search(r"BANDWIDTH=(\d+)", line)
|
||||
pending_bandwidth = int(match.group(1)) if match else 0
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
variants.append((pending_bandwidth, urljoin(playlist_url, line)))
|
||||
pending_bandwidth = 0
|
||||
|
||||
variants.sort(key=lambda item: item[0], reverse=True)
|
||||
return [url for _, url in variants]
|
||||
|
||||
def _parse_hls_segments(self, playlist_url: str, playlist_text: str) -> tuple[list[str], str | None]:
|
||||
segments: list[str] = []
|
||||
init_url: str | None = None
|
||||
|
||||
for raw_line in playlist_text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#EXT-X-MAP:"):
|
||||
match = re.search(r'URI="([^"]+)"', line)
|
||||
if match:
|
||||
init_url = urljoin(playlist_url, match.group(1))
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
segments.append(urljoin(playlist_url, line))
|
||||
|
||||
return segments, init_url
|
||||
|
||||
def _media_request_headers(self, url: str, *, referer: str | None = None) -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": self._ua(),
|
||||
"Accept": "*/*",
|
||||
"Referer": referer or "https://x.com/",
|
||||
"Origin": "https://x.com",
|
||||
}
|
||||
cookie_header = self._cookie_header(url)
|
||||
if cookie_header:
|
||||
headers["Cookie"] = cookie_header
|
||||
csrf_token = self._cookie_value("ct0")
|
||||
if csrf_token:
|
||||
headers["x-csrf-token"] = csrf_token
|
||||
return headers
|
||||
|
||||
def _cookie_header(self, url: str) -> str:
|
||||
pairs: list[str] = []
|
||||
for item in self._load_cookie_items():
|
||||
name = str(item.get("name") or "").strip()
|
||||
value = str(item.get("value") or "")
|
||||
if name:
|
||||
pairs.append(f"{name}={value}")
|
||||
return "; ".join(pairs)
|
||||
|
||||
def _cookie_value(self, name: str) -> str:
|
||||
for item in self._load_cookie_items():
|
||||
if str(item.get("name") or "") == name:
|
||||
return str(item.get("value") or "")
|
||||
return ""
|
||||
|
||||
def _load_cookie_items(self) -> list[dict[str, Any]]:
|
||||
cookie_path = Path(getattr(self.config, "cookie_file", "") or "")
|
||||
if not cookie_path.is_file():
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(cookie_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return []
|
||||
return payload if isinstance(payload, list) else []
|
||||
|
||||
async def _add_cookies_tolerant(self, cookies: list[dict[str, Any]]) -> int:
|
||||
assert self.context is not None
|
||||
try:
|
||||
await self.context.add_cookies(cookies)
|
||||
return len(cookies)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
loaded = 0
|
||||
for cookie in cookies:
|
||||
try:
|
||||
await self.context.add_cookies([cookie])
|
||||
loaded += 1
|
||||
except Exception:
|
||||
continue
|
||||
return loaded
|
||||
|
||||
@staticmethod
|
||||
def _prepare_cookie_items(cookies: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
prepared_by_key: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
for cookie in cookies:
|
||||
name = str(cookie.get("name") or "").strip()
|
||||
domain = str(cookie.get("domain") or "").strip()
|
||||
if not name or not domain:
|
||||
continue
|
||||
|
||||
item: dict[str, Any] = {
|
||||
"name": name,
|
||||
"value": str(cookie.get("value") or ""),
|
||||
"domain": domain,
|
||||
"path": str(cookie.get("path") or "/"),
|
||||
"secure": bool(cookie.get("secure", False)),
|
||||
"httpOnly": bool(cookie.get("httpOnly", False)),
|
||||
}
|
||||
|
||||
expires = cookie.get("expires")
|
||||
if expires is not None:
|
||||
try:
|
||||
expires_value = int(float(expires))
|
||||
if expires_value > 0:
|
||||
item["expires"] = expires_value
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
same_site = str(cookie.get("sameSite") or "").strip()
|
||||
if same_site in {"Strict", "Lax", "None"}:
|
||||
if same_site == "None" and not item["secure"]:
|
||||
item["sameSite"] = "Lax"
|
||||
else:
|
||||
item["sameSite"] = same_site
|
||||
|
||||
key = (domain.lower(), str(item["path"]), name)
|
||||
prepared_by_key[key] = item
|
||||
return list(prepared_by_key.values())
|
||||
|
||||
def _video_target_path(
|
||||
self,
|
||||
url: str,
|
||||
save_path: Path,
|
||||
*,
|
||||
segment_urls: list[str] | None = None,
|
||||
init_url: str | None = None,
|
||||
) -> Path:
|
||||
parsed = urlparse(url)
|
||||
name = Path(parsed.path).name or "x_video"
|
||||
|
||||
if name.endswith(".m3u8") or not Path(name).suffix:
|
||||
if segment_urls:
|
||||
seg_name = Path(urlparse(segment_urls[0]).path).name
|
||||
if seg_name:
|
||||
name = seg_name
|
||||
if init_url:
|
||||
init_name = Path(urlparse(init_url).path).name
|
||||
if init_name:
|
||||
name = init_name
|
||||
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix in {".m3u8", ".m4s", ".mpd", ""}:
|
||||
name = f"{Path(name).stem or 'x_video'}.mp4"
|
||||
return save_path / name
|
||||
|
||||
def _looks_like_hls(self, url: str, content_type: str, body: str) -> bool:
|
||||
low_url = url.lower()
|
||||
if ".m3u8" in low_url:
|
||||
return True
|
||||
if "mpegurl" in content_type or "vnd.apple.mpegurl" in content_type:
|
||||
return True
|
||||
return "#EXTM3U" in body[:256]
|
||||
|
||||
def _attach_response_collector(self, page: Page) -> None:
|
||||
page_id = id(page)
|
||||
bucket: list[dict[str, Any]] = []
|
||||
self._response_store[page_id] = bucket
|
||||
self._response_tasks[page_id] = set()
|
||||
|
||||
def handle_response(response: Any) -> None:
|
||||
task = asyncio.create_task(self._capture_response(page, response))
|
||||
task_bucket = self._response_tasks.setdefault(page_id, set())
|
||||
task_bucket.add(task)
|
||||
|
||||
def cleanup(done_task: asyncio.Task[Any]) -> None:
|
||||
current_bucket = self._response_tasks.get(page_id)
|
||||
if current_bucket is not None:
|
||||
current_bucket.discard(done_task)
|
||||
|
||||
task.add_done_callback(cleanup)
|
||||
|
||||
page.on("response", handle_response)
|
||||
|
||||
async def _capture_response(self, page: Page, response: Any) -> None:
|
||||
url = str(getattr(response, "url", "") or "")
|
||||
if not self._should_capture_response(url):
|
||||
return
|
||||
|
||||
try:
|
||||
headers = await response.all_headers()
|
||||
except Exception:
|
||||
headers = {}
|
||||
content_type = str(headers.get("content-type", "")).lower()
|
||||
if "json" not in content_type and "javascript" not in content_type and "text/plain" not in content_type:
|
||||
return
|
||||
|
||||
try:
|
||||
text = await response.text()
|
||||
except Exception:
|
||||
return
|
||||
if not text or len(text) < 2:
|
||||
return
|
||||
|
||||
parsed_json: Any = None
|
||||
try:
|
||||
parsed_json = json.loads(text)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
self._response_store.setdefault(id(page), []).append(
|
||||
{
|
||||
"url": url,
|
||||
"status": getattr(response, "status", None),
|
||||
"json": parsed_json,
|
||||
}
|
||||
)
|
||||
|
||||
def _should_capture_response(self, url: str) -> bool:
|
||||
low = url.lower()
|
||||
return any(
|
||||
token in low
|
||||
for token in (
|
||||
"/i/api/graphql/",
|
||||
"/graphql/",
|
||||
"tweetdetail",
|
||||
"tweetresultbyrestid",
|
||||
"conversation",
|
||||
"adaptive.json",
|
||||
)
|
||||
)
|
||||
|
||||
def _find_media_entities(self, payload: Any) -> list[dict[str, Any]]:
|
||||
media_entities: list[dict[str, Any]] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
if "media_url_https" in node or "video_info" in node:
|
||||
media_entities.append(node)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
return
|
||||
if isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
return media_entities
|
||||
|
||||
def _pick_best_video_variant(self, video_info: dict[str, Any]) -> str | None:
|
||||
variants = video_info.get("variants")
|
||||
if not isinstance(variants, list):
|
||||
return None
|
||||
|
||||
best_mp4: tuple[int, str] | None = None
|
||||
best_hls: str | None = None
|
||||
for item in variants:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
url = item.get("url")
|
||||
if not isinstance(url, str) or not url:
|
||||
continue
|
||||
low_url = url.lower()
|
||||
content_type = str(item.get("content_type") or "").lower()
|
||||
try:
|
||||
bitrate = int(item.get("bitrate") or 0)
|
||||
except (TypeError, ValueError):
|
||||
bitrate = 0
|
||||
if "mp4" in content_type or ".mp4" in low_url:
|
||||
if best_mp4 is None or bitrate > best_mp4[0]:
|
||||
best_mp4 = (bitrate, url)
|
||||
elif "mpegurl" in content_type or ".m3u8" in low_url:
|
||||
best_hls = url
|
||||
|
||||
if best_mp4 is not None:
|
||||
return best_mp4[1]
|
||||
return best_hls
|
||||
|
||||
def _extract_image_urls_from_html(self, html: str) -> list[str]:
|
||||
if not html:
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
urls: list[str] = []
|
||||
patterns = (
|
||||
r"https?://pbs\.twimg\.com/media/[^\s\"'<>\\]+",
|
||||
r"https:\\/\\/pbs\.twimg\.com\\/media\\/[^\s\"'<>]+",
|
||||
)
|
||||
for pattern in patterns:
|
||||
for match in re.findall(pattern, html):
|
||||
normalized = self._normalize_media_url(match)
|
||||
if "pbs.twimg.com/media" not in normalized:
|
||||
continue
|
||||
orig = self._original_tw_image_url(normalized)
|
||||
if orig not in seen:
|
||||
seen.add(orig)
|
||||
urls.append(orig)
|
||||
return urls
|
||||
|
||||
def _ua(self) -> str:
|
||||
if default_profile:
|
||||
return default_profile("x").user_agent
|
||||
return (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _original_tw_image_url(url: str) -> str:
|
||||
if "pbs.twimg.com/media" not in url:
|
||||
return url
|
||||
base = url.split("?")[0]
|
||||
return f"{base}?format=jpg&name=orig" if "format=" not in url else re.sub(
|
||||
r"name=\w+",
|
||||
"name=orig",
|
||||
url,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_media_url(url: str) -> str:
|
||||
value = url.strip().replace("\\/", "/")
|
||||
if value.startswith("//"):
|
||||
value = f"https:{value}"
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _is_video_url(url: str) -> bool:
|
||||
low = url.lower()
|
||||
return any(
|
||||
token in low
|
||||
for token in (
|
||||
".mp4",
|
||||
".m3u8",
|
||||
"video.twimg.com",
|
||||
"/amplify_video/",
|
||||
"/ext_tw_video/",
|
||||
)
|
||||
)
|
||||
Reference in new issue
Block a user