Files
SakuraHub/pixiv_app/core/x_media_service.py
T

405 lines
16 KiB
Python
Raw Normal View History

2026-11-16 20:45:22 +08:00
from __future__ import annotations
import json
import time
import uuid
from dataclasses import asdict
from datetime import datetime
from pathlib import Path
from typing import Any
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.cookie_import import (
cookie_summary,
cookies_to_playwright,
parse_cookie_text,
save_playwright_cookies,
)
from pixiv_app.core.paths import (
app_session_file,
downloads_root,
plugin_roots,
plugins_root as app_plugins_root,
runtime_path,
)
from pixiv_app.core.plugin.base import Resource
from pixiv_app.core.plugin.manager import PluginManager
def project_root() -> Path:
return app_plugins_root().parent
class XMediaService:
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
VIDEO_EXTS = {".mp4", ".mov", ".webm", ".m4v"}
def __init__(self, *, plugins_root: str | Path | None = None, save_root: str | Path | None = None) -> None:
self.plugins_root = Path(plugins_root) if plugins_root else app_plugins_root()
self.save_root = Path(save_root) if save_root else downloads_root() / "x"
self.cookie_path = runtime_path("x_cookies.json")
self.follow_store_path = runtime_path("x_followed_authors.json")
self._manager: PluginManager | None = None
self._preview_cache: dict[str, list[Resource]] = {}
def import_cookie_text(self, text: str) -> dict[str, Any]:
cookies = parse_cookie_text(text)
x_cookies = cookies_to_playwright(cookies, ("x.com", "twitter.com"))
path = save_playwright_cookies(x_cookies, self.cookie_path)
plugin = self._get_plugin()
if hasattr(plugin, "reset_playwright_session"):
plugin.reset_playwright_session()
return {"count": len(x_cookies), "path": str(path), "summary": cookie_summary(x_cookies)}
def preview_author_media(self, username: str, *, media_type: str = "all", max_items: int = 120) -> dict[str, Any]:
handle = _normalize_username(username)
media_type = _normalize_media_type(media_type)
max_items = max(1, min(int(max_items), 500))
url = f"https://x.com/{handle}/media"
plugin = self._get_plugin()
if not plugin.validate():
raise ValueError("X plugin is unavailable. Install playwright/httpx and run playwright install chromium first.")
resources = plugin.parse(url)
resources = _filter_resources(resources, media_type)[:max_items]
preview_id = uuid.uuid4().hex
self._preview_cache[preview_id] = resources
return {
"preview_id": preview_id,
"username": handle,
"media_type": media_type,
"count": len(resources),
"items": [_resource_to_json(item, i) for i, item in enumerate(resources)],
"created_at": int(time.time()),
}
def download_preview(self, preview_id: str, *, selected_ids: list[str] | None = None) -> dict[str, Any]:
resources = self._preview_cache.get(preview_id)
if resources is None:
raise ValueError("Preview expired. Re-run the X media preview.")
selected = set(selected_ids or [])
if selected:
resources = [item for item in resources if item.id in selected]
if not resources:
raise ValueError("No X media items selected.")
plugin = self._get_plugin()
self.save_root.mkdir(parents=True, exist_ok=True)
results: list[dict[str, Any]] = []
downloaded = 0
failed = 0
for resource in resources:
try:
paths = plugin.download(resource, self.save_root)
downloaded += len(paths)
results.append(
{
"id": resource.id,
"ok": bool(paths),
"paths": [str(path) for path in paths],
"title": resource.title,
"synced_files": 0,
}
)
if not paths:
failed += 1
except Exception as exc: # Keep batch downloads moving.
failed += 1
results.append({"id": resource.id, "ok": False, "error": str(exc), "title": resource.title})
return {
"requested": len(resources),
"downloaded_files": downloaded,
"synced_files": 0,
"failed_items": failed,
"items": results,
}
def list_local_media(self, *, limit: int = 160, offset: int = 0) -> dict[str, Any]:
self.save_root.mkdir(parents=True, exist_ok=True)
limit = max(1, min(int(limit), 500))
offset = max(0, int(offset))
files = [
path
for path in self.save_root.rglob("*")
if path.is_file() and path.suffix.lower() in self.IMAGE_EXTS | self.VIDEO_EXTS
]
files.sort(key=lambda path: path.stat().st_mtime, reverse=True)
page = files[offset : offset + limit]
return {
"total": len(files),
"items": [self._local_media_to_json(path) for path in page],
}
def delete_local_media(self, media_ids: list[str]) -> dict[str, Any]:
deleted: list[str] = []
failed: list[dict[str, str]] = []
for media_id in media_ids:
try:
path = self.resolve_local_media(media_id)
path.unlink()
deleted.append(media_id)
except Exception as exc:
failed.append({"id": media_id, "error": str(exc)})
return {"deleted": deleted, "failed": failed}
def resolve_local_media(self, media_id: str) -> Path:
rel_text = _decode_media_id(media_id)
target = (self.save_root / rel_text).resolve()
root = self.save_root.resolve()
if root != target and root not in target.parents:
raise ValueError("Illegal media path.")
if not target.is_file():
raise ValueError("Local media file does not exist.")
return target
def list_followed_authors(self) -> list[dict[str, Any]]:
rows = [item for item in self._load_follow_rows() if bool(item.get("enabled", True))]
rows.sort(key=lambda item: ((str(item.get("creator_name") or "") or str(item.get("username") or "")).lower()))
return rows
def set_followed_author(self, username: str, *, followed: bool = True, creator_name: str = "") -> dict[str, Any]:
handle = _normalize_username(username)
rows = self._load_follow_rows()
row = next((item for item in rows if str(item.get("username", "")).lower() == handle.lower()), None)
now = _now_text()
if row is None:
row = {
"username": handle,
"creator_name": creator_name.strip() or handle,
"enabled": True,
"profile_url": f"https://x.com/{handle}",
"latest_tweet_id": "",
"latest_published_at": "",
"last_checked_at": "",
"last_error": "",
"seen_tweet_ids": [],
"created_at": now,
}
rows.append(row)
row["username"] = handle
row["creator_name"] = creator_name.strip() or str(row.get("creator_name") or handle)
row["enabled"] = bool(followed)
row["profile_url"] = f"https://x.com/{handle}"
row["updated_at"] = now
if not followed:
row["last_error"] = ""
self._save_follow_rows(rows)
return dict(row)
def sync_followed_authors(self, *, download: bool = True, max_new_per_author: int = 20) -> dict[str, Any]:
followed = self.list_followed_authors()
if not followed:
return {"authors": []}
plugin = self._get_plugin()
if not plugin.validate():
raise ValueError("X plugin is unavailable. Import a valid X cookie and ensure Playwright is installed.")
results: list[dict[str, Any]] = []
rows = self._load_follow_rows()
for row in rows:
if not bool(row.get("enabled", True)):
continue
handle = str(row.get("username") or "").strip()
if not handle:
continue
try:
parse_for_sync = getattr(plugin, "parse_for_sync", plugin.parse)
resources = parse_for_sync(f"https://x.com/{handle}")
grouped = _group_resources_by_tweet(resources)
seen_ids = [str(item) for item in row.get("seen_tweet_ids", []) if str(item).strip()]
seen_set = set(seen_ids)
new_groups = [item for item in grouped if item["tweet_id"] and item["tweet_id"] not in seen_set]
new_groups = new_groups[: max(0, int(max_new_per_author))]
downloaded_files = 0
synced_files = 0
created_alerts = 0
new_ids: list[str] = []
for group in new_groups:
representative = group["resource"]
new_ids.append(group["tweet_id"])
if download:
for resource in group["resources"]:
paths = plugin.download(resource, self.save_root)
downloaded_files += len(paths)
latest_group = grouped[0] if grouped else None
if latest_group is not None:
representative = latest_group["resource"]
row["creator_name"] = representative.author or str(row.get("creator_name") or handle)
row["latest_tweet_id"] = latest_group["tweet_id"]
row["latest_published_at"] = representative.created_at or str(row.get("latest_published_at") or "")
row["last_checked_at"] = _now_text()
row["last_error"] = ""
row["updated_at"] = _now_text()
row["seen_tweet_ids"] = _merge_seen_ids(new_ids, seen_ids)
results.append(
{
"username": handle,
"creator_name": row.get("creator_name") or handle,
"new_posts": len(new_groups),
"downloaded_files": downloaded_files,
"synced_files": synced_files,
"alerts": created_alerts,
"error": "",
}
)
except Exception as exc:
row["last_checked_at"] = _now_text()
row["last_error"] = str(exc)
row["updated_at"] = _now_text()
results.append(
{
"username": handle,
"creator_name": row.get("creator_name") or handle,
"new_posts": 0,
"downloaded_files": 0,
"synced_files": 0,
"alerts": 0,
"error": str(exc),
}
)
self._save_follow_rows(rows)
return {"authors": results}
def _get_plugin(self):
if self._manager is None:
self._manager = PluginManager(plugin_roots() if self.plugins_root == app_plugins_root() else [self.plugins_root])
self._manager.load_all()
plugin = self._manager.plugins.get("X")
if plugin is None:
raise ValueError("X plugin is missing. Confirm plugins/x exists.")
return plugin
def _load_follow_rows(self) -> list[dict[str, Any]]:
if not self.follow_store_path.is_file():
return []
try:
payload = json.loads(self.follow_store_path.read_text(encoding="utf-8"))
except Exception:
return []
if isinstance(payload, list):
return [dict(item) for item in payload if isinstance(item, dict)]
return []
def _save_follow_rows(self, rows: list[dict[str, Any]]) -> None:
self.follow_store_path.parent.mkdir(parents=True, exist_ok=True)
self.follow_store_path.write_text(
json.dumps(rows, ensure_ascii=False, indent=2),
encoding="utf-8",
)
def _session_payload(self) -> dict[str, Any]:
return SessionStore(app_session_file()).load()
def _local_media_to_json(self, path: Path) -> dict[str, Any]:
rel = path.relative_to(self.save_root)
stat = path.stat()
kind = "video" if path.suffix.lower() in self.VIDEO_EXTS else "image"
media_id = _encode_media_id(str(rel).replace("\\", "/"))
return {
"id": media_id,
"name": path.name,
"relative_path": str(rel).replace("\\", "/"),
"kind": kind,
"size": stat.st_size,
"modified_at": int(stat.st_mtime),
"url": f"/media/x-local/{media_id}",
}
def _normalize_username(username: str) -> str:
value = username.strip()
if value.startswith("@"):
value = value[1:]
if "/" in value:
parts = [item for item in value.replace("https://", "").replace("http://", "").split("/") if item]
value = parts[1] if parts and parts[0].lower() in {"x.com", "twitter.com", "www.x.com", "www.twitter.com"} else parts[0]
if not value:
raise ValueError("Please enter an X username.")
if len(value) > 15 or not value.replace("_", "").isalnum():
raise ValueError("Invalid X username format.")
return value
def _normalize_media_type(media_type: str) -> str:
value = (media_type or "all").strip().lower()
if value not in {"all", "image", "video"}:
raise ValueError("media_type must be all, image, or video.")
return value
def _filter_resources(resources: list[Resource], media_type: str) -> list[Resource]:
if media_type == "all":
return resources
return [item for item in resources if any(file.get("type", "image") == media_type for file in item.files)]
def _resource_to_json(resource: Resource, index: int) -> dict[str, Any]:
payload = asdict(resource)
payload["index"] = index
payload["kind"] = "video" if any(file.get("type") == "video" for file in resource.files) else "image"
payload["file_count"] = len(resource.files)
return payload
def _group_resources_by_tweet(resources: list[Resource]) -> list[dict[str, Any]]:
grouped: dict[str, dict[str, Any]] = {}
ordered: list[str] = []
for resource in resources:
tweet_id = str(resource.metadata.get("tweet_id") or resource.id or "")
key = tweet_id or resource.id
if key not in grouped:
grouped[key] = {
"tweet_id": tweet_id,
"resource": resource,
"resources": [resource],
}
ordered.append(key)
else:
grouped[key]["resources"].append(resource)
return [grouped[key] for key in ordered]
def _merge_seen_ids(new_ids: list[str], existing_ids: list[str], *, limit: int = 200) -> list[str]:
merged: list[str] = []
seen: set[str] = set()
for item in [*new_ids, *existing_ids]:
value = str(item).strip()
if not value or value in seen:
continue
seen.add(value)
merged.append(value)
if len(merged) >= limit:
break
return merged
def _encode_media_id(value: str) -> str:
from base64 import urlsafe_b64encode
return urlsafe_b64encode(value.encode("utf-8")).decode("ascii").rstrip("=")
def _decode_media_id(value: str) -> str:
from base64 import urlsafe_b64decode
padding = "=" * (-len(value) % 4)
return urlsafe_b64decode((value + padding).encode("ascii")).decode("utf-8")
def _now_text() -> str:
return datetime.now().isoformat(timespec="seconds")