405 lines
16 KiB
Python
405 lines
16 KiB
Python
from __future__ import annotations
|
|||
|
|
|
||
|
|
import json
|
||
|
|
import time
|
||
|
|
import uuid
|
||
|
|
from dataclasses import asdict
|
||
|
|
from datetime import datetime
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
from pixiv_app.core.auth import SessionStore
|
||
|
|
from pixiv_app.core.cookie_import import (
|
||
|
|
cookie_summary,
|
||
|
|
cookies_to_playwright,
|
||
|
|
parse_cookie_text,
|
||
|
|
save_playwright_cookies,
|
||
|
|
)
|
||
|
|
from pixiv_app.core.paths import (
|
||
|
|
app_session_file,
|
||
|
|
downloads_root,
|
||
|
|
plugin_roots,
|
||
|
|
plugins_root as app_plugins_root,
|
||
|
|
runtime_path,
|
||
|
|
)
|
||
|
|
from pixiv_app.core.plugin.base import Resource
|
||
|
|
from pixiv_app.core.plugin.manager import PluginManager
|
||
|
|
|
||
|
|
|
||
|
|
def project_root() -> Path:
|
||
|
|
return app_plugins_root().parent
|
||
|
|
|
||
|
|
|
||
|
|
class XMediaService:
|
||
|
|
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
|
||
|
|
VIDEO_EXTS = {".mp4", ".mov", ".webm", ".m4v"}
|
||
|
|
|
||
|
|
def __init__(self, *, plugins_root: str | Path | None = None, save_root: str | Path | None = None) -> None:
|
||
|
|
self.plugins_root = Path(plugins_root) if plugins_root else app_plugins_root()
|
||
|
|
self.save_root = Path(save_root) if save_root else downloads_root() / "x"
|
||
|
|
self.cookie_path = runtime_path("x_cookies.json")
|
||
|
|
self.follow_store_path = runtime_path("x_followed_authors.json")
|
||
|
|
self._manager: PluginManager | None = None
|
||
|
|
self._preview_cache: dict[str, list[Resource]] = {}
|
||
|
|
|
||
|
|
def import_cookie_text(self, text: str) -> dict[str, Any]:
|
||
|
|
cookies = parse_cookie_text(text)
|
||
|
|
x_cookies = cookies_to_playwright(cookies, ("x.com", "twitter.com"))
|
||
|
|
path = save_playwright_cookies(x_cookies, self.cookie_path)
|
||
|
|
plugin = self._get_plugin()
|
||
|
|
if hasattr(plugin, "reset_playwright_session"):
|
||
|
|
plugin.reset_playwright_session()
|
||
|
|
return {"count": len(x_cookies), "path": str(path), "summary": cookie_summary(x_cookies)}
|
||
|
|
|
||
|
|
def preview_author_media(self, username: str, *, media_type: str = "all", max_items: int = 120) -> dict[str, Any]:
|
||
|
|
handle = _normalize_username(username)
|
||
|
|
media_type = _normalize_media_type(media_type)
|
||
|
|
max_items = max(1, min(int(max_items), 500))
|
||
|
|
url = f"https://x.com/{handle}/media"
|
||
|
|
|
||
|
|
plugin = self._get_plugin()
|
||
|
|
if not plugin.validate():
|
||
|
|
raise ValueError("X plugin is unavailable. Install playwright/httpx and run playwright install chromium first.")
|
||
|
|
|
||
|
|
resources = plugin.parse(url)
|
||
|
|
resources = _filter_resources(resources, media_type)[:max_items]
|
||
|
|
preview_id = uuid.uuid4().hex
|
||
|
|
self._preview_cache[preview_id] = resources
|
||
|
|
return {
|
||
|
|
"preview_id": preview_id,
|
||
|
|
"username": handle,
|
||
|
|
"media_type": media_type,
|
||
|
|
"count": len(resources),
|
||
|
|
"items": [_resource_to_json(item, i) for i, item in enumerate(resources)],
|
||
|
|
"created_at": int(time.time()),
|
||
|
|
}
|
||
|
|
|
||
|
|
def download_preview(self, preview_id: str, *, selected_ids: list[str] | None = None) -> dict[str, Any]:
|
||
|
|
resources = self._preview_cache.get(preview_id)
|
||
|
|
if resources is None:
|
||
|
|
raise ValueError("Preview expired. Re-run the X media preview.")
|
||
|
|
selected = set(selected_ids or [])
|
||
|
|
if selected:
|
||
|
|
resources = [item for item in resources if item.id in selected]
|
||
|
|
if not resources:
|
||
|
|
raise ValueError("No X media items selected.")
|
||
|
|
|
||
|
|
plugin = self._get_plugin()
|
||
|
|
self.save_root.mkdir(parents=True, exist_ok=True)
|
||
|
|
results: list[dict[str, Any]] = []
|
||
|
|
downloaded = 0
|
||
|
|
failed = 0
|
||
|
|
for resource in resources:
|
||
|
|
try:
|
||
|
|
paths = plugin.download(resource, self.save_root)
|
||
|
|
downloaded += len(paths)
|
||
|
|
results.append(
|
||
|
|
{
|
||
|
|
"id": resource.id,
|
||
|
|
"ok": bool(paths),
|
||
|
|
"paths": [str(path) for path in paths],
|
||
|
|
"title": resource.title,
|
||
|
|
"synced_files": 0,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
if not paths:
|
||
|
|
failed += 1
|
||
|
|
except Exception as exc: # Keep batch downloads moving.
|
||
|
|
failed += 1
|
||
|
|
results.append({"id": resource.id, "ok": False, "error": str(exc), "title": resource.title})
|
||
|
|
|
||
|
|
return {
|
||
|
|
"requested": len(resources),
|
||
|
|
"downloaded_files": downloaded,
|
||
|
|
"synced_files": 0,
|
||
|
|
"failed_items": failed,
|
||
|
|
"items": results,
|
||
|
|
}
|
||
|
|
|
||
|
|
def list_local_media(self, *, limit: int = 160, offset: int = 0) -> dict[str, Any]:
|
||
|
|
self.save_root.mkdir(parents=True, exist_ok=True)
|
||
|
|
limit = max(1, min(int(limit), 500))
|
||
|
|
offset = max(0, int(offset))
|
||
|
|
files = [
|
||
|
|
path
|
||
|
|
for path in self.save_root.rglob("*")
|
||
|
|
if path.is_file() and path.suffix.lower() in self.IMAGE_EXTS | self.VIDEO_EXTS
|
||
|
|
]
|
||
|
|
files.sort(key=lambda path: path.stat().st_mtime, reverse=True)
|
||
|
|
page = files[offset : offset + limit]
|
||
|
|
return {
|
||
|
|
"total": len(files),
|
||
|
|
"items": [self._local_media_to_json(path) for path in page],
|
||
|
|
}
|
||
|
|
|
||
|
|
def delete_local_media(self, media_ids: list[str]) -> dict[str, Any]:
|
||
|
|
deleted: list[str] = []
|
||
|
|
failed: list[dict[str, str]] = []
|
||
|
|
for media_id in media_ids:
|
||
|
|
try:
|
||
|
|
path = self.resolve_local_media(media_id)
|
||
|
|
path.unlink()
|
||
|
|
deleted.append(media_id)
|
||
|
|
except Exception as exc:
|
||
|
|
failed.append({"id": media_id, "error": str(exc)})
|
||
|
|
return {"deleted": deleted, "failed": failed}
|
||
|
|
|
||
|
|
def resolve_local_media(self, media_id: str) -> Path:
|
||
|
|
rel_text = _decode_media_id(media_id)
|
||
|
|
target = (self.save_root / rel_text).resolve()
|
||
|
|
root = self.save_root.resolve()
|
||
|
|
if root != target and root not in target.parents:
|
||
|
|
raise ValueError("Illegal media path.")
|
||
|
|
if not target.is_file():
|
||
|
|
raise ValueError("Local media file does not exist.")
|
||
|
|
return target
|
||
|
|
|
||
|
|
def list_followed_authors(self) -> list[dict[str, Any]]:
|
||
|
|
rows = [item for item in self._load_follow_rows() if bool(item.get("enabled", True))]
|
||
|
|
rows.sort(key=lambda item: ((str(item.get("creator_name") or "") or str(item.get("username") or "")).lower()))
|
||
|
|
return rows
|
||
|
|
|
||
|
|
def set_followed_author(self, username: str, *, followed: bool = True, creator_name: str = "") -> dict[str, Any]:
|
||
|
|
handle = _normalize_username(username)
|
||
|
|
rows = self._load_follow_rows()
|
||
|
|
row = next((item for item in rows if str(item.get("username", "")).lower() == handle.lower()), None)
|
||
|
|
now = _now_text()
|
||
|
|
if row is None:
|
||
|
|
row = {
|
||
|
|
"username": handle,
|
||
|
|
"creator_name": creator_name.strip() or handle,
|
||
|
|
"enabled": True,
|
||
|
|
"profile_url": f"https://x.com/{handle}",
|
||
|
|
"latest_tweet_id": "",
|
||
|
|
"latest_published_at": "",
|
||
|
|
"last_checked_at": "",
|
||
|
|
"last_error": "",
|
||
|
|
"seen_tweet_ids": [],
|
||
|
|
"created_at": now,
|
||
|
|
}
|
||
|
|
rows.append(row)
|
||
|
|
|
||
|
|
row["username"] = handle
|
||
|
|
row["creator_name"] = creator_name.strip() or str(row.get("creator_name") or handle)
|
||
|
|
row["enabled"] = bool(followed)
|
||
|
|
row["profile_url"] = f"https://x.com/{handle}"
|
||
|
|
row["updated_at"] = now
|
||
|
|
if not followed:
|
||
|
|
row["last_error"] = ""
|
||
|
|
|
||
|
|
self._save_follow_rows(rows)
|
||
|
|
return dict(row)
|
||
|
|
|
||
|
|
def sync_followed_authors(self, *, download: bool = True, max_new_per_author: int = 20) -> dict[str, Any]:
|
||
|
|
followed = self.list_followed_authors()
|
||
|
|
if not followed:
|
||
|
|
return {"authors": []}
|
||
|
|
|
||
|
|
plugin = self._get_plugin()
|
||
|
|
if not plugin.validate():
|
||
|
|
raise ValueError("X plugin is unavailable. Import a valid X cookie and ensure Playwright is installed.")
|
||
|
|
|
||
|
|
results: list[dict[str, Any]] = []
|
||
|
|
rows = self._load_follow_rows()
|
||
|
|
|
||
|
|
for row in rows:
|
||
|
|
if not bool(row.get("enabled", True)):
|
||
|
|
continue
|
||
|
|
|
||
|
|
handle = str(row.get("username") or "").strip()
|
||
|
|
if not handle:
|
||
|
|
continue
|
||
|
|
|
||
|
|
try:
|
||
|
|
parse_for_sync = getattr(plugin, "parse_for_sync", plugin.parse)
|
||
|
|
resources = parse_for_sync(f"https://x.com/{handle}")
|
||
|
|
grouped = _group_resources_by_tweet(resources)
|
||
|
|
seen_ids = [str(item) for item in row.get("seen_tweet_ids", []) if str(item).strip()]
|
||
|
|
seen_set = set(seen_ids)
|
||
|
|
new_groups = [item for item in grouped if item["tweet_id"] and item["tweet_id"] not in seen_set]
|
||
|
|
new_groups = new_groups[: max(0, int(max_new_per_author))]
|
||
|
|
|
||
|
|
downloaded_files = 0
|
||
|
|
synced_files = 0
|
||
|
|
created_alerts = 0
|
||
|
|
new_ids: list[str] = []
|
||
|
|
|
||
|
|
for group in new_groups:
|
||
|
|
representative = group["resource"]
|
||
|
|
new_ids.append(group["tweet_id"])
|
||
|
|
|
||
|
|
if download:
|
||
|
|
for resource in group["resources"]:
|
||
|
|
paths = plugin.download(resource, self.save_root)
|
||
|
|
downloaded_files += len(paths)
|
||
|
|
|
||
|
|
latest_group = grouped[0] if grouped else None
|
||
|
|
if latest_group is not None:
|
||
|
|
representative = latest_group["resource"]
|
||
|
|
row["creator_name"] = representative.author or str(row.get("creator_name") or handle)
|
||
|
|
row["latest_tweet_id"] = latest_group["tweet_id"]
|
||
|
|
row["latest_published_at"] = representative.created_at or str(row.get("latest_published_at") or "")
|
||
|
|
|
||
|
|
row["last_checked_at"] = _now_text()
|
||
|
|
row["last_error"] = ""
|
||
|
|
row["updated_at"] = _now_text()
|
||
|
|
row["seen_tweet_ids"] = _merge_seen_ids(new_ids, seen_ids)
|
||
|
|
|
||
|
|
results.append(
|
||
|
|
{
|
||
|
|
"username": handle,
|
||
|
|
"creator_name": row.get("creator_name") or handle,
|
||
|
|
"new_posts": len(new_groups),
|
||
|
|
"downloaded_files": downloaded_files,
|
||
|
|
"synced_files": synced_files,
|
||
|
|
"alerts": created_alerts,
|
||
|
|
"error": "",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
except Exception as exc:
|
||
|
|
row["last_checked_at"] = _now_text()
|
||
|
|
row["last_error"] = str(exc)
|
||
|
|
row["updated_at"] = _now_text()
|
||
|
|
results.append(
|
||
|
|
{
|
||
|
|
"username": handle,
|
||
|
|
"creator_name": row.get("creator_name") or handle,
|
||
|
|
"new_posts": 0,
|
||
|
|
"downloaded_files": 0,
|
||
|
|
"synced_files": 0,
|
||
|
|
"alerts": 0,
|
||
|
|
"error": str(exc),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
self._save_follow_rows(rows)
|
||
|
|
return {"authors": results}
|
||
|
|
|
||
|
|
def _get_plugin(self):
|
||
|
|
if self._manager is None:
|
||
|
|
self._manager = PluginManager(plugin_roots() if self.plugins_root == app_plugins_root() else [self.plugins_root])
|
||
|
|
self._manager.load_all()
|
||
|
|
plugin = self._manager.plugins.get("X")
|
||
|
|
if plugin is None:
|
||
|
|
raise ValueError("X plugin is missing. Confirm plugins/x exists.")
|
||
|
|
return plugin
|
||
|
|
|
||
|
|
def _load_follow_rows(self) -> list[dict[str, Any]]:
|
||
|
|
if not self.follow_store_path.is_file():
|
||
|
|
return []
|
||
|
|
try:
|
||
|
|
payload = json.loads(self.follow_store_path.read_text(encoding="utf-8"))
|
||
|
|
except Exception:
|
||
|
|
return []
|
||
|
|
if isinstance(payload, list):
|
||
|
|
return [dict(item) for item in payload if isinstance(item, dict)]
|
||
|
|
return []
|
||
|
|
|
||
|
|
def _save_follow_rows(self, rows: list[dict[str, Any]]) -> None:
|
||
|
|
self.follow_store_path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
self.follow_store_path.write_text(
|
||
|
|
json.dumps(rows, ensure_ascii=False, indent=2),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
|
||
|
|
def _session_payload(self) -> dict[str, Any]:
|
||
|
|
return SessionStore(app_session_file()).load()
|
||
|
|
|
||
|
|
def _local_media_to_json(self, path: Path) -> dict[str, Any]:
|
||
|
|
rel = path.relative_to(self.save_root)
|
||
|
|
stat = path.stat()
|
||
|
|
kind = "video" if path.suffix.lower() in self.VIDEO_EXTS else "image"
|
||
|
|
media_id = _encode_media_id(str(rel).replace("\\", "/"))
|
||
|
|
return {
|
||
|
|
"id": media_id,
|
||
|
|
"name": path.name,
|
||
|
|
"relative_path": str(rel).replace("\\", "/"),
|
||
|
|
"kind": kind,
|
||
|
|
"size": stat.st_size,
|
||
|
|
"modified_at": int(stat.st_mtime),
|
||
|
|
"url": f"/media/x-local/{media_id}",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_username(username: str) -> str:
|
||
|
|
value = username.strip()
|
||
|
|
if value.startswith("@"):
|
||
|
|
value = value[1:]
|
||
|
|
if "/" in value:
|
||
|
|
parts = [item for item in value.replace("https://", "").replace("http://", "").split("/") if item]
|
||
|
|
value = parts[1] if parts and parts[0].lower() in {"x.com", "twitter.com", "www.x.com", "www.twitter.com"} else parts[0]
|
||
|
|
if not value:
|
||
|
|
raise ValueError("Please enter an X username.")
|
||
|
|
if len(value) > 15 or not value.replace("_", "").isalnum():
|
||
|
|
raise ValueError("Invalid X username format.")
|
||
|
|
return value
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_media_type(media_type: str) -> str:
|
||
|
|
value = (media_type or "all").strip().lower()
|
||
|
|
if value not in {"all", "image", "video"}:
|
||
|
|
raise ValueError("media_type must be all, image, or video.")
|
||
|
|
return value
|
||
|
|
|
||
|
|
|
||
|
|
def _filter_resources(resources: list[Resource], media_type: str) -> list[Resource]:
|
||
|
|
if media_type == "all":
|
||
|
|
return resources
|
||
|
|
return [item for item in resources if any(file.get("type", "image") == media_type for file in item.files)]
|
||
|
|
|
||
|
|
|
||
|
|
def _resource_to_json(resource: Resource, index: int) -> dict[str, Any]:
|
||
|
|
payload = asdict(resource)
|
||
|
|
payload["index"] = index
|
||
|
|
payload["kind"] = "video" if any(file.get("type") == "video" for file in resource.files) else "image"
|
||
|
|
payload["file_count"] = len(resource.files)
|
||
|
|
return payload
|
||
|
|
|
||
|
|
|
||
|
|
def _group_resources_by_tweet(resources: list[Resource]) -> list[dict[str, Any]]:
|
||
|
|
grouped: dict[str, dict[str, Any]] = {}
|
||
|
|
ordered: list[str] = []
|
||
|
|
for resource in resources:
|
||
|
|
tweet_id = str(resource.metadata.get("tweet_id") or resource.id or "")
|
||
|
|
key = tweet_id or resource.id
|
||
|
|
if key not in grouped:
|
||
|
|
grouped[key] = {
|
||
|
|
"tweet_id": tweet_id,
|
||
|
|
"resource": resource,
|
||
|
|
"resources": [resource],
|
||
|
|
}
|
||
|
|
ordered.append(key)
|
||
|
|
else:
|
||
|
|
grouped[key]["resources"].append(resource)
|
||
|
|
return [grouped[key] for key in ordered]
|
||
|
|
|
||
|
|
|
||
|
|
def _merge_seen_ids(new_ids: list[str], existing_ids: list[str], *, limit: int = 200) -> list[str]:
|
||
|
|
merged: list[str] = []
|
||
|
|
seen: set[str] = set()
|
||
|
|
for item in [*new_ids, *existing_ids]:
|
||
|
|
value = str(item).strip()
|
||
|
|
if not value or value in seen:
|
||
|
|
continue
|
||
|
|
seen.add(value)
|
||
|
|
merged.append(value)
|
||
|
|
if len(merged) >= limit:
|
||
|
|
break
|
||
|
|
return merged
|
||
|
|
|
||
|
|
|
||
|
|
def _encode_media_id(value: str) -> str:
|
||
|
|
from base64 import urlsafe_b64encode
|
||
|
|
|
||
|
|
return urlsafe_b64encode(value.encode("utf-8")).decode("ascii").rstrip("=")
|
||
|
|
|
||
|
|
|
||
|
|
def _decode_media_id(value: str) -> str:
|
||
|
|
from base64 import urlsafe_b64decode
|
||
|
|
|
||
|
|
padding = "=" * (-len(value) % 4)
|
||
|
|
return urlsafe_b64decode((value + padding).encode("ascii")).decode("utf-8")
|
||
|
|
|
||
|
|
|
||
|
|
def _now_text() -> str:
|
||
|
|
return datetime.now().isoformat(timespec="seconds")
|