commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,405 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.cookie_import import (
|
||||
cookie_summary,
|
||||
cookies_to_playwright,
|
||||
parse_cookie_text,
|
||||
save_playwright_cookies,
|
||||
)
|
||||
from pixiv_app.core.paths import (
|
||||
app_session_file,
|
||||
downloads_root,
|
||||
plugin_roots,
|
||||
plugins_root as app_plugins_root,
|
||||
runtime_path,
|
||||
)
|
||||
from pixiv_app.core.plugin.base import Resource
|
||||
from pixiv_app.core.plugin.manager import PluginManager
|
||||
|
||||
|
||||
def project_root() -> Path:
|
||||
return app_plugins_root().parent
|
||||
|
||||
|
||||
class XMediaService:
|
||||
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
|
||||
VIDEO_EXTS = {".mp4", ".mov", ".webm", ".m4v"}
|
||||
|
||||
def __init__(self, *, plugins_root: str | Path | None = None, save_root: str | Path | None = None) -> None:
|
||||
self.plugins_root = Path(plugins_root) if plugins_root else app_plugins_root()
|
||||
self.save_root = Path(save_root) if save_root else downloads_root() / "x"
|
||||
self.cookie_path = runtime_path("x_cookies.json")
|
||||
self.follow_store_path = runtime_path("x_followed_authors.json")
|
||||
self._manager: PluginManager | None = None
|
||||
self._preview_cache: dict[str, list[Resource]] = {}
|
||||
|
||||
def import_cookie_text(self, text: str) -> dict[str, Any]:
|
||||
cookies = parse_cookie_text(text)
|
||||
x_cookies = cookies_to_playwright(cookies, ("x.com", "twitter.com"))
|
||||
path = save_playwright_cookies(x_cookies, self.cookie_path)
|
||||
plugin = self._get_plugin()
|
||||
if hasattr(plugin, "reset_playwright_session"):
|
||||
plugin.reset_playwright_session()
|
||||
return {"count": len(x_cookies), "path": str(path), "summary": cookie_summary(x_cookies)}
|
||||
|
||||
def preview_author_media(self, username: str, *, media_type: str = "all", max_items: int = 120) -> dict[str, Any]:
|
||||
handle = _normalize_username(username)
|
||||
media_type = _normalize_media_type(media_type)
|
||||
max_items = max(1, min(int(max_items), 500))
|
||||
url = f"https://x.com/{handle}/media"
|
||||
|
||||
plugin = self._get_plugin()
|
||||
if not plugin.validate():
|
||||
raise ValueError("X plugin is unavailable. Install playwright/httpx and run playwright install chromium first.")
|
||||
|
||||
resources = plugin.parse(url)
|
||||
resources = _filter_resources(resources, media_type)[:max_items]
|
||||
preview_id = uuid.uuid4().hex
|
||||
self._preview_cache[preview_id] = resources
|
||||
return {
|
||||
"preview_id": preview_id,
|
||||
"username": handle,
|
||||
"media_type": media_type,
|
||||
"count": len(resources),
|
||||
"items": [_resource_to_json(item, i) for i, item in enumerate(resources)],
|
||||
"created_at": int(time.time()),
|
||||
}
|
||||
|
||||
def download_preview(self, preview_id: str, *, selected_ids: list[str] | None = None) -> dict[str, Any]:
|
||||
resources = self._preview_cache.get(preview_id)
|
||||
if resources is None:
|
||||
raise ValueError("Preview expired. Re-run the X media preview.")
|
||||
selected = set(selected_ids or [])
|
||||
if selected:
|
||||
resources = [item for item in resources if item.id in selected]
|
||||
if not resources:
|
||||
raise ValueError("No X media items selected.")
|
||||
|
||||
plugin = self._get_plugin()
|
||||
self.save_root.mkdir(parents=True, exist_ok=True)
|
||||
results: list[dict[str, Any]] = []
|
||||
downloaded = 0
|
||||
failed = 0
|
||||
for resource in resources:
|
||||
try:
|
||||
paths = plugin.download(resource, self.save_root)
|
||||
downloaded += len(paths)
|
||||
results.append(
|
||||
{
|
||||
"id": resource.id,
|
||||
"ok": bool(paths),
|
||||
"paths": [str(path) for path in paths],
|
||||
"title": resource.title,
|
||||
"synced_files": 0,
|
||||
}
|
||||
)
|
||||
if not paths:
|
||||
failed += 1
|
||||
except Exception as exc: # Keep batch downloads moving.
|
||||
failed += 1
|
||||
results.append({"id": resource.id, "ok": False, "error": str(exc), "title": resource.title})
|
||||
|
||||
return {
|
||||
"requested": len(resources),
|
||||
"downloaded_files": downloaded,
|
||||
"synced_files": 0,
|
||||
"failed_items": failed,
|
||||
"items": results,
|
||||
}
|
||||
|
||||
def list_local_media(self, *, limit: int = 160, offset: int = 0) -> dict[str, Any]:
|
||||
self.save_root.mkdir(parents=True, exist_ok=True)
|
||||
limit = max(1, min(int(limit), 500))
|
||||
offset = max(0, int(offset))
|
||||
files = [
|
||||
path
|
||||
for path in self.save_root.rglob("*")
|
||||
if path.is_file() and path.suffix.lower() in self.IMAGE_EXTS | self.VIDEO_EXTS
|
||||
]
|
||||
files.sort(key=lambda path: path.stat().st_mtime, reverse=True)
|
||||
page = files[offset : offset + limit]
|
||||
return {
|
||||
"total": len(files),
|
||||
"items": [self._local_media_to_json(path) for path in page],
|
||||
}
|
||||
|
||||
def delete_local_media(self, media_ids: list[str]) -> dict[str, Any]:
|
||||
deleted: list[str] = []
|
||||
failed: list[dict[str, str]] = []
|
||||
for media_id in media_ids:
|
||||
try:
|
||||
path = self.resolve_local_media(media_id)
|
||||
path.unlink()
|
||||
deleted.append(media_id)
|
||||
except Exception as exc:
|
||||
failed.append({"id": media_id, "error": str(exc)})
|
||||
return {"deleted": deleted, "failed": failed}
|
||||
|
||||
def resolve_local_media(self, media_id: str) -> Path:
|
||||
rel_text = _decode_media_id(media_id)
|
||||
target = (self.save_root / rel_text).resolve()
|
||||
root = self.save_root.resolve()
|
||||
if root != target and root not in target.parents:
|
||||
raise ValueError("Illegal media path.")
|
||||
if not target.is_file():
|
||||
raise ValueError("Local media file does not exist.")
|
||||
return target
|
||||
|
||||
def list_followed_authors(self) -> list[dict[str, Any]]:
|
||||
rows = [item for item in self._load_follow_rows() if bool(item.get("enabled", True))]
|
||||
rows.sort(key=lambda item: ((str(item.get("creator_name") or "") or str(item.get("username") or "")).lower()))
|
||||
return rows
|
||||
|
||||
def set_followed_author(self, username: str, *, followed: bool = True, creator_name: str = "") -> dict[str, Any]:
|
||||
handle = _normalize_username(username)
|
||||
rows = self._load_follow_rows()
|
||||
row = next((item for item in rows if str(item.get("username", "")).lower() == handle.lower()), None)
|
||||
now = _now_text()
|
||||
if row is None:
|
||||
row = {
|
||||
"username": handle,
|
||||
"creator_name": creator_name.strip() or handle,
|
||||
"enabled": True,
|
||||
"profile_url": f"https://x.com/{handle}",
|
||||
"latest_tweet_id": "",
|
||||
"latest_published_at": "",
|
||||
"last_checked_at": "",
|
||||
"last_error": "",
|
||||
"seen_tweet_ids": [],
|
||||
"created_at": now,
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
row["username"] = handle
|
||||
row["creator_name"] = creator_name.strip() or str(row.get("creator_name") or handle)
|
||||
row["enabled"] = bool(followed)
|
||||
row["profile_url"] = f"https://x.com/{handle}"
|
||||
row["updated_at"] = now
|
||||
if not followed:
|
||||
row["last_error"] = ""
|
||||
|
||||
self._save_follow_rows(rows)
|
||||
return dict(row)
|
||||
|
||||
def sync_followed_authors(self, *, download: bool = True, max_new_per_author: int = 20) -> dict[str, Any]:
|
||||
followed = self.list_followed_authors()
|
||||
if not followed:
|
||||
return {"authors": []}
|
||||
|
||||
plugin = self._get_plugin()
|
||||
if not plugin.validate():
|
||||
raise ValueError("X plugin is unavailable. Import a valid X cookie and ensure Playwright is installed.")
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
rows = self._load_follow_rows()
|
||||
|
||||
for row in rows:
|
||||
if not bool(row.get("enabled", True)):
|
||||
continue
|
||||
|
||||
handle = str(row.get("username") or "").strip()
|
||||
if not handle:
|
||||
continue
|
||||
|
||||
try:
|
||||
parse_for_sync = getattr(plugin, "parse_for_sync", plugin.parse)
|
||||
resources = parse_for_sync(f"https://x.com/{handle}")
|
||||
grouped = _group_resources_by_tweet(resources)
|
||||
seen_ids = [str(item) for item in row.get("seen_tweet_ids", []) if str(item).strip()]
|
||||
seen_set = set(seen_ids)
|
||||
new_groups = [item for item in grouped if item["tweet_id"] and item["tweet_id"] not in seen_set]
|
||||
new_groups = new_groups[: max(0, int(max_new_per_author))]
|
||||
|
||||
downloaded_files = 0
|
||||
synced_files = 0
|
||||
created_alerts = 0
|
||||
new_ids: list[str] = []
|
||||
|
||||
for group in new_groups:
|
||||
representative = group["resource"]
|
||||
new_ids.append(group["tweet_id"])
|
||||
|
||||
if download:
|
||||
for resource in group["resources"]:
|
||||
paths = plugin.download(resource, self.save_root)
|
||||
downloaded_files += len(paths)
|
||||
|
||||
latest_group = grouped[0] if grouped else None
|
||||
if latest_group is not None:
|
||||
representative = latest_group["resource"]
|
||||
row["creator_name"] = representative.author or str(row.get("creator_name") or handle)
|
||||
row["latest_tweet_id"] = latest_group["tweet_id"]
|
||||
row["latest_published_at"] = representative.created_at or str(row.get("latest_published_at") or "")
|
||||
|
||||
row["last_checked_at"] = _now_text()
|
||||
row["last_error"] = ""
|
||||
row["updated_at"] = _now_text()
|
||||
row["seen_tweet_ids"] = _merge_seen_ids(new_ids, seen_ids)
|
||||
|
||||
results.append(
|
||||
{
|
||||
"username": handle,
|
||||
"creator_name": row.get("creator_name") or handle,
|
||||
"new_posts": len(new_groups),
|
||||
"downloaded_files": downloaded_files,
|
||||
"synced_files": synced_files,
|
||||
"alerts": created_alerts,
|
||||
"error": "",
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
row["last_checked_at"] = _now_text()
|
||||
row["last_error"] = str(exc)
|
||||
row["updated_at"] = _now_text()
|
||||
results.append(
|
||||
{
|
||||
"username": handle,
|
||||
"creator_name": row.get("creator_name") or handle,
|
||||
"new_posts": 0,
|
||||
"downloaded_files": 0,
|
||||
"synced_files": 0,
|
||||
"alerts": 0,
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
self._save_follow_rows(rows)
|
||||
return {"authors": results}
|
||||
|
||||
def _get_plugin(self):
|
||||
if self._manager is None:
|
||||
self._manager = PluginManager(plugin_roots() if self.plugins_root == app_plugins_root() else [self.plugins_root])
|
||||
self._manager.load_all()
|
||||
plugin = self._manager.plugins.get("X")
|
||||
if plugin is None:
|
||||
raise ValueError("X plugin is missing. Confirm plugins/x exists.")
|
||||
return plugin
|
||||
|
||||
def _load_follow_rows(self) -> list[dict[str, Any]]:
|
||||
if not self.follow_store_path.is_file():
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(self.follow_store_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return []
|
||||
if isinstance(payload, list):
|
||||
return [dict(item) for item in payload if isinstance(item, dict)]
|
||||
return []
|
||||
|
||||
def _save_follow_rows(self, rows: list[dict[str, Any]]) -> None:
|
||||
self.follow_store_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.follow_store_path.write_text(
|
||||
json.dumps(rows, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _session_payload(self) -> dict[str, Any]:
|
||||
return SessionStore(app_session_file()).load()
|
||||
|
||||
def _local_media_to_json(self, path: Path) -> dict[str, Any]:
|
||||
rel = path.relative_to(self.save_root)
|
||||
stat = path.stat()
|
||||
kind = "video" if path.suffix.lower() in self.VIDEO_EXTS else "image"
|
||||
media_id = _encode_media_id(str(rel).replace("\\", "/"))
|
||||
return {
|
||||
"id": media_id,
|
||||
"name": path.name,
|
||||
"relative_path": str(rel).replace("\\", "/"),
|
||||
"kind": kind,
|
||||
"size": stat.st_size,
|
||||
"modified_at": int(stat.st_mtime),
|
||||
"url": f"/media/x-local/{media_id}",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_username(username: str) -> str:
|
||||
value = username.strip()
|
||||
if value.startswith("@"):
|
||||
value = value[1:]
|
||||
if "/" in value:
|
||||
parts = [item for item in value.replace("https://", "").replace("http://", "").split("/") if item]
|
||||
value = parts[1] if parts and parts[0].lower() in {"x.com", "twitter.com", "www.x.com", "www.twitter.com"} else parts[0]
|
||||
if not value:
|
||||
raise ValueError("Please enter an X username.")
|
||||
if len(value) > 15 or not value.replace("_", "").isalnum():
|
||||
raise ValueError("Invalid X username format.")
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_media_type(media_type: str) -> str:
|
||||
value = (media_type or "all").strip().lower()
|
||||
if value not in {"all", "image", "video"}:
|
||||
raise ValueError("media_type must be all, image, or video.")
|
||||
return value
|
||||
|
||||
|
||||
def _filter_resources(resources: list[Resource], media_type: str) -> list[Resource]:
|
||||
if media_type == "all":
|
||||
return resources
|
||||
return [item for item in resources if any(file.get("type", "image") == media_type for file in item.files)]
|
||||
|
||||
|
||||
def _resource_to_json(resource: Resource, index: int) -> dict[str, Any]:
|
||||
payload = asdict(resource)
|
||||
payload["index"] = index
|
||||
payload["kind"] = "video" if any(file.get("type") == "video" for file in resource.files) else "image"
|
||||
payload["file_count"] = len(resource.files)
|
||||
return payload
|
||||
|
||||
|
||||
def _group_resources_by_tweet(resources: list[Resource]) -> list[dict[str, Any]]:
|
||||
grouped: dict[str, dict[str, Any]] = {}
|
||||
ordered: list[str] = []
|
||||
for resource in resources:
|
||||
tweet_id = str(resource.metadata.get("tweet_id") or resource.id or "")
|
||||
key = tweet_id or resource.id
|
||||
if key not in grouped:
|
||||
grouped[key] = {
|
||||
"tweet_id": tweet_id,
|
||||
"resource": resource,
|
||||
"resources": [resource],
|
||||
}
|
||||
ordered.append(key)
|
||||
else:
|
||||
grouped[key]["resources"].append(resource)
|
||||
return [grouped[key] for key in ordered]
|
||||
|
||||
|
||||
def _merge_seen_ids(new_ids: list[str], existing_ids: list[str], *, limit: int = 200) -> list[str]:
|
||||
merged: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in [*new_ids, *existing_ids]:
|
||||
value = str(item).strip()
|
||||
if not value or value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
merged.append(value)
|
||||
if len(merged) >= limit:
|
||||
break
|
||||
return merged
|
||||
|
||||
|
||||
def _encode_media_id(value: str) -> str:
|
||||
from base64 import urlsafe_b64encode
|
||||
|
||||
return urlsafe_b64encode(value.encode("utf-8")).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def _decode_media_id(value: str) -> str:
|
||||
from base64 import urlsafe_b64decode
|
||||
|
||||
padding = "=" * (-len(value) % 4)
|
||||
return urlsafe_b64decode((value + padding).encode("ascii")).decode("utf-8")
|
||||
|
||||
|
||||
def _now_text() -> str:
|
||||
return datetime.now().isoformat(timespec="seconds")
|
||||
Reference in new issue
Block a user