first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""Core business logic for Sakura downloader."""
+167
View File
@@ -0,0 +1,167 @@
from __future__ import annotations
import html
import json
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import urlencode
import requests
from pixiv_app.core.downloader import USER_AGENT, build_headers
LOGIN_PAGE_URL = "https://accounts.pixiv.net/login"
LOGIN_AJAX_URL = "https://accounts.pixiv.net/ajax/login?lang=zh"
VALIDATE_URL = "https://www.pixiv.net/setting_user.php"
@dataclass
class AuthResult:
success: bool
message: str
cookie: str = ""
requires_verification: bool = False
user_name: str = ""
user_id: str = ""
class SessionStore:
def __init__(self, path: str | Path = "pixiv_app_session.json") -> None:
self.path = Path(path)
def load(self) -> dict[str, Any]:
if not self.path.exists():
return {}
try:
return json.loads(self.path.read_text(encoding="utf-8"))
except Exception:
return {}
def save(self, payload: dict[str, Any]) -> None:
self.path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
class PixivAuthClient:
def __init__(self) -> None:
self.timeout = 20
def validate_cookie(self, cookie: str) -> AuthResult:
cookie = cookie.strip()
if not cookie:
return AuthResult(False, "Cookie 为空,请先填写或登录。")
session = requests.Session()
session.headers.update(build_headers(cookie=cookie, referer="https://www.pixiv.net/"))
try:
response = session.get(VALIDATE_URL, allow_redirects=True, timeout=self.timeout)
final_url = response.url
if "return_to=" in final_url or "accounts.pixiv.net" in final_url:
return AuthResult(False, "当前 Cookie 已失效或未登录,请更新会话。")
user_id, user_name = self._extract_user_info(response.text)
return AuthResult(True, "Cookie 有效,可直接开始抓取。", cookie=cookie, user_name=user_name, user_id=user_id)
except Exception as exc:
return AuthResult(False, f"Cookie 检测失败: {exc}")
def login_with_password(self, login_id: str, password: str) -> AuthResult:
login_id = login_id.strip()
password = password.strip()
if not login_id or not password:
return AuthResult(False, "账号和密码都不能为空。")
session = requests.Session()
session.headers.update(
{
"User-Agent": USER_AGENT,
"Accept": "application/json, text/plain, */*",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Origin": "https://accounts.pixiv.net",
"Referer": LOGIN_PAGE_URL,
"X-Requested-With": "XMLHttpRequest",
}
)
try:
login_page = session.get(LOGIN_PAGE_URL, timeout=self.timeout)
login_page.raise_for_status()
tt = self._extract_tt(login_page.text)
payload = {
"login_id": login_id,
"password": password,
"source": "accounts",
"app_ios": "0",
"ref": "",
"return_to": "https://www.pixiv.net/",
"g_recaptcha_response": "",
"recaptcha_enterprise_score_token": "",
}
headers = dict(session.headers)
if tt:
headers["X-CSRF-Token"] = tt
response = session.post(
LOGIN_AJAX_URL,
data=urlencode(payload),
headers={**headers, "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"},
timeout=self.timeout,
)
response.raise_for_status()
body = response.json()
if isinstance(body, dict) and body.get("body", {}).get("success"):
cookie = self._cookies_to_header(session.cookies)
validation = self.validate_cookie(cookie)
if validation.success:
validation.message = "账号密码登录成功,会话已可用。"
return validation
return AuthResult(True, "登录接口返回成功,但会话校验未完全通过,请手动检测。", cookie=cookie)
body_payload = body.get("body", {}) if isinstance(body, dict) else {}
errors = body_payload.get("errors", {}) if isinstance(body_payload, dict) else {}
if errors.get("recaptcha") or body_payload.get("requireExtraVerification") or body_payload.get("requireTwoFactorAuthentication"):
return AuthResult(
False,
"Pixiv 要求额外验证或二步验证,程序无法静默完成。请改用 Cookie 登录,或先在浏览器登录后导出 Cookie。",
requires_verification=True,
)
message = self._flatten_error_message(errors) or "账号密码登录失败,请检查账号、密码或改用 Cookie。"
return AuthResult(False, message)
except Exception as exc:
return AuthResult(False, f"账号密码登录失败: {exc}")
def _extract_tt(self, html_text: str) -> str:
match = re.search(r'"pixivAccount\.tt":"([^"]+)"', html_text)
return match.group(1) if match else ""
def _extract_user_info(self, html_text: str) -> tuple[str, str]:
user_id = ""
user_name = ""
patterns = [
r'"userData":\{"id":"(\d+)","pixivId":"[^"]*","name":"([^"]+)"',
r'"user_id":"(\d+)".*?"name":"([^"]+)"',
]
for pattern in patterns:
match = re.search(pattern, html_text)
if match:
user_id = match.group(1)
user_name = html.unescape(match.group(2))
break
return user_id, user_name
def _flatten_error_message(self, errors: dict[str, Any]) -> str:
messages: list[str] = []
for key, value in errors.items():
if isinstance(value, str) and value.strip():
messages.append(f"{key}: {value}")
elif isinstance(value, list):
text_items = [str(item).strip() for item in value if str(item).strip()]
if text_items:
messages.append(f"{key}: {'; '.join(text_items)}")
return " | ".join(messages)
def _cookies_to_header(self, jar: requests.cookies.RequestsCookieJar) -> str:
parts = [f"{cookie.name}={cookie.value}" for cookie in jar]
return "; ".join(parts)
+353
View File
@@ -0,0 +1,353 @@
"""
Playwright 浏览器代理池 - 支持动态换 IP
功能:
- 多代理轮换(轮询/随机/会话粘性)
- 健康检查(自动剔除失效代理)
- 延迟统计(加权选择最优代理)
- 线程安全
"""
import random
import threading
import time
from dataclasses import dataclass, field
from typing import Optional, List, Dict, Any
from enum import Enum
class RotationStrategy(Enum):
"""轮换策略"""
ROUND_ROBIN = "round_robin" # 轮询
RANDOM = "random" # 随机
WEIGHTED = "weighted" # 加权(按成功率+延迟)
SESSION_STICKY = "session_sticky" # 会话粘性(同会话同IP)
@dataclass
class ProxyInfo:
"""代理信息"""
server: str # http://ip:port 或 socks5://ip:port
username: str = ""
password: str = ""
# 统计数据
success_count: int = 0
fail_count: int = 0
total_latency_ms: float = 0.0
avg_latency_ms: float = 0.0
last_check_time: float = 0.0
last_fail_time: float = 0.0
# 状态
healthy: bool = True
cooldown_until: float = 0.0 # 冷却结束时间
@property
def proxy_url(self) -> str:
"""获取完整的代理URL(含认证)"""
if self.username and self.password:
# http://user:pass@ip:port
protocol = self.server.split("://")[0]
rest = self.server.split("://")[1]
return f"{protocol}://{self.username}:{self.password}@{rest}"
return self.server
def to_requests_proxies(self) -> dict[str, str]:
return {"http": self.proxy_url, "https": self.proxy_url}
def to_playwright_proxy(self) -> dict[str, str]:
data = {"server": self.server}
if self.username:
data["username"] = self.username
data["password"] = self.password
return data
@property
def success_rate(self) -> float:
"""成功率"""
total = self.success_count + self.fail_count
if total == 0:
return 1.0
return self.success_count / total
@property
def weight(self) -> float:
"""
计算代理权重(用于加权随机选择)
考虑因素:成功率、延迟、冷却状态
"""
if not self.healthy:
return 0.0
# 冷却中
if self.cooldown_until > time.time():
return 0.0
# 成功率权重
success_weight = self.success_rate
# 延迟权重(延迟越低权重越高)
if self.avg_latency_ms > 0:
latency_weight = max(0.1, 1000.0 / max(self.avg_latency_ms, 1))
else:
latency_weight = 1.0
# 失败惩罚
fail_penalty = 1.0 / (1.0 + self.fail_count * 0.5)
return success_weight * latency_weight * fail_penalty
class ProxyPool:
"""代理池 - 管理多个代理的获取和轮换"""
def __init__(
self,
strategy: RotationStrategy = RotationStrategy.WEIGHTED,
health_check_url: str = "https://httpbin.org/ip",
health_check_timeout: int = 8,
max_failures: int = 3,
cooldown_seconds: int = 60,
auto_health_check: bool = True,
):
"""
Args:
strategy: 轮换策略
health_check_url: 健康检查URL
health_check_timeout: 健康检查超时(秒)
max_failures: 最大失败次数,超过后标记为不健康
cooldown_seconds: 不健康代理的冷却时间(秒)
auto_health_check: 是否自动健康检查
"""
self.strategy = strategy
self.health_check_url = health_check_url
self.health_check_timeout = health_check_timeout
self.max_failures = max_failures
self.cooldown_seconds = cooldown_seconds
self.auto_health_check = auto_health_check
self._proxies: List[ProxyInfo] = []
self._session_proxy_map: Dict[str, ProxyInfo] = {} # 会话粘性用
self._round_robin_index = 0
self._lock = threading.Lock()
def add_proxy(self, server: str, username: str = "", password: str = "") -> None:
"""添加单个代理"""
with self._lock:
# 去重
existing = {p.server for p in self._proxies}
if server not in existing:
self._proxies.append(ProxyInfo(
server=server,
username=username,
password=password,
))
def add_proxy_url(self, value: str) -> None:
parsed = parse_proxy_url(value)
if parsed is not None:
self.add_proxy(parsed.server, parsed.username, parsed.password)
def add_proxies(self, proxies: List[dict]) -> None:
"""批量添加代理
Args:
proxies: [{"server": "http://1.2.3.4:8080", "username": "", "password": ""}, ...]
"""
for p in proxies:
self.add_proxy(
server=p["server"],
username=p.get("username", ""),
password=p.get("password", ""),
)
def add_proxy_text(self, text: str) -> int:
count = 0
for line in text.splitlines():
parsed = parse_proxy_url(line)
if parsed is None:
continue
self.add_proxy(parsed.server, parsed.username, parsed.password)
count += 1
return count
def remove_proxy(self, server: str) -> None:
"""移除代理"""
with self._lock:
self._proxies = [p for p in self._proxies if p.server != server]
def get_proxy(self, session_id: str = None) -> Optional[ProxyInfo]:
"""
获取一个代理
Args:
session_id: 会话ID,用于SESSION_STICKY策略
Returns:
代理信息,如果没有可用代理返回None
"""
with self._lock:
# 过滤出健康的代理
now = time.time()
healthy = [p for p in self._proxies if p.healthy and p.cooldown_until <= now]
if not healthy:
# 尝试从冷却中恢复
now = time.time()
recovered = []
for p in self._proxies:
if not p.healthy and p.cooldown_until <= now:
p.healthy = True
p.fail_count = 0
recovered.append(p)
if recovered:
healthy = recovered
if not healthy:
return None
# 根据策略选择代理
if self.strategy == RotationStrategy.RANDOM:
return random.choice(healthy)
elif self.strategy == RotationStrategy.WEIGHTED:
# 加权随机选择
weights = [p.weight for p in healthy]
total_weight = sum(weights)
if total_weight <= 0:
return random.choice(healthy)
r = random.uniform(0, total_weight)
cumsum = 0
for i, w in enumerate(weights):
cumsum += w
if r <= cumsum:
return healthy[i]
return healthy[0]
elif self.strategy == RotationStrategy.SESSION_STICKY:
if session_id and session_id in self._session_proxy_map:
proxy = self._session_proxy_map[session_id]
if proxy in healthy:
return proxy
# 新会话,分配一个代理
proxy = self._get_round_robin(healthy)
if session_id:
self._session_proxy_map[session_id] = proxy
return proxy
else: # ROUND_ROBIN
return self._get_round_robin(healthy)
def _get_round_robin(self, proxies: List[ProxyInfo]) -> ProxyInfo:
"""轮询获取代理"""
proxy = proxies[self._round_robin_index % len(proxies)]
self._round_robin_index += 1
return proxy
def mark_success(self, proxy: ProxyInfo, latency_ms: float = 0) -> None:
"""标记代理请求成功"""
if proxy is None:
return
with self._lock:
proxy.success_count += 1
proxy.fail_count = 0
proxy.healthy = True
if latency_ms > 0:
# 更新平均延迟(指数移动平均)
if proxy.avg_latency_ms == 0:
proxy.avg_latency_ms = latency_ms
else:
proxy.avg_latency_ms = proxy.avg_latency_ms * 0.7 + latency_ms * 0.3
proxy.last_check_time = time.time()
def mark_failed(self, proxy: ProxyInfo) -> None:
"""标记代理请求失败"""
if proxy is None:
return
with self._lock:
proxy.fail_count += 1
if proxy.fail_count >= self.max_failures:
proxy.healthy = False
proxy.cooldown_until = time.time() + self.cooldown_seconds
def health_check(self, proxy: ProxyInfo = None) -> bool:
"""
健康检查单个或所有代理
Returns:
如果检查通过返回True
"""
import requests
proxies_to_check = [proxy] if proxy else self._proxies
for p in proxies_to_check:
if p is None:
continue
try:
start = time.time()
resp = requests.get(
self.health_check_url,
proxies={"http": p.proxy_url, "https": p.proxy_url},
timeout=self.health_check_timeout,
)
if resp.status_code == 200:
latency_ms = (time.time() - start) * 1000
self.mark_success(p, latency_ms)
else:
self.mark_failed(p)
except Exception:
self.mark_failed(p)
return proxy.healthy if proxy else len(self.get_healthy_proxies()) > 0
def get_healthy_proxies(self) -> List[ProxyInfo]:
"""获取所有健康代理"""
with self._lock:
return [p for p in self._proxies if p.healthy]
def get_stats(self) -> dict:
"""获取代理池统计信息"""
with self._lock:
healthy = [p for p in self._proxies if p.healthy]
return {
"total": len(self._proxies),
"healthy": len(healthy),
"unhealthy": len(self._proxies) - len(healthy),
"proxies": [
{
"server": p.server,
"healthy": p.healthy,
"success_rate": p.success_rate,
"avg_latency_ms": round(p.avg_latency_ms, 2),
"fail_count": p.fail_count,
}
for p in self._proxies
]
}
def release_session(self, session_id: str) -> None:
"""释放会话绑定的代理"""
with self._lock:
self._session_proxy_map.pop(session_id, None)
def parse_proxy_url(value: str) -> Optional[ProxyInfo]:
text = value.strip()
if not text or text.startswith("#"):
return None
if "://" not in text:
text = "http://" + text
import re
match = re.match(r"^(https?|socks5)://(?:(?P<user>[^:@/]+):(?P<pwd>[^@/]+)@)?(?P<host>[\w.\-]+):(?P<port>\d{2,5})$", text, re.I)
if not match:
return None
scheme = text.split("://", 1)[0].lower()
host = match.group("host")
port = match.group("port")
user = match.group("user") or ""
pwd = match.group("pwd") or ""
return ProxyInfo(server=f"{scheme}://{host}:{port}", username=user, password=pwd)
+602
View File
@@ -0,0 +1,602 @@
from __future__ import annotations
import base64
import hashlib
import json
import logging
import threading
import time
from collections.abc import Callable
from dataclasses import asdict
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import requests
from pixiv_app.core.paths import runtime_path
logger = logging.getLogger(__name__)
def file_sha256(path: str | Path) -> str:
digest = hashlib.sha256()
with Path(path).open("rb") as fp:
for chunk in iter(lambda: fp.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def persist_cloud_session_tokens(access_token: str, refresh_token: str = "") -> None:
"""将刷新后的 Token 写回桌面会话文件,供 GUI 与 Gallery API 共用。"""
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.paths import app_session_file
store = SessionStore(app_session_file())
payload = store.load()
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
cloud["access_token"] = access_token
if refresh_token:
cloud["refresh_token"] = refresh_token
payload["cloud_sync"] = cloud
store.save(payload)
def _decode_jwt_exp(token: str) -> float:
"""解码 JWT 的 exp 字段,返回 Unix 时间戳;解码失败返回 0。"""
if not token or "." not in token:
return 0.0
try:
payload = token.split(".")[1]
payload += "=" * (4 - len(payload) % 4)
decoded = json.loads(base64.urlsafe_b64decode(payload))
return float(decoded.get("exp", 0))
except Exception:
return 0.0
class CloudSession:
"""全局单例,统一管理云同步 Token 的读写、刷新与持久化。
所有需要访问 Sakura Cloud API 的模块(GUI、Gallery、FollowSync、XMedia)
都通过此单例获取配置和创建客户端,确保 token 刷新互不冲突、变更全局可见。
"""
_instance: CloudSession | None = None
_instance_lock = threading.Lock()
def __new__(cls) -> CloudSession:
if cls._instance is None:
with cls._instance_lock:
if cls._instance is None:
instance = super().__new__(cls)
instance._base_url = ""
instance._access_token = ""
instance._refresh_token = ""
instance._enabled = False
instance._refresh_lock = threading.Lock()
instance._listeners: list[Callable[[str, str], None]] = []
instance._last_refresh_time = 0.0
cls._instance = instance
return cls._instance
# ------------------------------------------------------------------
# 配置入口
# ------------------------------------------------------------------
def configure(
self,
*,
base_url: str = "",
access_token: str = "",
refresh_token: str = "",
enabled: bool = False,
) -> None:
"""GUI 启动 / 登录 / 保存会话时调用,同步最新 token 到全局单例。"""
self._base_url = (base_url or "").strip()
self._access_token = (access_token or "").strip()
self._refresh_token = (refresh_token or "").strip()
self._enabled = bool(enabled)
def load_from_disk(self) -> dict[str, Any]:
"""从磁盘会话文件读取云端配置并应用到单例,返回原始 payload。"""
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.paths import app_session_file
store = SessionStore(app_session_file())
payload = store.load()
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
self.configure(
base_url=str(cloud.get("base_url") or ""),
access_token=str(cloud.get("access_token") or ""),
refresh_token=str(cloud.get("refresh_token") or ""),
enabled=bool(cloud.get("enabled", False)),
)
return payload
@property
def enabled(self) -> bool:
return self._enabled and bool(self._base_url) and bool(self._access_token)
# ------------------------------------------------------------------
# Config 快照
# ------------------------------------------------------------------
def get_config(self) -> CloudSyncConfig:
"""返回当前 token 的 CloudSyncConfig 快照。
调用时机:每次创建 SakuraCloudClient 前。
若 access_token 即将过期(< 5 分钟),主动触发一次刷新。
"""
if self._access_token and self._is_expired_or_near(threshold_seconds=300):
self.try_refresh()
return CloudSyncConfig(
base_url=self._base_url,
access_token=self._access_token,
refresh_token=self._refresh_token,
)
# ------------------------------------------------------------------
# 监听者管理
# ------------------------------------------------------------------
def add_listener(self, callback: Callable[[str, str], None]) -> None:
"""注册 token 刷新回调(如 GUI 的 _on_cloud_token_refreshed)。"""
if callback not in self._listeners:
self._listeners.append(callback)
def remove_listener(self, callback: Callable[[str, str], None]) -> None:
try:
self._listeners.remove(callback)
except ValueError:
pass
def notify_listeners(self, access_token: str, refresh_token: str) -> None:
"""通知所有监听者 token 已更新(成功时为新的 token 对,失败时为空字符串)。"""
for callback in self._listeners:
try:
callback(access_token, refresh_token)
except Exception:
logger.exception("CloudSession listener %s failed", getattr(callback, "__name__", callback))
# ------------------------------------------------------------------
# Token 刷新(核心)
# ------------------------------------------------------------------
def try_refresh(self) -> bool:
"""统一刷新入口。使用全局锁防止并发重复刷新。
Returns:
True: 刷新成功,access_token 已更新
False: 刷新失败,token 已被清空,监听者已收到空 token 通知
"""
refresh = self._refresh_token
if not refresh:
return False
with self._refresh_lock:
# 二次检查:可能其他线程已刷新
refresh = self._refresh_token
if not refresh:
return False
# 距上次刷新不足 5 秒,跳过(防抖)
elapsed = time.time() - self._last_refresh_time
if elapsed < 5.0 and self._access_token:
return True
try:
resp = requests.post(
f"{self._base_url}/api/v1/auth/refresh",
json={"refresh_token": refresh},
timeout=30.0,
)
if resp.status_code != 200:
logger.warning(
"CloudSession refresh failed HTTP %s: %s",
resp.status_code,
resp.text[:200],
)
return False
body = resp.json()
data = body.get("data", body) if isinstance(body, dict) else {}
if not isinstance(data, dict):
self._on_refresh_failed()
return False
new_access = (data.get("access_token") or "").strip()
new_refresh = (data.get("refresh_token") or "").strip()
if not new_access:
self._on_refresh_failed()
return False
# 更新内存
self._access_token = new_access
if new_refresh:
self._refresh_token = new_refresh
self._last_refresh_time = time.time()
# 写盘
self._persist()
# 通知监听者
self.notify_listeners(self._access_token, self._refresh_token)
logger.info("CloudSession token refreshed successfully")
return True
except Exception:
logger.exception("CloudSession refresh error")
self._on_refresh_failed()
return False
def _on_refresh_failed(self) -> None:
"""刷新失败:清空 token + 写盘 + 通知监听者。"""
self._access_token = ""
self._refresh_token = ""
self._last_refresh_time = time.time()
self._persist()
self.notify_listeners("", "")
def _persist(self) -> None:
"""将当前 token 写入磁盘会话文件。异常记录日志但不抛出。"""
try:
persist_cloud_session_tokens(self._access_token, self._refresh_token)
except Exception:
logger.exception("CloudSession persist to disk failed")
# ------------------------------------------------------------------
# JWT 辅助
# ------------------------------------------------------------------
def _is_expired_or_near(self, threshold_seconds: int = 300) -> bool:
exp = _decode_jwt_exp(self._access_token)
if exp <= 0:
return False
return (exp - threshold_seconds) <= time.time()
@dataclass
class CloudSyncConfig:
base_url: str
access_token: str
refresh_token: str = ""
chunk_size: int = 5 * 1024 * 1024
timeout: float = 60.0
@dataclass
class SyncMetadata:
resource_type: str = "artwork"
source_platform: str = "pixiv"
source_id: str | None = None
content_type: str | None = None
title: str = ""
author: str | None = None
author_id: str | None = None
tags: list[str] = field(default_factory=list)
metadata: dict[str, Any] = field(default_factory=dict)
page_count: int = 1
thumbnail_url: str | None = None
description: str | None = None
word_count: int = 0
series_id: str | None = None
series_title: str | None = None
chapter_title: str | None = None
chapter_order: int = 0
class SyncFailureQueue:
def __init__(self, path: str | Path | None = None) -> None:
self.path = Path(path) if path else runtime_path("cloud_sync_failures.json")
def load(self) -> list[dict[str, Any]]:
if not self.path.is_file():
return []
try:
data = json.loads(self.path.read_text(encoding="utf-8"))
return data if isinstance(data, list) else []
except Exception:
return []
def save(self, items: list[dict[str, Any]]) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8")
def add(self, file_path: str | Path, metadata: SyncMetadata, mime_type: str | None, error: str) -> None:
now = time.time()
file_path = str(Path(file_path))
items = self.load()
key = (file_path, metadata.source_platform, metadata.source_id)
for item in items:
item_key = (
str(item.get("file_path") or ""),
str(item.get("metadata", {}).get("source_platform") or ""),
item.get("metadata", {}).get("source_id"),
)
if item_key == key:
attempts = int(item.get("attempts") or 0)
item.update(
{
"metadata": asdict(metadata),
"mime_type": mime_type,
"attempts": attempts,
"last_error": error,
"updated_at": now,
"next_retry_at": now + min(3600, 60 * (2 ** min(attempts, 5))),
}
)
self.save(items)
return
items.append(
{
"file_path": file_path,
"metadata": asdict(metadata),
"mime_type": mime_type,
"attempts": 0,
"last_error": error,
"created_at": now,
"updated_at": now,
"next_retry_at": now + 60,
}
)
self.save(items)
class SakuraCloudClient:
def __init__(
self,
config: CloudSyncConfig,
*,
on_token_refreshed: Callable[[str, str], None] | None = None,
cloud_session: CloudSession | None = None,
) -> None:
self.config = config
self.base_url = config.base_url.rstrip("/")
self.session = requests.Session()
self.session.headers.update({"Authorization": f"Bearer {config.access_token}"})
self._cloud_session = cloud_session # 全局单例引用
# 仅在没有 cloud_session 时使用独立锁和回调(兼容旧路径)
if cloud_session is None:
self._on_token_refreshed = on_token_refreshed or persist_cloud_session_tokens
self._refresh_lock = threading.Lock()
else:
self._on_token_refreshed = None
self._refresh_lock = None
def check_file(self, sha256: str, file_size: int) -> dict[str, Any]:
return self._post("/api/v1/sync/check", {"sha256": sha256, "file_size": file_size})
def init_upload(
self,
*,
sha256: str,
file_name: str,
file_size: int,
mime_type: str | None,
resource_type: str,
) -> dict[str, Any]:
return self._post(
"/api/v1/upload/init",
{
"sha256": sha256,
"file_name": file_name,
"file_size": file_size,
"mime_type": mime_type,
"resource_type": resource_type,
},
)
def upload_chunk(self, upload_id: str, chunk: bytes) -> dict[str, Any]:
response = self._request(
"POST",
"/api/v1/upload/chunk",
data={"upload_id": upload_id},
files={"chunk": ("chunk.bin", chunk)},
)
return self._unwrap(response)
def complete_upload(self, upload_id: str) -> dict[str, Any]:
return self._post("/api/v1/upload/complete", {"upload_id": upload_id})
def sync_metadata(self, file_id: str, metadata: SyncMetadata) -> dict[str, Any]:
payload = {
"file_id": file_id,
"resource_type": metadata.resource_type,
"source_platform": metadata.source_platform,
"source_id": metadata.source_id,
"content_type": metadata.content_type,
"title": metadata.title,
"author": metadata.author,
"author_id": metadata.author_id,
"tags": metadata.tags,
"metadata": metadata.metadata,
"page_count": metadata.page_count,
"thumbnail_url": metadata.thumbnail_url,
"description": metadata.description,
"word_count": metadata.word_count,
"series_id": metadata.series_id,
"series_title": metadata.series_title,
"chapter_title": metadata.chapter_title,
"chapter_order": metadata.chapter_order,
}
return self._post("/api/v1/sync/metadata", payload)
def upsert_follow(self, payload: dict[str, Any]) -> dict[str, Any]:
return self._post("/api/v1/follows", payload)
def bulk_upsert_follows(self, items: list[dict[str, Any]]) -> dict[str, Any]:
return self._post("/api/v1/follows/bulk-upsert", {"items": items})
def create_alert(self, payload: dict[str, Any]) -> dict[str, Any]:
return self._post("/api/v1/follows/alerts", payload)
def _request(
self,
method: str,
path: str,
*,
json: dict[str, Any] | None = None,
data: dict[str, Any] | None = None,
files: dict[str, Any] | None = None,
_retry_on_401: bool = True,
) -> requests.Response:
url = f"{self.base_url}{path}"
response = self.session.request(
method,
url,
json=json,
data=data,
files=files,
timeout=self.config.timeout,
)
if response.status_code == 401 and _retry_on_401 and self._try_refresh_token():
response = self.session.request(
method,
url,
json=json,
data=data,
files=files,
timeout=self.config.timeout,
)
return response
def _post(self, path: str, payload: dict[str, Any], *, _retry_on_401: bool = True) -> dict[str, Any]:
response = self._request("POST", path, json=payload, _retry_on_401=_retry_on_401)
return self._unwrap(response)
def _try_refresh_token(self) -> bool:
# --- 新路径:委托给全局 CloudSession ---
if self._cloud_session is not None:
ok = self._cloud_session.try_refresh()
if ok:
# 刷新成功后同步更新本地 config 和 session headers
new_config = self._cloud_session.get_config()
self.config.access_token = new_config.access_token
self.config.refresh_token = new_config.refresh_token
self.session.headers.update({"Authorization": f"Bearer {new_config.access_token}"})
return ok
# --- 旧路径(向后兼容,无 cloud_session 时使用) ---
refresh = self.config.refresh_token.strip()
if not refresh:
return False
with self._refresh_lock:
refresh = self.config.refresh_token.strip()
if not refresh:
return False
try:
resp = requests.post(
f"{self.base_url}/api/v1/auth/refresh",
json={"refresh_token": refresh},
timeout=self.config.timeout,
)
if resp.status_code != 200:
return False
body = resp.json()
data = body.get("data", body) if isinstance(body, dict) else {}
if not isinstance(data, dict):
return False
new_access = (data.get("access_token") or "").strip()
new_refresh = (data.get("refresh_token") or "").strip()
if not new_access:
return False
self.config.access_token = new_access
self.session.headers.update({"Authorization": f"Bearer {new_access}"})
if new_refresh:
self.config.refresh_token = new_refresh
try:
self._on_token_refreshed(self.config.access_token, self.config.refresh_token)
except Exception:
logger.exception("SakuraCloudClient._on_token_refreshed failed")
return True
except Exception:
logger.exception("SakuraCloudClient._try_refresh_token error")
return False
@staticmethod
def _unwrap(response: requests.Response) -> dict[str, Any]:
response.raise_for_status()
body = response.json()
if isinstance(body, dict) and "data" in body:
return body["data"]
return body
class SyncManager:
def __init__(self, client: SakuraCloudClient, failure_queue: SyncFailureQueue | None = None) -> None:
self.client = client
self.failure_queue = failure_queue or SyncFailureQueue()
def sync_file(self, file_path: str | Path, metadata: SyncMetadata, *, mime_type: str | None = None) -> dict[str, Any]:
try:
return self._sync_file_once(file_path, metadata, mime_type=mime_type)
except Exception as exc:
self.failure_queue.add(file_path, metadata, mime_type, str(exc))
raise
def _sync_file_once(self, file_path: str | Path, metadata: SyncMetadata, *, mime_type: str | None = None) -> dict[str, Any]:
path = Path(file_path)
sha256 = file_sha256(path)
file_size = path.stat().st_size
checked = self.client.check_file(sha256, file_size)
file_id = checked.get("file_id") if checked.get("owned_by_user") else None
if not file_id:
init = self.client.init_upload(
sha256=sha256,
file_name=path.name,
file_size=file_size,
mime_type=mime_type,
resource_type=self._upload_resource_type(metadata, mime_type),
)
if init.get("exists") and init.get("file_id"):
file_id = str(init["file_id"])
else:
upload_id = str(init["upload_id"])
with path.open("rb") as fp:
for chunk in iter(lambda: fp.read(self.client.config.chunk_size), b""):
self.client.upload_chunk(upload_id, chunk)
completed = self.client.complete_upload(upload_id)
file_id = str(completed["file_id"])
return self.client.sync_metadata(str(file_id), metadata)
@staticmethod
def _upload_resource_type(metadata: SyncMetadata, mime_type: str | None) -> str:
if metadata.resource_type == "novel":
return "novel"
if (mime_type or "").lower().startswith("video/") or (metadata.content_type or "").lower() == "video":
return "video"
return "image"
def retry_failed(self, *, limit: int = 5) -> dict[str, int]:
now = time.time()
items = self.failure_queue.load()
kept: list[dict[str, Any]] = []
retried = 0
succeeded = 0
failed = 0
for item in items:
if retried >= limit or float(item.get("next_retry_at") or 0) > now:
kept.append(item)
continue
path = Path(str(item.get("file_path") or ""))
if not path.is_file():
failed += 1
continue
try:
metadata = SyncMetadata(**dict(item.get("metadata") or {}))
self._sync_file_once(path, metadata, mime_type=item.get("mime_type"))
succeeded += 1
except Exception as exc:
attempts = int(item.get("attempts") or 0) + 1
item["attempts"] = attempts
item["last_error"] = str(exc)
item["updated_at"] = now
item["next_retry_at"] = now + min(3600, 60 * (2 ** min(attempts, 5)))
kept.append(item)
failed += 1
retried += 1
self.failure_queue.save(kept)
return {"retried": retried, "succeeded": succeeded, "failed": failed, "remaining": len(kept)}
+175
View File
@@ -0,0 +1,175 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
Cookie = dict[str, Any]
def parse_cookie_text(text: str) -> list[Cookie]:
"""Parse JSON cookie exports or Netscape cookies.txt content."""
text = text.strip()
if not text:
raise ValueError("Cookie 文件为空。")
try:
payload = json.loads(text)
except json.JSONDecodeError:
payload = None
if payload is not None:
raw_items = payload.get("cookies", payload) if isinstance(payload, dict) else payload
if not isinstance(raw_items, list):
raise ValueError("Cookie JSON 必须是数组,或包含 cookies 数组。")
cookies = [_normalize_json_cookie(item) for item in raw_items if isinstance(item, dict)]
else:
cookies = _parse_netscape_cookie_text(text)
cookies = [item for item in cookies if item.get("name") and item.get("value") is not None]
if not cookies:
raise ValueError("没有解析到有效 Cookie。")
return cookies
def filter_cookies_for_domains(cookies: list[Cookie], domains: tuple[str, ...]) -> list[Cookie]:
result: list[Cookie] = []
for cookie in cookies:
domain = str(cookie.get("domain", "")).lower().lstrip(".")
if any(domain == item or domain.endswith("." + item) for item in domains):
result.append(cookie)
return result
def cookie_domains(cookies: list[Cookie]) -> list[str]:
domains = sorted({str(item.get("domain", "")).lower().lstrip(".") for item in cookies if item.get("domain")})
return [item for item in domains if item]
def has_cookie_domain(cookies: list[Cookie], domains: tuple[str, ...]) -> bool:
return bool(filter_cookies_for_domains(cookies, domains))
def cookies_to_header(cookies: list[Cookie], domains: tuple[str, ...] | None = None) -> str:
selected = filter_cookies_for_domains(cookies, domains) if domains else cookies
pairs: list[str] = []
seen: set[str] = set()
for cookie in selected:
name = str(cookie.get("name", "")).strip()
value = str(cookie.get("value", ""))
if not name or name in seen:
continue
seen.add(name)
pairs.append(f"{name}={value}")
if not pairs:
raise ValueError("没有找到匹配站点的 Cookie。")
return "; ".join(pairs)
def cookies_to_playwright(cookies: list[Cookie], domains: tuple[str, ...] | None = None) -> list[Cookie]:
selected = filter_cookies_for_domains(cookies, domains) if domains else cookies
output_by_key: dict[tuple[str, str, str], Cookie] = {}
for cookie in selected:
name = str(cookie.get("name") or "").strip()
domain = str(cookie.get("domain") or "").strip()
if not name or not domain:
continue
item: Cookie = {
"name": name,
"value": str(cookie.get("value", "")),
"domain": domain,
"path": str(cookie.get("path") or "/"),
"secure": bool(cookie.get("secure", False)),
"httpOnly": bool(cookie.get("httpOnly", False)),
}
expires = cookie.get("expires")
if expires is not None:
try:
expires_value = int(float(expires))
if expires_value > 0:
item["expires"] = expires_value
except (TypeError, ValueError):
pass
same_site = str(cookie.get("sameSite", "") or "").strip()
if same_site in {"Strict", "Lax", "None"}:
item["sameSite"] = same_site
key = (str(item["domain"]).lower(), str(item["path"]), name)
output_by_key[key] = item
output = list(output_by_key.values())
if not output:
raise ValueError("没有找到匹配站点的 Cookie。")
return output
def save_playwright_cookies(cookies: list[Cookie], path: str | Path) -> Path:
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(cookies, ensure_ascii=False, indent=2), encoding="utf-8")
return target
def cookie_summary(cookies: list[Cookie]) -> str:
domains = sorted({str(item.get("domain", "")).lstrip(".") or "-" for item in cookies})
shown = ", ".join(domains[:5])
suffix = "..." if len(domains) > 5 else ""
return f"{len(cookies)} 个 Cookie;域名:{shown}{suffix}"
def _parse_netscape_cookie_text(text: str) -> list[Cookie]:
cookies: list[Cookie] = []
for raw_line in text.splitlines():
line = raw_line.strip()
if not line or line.startswith("#HttpOnly_"):
http_only = line.startswith("#HttpOnly_")
line = line.removeprefix("#HttpOnly_")
elif line.startswith("#"):
continue
else:
http_only = False
parts = line.split("\t")
if len(parts) < 7:
parts = line.split()
if len(parts) < 7:
continue
domain, _include_subdomains, path, secure, expires, name = parts[:6]
value = "\t".join(parts[6:])
cookies.append(
{
"domain": domain,
"path": path or "/",
"secure": secure.upper() == "TRUE",
"expires": int(expires) if str(expires).isdigit() else -1,
"name": name,
"value": value,
"httpOnly": http_only,
"sameSite": "Lax",
}
)
return cookies
def _normalize_json_cookie(item: Cookie) -> Cookie:
expires = item.get("expires", item.get("expirationDate", item.get("expiry")))
same_site = _normalize_same_site(item.get("sameSite"))
return {
"domain": str(item.get("domain") or item.get("host") or ""),
"path": str(item.get("path") or "/"),
"secure": bool(item.get("secure", False)),
"expires": expires,
"name": str(item.get("name") or ""),
"value": str(item.get("value") or ""),
"httpOnly": bool(item.get("httpOnly", item.get("httponly", False))),
"sameSite": same_site,
}
def _normalize_same_site(value: Any) -> str:
text = str(value or "").strip().lower()
if text in {"strict", "samesitestrict"}:
return "Strict"
if text in {"none", "no_restriction", "samesitenone"}:
return "None"
return "Lax"
File diff suppressed because it is too large. Load diff
+159
View File
@@ -0,0 +1,159 @@
from __future__ import annotations
import copy
import os
import random
import threading
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import Any
class DeviceType(Enum):
WINDOWS_CHROME = "windows_chrome"
WINDOWS_EDGE = "windows_edge"
MAC_CHROME = "mac_chrome"
@dataclass
class BrowserProfile:
"""Stable browser context settings for one logical session.
This module intentionally avoids Canvas/WebGL spoofing or large fingerprint rotation.
The goal is consistency inside a session, not impersonating many different users.
"""
user_agent: str
platform: str = "Win32"
viewport: dict[str, int] = field(default_factory=lambda: {"width": 1920, "height": 1080})
locale: str = "zh-CN"
timezone_id: str = "Asia/Shanghai"
color_scheme: str = "light"
reduced_motion: str = "no-preference"
device_scale_factor: float = 1.0
is_mobile: bool = False
has_touch: bool = False
def to_playwright_context_options(self) -> dict[str, Any]:
return {
"user_agent": self.user_agent,
"viewport": dict(self.viewport),
"locale": self.locale,
"timezone_id": self.timezone_id,
"color_scheme": self.color_scheme,
"reduced_motion": self.reduced_motion,
"device_scale_factor": self.device_scale_factor,
"is_mobile": self.is_mobile,
"has_touch": self.has_touch,
}
def headers(self) -> dict[str, str]:
return {
"User-Agent": self.user_agent,
"Accept-Language": f"{self.locale},zh;q=0.9,en;q=0.8",
}
class BrowserProfileDatabase:
@staticmethod
def windows_chrome() -> BrowserProfile:
return BrowserProfile(
user_agent=real_browser_user_agent()
or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
platform="Win32",
viewport={"width": 1920, "height": 1080},
)
@staticmethod
def windows_edge() -> BrowserProfile:
return BrowserProfile(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
platform="Win32",
viewport={"width": 1536, "height": 864},
device_scale_factor=1.25,
)
@staticmethod
def mac_chrome() -> BrowserProfile:
return BrowserProfile(
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
platform="MacIntel",
viewport={"width": 1512, "height": 982},
device_scale_factor=2.0,
)
@staticmethod
def all_desktop() -> list[BrowserProfile]:
return [
BrowserProfileDatabase.windows_chrome(),
BrowserProfileDatabase.windows_edge(),
BrowserProfileDatabase.mac_chrome(),
]
@staticmethod
def by_device_type(device_type: DeviceType) -> BrowserProfile:
mapping = {
DeviceType.WINDOWS_CHROME: BrowserProfileDatabase.windows_chrome,
DeviceType.WINDOWS_EDGE: BrowserProfileDatabase.windows_edge,
DeviceType.MAC_CHROME: BrowserProfileDatabase.mac_chrome,
}
return mapping.get(device_type, BrowserProfileDatabase.windows_chrome)()
class SessionFingerprintManager:
"""Session-sticky browser profile manager."""
def __init__(self, profiles: list[BrowserProfile] | None = None, *, strategy: str = "sticky") -> None:
self.profiles = profiles or BrowserProfileDatabase.all_desktop()
self.strategy = strategy
self._session_map: dict[str, BrowserProfile] = {}
self._index = 0
self._lock = threading.Lock()
def get_fingerprint(self, session_id: str = "default") -> BrowserProfile:
with self._lock:
if self.strategy == "sticky" and session_id in self._session_map:
return copy.deepcopy(self._session_map[session_id])
if self.strategy == "round_robin":
profile = self.profiles[self._index % len(self.profiles)]
self._index += 1
elif self.strategy == "random":
profile = random.choice(self.profiles)
else:
stable_index = abs(hash(session_id)) % len(self.profiles)
profile = self.profiles[stable_index]
self._session_map[session_id] = copy.deepcopy(profile)
return copy.deepcopy(profile)
def get_stats(self) -> dict[str, Any]:
return {
"strategy": self.strategy,
"total_profiles": len(self.profiles),
"sticky_sessions": len(self._session_map),
}
Fingerprint = BrowserProfile
FingerprintManager = SessionFingerprintManager
FingerprintDatabase = BrowserProfileDatabase
def real_browser_user_agent() -> str:
"""Load a user-provided real browser UA, if configured."""
env_value = os.environ.get("PIXIV_REAL_USER_AGENT", "").strip()
if env_value:
return env_value
path_value = os.environ.get("PIXIV_REAL_USER_AGENT_FILE", "").strip()
if path_value:
path = Path(path_value)
if path.is_file():
return path.read_text(encoding="utf-8").strip()
return ""
def default_profile(session_id: str = "default") -> BrowserProfile:
return SessionFingerprintManager().get_fingerprint(session_id)
+597
View File
@@ -0,0 +1,597 @@
from __future__ import annotations
import re
from pathlib import Path
from typing import Any
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.downloader import download_user_works, fetch_user_profile, fetch_user_work_ids
from pixiv_app.core.library_importer import LegacyDownloadImporter
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
from pixiv_app.core.paths import app_session_file
from pixiv_app.core.thumbnails import DEFAULT_THUMBNAIL_DIR, ThumbnailCache
class LocalGalleryService:
def __init__(
self,
*,
db_path: str | Path = DEFAULT_LIBRARY_DB,
thumbnail_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
) -> None:
self.db_path = Path(db_path)
library = PixivLibrary(self.db_path)
library.close()
self.thumbnail_cache = ThumbnailCache(db_path=self.db_path, cache_dir=thumbnail_dir)
def list_artworks(
self,
*,
query: str = "",
search_scope: str = "all",
tag: str = "",
artist_id: int | None = None,
artwork_type: str = "",
include_restricted: bool = True,
limit: int = 60,
offset: int = 0,
ensure_thumbnails: bool = True,
) -> dict[str, Any]:
limit = max(1, min(int(limit), 200))
offset = max(0, int(offset))
library = PixivLibrary(self.db_path)
try:
where: list[str] = []
params: list[Any] = []
joins = """
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
LEFT JOIN thumbnail_cache tc ON tc.artwork_id = a.artwork_id AND tc.page_index = 0
LEFT JOIN artwork_files af ON af.artwork_id = a.artwork_id AND af.page_index = 0
LEFT JOIN artwork_ocr_texts ocr ON ocr.artwork_id = a.artwork_id
"""
if query:
like = f"%{query}%"
if search_scope == "ocr":
where.append("(ocr.status = 'done' AND ocr.ocr_text LIKE ?)")
params.append(like)
elif search_scope == "title":
where.append("(a.title LIKE ? OR a.caption LIKE ? OR ar.name LIKE ?)")
params.extend([like, like, like])
elif search_scope == "tags":
joins += """
LEFT JOIN artwork_tags at_search ON at_search.artwork_id = a.artwork_id
LEFT JOIN tags t_search ON t_search.tag_id = at_search.tag_id
"""
where.append("t_search.name LIKE ?")
params.append(like)
else:
joins += """
LEFT JOIN artwork_tags at_search ON at_search.artwork_id = a.artwork_id
LEFT JOIN tags t_search ON t_search.tag_id = at_search.tag_id
"""
where.append(
"(a.title LIKE ? OR a.caption LIKE ? OR ar.name LIKE ? OR t_search.name LIKE ? "
"OR (ocr.status = 'done' AND ocr.ocr_text LIKE ?))"
)
params.extend([like, like, like, like, like])
if tag:
joins += """
JOIN artwork_tags at_filter ON at_filter.artwork_id = a.artwork_id
JOIN tags t_filter ON t_filter.tag_id = at_filter.tag_id
"""
where.append("t_filter.name = ?")
params.append(tag)
if artist_id is not None:
where.append("a.artist_id = ?")
params.append(int(artist_id))
if artwork_type:
where.append("a.type = ?")
params.append(artwork_type)
else:
where.append("a.type IN ('illust', 'manga')")
if not include_restricted:
where.append("a.x_restrict = 0")
where_sql = "WHERE " + " AND ".join(where) if where else ""
total = library.connection.execute(
f"SELECT COUNT(DISTINCT a.artwork_id) AS total FROM artworks a {joins} {where_sql}",
params,
).fetchone()["total"]
rows = library.connection.execute(
f"""
SELECT
a.artwork_id,
a.type,
a.title,
a.artist_id,
ar.name AS artist_name,
ar.account AS artist_account,
a.page_count,
a.bookmark_count,
a.like_count,
a.view_count,
a.x_restrict,
a.ai_type,
a.thumbnail_url,
CASE WHEN fav.artwork_id IS NULL THEN 0 ELSE 1 END AS is_favorite,
CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed,
COALESCE(tc.thumbnail_path, '') AS thumbnail_path,
COALESCE(af.file_path, '') AS primary_file_path,
CASE
WHEN ocr.status = 'done' AND ocr.ocr_text LIKE ? THEN 'ocr'
ELSE ''
END AS matched_by,
a.updated_at
FROM artworks a
{joins}
LEFT JOIN favorites fav ON fav.artwork_id = a.artwork_id
{where_sql}
GROUP BY a.artwork_id
ORDER BY a.updated_at DESC, a.artwork_id DESC
LIMIT ? OFFSET ?
""",
[f"%{query}%" if query else "", *params, limit, offset],
).fetchall()
items = [dict(row) for row in rows]
if ensure_thumbnails:
self._ensure_list_thumbnails(items)
items = self._refresh_thumbnail_paths(library, items)
return {"total": int(total), "limit": limit, "offset": offset, "items": items}
finally:
library.close()
def get_artwork(self, artwork_id: int, *, ensure_thumbnails: bool = True) -> dict[str, Any] | None:
if ensure_thumbnails:
self.thumbnail_cache.ensure_for_artwork(artwork_id)
library = PixivLibrary(self.db_path)
try:
row = library.connection.execute(
"""
SELECT a.*, ar.name AS artist_name, ar.account AS artist_account
, CASE WHEN fav.artwork_id IS NULL THEN 0 ELSE 1 END AS is_favorite
, CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed
FROM artworks a
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
LEFT JOIN favorites fav ON fav.artwork_id = a.artwork_id
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
WHERE a.artwork_id = ?
""",
(artwork_id,),
).fetchone()
if row is None:
return None
files = library.connection.execute(
"""
SELECT f.*, tc.thumbnail_path, tc.width AS thumbnail_width, tc.height AS thumbnail_height
FROM artwork_files f
LEFT JOIN thumbnail_cache tc
ON tc.artwork_id = f.artwork_id AND tc.page_index = f.page_index
WHERE f.artwork_id = ?
ORDER BY f.page_index ASC
""",
(artwork_id,),
).fetchall()
tags = library.connection.execute(
"""
SELECT t.name, t.translated_name
FROM tags t
JOIN artwork_tags at ON at.tag_id = t.tag_id
WHERE at.artwork_id = ?
ORDER BY t.name ASC
""",
(artwork_id,),
).fetchall()
history = library.connection.execute(
"""
SELECT status, downloaded_pages, skipped_pages, failed_pages, message, finished_at
FROM download_history
WHERE artwork_id = ?
ORDER BY finished_at DESC
LIMIT 20
""",
(artwork_id,),
).fetchall()
return {
"artwork": dict(row),
"files": [dict(item) for item in files],
"tags": [dict(item) for item in tags],
"history": [dict(item) for item in history],
}
finally:
library.close()
def list_tags(self, *, query: str = "", limit: int = 100) -> list[dict[str, Any]]:
library = PixivLibrary(self.db_path)
try:
params: list[Any] = []
where = ""
if query:
where = "WHERE t.name LIKE ? OR t.translated_name LIKE ?"
params.extend([f"%{query}%", f"%{query}%"])
rows = library.connection.execute(
f"""
SELECT t.name, t.translated_name, COUNT(at.artwork_id) AS artwork_count
FROM tags t
LEFT JOIN artwork_tags at ON at.tag_id = t.tag_id
{where}
GROUP BY t.tag_id
ORDER BY artwork_count DESC, t.name ASC
LIMIT ?
""",
[*params, max(1, min(int(limit), 500))],
).fetchall()
return [dict(row) for row in rows]
finally:
library.close()
def list_artists(self, *, query: str = "", limit: int = 100) -> list[dict[str, Any]]:
library = PixivLibrary(self.db_path)
try:
params: list[Any] = []
where = ""
if query:
where = "WHERE ar.name LIKE ? OR ar.account LIKE ?"
params.extend([f"%{query}%", f"%{query}%"])
rows = library.connection.execute(
f"""
SELECT ar.artist_id, ar.name, ar.account, COUNT(a.artwork_id) AS artwork_count
FROM artists ar
LEFT JOIN artworks a ON a.artist_id = ar.artist_id
{where}
GROUP BY ar.artist_id
ORDER BY
CASE WHEN COALESCE(NULLIF(ar.name, ''), NULLIF(ar.account, '')) = '' THEN 1 ELSE 0 END,
COALESCE(NULLIF(ar.name, ''), NULLIF(ar.account, ''), CAST(ar.artist_id AS TEXT)) ASC,
artwork_count DESC
LIMIT ?
""",
[*params, max(1, min(int(limit), 500))],
).fetchall()
return [dict(row) for row in rows]
finally:
library.close()
def get_artist(self, artist_id: int, *, limit: int = 60, offset: int = 0) -> dict[str, Any] | None:
library = PixivLibrary(self.db_path)
try:
row = library.connection.execute(
"""
SELECT ar.*, COUNT(a.artwork_id) AS artwork_count,
CASE WHEN fa.enabled = 1 THEN 1 ELSE 0 END AS is_followed,
fa.last_synced_at, fa.last_error
FROM artists ar
LEFT JOIN artworks a ON a.artist_id = ar.artist_id
LEFT JOIN followed_artists fa ON fa.artist_id = ar.artist_id
WHERE ar.artist_id = ?
GROUP BY ar.artist_id
""",
(artist_id,),
).fetchone()
if row is None:
return None
return {
"artist": dict(row),
"artworks": self.list_artworks(artist_id=artist_id, limit=limit, offset=offset),
}
finally:
library.close()
def list_favorites(self, *, limit: int = 60, offset: int = 0) -> dict[str, Any]:
library = PixivLibrary(self.db_path)
try:
total = library.connection.execute(
"""
SELECT COUNT(*) AS total
FROM favorites fav
JOIN artworks a ON a.artwork_id = fav.artwork_id
WHERE a.type IN ('illust', 'manga')
"""
).fetchone()["total"]
rows = library.connection.execute(
"""
SELECT
a.artwork_id, a.type, a.title, a.artist_id, ar.name AS artist_name,
ar.account AS artist_account,
a.page_count, a.bookmark_count, a.like_count, a.view_count,
a.x_restrict, a.ai_type, a.thumbnail_url,
1 AS is_favorite,
CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed,
COALESCE(tc.thumbnail_path, '') AS thumbnail_path,
COALESCE(af.file_path, '') AS primary_file_path,
fav.created_at AS favorite_at
FROM favorites fav
JOIN artworks a ON a.artwork_id = fav.artwork_id
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
LEFT JOIN thumbnail_cache tc ON tc.artwork_id = a.artwork_id AND tc.page_index = 0
LEFT JOIN artwork_files af ON af.artwork_id = a.artwork_id AND af.page_index = 0
WHERE a.type IN ('illust', 'manga')
ORDER BY fav.created_at DESC
LIMIT ? OFFSET ?
""",
(max(1, min(int(limit), 200)), max(0, int(offset))),
).fetchall()
items = [dict(row) for row in rows]
self._ensure_list_thumbnails(items)
items = self._refresh_thumbnail_paths(library, items)
return {"total": int(total), "limit": limit, "offset": offset, "items": items}
finally:
library.close()
def set_favorite(self, artwork_id: int, favorite: bool, note: str = "") -> dict[str, Any]:
library = PixivLibrary(self.db_path)
try:
library.set_favorite(artwork_id, favorite, note)
return {"artwork_id": artwork_id, "favorite": favorite}
finally:
library.close()
def list_followed_artists(self) -> list[dict[str, Any]]:
library = PixivLibrary(self.db_path)
try:
rows = library.connection.execute(
"""
SELECT fa.artist_id, COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), '') AS name,
COALESCE(NULLIF(ar.account, ''), '') AS account,
fa.enabled, fa.last_synced_at, fa.last_error,
COUNT(a.artwork_id) AS artwork_count
FROM followed_artists fa
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
LEFT JOIN artworks a ON a.artist_id = fa.artist_id
WHERE fa.enabled = 1
GROUP BY fa.artist_id
ORDER BY
CASE WHEN COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, '')) = '' THEN 1 ELSE 0 END,
COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, ''), CAST(fa.artist_id AS TEXT)) ASC,
fa.artist_id ASC
"""
).fetchall()
return [dict(row) for row in rows]
finally:
library.close()
def set_followed_artist(self, artist_ref: int | str, followed: bool, name: str = "") -> dict[str, Any]:
artist_id = self._resolve_artist_id(artist_ref)
clean_name = self._clean_name(name)
account = ""
if followed and not clean_name:
try:
profile = fetch_user_profile(
artist_id,
cookie=str(SessionStore(app_session_file()).load().get("cookie", "")).strip(),
)
profile_info = profile.get("profile") if isinstance(profile.get("profile"), dict) else {}
clean_name = self._clean_name(
profile.get("name")
or profile.get("userName")
or profile.get("displayName")
or profile_info.get("name")
)
account = self._clean_name(profile.get("account") or profile.get("userAccount"))
except Exception:
clean_name = ""
library = PixivLibrary(self.db_path)
try:
if clean_name or account:
library.upsert_artist_profile(artist_id, name=clean_name, account=account)
library.set_followed_artist(artist_id, followed, clean_name)
return {
"artist_id": artist_id,
"followed": followed,
"name": clean_name,
"account": account,
"cloud_synced": False,
}
finally:
library.close()
def _resolve_artist_id(self, artist_ref: int | str) -> int:
text = str(artist_ref).strip()
if not text:
raise ValueError("Please enter a Pixiv artist ID, profile URL, or synced artist name.")
if text.lower().startswith("artist_") and text[7:].isdigit():
return int(text[7:])
match = re.search(r"/users/(\d+)", text)
if match:
return int(match.group(1))
if text.isdigit():
return int(text)
lookup = text.lstrip("@")
library = PixivLibrary(self.db_path)
try:
rows = [
dict(row)
for row in library.connection.execute(
"""
SELECT artist_id, name, account
FROM artists
WHERE lower(name) = lower(?) OR lower(account) = lower(?)
ORDER BY
CASE
WHEN lower(account) = lower(?) THEN 0
WHEN lower(name) = lower(?) THEN 1
ELSE 2
END,
artist_id ASC
""",
(lookup, lookup, lookup, lookup),
).fetchall()
]
finally:
library.close()
if len(rows) == 1:
return int(rows[0]["artist_id"])
if not rows:
raise ValueError("Could not resolve Pixiv artist ID from that name. Enter an ID/profile URL or sync the artist first.")
choices = ", ".join(str(int(row["artist_id"])) for row in rows[:5])
raise ValueError(f"Artist name matched multiple Pixiv users; use an ID instead: {choices}")
@staticmethod
def _clean_name(value: Any) -> str:
text = str(value or "").strip()
return "" if text.lower() in {"none", "null", "undefined"} else text
def scan_legacy_downloads(self, *, root: str | Path = "Sakura_Downloads") -> dict[str, Any]:
summary = LegacyDownloadImporter(db_path=self.db_path).scan(root)
self.thumbnail_cache.build_missing(limit=500)
return summary.__dict__
def sync_followed_artists(
self,
*,
cookie: str = "",
max_new_per_artist: int = 20,
download: bool = True,
save_root: str | Path = "Sakura_Downloads",
) -> dict[str, Any]:
if not cookie:
cookie = str(SessionStore(app_session_file()).load().get("cookie", "")).strip()
if not cookie:
raise ValueError("No Pixiv cookie available. Save a session in the GUI first or pass cookie explicitly.")
library = PixivLibrary(self.db_path)
try:
followed = library.connection.execute(
"""
SELECT fa.artist_id,
COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, ''), '') AS name,
COALESCE(NULLIF(ar.account, ''), '') AS account
FROM followed_artists fa
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
WHERE fa.enabled = 1
ORDER BY fa.artist_id
"""
).fetchall()
finally:
library.close()
results: list[dict[str, Any]] = []
for row in followed:
artist_id = int(row["artist_id"])
artist_name = self._clean_name(row["name"])
artist_account = self._clean_name(row["account"])
library = PixivLibrary(self.db_path)
try:
existing = {
int(item["artwork_id"])
for item in library.connection.execute(
"SELECT artwork_id FROM artworks WHERE artist_id = ? AND type IN ('illust', 'manga')",
(artist_id,),
).fetchall()
}
finally:
library.close()
try:
_, work_ids = fetch_user_work_ids(user_id=artist_id, cookie=cookie)
new_ids = [work_id for work_id in work_ids if work_id not in existing][: max(0, int(max_new_per_artist))]
downloaded = 0
if download and new_ids:
_, download_results = download_user_works(
user_id=artist_id,
cookie=cookie,
max_workers=3,
request_delay=0.2,
save_root=save_root,
work_ids=new_ids,
)
downloaded = len([item for item in download_results if item.ok])
library = PixivLibrary(self.db_path)
try:
library.mark_artist_synced(artist_id)
finally:
library.close()
cloud_synced = False
cloud_alerts = 0
results.append(
{
"artist_id": artist_id,
"remote_count": len(work_ids),
"new_count": len(new_ids),
"downloaded": downloaded,
"cloud_synced": cloud_synced,
"alerts": cloud_alerts,
"error": "",
}
)
except Exception as exc:
library = PixivLibrary(self.db_path)
try:
library.mark_artist_synced(artist_id, error=str(exc))
finally:
library.close()
results.append({"artist_id": artist_id, "remote_count": 0, "new_count": 0, "downloaded": 0, "alerts": 0, "error": str(exc)})
return {"artists": results}
def build_missing_thumbnails(self, *, limit: int = 200) -> list[dict[str, Any]]:
return [item.__dict__ for item in self.thumbnail_cache.build_missing(limit=limit)]
def get_thumbnail_path(self, artwork_id: int, *, page_index: int = 0) -> Path | None:
self.thumbnail_cache.ensure_for_artwork(artwork_id)
library = PixivLibrary(self.db_path)
try:
row = library.connection.execute(
"""
SELECT thumbnail_path
FROM thumbnail_cache
WHERE artwork_id = ? AND page_index = ?
""",
(artwork_id, page_index),
).fetchone()
if row is None:
return None
path = Path(str(row["thumbnail_path"]))
return path if path.exists() else None
finally:
library.close()
def get_source_file_path(self, artwork_id: int, *, page_index: int = 0) -> Path | None:
library = PixivLibrary(self.db_path)
try:
row = library.connection.execute(
"""
SELECT file_path
FROM artwork_files
WHERE artwork_id = ? AND page_index = ?
""",
(artwork_id, page_index),
).fetchone()
if row is None:
return None
path = Path(str(row["file_path"]))
return path if path.exists() else None
finally:
library.close()
def _ensure_list_thumbnails(self, items: list[dict[str, Any]]) -> None:
for item in items:
if item.get("thumbnail_path"):
continue
if not item.get("primary_file_path"):
continue
self.thumbnail_cache.ensure_for_artwork(int(item["artwork_id"]))
def _refresh_thumbnail_paths(self, library: PixivLibrary, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
artwork_ids = [int(item["artwork_id"]) for item in items]
if not artwork_ids:
return items
placeholders = ",".join("?" for _ in artwork_ids)
rows = library.connection.execute(
f"""
SELECT artwork_id, thumbnail_path
FROM thumbnail_cache
WHERE page_index = 0 AND artwork_id IN ({placeholders})
""",
artwork_ids,
).fetchall()
thumbs = {int(row["artwork_id"]): str(row["thumbnail_path"]) for row in rows}
for item in items:
item["thumbnail_path"] = thumbs.get(int(item["artwork_id"]), item.get("thumbnail_path", ""))
return items
+703
View File
@@ -0,0 +1,703 @@
from __future__ import annotations
import json
import re
import sqlite3
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from pixiv_app.core.paths import app_library_file
DEFAULT_LIBRARY_DB = app_library_file()
@dataclass
class DownloadFileRecord:
page_index: int
file_path: str
source_url: str
status: str
size_bytes: int = 0
content_hash: str = ""
class PixivLibrary:
def __init__(self, db_path: str | Path = DEFAULT_LIBRARY_DB) -> None:
self.db_path = Path(db_path)
self.db_path.parent.mkdir(parents=True, exist_ok=True) if self.db_path.parent != Path(".") else None
# Allow sharing one connection across threads when guarded by a lock (e.g. TaskQueue).
self.connection = sqlite3.connect(self.db_path, timeout=30, check_same_thread=False)
self.connection.row_factory = sqlite3.Row
self.connection.execute("PRAGMA journal_mode=WAL")
self.connection.execute("PRAGMA foreign_keys=ON")
self.setup()
def close(self) -> None:
self.connection.close()
def setup(self) -> None:
with self.connection:
self.connection.executescript(
"""
CREATE TABLE IF NOT EXISTS artists (
artist_id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
account TEXT,
avatar_url TEXT,
metadata_json TEXT NOT NULL DEFAULT '{}',
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS artworks (
artwork_id INTEGER PRIMARY KEY,
artist_id INTEGER,
type TEXT NOT NULL DEFAULT 'illust',
title TEXT NOT NULL DEFAULT '',
caption TEXT NOT NULL DEFAULT '',
page_count INTEGER NOT NULL DEFAULT 0,
bookmark_count INTEGER NOT NULL DEFAULT 0,
like_count INTEGER NOT NULL DEFAULT 0,
view_count INTEGER NOT NULL DEFAULT 0,
x_restrict INTEGER NOT NULL DEFAULT 0,
ai_type INTEGER NOT NULL DEFAULT 0,
thumbnail_url TEXT,
metadata_json TEXT NOT NULL DEFAULT '{}',
created_at TEXT,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (artist_id) REFERENCES artists(artist_id)
);
CREATE TABLE IF NOT EXISTS tags (
tag_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
translated_name TEXT
);
CREATE TABLE IF NOT EXISTS artwork_tags (
artwork_id INTEGER NOT NULL,
tag_id INTEGER NOT NULL,
PRIMARY KEY (artwork_id, tag_id),
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE,
FOREIGN KEY (tag_id) REFERENCES tags(tag_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS artwork_files (
artwork_id INTEGER NOT NULL,
page_index INTEGER NOT NULL,
file_path TEXT NOT NULL,
source_url TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'unknown',
size_bytes INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (artwork_id, page_index),
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS download_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
artwork_id INTEGER NOT NULL,
status TEXT NOT NULL,
downloaded_pages INTEGER NOT NULL DEFAULT 0,
skipped_pages INTEGER NOT NULL DEFAULT 0,
failed_pages INTEGER NOT NULL DEFAULT 0,
message TEXT NOT NULL DEFAULT '',
started_at TEXT,
finished_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS sync_state (
key TEXT PRIMARY KEY,
value_json TEXT NOT NULL DEFAULT '{}',
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS thumbnail_cache (
artwork_id INTEGER NOT NULL,
page_index INTEGER NOT NULL,
source_path TEXT NOT NULL,
thumbnail_path TEXT NOT NULL,
width INTEGER NOT NULL DEFAULT 0,
height INTEGER NOT NULL DEFAULT 0,
source_mtime REAL NOT NULL DEFAULT 0,
source_size INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (artwork_id, page_index),
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS artwork_ocr_texts (
artwork_id INTEGER PRIMARY KEY,
language TEXT NOT NULL DEFAULT 'mixed',
ocr_text TEXT NOT NULL DEFAULT '',
confidence REAL,
engine TEXT NOT NULL DEFAULT 'paddleocr',
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS favorites (
artwork_id INTEGER PRIMARY KEY,
note TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS followed_artists (
artist_id INTEGER PRIMARY KEY,
name TEXT NOT NULL DEFAULT '',
enabled INTEGER NOT NULL DEFAULT 1,
last_synced_at TEXT,
last_error TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_artworks_artist_id ON artworks(artist_id);
CREATE INDEX IF NOT EXISTS idx_artworks_type ON artworks(type);
CREATE INDEX IF NOT EXISTS idx_download_history_artwork_id ON download_history(artwork_id);
CREATE INDEX IF NOT EXISTS idx_artwork_files_path ON artwork_files(file_path);
CREATE INDEX IF NOT EXISTS idx_thumbnail_cache_path ON thumbnail_cache(thumbnail_path);
CREATE INDEX IF NOT EXISTS idx_artwork_ocr_texts_status ON artwork_ocr_texts(status);
CREATE INDEX IF NOT EXISTS idx_artwork_ocr_texts_text ON artwork_ocr_texts(ocr_text);
CREATE INDEX IF NOT EXISTS idx_favorites_created_at ON favorites(created_at);
CREATE INDEX IF NOT EXISTS idx_followed_artists_enabled ON followed_artists(enabled);
CREATE TABLE IF NOT EXISTS download_tasks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
fingerprint TEXT NOT NULL UNIQUE,
target_type TEXT NOT NULL,
target_id INTEGER,
page_index INTEGER,
task_kind TEXT NOT NULL,
priority INTEGER NOT NULL DEFAULT 2,
status TEXT NOT NULL DEFAULT 'pending',
payload_json TEXT NOT NULL DEFAULT '{}',
attempts INTEGER NOT NULL DEFAULT 0,
leased_until REAL,
worker_id TEXT,
last_error TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_download_tasks_status_pri_id
ON download_tasks(status, priority DESC, id ASC);
CREATE TABLE IF NOT EXISTS artist_sync_watermark (
artist_id INTEGER PRIMARY KEY,
last_synced_illust_id INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
"""
)
self._apply_schema_migrations()
def _apply_schema_migrations(self) -> None:
"""Incremental ALTER for existing databases created before new columns/tables."""
with self.connection:
cols = {row[1] for row in self.connection.execute("PRAGMA table_info(artwork_files)")}
if "content_hash" not in cols:
self.connection.execute(
"ALTER TABLE artwork_files ADD COLUMN content_hash TEXT NOT NULL DEFAULT ''"
)
# Drop legacy placeholder names like artist_12345 so UI can fall back cleanly.
self.connection.execute(
"UPDATE artists SET name = '' WHERE name = 'artist_' || artist_id"
)
self.connection.execute(
"UPDATE followed_artists SET name = '' WHERE name = 'artist_' || artist_id"
)
self.connection.execute(
"UPDATE artists SET name = '' WHERE lower(name) IN ('none', 'null', 'undefined')"
)
self.connection.execute(
"UPDATE followed_artists SET name = '' WHERE lower(name) IN ('none', 'null', 'undefined')"
)
rows = self.connection.execute(
"SELECT artwork_id, metadata_json FROM artworks WHERE artist_id IS NULL OR artist_id = 0"
).fetchall()
for row in rows:
try:
metadata = json.loads(str(row["metadata_json"] or "{}"))
except Exception:
continue
artist_id = _as_int(metadata.get("userId") or metadata.get("author_id") or metadata.get("artistId"))
if artist_id is None:
continue
self.ensure_artist_stub(artist_id, name=str(metadata.get("userName") or metadata.get("artistName") or ""))
self.connection.execute(
"UPDATE artworks SET artist_id = ?, updated_at = CURRENT_TIMESTAMP WHERE artwork_id = ?",
(artist_id, int(row["artwork_id"])),
)
def upsert_artwork(self, detail: dict[str, Any], *, artwork_type: str = "illust") -> None:
source_id = _as_int(detail.get("illustId") or detail.get("id") or detail.get("novelId"))
if source_id is None:
return
source_platform = str(detail.get("source_platform") or detail.get("source") or "")
artwork_id = library_artwork_id(source_id, artwork_type, source_platform=source_platform)
artist_id = _as_int(detail.get("userId"))
if artist_id is not None:
self._upsert_artist(detail, artist_id)
tags = _extract_tags(detail)
urls = detail.get("urls") if isinstance(detail.get("urls"), dict) else {}
thumbnail_url = (
detail.get("url")
or detail.get("coverUrl")
or urls.get("small")
or urls.get("thumb")
or urls.get("regular")
or urls.get("original")
)
with self.connection:
self.connection.execute(
"""
INSERT INTO artworks (
artwork_id, artist_id, type, title, caption, page_count,
bookmark_count, like_count, view_count, x_restrict, ai_type,
thumbnail_url, metadata_json, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id) DO UPDATE SET
artist_id = excluded.artist_id,
type = excluded.type,
title = excluded.title,
caption = excluded.caption,
page_count = excluded.page_count,
bookmark_count = excluded.bookmark_count,
like_count = excluded.like_count,
view_count = excluded.view_count,
x_restrict = excluded.x_restrict,
ai_type = excluded.ai_type,
thumbnail_url = excluded.thumbnail_url,
metadata_json = excluded.metadata_json,
created_at = COALESCE(excluded.created_at, artworks.created_at),
updated_at = CURRENT_TIMESTAMP
""",
(
artwork_id,
artist_id,
artwork_type,
str(detail.get("title") or ""),
str(detail.get("description") or detail.get("caption") or ""),
_as_int(detail.get("pageCount")) or 0,
_as_int(detail.get("bookmarkCount")) or 0,
_as_int(detail.get("likeCount")) or 0,
_as_int(detail.get("viewCount")) or 0,
_as_int(detail.get("xRestrict")) or 0,
_as_int(detail.get("aiType")) or 0,
str(thumbnail_url) if thumbnail_url else None,
json.dumps({**detail, "sourceId": source_id}, ensure_ascii=False),
str(detail.get("createDate") or detail.get("uploadDate") or "") or None,
),
)
self.connection.execute("DELETE FROM artwork_tags WHERE artwork_id = ?", (artwork_id,))
for name, translated_name in tags:
tag_id = self._upsert_tag(name, translated_name)
self.connection.execute(
"INSERT OR IGNORE INTO artwork_tags (artwork_id, tag_id) VALUES (?, ?)",
(artwork_id, tag_id),
)
def ensure_artwork_stub(self, artwork_id: int, *, artwork_type: str = "illust", title: str = "", source_platform: str = "") -> int:
db_artwork_id = library_artwork_id(artwork_id, artwork_type, source_platform=source_platform)
with self.connection:
self.connection.execute(
"""
INSERT OR IGNORE INTO artworks (artwork_id, type, title, updated_at)
VALUES (?, ?, ?, CURRENT_TIMESTAMP)
""",
(db_artwork_id, artwork_type, title),
)
return db_artwork_id
def ensure_artist_stub(self, artist_id: int, *, name: str = "") -> None:
with self.connection:
self.connection.execute(
"""
INSERT OR IGNORE INTO artists (artist_id, name, updated_at)
VALUES (?, ?, CURRENT_TIMESTAMP)
""",
(artist_id, name or ""),
)
def upsert_artist_profile(self, artist_id: int, *, name: str = "", account: str = "", avatar_url: str = "") -> None:
clean_name = _clean_optional_text(name)
clean_account = _clean_optional_text(account)
clean_avatar = _clean_optional_text(avatar_url)
with self.connection:
self.connection.execute(
"""
INSERT INTO artists (artist_id, name, account, avatar_url, metadata_json, updated_at)
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artist_id) DO UPDATE SET
name = COALESCE(NULLIF(excluded.name, ''), artists.name),
account = COALESCE(NULLIF(excluded.account, ''), artists.account),
avatar_url = COALESCE(NULLIF(excluded.avatar_url, ''), artists.avatar_url),
metadata_json = CASE
WHEN excluded.metadata_json != '{}' THEN excluded.metadata_json
ELSE artists.metadata_json
END,
updated_at = CURRENT_TIMESTAMP
""",
(
int(artist_id),
clean_name,
clean_account,
clean_avatar,
json.dumps(
{
"userId": int(artist_id),
"userName": clean_name,
"userAccount": clean_account,
},
ensure_ascii=False,
),
),
)
def set_artwork_artist(self, artwork_id: int, artist_id: int) -> None:
with self.connection:
self.ensure_artist_stub(artist_id)
self.connection.execute(
"""
UPDATE artworks
SET artist_id = ?, updated_at = CURRENT_TIMESTAMP
WHERE artwork_id = ?
""",
(artist_id, artwork_id),
)
def record_files(self, artwork_id: int, files: list[DownloadFileRecord]) -> None:
with self.connection:
for item in files:
self.connection.execute(
"""
INSERT INTO artwork_files (
artwork_id, page_index, file_path, source_url, status, size_bytes,
content_hash, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id, page_index) DO UPDATE SET
file_path = excluded.file_path,
source_url = excluded.source_url,
status = excluded.status,
size_bytes = excluded.size_bytes,
content_hash = CASE
WHEN excluded.content_hash != '' THEN excluded.content_hash
ELSE artwork_files.content_hash
END,
updated_at = CURRENT_TIMESTAMP
""",
(
artwork_id,
item.page_index,
item.file_path,
item.source_url,
item.status,
item.size_bytes,
item.content_hash or "",
),
)
def record_download_history(
self,
*,
artwork_id: int,
status: str,
downloaded_pages: int,
skipped_pages: int,
failed_pages: int,
message: str,
started_at: str | None = None,
) -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO download_history (
artwork_id, status, downloaded_pages, skipped_pages, failed_pages, message, started_at
)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(artwork_id, status, downloaded_pages, skipped_pages, failed_pages, message, started_at),
)
def update_sync_state(self, key: str, value: dict[str, Any]) -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO sync_state (key, value_json, updated_at)
VALUES (?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(key) DO UPDATE SET
value_json = excluded.value_json,
updated_at = CURRENT_TIMESTAMP
""",
(key, json.dumps(value, ensure_ascii=False)),
)
def upsert_thumbnail(
self,
*,
artwork_id: int,
page_index: int,
source_path: str,
thumbnail_path: str,
width: int,
height: int,
source_mtime: float,
source_size: int,
) -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO thumbnail_cache (
artwork_id, page_index, source_path, thumbnail_path, width, height,
source_mtime, source_size, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id, page_index) DO UPDATE SET
source_path = excluded.source_path,
thumbnail_path = excluded.thumbnail_path,
width = excluded.width,
height = excluded.height,
source_mtime = excluded.source_mtime,
source_size = excluded.source_size,
updated_at = CURRENT_TIMESTAMP
""",
(artwork_id, page_index, source_path, thumbnail_path, width, height, source_mtime, source_size),
)
def upsert_ocr_text(
self,
artwork_id: int,
*,
text: str,
language: str = "mixed",
confidence: float | None = None,
engine: str = "paddleocr",
status: str = "done",
attempts: int = 1,
) -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO artwork_ocr_texts (
artwork_id, language, ocr_text, confidence, engine, status, attempts, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id) DO UPDATE SET
language = excluded.language,
ocr_text = excluded.ocr_text,
confidence = excluded.confidence,
engine = excluded.engine,
status = excluded.status,
attempts = excluded.attempts,
updated_at = CURRENT_TIMESTAMP
""",
(int(artwork_id), language, text, confidence, engine, status, attempts),
)
def get_ocr_text(self, artwork_id: int) -> sqlite3.Row | None:
return self.connection.execute(
"SELECT * FROM artwork_ocr_texts WHERE artwork_id = ?",
(int(artwork_id),),
).fetchone()
def set_favorite(self, artwork_id: int, favorite: bool, note: str = "") -> None:
with self.connection:
if favorite:
self.connection.execute(
"""
INSERT INTO favorites (artwork_id, note, created_at)
VALUES (?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id) DO UPDATE SET note = excluded.note
""",
(artwork_id, note),
)
else:
self.connection.execute("DELETE FROM favorites WHERE artwork_id = ?", (artwork_id,))
def set_followed_artist(self, artist_id: int, followed: bool, name: str = "") -> None:
with self.connection:
self.ensure_artist_stub(artist_id, name=name)
if followed:
self.connection.execute(
"""
INSERT INTO followed_artists (artist_id, name, enabled, created_at)
VALUES (?, ?, 1, CURRENT_TIMESTAMP)
ON CONFLICT(artist_id) DO UPDATE SET
name = COALESCE(NULLIF(excluded.name, ''), followed_artists.name),
enabled = 1
""",
(artist_id, name),
)
else:
self.connection.execute(
"UPDATE followed_artists SET enabled = 0 WHERE artist_id = ?",
(artist_id,),
)
def upsert_origin_followed_artist(
self,
artist_id: int,
*,
name: str = "",
account: str = "",
avatar_url: str = "",
) -> str:
clean_name = _clean_optional_text(name)
before = self.connection.execute(
"""
SELECT fa.enabled, fa.name AS follow_name, ar.name AS artist_name,
ar.account, ar.avatar_url
FROM followed_artists fa
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
WHERE fa.artist_id = ?
""",
(int(artist_id),),
).fetchone()
status = "created"
if before is not None:
changed = (
not bool(before["enabled"])
or (clean_name and clean_name != str(before["follow_name"] or before["artist_name"] or ""))
or (account and account != str(before["account"] or ""))
or (avatar_url and avatar_url != str(before["avatar_url"] or ""))
)
status = "updated" if changed else "skipped"
with self.connection:
self.upsert_artist_profile(
int(artist_id),
name=clean_name,
account=account,
avatar_url=avatar_url,
)
self.connection.execute(
"""
INSERT INTO followed_artists (artist_id, name, enabled, last_synced_at, last_error, created_at)
VALUES (?, ?, 1, CURRENT_TIMESTAMP, '', CURRENT_TIMESTAMP)
ON CONFLICT(artist_id) DO UPDATE SET
name = COALESCE(NULLIF(excluded.name, ''), followed_artists.name),
enabled = 1,
last_synced_at = CURRENT_TIMESTAMP,
last_error = ''
""",
(int(artist_id), clean_name),
)
return status
def mark_artist_synced(self, artist_id: int, *, error: str = "") -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO followed_artists (artist_id, enabled, last_synced_at, last_error)
VALUES (?, 1, CURRENT_TIMESTAMP, ?)
ON CONFLICT(artist_id) DO UPDATE SET
last_synced_at = CURRENT_TIMESTAMP,
last_error = excluded.last_error
""",
(artist_id, error),
)
def _upsert_artist(self, detail: dict[str, Any], artist_id: int) -> None:
self.connection.execute(
"""
INSERT INTO artists (artist_id, name, account, avatar_url, metadata_json, updated_at)
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artist_id) DO UPDATE SET
name = excluded.name,
account = COALESCE(excluded.account, artists.account),
avatar_url = COALESCE(excluded.avatar_url, artists.avatar_url),
metadata_json = excluded.metadata_json,
updated_at = CURRENT_TIMESTAMP
""",
(
artist_id,
str(detail.get("userName") or detail.get("artistName") or ""),
str(detail.get("userAccount") or "") or None,
str(detail.get("profileImageUrl") or detail.get("userImage") or "") or None,
json.dumps(
{
"userId": artist_id,
"userName": detail.get("userName"),
"userAccount": detail.get("userAccount"),
},
ensure_ascii=False,
),
),
)
def _upsert_tag(self, name: str, translated_name: str | None) -> int:
self.connection.execute(
"""
INSERT INTO tags (name, translated_name)
VALUES (?, ?)
ON CONFLICT(name) DO UPDATE SET
translated_name = COALESCE(excluded.translated_name, tags.translated_name)
""",
(name, translated_name),
)
row = self.connection.execute("SELECT tag_id FROM tags WHERE name = ?", (name,)).fetchone()
return int(row["tag_id"])
def _as_int(value: Any) -> int | None:
if value is None:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _clean_optional_text(value: Any) -> str:
text = str(value or "").strip()
return "" if text.lower() in {"none", "null", "undefined"} else text
def library_artwork_id(source_id: int, artwork_type: str = "illust", source_platform: str = "") -> int:
"""Map a source-native ID into the unified library artwork_id namespace.
Pixiv novels use negative source_id; AO3 works use a negative offset
range to avoid collisions with Pixiv novel IDs.
"""
if artwork_type == "novel":
if source_platform == "ao3":
# AO3: use negative offset range -(200M + work_id) to stay clear
# of Pixiv novel IDs (currently -1 .. -199_999_999).
AO3_OFFSET = 200_000_000
return -(AO3_OFFSET + abs(int(source_id)))
return -abs(int(source_id))
return int(source_id)
def _extract_tags(detail: dict[str, Any]) -> list[tuple[str, str | None]]:
raw_tags = detail.get("tags")
if isinstance(raw_tags, dict):
raw_tags = raw_tags.get("tags")
if not isinstance(raw_tags, list):
return []
result: list[tuple[str, str | None]] = []
for item in raw_tags:
if isinstance(item, str):
result.append((item, None))
continue
if not isinstance(item, dict):
continue
name = item.get("tag") or item.get("name")
if not name:
continue
translation = item.get("translation")
translated_name = None
if isinstance(translation, dict):
translated_name = translation.get("en") or translation.get("zh") or translation.get("romaji")
elif isinstance(translation, str):
translated_name = translation
result.append((str(name), str(translated_name) if translated_name else None))
return result
+76
View File
@@ -0,0 +1,76 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from pathlib import Path
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, DownloadFileRecord, PixivLibrary
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
ILLUST_FILE_PATTERN = re.compile(r"(?P<artwork_id>\d+)_p(?P<page_index>\d+)", re.IGNORECASE)
@dataclass
class ImportSummary:
scanned_files: int = 0
imported_files: int = 0
skipped_files: int = 0
imported_artworks: int = 0
class LegacyDownloadImporter:
def __init__(self, *, db_path: str | Path = DEFAULT_LIBRARY_DB) -> None:
self.db_path = Path(db_path)
def scan(self, root: str | Path = "Sakura_Downloads") -> ImportSummary:
root_path = Path(root)
summary = ImportSummary()
if not root_path.exists():
return summary
library = PixivLibrary(self.db_path)
imported_artworks: set[int] = set()
try:
for file_path in root_path.rglob("*"):
if not file_path.is_file() or file_path.suffix.lower() not in IMAGE_SUFFIXES:
continue
summary.scanned_files += 1
match = ILLUST_FILE_PATTERN.search(file_path.stem)
if match is None:
summary.skipped_files += 1
continue
artwork_id = int(match.group("artwork_id"))
page_index = int(match.group("page_index"))
library.ensure_artwork_stub(artwork_id, artwork_type="illust", title=str(artwork_id))
artist_id = self._infer_artist_id(root_path, file_path)
if artist_id is not None:
library.set_artwork_artist(artwork_id, artist_id)
library.record_files(
artwork_id,
[
DownloadFileRecord(
page_index=page_index,
file_path=str(file_path.resolve()),
source_url="",
status="imported",
size_bytes=file_path.stat().st_size,
)
],
)
imported_artworks.add(artwork_id)
summary.imported_files += 1
summary.imported_artworks = len(imported_artworks)
return summary
finally:
library.close()
def _infer_artist_id(self, root_path: Path, file_path: Path) -> int | None:
try:
relative = file_path.relative_to(root_path)
except ValueError:
return None
if not relative.parts:
return None
first = relative.parts[0]
return int(first) if first.isdigit() else None
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
import logging
import os
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Callable
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
OcrExtractor = Callable[[Path], tuple[str, float | None, str]]
@dataclass
class LocalOcrResult:
artwork_id: int
status: str
text_length: int = 0
confidence: float | None = None
error: str | None = None
class LocalOcrIndexer:
def __init__(
self,
*,
db_path: str | Path = DEFAULT_LIBRARY_DB,
enabled: bool | None = None,
max_dimension: int | None = None,
language: str | None = None,
max_concurrency: int | None = None,
) -> None:
self.db_path = Path(db_path)
self.enabled = enabled if enabled is not None else _env_bool("SAKURA_LOCAL_OCR_ENABLED", False)
self.max_dimension = max_dimension or int(os.getenv("SAKURA_LOCAL_OCR_MAX_DIMENSION", "1800"))
self.language = language or os.getenv("SAKURA_LOCAL_OCR_LANGUAGE", "ch")
self._semaphore = threading.BoundedSemaphore(
max_concurrency or int(os.getenv("SAKURA_LOCAL_OCR_MAX_CONCURRENCY", "1"))
)
def index_artwork(self, artwork_id: int, *, extractor: OcrExtractor | None = None) -> LocalOcrResult:
if not self.enabled and extractor is None:
return LocalOcrResult(artwork_id=artwork_id, status="disabled")
if not self._semaphore.acquire(blocking=False):
return LocalOcrResult(artwork_id=artwork_id, status="busy")
library = PixivLibrary(self.db_path)
try:
existing = library.get_ocr_text(artwork_id)
if existing is not None and str(existing["status"]) == "done" and str(existing["ocr_text"]).strip():
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(str(existing["ocr_text"])))
row = library.connection.execute(
"""
SELECT file_path
FROM artwork_files
WHERE artwork_id = ?
ORDER BY page_index ASC
LIMIT 1
""",
(int(artwork_id),),
).fetchone()
if row is None:
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="file_not_found")
path = Path(str(row["file_path"]))
if path.suffix.lower() not in {".jpg", ".jpeg", ".png", ".webp"}:
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="unsupported_file_type")
text, confidence, language = (extractor or self._paddle_ocr_extract)(path)
library.upsert_ocr_text(
artwork_id,
text=text.strip(),
confidence=confidence,
language=language or self.language,
status="done",
attempts=1,
)
logging.info("Local OCR done artwork=%s text_len=%s confidence=%s", artwork_id, len(text.strip()), confidence)
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(text.strip()), confidence=confidence)
except Exception as exc:
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
logging.warning("Local OCR failed artwork=%s error=%s", artwork_id, exc.__class__.__name__)
return LocalOcrResult(artwork_id=artwork_id, status="failed", error=exc.__class__.__name__)
finally:
library.close()
self._semaphore.release()
def _paddle_ocr_extract(self, path: Path) -> tuple[str, float | None, str]:
from PIL import Image
from paddleocr import PaddleOCR
work_path = path
with Image.open(path) as image:
original_size = image.size
image.thumbnail((self.max_dimension, self.max_dimension))
if image.size != original_size:
tmp = path.with_suffix(".local-ocr-resized.jpg")
image.convert("RGB").save(tmp, quality=90)
work_path = tmp
try:
ocr = PaddleOCR(use_angle_cls=True, lang=self.language, use_gpu=False)
result = ocr.ocr(str(work_path), cls=True)
texts: list[str] = []
scores: list[float] = []
for page in result or []:
for line in page or []:
if len(line) >= 2 and isinstance(line[1], (list, tuple)):
text = str(line[1][0]).strip()
if text:
texts.append(text)
try:
scores.append(float(line[1][1]))
except (TypeError, ValueError, IndexError):
pass
confidence = sum(scores) / len(scores) if scores else None
return "\n".join(texts), confidence, self.language
finally:
if work_path != path:
work_path.unlink(missing_ok=True)
def _env_bool(name: str, default: bool) -> bool:
value = os.getenv(name)
if value is None:
return default
return value.strip().lower() in {"1", "true", "yes", "on"}
+50
View File
@@ -0,0 +1,50 @@
"""
Bridge URL-level plugin parsing with the rest of the app (tasks/GUI logging).
Full TaskQueue integration can map Resource → DownloadTaskSpec per site.
"""
from __future__ import annotations
from dataclasses import dataclass
import logging
from pathlib import Path
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
from pixiv_app.core.plugin.manager import PluginManager
logger = logging.getLogger(__name__)
@dataclass
class OrchestrationResult:
plugin_name: str
resources: list[Resource]
class PluginTaskOrchestrator:
"""Resolve plugins and run parse/download without replacing the legacy Pixiv GUI flow."""
def __init__(self, plugin_manager: PluginManager) -> None:
self.plugin_manager = plugin_manager
def get_plugin_for_url(self, url: str) -> BasePlugin | None:
return self.plugin_manager.get_plugin_for_url(url)
def parse_url(self, url: str) -> OrchestrationResult:
plugin = self.get_plugin_for_url(url)
if plugin is None:
raise PluginParseError(f"No plugin registered for: {url}")
resources = plugin.parse(url)
return OrchestrationResult(plugin_name=plugin.name, resources=resources)
def download_resource(
self,
plugin_name: str,
resource: Resource,
save_path: Path,
) -> list[Path]:
plugin = self.plugin_manager.plugins.get(plugin_name)
if plugin is None:
raise PluginParseError(f"Plugin not loaded: {plugin_name}")
return plugin.download(resource, save_path)
+423
View File
@@ -0,0 +1,423 @@
from __future__ import annotations
import asyncio
import re
from dataclasses import dataclass, field
from typing import Any, Callable
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.downloader import create_session, request_json
from pixiv_app.core.gallery_service import LocalGalleryService
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
from pixiv_app.core.paths import app_session_file
from pixiv_app.core.x_media_service import XMediaService, _normalize_username
ProgressCallback = Callable[[dict[str, Any]], None]
@dataclass
class OriginFollow:
platform: str
external_id: str
name: str = ""
handle: str = ""
avatar_url: str = ""
profile_url: str = ""
@dataclass
class OriginFollowSyncResult:
platform: str
created: int = 0
updated: int = 0
skipped: int = 0
failed: int = 0
cloud_created: int = 0
cloud_updated: int = 0
cloud_skipped: int = 0
pages: int = 0
errors: list[dict[str, str]] = field(default_factory=list)
progress: list[dict[str, Any]] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"platform": self.platform,
"created": self.created,
"updated": self.updated,
"skipped": self.skipped,
"failed": self.failed,
"cloud_created": self.cloud_created,
"cloud_updated": self.cloud_updated,
"cloud_skipped": self.cloud_skipped,
"pages": self.pages,
"errors": self.errors,
"progress": self.progress,
}
class OriginFollowSyncService:
def __init__(
self,
*,
db_path: str = DEFAULT_LIBRARY_DB,
gallery_service: LocalGalleryService | None = None,
x_media_service: XMediaService | None = None,
) -> None:
self.db_path = db_path
self.gallery_service = gallery_service or LocalGalleryService(db_path=db_path)
self.x_media_service = x_media_service or XMediaService()
def sync(self, platform: str, *, progress: ProgressCallback | None = None) -> dict[str, Any]:
normalized = platform.strip().lower()
if normalized == "pixiv":
return self.sync_pixiv(progress=progress).to_dict()
if normalized == "x":
return self.sync_x(progress=progress).to_dict()
raise ValueError("请选择 Pixiv 或 X。")
def sync_pixiv(self, *, progress: ProgressCallback | None = None) -> OriginFollowSyncResult:
cookie = str(SessionStore(app_session_file()).load().get("cookie", "")).strip()
if not cookie:
raise ValueError("Pixiv Cookie 不存在,请先导入或保存 Pixiv 登录态。")
result = OriginFollowSyncResult(platform="pixiv")
session = create_session(cookie)
try:
user_id = self._fetch_pixiv_current_user_id(session, cookie)
follows = self._fetch_pixiv_following(session, cookie, user_id, result, progress=progress)
finally:
session.close()
self._upsert_pixiv_local(follows, result)
return result
def sync_x(self, *, progress: ProgressCallback | None = None) -> OriginFollowSyncResult:
plugin = self.x_media_service._get_plugin()
if not plugin.validate():
raise ValueError("X Playwright 不可用,请先安装 Playwright 并完成 X 登录。")
result = OriginFollowSyncResult(platform="x")
with plugin._playwright_lock:
follows = asyncio.run(self._fetch_x_following(plugin, result, progress=progress))
self._upsert_x_local(follows, result)
return result
def _emit(
self,
result: OriginFollowSyncResult,
progress: ProgressCallback | None,
payload: dict[str, Any],
) -> None:
result.progress.append(payload)
if progress is not None:
progress(payload)
def _fetch_pixiv_current_user_id(self, session: Any, cookie: str) -> str:
for url in (
"https://www.pixiv.net/ajax/user/self?lang=zh",
"https://www.pixiv.net/ajax/user/self",
):
try:
payload = request_json(session, url, cookie=cookie, context="Pixiv 当前账号")
body = payload.get("body") if isinstance(payload, dict) else {}
user_id = _first_text(body, "userId", "id")
if user_id:
return user_id
except Exception:
continue
try:
response = session.get("https://www.pixiv.net/", timeout=30)
response.raise_for_status()
match = re.search(r'"userData"\s*:\s*\{[^}]*"id"\s*:\s*"?(?P<id>\d+)"?', response.text)
if match:
return match.group("id")
except Exception:
pass
raise ValueError("Pixiv Cookie 可能已失效:无法识别当前登录账号。")
def _fetch_pixiv_following(
self,
session: Any,
cookie: str,
user_id: str,
result: OriginFollowSyncResult,
*,
progress: ProgressCallback | None = None,
) -> list[OriginFollow]:
follows: list[OriginFollow] = []
seen: set[str] = set()
limit = 100
offset = 0
page = 1
while True:
self._emit(result, progress, {"platform": "pixiv", "page": page, "message": f"正在同步第 {page} 页"})
url = (
f"https://www.pixiv.net/ajax/user/{user_id}/following"
f"?offset={offset}&limit={limit}&rest=show&tag=&acceptingRequests=0&lang=zh"
)
try:
payload = request_json(
session,
url,
cookie=cookie,
referer=f"https://www.pixiv.net/users/{user_id}/following",
context=f"Pixiv 关注列表第 {page} 页",
)
except Exception as exc:
if page == 1:
raise ValueError("Pixiv Cookie 可能已失效:无法读取关注列表。") from exc
result.failed += 1
result.errors.append({"id": f"page:{page}", "reason": str(exc)})
break
users = _extract_pixiv_users(payload)
result.pages = page
if not users:
break
for item in users:
follow = _pixiv_follow_from_payload(item)
if follow is None:
result.failed += 1
result.errors.append({"id": "unknown", "reason": "Pixiv 作者记录缺少 userId"})
continue
if follow.external_id in seen:
result.skipped += 1
continue
seen.add(follow.external_id)
follows.append(follow)
if len(users) < limit:
break
offset += limit
page += 1
return follows
async def _fetch_x_following(
self,
plugin: Any,
result: OriginFollowSyncResult,
*,
progress: ProgressCallback | None = None,
) -> list[OriginFollow]:
from plugins.x import selectors
from plugins.x.playwright_client import XPlaywrightClient
client = XPlaywrightClient(plugin.config)
follows: list[OriginFollow] = []
seen: set[str] = set()
try:
await client.start()
page = await client.new_page_goto("https://x.com/home")
try:
await client.raise_if_guest_wall(page)
profile_href = await page.locator(selectors.PROFILE_LINK).first.get_attribute("href", timeout=8000)
username = _x_username_from_href(profile_href or "")
if not username:
raise ValueError("X 未登录:请先在插件管理里完成 X 登录。")
await page.goto(f"https://x.com/{username}{selectors.FOLLOWING_URL_SUFFIX}", wait_until="domcontentloaded")
await client.raise_if_guest_wall(page)
stale_rounds = 0
page_no = 1
while stale_rounds < 4 and page_no <= 80:
self._emit(result, progress, {"platform": "x", "page": page_no, "message": f"正在同步第 {page_no} 页"})
added = await _collect_x_user_cells(page, seen, follows)
result.pages = page_no
stale_rounds = stale_rounds + 1 if added == 0 else 0
await client.human_scroll_page(page)
page_no += 1
if not follows:
raise ValueError("X 未读取到 following 列表,请确认账号已登录且 following 页面可访问。")
finally:
await page.close()
finally:
await client.close()
return follows
def _upsert_pixiv_local(self, follows: list[OriginFollow], result: OriginFollowSyncResult) -> None:
library = PixivLibrary(self.db_path)
try:
for follow in follows:
try:
status = library.upsert_origin_followed_artist(
int(follow.external_id),
name=follow.name,
account=follow.handle,
avatar_url=follow.avatar_url,
)
_apply_local_status(result, status)
except Exception as exc:
result.failed += 1
result.errors.append({"id": follow.external_id, "reason": str(exc)})
finally:
library.close()
def _upsert_x_local(self, follows: list[OriginFollow], result: OriginFollowSyncResult) -> None:
rows = self.x_media_service._load_follow_rows()
now = _now_text()
by_handle = {str(item.get("username", "")).lower(): item for item in rows}
for follow in follows:
handle = _normalize_username(follow.handle or follow.external_id)
if not handle:
result.failed += 1
result.errors.append({"id": follow.external_id, "reason": "X 用户名为空"})
continue
row = by_handle.get(handle.lower())
status = "created"
if row is None:
row = {
"username": handle,
"creator_name": follow.name or handle,
"enabled": True,
"profile_url": follow.profile_url or f"https://x.com/{handle}",
"avatar_url": follow.avatar_url,
"latest_tweet_id": "",
"latest_published_at": "",
"last_checked_at": now,
"last_error": "",
"seen_tweet_ids": [],
"created_at": now,
"updated_at": now,
}
rows.append(row)
by_handle[handle.lower()] = row
else:
changed = (
not bool(row.get("enabled", True))
or (follow.name and follow.name != str(row.get("creator_name") or ""))
or (follow.avatar_url and follow.avatar_url != str(row.get("avatar_url") or ""))
)
status = "updated" if changed else "skipped"
row["username"] = handle
row["creator_name"] = follow.name or str(row.get("creator_name") or handle)
row["enabled"] = True
row["profile_url"] = follow.profile_url or f"https://x.com/{handle}"
row["avatar_url"] = follow.avatar_url or str(row.get("avatar_url") or "")
row["last_checked_at"] = now
row["last_error"] = ""
row["updated_at"] = now
_apply_local_status(result, status)
self.x_media_service._save_follow_rows(rows)
def _apply_local_status(result: OriginFollowSyncResult, status: str) -> None:
if status == "created":
result.created += 1
elif status == "updated":
result.updated += 1
else:
result.skipped += 1
def _extract_pixiv_users(payload: dict[str, Any]) -> list[dict[str, Any]]:
body = payload.get("body") if isinstance(payload, dict) else payload
if isinstance(body, list):
return [item for item in body if isinstance(item, dict)]
if not isinstance(body, dict):
return []
candidates = (
body.get("users"),
body.get("followUsers"),
body.get("userPreviews"),
body.get("thumbnails", {}).get("user") if isinstance(body.get("thumbnails"), dict) else None,
)
for candidate in candidates:
if isinstance(candidate, list):
return [item for item in candidate if isinstance(item, dict)]
return []
def _pixiv_follow_from_payload(item: dict[str, Any]) -> OriginFollow | None:
user_id = _first_text(item, "userId", "user_id", "id")
if not user_id:
user = item.get("user") if isinstance(item.get("user"), dict) else {}
user_id = _first_text(user, "userId", "user_id", "id")
item = {**item, **user}
if not user_id:
return None
name = _first_text(item, "name", "userName", "displayName")
handle = _first_text(item, "userAccount", "account", "screenName")
avatar = _first_text(item, "imageBig", "image", "profileImageUrl", "profile_image_url")
return OriginFollow(
platform="pixiv",
external_id=str(user_id),
name=name,
handle=handle,
avatar_url=avatar,
profile_url=f"https://www.pixiv.net/users/{user_id}",
)
async def _collect_x_user_cells(page: Any, seen: set[str], follows: list[OriginFollow]) -> int:
from plugins.x import selectors
added = 0
cells = await page.locator(selectors.USER_CELL).all()
for cell in cells:
try:
data = await cell.evaluate(
"""(node) => {
const text = node.innerText || "";
const links = Array.from(node.querySelectorAll('a[href^="/"]')).map(a => a.getAttribute("href") || "");
const img = node.querySelector('img[src*="profile_images"]');
return { text, links, avatar: img ? img.getAttribute("src") : "" };
}"""
)
handle = _x_handle_from_cell(data)
if not handle or handle.lower() in seen:
continue
seen.add(handle.lower())
follows.append(
OriginFollow(
platform="x",
external_id=handle,
name=_x_display_name_from_cell(data, handle),
handle=handle,
avatar_url=str(data.get("avatar") or ""),
profile_url=f"https://x.com/{handle}",
)
)
added += 1
except Exception:
continue
return added
def _x_handle_from_cell(data: dict[str, Any]) -> str:
for href in data.get("links") or []:
username = _x_username_from_href(str(href))
if username:
return username
text = str(data.get("text") or "")
match = re.search(r"@([A-Za-z0-9_]{1,15})", text)
return match.group(1) if match else ""
def _x_display_name_from_cell(data: dict[str, Any], handle: str) -> str:
text = str(data.get("text") or "")
for line in [item.strip() for item in text.splitlines() if item.strip()]:
if not line.startswith("@") and line.lower() not in {"follow", "following"}:
return line
return handle
def _x_username_from_href(href: str) -> str:
match = re.match(r"^/?([A-Za-z0-9_]{1,15})(?:/)?$", href.strip())
if not match:
return ""
username = match.group(1)
if username.lower() in {"home", "explore", "notifications", "messages", "settings", "i"}:
return ""
return username
def _first_text(payload: Any, *keys: str) -> str:
if not isinstance(payload, dict):
return ""
for key in keys:
value = payload.get(key)
if value is not None and str(value).strip():
return str(value).strip()
return ""
def _now_text() -> str:
from datetime import datetime, timezone
return datetime.now(timezone.utc).isoformat()
+113
View File
@@ -0,0 +1,113 @@
from __future__ import annotations
import os
import shutil
import sys
from pathlib import Path
APP_NAME = "SakuraDownloader"
LEGACY_APP_NAME = "PixivDownloader"
def is_frozen() -> bool:
return bool(getattr(sys, "frozen", False))
def resource_root() -> Path:
if is_frozen():
return Path(getattr(sys, "_MEIPASS")).resolve()
return Path(__file__).resolve().parents[2]
def executable_dir() -> Path:
if is_frozen():
return Path(sys.executable).resolve().parent
return resource_root()
def data_root() -> Path:
override = os.environ.get("SAKURA_APP_DATA_DIR") or os.environ.get("PIXIV_APP_DATA_DIR")
if override:
root = Path(override)
elif is_frozen():
local_appdata = os.environ.get("LOCALAPPDATA")
root = Path(local_appdata) / APP_NAME if local_appdata else executable_dir() / "data"
legacy_root = Path(local_appdata) / LEGACY_APP_NAME if local_appdata else None
if legacy_root and legacy_root.exists() and not root.exists():
root.mkdir(parents=True, exist_ok=True)
for name in ("pixiv_app_session.json", "runtime", "profiles", "plugins"):
src = legacy_root / name
dst = root / name
if src.exists() and not dst.exists():
if src.is_dir():
import shutil
shutil.copytree(src, dst)
else:
dst.write_bytes(src.read_bytes())
else:
root = resource_root()
root.mkdir(parents=True, exist_ok=True)
return root
def runtime_path(*parts: str) -> Path:
path = data_root() / "runtime"
path.mkdir(parents=True, exist_ok=True)
return path.joinpath(*parts)
def profiles_root() -> Path:
path = data_root() / "profiles"
path.mkdir(parents=True, exist_ok=True)
return path
def plugins_root() -> Path:
return resource_root() / "plugins"
def external_plugins_root() -> Path:
"""Plugins shipped beside the executable, outside the PyInstaller bundle."""
return executable_dir() / "plugins"
def user_plugins_root() -> Path:
path = data_root() / "plugins"
path.mkdir(parents=True, exist_ok=True)
return path
def plugin_roots() -> list[Path]:
roots = [plugins_root(), external_plugins_root(), user_plugins_root()]
result: list[Path] = []
for root in roots:
if root not in result:
result.append(root)
return result
def webui_root() -> Path:
return resource_root() / "pixiv_app" / "webui"
def app_session_file() -> Path:
return data_root() / "pixiv_app_session.json"
def app_library_file() -> Path:
"""Persistent local library DB, writable even when the EXE directory is read-only."""
target = data_root() / "pixiv_app_library.db"
if is_frozen() and not target.exists():
legacy = executable_dir() / "pixiv_app_library.db"
if legacy.is_file():
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(legacy, target)
return target
def downloads_root() -> Path:
path = executable_dir() / "Sakura_Downloads"
path.mkdir(parents=True, exist_ok=True)
return path
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
import os
from pathlib import Path
def playwright_browsers_root() -> Path | None:
local_appdata = os.environ.get("LOCALAPPDATA", "").strip()
if not local_appdata:
return None
return Path(local_appdata) / "ms-playwright"
def configure_playwright_runtime_env() -> Path | None:
root = playwright_browsers_root()
if root is None:
return None
if root.exists() and "PLAYWRIGHT_BROWSERS_PATH" not in os.environ:
os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(root)
return root
def preferred_chromium_executable() -> Path | None:
root = configure_playwright_runtime_env()
if root is None or not root.exists():
return None
candidates: list[tuple[int, float, Path]] = []
for browser_dir in root.glob("chromium-*"):
if not browser_dir.is_dir():
continue
revision = _parse_revision(browser_dir.name)
for relative in (
Path("chrome-win64") / "chrome.exe",
Path("chrome-win") / "chrome.exe",
):
executable = browser_dir / relative
if executable.is_file():
try:
mtime = executable.stat().st_mtime
except OSError:
mtime = 0.0
candidates.append((revision, mtime, executable))
break
if not candidates:
return None
candidates.sort(key=lambda item: (item[0], item[1]), reverse=True)
return candidates[0][2]
def explain_missing_browser(exc: Exception, *, feature_name: str) -> str:
text = str(exc)
lowered = text.lower()
if "executable doesn't exist" not in lowered and "browsertype.launch_persistent_context" not in lowered:
return text
return (
f"{feature_name} 需要 Playwright Chromium 浏览器,但当前系统里没有可用的浏览器文件。\n"
"请先运行同目录下的 install_playwright_chromium.bat,安装完成后重新打开程序再试。"
)
def _parse_revision(name: str) -> int:
try:
return int(name.rsplit("-", 1)[-1])
except (TypeError, ValueError):
return 0
+24
View File
@@ -0,0 +1,24 @@
"""Plugin system: base types, manager, generator, optional market client."""
from pixiv_app.core.plugin.base import (
BasePlugin,
PluginAuthError,
PluginDownloadError,
PluginError,
PluginManifest,
PluginParseError,
Resource,
)
from pixiv_app.core.plugin.manager import PluginEvent, PluginManager
__all__ = [
"BasePlugin",
"PluginAuthError",
"PluginDownloadError",
"PluginError",
"PluginEvent",
"PluginManager",
"PluginManifest",
"PluginParseError",
"Resource",
]
+119
View File
@@ -0,0 +1,119 @@
"""
Multi-site plugin core abstractions.
解析与下载采用同步接口,便于与现有 requests / 线程模型集成;
若站点 SDK 仅提供 asyncio,可在插件内部使用 asyncio.run 包装。
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
@dataclass
class Resource:
"""Unified resource model returned by plugins."""
id: str
url: str
title: str
author: str
author_id: str
files: list[dict[str, Any]]
metadata: dict[str, Any] = field(default_factory=dict)
thumbnail: str | None = None
created_at: str | None = None
@dataclass
class PluginManifest:
"""Optional manifest shipped as manifest.json next to plugin.py."""
name: str
version: str
domain: str
author: str = ""
description: str = ""
requires_auth: bool = False
min_platform_version: str = "1.0.0"
tags: list[str] = field(default_factory=list)
dependencies: list[str] = field(default_factory=list)
class PluginError(Exception):
"""Base class for plugin failures."""
class PluginAuthError(PluginError):
"""Authentication or authorization failed."""
class PluginParseError(PluginError):
"""URL or API response could not be parsed."""
class PluginDownloadError(PluginError):
"""Download or filesystem error."""
class BasePlugin(ABC):
"""Interface implemented by every site plugin."""
@property
@abstractmethod
def domain(self) -> str:
"""Primary domain, e.g. pixiv.net."""
@property
@abstractmethod
def name(self) -> str:
"""Human-readable plugin name."""
@property
@abstractmethod
def version(self) -> str:
"""Semantic version string."""
@abstractmethod
def can_handle(self, url: str) -> bool:
"""Return True if this plugin should process the URL or raw token."""
@abstractmethod
def parse(self, url: str) -> list[Resource]:
"""Resolve URL into one or more downloadable resources."""
@abstractmethod
def download(self, resource: Resource, save_path: Path) -> list[Path]:
"""Download files for ``resource`` under ``save_path``."""
@abstractmethod
def get_headers(self) -> dict[str, str]:
"""HTTP headers for requests (Cookie, Bearer token, etc.)."""
def validate(self) -> bool:
"""Optional self-check (credentials present, deps installed)."""
return True
def get_config_schema(self) -> dict[str, Any]:
"""Optional JSON-schema-like dict for dynamic GUI forms."""
return {}
def load_config(self) -> dict[str, Any]:
"""Override to load persisted plugin settings."""
return {}
def save_config(self, data: dict[str, Any]) -> None:
"""Override to persist plugin settings."""
__all__ = [
"BasePlugin",
"PluginAuthError",
"PluginDownloadError",
"PluginError",
"PluginManifest",
"PluginParseError",
"Resource",
]
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
import re
from datetime import datetime
from pathlib import Path
from typing import Any
from jinja2 import Environment, FileSystemLoader, select_autoescape
def _class_name_from_plugin_name(name: str) -> str:
parts = re.split(r"[\s_]+", name.strip())
stem = "".join(p[:1].upper() + p[1:].lower() if p else "" for p in parts if p)
return f"{stem}Plugin" if stem else "GeneratedPlugin"
class PluginGenerator:
"""Generate a skeleton plugin directory from Jinja2 templates."""
def __init__(self, template_dir: str | Path | None = None) -> None:
base = Path(__file__).resolve().parent / "templates"
self.template_dir = Path(template_dir) if template_dir else base
self.env = Environment(
loader=FileSystemLoader(str(self.template_dir)),
autoescape=select_autoescape(enabled_extensions=()),
)
def generate(
self,
name: str,
domain: str,
author: str,
description: str,
*,
output_root: str | Path = "plugins",
) -> Path:
slug = name.strip().lower().replace(" ", "_")
plugin_dir = Path(output_root) / slug
plugin_dir.mkdir(parents=True, exist_ok=True)
context: dict[str, Any] = {
"name": name.strip(),
"domain": domain.strip(),
"author": author.strip(),
"description": description.strip(),
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"class_name": _class_name_from_plugin_name(name),
}
py_template = self.env.get_template("base_plugin.py.jinja")
(plugin_dir / "plugin.py").write_text(py_template.render(**context), encoding="utf-8")
manifest_template = self.env.get_template("manifest.json.jinja")
(plugin_dir / "manifest.json").write_text(manifest_template.render(**context), encoding="utf-8")
init_file = plugin_dir / "__init__.py"
if not init_file.exists():
init_file.write_text('"""Generated plugin."""\n', encoding="utf-8")
return plugin_dir.resolve()
def generate_from_gui(self, form_data: dict[str, str], *, output_root: str | Path = "plugins") -> Path:
return self.generate(
name=form_data["name"],
domain=form_data["domain"],
author=form_data["author"],
description=form_data["description"],
output_root=output_root,
)
+194
View File
@@ -0,0 +1,194 @@
from __future__ import annotations
import importlib.util
import json
import logging
import sys
from collections import defaultdict
from enum import Enum
from pathlib import Path
from typing import Any, Callable
from pixiv_app.core.plugin.base import BasePlugin, PluginManifest
logger = logging.getLogger(__name__)
class PluginEvent(str, Enum):
BEFORE_LOAD = "before_load"
AFTER_LOAD = "after_load"
BEFORE_UNLOAD = "before_unload"
AFTER_UNLOAD = "after_unload"
ERROR = "error"
class PluginManager:
"""Discover and load plugins from filesystem directories."""
def __init__(self, plugin_dirs: list[str | Path]) -> None:
self.plugin_dirs = [Path(d).resolve() for d in plugin_dirs]
self.plugins: dict[str, BasePlugin] = {}
self.manifests: dict[str, PluginManifest] = {}
self.loaded_modules: dict[str, Any] = {}
self._paths: dict[str, Path] = {}
self.load_errors: dict[Path, str] = {}
self.event_handlers: dict[PluginEvent, list[Callable[..., None]]] = defaultdict(list)
def on_event(self, event: PluginEvent, callback: Callable[..., None]) -> None:
self.event_handlers[event].append(callback)
def _emit(self, event: PluginEvent, *args: Any, **kwargs: Any) -> None:
for cb in self.event_handlers.get(event, []):
try:
cb(*args, **kwargs)
except Exception as exc:
logger.exception("Plugin event handler failed: %s", exc)
def discover(self) -> list[Path]:
"""Return paths to plugin.py files."""
found: list[Path] = []
for root in self.plugin_dirs:
if not root.is_dir():
continue
for subdir in sorted(root.iterdir()):
if not subdir.is_dir():
continue
candidate = subdir / "plugin.py"
if candidate.is_file():
found.append(candidate)
return found
def _load_manifest(self, plugin_dir: Path) -> PluginManifest | None:
manifest_path = plugin_dir / "manifest.json"
if not manifest_path.is_file():
return None
try:
data = json.loads(manifest_path.read_text(encoding="utf-8"))
return PluginManifest(
name=str(data.get("name", plugin_dir.name)),
version=str(data.get("version", "0.0.0")),
domain=str(data.get("domain", "")),
author=str(data.get("author", "")),
description=str(data.get("description", "")),
requires_auth=bool(data.get("requires_auth", False)),
min_platform_version=str(data.get("min_platform_version", "1.0.0")),
tags=list(data.get("tags") or []),
dependencies=list(data.get("dependencies") or []),
)
except Exception as exc:
logger.warning("Invalid manifest %s: %s", manifest_path, exc)
return None
def load_plugin(self, plugin_path: Path) -> BasePlugin | None:
"""Load a single plugin from ``.../plugin.py``."""
plugin_path = plugin_path.resolve()
plugin_dir = plugin_path.parent
module_name = f"sakura_dynamic_plugin_{plugin_dir.name}"
module_stem = module_name.rsplit(".", 1)[0]
for loaded_name in list(sys.modules):
if loaded_name == module_name or loaded_name.startswith(f"{module_name}."):
del sys.modules[loaded_name]
self.load_errors.pop(plugin_path, None)
self._emit(PluginEvent.BEFORE_LOAD, plugin_path)
spec = importlib.util.spec_from_file_location(
module_name,
plugin_path,
submodule_search_locations=[str(plugin_dir)],
)
if spec is None or spec.loader is None:
self._emit(PluginEvent.ERROR, plugin_path, "invalid spec")
self.load_errors[plugin_path] = "invalid spec"
return None
module = importlib.util.module_from_spec(spec)
module.__package__ = module_stem
module.__file__ = str(plugin_path)
sys.modules[module_name] = module
try:
spec.loader.exec_module(module)
except Exception as exc:
logger.exception("Failed to load %s", plugin_path)
self._emit(PluginEvent.ERROR, plugin_path, exc)
self.load_errors[plugin_path] = f"{type(exc).__name__}: {exc}"
return None
plugin_class = getattr(module, "plugin_class", None)
if plugin_class is None:
logger.error("Module %s has no plugin_class export", plugin_path)
self._emit(PluginEvent.ERROR, plugin_path, "missing plugin_class")
self.load_errors[plugin_path] = "missing plugin_class"
return None
try:
plugin = plugin_class()
except Exception as exc:
logger.exception("Failed to instantiate plugin from %s", plugin_path)
self._emit(PluginEvent.ERROR, plugin_path, exc)
self.load_errors[plugin_path] = f"{type(exc).__name__}: {exc}"
return None
if not isinstance(plugin, BasePlugin):
logger.error("plugin_class is not BasePlugin: %s", plugin_path)
self.load_errors[plugin_path] = "plugin_class is not BasePlugin"
return None
key = plugin.name
manifest = self._load_manifest(plugin_dir)
if manifest:
self.manifests[key] = manifest
self.plugins[key] = plugin
self.loaded_modules[key] = module
self._paths[key] = plugin_path
self.load_errors.pop(plugin_path, None)
self._emit(PluginEvent.AFTER_LOAD, plugin)
return plugin
def load_all(self) -> dict[str, BasePlugin]:
for path in self.discover():
try:
self.load_plugin(path)
except Exception as exc:
logger.exception("Failed during load_all %s: %s", path, exc)
return self.plugins
def reload_all_from_disk(self) -> None:
"""Unload every plugin entry then scan directories again."""
for name in list(self.plugins.keys()):
self.unload_plugin(name)
self.load_errors.clear()
self.load_all()
def reload_plugin(self, name: str) -> bool:
path = self._paths.get(name)
if path is None:
return False
self.unload_plugin(name)
return self.load_plugin(path) is not None
def get_plugin_path(self, name: str) -> Path | None:
return self._paths.get(name)
def unload_plugin(self, name: str) -> None:
self._emit(PluginEvent.BEFORE_UNLOAD, name)
self.plugins.pop(name, None)
mod = self.loaded_modules.pop(name, None)
self.manifests.pop(name, None)
self._paths.pop(name, None)
if mod is not None and getattr(mod, "__name__", None) in sys.modules:
del sys.modules[mod.__name__]
self._emit(PluginEvent.AFTER_UNLOAD, name)
def get_plugin_for_url(self, url: str) -> BasePlugin | None:
for plugin in self.plugins.values():
if plugin.can_handle(url):
return plugin
return None
def check_dependencies(self, manifest: PluginManifest) -> bool:
for dep in manifest.dependencies:
if dep not in self.plugins:
return False
return True
+29
View File
@@ -0,0 +1,29 @@
"""
Optional plugin market client — stub for future HTTP catalog integration.
"""
from __future__ import annotations
from typing import Any
import requests
class PluginMarketClient:
def __init__(self, api_url: str = "") -> None:
self.api_url = api_url.rstrip("/")
def list_plugins(self) -> list[dict[str, Any]]:
if not self.api_url:
return []
response = requests.get(f"{self.api_url}/plugins", timeout=15)
response.raise_for_status()
data = response.json()
return list(data.get("plugins", []))
def install_plugin(self, _plugin_name: str) -> bool:
"""Download zip and extract into plugins/ — not implemented."""
return False
def check_updates(self) -> list[dict[str, Any]]:
return []
@@ -0,0 +1,31 @@
"""
{{ name }} plugin — auto-generated at {{ generated_at }}.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from pixiv_app.core.plugin.base import BasePlugin, Resource
class {{ class_name }}(BasePlugin):
domain = "{{ domain }}"
name = "{{ name }}"
version = "1.0.0"
def can_handle(self, url: str) -> bool:
return "{{ domain }}" in url.lower()
def parse(self, url: str) -> list[Resource]:
raise NotImplementedError("Implement URL parsing and build Resource list.")
def download(self, resource: Resource, save_path: Path) -> list[Path]:
raise NotImplementedError("Download files for resource into save_path.")
def get_headers(self) -> dict[str, str]:
return {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
plugin_class = {{ class_name }}
@@ -0,0 +1,11 @@
{
"name": "{{ name }}",
"version": "1.0.0",
"domain": "{{ domain }}",
"author": "{{ author }}",
"description": "{{ description }}",
"requires_auth": false,
"tags": ["general"],
"min_platform_version": "1.0.0",
"dependencies": []
}
+292
View File
@@ -0,0 +1,292 @@
from __future__ import annotations
import random
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from typing import Any, Optional
import requests
@dataclass
class ProxyInfo:
ip: str
port: int
protocol: str = "http"
source: str = "manual"
location: str = ""
org: str = ""
last_check: float = 0.0
success_count: int = 0
fail_count: int = 0
@property
def proxy_url(self) -> str:
return f"{self.protocol}://{self.ip}:{self.port}"
@property
def is_alive(self) -> bool:
return self.success_count > 0 or self.fail_count < 3
def parse_proxy_text(text: str) -> list[ProxyInfo]:
proxies: list[ProxyInfo] = []
seen: set[tuple[str, int, str]] = set()
for raw_line in text.splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
if "://" not in line:
line = f"http://{line}"
match = re.match(r"^(https?)://([\w\.\-]+):(\d{2,5})$", line, re.IGNORECASE)
if not match:
continue
protocol, ip, port_text = match.groups()
key = (ip, int(port_text), protocol.lower())
if key in seen:
continue
seen.add(key)
proxies.append(ProxyInfo(ip=ip, port=int(port_text), protocol=protocol.lower(), source="manual"))
return proxies
class QuakeClient:
def __init__(
self,
api_key: str = "",
cookie: str = "",
mode: str = "api_v3",
request_kwargs: Optional[dict[str, Any]] = None,
):
self.api_key = api_key.strip()
self.cookie = cookie.strip()
self.mode = mode
self.request_kwargs = dict(request_kwargs or {})
self.base_url = "https://quake.360.net/api/v3"
self.web_assoc_url = "https://quake.360.net/api/search/field/association/quake_service"
def _api_headers(self) -> dict[str, str]:
return {
"X-QuakeToken": self.api_key,
"Content-Type": "application/json",
}
def _web_headers(self) -> dict[str, str]:
headers = {
"Accept": "application/json, text/plain, */*",
"Content-Type": "application/json",
"Origin": "https://quake.360.net",
"Referer": "https://quake.360.net/quake/",
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
),
}
if self.cookie:
headers["Cookie"] = self.cookie
return headers
def search_ips_api(self, query: str, size: int = 100, start: int = 0) -> list[dict[str, Any]]:
if not self.api_key:
return []
url = f"{self.base_url}/search/quake_service"
payload = {"query": query, "start": start, "size": min(size, 500)}
kwargs = dict(self.request_kwargs)
kwargs.setdefault("timeout", 30)
try:
response = requests.post(url, headers=self._api_headers(), json=payload, **kwargs)
if response.status_code != 200:
return []
body = response.json()
return body.get("data", []) if isinstance(body, dict) else []
except Exception:
return []
def search_ips_web_assoc(self, search_content: str) -> list[dict[str, Any]]:
if not self.cookie:
return []
payload = {
"search_content": search_content,
"device": {
"device_type": "PC",
"os": "Windows",
"os_version": "10.0",
"language": "zh_CN",
"network": "4g",
"browser_info": "Chrome",
"fingerprint": "pixiv-proxy-pool",
"user_agent": self._web_headers()["User-Agent"],
"date": time.strftime("%Y/%m/%d %H:%M:%S"),
"UUID": "pixiv-proxy-association",
},
}
kwargs = dict(self.request_kwargs)
kwargs.setdefault("timeout", 30)
try:
response = requests.post(
self.web_assoc_url,
headers=self._web_headers(),
json=payload,
**kwargs,
)
if response.status_code != 200:
return []
text = response.text
ips = sorted(set(re.findall(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", text)))
return [{"ip": ip, "port": 80, "organization": "", "location": {}} for ip in ips]
except Exception:
return []
def get_foreign_proxies(
self,
size: int = 120,
countries: Optional[list[str]] = None,
) -> list[ProxyInfo]:
proxy_ports = [80, 443, 8080, 3128, 8888, 1080, 8118]
port_query = " OR ".join([f"port:{port}" for port in proxy_ports])
country_codes = countries or ["US", "JP", "KR", "SG", "HK", "TW", "DE", "FR", "GB", "CA", "AU"]
country_query = " OR ".join([f'country:"{code}"' for code in country_codes])
query = f"({port_query}) AND ({country_query}) AND response:ok"
if self.mode == "web_assoc":
data = self.search_ips_web_assoc(query)
else:
data = self.search_ips_api(query=query, size=size)
proxies: list[ProxyInfo] = []
seen: set[tuple[str, int, str]] = set()
for item in data:
if not isinstance(item, dict):
continue
ip = item.get("ip")
port = item.get("port")
if not ip or not isinstance(port, int):
continue
protocol = "https" if port in (443, 8443) else "http"
key = (ip, port, protocol)
if key in seen:
continue
seen.add(key)
location_obj = item.get("location", {}) if isinstance(item.get("location"), dict) else {}
location = "/".join(
[str(location_obj.get("country", "")).strip(), str(location_obj.get("province", "")).strip()]
).strip("/")
proxies.append(
ProxyInfo(
ip=ip,
port=port,
protocol=protocol,
source=f"quake:{self.mode}",
location=location,
org=str(item.get("organization", "") or item.get("org", "")),
)
)
return proxies
class ProxyPool:
def __init__(self, test_url: str = "https://httpbin.org/ip", test_timeout: int = 6):
self.proxies: list[ProxyInfo] = []
self.working_proxies: list[ProxyInfo] = []
self.current_index = 0
self.test_url = test_url
self.test_timeout = test_timeout
self.session = requests.Session()
self.session.headers.update({"User-Agent": "SakuraDownloaderProxyPool/1.0"})
self._lock = threading.Lock()
def add_proxies(self, proxies: list[ProxyInfo]) -> None:
with self._lock:
existing = {(p.ip, p.port, p.protocol) for p in self.proxies}
for proxy in proxies:
key = (proxy.ip, proxy.port, proxy.protocol)
if key not in existing:
existing.add(key)
self.proxies.append(proxy)
def verify_proxy(self, proxy: ProxyInfo) -> bool:
target_proxies = {"http": proxy.proxy_url, "https": proxy.proxy_url}
try:
response = self.session.get(
self.test_url,
proxies=target_proxies,
timeout=self.test_timeout,
verify=False,
)
if response.status_code == 200:
with self._lock:
proxy.last_check = time.time()
proxy.success_count += 1
proxy.fail_count = 0
return True
except Exception:
pass
with self._lock:
proxy.fail_count += 1
return False
def verify_all(self, max_workers: int = 16, max_proxies: int = 80) -> list[ProxyInfo]:
with self._lock:
to_verify = list(self.proxies[:max_proxies])
self.working_proxies = []
if not to_verify:
return self.working_proxies
with ThreadPoolExecutor(max_workers=max_workers) as executor:
future_map = {executor.submit(self.verify_proxy, proxy): proxy for proxy in to_verify}
for future in as_completed(future_map):
proxy = future_map[future]
try:
if future.result():
with self._lock:
self.working_proxies.append(proxy)
except Exception:
continue
with self._lock:
return list(self.working_proxies)
def collect_until_target(
self,
quake_client: QuakeClient,
target_working: int = 5,
stable_rounds: int = 2,
fetch_batch_size: int = 60,
verify_batch_size: int = 30,
max_rounds: int = 6,
max_workers: int = 16,
countries: Optional[list[str]] = None,
) -> list[ProxyInfo]:
stable_hits = 0
for _ in range(max_rounds):
if stable_hits >= stable_rounds:
break
new_items = quake_client.get_foreign_proxies(size=fetch_batch_size, countries=countries)
if new_items:
self.add_proxies(new_items)
self.verify_all(max_workers=max_workers, max_proxies=verify_batch_size)
with self._lock:
working_count = len(self.working_proxies)
if working_count >= target_working:
stable_hits += 1
else:
stable_hits = 0
with self._lock:
return list(self.working_proxies)
def get_proxy(self, rotate: bool = True) -> Optional[ProxyInfo]:
with self._lock:
if not self.working_proxies:
return None
if not rotate:
return random.choice(self.working_proxies)
proxy = self.working_proxies[self.current_index % len(self.working_proxies)]
self.current_index += 1
return proxy
def mark_failed(self, proxy: ProxyInfo) -> None:
with self._lock:
proxy.fail_count += 1
if proxy.fail_count >= 3 and proxy in self.working_proxies:
self.working_proxies.remove(proxy)
+145
View File
@@ -0,0 +1,145 @@
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
from pixiv_app.core.paths import runtime_path
DEFAULT_THUMBNAIL_DIR = runtime_path("thumbnails")
SUPPORTED_IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
@dataclass
class ThumbnailResult:
artwork_id: int
page_index: int
source_path: str
thumbnail_path: str
width: int
height: int
generated: bool
class ThumbnailCache:
def __init__(
self,
*,
db_path: str | Path = DEFAULT_LIBRARY_DB,
cache_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
size: tuple[int, int] = (360, 360),
quality: int = 86,
) -> None:
self.db_path = Path(db_path)
self.cache_dir = Path(cache_dir)
self.size = size
self.quality = quality
self.cache_dir.mkdir(parents=True, exist_ok=True)
def ensure_for_artwork(self, artwork_id: int) -> list[ThumbnailResult]:
library = PixivLibrary(self.db_path)
try:
rows = library.connection.execute(
"""
SELECT artwork_id, page_index, file_path
FROM artwork_files
WHERE artwork_id = ?
ORDER BY page_index ASC
""",
(artwork_id,),
).fetchall()
return [result for row in rows if (result := self.ensure_for_file(library, row)) is not None]
finally:
library.close()
def build_missing(self, *, limit: int = 200) -> list[ThumbnailResult]:
library = PixivLibrary(self.db_path)
try:
rows = library.connection.execute(
"""
SELECT f.artwork_id, f.page_index, f.file_path
FROM artwork_files f
LEFT JOIN thumbnail_cache t
ON t.artwork_id = f.artwork_id AND t.page_index = f.page_index
WHERE t.thumbnail_path IS NULL
OR t.source_path != f.file_path
ORDER BY f.updated_at DESC
LIMIT ?
""",
(limit,),
).fetchall()
return [result for row in rows if (result := self.ensure_for_file(library, row)) is not None]
finally:
library.close()
def ensure_for_file(self, library: PixivLibrary, row) -> ThumbnailResult | None:
source_path = Path(str(row["file_path"]))
if source_path.suffix.lower() not in SUPPORTED_IMAGE_SUFFIXES or not source_path.exists():
return None
stat = source_path.stat()
thumbnail_path = self._thumbnail_path(int(row["artwork_id"]), int(row["page_index"]), source_path)
cached = library.connection.execute(
"""
SELECT thumbnail_path, width, height, source_mtime, source_size
FROM thumbnail_cache
WHERE artwork_id = ? AND page_index = ?
""",
(int(row["artwork_id"]), int(row["page_index"])),
).fetchone()
if (
cached
and Path(str(cached["thumbnail_path"])).exists()
and float(cached["source_mtime"]) == stat.st_mtime
and int(cached["source_size"]) == stat.st_size
):
return ThumbnailResult(
artwork_id=int(row["artwork_id"]),
page_index=int(row["page_index"]),
source_path=str(source_path),
thumbnail_path=str(cached["thumbnail_path"]),
width=int(cached["width"]),
height=int(cached["height"]),
generated=False,
)
width, height = self._generate(source_path, thumbnail_path)
library.upsert_thumbnail(
artwork_id=int(row["artwork_id"]),
page_index=int(row["page_index"]),
source_path=str(source_path),
thumbnail_path=str(thumbnail_path),
width=width,
height=height,
source_mtime=stat.st_mtime,
source_size=stat.st_size,
)
return ThumbnailResult(
artwork_id=int(row["artwork_id"]),
page_index=int(row["page_index"]),
source_path=str(source_path),
thumbnail_path=str(thumbnail_path),
width=width,
height=height,
generated=True,
)
def _generate(self, source_path: Path, thumbnail_path: Path) -> tuple[int, int]:
try:
from PIL import Image, ImageOps
except ImportError as exc:
raise RuntimeError("Pillow is required to generate thumbnails. Install it with: pip install Pillow") from exc
thumbnail_path.parent.mkdir(parents=True, exist_ok=True)
with Image.open(source_path) as image:
image = ImageOps.exif_transpose(image)
image.thumbnail(self.size, Image.Resampling.LANCZOS)
if image.mode not in {"RGB", "L"}:
image = image.convert("RGB")
image.save(thumbnail_path, format="JPEG", quality=self.quality, optimize=True)
return image.size
def _thumbnail_path(self, artwork_id: int, page_index: int, source_path: Path) -> Path:
bucket = str(abs(artwork_id) % 1000).zfill(3)
return self.cache_dir / bucket / f"{artwork_id}_{page_index}_{source_path.stem}.jpg"
+405
View File
@@ -0,0 +1,405 @@
from __future__ import annotations
import json
import time
import uuid
from dataclasses import asdict
from datetime import datetime
from pathlib import Path
from typing import Any
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.cookie_import import (
cookie_summary,
cookies_to_playwright,
parse_cookie_text,
save_playwright_cookies,
)
from pixiv_app.core.paths import (
app_session_file,
downloads_root,
plugin_roots,
plugins_root as app_plugins_root,
runtime_path,
)
from pixiv_app.core.plugin.base import Resource
from pixiv_app.core.plugin.manager import PluginManager
def project_root() -> Path:
return app_plugins_root().parent
class XMediaService:
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
VIDEO_EXTS = {".mp4", ".mov", ".webm", ".m4v"}
def __init__(self, *, plugins_root: str | Path | None = None, save_root: str | Path | None = None) -> None:
self.plugins_root = Path(plugins_root) if plugins_root else app_plugins_root()
self.save_root = Path(save_root) if save_root else downloads_root() / "x"
self.cookie_path = runtime_path("x_cookies.json")
self.follow_store_path = runtime_path("x_followed_authors.json")
self._manager: PluginManager | None = None
self._preview_cache: dict[str, list[Resource]] = {}
def import_cookie_text(self, text: str) -> dict[str, Any]:
cookies = parse_cookie_text(text)
x_cookies = cookies_to_playwright(cookies, ("x.com", "twitter.com"))
path = save_playwright_cookies(x_cookies, self.cookie_path)
plugin = self._get_plugin()
if hasattr(plugin, "reset_playwright_session"):
plugin.reset_playwright_session()
return {"count": len(x_cookies), "path": str(path), "summary": cookie_summary(x_cookies)}
def preview_author_media(self, username: str, *, media_type: str = "all", max_items: int = 120) -> dict[str, Any]:
handle = _normalize_username(username)
media_type = _normalize_media_type(media_type)
max_items = max(1, min(int(max_items), 500))
url = f"https://x.com/{handle}/media"
plugin = self._get_plugin()
if not plugin.validate():
raise ValueError("X plugin is unavailable. Install playwright/httpx and run playwright install chromium first.")
resources = plugin.parse(url)
resources = _filter_resources(resources, media_type)[:max_items]
preview_id = uuid.uuid4().hex
self._preview_cache[preview_id] = resources
return {
"preview_id": preview_id,
"username": handle,
"media_type": media_type,
"count": len(resources),
"items": [_resource_to_json(item, i) for i, item in enumerate(resources)],
"created_at": int(time.time()),
}
def download_preview(self, preview_id: str, *, selected_ids: list[str] | None = None) -> dict[str, Any]:
resources = self._preview_cache.get(preview_id)
if resources is None:
raise ValueError("Preview expired. Re-run the X media preview.")
selected = set(selected_ids or [])
if selected:
resources = [item for item in resources if item.id in selected]
if not resources:
raise ValueError("No X media items selected.")
plugin = self._get_plugin()
self.save_root.mkdir(parents=True, exist_ok=True)
results: list[dict[str, Any]] = []
downloaded = 0
failed = 0
for resource in resources:
try:
paths = plugin.download(resource, self.save_root)
downloaded += len(paths)
results.append(
{
"id": resource.id,
"ok": bool(paths),
"paths": [str(path) for path in paths],
"title": resource.title,
"synced_files": 0,
}
)
if not paths:
failed += 1
except Exception as exc: # Keep batch downloads moving.
failed += 1
results.append({"id": resource.id, "ok": False, "error": str(exc), "title": resource.title})
return {
"requested": len(resources),
"downloaded_files": downloaded,
"synced_files": 0,
"failed_items": failed,
"items": results,
}
def list_local_media(self, *, limit: int = 160, offset: int = 0) -> dict[str, Any]:
self.save_root.mkdir(parents=True, exist_ok=True)
limit = max(1, min(int(limit), 500))
offset = max(0, int(offset))
files = [
path
for path in self.save_root.rglob("*")
if path.is_file() and path.suffix.lower() in self.IMAGE_EXTS | self.VIDEO_EXTS
]
files.sort(key=lambda path: path.stat().st_mtime, reverse=True)
page = files[offset : offset + limit]
return {
"total": len(files),
"items": [self._local_media_to_json(path) for path in page],
}
def delete_local_media(self, media_ids: list[str]) -> dict[str, Any]:
deleted: list[str] = []
failed: list[dict[str, str]] = []
for media_id in media_ids:
try:
path = self.resolve_local_media(media_id)
path.unlink()
deleted.append(media_id)
except Exception as exc:
failed.append({"id": media_id, "error": str(exc)})
return {"deleted": deleted, "failed": failed}
def resolve_local_media(self, media_id: str) -> Path:
rel_text = _decode_media_id(media_id)
target = (self.save_root / rel_text).resolve()
root = self.save_root.resolve()
if root != target and root not in target.parents:
raise ValueError("Illegal media path.")
if not target.is_file():
raise ValueError("Local media file does not exist.")
return target
def list_followed_authors(self) -> list[dict[str, Any]]:
rows = [item for item in self._load_follow_rows() if bool(item.get("enabled", True))]
rows.sort(key=lambda item: ((str(item.get("creator_name") or "") or str(item.get("username") or "")).lower()))
return rows
def set_followed_author(self, username: str, *, followed: bool = True, creator_name: str = "") -> dict[str, Any]:
handle = _normalize_username(username)
rows = self._load_follow_rows()
row = next((item for item in rows if str(item.get("username", "")).lower() == handle.lower()), None)
now = _now_text()
if row is None:
row = {
"username": handle,
"creator_name": creator_name.strip() or handle,
"enabled": True,
"profile_url": f"https://x.com/{handle}",
"latest_tweet_id": "",
"latest_published_at": "",
"last_checked_at": "",
"last_error": "",
"seen_tweet_ids": [],
"created_at": now,
}
rows.append(row)
row["username"] = handle
row["creator_name"] = creator_name.strip() or str(row.get("creator_name") or handle)
row["enabled"] = bool(followed)
row["profile_url"] = f"https://x.com/{handle}"
row["updated_at"] = now
if not followed:
row["last_error"] = ""
self._save_follow_rows(rows)
return dict(row)
def sync_followed_authors(self, *, download: bool = True, max_new_per_author: int = 20) -> dict[str, Any]:
followed = self.list_followed_authors()
if not followed:
return {"authors": []}
plugin = self._get_plugin()
if not plugin.validate():
raise ValueError("X plugin is unavailable. Import a valid X cookie and ensure Playwright is installed.")
results: list[dict[str, Any]] = []
rows = self._load_follow_rows()
for row in rows:
if not bool(row.get("enabled", True)):
continue
handle = str(row.get("username") or "").strip()
if not handle:
continue
try:
parse_for_sync = getattr(plugin, "parse_for_sync", plugin.parse)
resources = parse_for_sync(f"https://x.com/{handle}")
grouped = _group_resources_by_tweet(resources)
seen_ids = [str(item) for item in row.get("seen_tweet_ids", []) if str(item).strip()]
seen_set = set(seen_ids)
new_groups = [item for item in grouped if item["tweet_id"] and item["tweet_id"] not in seen_set]
new_groups = new_groups[: max(0, int(max_new_per_author))]
downloaded_files = 0
synced_files = 0
created_alerts = 0
new_ids: list[str] = []
for group in new_groups:
representative = group["resource"]
new_ids.append(group["tweet_id"])
if download:
for resource in group["resources"]:
paths = plugin.download(resource, self.save_root)
downloaded_files += len(paths)
latest_group = grouped[0] if grouped else None
if latest_group is not None:
representative = latest_group["resource"]
row["creator_name"] = representative.author or str(row.get("creator_name") or handle)
row["latest_tweet_id"] = latest_group["tweet_id"]
row["latest_published_at"] = representative.created_at or str(row.get("latest_published_at") or "")
row["last_checked_at"] = _now_text()
row["last_error"] = ""
row["updated_at"] = _now_text()
row["seen_tweet_ids"] = _merge_seen_ids(new_ids, seen_ids)
results.append(
{
"username": handle,
"creator_name": row.get("creator_name") or handle,
"new_posts": len(new_groups),
"downloaded_files": downloaded_files,
"synced_files": synced_files,
"alerts": created_alerts,
"error": "",
}
)
except Exception as exc:
row["last_checked_at"] = _now_text()
row["last_error"] = str(exc)
row["updated_at"] = _now_text()
results.append(
{
"username": handle,
"creator_name": row.get("creator_name") or handle,
"new_posts": 0,
"downloaded_files": 0,
"synced_files": 0,
"alerts": 0,
"error": str(exc),
}
)
self._save_follow_rows(rows)
return {"authors": results}
def _get_plugin(self):
if self._manager is None:
self._manager = PluginManager(plugin_roots() if self.plugins_root == app_plugins_root() else [self.plugins_root])
self._manager.load_all()
plugin = self._manager.plugins.get("X")
if plugin is None:
raise ValueError("X plugin is missing. Confirm plugins/x exists.")
return plugin
def _load_follow_rows(self) -> list[dict[str, Any]]:
if not self.follow_store_path.is_file():
return []
try:
payload = json.loads(self.follow_store_path.read_text(encoding="utf-8"))
except Exception:
return []
if isinstance(payload, list):
return [dict(item) for item in payload if isinstance(item, dict)]
return []
def _save_follow_rows(self, rows: list[dict[str, Any]]) -> None:
self.follow_store_path.parent.mkdir(parents=True, exist_ok=True)
self.follow_store_path.write_text(
json.dumps(rows, ensure_ascii=False, indent=2),
encoding="utf-8",
)
def _session_payload(self) -> dict[str, Any]:
return SessionStore(app_session_file()).load()
def _local_media_to_json(self, path: Path) -> dict[str, Any]:
rel = path.relative_to(self.save_root)
stat = path.stat()
kind = "video" if path.suffix.lower() in self.VIDEO_EXTS else "image"
media_id = _encode_media_id(str(rel).replace("\\", "/"))
return {
"id": media_id,
"name": path.name,
"relative_path": str(rel).replace("\\", "/"),
"kind": kind,
"size": stat.st_size,
"modified_at": int(stat.st_mtime),
"url": f"/media/x-local/{media_id}",
}
def _normalize_username(username: str) -> str:
value = username.strip()
if value.startswith("@"):
value = value[1:]
if "/" in value:
parts = [item for item in value.replace("https://", "").replace("http://", "").split("/") if item]
value = parts[1] if parts and parts[0].lower() in {"x.com", "twitter.com", "www.x.com", "www.twitter.com"} else parts[0]
if not value:
raise ValueError("Please enter an X username.")
if len(value) > 15 or not value.replace("_", "").isalnum():
raise ValueError("Invalid X username format.")
return value
def _normalize_media_type(media_type: str) -> str:
value = (media_type or "all").strip().lower()
if value not in {"all", "image", "video"}:
raise ValueError("media_type must be all, image, or video.")
return value
def _filter_resources(resources: list[Resource], media_type: str) -> list[Resource]:
if media_type == "all":
return resources
return [item for item in resources if any(file.get("type", "image") == media_type for file in item.files)]
def _resource_to_json(resource: Resource, index: int) -> dict[str, Any]:
payload = asdict(resource)
payload["index"] = index
payload["kind"] = "video" if any(file.get("type") == "video" for file in resource.files) else "image"
payload["file_count"] = len(resource.files)
return payload
def _group_resources_by_tweet(resources: list[Resource]) -> list[dict[str, Any]]:
grouped: dict[str, dict[str, Any]] = {}
ordered: list[str] = []
for resource in resources:
tweet_id = str(resource.metadata.get("tweet_id") or resource.id or "")
key = tweet_id or resource.id
if key not in grouped:
grouped[key] = {
"tweet_id": tweet_id,
"resource": resource,
"resources": [resource],
}
ordered.append(key)
else:
grouped[key]["resources"].append(resource)
return [grouped[key] for key in ordered]
def _merge_seen_ids(new_ids: list[str], existing_ids: list[str], *, limit: int = 200) -> list[str]:
merged: list[str] = []
seen: set[str] = set()
for item in [*new_ids, *existing_ids]:
value = str(item).strip()
if not value or value in seen:
continue
seen.add(value)
merged.append(value)
if len(merged) >= limit:
break
return merged
def _encode_media_id(value: str) -> str:
from base64 import urlsafe_b64encode
return urlsafe_b64encode(value.encode("utf-8")).decode("ascii").rstrip("=")
def _decode_media_id(value: str) -> str:
from base64 import urlsafe_b64decode
padding = "=" * (-len(value) % 4)
return urlsafe_b64decode((value + padding).encode("ascii")).decode("utf-8")
def _now_text() -> str:
return datetime.now().isoformat(timespec="seconds")