commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Core business logic for Sakura downloader."""
|
||||
@@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import requests
|
||||
|
||||
from pixiv_app.core.downloader import USER_AGENT, build_headers
|
||||
|
||||
|
||||
LOGIN_PAGE_URL = "https://accounts.pixiv.net/login"
|
||||
LOGIN_AJAX_URL = "https://accounts.pixiv.net/ajax/login?lang=zh"
|
||||
VALIDATE_URL = "https://www.pixiv.net/setting_user.php"
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuthResult:
|
||||
success: bool
|
||||
message: str
|
||||
cookie: str = ""
|
||||
requires_verification: bool = False
|
||||
user_name: str = ""
|
||||
user_id: str = ""
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, path: str | Path = "pixiv_app_session.json") -> None:
|
||||
self.path = Path(path)
|
||||
|
||||
def load(self) -> dict[str, Any]:
|
||||
if not self.path.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(self.path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def save(self, payload: dict[str, Any]) -> None:
|
||||
self.path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
class PixivAuthClient:
|
||||
def __init__(self) -> None:
|
||||
self.timeout = 20
|
||||
|
||||
def validate_cookie(self, cookie: str) -> AuthResult:
|
||||
cookie = cookie.strip()
|
||||
if not cookie:
|
||||
return AuthResult(False, "Cookie 为空,请先填写或登录。")
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update(build_headers(cookie=cookie, referer="https://www.pixiv.net/"))
|
||||
try:
|
||||
response = session.get(VALIDATE_URL, allow_redirects=True, timeout=self.timeout)
|
||||
final_url = response.url
|
||||
if "return_to=" in final_url or "accounts.pixiv.net" in final_url:
|
||||
return AuthResult(False, "当前 Cookie 已失效或未登录,请更新会话。")
|
||||
user_id, user_name = self._extract_user_info(response.text)
|
||||
return AuthResult(True, "Cookie 有效,可直接开始抓取。", cookie=cookie, user_name=user_name, user_id=user_id)
|
||||
except Exception as exc:
|
||||
return AuthResult(False, f"Cookie 检测失败: {exc}")
|
||||
|
||||
def login_with_password(self, login_id: str, password: str) -> AuthResult:
|
||||
login_id = login_id.strip()
|
||||
password = password.strip()
|
||||
if not login_id or not password:
|
||||
return AuthResult(False, "账号和密码都不能为空。")
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update(
|
||||
{
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Origin": "https://accounts.pixiv.net",
|
||||
"Referer": LOGIN_PAGE_URL,
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
login_page = session.get(LOGIN_PAGE_URL, timeout=self.timeout)
|
||||
login_page.raise_for_status()
|
||||
tt = self._extract_tt(login_page.text)
|
||||
|
||||
payload = {
|
||||
"login_id": login_id,
|
||||
"password": password,
|
||||
"source": "accounts",
|
||||
"app_ios": "0",
|
||||
"ref": "",
|
||||
"return_to": "https://www.pixiv.net/",
|
||||
"g_recaptcha_response": "",
|
||||
"recaptcha_enterprise_score_token": "",
|
||||
}
|
||||
headers = dict(session.headers)
|
||||
if tt:
|
||||
headers["X-CSRF-Token"] = tt
|
||||
|
||||
response = session.post(
|
||||
LOGIN_AJAX_URL,
|
||||
data=urlencode(payload),
|
||||
headers={**headers, "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
|
||||
if isinstance(body, dict) and body.get("body", {}).get("success"):
|
||||
cookie = self._cookies_to_header(session.cookies)
|
||||
validation = self.validate_cookie(cookie)
|
||||
if validation.success:
|
||||
validation.message = "账号密码登录成功,会话已可用。"
|
||||
return validation
|
||||
return AuthResult(True, "登录接口返回成功,但会话校验未完全通过,请手动检测。", cookie=cookie)
|
||||
|
||||
body_payload = body.get("body", {}) if isinstance(body, dict) else {}
|
||||
errors = body_payload.get("errors", {}) if isinstance(body_payload, dict) else {}
|
||||
if errors.get("recaptcha") or body_payload.get("requireExtraVerification") or body_payload.get("requireTwoFactorAuthentication"):
|
||||
return AuthResult(
|
||||
False,
|
||||
"Pixiv 要求额外验证或二步验证,程序无法静默完成。请改用 Cookie 登录,或先在浏览器登录后导出 Cookie。",
|
||||
requires_verification=True,
|
||||
)
|
||||
message = self._flatten_error_message(errors) or "账号密码登录失败,请检查账号、密码或改用 Cookie。"
|
||||
return AuthResult(False, message)
|
||||
except Exception as exc:
|
||||
return AuthResult(False, f"账号密码登录失败: {exc}")
|
||||
|
||||
def _extract_tt(self, html_text: str) -> str:
|
||||
match = re.search(r'"pixivAccount\.tt":"([^"]+)"', html_text)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
def _extract_user_info(self, html_text: str) -> tuple[str, str]:
|
||||
user_id = ""
|
||||
user_name = ""
|
||||
patterns = [
|
||||
r'"userData":\{"id":"(\d+)","pixivId":"[^"]*","name":"([^"]+)"',
|
||||
r'"user_id":"(\d+)".*?"name":"([^"]+)"',
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, html_text)
|
||||
if match:
|
||||
user_id = match.group(1)
|
||||
user_name = html.unescape(match.group(2))
|
||||
break
|
||||
return user_id, user_name
|
||||
|
||||
def _flatten_error_message(self, errors: dict[str, Any]) -> str:
|
||||
messages: list[str] = []
|
||||
for key, value in errors.items():
|
||||
if isinstance(value, str) and value.strip():
|
||||
messages.append(f"{key}: {value}")
|
||||
elif isinstance(value, list):
|
||||
text_items = [str(item).strip() for item in value if str(item).strip()]
|
||||
if text_items:
|
||||
messages.append(f"{key}: {'; '.join(text_items)}")
|
||||
return " | ".join(messages)
|
||||
|
||||
def _cookies_to_header(self, jar: requests.cookies.RequestsCookieJar) -> str:
|
||||
parts = [f"{cookie.name}={cookie.value}" for cookie in jar]
|
||||
return "; ".join(parts)
|
||||
@@ -0,0 +1,353 @@
|
||||
"""
|
||||
Playwright 浏览器代理池 - 支持动态换 IP
|
||||
功能:
|
||||
- 多代理轮换(轮询/随机/会话粘性)
|
||||
- 健康检查(自动剔除失效代理)
|
||||
- 延迟统计(加权选择最优代理)
|
||||
- 线程安全
|
||||
"""
|
||||
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, List, Dict, Any
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class RotationStrategy(Enum):
|
||||
"""轮换策略"""
|
||||
ROUND_ROBIN = "round_robin" # 轮询
|
||||
RANDOM = "random" # 随机
|
||||
WEIGHTED = "weighted" # 加权(按成功率+延迟)
|
||||
SESSION_STICKY = "session_sticky" # 会话粘性(同会话同IP)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProxyInfo:
|
||||
"""代理信息"""
|
||||
server: str # http://ip:port 或 socks5://ip:port
|
||||
username: str = ""
|
||||
password: str = ""
|
||||
|
||||
# 统计数据
|
||||
success_count: int = 0
|
||||
fail_count: int = 0
|
||||
total_latency_ms: float = 0.0
|
||||
avg_latency_ms: float = 0.0
|
||||
last_check_time: float = 0.0
|
||||
last_fail_time: float = 0.0
|
||||
|
||||
# 状态
|
||||
healthy: bool = True
|
||||
cooldown_until: float = 0.0 # 冷却结束时间
|
||||
|
||||
@property
|
||||
def proxy_url(self) -> str:
|
||||
"""获取完整的代理URL(含认证)"""
|
||||
if self.username and self.password:
|
||||
# http://user:pass@ip:port
|
||||
protocol = self.server.split("://")[0]
|
||||
rest = self.server.split("://")[1]
|
||||
return f"{protocol}://{self.username}:{self.password}@{rest}"
|
||||
return self.server
|
||||
|
||||
def to_requests_proxies(self) -> dict[str, str]:
|
||||
return {"http": self.proxy_url, "https": self.proxy_url}
|
||||
|
||||
def to_playwright_proxy(self) -> dict[str, str]:
|
||||
data = {"server": self.server}
|
||||
if self.username:
|
||||
data["username"] = self.username
|
||||
data["password"] = self.password
|
||||
return data
|
||||
|
||||
@property
|
||||
def success_rate(self) -> float:
|
||||
"""成功率"""
|
||||
total = self.success_count + self.fail_count
|
||||
if total == 0:
|
||||
return 1.0
|
||||
return self.success_count / total
|
||||
|
||||
@property
|
||||
def weight(self) -> float:
|
||||
"""
|
||||
计算代理权重(用于加权随机选择)
|
||||
考虑因素:成功率、延迟、冷却状态
|
||||
"""
|
||||
if not self.healthy:
|
||||
return 0.0
|
||||
|
||||
# 冷却中
|
||||
if self.cooldown_until > time.time():
|
||||
return 0.0
|
||||
|
||||
# 成功率权重
|
||||
success_weight = self.success_rate
|
||||
|
||||
# 延迟权重(延迟越低权重越高)
|
||||
if self.avg_latency_ms > 0:
|
||||
latency_weight = max(0.1, 1000.0 / max(self.avg_latency_ms, 1))
|
||||
else:
|
||||
latency_weight = 1.0
|
||||
|
||||
# 失败惩罚
|
||||
fail_penalty = 1.0 / (1.0 + self.fail_count * 0.5)
|
||||
|
||||
return success_weight * latency_weight * fail_penalty
|
||||
|
||||
|
||||
class ProxyPool:
|
||||
"""代理池 - 管理多个代理的获取和轮换"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
strategy: RotationStrategy = RotationStrategy.WEIGHTED,
|
||||
health_check_url: str = "https://httpbin.org/ip",
|
||||
health_check_timeout: int = 8,
|
||||
max_failures: int = 3,
|
||||
cooldown_seconds: int = 60,
|
||||
auto_health_check: bool = True,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
strategy: 轮换策略
|
||||
health_check_url: 健康检查URL
|
||||
health_check_timeout: 健康检查超时(秒)
|
||||
max_failures: 最大失败次数,超过后标记为不健康
|
||||
cooldown_seconds: 不健康代理的冷却时间(秒)
|
||||
auto_health_check: 是否自动健康检查
|
||||
"""
|
||||
self.strategy = strategy
|
||||
self.health_check_url = health_check_url
|
||||
self.health_check_timeout = health_check_timeout
|
||||
self.max_failures = max_failures
|
||||
self.cooldown_seconds = cooldown_seconds
|
||||
self.auto_health_check = auto_health_check
|
||||
|
||||
self._proxies: List[ProxyInfo] = []
|
||||
self._session_proxy_map: Dict[str, ProxyInfo] = {} # 会话粘性用
|
||||
self._round_robin_index = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add_proxy(self, server: str, username: str = "", password: str = "") -> None:
|
||||
"""添加单个代理"""
|
||||
with self._lock:
|
||||
# 去重
|
||||
existing = {p.server for p in self._proxies}
|
||||
if server not in existing:
|
||||
self._proxies.append(ProxyInfo(
|
||||
server=server,
|
||||
username=username,
|
||||
password=password,
|
||||
))
|
||||
|
||||
def add_proxy_url(self, value: str) -> None:
|
||||
parsed = parse_proxy_url(value)
|
||||
if parsed is not None:
|
||||
self.add_proxy(parsed.server, parsed.username, parsed.password)
|
||||
|
||||
def add_proxies(self, proxies: List[dict]) -> None:
|
||||
"""批量添加代理
|
||||
|
||||
Args:
|
||||
proxies: [{"server": "http://1.2.3.4:8080", "username": "", "password": ""}, ...]
|
||||
"""
|
||||
for p in proxies:
|
||||
self.add_proxy(
|
||||
server=p["server"],
|
||||
username=p.get("username", ""),
|
||||
password=p.get("password", ""),
|
||||
)
|
||||
|
||||
def add_proxy_text(self, text: str) -> int:
|
||||
count = 0
|
||||
for line in text.splitlines():
|
||||
parsed = parse_proxy_url(line)
|
||||
if parsed is None:
|
||||
continue
|
||||
self.add_proxy(parsed.server, parsed.username, parsed.password)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def remove_proxy(self, server: str) -> None:
|
||||
"""移除代理"""
|
||||
with self._lock:
|
||||
self._proxies = [p for p in self._proxies if p.server != server]
|
||||
|
||||
def get_proxy(self, session_id: str = None) -> Optional[ProxyInfo]:
|
||||
"""
|
||||
获取一个代理
|
||||
|
||||
Args:
|
||||
session_id: 会话ID,用于SESSION_STICKY策略
|
||||
|
||||
Returns:
|
||||
代理信息,如果没有可用代理返回None
|
||||
"""
|
||||
with self._lock:
|
||||
# 过滤出健康的代理
|
||||
now = time.time()
|
||||
healthy = [p for p in self._proxies if p.healthy and p.cooldown_until <= now]
|
||||
|
||||
if not healthy:
|
||||
# 尝试从冷却中恢复
|
||||
now = time.time()
|
||||
recovered = []
|
||||
for p in self._proxies:
|
||||
if not p.healthy and p.cooldown_until <= now:
|
||||
p.healthy = True
|
||||
p.fail_count = 0
|
||||
recovered.append(p)
|
||||
if recovered:
|
||||
healthy = recovered
|
||||
|
||||
if not healthy:
|
||||
return None
|
||||
|
||||
# 根据策略选择代理
|
||||
if self.strategy == RotationStrategy.RANDOM:
|
||||
return random.choice(healthy)
|
||||
|
||||
elif self.strategy == RotationStrategy.WEIGHTED:
|
||||
# 加权随机选择
|
||||
weights = [p.weight for p in healthy]
|
||||
total_weight = sum(weights)
|
||||
if total_weight <= 0:
|
||||
return random.choice(healthy)
|
||||
r = random.uniform(0, total_weight)
|
||||
cumsum = 0
|
||||
for i, w in enumerate(weights):
|
||||
cumsum += w
|
||||
if r <= cumsum:
|
||||
return healthy[i]
|
||||
return healthy[0]
|
||||
|
||||
elif self.strategy == RotationStrategy.SESSION_STICKY:
|
||||
if session_id and session_id in self._session_proxy_map:
|
||||
proxy = self._session_proxy_map[session_id]
|
||||
if proxy in healthy:
|
||||
return proxy
|
||||
# 新会话,分配一个代理
|
||||
proxy = self._get_round_robin(healthy)
|
||||
if session_id:
|
||||
self._session_proxy_map[session_id] = proxy
|
||||
return proxy
|
||||
|
||||
else: # ROUND_ROBIN
|
||||
return self._get_round_robin(healthy)
|
||||
|
||||
def _get_round_robin(self, proxies: List[ProxyInfo]) -> ProxyInfo:
|
||||
"""轮询获取代理"""
|
||||
proxy = proxies[self._round_robin_index % len(proxies)]
|
||||
self._round_robin_index += 1
|
||||
return proxy
|
||||
|
||||
def mark_success(self, proxy: ProxyInfo, latency_ms: float = 0) -> None:
|
||||
"""标记代理请求成功"""
|
||||
if proxy is None:
|
||||
return
|
||||
with self._lock:
|
||||
proxy.success_count += 1
|
||||
proxy.fail_count = 0
|
||||
proxy.healthy = True
|
||||
if latency_ms > 0:
|
||||
# 更新平均延迟(指数移动平均)
|
||||
if proxy.avg_latency_ms == 0:
|
||||
proxy.avg_latency_ms = latency_ms
|
||||
else:
|
||||
proxy.avg_latency_ms = proxy.avg_latency_ms * 0.7 + latency_ms * 0.3
|
||||
proxy.last_check_time = time.time()
|
||||
|
||||
def mark_failed(self, proxy: ProxyInfo) -> None:
|
||||
"""标记代理请求失败"""
|
||||
if proxy is None:
|
||||
return
|
||||
with self._lock:
|
||||
proxy.fail_count += 1
|
||||
if proxy.fail_count >= self.max_failures:
|
||||
proxy.healthy = False
|
||||
proxy.cooldown_until = time.time() + self.cooldown_seconds
|
||||
|
||||
def health_check(self, proxy: ProxyInfo = None) -> bool:
|
||||
"""
|
||||
健康检查单个或所有代理
|
||||
|
||||
Returns:
|
||||
如果检查通过返回True
|
||||
"""
|
||||
import requests
|
||||
|
||||
proxies_to_check = [proxy] if proxy else self._proxies
|
||||
|
||||
for p in proxies_to_check:
|
||||
if p is None:
|
||||
continue
|
||||
try:
|
||||
start = time.time()
|
||||
resp = requests.get(
|
||||
self.health_check_url,
|
||||
proxies={"http": p.proxy_url, "https": p.proxy_url},
|
||||
timeout=self.health_check_timeout,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
latency_ms = (time.time() - start) * 1000
|
||||
self.mark_success(p, latency_ms)
|
||||
else:
|
||||
self.mark_failed(p)
|
||||
except Exception:
|
||||
self.mark_failed(p)
|
||||
|
||||
return proxy.healthy if proxy else len(self.get_healthy_proxies()) > 0
|
||||
|
||||
def get_healthy_proxies(self) -> List[ProxyInfo]:
|
||||
"""获取所有健康代理"""
|
||||
with self._lock:
|
||||
return [p for p in self._proxies if p.healthy]
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""获取代理池统计信息"""
|
||||
with self._lock:
|
||||
healthy = [p for p in self._proxies if p.healthy]
|
||||
return {
|
||||
"total": len(self._proxies),
|
||||
"healthy": len(healthy),
|
||||
"unhealthy": len(self._proxies) - len(healthy),
|
||||
"proxies": [
|
||||
{
|
||||
"server": p.server,
|
||||
"healthy": p.healthy,
|
||||
"success_rate": p.success_rate,
|
||||
"avg_latency_ms": round(p.avg_latency_ms, 2),
|
||||
"fail_count": p.fail_count,
|
||||
}
|
||||
for p in self._proxies
|
||||
]
|
||||
}
|
||||
|
||||
def release_session(self, session_id: str) -> None:
|
||||
"""释放会话绑定的代理"""
|
||||
with self._lock:
|
||||
self._session_proxy_map.pop(session_id, None)
|
||||
|
||||
|
||||
def parse_proxy_url(value: str) -> Optional[ProxyInfo]:
|
||||
text = value.strip()
|
||||
if not text or text.startswith("#"):
|
||||
return None
|
||||
if "://" not in text:
|
||||
text = "http://" + text
|
||||
import re
|
||||
|
||||
match = re.match(r"^(https?|socks5)://(?:(?P<user>[^:@/]+):(?P<pwd>[^@/]+)@)?(?P<host>[\w.\-]+):(?P<port>\d{2,5})$", text, re.I)
|
||||
if not match:
|
||||
return None
|
||||
scheme = text.split("://", 1)[0].lower()
|
||||
host = match.group("host")
|
||||
port = match.group("port")
|
||||
user = match.group("user") or ""
|
||||
pwd = match.group("pwd") or ""
|
||||
return ProxyInfo(server=f"{scheme}://{host}:{port}", username=user, password=pwd)
|
||||
|
||||
|
||||
@@ -0,0 +1,602 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def file_sha256(path: str | Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open("rb") as fp:
|
||||
for chunk in iter(lambda: fp.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def persist_cloud_session_tokens(access_token: str, refresh_token: str = "") -> None:
|
||||
"""将刷新后的 Token 写回桌面会话文件,供 GUI 与 Gallery API 共用。"""
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.paths import app_session_file
|
||||
|
||||
store = SessionStore(app_session_file())
|
||||
payload = store.load()
|
||||
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
|
||||
cloud["access_token"] = access_token
|
||||
if refresh_token:
|
||||
cloud["refresh_token"] = refresh_token
|
||||
payload["cloud_sync"] = cloud
|
||||
store.save(payload)
|
||||
|
||||
|
||||
def _decode_jwt_exp(token: str) -> float:
|
||||
"""解码 JWT 的 exp 字段,返回 Unix 时间戳;解码失败返回 0。"""
|
||||
if not token or "." not in token:
|
||||
return 0.0
|
||||
try:
|
||||
payload = token.split(".")[1]
|
||||
payload += "=" * (4 - len(payload) % 4)
|
||||
decoded = json.loads(base64.urlsafe_b64decode(payload))
|
||||
return float(decoded.get("exp", 0))
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
class CloudSession:
|
||||
"""全局单例,统一管理云同步 Token 的读写、刷新与持久化。
|
||||
|
||||
所有需要访问 Sakura Cloud API 的模块(GUI、Gallery、FollowSync、XMedia)
|
||||
都通过此单例获取配置和创建客户端,确保 token 刷新互不冲突、变更全局可见。
|
||||
"""
|
||||
|
||||
_instance: CloudSession | None = None
|
||||
_instance_lock = threading.Lock()
|
||||
|
||||
def __new__(cls) -> CloudSession:
|
||||
if cls._instance is None:
|
||||
with cls._instance_lock:
|
||||
if cls._instance is None:
|
||||
instance = super().__new__(cls)
|
||||
instance._base_url = ""
|
||||
instance._access_token = ""
|
||||
instance._refresh_token = ""
|
||||
instance._enabled = False
|
||||
instance._refresh_lock = threading.Lock()
|
||||
instance._listeners: list[Callable[[str, str], None]] = []
|
||||
instance._last_refresh_time = 0.0
|
||||
cls._instance = instance
|
||||
return cls._instance
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 配置入口
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def configure(
|
||||
self,
|
||||
*,
|
||||
base_url: str = "",
|
||||
access_token: str = "",
|
||||
refresh_token: str = "",
|
||||
enabled: bool = False,
|
||||
) -> None:
|
||||
"""GUI 启动 / 登录 / 保存会话时调用,同步最新 token 到全局单例。"""
|
||||
self._base_url = (base_url or "").strip()
|
||||
self._access_token = (access_token or "").strip()
|
||||
self._refresh_token = (refresh_token or "").strip()
|
||||
self._enabled = bool(enabled)
|
||||
|
||||
def load_from_disk(self) -> dict[str, Any]:
|
||||
"""从磁盘会话文件读取云端配置并应用到单例,返回原始 payload。"""
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.paths import app_session_file
|
||||
|
||||
store = SessionStore(app_session_file())
|
||||
payload = store.load()
|
||||
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
|
||||
self.configure(
|
||||
base_url=str(cloud.get("base_url") or ""),
|
||||
access_token=str(cloud.get("access_token") or ""),
|
||||
refresh_token=str(cloud.get("refresh_token") or ""),
|
||||
enabled=bool(cloud.get("enabled", False)),
|
||||
)
|
||||
return payload
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self._enabled and bool(self._base_url) and bool(self._access_token)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Config 快照
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_config(self) -> CloudSyncConfig:
|
||||
"""返回当前 token 的 CloudSyncConfig 快照。
|
||||
|
||||
调用时机:每次创建 SakuraCloudClient 前。
|
||||
若 access_token 即将过期(< 5 分钟),主动触发一次刷新。
|
||||
"""
|
||||
if self._access_token and self._is_expired_or_near(threshold_seconds=300):
|
||||
self.try_refresh()
|
||||
return CloudSyncConfig(
|
||||
base_url=self._base_url,
|
||||
access_token=self._access_token,
|
||||
refresh_token=self._refresh_token,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 监听者管理
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_listener(self, callback: Callable[[str, str], None]) -> None:
|
||||
"""注册 token 刷新回调(如 GUI 的 _on_cloud_token_refreshed)。"""
|
||||
if callback not in self._listeners:
|
||||
self._listeners.append(callback)
|
||||
|
||||
def remove_listener(self, callback: Callable[[str, str], None]) -> None:
|
||||
try:
|
||||
self._listeners.remove(callback)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def notify_listeners(self, access_token: str, refresh_token: str) -> None:
|
||||
"""通知所有监听者 token 已更新(成功时为新的 token 对,失败时为空字符串)。"""
|
||||
for callback in self._listeners:
|
||||
try:
|
||||
callback(access_token, refresh_token)
|
||||
except Exception:
|
||||
logger.exception("CloudSession listener %s failed", getattr(callback, "__name__", callback))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Token 刷新(核心)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def try_refresh(self) -> bool:
|
||||
"""统一刷新入口。使用全局锁防止并发重复刷新。
|
||||
|
||||
Returns:
|
||||
True: 刷新成功,access_token 已更新
|
||||
False: 刷新失败,token 已被清空,监听者已收到空 token 通知
|
||||
"""
|
||||
refresh = self._refresh_token
|
||||
if not refresh:
|
||||
return False
|
||||
|
||||
with self._refresh_lock:
|
||||
# 二次检查:可能其他线程已刷新
|
||||
refresh = self._refresh_token
|
||||
if not refresh:
|
||||
return False
|
||||
|
||||
# 距上次刷新不足 5 秒,跳过(防抖)
|
||||
elapsed = time.time() - self._last_refresh_time
|
||||
if elapsed < 5.0 and self._access_token:
|
||||
return True
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._base_url}/api/v1/auth/refresh",
|
||||
json={"refresh_token": refresh},
|
||||
timeout=30.0,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.warning(
|
||||
"CloudSession refresh failed HTTP %s: %s",
|
||||
resp.status_code,
|
||||
resp.text[:200],
|
||||
)
|
||||
return False
|
||||
|
||||
body = resp.json()
|
||||
data = body.get("data", body) if isinstance(body, dict) else {}
|
||||
if not isinstance(data, dict):
|
||||
self._on_refresh_failed()
|
||||
return False
|
||||
|
||||
new_access = (data.get("access_token") or "").strip()
|
||||
new_refresh = (data.get("refresh_token") or "").strip()
|
||||
if not new_access:
|
||||
self._on_refresh_failed()
|
||||
return False
|
||||
|
||||
# 更新内存
|
||||
self._access_token = new_access
|
||||
if new_refresh:
|
||||
self._refresh_token = new_refresh
|
||||
self._last_refresh_time = time.time()
|
||||
|
||||
# 写盘
|
||||
self._persist()
|
||||
|
||||
# 通知监听者
|
||||
self.notify_listeners(self._access_token, self._refresh_token)
|
||||
logger.info("CloudSession token refreshed successfully")
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
logger.exception("CloudSession refresh error")
|
||||
self._on_refresh_failed()
|
||||
return False
|
||||
|
||||
def _on_refresh_failed(self) -> None:
|
||||
"""刷新失败:清空 token + 写盘 + 通知监听者。"""
|
||||
self._access_token = ""
|
||||
self._refresh_token = ""
|
||||
self._last_refresh_time = time.time()
|
||||
self._persist()
|
||||
self.notify_listeners("", "")
|
||||
|
||||
def _persist(self) -> None:
|
||||
"""将当前 token 写入磁盘会话文件。异常记录日志但不抛出。"""
|
||||
try:
|
||||
persist_cloud_session_tokens(self._access_token, self._refresh_token)
|
||||
except Exception:
|
||||
logger.exception("CloudSession persist to disk failed")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# JWT 辅助
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _is_expired_or_near(self, threshold_seconds: int = 300) -> bool:
|
||||
exp = _decode_jwt_exp(self._access_token)
|
||||
if exp <= 0:
|
||||
return False
|
||||
return (exp - threshold_seconds) <= time.time()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CloudSyncConfig:
|
||||
base_url: str
|
||||
access_token: str
|
||||
refresh_token: str = ""
|
||||
chunk_size: int = 5 * 1024 * 1024
|
||||
timeout: float = 60.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncMetadata:
|
||||
resource_type: str = "artwork"
|
||||
source_platform: str = "pixiv"
|
||||
source_id: str | None = None
|
||||
content_type: str | None = None
|
||||
title: str = ""
|
||||
author: str | None = None
|
||||
author_id: str | None = None
|
||||
tags: list[str] = field(default_factory=list)
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
page_count: int = 1
|
||||
thumbnail_url: str | None = None
|
||||
description: str | None = None
|
||||
word_count: int = 0
|
||||
series_id: str | None = None
|
||||
series_title: str | None = None
|
||||
chapter_title: str | None = None
|
||||
chapter_order: int = 0
|
||||
|
||||
|
||||
class SyncFailureQueue:
|
||||
def __init__(self, path: str | Path | None = None) -> None:
|
||||
self.path = Path(path) if path else runtime_path("cloud_sync_failures.json")
|
||||
|
||||
def load(self) -> list[dict[str, Any]]:
|
||||
if not self.path.is_file():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def save(self, items: list[dict[str, Any]]) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text(json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
def add(self, file_path: str | Path, metadata: SyncMetadata, mime_type: str | None, error: str) -> None:
|
||||
now = time.time()
|
||||
file_path = str(Path(file_path))
|
||||
items = self.load()
|
||||
key = (file_path, metadata.source_platform, metadata.source_id)
|
||||
for item in items:
|
||||
item_key = (
|
||||
str(item.get("file_path") or ""),
|
||||
str(item.get("metadata", {}).get("source_platform") or ""),
|
||||
item.get("metadata", {}).get("source_id"),
|
||||
)
|
||||
if item_key == key:
|
||||
attempts = int(item.get("attempts") or 0)
|
||||
item.update(
|
||||
{
|
||||
"metadata": asdict(metadata),
|
||||
"mime_type": mime_type,
|
||||
"attempts": attempts,
|
||||
"last_error": error,
|
||||
"updated_at": now,
|
||||
"next_retry_at": now + min(3600, 60 * (2 ** min(attempts, 5))),
|
||||
}
|
||||
)
|
||||
self.save(items)
|
||||
return
|
||||
items.append(
|
||||
{
|
||||
"file_path": file_path,
|
||||
"metadata": asdict(metadata),
|
||||
"mime_type": mime_type,
|
||||
"attempts": 0,
|
||||
"last_error": error,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"next_retry_at": now + 60,
|
||||
}
|
||||
)
|
||||
self.save(items)
|
||||
|
||||
|
||||
class SakuraCloudClient:
|
||||
def __init__(
|
||||
self,
|
||||
config: CloudSyncConfig,
|
||||
*,
|
||||
on_token_refreshed: Callable[[str, str], None] | None = None,
|
||||
cloud_session: CloudSession | None = None,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.base_url = config.base_url.rstrip("/")
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({"Authorization": f"Bearer {config.access_token}"})
|
||||
self._cloud_session = cloud_session # 全局单例引用
|
||||
# 仅在没有 cloud_session 时使用独立锁和回调(兼容旧路径)
|
||||
if cloud_session is None:
|
||||
self._on_token_refreshed = on_token_refreshed or persist_cloud_session_tokens
|
||||
self._refresh_lock = threading.Lock()
|
||||
else:
|
||||
self._on_token_refreshed = None
|
||||
self._refresh_lock = None
|
||||
|
||||
def check_file(self, sha256: str, file_size: int) -> dict[str, Any]:
|
||||
return self._post("/api/v1/sync/check", {"sha256": sha256, "file_size": file_size})
|
||||
|
||||
def init_upload(
|
||||
self,
|
||||
*,
|
||||
sha256: str,
|
||||
file_name: str,
|
||||
file_size: int,
|
||||
mime_type: str | None,
|
||||
resource_type: str,
|
||||
) -> dict[str, Any]:
|
||||
return self._post(
|
||||
"/api/v1/upload/init",
|
||||
{
|
||||
"sha256": sha256,
|
||||
"file_name": file_name,
|
||||
"file_size": file_size,
|
||||
"mime_type": mime_type,
|
||||
"resource_type": resource_type,
|
||||
},
|
||||
)
|
||||
|
||||
def upload_chunk(self, upload_id: str, chunk: bytes) -> dict[str, Any]:
|
||||
response = self._request(
|
||||
"POST",
|
||||
"/api/v1/upload/chunk",
|
||||
data={"upload_id": upload_id},
|
||||
files={"chunk": ("chunk.bin", chunk)},
|
||||
)
|
||||
return self._unwrap(response)
|
||||
|
||||
def complete_upload(self, upload_id: str) -> dict[str, Any]:
|
||||
return self._post("/api/v1/upload/complete", {"upload_id": upload_id})
|
||||
|
||||
def sync_metadata(self, file_id: str, metadata: SyncMetadata) -> dict[str, Any]:
|
||||
payload = {
|
||||
"file_id": file_id,
|
||||
"resource_type": metadata.resource_type,
|
||||
"source_platform": metadata.source_platform,
|
||||
"source_id": metadata.source_id,
|
||||
"content_type": metadata.content_type,
|
||||
"title": metadata.title,
|
||||
"author": metadata.author,
|
||||
"author_id": metadata.author_id,
|
||||
"tags": metadata.tags,
|
||||
"metadata": metadata.metadata,
|
||||
"page_count": metadata.page_count,
|
||||
"thumbnail_url": metadata.thumbnail_url,
|
||||
"description": metadata.description,
|
||||
"word_count": metadata.word_count,
|
||||
"series_id": metadata.series_id,
|
||||
"series_title": metadata.series_title,
|
||||
"chapter_title": metadata.chapter_title,
|
||||
"chapter_order": metadata.chapter_order,
|
||||
}
|
||||
return self._post("/api/v1/sync/metadata", payload)
|
||||
|
||||
def upsert_follow(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return self._post("/api/v1/follows", payload)
|
||||
|
||||
def bulk_upsert_follows(self, items: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
return self._post("/api/v1/follows/bulk-upsert", {"items": items})
|
||||
|
||||
def create_alert(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return self._post("/api/v1/follows/alerts", payload)
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
json: dict[str, Any] | None = None,
|
||||
data: dict[str, Any] | None = None,
|
||||
files: dict[str, Any] | None = None,
|
||||
_retry_on_401: bool = True,
|
||||
) -> requests.Response:
|
||||
url = f"{self.base_url}{path}"
|
||||
response = self.session.request(
|
||||
method,
|
||||
url,
|
||||
json=json,
|
||||
data=data,
|
||||
files=files,
|
||||
timeout=self.config.timeout,
|
||||
)
|
||||
if response.status_code == 401 and _retry_on_401 and self._try_refresh_token():
|
||||
response = self.session.request(
|
||||
method,
|
||||
url,
|
||||
json=json,
|
||||
data=data,
|
||||
files=files,
|
||||
timeout=self.config.timeout,
|
||||
)
|
||||
return response
|
||||
|
||||
def _post(self, path: str, payload: dict[str, Any], *, _retry_on_401: bool = True) -> dict[str, Any]:
|
||||
response = self._request("POST", path, json=payload, _retry_on_401=_retry_on_401)
|
||||
return self._unwrap(response)
|
||||
|
||||
def _try_refresh_token(self) -> bool:
|
||||
# --- 新路径:委托给全局 CloudSession ---
|
||||
if self._cloud_session is not None:
|
||||
ok = self._cloud_session.try_refresh()
|
||||
if ok:
|
||||
# 刷新成功后同步更新本地 config 和 session headers
|
||||
new_config = self._cloud_session.get_config()
|
||||
self.config.access_token = new_config.access_token
|
||||
self.config.refresh_token = new_config.refresh_token
|
||||
self.session.headers.update({"Authorization": f"Bearer {new_config.access_token}"})
|
||||
return ok
|
||||
|
||||
# --- 旧路径(向后兼容,无 cloud_session 时使用) ---
|
||||
refresh = self.config.refresh_token.strip()
|
||||
if not refresh:
|
||||
return False
|
||||
with self._refresh_lock:
|
||||
refresh = self.config.refresh_token.strip()
|
||||
if not refresh:
|
||||
return False
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self.base_url}/api/v1/auth/refresh",
|
||||
json={"refresh_token": refresh},
|
||||
timeout=self.config.timeout,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return False
|
||||
body = resp.json()
|
||||
data = body.get("data", body) if isinstance(body, dict) else {}
|
||||
if not isinstance(data, dict):
|
||||
return False
|
||||
new_access = (data.get("access_token") or "").strip()
|
||||
new_refresh = (data.get("refresh_token") or "").strip()
|
||||
if not new_access:
|
||||
return False
|
||||
self.config.access_token = new_access
|
||||
self.session.headers.update({"Authorization": f"Bearer {new_access}"})
|
||||
if new_refresh:
|
||||
self.config.refresh_token = new_refresh
|
||||
try:
|
||||
self._on_token_refreshed(self.config.access_token, self.config.refresh_token)
|
||||
except Exception:
|
||||
logger.exception("SakuraCloudClient._on_token_refreshed failed")
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("SakuraCloudClient._try_refresh_token error")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _unwrap(response: requests.Response) -> dict[str, Any]:
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
if isinstance(body, dict) and "data" in body:
|
||||
return body["data"]
|
||||
return body
|
||||
|
||||
|
||||
class SyncManager:
|
||||
def __init__(self, client: SakuraCloudClient, failure_queue: SyncFailureQueue | None = None) -> None:
|
||||
self.client = client
|
||||
self.failure_queue = failure_queue or SyncFailureQueue()
|
||||
|
||||
def sync_file(self, file_path: str | Path, metadata: SyncMetadata, *, mime_type: str | None = None) -> dict[str, Any]:
|
||||
try:
|
||||
return self._sync_file_once(file_path, metadata, mime_type=mime_type)
|
||||
except Exception as exc:
|
||||
self.failure_queue.add(file_path, metadata, mime_type, str(exc))
|
||||
raise
|
||||
|
||||
def _sync_file_once(self, file_path: str | Path, metadata: SyncMetadata, *, mime_type: str | None = None) -> dict[str, Any]:
|
||||
path = Path(file_path)
|
||||
sha256 = file_sha256(path)
|
||||
file_size = path.stat().st_size
|
||||
checked = self.client.check_file(sha256, file_size)
|
||||
file_id = checked.get("file_id") if checked.get("owned_by_user") else None
|
||||
|
||||
if not file_id:
|
||||
init = self.client.init_upload(
|
||||
sha256=sha256,
|
||||
file_name=path.name,
|
||||
file_size=file_size,
|
||||
mime_type=mime_type,
|
||||
resource_type=self._upload_resource_type(metadata, mime_type),
|
||||
)
|
||||
if init.get("exists") and init.get("file_id"):
|
||||
file_id = str(init["file_id"])
|
||||
else:
|
||||
upload_id = str(init["upload_id"])
|
||||
with path.open("rb") as fp:
|
||||
for chunk in iter(lambda: fp.read(self.client.config.chunk_size), b""):
|
||||
self.client.upload_chunk(upload_id, chunk)
|
||||
completed = self.client.complete_upload(upload_id)
|
||||
file_id = str(completed["file_id"])
|
||||
|
||||
return self.client.sync_metadata(str(file_id), metadata)
|
||||
|
||||
@staticmethod
|
||||
def _upload_resource_type(metadata: SyncMetadata, mime_type: str | None) -> str:
|
||||
if metadata.resource_type == "novel":
|
||||
return "novel"
|
||||
if (mime_type or "").lower().startswith("video/") or (metadata.content_type or "").lower() == "video":
|
||||
return "video"
|
||||
return "image"
|
||||
|
||||
def retry_failed(self, *, limit: int = 5) -> dict[str, int]:
|
||||
now = time.time()
|
||||
items = self.failure_queue.load()
|
||||
kept: list[dict[str, Any]] = []
|
||||
retried = 0
|
||||
succeeded = 0
|
||||
failed = 0
|
||||
for item in items:
|
||||
if retried >= limit or float(item.get("next_retry_at") or 0) > now:
|
||||
kept.append(item)
|
||||
continue
|
||||
path = Path(str(item.get("file_path") or ""))
|
||||
if not path.is_file():
|
||||
failed += 1
|
||||
continue
|
||||
try:
|
||||
metadata = SyncMetadata(**dict(item.get("metadata") or {}))
|
||||
self._sync_file_once(path, metadata, mime_type=item.get("mime_type"))
|
||||
succeeded += 1
|
||||
except Exception as exc:
|
||||
attempts = int(item.get("attempts") or 0) + 1
|
||||
item["attempts"] = attempts
|
||||
item["last_error"] = str(exc)
|
||||
item["updated_at"] = now
|
||||
item["next_retry_at"] = now + min(3600, 60 * (2 ** min(attempts, 5)))
|
||||
kept.append(item)
|
||||
failed += 1
|
||||
retried += 1
|
||||
self.failure_queue.save(kept)
|
||||
return {"retried": retried, "succeeded": succeeded, "failed": failed, "remaining": len(kept)}
|
||||
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
Cookie = dict[str, Any]
|
||||
|
||||
|
||||
def parse_cookie_text(text: str) -> list[Cookie]:
|
||||
"""Parse JSON cookie exports or Netscape cookies.txt content."""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
raise ValueError("Cookie 文件为空。")
|
||||
|
||||
try:
|
||||
payload = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
|
||||
if payload is not None:
|
||||
raw_items = payload.get("cookies", payload) if isinstance(payload, dict) else payload
|
||||
if not isinstance(raw_items, list):
|
||||
raise ValueError("Cookie JSON 必须是数组,或包含 cookies 数组。")
|
||||
cookies = [_normalize_json_cookie(item) for item in raw_items if isinstance(item, dict)]
|
||||
else:
|
||||
cookies = _parse_netscape_cookie_text(text)
|
||||
|
||||
cookies = [item for item in cookies if item.get("name") and item.get("value") is not None]
|
||||
if not cookies:
|
||||
raise ValueError("没有解析到有效 Cookie。")
|
||||
return cookies
|
||||
|
||||
|
||||
def filter_cookies_for_domains(cookies: list[Cookie], domains: tuple[str, ...]) -> list[Cookie]:
|
||||
result: list[Cookie] = []
|
||||
for cookie in cookies:
|
||||
domain = str(cookie.get("domain", "")).lower().lstrip(".")
|
||||
if any(domain == item or domain.endswith("." + item) for item in domains):
|
||||
result.append(cookie)
|
||||
return result
|
||||
|
||||
|
||||
def cookie_domains(cookies: list[Cookie]) -> list[str]:
|
||||
domains = sorted({str(item.get("domain", "")).lower().lstrip(".") for item in cookies if item.get("domain")})
|
||||
return [item for item in domains if item]
|
||||
|
||||
|
||||
def has_cookie_domain(cookies: list[Cookie], domains: tuple[str, ...]) -> bool:
|
||||
return bool(filter_cookies_for_domains(cookies, domains))
|
||||
|
||||
|
||||
def cookies_to_header(cookies: list[Cookie], domains: tuple[str, ...] | None = None) -> str:
|
||||
selected = filter_cookies_for_domains(cookies, domains) if domains else cookies
|
||||
pairs: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for cookie in selected:
|
||||
name = str(cookie.get("name", "")).strip()
|
||||
value = str(cookie.get("value", ""))
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
pairs.append(f"{name}={value}")
|
||||
if not pairs:
|
||||
raise ValueError("没有找到匹配站点的 Cookie。")
|
||||
return "; ".join(pairs)
|
||||
|
||||
|
||||
def cookies_to_playwright(cookies: list[Cookie], domains: tuple[str, ...] | None = None) -> list[Cookie]:
|
||||
selected = filter_cookies_for_domains(cookies, domains) if domains else cookies
|
||||
output_by_key: dict[tuple[str, str, str], Cookie] = {}
|
||||
for cookie in selected:
|
||||
name = str(cookie.get("name") or "").strip()
|
||||
domain = str(cookie.get("domain") or "").strip()
|
||||
if not name or not domain:
|
||||
continue
|
||||
item: Cookie = {
|
||||
"name": name,
|
||||
"value": str(cookie.get("value", "")),
|
||||
"domain": domain,
|
||||
"path": str(cookie.get("path") or "/"),
|
||||
"secure": bool(cookie.get("secure", False)),
|
||||
"httpOnly": bool(cookie.get("httpOnly", False)),
|
||||
}
|
||||
expires = cookie.get("expires")
|
||||
if expires is not None:
|
||||
try:
|
||||
expires_value = int(float(expires))
|
||||
if expires_value > 0:
|
||||
item["expires"] = expires_value
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
same_site = str(cookie.get("sameSite", "") or "").strip()
|
||||
if same_site in {"Strict", "Lax", "None"}:
|
||||
item["sameSite"] = same_site
|
||||
key = (str(item["domain"]).lower(), str(item["path"]), name)
|
||||
output_by_key[key] = item
|
||||
output = list(output_by_key.values())
|
||||
if not output:
|
||||
raise ValueError("没有找到匹配站点的 Cookie。")
|
||||
return output
|
||||
|
||||
|
||||
def save_playwright_cookies(cookies: list[Cookie], path: str | Path) -> Path:
|
||||
target = Path(path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json.dumps(cookies, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
def cookie_summary(cookies: list[Cookie]) -> str:
|
||||
domains = sorted({str(item.get("domain", "")).lstrip(".") or "-" for item in cookies})
|
||||
shown = ", ".join(domains[:5])
|
||||
suffix = "..." if len(domains) > 5 else ""
|
||||
return f"{len(cookies)} 个 Cookie;域名:{shown}{suffix}"
|
||||
|
||||
|
||||
def _parse_netscape_cookie_text(text: str) -> list[Cookie]:
|
||||
cookies: list[Cookie] = []
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#HttpOnly_"):
|
||||
http_only = line.startswith("#HttpOnly_")
|
||||
line = line.removeprefix("#HttpOnly_")
|
||||
elif line.startswith("#"):
|
||||
continue
|
||||
else:
|
||||
http_only = False
|
||||
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 7:
|
||||
parts = line.split()
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
|
||||
domain, _include_subdomains, path, secure, expires, name = parts[:6]
|
||||
value = "\t".join(parts[6:])
|
||||
cookies.append(
|
||||
{
|
||||
"domain": domain,
|
||||
"path": path or "/",
|
||||
"secure": secure.upper() == "TRUE",
|
||||
"expires": int(expires) if str(expires).isdigit() else -1,
|
||||
"name": name,
|
||||
"value": value,
|
||||
"httpOnly": http_only,
|
||||
"sameSite": "Lax",
|
||||
}
|
||||
)
|
||||
return cookies
|
||||
|
||||
|
||||
def _normalize_json_cookie(item: Cookie) -> Cookie:
|
||||
expires = item.get("expires", item.get("expirationDate", item.get("expiry")))
|
||||
same_site = _normalize_same_site(item.get("sameSite"))
|
||||
return {
|
||||
"domain": str(item.get("domain") or item.get("host") or ""),
|
||||
"path": str(item.get("path") or "/"),
|
||||
"secure": bool(item.get("secure", False)),
|
||||
"expires": expires,
|
||||
"name": str(item.get("name") or ""),
|
||||
"value": str(item.get("value") or ""),
|
||||
"httpOnly": bool(item.get("httpOnly", item.get("httponly", False))),
|
||||
"sameSite": same_site,
|
||||
}
|
||||
|
||||
|
||||
def _normalize_same_site(value: Any) -> str:
|
||||
text = str(value or "").strip().lower()
|
||||
if text in {"strict", "samesitestrict"}:
|
||||
return "Strict"
|
||||
if text in {"none", "no_restriction", "samesitenone"}:
|
||||
return "None"
|
||||
return "Lax"
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
class DeviceType(Enum):
|
||||
WINDOWS_CHROME = "windows_chrome"
|
||||
WINDOWS_EDGE = "windows_edge"
|
||||
MAC_CHROME = "mac_chrome"
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrowserProfile:
|
||||
"""Stable browser context settings for one logical session.
|
||||
|
||||
This module intentionally avoids Canvas/WebGL spoofing or large fingerprint rotation.
|
||||
The goal is consistency inside a session, not impersonating many different users.
|
||||
"""
|
||||
|
||||
user_agent: str
|
||||
platform: str = "Win32"
|
||||
viewport: dict[str, int] = field(default_factory=lambda: {"width": 1920, "height": 1080})
|
||||
locale: str = "zh-CN"
|
||||
timezone_id: str = "Asia/Shanghai"
|
||||
color_scheme: str = "light"
|
||||
reduced_motion: str = "no-preference"
|
||||
device_scale_factor: float = 1.0
|
||||
is_mobile: bool = False
|
||||
has_touch: bool = False
|
||||
|
||||
def to_playwright_context_options(self) -> dict[str, Any]:
|
||||
return {
|
||||
"user_agent": self.user_agent,
|
||||
"viewport": dict(self.viewport),
|
||||
"locale": self.locale,
|
||||
"timezone_id": self.timezone_id,
|
||||
"color_scheme": self.color_scheme,
|
||||
"reduced_motion": self.reduced_motion,
|
||||
"device_scale_factor": self.device_scale_factor,
|
||||
"is_mobile": self.is_mobile,
|
||||
"has_touch": self.has_touch,
|
||||
}
|
||||
|
||||
def headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"User-Agent": self.user_agent,
|
||||
"Accept-Language": f"{self.locale},zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
|
||||
class BrowserProfileDatabase:
|
||||
@staticmethod
|
||||
def windows_chrome() -> BrowserProfile:
|
||||
return BrowserProfile(
|
||||
user_agent=real_browser_user_agent()
|
||||
or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
platform="Win32",
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def windows_edge() -> BrowserProfile:
|
||||
return BrowserProfile(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
platform="Win32",
|
||||
viewport={"width": 1536, "height": 864},
|
||||
device_scale_factor=1.25,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def mac_chrome() -> BrowserProfile:
|
||||
return BrowserProfile(
|
||||
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
platform="MacIntel",
|
||||
viewport={"width": 1512, "height": 982},
|
||||
device_scale_factor=2.0,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def all_desktop() -> list[BrowserProfile]:
|
||||
return [
|
||||
BrowserProfileDatabase.windows_chrome(),
|
||||
BrowserProfileDatabase.windows_edge(),
|
||||
BrowserProfileDatabase.mac_chrome(),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def by_device_type(device_type: DeviceType) -> BrowserProfile:
|
||||
mapping = {
|
||||
DeviceType.WINDOWS_CHROME: BrowserProfileDatabase.windows_chrome,
|
||||
DeviceType.WINDOWS_EDGE: BrowserProfileDatabase.windows_edge,
|
||||
DeviceType.MAC_CHROME: BrowserProfileDatabase.mac_chrome,
|
||||
}
|
||||
return mapping.get(device_type, BrowserProfileDatabase.windows_chrome)()
|
||||
|
||||
|
||||
class SessionFingerprintManager:
|
||||
"""Session-sticky browser profile manager."""
|
||||
|
||||
def __init__(self, profiles: list[BrowserProfile] | None = None, *, strategy: str = "sticky") -> None:
|
||||
self.profiles = profiles or BrowserProfileDatabase.all_desktop()
|
||||
self.strategy = strategy
|
||||
self._session_map: dict[str, BrowserProfile] = {}
|
||||
self._index = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def get_fingerprint(self, session_id: str = "default") -> BrowserProfile:
|
||||
with self._lock:
|
||||
if self.strategy == "sticky" and session_id in self._session_map:
|
||||
return copy.deepcopy(self._session_map[session_id])
|
||||
if self.strategy == "round_robin":
|
||||
profile = self.profiles[self._index % len(self.profiles)]
|
||||
self._index += 1
|
||||
elif self.strategy == "random":
|
||||
profile = random.choice(self.profiles)
|
||||
else:
|
||||
stable_index = abs(hash(session_id)) % len(self.profiles)
|
||||
profile = self.profiles[stable_index]
|
||||
self._session_map[session_id] = copy.deepcopy(profile)
|
||||
return copy.deepcopy(profile)
|
||||
|
||||
def get_stats(self) -> dict[str, Any]:
|
||||
return {
|
||||
"strategy": self.strategy,
|
||||
"total_profiles": len(self.profiles),
|
||||
"sticky_sessions": len(self._session_map),
|
||||
}
|
||||
|
||||
|
||||
Fingerprint = BrowserProfile
|
||||
FingerprintManager = SessionFingerprintManager
|
||||
FingerprintDatabase = BrowserProfileDatabase
|
||||
|
||||
|
||||
def real_browser_user_agent() -> str:
|
||||
"""Load a user-provided real browser UA, if configured."""
|
||||
env_value = os.environ.get("PIXIV_REAL_USER_AGENT", "").strip()
|
||||
if env_value:
|
||||
return env_value
|
||||
path_value = os.environ.get("PIXIV_REAL_USER_AGENT_FILE", "").strip()
|
||||
if path_value:
|
||||
path = Path(path_value)
|
||||
if path.is_file():
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
return ""
|
||||
|
||||
|
||||
def default_profile(session_id: str = "default") -> BrowserProfile:
|
||||
return SessionFingerprintManager().get_fingerprint(session_id)
|
||||
@@ -0,0 +1,597 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.downloader import download_user_works, fetch_user_profile, fetch_user_work_ids
|
||||
from pixiv_app.core.library_importer import LegacyDownloadImporter
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
|
||||
from pixiv_app.core.paths import app_session_file
|
||||
from pixiv_app.core.thumbnails import DEFAULT_THUMBNAIL_DIR, ThumbnailCache
|
||||
|
||||
|
||||
class LocalGalleryService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db_path: str | Path = DEFAULT_LIBRARY_DB,
|
||||
thumbnail_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
|
||||
) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
library = PixivLibrary(self.db_path)
|
||||
library.close()
|
||||
self.thumbnail_cache = ThumbnailCache(db_path=self.db_path, cache_dir=thumbnail_dir)
|
||||
|
||||
def list_artworks(
|
||||
self,
|
||||
*,
|
||||
query: str = "",
|
||||
search_scope: str = "all",
|
||||
tag: str = "",
|
||||
artist_id: int | None = None,
|
||||
artwork_type: str = "",
|
||||
include_restricted: bool = True,
|
||||
limit: int = 60,
|
||||
offset: int = 0,
|
||||
ensure_thumbnails: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
limit = max(1, min(int(limit), 200))
|
||||
offset = max(0, int(offset))
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
where: list[str] = []
|
||||
params: list[Any] = []
|
||||
joins = """
|
||||
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
|
||||
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
|
||||
LEFT JOIN thumbnail_cache tc ON tc.artwork_id = a.artwork_id AND tc.page_index = 0
|
||||
LEFT JOIN artwork_files af ON af.artwork_id = a.artwork_id AND af.page_index = 0
|
||||
LEFT JOIN artwork_ocr_texts ocr ON ocr.artwork_id = a.artwork_id
|
||||
"""
|
||||
if query:
|
||||
like = f"%{query}%"
|
||||
if search_scope == "ocr":
|
||||
where.append("(ocr.status = 'done' AND ocr.ocr_text LIKE ?)")
|
||||
params.append(like)
|
||||
elif search_scope == "title":
|
||||
where.append("(a.title LIKE ? OR a.caption LIKE ? OR ar.name LIKE ?)")
|
||||
params.extend([like, like, like])
|
||||
elif search_scope == "tags":
|
||||
joins += """
|
||||
LEFT JOIN artwork_tags at_search ON at_search.artwork_id = a.artwork_id
|
||||
LEFT JOIN tags t_search ON t_search.tag_id = at_search.tag_id
|
||||
"""
|
||||
where.append("t_search.name LIKE ?")
|
||||
params.append(like)
|
||||
else:
|
||||
joins += """
|
||||
LEFT JOIN artwork_tags at_search ON at_search.artwork_id = a.artwork_id
|
||||
LEFT JOIN tags t_search ON t_search.tag_id = at_search.tag_id
|
||||
"""
|
||||
where.append(
|
||||
"(a.title LIKE ? OR a.caption LIKE ? OR ar.name LIKE ? OR t_search.name LIKE ? "
|
||||
"OR (ocr.status = 'done' AND ocr.ocr_text LIKE ?))"
|
||||
)
|
||||
params.extend([like, like, like, like, like])
|
||||
if tag:
|
||||
joins += """
|
||||
JOIN artwork_tags at_filter ON at_filter.artwork_id = a.artwork_id
|
||||
JOIN tags t_filter ON t_filter.tag_id = at_filter.tag_id
|
||||
"""
|
||||
where.append("t_filter.name = ?")
|
||||
params.append(tag)
|
||||
if artist_id is not None:
|
||||
where.append("a.artist_id = ?")
|
||||
params.append(int(artist_id))
|
||||
if artwork_type:
|
||||
where.append("a.type = ?")
|
||||
params.append(artwork_type)
|
||||
else:
|
||||
where.append("a.type IN ('illust', 'manga')")
|
||||
if not include_restricted:
|
||||
where.append("a.x_restrict = 0")
|
||||
|
||||
where_sql = "WHERE " + " AND ".join(where) if where else ""
|
||||
total = library.connection.execute(
|
||||
f"SELECT COUNT(DISTINCT a.artwork_id) AS total FROM artworks a {joins} {where_sql}",
|
||||
params,
|
||||
).fetchone()["total"]
|
||||
rows = library.connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
a.artwork_id,
|
||||
a.type,
|
||||
a.title,
|
||||
a.artist_id,
|
||||
ar.name AS artist_name,
|
||||
ar.account AS artist_account,
|
||||
a.page_count,
|
||||
a.bookmark_count,
|
||||
a.like_count,
|
||||
a.view_count,
|
||||
a.x_restrict,
|
||||
a.ai_type,
|
||||
a.thumbnail_url,
|
||||
CASE WHEN fav.artwork_id IS NULL THEN 0 ELSE 1 END AS is_favorite,
|
||||
CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed,
|
||||
COALESCE(tc.thumbnail_path, '') AS thumbnail_path,
|
||||
COALESCE(af.file_path, '') AS primary_file_path,
|
||||
CASE
|
||||
WHEN ocr.status = 'done' AND ocr.ocr_text LIKE ? THEN 'ocr'
|
||||
ELSE ''
|
||||
END AS matched_by,
|
||||
a.updated_at
|
||||
FROM artworks a
|
||||
{joins}
|
||||
LEFT JOIN favorites fav ON fav.artwork_id = a.artwork_id
|
||||
{where_sql}
|
||||
GROUP BY a.artwork_id
|
||||
ORDER BY a.updated_at DESC, a.artwork_id DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
[f"%{query}%" if query else "", *params, limit, offset],
|
||||
).fetchall()
|
||||
|
||||
items = [dict(row) for row in rows]
|
||||
if ensure_thumbnails:
|
||||
self._ensure_list_thumbnails(items)
|
||||
items = self._refresh_thumbnail_paths(library, items)
|
||||
return {"total": int(total), "limit": limit, "offset": offset, "items": items}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def get_artwork(self, artwork_id: int, *, ensure_thumbnails: bool = True) -> dict[str, Any] | None:
|
||||
if ensure_thumbnails:
|
||||
self.thumbnail_cache.ensure_for_artwork(artwork_id)
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT a.*, ar.name AS artist_name, ar.account AS artist_account
|
||||
, CASE WHEN fav.artwork_id IS NULL THEN 0 ELSE 1 END AS is_favorite
|
||||
, CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed
|
||||
FROM artworks a
|
||||
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
|
||||
LEFT JOIN favorites fav ON fav.artwork_id = a.artwork_id
|
||||
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
|
||||
WHERE a.artwork_id = ?
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
files = library.connection.execute(
|
||||
"""
|
||||
SELECT f.*, tc.thumbnail_path, tc.width AS thumbnail_width, tc.height AS thumbnail_height
|
||||
FROM artwork_files f
|
||||
LEFT JOIN thumbnail_cache tc
|
||||
ON tc.artwork_id = f.artwork_id AND tc.page_index = f.page_index
|
||||
WHERE f.artwork_id = ?
|
||||
ORDER BY f.page_index ASC
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchall()
|
||||
tags = library.connection.execute(
|
||||
"""
|
||||
SELECT t.name, t.translated_name
|
||||
FROM tags t
|
||||
JOIN artwork_tags at ON at.tag_id = t.tag_id
|
||||
WHERE at.artwork_id = ?
|
||||
ORDER BY t.name ASC
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchall()
|
||||
history = library.connection.execute(
|
||||
"""
|
||||
SELECT status, downloaded_pages, skipped_pages, failed_pages, message, finished_at
|
||||
FROM download_history
|
||||
WHERE artwork_id = ?
|
||||
ORDER BY finished_at DESC
|
||||
LIMIT 20
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchall()
|
||||
return {
|
||||
"artwork": dict(row),
|
||||
"files": [dict(item) for item in files],
|
||||
"tags": [dict(item) for item in tags],
|
||||
"history": [dict(item) for item in history],
|
||||
}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def list_tags(self, *, query: str = "", limit: int = 100) -> list[dict[str, Any]]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
params: list[Any] = []
|
||||
where = ""
|
||||
if query:
|
||||
where = "WHERE t.name LIKE ? OR t.translated_name LIKE ?"
|
||||
params.extend([f"%{query}%", f"%{query}%"])
|
||||
rows = library.connection.execute(
|
||||
f"""
|
||||
SELECT t.name, t.translated_name, COUNT(at.artwork_id) AS artwork_count
|
||||
FROM tags t
|
||||
LEFT JOIN artwork_tags at ON at.tag_id = t.tag_id
|
||||
{where}
|
||||
GROUP BY t.tag_id
|
||||
ORDER BY artwork_count DESC, t.name ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
[*params, max(1, min(int(limit), 500))],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def list_artists(self, *, query: str = "", limit: int = 100) -> list[dict[str, Any]]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
params: list[Any] = []
|
||||
where = ""
|
||||
if query:
|
||||
where = "WHERE ar.name LIKE ? OR ar.account LIKE ?"
|
||||
params.extend([f"%{query}%", f"%{query}%"])
|
||||
rows = library.connection.execute(
|
||||
f"""
|
||||
SELECT ar.artist_id, ar.name, ar.account, COUNT(a.artwork_id) AS artwork_count
|
||||
FROM artists ar
|
||||
LEFT JOIN artworks a ON a.artist_id = ar.artist_id
|
||||
{where}
|
||||
GROUP BY ar.artist_id
|
||||
ORDER BY
|
||||
CASE WHEN COALESCE(NULLIF(ar.name, ''), NULLIF(ar.account, '')) = '' THEN 1 ELSE 0 END,
|
||||
COALESCE(NULLIF(ar.name, ''), NULLIF(ar.account, ''), CAST(ar.artist_id AS TEXT)) ASC,
|
||||
artwork_count DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
[*params, max(1, min(int(limit), 500))],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def get_artist(self, artist_id: int, *, limit: int = 60, offset: int = 0) -> dict[str, Any] | None:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT ar.*, COUNT(a.artwork_id) AS artwork_count,
|
||||
CASE WHEN fa.enabled = 1 THEN 1 ELSE 0 END AS is_followed,
|
||||
fa.last_synced_at, fa.last_error
|
||||
FROM artists ar
|
||||
LEFT JOIN artworks a ON a.artist_id = ar.artist_id
|
||||
LEFT JOIN followed_artists fa ON fa.artist_id = ar.artist_id
|
||||
WHERE ar.artist_id = ?
|
||||
GROUP BY ar.artist_id
|
||||
""",
|
||||
(artist_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"artist": dict(row),
|
||||
"artworks": self.list_artworks(artist_id=artist_id, limit=limit, offset=offset),
|
||||
}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def list_favorites(self, *, limit: int = 60, offset: int = 0) -> dict[str, Any]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
total = library.connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS total
|
||||
FROM favorites fav
|
||||
JOIN artworks a ON a.artwork_id = fav.artwork_id
|
||||
WHERE a.type IN ('illust', 'manga')
|
||||
"""
|
||||
).fetchone()["total"]
|
||||
rows = library.connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
a.artwork_id, a.type, a.title, a.artist_id, ar.name AS artist_name,
|
||||
ar.account AS artist_account,
|
||||
a.page_count, a.bookmark_count, a.like_count, a.view_count,
|
||||
a.x_restrict, a.ai_type, a.thumbnail_url,
|
||||
1 AS is_favorite,
|
||||
CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed,
|
||||
COALESCE(tc.thumbnail_path, '') AS thumbnail_path,
|
||||
COALESCE(af.file_path, '') AS primary_file_path,
|
||||
fav.created_at AS favorite_at
|
||||
FROM favorites fav
|
||||
JOIN artworks a ON a.artwork_id = fav.artwork_id
|
||||
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
|
||||
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
|
||||
LEFT JOIN thumbnail_cache tc ON tc.artwork_id = a.artwork_id AND tc.page_index = 0
|
||||
LEFT JOIN artwork_files af ON af.artwork_id = a.artwork_id AND af.page_index = 0
|
||||
WHERE a.type IN ('illust', 'manga')
|
||||
ORDER BY fav.created_at DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
(max(1, min(int(limit), 200)), max(0, int(offset))),
|
||||
).fetchall()
|
||||
items = [dict(row) for row in rows]
|
||||
self._ensure_list_thumbnails(items)
|
||||
items = self._refresh_thumbnail_paths(library, items)
|
||||
return {"total": int(total), "limit": limit, "offset": offset, "items": items}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def set_favorite(self, artwork_id: int, favorite: bool, note: str = "") -> dict[str, Any]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
library.set_favorite(artwork_id, favorite, note)
|
||||
return {"artwork_id": artwork_id, "favorite": favorite}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def list_followed_artists(self) -> list[dict[str, Any]]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
rows = library.connection.execute(
|
||||
"""
|
||||
SELECT fa.artist_id, COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), '') AS name,
|
||||
COALESCE(NULLIF(ar.account, ''), '') AS account,
|
||||
fa.enabled, fa.last_synced_at, fa.last_error,
|
||||
COUNT(a.artwork_id) AS artwork_count
|
||||
FROM followed_artists fa
|
||||
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
|
||||
LEFT JOIN artworks a ON a.artist_id = fa.artist_id
|
||||
WHERE fa.enabled = 1
|
||||
GROUP BY fa.artist_id
|
||||
ORDER BY
|
||||
CASE WHEN COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, '')) = '' THEN 1 ELSE 0 END,
|
||||
COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, ''), CAST(fa.artist_id AS TEXT)) ASC,
|
||||
fa.artist_id ASC
|
||||
"""
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def set_followed_artist(self, artist_ref: int | str, followed: bool, name: str = "") -> dict[str, Any]:
|
||||
artist_id = self._resolve_artist_id(artist_ref)
|
||||
clean_name = self._clean_name(name)
|
||||
account = ""
|
||||
if followed and not clean_name:
|
||||
try:
|
||||
profile = fetch_user_profile(
|
||||
artist_id,
|
||||
cookie=str(SessionStore(app_session_file()).load().get("cookie", "")).strip(),
|
||||
)
|
||||
profile_info = profile.get("profile") if isinstance(profile.get("profile"), dict) else {}
|
||||
clean_name = self._clean_name(
|
||||
profile.get("name")
|
||||
or profile.get("userName")
|
||||
or profile.get("displayName")
|
||||
or profile_info.get("name")
|
||||
)
|
||||
account = self._clean_name(profile.get("account") or profile.get("userAccount"))
|
||||
except Exception:
|
||||
clean_name = ""
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
if clean_name or account:
|
||||
library.upsert_artist_profile(artist_id, name=clean_name, account=account)
|
||||
library.set_followed_artist(artist_id, followed, clean_name)
|
||||
return {
|
||||
"artist_id": artist_id,
|
||||
"followed": followed,
|
||||
"name": clean_name,
|
||||
"account": account,
|
||||
"cloud_synced": False,
|
||||
}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def _resolve_artist_id(self, artist_ref: int | str) -> int:
|
||||
text = str(artist_ref).strip()
|
||||
if not text:
|
||||
raise ValueError("Please enter a Pixiv artist ID, profile URL, or synced artist name.")
|
||||
|
||||
if text.lower().startswith("artist_") and text[7:].isdigit():
|
||||
return int(text[7:])
|
||||
|
||||
match = re.search(r"/users/(\d+)", text)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
if text.isdigit():
|
||||
return int(text)
|
||||
|
||||
lookup = text.lstrip("@")
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
rows = [
|
||||
dict(row)
|
||||
for row in library.connection.execute(
|
||||
"""
|
||||
SELECT artist_id, name, account
|
||||
FROM artists
|
||||
WHERE lower(name) = lower(?) OR lower(account) = lower(?)
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN lower(account) = lower(?) THEN 0
|
||||
WHEN lower(name) = lower(?) THEN 1
|
||||
ELSE 2
|
||||
END,
|
||||
artist_id ASC
|
||||
""",
|
||||
(lookup, lookup, lookup, lookup),
|
||||
).fetchall()
|
||||
]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
if len(rows) == 1:
|
||||
return int(rows[0]["artist_id"])
|
||||
if not rows:
|
||||
raise ValueError("Could not resolve Pixiv artist ID from that name. Enter an ID/profile URL or sync the artist first.")
|
||||
choices = ", ".join(str(int(row["artist_id"])) for row in rows[:5])
|
||||
raise ValueError(f"Artist name matched multiple Pixiv users; use an ID instead: {choices}")
|
||||
|
||||
@staticmethod
|
||||
def _clean_name(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
return "" if text.lower() in {"none", "null", "undefined"} else text
|
||||
|
||||
def scan_legacy_downloads(self, *, root: str | Path = "Sakura_Downloads") -> dict[str, Any]:
|
||||
summary = LegacyDownloadImporter(db_path=self.db_path).scan(root)
|
||||
self.thumbnail_cache.build_missing(limit=500)
|
||||
return summary.__dict__
|
||||
|
||||
def sync_followed_artists(
|
||||
self,
|
||||
*,
|
||||
cookie: str = "",
|
||||
max_new_per_artist: int = 20,
|
||||
download: bool = True,
|
||||
save_root: str | Path = "Sakura_Downloads",
|
||||
) -> dict[str, Any]:
|
||||
if not cookie:
|
||||
cookie = str(SessionStore(app_session_file()).load().get("cookie", "")).strip()
|
||||
if not cookie:
|
||||
raise ValueError("No Pixiv cookie available. Save a session in the GUI first or pass cookie explicitly.")
|
||||
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
followed = library.connection.execute(
|
||||
"""
|
||||
SELECT fa.artist_id,
|
||||
COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, ''), '') AS name,
|
||||
COALESCE(NULLIF(ar.account, ''), '') AS account
|
||||
FROM followed_artists fa
|
||||
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
|
||||
WHERE fa.enabled = 1
|
||||
ORDER BY fa.artist_id
|
||||
"""
|
||||
).fetchall()
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for row in followed:
|
||||
artist_id = int(row["artist_id"])
|
||||
artist_name = self._clean_name(row["name"])
|
||||
artist_account = self._clean_name(row["account"])
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
existing = {
|
||||
int(item["artwork_id"])
|
||||
for item in library.connection.execute(
|
||||
"SELECT artwork_id FROM artworks WHERE artist_id = ? AND type IN ('illust', 'manga')",
|
||||
(artist_id,),
|
||||
).fetchall()
|
||||
}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
try:
|
||||
_, work_ids = fetch_user_work_ids(user_id=artist_id, cookie=cookie)
|
||||
new_ids = [work_id for work_id in work_ids if work_id not in existing][: max(0, int(max_new_per_artist))]
|
||||
downloaded = 0
|
||||
if download and new_ids:
|
||||
_, download_results = download_user_works(
|
||||
user_id=artist_id,
|
||||
cookie=cookie,
|
||||
max_workers=3,
|
||||
request_delay=0.2,
|
||||
save_root=save_root,
|
||||
work_ids=new_ids,
|
||||
)
|
||||
downloaded = len([item for item in download_results if item.ok])
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
library.mark_artist_synced(artist_id)
|
||||
finally:
|
||||
library.close()
|
||||
cloud_synced = False
|
||||
cloud_alerts = 0
|
||||
results.append(
|
||||
{
|
||||
"artist_id": artist_id,
|
||||
"remote_count": len(work_ids),
|
||||
"new_count": len(new_ids),
|
||||
"downloaded": downloaded,
|
||||
"cloud_synced": cloud_synced,
|
||||
"alerts": cloud_alerts,
|
||||
"error": "",
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
library.mark_artist_synced(artist_id, error=str(exc))
|
||||
finally:
|
||||
library.close()
|
||||
results.append({"artist_id": artist_id, "remote_count": 0, "new_count": 0, "downloaded": 0, "alerts": 0, "error": str(exc)})
|
||||
return {"artists": results}
|
||||
|
||||
def build_missing_thumbnails(self, *, limit: int = 200) -> list[dict[str, Any]]:
|
||||
return [item.__dict__ for item in self.thumbnail_cache.build_missing(limit=limit)]
|
||||
|
||||
def get_thumbnail_path(self, artwork_id: int, *, page_index: int = 0) -> Path | None:
|
||||
self.thumbnail_cache.ensure_for_artwork(artwork_id)
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT thumbnail_path
|
||||
FROM thumbnail_cache
|
||||
WHERE artwork_id = ? AND page_index = ?
|
||||
""",
|
||||
(artwork_id, page_index),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
path = Path(str(row["thumbnail_path"]))
|
||||
return path if path.exists() else None
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def get_source_file_path(self, artwork_id: int, *, page_index: int = 0) -> Path | None:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT file_path
|
||||
FROM artwork_files
|
||||
WHERE artwork_id = ? AND page_index = ?
|
||||
""",
|
||||
(artwork_id, page_index),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
path = Path(str(row["file_path"]))
|
||||
return path if path.exists() else None
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def _ensure_list_thumbnails(self, items: list[dict[str, Any]]) -> None:
|
||||
for item in items:
|
||||
if item.get("thumbnail_path"):
|
||||
continue
|
||||
if not item.get("primary_file_path"):
|
||||
continue
|
||||
self.thumbnail_cache.ensure_for_artwork(int(item["artwork_id"]))
|
||||
|
||||
def _refresh_thumbnail_paths(self, library: PixivLibrary, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
artwork_ids = [int(item["artwork_id"]) for item in items]
|
||||
if not artwork_ids:
|
||||
return items
|
||||
placeholders = ",".join("?" for _ in artwork_ids)
|
||||
rows = library.connection.execute(
|
||||
f"""
|
||||
SELECT artwork_id, thumbnail_path
|
||||
FROM thumbnail_cache
|
||||
WHERE page_index = 0 AND artwork_id IN ({placeholders})
|
||||
""",
|
||||
artwork_ids,
|
||||
).fetchall()
|
||||
thumbs = {int(row["artwork_id"]): str(row["thumbnail_path"]) for row in rows}
|
||||
for item in items:
|
||||
item["thumbnail_path"] = thumbs.get(int(item["artwork_id"]), item.get("thumbnail_path", ""))
|
||||
return items
|
||||
@@ -0,0 +1,703 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pixiv_app.core.paths import app_library_file
|
||||
|
||||
DEFAULT_LIBRARY_DB = app_library_file()
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadFileRecord:
|
||||
page_index: int
|
||||
file_path: str
|
||||
source_url: str
|
||||
status: str
|
||||
size_bytes: int = 0
|
||||
content_hash: str = ""
|
||||
|
||||
|
||||
class PixivLibrary:
|
||||
def __init__(self, db_path: str | Path = DEFAULT_LIBRARY_DB) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True) if self.db_path.parent != Path(".") else None
|
||||
# Allow sharing one connection across threads when guarded by a lock (e.g. TaskQueue).
|
||||
self.connection = sqlite3.connect(self.db_path, timeout=30, check_same_thread=False)
|
||||
self.connection.row_factory = sqlite3.Row
|
||||
self.connection.execute("PRAGMA journal_mode=WAL")
|
||||
self.connection.execute("PRAGMA foreign_keys=ON")
|
||||
self.setup()
|
||||
|
||||
def close(self) -> None:
|
||||
self.connection.close()
|
||||
|
||||
def setup(self) -> None:
|
||||
with self.connection:
|
||||
self.connection.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS artists (
|
||||
artist_id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
account TEXT,
|
||||
avatar_url TEXT,
|
||||
metadata_json TEXT NOT NULL DEFAULT '{}',
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artworks (
|
||||
artwork_id INTEGER PRIMARY KEY,
|
||||
artist_id INTEGER,
|
||||
type TEXT NOT NULL DEFAULT 'illust',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
caption TEXT NOT NULL DEFAULT '',
|
||||
page_count INTEGER NOT NULL DEFAULT 0,
|
||||
bookmark_count INTEGER NOT NULL DEFAULT 0,
|
||||
like_count INTEGER NOT NULL DEFAULT 0,
|
||||
view_count INTEGER NOT NULL DEFAULT 0,
|
||||
x_restrict INTEGER NOT NULL DEFAULT 0,
|
||||
ai_type INTEGER NOT NULL DEFAULT 0,
|
||||
thumbnail_url TEXT,
|
||||
metadata_json TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (artist_id) REFERENCES artists(artist_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
tag_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
translated_name TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artwork_tags (
|
||||
artwork_id INTEGER NOT NULL,
|
||||
tag_id INTEGER NOT NULL,
|
||||
PRIMARY KEY (artwork_id, tag_id),
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (tag_id) REFERENCES tags(tag_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artwork_files (
|
||||
artwork_id INTEGER NOT NULL,
|
||||
page_index INTEGER NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
source_url TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'unknown',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (artwork_id, page_index),
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
artwork_id INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
downloaded_pages INTEGER NOT NULL DEFAULT 0,
|
||||
skipped_pages INTEGER NOT NULL DEFAULT 0,
|
||||
failed_pages INTEGER NOT NULL DEFAULT 0,
|
||||
message TEXT NOT NULL DEFAULT '',
|
||||
started_at TEXT,
|
||||
finished_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value_json TEXT NOT NULL DEFAULT '{}',
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS thumbnail_cache (
|
||||
artwork_id INTEGER NOT NULL,
|
||||
page_index INTEGER NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
thumbnail_path TEXT NOT NULL,
|
||||
width INTEGER NOT NULL DEFAULT 0,
|
||||
height INTEGER NOT NULL DEFAULT 0,
|
||||
source_mtime REAL NOT NULL DEFAULT 0,
|
||||
source_size INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (artwork_id, page_index),
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artwork_ocr_texts (
|
||||
artwork_id INTEGER PRIMARY KEY,
|
||||
language TEXT NOT NULL DEFAULT 'mixed',
|
||||
ocr_text TEXT NOT NULL DEFAULT '',
|
||||
confidence REAL,
|
||||
engine TEXT NOT NULL DEFAULT 'paddleocr',
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS favorites (
|
||||
artwork_id INTEGER PRIMARY KEY,
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS followed_artists (
|
||||
artist_id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
last_synced_at TEXT,
|
||||
last_error TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artworks_artist_id ON artworks(artist_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_artworks_type ON artworks(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_download_history_artwork_id ON download_history(artwork_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_artwork_files_path ON artwork_files(file_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_thumbnail_cache_path ON thumbnail_cache(thumbnail_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_artwork_ocr_texts_status ON artwork_ocr_texts(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_artwork_ocr_texts_text ON artwork_ocr_texts(ocr_text);
|
||||
CREATE INDEX IF NOT EXISTS idx_favorites_created_at ON favorites(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_followed_artists_enabled ON followed_artists(enabled);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_tasks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
fingerprint TEXT NOT NULL UNIQUE,
|
||||
target_type TEXT NOT NULL,
|
||||
target_id INTEGER,
|
||||
page_index INTEGER,
|
||||
task_kind TEXT NOT NULL,
|
||||
priority INTEGER NOT NULL DEFAULT 2,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
leased_until REAL,
|
||||
worker_id TEXT,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_tasks_status_pri_id
|
||||
ON download_tasks(status, priority DESC, id ASC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artist_sync_watermark (
|
||||
artist_id INTEGER PRIMARY KEY,
|
||||
last_synced_illust_id INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
"""
|
||||
)
|
||||
self._apply_schema_migrations()
|
||||
|
||||
def _apply_schema_migrations(self) -> None:
|
||||
"""Incremental ALTER for existing databases created before new columns/tables."""
|
||||
with self.connection:
|
||||
cols = {row[1] for row in self.connection.execute("PRAGMA table_info(artwork_files)")}
|
||||
if "content_hash" not in cols:
|
||||
self.connection.execute(
|
||||
"ALTER TABLE artwork_files ADD COLUMN content_hash TEXT NOT NULL DEFAULT ''"
|
||||
)
|
||||
# Drop legacy placeholder names like artist_12345 so UI can fall back cleanly.
|
||||
self.connection.execute(
|
||||
"UPDATE artists SET name = '' WHERE name = 'artist_' || artist_id"
|
||||
)
|
||||
self.connection.execute(
|
||||
"UPDATE followed_artists SET name = '' WHERE name = 'artist_' || artist_id"
|
||||
)
|
||||
self.connection.execute(
|
||||
"UPDATE artists SET name = '' WHERE lower(name) IN ('none', 'null', 'undefined')"
|
||||
)
|
||||
self.connection.execute(
|
||||
"UPDATE followed_artists SET name = '' WHERE lower(name) IN ('none', 'null', 'undefined')"
|
||||
)
|
||||
rows = self.connection.execute(
|
||||
"SELECT artwork_id, metadata_json FROM artworks WHERE artist_id IS NULL OR artist_id = 0"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
try:
|
||||
metadata = json.loads(str(row["metadata_json"] or "{}"))
|
||||
except Exception:
|
||||
continue
|
||||
artist_id = _as_int(metadata.get("userId") or metadata.get("author_id") or metadata.get("artistId"))
|
||||
if artist_id is None:
|
||||
continue
|
||||
self.ensure_artist_stub(artist_id, name=str(metadata.get("userName") or metadata.get("artistName") or ""))
|
||||
self.connection.execute(
|
||||
"UPDATE artworks SET artist_id = ?, updated_at = CURRENT_TIMESTAMP WHERE artwork_id = ?",
|
||||
(artist_id, int(row["artwork_id"])),
|
||||
)
|
||||
|
||||
def upsert_artwork(self, detail: dict[str, Any], *, artwork_type: str = "illust") -> None:
|
||||
source_id = _as_int(detail.get("illustId") or detail.get("id") or detail.get("novelId"))
|
||||
if source_id is None:
|
||||
return
|
||||
source_platform = str(detail.get("source_platform") or detail.get("source") or "")
|
||||
artwork_id = library_artwork_id(source_id, artwork_type, source_platform=source_platform)
|
||||
|
||||
artist_id = _as_int(detail.get("userId"))
|
||||
if artist_id is not None:
|
||||
self._upsert_artist(detail, artist_id)
|
||||
|
||||
tags = _extract_tags(detail)
|
||||
urls = detail.get("urls") if isinstance(detail.get("urls"), dict) else {}
|
||||
thumbnail_url = (
|
||||
detail.get("url")
|
||||
or detail.get("coverUrl")
|
||||
or urls.get("small")
|
||||
or urls.get("thumb")
|
||||
or urls.get("regular")
|
||||
or urls.get("original")
|
||||
)
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artworks (
|
||||
artwork_id, artist_id, type, title, caption, page_count,
|
||||
bookmark_count, like_count, view_count, x_restrict, ai_type,
|
||||
thumbnail_url, metadata_json, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id) DO UPDATE SET
|
||||
artist_id = excluded.artist_id,
|
||||
type = excluded.type,
|
||||
title = excluded.title,
|
||||
caption = excluded.caption,
|
||||
page_count = excluded.page_count,
|
||||
bookmark_count = excluded.bookmark_count,
|
||||
like_count = excluded.like_count,
|
||||
view_count = excluded.view_count,
|
||||
x_restrict = excluded.x_restrict,
|
||||
ai_type = excluded.ai_type,
|
||||
thumbnail_url = excluded.thumbnail_url,
|
||||
metadata_json = excluded.metadata_json,
|
||||
created_at = COALESCE(excluded.created_at, artworks.created_at),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
artwork_id,
|
||||
artist_id,
|
||||
artwork_type,
|
||||
str(detail.get("title") or ""),
|
||||
str(detail.get("description") or detail.get("caption") or ""),
|
||||
_as_int(detail.get("pageCount")) or 0,
|
||||
_as_int(detail.get("bookmarkCount")) or 0,
|
||||
_as_int(detail.get("likeCount")) or 0,
|
||||
_as_int(detail.get("viewCount")) or 0,
|
||||
_as_int(detail.get("xRestrict")) or 0,
|
||||
_as_int(detail.get("aiType")) or 0,
|
||||
str(thumbnail_url) if thumbnail_url else None,
|
||||
json.dumps({**detail, "sourceId": source_id}, ensure_ascii=False),
|
||||
str(detail.get("createDate") or detail.get("uploadDate") or "") or None,
|
||||
),
|
||||
)
|
||||
self.connection.execute("DELETE FROM artwork_tags WHERE artwork_id = ?", (artwork_id,))
|
||||
for name, translated_name in tags:
|
||||
tag_id = self._upsert_tag(name, translated_name)
|
||||
self.connection.execute(
|
||||
"INSERT OR IGNORE INTO artwork_tags (artwork_id, tag_id) VALUES (?, ?)",
|
||||
(artwork_id, tag_id),
|
||||
)
|
||||
|
||||
def ensure_artwork_stub(self, artwork_id: int, *, artwork_type: str = "illust", title: str = "", source_platform: str = "") -> int:
|
||||
db_artwork_id = library_artwork_id(artwork_id, artwork_type, source_platform=source_platform)
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO artworks (artwork_id, type, title, updated_at)
|
||||
VALUES (?, ?, ?, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
(db_artwork_id, artwork_type, title),
|
||||
)
|
||||
return db_artwork_id
|
||||
|
||||
def ensure_artist_stub(self, artist_id: int, *, name: str = "") -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO artists (artist_id, name, updated_at)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
(artist_id, name or ""),
|
||||
)
|
||||
|
||||
def upsert_artist_profile(self, artist_id: int, *, name: str = "", account: str = "", avatar_url: str = "") -> None:
|
||||
clean_name = _clean_optional_text(name)
|
||||
clean_account = _clean_optional_text(account)
|
||||
clean_avatar = _clean_optional_text(avatar_url)
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artists (artist_id, name, account, avatar_url, metadata_json, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
name = COALESCE(NULLIF(excluded.name, ''), artists.name),
|
||||
account = COALESCE(NULLIF(excluded.account, ''), artists.account),
|
||||
avatar_url = COALESCE(NULLIF(excluded.avatar_url, ''), artists.avatar_url),
|
||||
metadata_json = CASE
|
||||
WHEN excluded.metadata_json != '{}' THEN excluded.metadata_json
|
||||
ELSE artists.metadata_json
|
||||
END,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
int(artist_id),
|
||||
clean_name,
|
||||
clean_account,
|
||||
clean_avatar,
|
||||
json.dumps(
|
||||
{
|
||||
"userId": int(artist_id),
|
||||
"userName": clean_name,
|
||||
"userAccount": clean_account,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def set_artwork_artist(self, artwork_id: int, artist_id: int) -> None:
|
||||
with self.connection:
|
||||
self.ensure_artist_stub(artist_id)
|
||||
self.connection.execute(
|
||||
"""
|
||||
UPDATE artworks
|
||||
SET artist_id = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE artwork_id = ?
|
||||
""",
|
||||
(artist_id, artwork_id),
|
||||
)
|
||||
|
||||
def record_files(self, artwork_id: int, files: list[DownloadFileRecord]) -> None:
|
||||
with self.connection:
|
||||
for item in files:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artwork_files (
|
||||
artwork_id, page_index, file_path, source_url, status, size_bytes,
|
||||
content_hash, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id, page_index) DO UPDATE SET
|
||||
file_path = excluded.file_path,
|
||||
source_url = excluded.source_url,
|
||||
status = excluded.status,
|
||||
size_bytes = excluded.size_bytes,
|
||||
content_hash = CASE
|
||||
WHEN excluded.content_hash != '' THEN excluded.content_hash
|
||||
ELSE artwork_files.content_hash
|
||||
END,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
artwork_id,
|
||||
item.page_index,
|
||||
item.file_path,
|
||||
item.source_url,
|
||||
item.status,
|
||||
item.size_bytes,
|
||||
item.content_hash or "",
|
||||
),
|
||||
)
|
||||
|
||||
def record_download_history(
|
||||
self,
|
||||
*,
|
||||
artwork_id: int,
|
||||
status: str,
|
||||
downloaded_pages: int,
|
||||
skipped_pages: int,
|
||||
failed_pages: int,
|
||||
message: str,
|
||||
started_at: str | None = None,
|
||||
) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO download_history (
|
||||
artwork_id, status, downloaded_pages, skipped_pages, failed_pages, message, started_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(artwork_id, status, downloaded_pages, skipped_pages, failed_pages, message, started_at),
|
||||
)
|
||||
|
||||
def update_sync_state(self, key: str, value: dict[str, Any]) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO sync_state (key, value_json, updated_at)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
value_json = excluded.value_json,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(key, json.dumps(value, ensure_ascii=False)),
|
||||
)
|
||||
|
||||
def upsert_thumbnail(
|
||||
self,
|
||||
*,
|
||||
artwork_id: int,
|
||||
page_index: int,
|
||||
source_path: str,
|
||||
thumbnail_path: str,
|
||||
width: int,
|
||||
height: int,
|
||||
source_mtime: float,
|
||||
source_size: int,
|
||||
) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO thumbnail_cache (
|
||||
artwork_id, page_index, source_path, thumbnail_path, width, height,
|
||||
source_mtime, source_size, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id, page_index) DO UPDATE SET
|
||||
source_path = excluded.source_path,
|
||||
thumbnail_path = excluded.thumbnail_path,
|
||||
width = excluded.width,
|
||||
height = excluded.height,
|
||||
source_mtime = excluded.source_mtime,
|
||||
source_size = excluded.source_size,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(artwork_id, page_index, source_path, thumbnail_path, width, height, source_mtime, source_size),
|
||||
)
|
||||
|
||||
def upsert_ocr_text(
|
||||
self,
|
||||
artwork_id: int,
|
||||
*,
|
||||
text: str,
|
||||
language: str = "mixed",
|
||||
confidence: float | None = None,
|
||||
engine: str = "paddleocr",
|
||||
status: str = "done",
|
||||
attempts: int = 1,
|
||||
) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artwork_ocr_texts (
|
||||
artwork_id, language, ocr_text, confidence, engine, status, attempts, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id) DO UPDATE SET
|
||||
language = excluded.language,
|
||||
ocr_text = excluded.ocr_text,
|
||||
confidence = excluded.confidence,
|
||||
engine = excluded.engine,
|
||||
status = excluded.status,
|
||||
attempts = excluded.attempts,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(int(artwork_id), language, text, confidence, engine, status, attempts),
|
||||
)
|
||||
|
||||
def get_ocr_text(self, artwork_id: int) -> sqlite3.Row | None:
|
||||
return self.connection.execute(
|
||||
"SELECT * FROM artwork_ocr_texts WHERE artwork_id = ?",
|
||||
(int(artwork_id),),
|
||||
).fetchone()
|
||||
|
||||
def set_favorite(self, artwork_id: int, favorite: bool, note: str = "") -> None:
|
||||
with self.connection:
|
||||
if favorite:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO favorites (artwork_id, note, created_at)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id) DO UPDATE SET note = excluded.note
|
||||
""",
|
||||
(artwork_id, note),
|
||||
)
|
||||
else:
|
||||
self.connection.execute("DELETE FROM favorites WHERE artwork_id = ?", (artwork_id,))
|
||||
|
||||
def set_followed_artist(self, artist_id: int, followed: bool, name: str = "") -> None:
|
||||
with self.connection:
|
||||
self.ensure_artist_stub(artist_id, name=name)
|
||||
if followed:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO followed_artists (artist_id, name, enabled, created_at)
|
||||
VALUES (?, ?, 1, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
name = COALESCE(NULLIF(excluded.name, ''), followed_artists.name),
|
||||
enabled = 1
|
||||
""",
|
||||
(artist_id, name),
|
||||
)
|
||||
else:
|
||||
self.connection.execute(
|
||||
"UPDATE followed_artists SET enabled = 0 WHERE artist_id = ?",
|
||||
(artist_id,),
|
||||
)
|
||||
|
||||
def upsert_origin_followed_artist(
|
||||
self,
|
||||
artist_id: int,
|
||||
*,
|
||||
name: str = "",
|
||||
account: str = "",
|
||||
avatar_url: str = "",
|
||||
) -> str:
|
||||
clean_name = _clean_optional_text(name)
|
||||
before = self.connection.execute(
|
||||
"""
|
||||
SELECT fa.enabled, fa.name AS follow_name, ar.name AS artist_name,
|
||||
ar.account, ar.avatar_url
|
||||
FROM followed_artists fa
|
||||
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
|
||||
WHERE fa.artist_id = ?
|
||||
""",
|
||||
(int(artist_id),),
|
||||
).fetchone()
|
||||
status = "created"
|
||||
if before is not None:
|
||||
changed = (
|
||||
not bool(before["enabled"])
|
||||
or (clean_name and clean_name != str(before["follow_name"] or before["artist_name"] or ""))
|
||||
or (account and account != str(before["account"] or ""))
|
||||
or (avatar_url and avatar_url != str(before["avatar_url"] or ""))
|
||||
)
|
||||
status = "updated" if changed else "skipped"
|
||||
with self.connection:
|
||||
self.upsert_artist_profile(
|
||||
int(artist_id),
|
||||
name=clean_name,
|
||||
account=account,
|
||||
avatar_url=avatar_url,
|
||||
)
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO followed_artists (artist_id, name, enabled, last_synced_at, last_error, created_at)
|
||||
VALUES (?, ?, 1, CURRENT_TIMESTAMP, '', CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
name = COALESCE(NULLIF(excluded.name, ''), followed_artists.name),
|
||||
enabled = 1,
|
||||
last_synced_at = CURRENT_TIMESTAMP,
|
||||
last_error = ''
|
||||
""",
|
||||
(int(artist_id), clean_name),
|
||||
)
|
||||
return status
|
||||
|
||||
def mark_artist_synced(self, artist_id: int, *, error: str = "") -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO followed_artists (artist_id, enabled, last_synced_at, last_error)
|
||||
VALUES (?, 1, CURRENT_TIMESTAMP, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
last_synced_at = CURRENT_TIMESTAMP,
|
||||
last_error = excluded.last_error
|
||||
""",
|
||||
(artist_id, error),
|
||||
)
|
||||
|
||||
def _upsert_artist(self, detail: dict[str, Any], artist_id: int) -> None:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artists (artist_id, name, account, avatar_url, metadata_json, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
account = COALESCE(excluded.account, artists.account),
|
||||
avatar_url = COALESCE(excluded.avatar_url, artists.avatar_url),
|
||||
metadata_json = excluded.metadata_json,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
artist_id,
|
||||
str(detail.get("userName") or detail.get("artistName") or ""),
|
||||
str(detail.get("userAccount") or "") or None,
|
||||
str(detail.get("profileImageUrl") or detail.get("userImage") or "") or None,
|
||||
json.dumps(
|
||||
{
|
||||
"userId": artist_id,
|
||||
"userName": detail.get("userName"),
|
||||
"userAccount": detail.get("userAccount"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def _upsert_tag(self, name: str, translated_name: str | None) -> int:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO tags (name, translated_name)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
translated_name = COALESCE(excluded.translated_name, tags.translated_name)
|
||||
""",
|
||||
(name, translated_name),
|
||||
)
|
||||
row = self.connection.execute("SELECT tag_id FROM tags WHERE name = ?", (name,)).fetchone()
|
||||
return int(row["tag_id"])
|
||||
|
||||
|
||||
def _as_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _clean_optional_text(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
return "" if text.lower() in {"none", "null", "undefined"} else text
|
||||
|
||||
|
||||
def library_artwork_id(source_id: int, artwork_type: str = "illust", source_platform: str = "") -> int:
|
||||
"""Map a source-native ID into the unified library artwork_id namespace.
|
||||
|
||||
Pixiv novels use negative source_id; AO3 works use a negative offset
|
||||
range to avoid collisions with Pixiv novel IDs.
|
||||
"""
|
||||
if artwork_type == "novel":
|
||||
if source_platform == "ao3":
|
||||
# AO3: use negative offset range -(200M + work_id) to stay clear
|
||||
# of Pixiv novel IDs (currently -1 .. -199_999_999).
|
||||
AO3_OFFSET = 200_000_000
|
||||
return -(AO3_OFFSET + abs(int(source_id)))
|
||||
return -abs(int(source_id))
|
||||
return int(source_id)
|
||||
|
||||
|
||||
def _extract_tags(detail: dict[str, Any]) -> list[tuple[str, str | None]]:
|
||||
raw_tags = detail.get("tags")
|
||||
if isinstance(raw_tags, dict):
|
||||
raw_tags = raw_tags.get("tags")
|
||||
if not isinstance(raw_tags, list):
|
||||
return []
|
||||
|
||||
result: list[tuple[str, str | None]] = []
|
||||
for item in raw_tags:
|
||||
if isinstance(item, str):
|
||||
result.append((item, None))
|
||||
continue
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
name = item.get("tag") or item.get("name")
|
||||
if not name:
|
||||
continue
|
||||
translation = item.get("translation")
|
||||
translated_name = None
|
||||
if isinstance(translation, dict):
|
||||
translated_name = translation.get("en") or translation.get("zh") or translation.get("romaji")
|
||||
elif isinstance(translation, str):
|
||||
translated_name = translation
|
||||
result.append((str(name), str(translated_name) if translated_name else None))
|
||||
return result
|
||||
@@ -0,0 +1,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, DownloadFileRecord, PixivLibrary
|
||||
|
||||
|
||||
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
|
||||
ILLUST_FILE_PATTERN = re.compile(r"(?P<artwork_id>\d+)_p(?P<page_index>\d+)", re.IGNORECASE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportSummary:
|
||||
scanned_files: int = 0
|
||||
imported_files: int = 0
|
||||
skipped_files: int = 0
|
||||
imported_artworks: int = 0
|
||||
|
||||
|
||||
class LegacyDownloadImporter:
|
||||
def __init__(self, *, db_path: str | Path = DEFAULT_LIBRARY_DB) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
|
||||
def scan(self, root: str | Path = "Sakura_Downloads") -> ImportSummary:
|
||||
root_path = Path(root)
|
||||
summary = ImportSummary()
|
||||
if not root_path.exists():
|
||||
return summary
|
||||
|
||||
library = PixivLibrary(self.db_path)
|
||||
imported_artworks: set[int] = set()
|
||||
try:
|
||||
for file_path in root_path.rglob("*"):
|
||||
if not file_path.is_file() or file_path.suffix.lower() not in IMAGE_SUFFIXES:
|
||||
continue
|
||||
summary.scanned_files += 1
|
||||
match = ILLUST_FILE_PATTERN.search(file_path.stem)
|
||||
if match is None:
|
||||
summary.skipped_files += 1
|
||||
continue
|
||||
artwork_id = int(match.group("artwork_id"))
|
||||
page_index = int(match.group("page_index"))
|
||||
library.ensure_artwork_stub(artwork_id, artwork_type="illust", title=str(artwork_id))
|
||||
artist_id = self._infer_artist_id(root_path, file_path)
|
||||
if artist_id is not None:
|
||||
library.set_artwork_artist(artwork_id, artist_id)
|
||||
library.record_files(
|
||||
artwork_id,
|
||||
[
|
||||
DownloadFileRecord(
|
||||
page_index=page_index,
|
||||
file_path=str(file_path.resolve()),
|
||||
source_url="",
|
||||
status="imported",
|
||||
size_bytes=file_path.stat().st_size,
|
||||
)
|
||||
],
|
||||
)
|
||||
imported_artworks.add(artwork_id)
|
||||
summary.imported_files += 1
|
||||
summary.imported_artworks = len(imported_artworks)
|
||||
return summary
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def _infer_artist_id(self, root_path: Path, file_path: Path) -> int | None:
|
||||
try:
|
||||
relative = file_path.relative_to(root_path)
|
||||
except ValueError:
|
||||
return None
|
||||
if not relative.parts:
|
||||
return None
|
||||
first = relative.parts[0]
|
||||
return int(first) if first.isdigit() else None
|
||||
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
|
||||
|
||||
|
||||
OcrExtractor = Callable[[Path], tuple[str, float | None, str]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class LocalOcrResult:
|
||||
artwork_id: int
|
||||
status: str
|
||||
text_length: int = 0
|
||||
confidence: float | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class LocalOcrIndexer:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db_path: str | Path = DEFAULT_LIBRARY_DB,
|
||||
enabled: bool | None = None,
|
||||
max_dimension: int | None = None,
|
||||
language: str | None = None,
|
||||
max_concurrency: int | None = None,
|
||||
) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.enabled = enabled if enabled is not None else _env_bool("SAKURA_LOCAL_OCR_ENABLED", False)
|
||||
self.max_dimension = max_dimension or int(os.getenv("SAKURA_LOCAL_OCR_MAX_DIMENSION", "1800"))
|
||||
self.language = language or os.getenv("SAKURA_LOCAL_OCR_LANGUAGE", "ch")
|
||||
self._semaphore = threading.BoundedSemaphore(
|
||||
max_concurrency or int(os.getenv("SAKURA_LOCAL_OCR_MAX_CONCURRENCY", "1"))
|
||||
)
|
||||
|
||||
def index_artwork(self, artwork_id: int, *, extractor: OcrExtractor | None = None) -> LocalOcrResult:
|
||||
if not self.enabled and extractor is None:
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="disabled")
|
||||
if not self._semaphore.acquire(blocking=False):
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="busy")
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
existing = library.get_ocr_text(artwork_id)
|
||||
if existing is not None and str(existing["status"]) == "done" and str(existing["ocr_text"]).strip():
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(str(existing["ocr_text"])))
|
||||
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT file_path
|
||||
FROM artwork_files
|
||||
WHERE artwork_id = ?
|
||||
ORDER BY page_index ASC
|
||||
LIMIT 1
|
||||
""",
|
||||
(int(artwork_id),),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="file_not_found")
|
||||
path = Path(str(row["file_path"]))
|
||||
if path.suffix.lower() not in {".jpg", ".jpeg", ".png", ".webp"}:
|
||||
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="unsupported_file_type")
|
||||
|
||||
text, confidence, language = (extractor or self._paddle_ocr_extract)(path)
|
||||
library.upsert_ocr_text(
|
||||
artwork_id,
|
||||
text=text.strip(),
|
||||
confidence=confidence,
|
||||
language=language or self.language,
|
||||
status="done",
|
||||
attempts=1,
|
||||
)
|
||||
logging.info("Local OCR done artwork=%s text_len=%s confidence=%s", artwork_id, len(text.strip()), confidence)
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(text.strip()), confidence=confidence)
|
||||
except Exception as exc:
|
||||
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
|
||||
logging.warning("Local OCR failed artwork=%s error=%s", artwork_id, exc.__class__.__name__)
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="failed", error=exc.__class__.__name__)
|
||||
finally:
|
||||
library.close()
|
||||
self._semaphore.release()
|
||||
|
||||
def _paddle_ocr_extract(self, path: Path) -> tuple[str, float | None, str]:
|
||||
from PIL import Image
|
||||
from paddleocr import PaddleOCR
|
||||
|
||||
work_path = path
|
||||
with Image.open(path) as image:
|
||||
original_size = image.size
|
||||
image.thumbnail((self.max_dimension, self.max_dimension))
|
||||
if image.size != original_size:
|
||||
tmp = path.with_suffix(".local-ocr-resized.jpg")
|
||||
image.convert("RGB").save(tmp, quality=90)
|
||||
work_path = tmp
|
||||
try:
|
||||
ocr = PaddleOCR(use_angle_cls=True, lang=self.language, use_gpu=False)
|
||||
result = ocr.ocr(str(work_path), cls=True)
|
||||
texts: list[str] = []
|
||||
scores: list[float] = []
|
||||
for page in result or []:
|
||||
for line in page or []:
|
||||
if len(line) >= 2 and isinstance(line[1], (list, tuple)):
|
||||
text = str(line[1][0]).strip()
|
||||
if text:
|
||||
texts.append(text)
|
||||
try:
|
||||
scores.append(float(line[1][1]))
|
||||
except (TypeError, ValueError, IndexError):
|
||||
pass
|
||||
confidence = sum(scores) / len(scores) if scores else None
|
||||
return "\n".join(texts), confidence, self.language
|
||||
finally:
|
||||
if work_path != path:
|
||||
work_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
value = os.getenv(name)
|
||||
if value is None:
|
||||
return default
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Bridge URL-level plugin parsing with the rest of the app (tasks/GUI logging).
|
||||
|
||||
Full TaskQueue integration can map Resource → DownloadTaskSpec per site.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
|
||||
from pixiv_app.core.plugin.manager import PluginManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrchestrationResult:
|
||||
plugin_name: str
|
||||
resources: list[Resource]
|
||||
|
||||
|
||||
class PluginTaskOrchestrator:
|
||||
"""Resolve plugins and run parse/download without replacing the legacy Pixiv GUI flow."""
|
||||
|
||||
def __init__(self, plugin_manager: PluginManager) -> None:
|
||||
self.plugin_manager = plugin_manager
|
||||
|
||||
def get_plugin_for_url(self, url: str) -> BasePlugin | None:
|
||||
return self.plugin_manager.get_plugin_for_url(url)
|
||||
|
||||
def parse_url(self, url: str) -> OrchestrationResult:
|
||||
plugin = self.get_plugin_for_url(url)
|
||||
if plugin is None:
|
||||
raise PluginParseError(f"No plugin registered for: {url}")
|
||||
resources = plugin.parse(url)
|
||||
return OrchestrationResult(plugin_name=plugin.name, resources=resources)
|
||||
|
||||
def download_resource(
|
||||
self,
|
||||
plugin_name: str,
|
||||
resource: Resource,
|
||||
save_path: Path,
|
||||
) -> list[Path]:
|
||||
plugin = self.plugin_manager.plugins.get(plugin_name)
|
||||
if plugin is None:
|
||||
raise PluginParseError(f"Plugin not loaded: {plugin_name}")
|
||||
return plugin.download(resource, save_path)
|
||||
@@ -0,0 +1,423 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.downloader import create_session, request_json
|
||||
from pixiv_app.core.gallery_service import LocalGalleryService
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
|
||||
from pixiv_app.core.paths import app_session_file
|
||||
from pixiv_app.core.x_media_service import XMediaService, _normalize_username
|
||||
|
||||
|
||||
ProgressCallback = Callable[[dict[str, Any]], None]
|
||||
|
||||
|
||||
@dataclass
|
||||
class OriginFollow:
|
||||
platform: str
|
||||
external_id: str
|
||||
name: str = ""
|
||||
handle: str = ""
|
||||
avatar_url: str = ""
|
||||
profile_url: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class OriginFollowSyncResult:
|
||||
platform: str
|
||||
created: int = 0
|
||||
updated: int = 0
|
||||
skipped: int = 0
|
||||
failed: int = 0
|
||||
cloud_created: int = 0
|
||||
cloud_updated: int = 0
|
||||
cloud_skipped: int = 0
|
||||
pages: int = 0
|
||||
errors: list[dict[str, str]] = field(default_factory=list)
|
||||
progress: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"platform": self.platform,
|
||||
"created": self.created,
|
||||
"updated": self.updated,
|
||||
"skipped": self.skipped,
|
||||
"failed": self.failed,
|
||||
"cloud_created": self.cloud_created,
|
||||
"cloud_updated": self.cloud_updated,
|
||||
"cloud_skipped": self.cloud_skipped,
|
||||
"pages": self.pages,
|
||||
"errors": self.errors,
|
||||
"progress": self.progress,
|
||||
}
|
||||
|
||||
|
||||
class OriginFollowSyncService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db_path: str = DEFAULT_LIBRARY_DB,
|
||||
gallery_service: LocalGalleryService | None = None,
|
||||
x_media_service: XMediaService | None = None,
|
||||
) -> None:
|
||||
self.db_path = db_path
|
||||
self.gallery_service = gallery_service or LocalGalleryService(db_path=db_path)
|
||||
self.x_media_service = x_media_service or XMediaService()
|
||||
|
||||
def sync(self, platform: str, *, progress: ProgressCallback | None = None) -> dict[str, Any]:
|
||||
normalized = platform.strip().lower()
|
||||
if normalized == "pixiv":
|
||||
return self.sync_pixiv(progress=progress).to_dict()
|
||||
if normalized == "x":
|
||||
return self.sync_x(progress=progress).to_dict()
|
||||
raise ValueError("请选择 Pixiv 或 X。")
|
||||
|
||||
def sync_pixiv(self, *, progress: ProgressCallback | None = None) -> OriginFollowSyncResult:
|
||||
cookie = str(SessionStore(app_session_file()).load().get("cookie", "")).strip()
|
||||
if not cookie:
|
||||
raise ValueError("Pixiv Cookie 不存在,请先导入或保存 Pixiv 登录态。")
|
||||
|
||||
result = OriginFollowSyncResult(platform="pixiv")
|
||||
session = create_session(cookie)
|
||||
try:
|
||||
user_id = self._fetch_pixiv_current_user_id(session, cookie)
|
||||
follows = self._fetch_pixiv_following(session, cookie, user_id, result, progress=progress)
|
||||
finally:
|
||||
session.close()
|
||||
self._upsert_pixiv_local(follows, result)
|
||||
return result
|
||||
|
||||
def sync_x(self, *, progress: ProgressCallback | None = None) -> OriginFollowSyncResult:
|
||||
plugin = self.x_media_service._get_plugin()
|
||||
if not plugin.validate():
|
||||
raise ValueError("X Playwright 不可用,请先安装 Playwright 并完成 X 登录。")
|
||||
|
||||
result = OriginFollowSyncResult(platform="x")
|
||||
with plugin._playwright_lock:
|
||||
follows = asyncio.run(self._fetch_x_following(plugin, result, progress=progress))
|
||||
self._upsert_x_local(follows, result)
|
||||
return result
|
||||
|
||||
def _emit(
|
||||
self,
|
||||
result: OriginFollowSyncResult,
|
||||
progress: ProgressCallback | None,
|
||||
payload: dict[str, Any],
|
||||
) -> None:
|
||||
result.progress.append(payload)
|
||||
if progress is not None:
|
||||
progress(payload)
|
||||
|
||||
def _fetch_pixiv_current_user_id(self, session: Any, cookie: str) -> str:
|
||||
for url in (
|
||||
"https://www.pixiv.net/ajax/user/self?lang=zh",
|
||||
"https://www.pixiv.net/ajax/user/self",
|
||||
):
|
||||
try:
|
||||
payload = request_json(session, url, cookie=cookie, context="Pixiv 当前账号")
|
||||
body = payload.get("body") if isinstance(payload, dict) else {}
|
||||
user_id = _first_text(body, "userId", "id")
|
||||
if user_id:
|
||||
return user_id
|
||||
except Exception:
|
||||
continue
|
||||
try:
|
||||
response = session.get("https://www.pixiv.net/", timeout=30)
|
||||
response.raise_for_status()
|
||||
match = re.search(r'"userData"\s*:\s*\{[^}]*"id"\s*:\s*"?(?P<id>\d+)"?', response.text)
|
||||
if match:
|
||||
return match.group("id")
|
||||
except Exception:
|
||||
pass
|
||||
raise ValueError("Pixiv Cookie 可能已失效:无法识别当前登录账号。")
|
||||
|
||||
def _fetch_pixiv_following(
|
||||
self,
|
||||
session: Any,
|
||||
cookie: str,
|
||||
user_id: str,
|
||||
result: OriginFollowSyncResult,
|
||||
*,
|
||||
progress: ProgressCallback | None = None,
|
||||
) -> list[OriginFollow]:
|
||||
follows: list[OriginFollow] = []
|
||||
seen: set[str] = set()
|
||||
limit = 100
|
||||
offset = 0
|
||||
page = 1
|
||||
while True:
|
||||
self._emit(result, progress, {"platform": "pixiv", "page": page, "message": f"正在同步第 {page} 页"})
|
||||
url = (
|
||||
f"https://www.pixiv.net/ajax/user/{user_id}/following"
|
||||
f"?offset={offset}&limit={limit}&rest=show&tag=&acceptingRequests=0&lang=zh"
|
||||
)
|
||||
try:
|
||||
payload = request_json(
|
||||
session,
|
||||
url,
|
||||
cookie=cookie,
|
||||
referer=f"https://www.pixiv.net/users/{user_id}/following",
|
||||
context=f"Pixiv 关注列表第 {page} 页",
|
||||
)
|
||||
except Exception as exc:
|
||||
if page == 1:
|
||||
raise ValueError("Pixiv Cookie 可能已失效:无法读取关注列表。") from exc
|
||||
result.failed += 1
|
||||
result.errors.append({"id": f"page:{page}", "reason": str(exc)})
|
||||
break
|
||||
users = _extract_pixiv_users(payload)
|
||||
result.pages = page
|
||||
if not users:
|
||||
break
|
||||
for item in users:
|
||||
follow = _pixiv_follow_from_payload(item)
|
||||
if follow is None:
|
||||
result.failed += 1
|
||||
result.errors.append({"id": "unknown", "reason": "Pixiv 作者记录缺少 userId"})
|
||||
continue
|
||||
if follow.external_id in seen:
|
||||
result.skipped += 1
|
||||
continue
|
||||
seen.add(follow.external_id)
|
||||
follows.append(follow)
|
||||
if len(users) < limit:
|
||||
break
|
||||
offset += limit
|
||||
page += 1
|
||||
return follows
|
||||
|
||||
async def _fetch_x_following(
|
||||
self,
|
||||
plugin: Any,
|
||||
result: OriginFollowSyncResult,
|
||||
*,
|
||||
progress: ProgressCallback | None = None,
|
||||
) -> list[OriginFollow]:
|
||||
from plugins.x import selectors
|
||||
from plugins.x.playwright_client import XPlaywrightClient
|
||||
|
||||
client = XPlaywrightClient(plugin.config)
|
||||
follows: list[OriginFollow] = []
|
||||
seen: set[str] = set()
|
||||
try:
|
||||
await client.start()
|
||||
page = await client.new_page_goto("https://x.com/home")
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
profile_href = await page.locator(selectors.PROFILE_LINK).first.get_attribute("href", timeout=8000)
|
||||
username = _x_username_from_href(profile_href or "")
|
||||
if not username:
|
||||
raise ValueError("X 未登录:请先在插件管理里完成 X 登录。")
|
||||
await page.goto(f"https://x.com/{username}{selectors.FOLLOWING_URL_SUFFIX}", wait_until="domcontentloaded")
|
||||
await client.raise_if_guest_wall(page)
|
||||
stale_rounds = 0
|
||||
page_no = 1
|
||||
while stale_rounds < 4 and page_no <= 80:
|
||||
self._emit(result, progress, {"platform": "x", "page": page_no, "message": f"正在同步第 {page_no} 页"})
|
||||
added = await _collect_x_user_cells(page, seen, follows)
|
||||
result.pages = page_no
|
||||
stale_rounds = stale_rounds + 1 if added == 0 else 0
|
||||
await client.human_scroll_page(page)
|
||||
page_no += 1
|
||||
if not follows:
|
||||
raise ValueError("X 未读取到 following 列表,请确认账号已登录且 following 页面可访问。")
|
||||
finally:
|
||||
await page.close()
|
||||
finally:
|
||||
await client.close()
|
||||
return follows
|
||||
|
||||
def _upsert_pixiv_local(self, follows: list[OriginFollow], result: OriginFollowSyncResult) -> None:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
for follow in follows:
|
||||
try:
|
||||
status = library.upsert_origin_followed_artist(
|
||||
int(follow.external_id),
|
||||
name=follow.name,
|
||||
account=follow.handle,
|
||||
avatar_url=follow.avatar_url,
|
||||
)
|
||||
_apply_local_status(result, status)
|
||||
except Exception as exc:
|
||||
result.failed += 1
|
||||
result.errors.append({"id": follow.external_id, "reason": str(exc)})
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def _upsert_x_local(self, follows: list[OriginFollow], result: OriginFollowSyncResult) -> None:
|
||||
rows = self.x_media_service._load_follow_rows()
|
||||
now = _now_text()
|
||||
by_handle = {str(item.get("username", "")).lower(): item for item in rows}
|
||||
for follow in follows:
|
||||
handle = _normalize_username(follow.handle or follow.external_id)
|
||||
if not handle:
|
||||
result.failed += 1
|
||||
result.errors.append({"id": follow.external_id, "reason": "X 用户名为空"})
|
||||
continue
|
||||
row = by_handle.get(handle.lower())
|
||||
status = "created"
|
||||
if row is None:
|
||||
row = {
|
||||
"username": handle,
|
||||
"creator_name": follow.name or handle,
|
||||
"enabled": True,
|
||||
"profile_url": follow.profile_url or f"https://x.com/{handle}",
|
||||
"avatar_url": follow.avatar_url,
|
||||
"latest_tweet_id": "",
|
||||
"latest_published_at": "",
|
||||
"last_checked_at": now,
|
||||
"last_error": "",
|
||||
"seen_tweet_ids": [],
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
rows.append(row)
|
||||
by_handle[handle.lower()] = row
|
||||
else:
|
||||
changed = (
|
||||
not bool(row.get("enabled", True))
|
||||
or (follow.name and follow.name != str(row.get("creator_name") or ""))
|
||||
or (follow.avatar_url and follow.avatar_url != str(row.get("avatar_url") or ""))
|
||||
)
|
||||
status = "updated" if changed else "skipped"
|
||||
row["username"] = handle
|
||||
row["creator_name"] = follow.name or str(row.get("creator_name") or handle)
|
||||
row["enabled"] = True
|
||||
row["profile_url"] = follow.profile_url or f"https://x.com/{handle}"
|
||||
row["avatar_url"] = follow.avatar_url or str(row.get("avatar_url") or "")
|
||||
row["last_checked_at"] = now
|
||||
row["last_error"] = ""
|
||||
row["updated_at"] = now
|
||||
_apply_local_status(result, status)
|
||||
self.x_media_service._save_follow_rows(rows)
|
||||
|
||||
def _apply_local_status(result: OriginFollowSyncResult, status: str) -> None:
|
||||
if status == "created":
|
||||
result.created += 1
|
||||
elif status == "updated":
|
||||
result.updated += 1
|
||||
else:
|
||||
result.skipped += 1
|
||||
|
||||
|
||||
def _extract_pixiv_users(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
body = payload.get("body") if isinstance(payload, dict) else payload
|
||||
if isinstance(body, list):
|
||||
return [item for item in body if isinstance(item, dict)]
|
||||
if not isinstance(body, dict):
|
||||
return []
|
||||
candidates = (
|
||||
body.get("users"),
|
||||
body.get("followUsers"),
|
||||
body.get("userPreviews"),
|
||||
body.get("thumbnails", {}).get("user") if isinstance(body.get("thumbnails"), dict) else None,
|
||||
)
|
||||
for candidate in candidates:
|
||||
if isinstance(candidate, list):
|
||||
return [item for item in candidate if isinstance(item, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def _pixiv_follow_from_payload(item: dict[str, Any]) -> OriginFollow | None:
|
||||
user_id = _first_text(item, "userId", "user_id", "id")
|
||||
if not user_id:
|
||||
user = item.get("user") if isinstance(item.get("user"), dict) else {}
|
||||
user_id = _first_text(user, "userId", "user_id", "id")
|
||||
item = {**item, **user}
|
||||
if not user_id:
|
||||
return None
|
||||
name = _first_text(item, "name", "userName", "displayName")
|
||||
handle = _first_text(item, "userAccount", "account", "screenName")
|
||||
avatar = _first_text(item, "imageBig", "image", "profileImageUrl", "profile_image_url")
|
||||
return OriginFollow(
|
||||
platform="pixiv",
|
||||
external_id=str(user_id),
|
||||
name=name,
|
||||
handle=handle,
|
||||
avatar_url=avatar,
|
||||
profile_url=f"https://www.pixiv.net/users/{user_id}",
|
||||
)
|
||||
|
||||
|
||||
async def _collect_x_user_cells(page: Any, seen: set[str], follows: list[OriginFollow]) -> int:
|
||||
from plugins.x import selectors
|
||||
|
||||
added = 0
|
||||
cells = await page.locator(selectors.USER_CELL).all()
|
||||
for cell in cells:
|
||||
try:
|
||||
data = await cell.evaluate(
|
||||
"""(node) => {
|
||||
const text = node.innerText || "";
|
||||
const links = Array.from(node.querySelectorAll('a[href^="/"]')).map(a => a.getAttribute("href") || "");
|
||||
const img = node.querySelector('img[src*="profile_images"]');
|
||||
return { text, links, avatar: img ? img.getAttribute("src") : "" };
|
||||
}"""
|
||||
)
|
||||
handle = _x_handle_from_cell(data)
|
||||
if not handle or handle.lower() in seen:
|
||||
continue
|
||||
seen.add(handle.lower())
|
||||
follows.append(
|
||||
OriginFollow(
|
||||
platform="x",
|
||||
external_id=handle,
|
||||
name=_x_display_name_from_cell(data, handle),
|
||||
handle=handle,
|
||||
avatar_url=str(data.get("avatar") or ""),
|
||||
profile_url=f"https://x.com/{handle}",
|
||||
)
|
||||
)
|
||||
added += 1
|
||||
except Exception:
|
||||
continue
|
||||
return added
|
||||
|
||||
|
||||
def _x_handle_from_cell(data: dict[str, Any]) -> str:
|
||||
for href in data.get("links") or []:
|
||||
username = _x_username_from_href(str(href))
|
||||
if username:
|
||||
return username
|
||||
text = str(data.get("text") or "")
|
||||
match = re.search(r"@([A-Za-z0-9_]{1,15})", text)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def _x_display_name_from_cell(data: dict[str, Any], handle: str) -> str:
|
||||
text = str(data.get("text") or "")
|
||||
for line in [item.strip() for item in text.splitlines() if item.strip()]:
|
||||
if not line.startswith("@") and line.lower() not in {"follow", "following"}:
|
||||
return line
|
||||
return handle
|
||||
|
||||
|
||||
def _x_username_from_href(href: str) -> str:
|
||||
match = re.match(r"^/?([A-Za-z0-9_]{1,15})(?:/)?$", href.strip())
|
||||
if not match:
|
||||
return ""
|
||||
username = match.group(1)
|
||||
if username.lower() in {"home", "explore", "notifications", "messages", "settings", "i"}:
|
||||
return ""
|
||||
return username
|
||||
|
||||
|
||||
def _first_text(payload: Any, *keys: str) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
return ""
|
||||
for key in keys:
|
||||
value = payload.get(key)
|
||||
if value is not None and str(value).strip():
|
||||
return str(value).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _now_text() -> str:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
APP_NAME = "SakuraDownloader"
|
||||
LEGACY_APP_NAME = "PixivDownloader"
|
||||
|
||||
|
||||
def is_frozen() -> bool:
|
||||
return bool(getattr(sys, "frozen", False))
|
||||
|
||||
|
||||
def resource_root() -> Path:
|
||||
if is_frozen():
|
||||
return Path(getattr(sys, "_MEIPASS")).resolve()
|
||||
return Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def executable_dir() -> Path:
|
||||
if is_frozen():
|
||||
return Path(sys.executable).resolve().parent
|
||||
return resource_root()
|
||||
|
||||
|
||||
def data_root() -> Path:
|
||||
override = os.environ.get("SAKURA_APP_DATA_DIR") or os.environ.get("PIXIV_APP_DATA_DIR")
|
||||
if override:
|
||||
root = Path(override)
|
||||
elif is_frozen():
|
||||
local_appdata = os.environ.get("LOCALAPPDATA")
|
||||
root = Path(local_appdata) / APP_NAME if local_appdata else executable_dir() / "data"
|
||||
legacy_root = Path(local_appdata) / LEGACY_APP_NAME if local_appdata else None
|
||||
if legacy_root and legacy_root.exists() and not root.exists():
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
for name in ("pixiv_app_session.json", "runtime", "profiles", "plugins"):
|
||||
src = legacy_root / name
|
||||
dst = root / name
|
||||
if src.exists() and not dst.exists():
|
||||
if src.is_dir():
|
||||
import shutil
|
||||
|
||||
shutil.copytree(src, dst)
|
||||
else:
|
||||
dst.write_bytes(src.read_bytes())
|
||||
else:
|
||||
root = resource_root()
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
def runtime_path(*parts: str) -> Path:
|
||||
path = data_root() / "runtime"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path.joinpath(*parts)
|
||||
|
||||
|
||||
def profiles_root() -> Path:
|
||||
path = data_root() / "profiles"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def plugins_root() -> Path:
|
||||
return resource_root() / "plugins"
|
||||
|
||||
|
||||
def external_plugins_root() -> Path:
|
||||
"""Plugins shipped beside the executable, outside the PyInstaller bundle."""
|
||||
return executable_dir() / "plugins"
|
||||
|
||||
|
||||
def user_plugins_root() -> Path:
|
||||
path = data_root() / "plugins"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def plugin_roots() -> list[Path]:
|
||||
roots = [plugins_root(), external_plugins_root(), user_plugins_root()]
|
||||
result: list[Path] = []
|
||||
for root in roots:
|
||||
if root not in result:
|
||||
result.append(root)
|
||||
return result
|
||||
|
||||
|
||||
def webui_root() -> Path:
|
||||
return resource_root() / "pixiv_app" / "webui"
|
||||
|
||||
|
||||
def app_session_file() -> Path:
|
||||
return data_root() / "pixiv_app_session.json"
|
||||
|
||||
|
||||
def app_library_file() -> Path:
|
||||
"""Persistent local library DB, writable even when the EXE directory is read-only."""
|
||||
target = data_root() / "pixiv_app_library.db"
|
||||
if is_frozen() and not target.exists():
|
||||
legacy = executable_dir() / "pixiv_app_library.db"
|
||||
if legacy.is_file():
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(legacy, target)
|
||||
return target
|
||||
|
||||
|
||||
def downloads_root() -> Path:
|
||||
path = executable_dir() / "Sakura_Downloads"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def playwright_browsers_root() -> Path | None:
|
||||
local_appdata = os.environ.get("LOCALAPPDATA", "").strip()
|
||||
if not local_appdata:
|
||||
return None
|
||||
return Path(local_appdata) / "ms-playwright"
|
||||
|
||||
|
||||
def configure_playwright_runtime_env() -> Path | None:
|
||||
root = playwright_browsers_root()
|
||||
if root is None:
|
||||
return None
|
||||
if root.exists() and "PLAYWRIGHT_BROWSERS_PATH" not in os.environ:
|
||||
os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(root)
|
||||
return root
|
||||
|
||||
|
||||
def preferred_chromium_executable() -> Path | None:
|
||||
root = configure_playwright_runtime_env()
|
||||
if root is None or not root.exists():
|
||||
return None
|
||||
|
||||
candidates: list[tuple[int, float, Path]] = []
|
||||
for browser_dir in root.glob("chromium-*"):
|
||||
if not browser_dir.is_dir():
|
||||
continue
|
||||
revision = _parse_revision(browser_dir.name)
|
||||
for relative in (
|
||||
Path("chrome-win64") / "chrome.exe",
|
||||
Path("chrome-win") / "chrome.exe",
|
||||
):
|
||||
executable = browser_dir / relative
|
||||
if executable.is_file():
|
||||
try:
|
||||
mtime = executable.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = 0.0
|
||||
candidates.append((revision, mtime, executable))
|
||||
break
|
||||
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
candidates.sort(key=lambda item: (item[0], item[1]), reverse=True)
|
||||
return candidates[0][2]
|
||||
|
||||
|
||||
def explain_missing_browser(exc: Exception, *, feature_name: str) -> str:
|
||||
text = str(exc)
|
||||
lowered = text.lower()
|
||||
if "executable doesn't exist" not in lowered and "browsertype.launch_persistent_context" not in lowered:
|
||||
return text
|
||||
|
||||
return (
|
||||
f"{feature_name} 需要 Playwright Chromium 浏览器,但当前系统里没有可用的浏览器文件。\n"
|
||||
"请先运行同目录下的 install_playwright_chromium.bat,安装完成后重新打开程序再试。"
|
||||
)
|
||||
|
||||
|
||||
def _parse_revision(name: str) -> int:
|
||||
try:
|
||||
return int(name.rsplit("-", 1)[-1])
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Plugin system: base types, manager, generator, optional market client."""
|
||||
|
||||
from pixiv_app.core.plugin.base import (
|
||||
BasePlugin,
|
||||
PluginAuthError,
|
||||
PluginDownloadError,
|
||||
PluginError,
|
||||
PluginManifest,
|
||||
PluginParseError,
|
||||
Resource,
|
||||
)
|
||||
from pixiv_app.core.plugin.manager import PluginEvent, PluginManager
|
||||
|
||||
__all__ = [
|
||||
"BasePlugin",
|
||||
"PluginAuthError",
|
||||
"PluginDownloadError",
|
||||
"PluginError",
|
||||
"PluginEvent",
|
||||
"PluginManager",
|
||||
"PluginManifest",
|
||||
"PluginParseError",
|
||||
"Resource",
|
||||
]
|
||||
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
Multi-site plugin core abstractions.
|
||||
|
||||
解析与下载采用同步接口,便于与现有 requests / 线程模型集成;
|
||||
若站点 SDK 仅提供 asyncio,可在插件内部使用 asyncio.run 包装。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class Resource:
|
||||
"""Unified resource model returned by plugins."""
|
||||
|
||||
id: str
|
||||
url: str
|
||||
title: str
|
||||
author: str
|
||||
author_id: str
|
||||
files: list[dict[str, Any]]
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
thumbnail: str | None = None
|
||||
created_at: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PluginManifest:
|
||||
"""Optional manifest shipped as manifest.json next to plugin.py."""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
domain: str
|
||||
author: str = ""
|
||||
description: str = ""
|
||||
requires_auth: bool = False
|
||||
min_platform_version: str = "1.0.0"
|
||||
tags: list[str] = field(default_factory=list)
|
||||
dependencies: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class PluginError(Exception):
|
||||
"""Base class for plugin failures."""
|
||||
|
||||
|
||||
class PluginAuthError(PluginError):
|
||||
"""Authentication or authorization failed."""
|
||||
|
||||
|
||||
class PluginParseError(PluginError):
|
||||
"""URL or API response could not be parsed."""
|
||||
|
||||
|
||||
class PluginDownloadError(PluginError):
|
||||
"""Download or filesystem error."""
|
||||
|
||||
|
||||
class BasePlugin(ABC):
|
||||
"""Interface implemented by every site plugin."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def domain(self) -> str:
|
||||
"""Primary domain, e.g. pixiv.net."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""Human-readable plugin name."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def version(self) -> str:
|
||||
"""Semantic version string."""
|
||||
|
||||
@abstractmethod
|
||||
def can_handle(self, url: str) -> bool:
|
||||
"""Return True if this plugin should process the URL or raw token."""
|
||||
|
||||
@abstractmethod
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
"""Resolve URL into one or more downloadable resources."""
|
||||
|
||||
@abstractmethod
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
"""Download files for ``resource`` under ``save_path``."""
|
||||
|
||||
@abstractmethod
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
"""HTTP headers for requests (Cookie, Bearer token, etc.)."""
|
||||
|
||||
def validate(self) -> bool:
|
||||
"""Optional self-check (credentials present, deps installed)."""
|
||||
return True
|
||||
|
||||
def get_config_schema(self) -> dict[str, Any]:
|
||||
"""Optional JSON-schema-like dict for dynamic GUI forms."""
|
||||
return {}
|
||||
|
||||
def load_config(self) -> dict[str, Any]:
|
||||
"""Override to load persisted plugin settings."""
|
||||
return {}
|
||||
|
||||
def save_config(self, data: dict[str, Any]) -> None:
|
||||
"""Override to persist plugin settings."""
|
||||
|
||||
__all__ = [
|
||||
"BasePlugin",
|
||||
"PluginAuthError",
|
||||
"PluginDownloadError",
|
||||
"PluginError",
|
||||
"PluginManifest",
|
||||
"PluginParseError",
|
||||
"Resource",
|
||||
]
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||
|
||||
|
||||
def _class_name_from_plugin_name(name: str) -> str:
|
||||
parts = re.split(r"[\s_]+", name.strip())
|
||||
stem = "".join(p[:1].upper() + p[1:].lower() if p else "" for p in parts if p)
|
||||
return f"{stem}Plugin" if stem else "GeneratedPlugin"
|
||||
|
||||
|
||||
class PluginGenerator:
|
||||
"""Generate a skeleton plugin directory from Jinja2 templates."""
|
||||
|
||||
def __init__(self, template_dir: str | Path | None = None) -> None:
|
||||
base = Path(__file__).resolve().parent / "templates"
|
||||
self.template_dir = Path(template_dir) if template_dir else base
|
||||
self.env = Environment(
|
||||
loader=FileSystemLoader(str(self.template_dir)),
|
||||
autoescape=select_autoescape(enabled_extensions=()),
|
||||
)
|
||||
|
||||
def generate(
|
||||
self,
|
||||
name: str,
|
||||
domain: str,
|
||||
author: str,
|
||||
description: str,
|
||||
*,
|
||||
output_root: str | Path = "plugins",
|
||||
) -> Path:
|
||||
slug = name.strip().lower().replace(" ", "_")
|
||||
plugin_dir = Path(output_root) / slug
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
context: dict[str, Any] = {
|
||||
"name": name.strip(),
|
||||
"domain": domain.strip(),
|
||||
"author": author.strip(),
|
||||
"description": description.strip(),
|
||||
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"class_name": _class_name_from_plugin_name(name),
|
||||
}
|
||||
|
||||
py_template = self.env.get_template("base_plugin.py.jinja")
|
||||
(plugin_dir / "plugin.py").write_text(py_template.render(**context), encoding="utf-8")
|
||||
|
||||
manifest_template = self.env.get_template("manifest.json.jinja")
|
||||
(plugin_dir / "manifest.json").write_text(manifest_template.render(**context), encoding="utf-8")
|
||||
|
||||
init_file = plugin_dir / "__init__.py"
|
||||
if not init_file.exists():
|
||||
init_file.write_text('"""Generated plugin."""\n', encoding="utf-8")
|
||||
|
||||
return plugin_dir.resolve()
|
||||
|
||||
def generate_from_gui(self, form_data: dict[str, str], *, output_root: str | Path = "plugins") -> Path:
|
||||
return self.generate(
|
||||
name=form_data["name"],
|
||||
domain=form_data["domain"],
|
||||
author=form_data["author"],
|
||||
description=form_data["description"],
|
||||
output_root=output_root,
|
||||
)
|
||||
@@ -0,0 +1,194 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginManifest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PluginEvent(str, Enum):
|
||||
BEFORE_LOAD = "before_load"
|
||||
AFTER_LOAD = "after_load"
|
||||
BEFORE_UNLOAD = "before_unload"
|
||||
AFTER_UNLOAD = "after_unload"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
class PluginManager:
|
||||
"""Discover and load plugins from filesystem directories."""
|
||||
|
||||
def __init__(self, plugin_dirs: list[str | Path]) -> None:
|
||||
self.plugin_dirs = [Path(d).resolve() for d in plugin_dirs]
|
||||
self.plugins: dict[str, BasePlugin] = {}
|
||||
self.manifests: dict[str, PluginManifest] = {}
|
||||
self.loaded_modules: dict[str, Any] = {}
|
||||
self._paths: dict[str, Path] = {}
|
||||
self.load_errors: dict[Path, str] = {}
|
||||
self.event_handlers: dict[PluginEvent, list[Callable[..., None]]] = defaultdict(list)
|
||||
|
||||
def on_event(self, event: PluginEvent, callback: Callable[..., None]) -> None:
|
||||
self.event_handlers[event].append(callback)
|
||||
|
||||
def _emit(self, event: PluginEvent, *args: Any, **kwargs: Any) -> None:
|
||||
for cb in self.event_handlers.get(event, []):
|
||||
try:
|
||||
cb(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
logger.exception("Plugin event handler failed: %s", exc)
|
||||
|
||||
def discover(self) -> list[Path]:
|
||||
"""Return paths to plugin.py files."""
|
||||
found: list[Path] = []
|
||||
for root in self.plugin_dirs:
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for subdir in sorted(root.iterdir()):
|
||||
if not subdir.is_dir():
|
||||
continue
|
||||
candidate = subdir / "plugin.py"
|
||||
if candidate.is_file():
|
||||
found.append(candidate)
|
||||
return found
|
||||
|
||||
def _load_manifest(self, plugin_dir: Path) -> PluginManifest | None:
|
||||
manifest_path = plugin_dir / "manifest.json"
|
||||
if not manifest_path.is_file():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
return PluginManifest(
|
||||
name=str(data.get("name", plugin_dir.name)),
|
||||
version=str(data.get("version", "0.0.0")),
|
||||
domain=str(data.get("domain", "")),
|
||||
author=str(data.get("author", "")),
|
||||
description=str(data.get("description", "")),
|
||||
requires_auth=bool(data.get("requires_auth", False)),
|
||||
min_platform_version=str(data.get("min_platform_version", "1.0.0")),
|
||||
tags=list(data.get("tags") or []),
|
||||
dependencies=list(data.get("dependencies") or []),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Invalid manifest %s: %s", manifest_path, exc)
|
||||
return None
|
||||
|
||||
def load_plugin(self, plugin_path: Path) -> BasePlugin | None:
|
||||
"""Load a single plugin from ``.../plugin.py``."""
|
||||
plugin_path = plugin_path.resolve()
|
||||
plugin_dir = plugin_path.parent
|
||||
module_name = f"sakura_dynamic_plugin_{plugin_dir.name}"
|
||||
module_stem = module_name.rsplit(".", 1)[0]
|
||||
for loaded_name in list(sys.modules):
|
||||
if loaded_name == module_name or loaded_name.startswith(f"{module_name}."):
|
||||
del sys.modules[loaded_name]
|
||||
self.load_errors.pop(plugin_path, None)
|
||||
|
||||
self._emit(PluginEvent.BEFORE_LOAD, plugin_path)
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name,
|
||||
plugin_path,
|
||||
submodule_search_locations=[str(plugin_dir)],
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
self._emit(PluginEvent.ERROR, plugin_path, "invalid spec")
|
||||
self.load_errors[plugin_path] = "invalid spec"
|
||||
return None
|
||||
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
module.__package__ = module_stem
|
||||
module.__file__ = str(plugin_path)
|
||||
sys.modules[module_name] = module
|
||||
try:
|
||||
spec.loader.exec_module(module)
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to load %s", plugin_path)
|
||||
self._emit(PluginEvent.ERROR, plugin_path, exc)
|
||||
self.load_errors[plugin_path] = f"{type(exc).__name__}: {exc}"
|
||||
return None
|
||||
|
||||
plugin_class = getattr(module, "plugin_class", None)
|
||||
if plugin_class is None:
|
||||
logger.error("Module %s has no plugin_class export", plugin_path)
|
||||
self._emit(PluginEvent.ERROR, plugin_path, "missing plugin_class")
|
||||
self.load_errors[plugin_path] = "missing plugin_class"
|
||||
return None
|
||||
|
||||
try:
|
||||
plugin = plugin_class()
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to instantiate plugin from %s", plugin_path)
|
||||
self._emit(PluginEvent.ERROR, plugin_path, exc)
|
||||
self.load_errors[plugin_path] = f"{type(exc).__name__}: {exc}"
|
||||
return None
|
||||
|
||||
if not isinstance(plugin, BasePlugin):
|
||||
logger.error("plugin_class is not BasePlugin: %s", plugin_path)
|
||||
self.load_errors[plugin_path] = "plugin_class is not BasePlugin"
|
||||
return None
|
||||
|
||||
key = plugin.name
|
||||
manifest = self._load_manifest(plugin_dir)
|
||||
if manifest:
|
||||
self.manifests[key] = manifest
|
||||
|
||||
self.plugins[key] = plugin
|
||||
self.loaded_modules[key] = module
|
||||
self._paths[key] = plugin_path
|
||||
self.load_errors.pop(plugin_path, None)
|
||||
self._emit(PluginEvent.AFTER_LOAD, plugin)
|
||||
return plugin
|
||||
|
||||
def load_all(self) -> dict[str, BasePlugin]:
|
||||
for path in self.discover():
|
||||
try:
|
||||
self.load_plugin(path)
|
||||
except Exception as exc:
|
||||
logger.exception("Failed during load_all %s: %s", path, exc)
|
||||
return self.plugins
|
||||
|
||||
def reload_all_from_disk(self) -> None:
|
||||
"""Unload every plugin entry then scan directories again."""
|
||||
for name in list(self.plugins.keys()):
|
||||
self.unload_plugin(name)
|
||||
self.load_errors.clear()
|
||||
self.load_all()
|
||||
|
||||
def reload_plugin(self, name: str) -> bool:
|
||||
path = self._paths.get(name)
|
||||
if path is None:
|
||||
return False
|
||||
self.unload_plugin(name)
|
||||
return self.load_plugin(path) is not None
|
||||
|
||||
def get_plugin_path(self, name: str) -> Path | None:
|
||||
return self._paths.get(name)
|
||||
|
||||
def unload_plugin(self, name: str) -> None:
|
||||
self._emit(PluginEvent.BEFORE_UNLOAD, name)
|
||||
self.plugins.pop(name, None)
|
||||
mod = self.loaded_modules.pop(name, None)
|
||||
self.manifests.pop(name, None)
|
||||
self._paths.pop(name, None)
|
||||
if mod is not None and getattr(mod, "__name__", None) in sys.modules:
|
||||
del sys.modules[mod.__name__]
|
||||
self._emit(PluginEvent.AFTER_UNLOAD, name)
|
||||
|
||||
def get_plugin_for_url(self, url: str) -> BasePlugin | None:
|
||||
for plugin in self.plugins.values():
|
||||
if plugin.can_handle(url):
|
||||
return plugin
|
||||
return None
|
||||
|
||||
def check_dependencies(self, manifest: PluginManifest) -> bool:
|
||||
for dep in manifest.dependencies:
|
||||
if dep not in self.plugins:
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,29 @@
|
||||
"""
|
||||
Optional plugin market client — stub for future HTTP catalog integration.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
class PluginMarketClient:
|
||||
def __init__(self, api_url: str = "") -> None:
|
||||
self.api_url = api_url.rstrip("/")
|
||||
|
||||
def list_plugins(self) -> list[dict[str, Any]]:
|
||||
if not self.api_url:
|
||||
return []
|
||||
response = requests.get(f"{self.api_url}/plugins", timeout=15)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return list(data.get("plugins", []))
|
||||
|
||||
def install_plugin(self, _plugin_name: str) -> bool:
|
||||
"""Download zip and extract into plugins/ — not implemented."""
|
||||
return False
|
||||
|
||||
def check_updates(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
{{ name }} plugin — auto-generated at {{ generated_at }}.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pixiv_app.core.plugin.base import BasePlugin, Resource
|
||||
|
||||
|
||||
class {{ class_name }}(BasePlugin):
|
||||
domain = "{{ domain }}"
|
||||
name = "{{ name }}"
|
||||
version = "1.0.0"
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
return "{{ domain }}" in url.lower()
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
raise NotImplementedError("Implement URL parsing and build Resource list.")
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
raise NotImplementedError("Download files for resource into save_path.")
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
return {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
|
||||
|
||||
|
||||
plugin_class = {{ class_name }}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "{{ name }}",
|
||||
"version": "1.0.0",
|
||||
"domain": "{{ domain }}",
|
||||
"author": "{{ author }}",
|
||||
"description": "{{ description }}",
|
||||
"requires_auth": false,
|
||||
"tags": ["general"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProxyInfo:
|
||||
ip: str
|
||||
port: int
|
||||
protocol: str = "http"
|
||||
source: str = "manual"
|
||||
location: str = ""
|
||||
org: str = ""
|
||||
last_check: float = 0.0
|
||||
success_count: int = 0
|
||||
fail_count: int = 0
|
||||
|
||||
@property
|
||||
def proxy_url(self) -> str:
|
||||
return f"{self.protocol}://{self.ip}:{self.port}"
|
||||
|
||||
@property
|
||||
def is_alive(self) -> bool:
|
||||
return self.success_count > 0 or self.fail_count < 3
|
||||
|
||||
|
||||
def parse_proxy_text(text: str) -> list[ProxyInfo]:
|
||||
proxies: list[ProxyInfo] = []
|
||||
seen: set[tuple[str, int, str]] = set()
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "://" not in line:
|
||||
line = f"http://{line}"
|
||||
match = re.match(r"^(https?)://([\w\.\-]+):(\d{2,5})$", line, re.IGNORECASE)
|
||||
if not match:
|
||||
continue
|
||||
protocol, ip, port_text = match.groups()
|
||||
key = (ip, int(port_text), protocol.lower())
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
proxies.append(ProxyInfo(ip=ip, port=int(port_text), protocol=protocol.lower(), source="manual"))
|
||||
return proxies
|
||||
|
||||
|
||||
class QuakeClient:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str = "",
|
||||
cookie: str = "",
|
||||
mode: str = "api_v3",
|
||||
request_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
self.api_key = api_key.strip()
|
||||
self.cookie = cookie.strip()
|
||||
self.mode = mode
|
||||
self.request_kwargs = dict(request_kwargs or {})
|
||||
self.base_url = "https://quake.360.net/api/v3"
|
||||
self.web_assoc_url = "https://quake.360.net/api/search/field/association/quake_service"
|
||||
|
||||
def _api_headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"X-QuakeToken": self.api_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def _web_headers(self) -> dict[str, str]:
|
||||
headers = {
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Content-Type": "application/json",
|
||||
"Origin": "https://quake.360.net",
|
||||
"Referer": "https://quake.360.net/quake/",
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
if self.cookie:
|
||||
headers["Cookie"] = self.cookie
|
||||
return headers
|
||||
|
||||
def search_ips_api(self, query: str, size: int = 100, start: int = 0) -> list[dict[str, Any]]:
|
||||
if not self.api_key:
|
||||
return []
|
||||
url = f"{self.base_url}/search/quake_service"
|
||||
payload = {"query": query, "start": start, "size": min(size, 500)}
|
||||
kwargs = dict(self.request_kwargs)
|
||||
kwargs.setdefault("timeout", 30)
|
||||
try:
|
||||
response = requests.post(url, headers=self._api_headers(), json=payload, **kwargs)
|
||||
if response.status_code != 200:
|
||||
return []
|
||||
body = response.json()
|
||||
return body.get("data", []) if isinstance(body, dict) else []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def search_ips_web_assoc(self, search_content: str) -> list[dict[str, Any]]:
|
||||
if not self.cookie:
|
||||
return []
|
||||
payload = {
|
||||
"search_content": search_content,
|
||||
"device": {
|
||||
"device_type": "PC",
|
||||
"os": "Windows",
|
||||
"os_version": "10.0",
|
||||
"language": "zh_CN",
|
||||
"network": "4g",
|
||||
"browser_info": "Chrome",
|
||||
"fingerprint": "pixiv-proxy-pool",
|
||||
"user_agent": self._web_headers()["User-Agent"],
|
||||
"date": time.strftime("%Y/%m/%d %H:%M:%S"),
|
||||
"UUID": "pixiv-proxy-association",
|
||||
},
|
||||
}
|
||||
kwargs = dict(self.request_kwargs)
|
||||
kwargs.setdefault("timeout", 30)
|
||||
try:
|
||||
response = requests.post(
|
||||
self.web_assoc_url,
|
||||
headers=self._web_headers(),
|
||||
json=payload,
|
||||
**kwargs,
|
||||
)
|
||||
if response.status_code != 200:
|
||||
return []
|
||||
text = response.text
|
||||
ips = sorted(set(re.findall(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", text)))
|
||||
return [{"ip": ip, "port": 80, "organization": "", "location": {}} for ip in ips]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def get_foreign_proxies(
|
||||
self,
|
||||
size: int = 120,
|
||||
countries: Optional[list[str]] = None,
|
||||
) -> list[ProxyInfo]:
|
||||
proxy_ports = [80, 443, 8080, 3128, 8888, 1080, 8118]
|
||||
port_query = " OR ".join([f"port:{port}" for port in proxy_ports])
|
||||
country_codes = countries or ["US", "JP", "KR", "SG", "HK", "TW", "DE", "FR", "GB", "CA", "AU"]
|
||||
country_query = " OR ".join([f'country:"{code}"' for code in country_codes])
|
||||
query = f"({port_query}) AND ({country_query}) AND response:ok"
|
||||
|
||||
if self.mode == "web_assoc":
|
||||
data = self.search_ips_web_assoc(query)
|
||||
else:
|
||||
data = self.search_ips_api(query=query, size=size)
|
||||
|
||||
proxies: list[ProxyInfo] = []
|
||||
seen: set[tuple[str, int, str]] = set()
|
||||
for item in data:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
ip = item.get("ip")
|
||||
port = item.get("port")
|
||||
if not ip or not isinstance(port, int):
|
||||
continue
|
||||
protocol = "https" if port in (443, 8443) else "http"
|
||||
key = (ip, port, protocol)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
location_obj = item.get("location", {}) if isinstance(item.get("location"), dict) else {}
|
||||
location = "/".join(
|
||||
[str(location_obj.get("country", "")).strip(), str(location_obj.get("province", "")).strip()]
|
||||
).strip("/")
|
||||
proxies.append(
|
||||
ProxyInfo(
|
||||
ip=ip,
|
||||
port=port,
|
||||
protocol=protocol,
|
||||
source=f"quake:{self.mode}",
|
||||
location=location,
|
||||
org=str(item.get("organization", "") or item.get("org", "")),
|
||||
)
|
||||
)
|
||||
return proxies
|
||||
|
||||
|
||||
class ProxyPool:
|
||||
def __init__(self, test_url: str = "https://httpbin.org/ip", test_timeout: int = 6):
|
||||
self.proxies: list[ProxyInfo] = []
|
||||
self.working_proxies: list[ProxyInfo] = []
|
||||
self.current_index = 0
|
||||
self.test_url = test_url
|
||||
self.test_timeout = test_timeout
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({"User-Agent": "SakuraDownloaderProxyPool/1.0"})
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add_proxies(self, proxies: list[ProxyInfo]) -> None:
|
||||
with self._lock:
|
||||
existing = {(p.ip, p.port, p.protocol) for p in self.proxies}
|
||||
for proxy in proxies:
|
||||
key = (proxy.ip, proxy.port, proxy.protocol)
|
||||
if key not in existing:
|
||||
existing.add(key)
|
||||
self.proxies.append(proxy)
|
||||
|
||||
def verify_proxy(self, proxy: ProxyInfo) -> bool:
|
||||
target_proxies = {"http": proxy.proxy_url, "https": proxy.proxy_url}
|
||||
try:
|
||||
response = self.session.get(
|
||||
self.test_url,
|
||||
proxies=target_proxies,
|
||||
timeout=self.test_timeout,
|
||||
verify=False,
|
||||
)
|
||||
if response.status_code == 200:
|
||||
with self._lock:
|
||||
proxy.last_check = time.time()
|
||||
proxy.success_count += 1
|
||||
proxy.fail_count = 0
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
with self._lock:
|
||||
proxy.fail_count += 1
|
||||
return False
|
||||
|
||||
def verify_all(self, max_workers: int = 16, max_proxies: int = 80) -> list[ProxyInfo]:
|
||||
with self._lock:
|
||||
to_verify = list(self.proxies[:max_proxies])
|
||||
self.working_proxies = []
|
||||
if not to_verify:
|
||||
return self.working_proxies
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
future_map = {executor.submit(self.verify_proxy, proxy): proxy for proxy in to_verify}
|
||||
for future in as_completed(future_map):
|
||||
proxy = future_map[future]
|
||||
try:
|
||||
if future.result():
|
||||
with self._lock:
|
||||
self.working_proxies.append(proxy)
|
||||
except Exception:
|
||||
continue
|
||||
with self._lock:
|
||||
return list(self.working_proxies)
|
||||
|
||||
def collect_until_target(
|
||||
self,
|
||||
quake_client: QuakeClient,
|
||||
target_working: int = 5,
|
||||
stable_rounds: int = 2,
|
||||
fetch_batch_size: int = 60,
|
||||
verify_batch_size: int = 30,
|
||||
max_rounds: int = 6,
|
||||
max_workers: int = 16,
|
||||
countries: Optional[list[str]] = None,
|
||||
) -> list[ProxyInfo]:
|
||||
stable_hits = 0
|
||||
for _ in range(max_rounds):
|
||||
if stable_hits >= stable_rounds:
|
||||
break
|
||||
new_items = quake_client.get_foreign_proxies(size=fetch_batch_size, countries=countries)
|
||||
if new_items:
|
||||
self.add_proxies(new_items)
|
||||
self.verify_all(max_workers=max_workers, max_proxies=verify_batch_size)
|
||||
with self._lock:
|
||||
working_count = len(self.working_proxies)
|
||||
if working_count >= target_working:
|
||||
stable_hits += 1
|
||||
else:
|
||||
stable_hits = 0
|
||||
with self._lock:
|
||||
return list(self.working_proxies)
|
||||
|
||||
def get_proxy(self, rotate: bool = True) -> Optional[ProxyInfo]:
|
||||
with self._lock:
|
||||
if not self.working_proxies:
|
||||
return None
|
||||
if not rotate:
|
||||
return random.choice(self.working_proxies)
|
||||
proxy = self.working_proxies[self.current_index % len(self.working_proxies)]
|
||||
self.current_index += 1
|
||||
return proxy
|
||||
|
||||
def mark_failed(self, proxy: ProxyInfo) -> None:
|
||||
with self._lock:
|
||||
proxy.fail_count += 1
|
||||
if proxy.fail_count >= 3 and proxy in self.working_proxies:
|
||||
self.working_proxies.remove(proxy)
|
||||
@@ -0,0 +1,145 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
|
||||
|
||||
DEFAULT_THUMBNAIL_DIR = runtime_path("thumbnails")
|
||||
SUPPORTED_IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ThumbnailResult:
|
||||
artwork_id: int
|
||||
page_index: int
|
||||
source_path: str
|
||||
thumbnail_path: str
|
||||
width: int
|
||||
height: int
|
||||
generated: bool
|
||||
|
||||
|
||||
class ThumbnailCache:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db_path: str | Path = DEFAULT_LIBRARY_DB,
|
||||
cache_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
|
||||
size: tuple[int, int] = (360, 360),
|
||||
quality: int = 86,
|
||||
) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.cache_dir = Path(cache_dir)
|
||||
self.size = size
|
||||
self.quality = quality
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def ensure_for_artwork(self, artwork_id: int) -> list[ThumbnailResult]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
rows = library.connection.execute(
|
||||
"""
|
||||
SELECT artwork_id, page_index, file_path
|
||||
FROM artwork_files
|
||||
WHERE artwork_id = ?
|
||||
ORDER BY page_index ASC
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchall()
|
||||
return [result for row in rows if (result := self.ensure_for_file(library, row)) is not None]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def build_missing(self, *, limit: int = 200) -> list[ThumbnailResult]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
rows = library.connection.execute(
|
||||
"""
|
||||
SELECT f.artwork_id, f.page_index, f.file_path
|
||||
FROM artwork_files f
|
||||
LEFT JOIN thumbnail_cache t
|
||||
ON t.artwork_id = f.artwork_id AND t.page_index = f.page_index
|
||||
WHERE t.thumbnail_path IS NULL
|
||||
OR t.source_path != f.file_path
|
||||
ORDER BY f.updated_at DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [result for row in rows if (result := self.ensure_for_file(library, row)) is not None]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def ensure_for_file(self, library: PixivLibrary, row) -> ThumbnailResult | None:
|
||||
source_path = Path(str(row["file_path"]))
|
||||
if source_path.suffix.lower() not in SUPPORTED_IMAGE_SUFFIXES or not source_path.exists():
|
||||
return None
|
||||
stat = source_path.stat()
|
||||
thumbnail_path = self._thumbnail_path(int(row["artwork_id"]), int(row["page_index"]), source_path)
|
||||
|
||||
cached = library.connection.execute(
|
||||
"""
|
||||
SELECT thumbnail_path, width, height, source_mtime, source_size
|
||||
FROM thumbnail_cache
|
||||
WHERE artwork_id = ? AND page_index = ?
|
||||
""",
|
||||
(int(row["artwork_id"]), int(row["page_index"])),
|
||||
).fetchone()
|
||||
if (
|
||||
cached
|
||||
and Path(str(cached["thumbnail_path"])).exists()
|
||||
and float(cached["source_mtime"]) == stat.st_mtime
|
||||
and int(cached["source_size"]) == stat.st_size
|
||||
):
|
||||
return ThumbnailResult(
|
||||
artwork_id=int(row["artwork_id"]),
|
||||
page_index=int(row["page_index"]),
|
||||
source_path=str(source_path),
|
||||
thumbnail_path=str(cached["thumbnail_path"]),
|
||||
width=int(cached["width"]),
|
||||
height=int(cached["height"]),
|
||||
generated=False,
|
||||
)
|
||||
|
||||
width, height = self._generate(source_path, thumbnail_path)
|
||||
library.upsert_thumbnail(
|
||||
artwork_id=int(row["artwork_id"]),
|
||||
page_index=int(row["page_index"]),
|
||||
source_path=str(source_path),
|
||||
thumbnail_path=str(thumbnail_path),
|
||||
width=width,
|
||||
height=height,
|
||||
source_mtime=stat.st_mtime,
|
||||
source_size=stat.st_size,
|
||||
)
|
||||
return ThumbnailResult(
|
||||
artwork_id=int(row["artwork_id"]),
|
||||
page_index=int(row["page_index"]),
|
||||
source_path=str(source_path),
|
||||
thumbnail_path=str(thumbnail_path),
|
||||
width=width,
|
||||
height=height,
|
||||
generated=True,
|
||||
)
|
||||
|
||||
def _generate(self, source_path: Path, thumbnail_path: Path) -> tuple[int, int]:
|
||||
try:
|
||||
from PIL import Image, ImageOps
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("Pillow is required to generate thumbnails. Install it with: pip install Pillow") from exc
|
||||
|
||||
thumbnail_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with Image.open(source_path) as image:
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image.thumbnail(self.size, Image.Resampling.LANCZOS)
|
||||
if image.mode not in {"RGB", "L"}:
|
||||
image = image.convert("RGB")
|
||||
image.save(thumbnail_path, format="JPEG", quality=self.quality, optimize=True)
|
||||
return image.size
|
||||
|
||||
def _thumbnail_path(self, artwork_id: int, page_index: int, source_path: Path) -> Path:
|
||||
bucket = str(abs(artwork_id) % 1000).zfill(3)
|
||||
return self.cache_dir / bucket / f"{artwork_id}_{page_index}_{source_path.stem}.jpg"
|
||||
@@ -0,0 +1,405 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.cookie_import import (
|
||||
cookie_summary,
|
||||
cookies_to_playwright,
|
||||
parse_cookie_text,
|
||||
save_playwright_cookies,
|
||||
)
|
||||
from pixiv_app.core.paths import (
|
||||
app_session_file,
|
||||
downloads_root,
|
||||
plugin_roots,
|
||||
plugins_root as app_plugins_root,
|
||||
runtime_path,
|
||||
)
|
||||
from pixiv_app.core.plugin.base import Resource
|
||||
from pixiv_app.core.plugin.manager import PluginManager
|
||||
|
||||
|
||||
def project_root() -> Path:
|
||||
return app_plugins_root().parent
|
||||
|
||||
|
||||
class XMediaService:
|
||||
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
|
||||
VIDEO_EXTS = {".mp4", ".mov", ".webm", ".m4v"}
|
||||
|
||||
def __init__(self, *, plugins_root: str | Path | None = None, save_root: str | Path | None = None) -> None:
|
||||
self.plugins_root = Path(plugins_root) if plugins_root else app_plugins_root()
|
||||
self.save_root = Path(save_root) if save_root else downloads_root() / "x"
|
||||
self.cookie_path = runtime_path("x_cookies.json")
|
||||
self.follow_store_path = runtime_path("x_followed_authors.json")
|
||||
self._manager: PluginManager | None = None
|
||||
self._preview_cache: dict[str, list[Resource]] = {}
|
||||
|
||||
def import_cookie_text(self, text: str) -> dict[str, Any]:
|
||||
cookies = parse_cookie_text(text)
|
||||
x_cookies = cookies_to_playwright(cookies, ("x.com", "twitter.com"))
|
||||
path = save_playwright_cookies(x_cookies, self.cookie_path)
|
||||
plugin = self._get_plugin()
|
||||
if hasattr(plugin, "reset_playwright_session"):
|
||||
plugin.reset_playwright_session()
|
||||
return {"count": len(x_cookies), "path": str(path), "summary": cookie_summary(x_cookies)}
|
||||
|
||||
def preview_author_media(self, username: str, *, media_type: str = "all", max_items: int = 120) -> dict[str, Any]:
|
||||
handle = _normalize_username(username)
|
||||
media_type = _normalize_media_type(media_type)
|
||||
max_items = max(1, min(int(max_items), 500))
|
||||
url = f"https://x.com/{handle}/media"
|
||||
|
||||
plugin = self._get_plugin()
|
||||
if not plugin.validate():
|
||||
raise ValueError("X plugin is unavailable. Install playwright/httpx and run playwright install chromium first.")
|
||||
|
||||
resources = plugin.parse(url)
|
||||
resources = _filter_resources(resources, media_type)[:max_items]
|
||||
preview_id = uuid.uuid4().hex
|
||||
self._preview_cache[preview_id] = resources
|
||||
return {
|
||||
"preview_id": preview_id,
|
||||
"username": handle,
|
||||
"media_type": media_type,
|
||||
"count": len(resources),
|
||||
"items": [_resource_to_json(item, i) for i, item in enumerate(resources)],
|
||||
"created_at": int(time.time()),
|
||||
}
|
||||
|
||||
def download_preview(self, preview_id: str, *, selected_ids: list[str] | None = None) -> dict[str, Any]:
|
||||
resources = self._preview_cache.get(preview_id)
|
||||
if resources is None:
|
||||
raise ValueError("Preview expired. Re-run the X media preview.")
|
||||
selected = set(selected_ids or [])
|
||||
if selected:
|
||||
resources = [item for item in resources if item.id in selected]
|
||||
if not resources:
|
||||
raise ValueError("No X media items selected.")
|
||||
|
||||
plugin = self._get_plugin()
|
||||
self.save_root.mkdir(parents=True, exist_ok=True)
|
||||
results: list[dict[str, Any]] = []
|
||||
downloaded = 0
|
||||
failed = 0
|
||||
for resource in resources:
|
||||
try:
|
||||
paths = plugin.download(resource, self.save_root)
|
||||
downloaded += len(paths)
|
||||
results.append(
|
||||
{
|
||||
"id": resource.id,
|
||||
"ok": bool(paths),
|
||||
"paths": [str(path) for path in paths],
|
||||
"title": resource.title,
|
||||
"synced_files": 0,
|
||||
}
|
||||
)
|
||||
if not paths:
|
||||
failed += 1
|
||||
except Exception as exc: # Keep batch downloads moving.
|
||||
failed += 1
|
||||
results.append({"id": resource.id, "ok": False, "error": str(exc), "title": resource.title})
|
||||
|
||||
return {
|
||||
"requested": len(resources),
|
||||
"downloaded_files": downloaded,
|
||||
"synced_files": 0,
|
||||
"failed_items": failed,
|
||||
"items": results,
|
||||
}
|
||||
|
||||
def list_local_media(self, *, limit: int = 160, offset: int = 0) -> dict[str, Any]:
|
||||
self.save_root.mkdir(parents=True, exist_ok=True)
|
||||
limit = max(1, min(int(limit), 500))
|
||||
offset = max(0, int(offset))
|
||||
files = [
|
||||
path
|
||||
for path in self.save_root.rglob("*")
|
||||
if path.is_file() and path.suffix.lower() in self.IMAGE_EXTS | self.VIDEO_EXTS
|
||||
]
|
||||
files.sort(key=lambda path: path.stat().st_mtime, reverse=True)
|
||||
page = files[offset : offset + limit]
|
||||
return {
|
||||
"total": len(files),
|
||||
"items": [self._local_media_to_json(path) for path in page],
|
||||
}
|
||||
|
||||
def delete_local_media(self, media_ids: list[str]) -> dict[str, Any]:
|
||||
deleted: list[str] = []
|
||||
failed: list[dict[str, str]] = []
|
||||
for media_id in media_ids:
|
||||
try:
|
||||
path = self.resolve_local_media(media_id)
|
||||
path.unlink()
|
||||
deleted.append(media_id)
|
||||
except Exception as exc:
|
||||
failed.append({"id": media_id, "error": str(exc)})
|
||||
return {"deleted": deleted, "failed": failed}
|
||||
|
||||
def resolve_local_media(self, media_id: str) -> Path:
|
||||
rel_text = _decode_media_id(media_id)
|
||||
target = (self.save_root / rel_text).resolve()
|
||||
root = self.save_root.resolve()
|
||||
if root != target and root not in target.parents:
|
||||
raise ValueError("Illegal media path.")
|
||||
if not target.is_file():
|
||||
raise ValueError("Local media file does not exist.")
|
||||
return target
|
||||
|
||||
def list_followed_authors(self) -> list[dict[str, Any]]:
|
||||
rows = [item for item in self._load_follow_rows() if bool(item.get("enabled", True))]
|
||||
rows.sort(key=lambda item: ((str(item.get("creator_name") or "") or str(item.get("username") or "")).lower()))
|
||||
return rows
|
||||
|
||||
def set_followed_author(self, username: str, *, followed: bool = True, creator_name: str = "") -> dict[str, Any]:
|
||||
handle = _normalize_username(username)
|
||||
rows = self._load_follow_rows()
|
||||
row = next((item for item in rows if str(item.get("username", "")).lower() == handle.lower()), None)
|
||||
now = _now_text()
|
||||
if row is None:
|
||||
row = {
|
||||
"username": handle,
|
||||
"creator_name": creator_name.strip() or handle,
|
||||
"enabled": True,
|
||||
"profile_url": f"https://x.com/{handle}",
|
||||
"latest_tweet_id": "",
|
||||
"latest_published_at": "",
|
||||
"last_checked_at": "",
|
||||
"last_error": "",
|
||||
"seen_tweet_ids": [],
|
||||
"created_at": now,
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
row["username"] = handle
|
||||
row["creator_name"] = creator_name.strip() or str(row.get("creator_name") or handle)
|
||||
row["enabled"] = bool(followed)
|
||||
row["profile_url"] = f"https://x.com/{handle}"
|
||||
row["updated_at"] = now
|
||||
if not followed:
|
||||
row["last_error"] = ""
|
||||
|
||||
self._save_follow_rows(rows)
|
||||
return dict(row)
|
||||
|
||||
def sync_followed_authors(self, *, download: bool = True, max_new_per_author: int = 20) -> dict[str, Any]:
|
||||
followed = self.list_followed_authors()
|
||||
if not followed:
|
||||
return {"authors": []}
|
||||
|
||||
plugin = self._get_plugin()
|
||||
if not plugin.validate():
|
||||
raise ValueError("X plugin is unavailable. Import a valid X cookie and ensure Playwright is installed.")
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
rows = self._load_follow_rows()
|
||||
|
||||
for row in rows:
|
||||
if not bool(row.get("enabled", True)):
|
||||
continue
|
||||
|
||||
handle = str(row.get("username") or "").strip()
|
||||
if not handle:
|
||||
continue
|
||||
|
||||
try:
|
||||
parse_for_sync = getattr(plugin, "parse_for_sync", plugin.parse)
|
||||
resources = parse_for_sync(f"https://x.com/{handle}")
|
||||
grouped = _group_resources_by_tweet(resources)
|
||||
seen_ids = [str(item) for item in row.get("seen_tweet_ids", []) if str(item).strip()]
|
||||
seen_set = set(seen_ids)
|
||||
new_groups = [item for item in grouped if item["tweet_id"] and item["tweet_id"] not in seen_set]
|
||||
new_groups = new_groups[: max(0, int(max_new_per_author))]
|
||||
|
||||
downloaded_files = 0
|
||||
synced_files = 0
|
||||
created_alerts = 0
|
||||
new_ids: list[str] = []
|
||||
|
||||
for group in new_groups:
|
||||
representative = group["resource"]
|
||||
new_ids.append(group["tweet_id"])
|
||||
|
||||
if download:
|
||||
for resource in group["resources"]:
|
||||
paths = plugin.download(resource, self.save_root)
|
||||
downloaded_files += len(paths)
|
||||
|
||||
latest_group = grouped[0] if grouped else None
|
||||
if latest_group is not None:
|
||||
representative = latest_group["resource"]
|
||||
row["creator_name"] = representative.author or str(row.get("creator_name") or handle)
|
||||
row["latest_tweet_id"] = latest_group["tweet_id"]
|
||||
row["latest_published_at"] = representative.created_at or str(row.get("latest_published_at") or "")
|
||||
|
||||
row["last_checked_at"] = _now_text()
|
||||
row["last_error"] = ""
|
||||
row["updated_at"] = _now_text()
|
||||
row["seen_tweet_ids"] = _merge_seen_ids(new_ids, seen_ids)
|
||||
|
||||
results.append(
|
||||
{
|
||||
"username": handle,
|
||||
"creator_name": row.get("creator_name") or handle,
|
||||
"new_posts": len(new_groups),
|
||||
"downloaded_files": downloaded_files,
|
||||
"synced_files": synced_files,
|
||||
"alerts": created_alerts,
|
||||
"error": "",
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
row["last_checked_at"] = _now_text()
|
||||
row["last_error"] = str(exc)
|
||||
row["updated_at"] = _now_text()
|
||||
results.append(
|
||||
{
|
||||
"username": handle,
|
||||
"creator_name": row.get("creator_name") or handle,
|
||||
"new_posts": 0,
|
||||
"downloaded_files": 0,
|
||||
"synced_files": 0,
|
||||
"alerts": 0,
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
self._save_follow_rows(rows)
|
||||
return {"authors": results}
|
||||
|
||||
def _get_plugin(self):
|
||||
if self._manager is None:
|
||||
self._manager = PluginManager(plugin_roots() if self.plugins_root == app_plugins_root() else [self.plugins_root])
|
||||
self._manager.load_all()
|
||||
plugin = self._manager.plugins.get("X")
|
||||
if plugin is None:
|
||||
raise ValueError("X plugin is missing. Confirm plugins/x exists.")
|
||||
return plugin
|
||||
|
||||
def _load_follow_rows(self) -> list[dict[str, Any]]:
|
||||
if not self.follow_store_path.is_file():
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(self.follow_store_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return []
|
||||
if isinstance(payload, list):
|
||||
return [dict(item) for item in payload if isinstance(item, dict)]
|
||||
return []
|
||||
|
||||
def _save_follow_rows(self, rows: list[dict[str, Any]]) -> None:
|
||||
self.follow_store_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.follow_store_path.write_text(
|
||||
json.dumps(rows, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _session_payload(self) -> dict[str, Any]:
|
||||
return SessionStore(app_session_file()).load()
|
||||
|
||||
def _local_media_to_json(self, path: Path) -> dict[str, Any]:
|
||||
rel = path.relative_to(self.save_root)
|
||||
stat = path.stat()
|
||||
kind = "video" if path.suffix.lower() in self.VIDEO_EXTS else "image"
|
||||
media_id = _encode_media_id(str(rel).replace("\\", "/"))
|
||||
return {
|
||||
"id": media_id,
|
||||
"name": path.name,
|
||||
"relative_path": str(rel).replace("\\", "/"),
|
||||
"kind": kind,
|
||||
"size": stat.st_size,
|
||||
"modified_at": int(stat.st_mtime),
|
||||
"url": f"/media/x-local/{media_id}",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_username(username: str) -> str:
|
||||
value = username.strip()
|
||||
if value.startswith("@"):
|
||||
value = value[1:]
|
||||
if "/" in value:
|
||||
parts = [item for item in value.replace("https://", "").replace("http://", "").split("/") if item]
|
||||
value = parts[1] if parts and parts[0].lower() in {"x.com", "twitter.com", "www.x.com", "www.twitter.com"} else parts[0]
|
||||
if not value:
|
||||
raise ValueError("Please enter an X username.")
|
||||
if len(value) > 15 or not value.replace("_", "").isalnum():
|
||||
raise ValueError("Invalid X username format.")
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_media_type(media_type: str) -> str:
|
||||
value = (media_type or "all").strip().lower()
|
||||
if value not in {"all", "image", "video"}:
|
||||
raise ValueError("media_type must be all, image, or video.")
|
||||
return value
|
||||
|
||||
|
||||
def _filter_resources(resources: list[Resource], media_type: str) -> list[Resource]:
|
||||
if media_type == "all":
|
||||
return resources
|
||||
return [item for item in resources if any(file.get("type", "image") == media_type for file in item.files)]
|
||||
|
||||
|
||||
def _resource_to_json(resource: Resource, index: int) -> dict[str, Any]:
|
||||
payload = asdict(resource)
|
||||
payload["index"] = index
|
||||
payload["kind"] = "video" if any(file.get("type") == "video" for file in resource.files) else "image"
|
||||
payload["file_count"] = len(resource.files)
|
||||
return payload
|
||||
|
||||
|
||||
def _group_resources_by_tweet(resources: list[Resource]) -> list[dict[str, Any]]:
|
||||
grouped: dict[str, dict[str, Any]] = {}
|
||||
ordered: list[str] = []
|
||||
for resource in resources:
|
||||
tweet_id = str(resource.metadata.get("tweet_id") or resource.id or "")
|
||||
key = tweet_id or resource.id
|
||||
if key not in grouped:
|
||||
grouped[key] = {
|
||||
"tweet_id": tweet_id,
|
||||
"resource": resource,
|
||||
"resources": [resource],
|
||||
}
|
||||
ordered.append(key)
|
||||
else:
|
||||
grouped[key]["resources"].append(resource)
|
||||
return [grouped[key] for key in ordered]
|
||||
|
||||
|
||||
def _merge_seen_ids(new_ids: list[str], existing_ids: list[str], *, limit: int = 200) -> list[str]:
|
||||
merged: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in [*new_ids, *existing_ids]:
|
||||
value = str(item).strip()
|
||||
if not value or value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
merged.append(value)
|
||||
if len(merged) >= limit:
|
||||
break
|
||||
return merged
|
||||
|
||||
|
||||
def _encode_media_id(value: str) -> str:
|
||||
from base64 import urlsafe_b64encode
|
||||
|
||||
return urlsafe_b64encode(value.encode("utf-8")).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def _decode_media_id(value: str) -> str:
|
||||
from base64 import urlsafe_b64decode
|
||||
|
||||
padding = "=" * (-len(value) % 4)
|
||||
return urlsafe_b64decode((value + padding).encode("ascii")).decode("utf-8")
|
||||
|
||||
|
||||
def _now_text() -> str:
|
||||
return datetime.now().isoformat(timespec="seconds")
|
||||
Reference in new issue
Block a user