first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+42
View File
@@ -0,0 +1,42 @@
# Sakura Cloud example config. Copy to .env for local backend development.
APP_NAME=Sakura Cloud
APP_VERSION=0.1.0
API_PREFIX=/api/v1
DEBUG=true
DATABASE_URL=postgresql+psycopg://sakura:sakura@localhost:5432/sakura_cloud
REDIS_URL=redis://localhost:6379/0
STORAGE_ROOT=data/sakura
LOG_DIR=data/sakura/logs
JWT_SECRET_KEY=change-this-secret-in-production
JWT_ALGORITHM=HS256
JWT_ACCESS_TOKEN_EXPIRE_HOURS=24
JWT_REFRESH_TOKEN_EXPIRE_DAYS=30
DEFAULT_STORAGE_LIMIT_GB=5
MAX_UPLOAD_SIZE_MB=100
MAX_VIDEO_UPLOAD_SIZE_MB=2048
VIDEO_STORAGE_LIMIT_GB=30
MAX_BATCH_SIZE_MB=500
UPLOAD_CHUNK_SIZE=5242880
TEMP_UPLOAD_TTL_HOURS=24
THUMBNAIL_MAX_SIZE=640
THUMBNAIL_QUALITY=82
FCM_ENABLED=true
FIREBASE_CREDENTIALS_PATH=
FIREBASE_CREDENTIALS_JSON=
OCR_ENABLED=false
OCR_LANGUAGE=ch
OCR_MAX_IMAGE_DIMENSION=1800
OCR_MAX_ATTEMPTS=3
OCR_MAX_CONCURRENCY=1
ADMIN_LOGIN_RATE_LIMIT_PER_MINUTE=5
ADMIN_FRONTEND_ENABLED=true
CORS_ORIGINS=["http://localhost:5173","http://localhost:3000"]
+60
View File
@@ -0,0 +1,60 @@
# Python cache and env
__pycache__/
*.pyc
.venv/
# IDE
.idea/
.vscode/
# Local toolchains (SDKs, caches) — keep only scripts in tools/
tools/android-sdk/
tools/android-commandlinetools.zip
tools/flutter/
tools/pub-cache/
tools/python314/
# Build artifacts
build/
dist/
release/
releases/*
!releases/.gitkeep
*.spec
!build_assets/PixivDownloader.spec
!build_assets/SakuraDownloader.spec
# Runtime and sensitive local data
runtime/
profiles/
Pixiv_Downloads/
Sakura_Downloads/
data/
pixiv_app_session.json
pixiv_app_library.db*
.env
.env.*
!.env.cloud.example
firebase-service-account*.json
sakura_mobile/android/app/google-services.json
# Flutter generated files
sakura_mobile/.dart_tool/
sakura_mobile/build/
sakura_mobile/.flutter-plugins
sakura_mobile/.flutter-plugins-dependencies
# Local project docs and dev logs (not code)
项目文件/
日志/
# AI assistant / debug local files
.claude/
.chatgpt/
*.log
debug/
# Web frontend generated files
sakura_admin_web/node_modules/
sakura_admin_web/dist/
sakura_admin_web/.npm-cache/
+104
View File
@@ -0,0 +1,104 @@
# Sakura Downloader(樱云下载器)
个人媒体采集、本地图库管理与跨设备云同步系统。
| 端 | 角色 |
|----|------|
| Windows 桌面端 | 采集与整理中枢 |
| Sakura Cloud | 数据与账号中心 |
| Android 客户端 | 跨设备阅读与提醒 |
## 成品文件(`releases/`)
| 文件 | 说明 | 大小(约) |
|------|------|------------|
| `releases/sakuradownloader.exe` | Windows 桌面端,双击即用 | ~68 MB |
| `releases/app-release.apk` | Android 客户端(调试签名,适合自测) | ~54 MB |
构建产物来源:
- EXE:`dist/sakuradownloader.exe` → 复制到 `releases/`
- APK:`sakura_mobile/build/app/outputs/flutter-apk/app-release.apk` → 复制到 `releases/`
## 文档索引
| 文档 | 面向读者 | 内容 |
|------|----------|------|
| [使用说明-桌面版与安卓版.md](使用说明-桌面版与安卓版.md) | 终端用户 | 登录、下载、云同步、手机端操作 |
| [docs/项目功能交付书.md](docs/项目功能交付书.md) | 产品 / 验收 | 功能模块、架构、API、代码索引 |
| [docs/项目文档.md](docs/项目文档.md) | 开发者 | 环境、目录、构建、部署、开发流程 |
| [项目文件/BUILD.md](项目文件/BUILD.md) | 开发者 | EXE / APK 打包细节与排错 |
## 快速开始(源码)
### 桌面端
```powershell
python -m venv .venv
.\.venv\Scripts\activate
pip install -r requirements.txt
python main.py
```
### 云端(可选)
```powershell
pip install -r sakura_cloud/requirements.txt
uvicorn sakura_cloud.main:app --reload
```
### 移动端(开发)
```powershell
cd sakura_mobile
flutter pub get
flutter run
```
## 一键打包
```powershell
# 桌面 EXE
scripts\build_desktop_exe.bat
# Android APK(需 JDK 17+,仓库内 tools\flutter 与 tools\android-sdk)
scripts\build_mobile_android.bat
# 打包后复制到 releases/
scripts\copy_release_artifacts.bat
# 或依次执行 EXE + APK + 复制
scripts\build_all.bat
```
## 技术栈摘要
- 桌面:Python 3.8+、CustomTkinter、SQLite、Playwright(X 插件)
- 云端:FastAPI、PostgreSQL、JWT
- 移动:Flutter、Riverpod、Dio
## 目录结构(精简)
```text
Pixiv/
├── main.py # 桌面入口
├── pixiv_app/ # 桌面核心(GUI、下载、图库、云同步)
├── plugins/ # Pixiv / X 等插件
├── sakura_cloud/ # 云端 API、docker-compose、Alembic
├── sakura_mobile/ # Flutter 安卓端
├── releases/ # 可分发成品(EXE / APK)
├── docs/ # 交付书、开发文档
├── scripts/ # 构建与依赖安装脚本
├── tools/ # 辅助脚本(如代码统计)
└── 项目文件/ # 迭代与专题说明
```
## 许可与说明
- APK 当前为 **调试签名**,仅供个人测试,不宜直接上架分发。
- 使用 X 插件前需安装 Playwright Chromium(见 `项目文件/BUILD.md`)。
- 会话与图库数据默认保存在 `%LOCALAPPDATA%\PixivDownloader`(打包版)。
---
*最后构建:2026-06-05*
+115
View File
@@ -0,0 +1,115 @@
# -*- mode: python ; coding: utf-8 -*-
from __future__ import annotations
from pathlib import Path
block_cipher = None
spec_path = Path(SPECPATH).resolve()
root = spec_path.parent.parent if spec_path.suffix == ".spec" else spec_path.parent
def tree_datas(source: str, dest: str, suffixes: tuple[str, ...] | None = None):
base = root / source
items = []
if not base.exists():
return items
for path in base.rglob("*"):
if not path.is_file():
continue
parts = set(path.parts)
if "__pycache__" in parts:
continue
if path.suffix.lower() in {".pyc", ".pyo"}:
continue
if suffixes and path.suffix.lower() not in suffixes:
continue
rel_parent = path.parent.relative_to(base)
items.append((str(path), str(Path(dest) / rel_parent)))
return items
datas = []
datas += tree_datas("pixiv_app/webui", "pixiv_app/webui", (".html", ".css", ".js", ".png", ".jpg", ".jpeg", ".webp", ".svg", ".ico"))
datas += [
(src, dst)
for src, dst in tree_datas("plugins", "plugins", (".py", ".json", ".md", ".txt"))
if not any(
marker in src
for marker in (
"plugins\\ao3",
"plugins/ao3",
"plugins\\changpei",
"plugins/changpei",
"plugins\\taobao",
"plugins/taobao",
)
)
]
datas += [(str(root / "build_assets" / "app_icon.ico"), "build_assets")]
hiddenimports = [
"pixiv_app.core.fingerprint",
"pixiv_app.core.browser_proxy_pool",
"PIL",
"PIL.Image",
"cryptography",
"cryptography.hazmat.primitives",
"cryptography.hazmat.primitives.padding",
"cryptography.hazmat.primitives.ciphers",
"customtkinter",
"requests",
"httpx",
"playwright",
"playwright.async_api",
"playwright.sync_api",
"pixiv_app.core.playwright_runtime",
"pixiv_app.services.x_login_flow",
]
a = Analysis(
[str(root / "main.py")],
pathex=[str(root)],
binaries=[],
datas=datas,
hiddenimports=hiddenimports,
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[
"pytest",
"unittest",
"tkinter.test",
],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False,
)
pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
[],
name="sakuradownloader",
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
upx_exclude=[],
runtime_tmpdir=None,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
icon=str(root / "build_assets" / "app_icon.ico"),
)
Binary file not shown.

After

Width:  |  Height:  |  Size: 176 KiB

+116
View File
@@ -0,0 +1,116 @@
# Distributed Crawler
An enterprise-oriented distributed crawler scaffold built around `aiohttp` and `asyncio`.
## Included capabilities
- Master + Worker topology
- Redis-backed priority queues
- Consistent-hash worker assignment
- Bloom-filter task dedupe
- Dead-letter queue for exhausted failures
- Checkpoint + state recovery
- Task/request/worker trace headers
- Exponential backoff retries
- Domain-aware rate limiting
- Domain-level circuit breaker
- Proxy pool with health verification
- Proxy scoring with weighted selection
- Shared session store with Redis + local backup
- Result aggregation with JSONL, MongoDB, or PostgreSQL sinks and URL-hash dedupe
- APScheduler-driven periodic task submission
## Safety notes
This project intentionally keeps high-risk account automation, captcha bypass, and anti-protection evasion as extension points only.
Use it only against systems you are authorized to access and collect from.
## Directory layout
```text
distributed_crawler/
config.example.yaml
requirements.txt
crawler/
auth.py
backoff.py
bloom.py
config.py
hash_ring.py
http.py
master.py
models.py
proxy.py
queue.py
rate_limit.py
results.py
scheduler.py
sessions.py
signatures.py
spider.py
state.py
worker.py
scripts/
run_master.py
run_worker.py
```
## Quick start
1. Copy `config.example.yaml` to `config.yaml`.
2. Start Redis.
3. Install dependencies from `requirements.txt`.
4. Start the master:
```bash
python scripts/run_master.py --config config.yaml
```
5. Start one or more workers:
```bash
python scripts/run_worker.py --config config.yaml --worker-id worker-1
python scripts/run_worker.py --config config.yaml --worker-id worker-2
```
6. Submit a task:
```bash
curl -X POST http://127.0.0.1:8088/tasks ^
-H "Content-Type: application/json" ^
-d "{\"url\":\"https://httpbin.org/get\",\"spider\":\"generic_fetch\"}"
```
## Master endpoints
- `POST /tasks`
- `POST /heartbeat`
- `GET /workers`
- `GET /stats`
- `GET /health`
- `GET /dashboard` (feature and runtime panel)
## Failure handling and tracing
- Bloom sizing uses `master.bloom_capacity` and `master.bloom_false_positive_rate` to calculate bit count and hash count automatically. Set `master.bloom_hashes` only when you need to override the computed optimum.
- Final failed tasks are appended to `master.dead_letter_queue_name` with the original task, final result, retry count, worker ID, error, and HTTP status.
- Worker requests include `X-Task-Id`, `X-Request-Id`, and `X-Worker-Id` headers. Result payloads also include `request_id` for easier log/result correlation.
## Stability controls
- Proxy endpoints track success count, failure count, latency, cooldown, and a computed score. Workers choose healthy or cooled-down proxies with weighted random selection.
- `circuit_breaker` protects each domain independently. After repeated transport errors or 5xx responses, the domain opens briefly, then moves to half-open for recovery probing.
- Result storage writes a stable `url_hash`. MongoDB and PostgreSQL enforce uniqueness with upsert behavior; JSONL keeps a sidecar `.url_hashes` index.
## Auto login and shared sessions
- Enable `auth.enabled: true` and configure per-domain credentials in `config.yaml`.
- Worker marks sessions expired based on configured status codes/body keywords.
- When expired, worker triggers `AuthProvider.refresh_session()` and writes refreshed cookie to Redis + local backup via `DualSessionStore`.
- All workers read the same session keys (`session.redis_prefix`), so login state is shared.
## Extension points
- `crawler.auth.AuthProvider`
- `crawler.signatures.SignatureProvider`
- `crawler.spider.BaseSpider`
+93
View File
@@ -0,0 +1,93 @@
app_name: distributed-crawler
redis_dsn: redis://127.0.0.1:6379/0
checkpoint_dir: ./runtime/checkpoints
master:
host: 127.0.0.1
port: 8088
worker_ttl_seconds: 30
stale_task_seconds: 120
bloom_capacity: 1000000
bloom_false_positive_rate: 0.01
bloom_hashes:
queue_name: crawler
result_queue_name: crawler:results
dead_letter_queue_name: crawler:dead_letter
default_spider: generic_fetch
worker:
worker_id: worker-1
concurrency: 100
heartbeat_interval_seconds: 5
request_timeout_seconds: 20
connect_timeout_seconds: 10
retries: 4
base_backoff_seconds: 1
max_backoff_seconds: 16
download_dir: ./runtime/downloads
storage:
backend: jsonl
jsonl_path: ./runtime/results/results.jsonl
mongodb_dsn: mongodb://127.0.0.1:27017
mongodb_database: crawler
mongodb_collection: results
postgres_dsn: postgresql://postgres:postgres@127.0.0.1:5432/crawler
postgres_table: crawl_results
proxy:
enabled: false
verify_url: https://httpbin.org/ip
verify_timeout_seconds: 8
refresh_interval_seconds: 300
failure_threshold: 3
cooldown_seconds: 60
static_proxies: []
provider_api_url: ""
provider_auth_header: ""
circuit_breaker:
enabled: true
failure_threshold: 8
recovery_timeout_seconds: 30
half_open_success_threshold: 2
session:
enabled: true
redis_prefix: crawler:sessions
local_backup_path: ./runtime/sessions/sessions.json
auth:
enabled: false
domains:
example.com:
login_url: https://example.com/api/login
username: your_username
password: your_password
username_field: username
password_field: password
submit_type: json # json or form
extra_payload: {}
success_status_codes: [200]
expire_status_codes: [401, 403]
expire_body_keywords: ["login required", "please login", "unauthorized"]
rate_limit:
default_qps: 3.0
default_burst: 6
per_domain:
httpbin.org:
qps: 2.0
burst: 4
schedules:
- name: sample-httpbin
trigger: interval
seconds: 300
priority: medium
task:
spider: generic_fetch
url: https://httpbin.org/get
method: GET
metadata:
source: schedule
+5
View File
@@ -0,0 +1,5 @@
from .config import AppSettings, load_settings
from .master import MasterNode
from .worker import WorkerNode
__all__ = ["AppSettings", "MasterNode", "WorkerNode", "load_settings"]
+75
View File
@@ -0,0 +1,75 @@
from __future__ import annotations
import logging
from abc import ABC, abstractmethod
from typing import Any
import aiohttp
from .config import AuthSettings
LOGGER = logging.getLogger(__name__)
class AuthProvider(ABC):
@abstractmethod
async def refresh_session(self, domain: str, existing_session: dict[str, Any]) -> dict[str, Any]:
"""
Provide an authorized and compliant session refresh implementation for the target system.
"""
class ManualAuthProvider(AuthProvider):
async def refresh_session(self, domain: str, existing_session: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError(
f"Session refresh for {domain} must be implemented with authorized credentials and target-specific logic."
)
class ConfigurableAuthProvider(AuthProvider):
"""
Generic username/password login provider.
This intentionally keeps site-specific anti-bot behavior out of scope.
"""
def __init__(self, settings: AuthSettings, timeout_seconds: int = 20) -> None:
self.settings = settings
self.timeout_seconds = timeout_seconds
async def refresh_session(self, domain: str, existing_session: dict[str, Any]) -> dict[str, Any]:
if not self.settings.enabled:
return {**existing_session, "expired": False}
cfg = self.settings.domains.get(domain)
if cfg is None or not cfg.login_url or not cfg.username or not cfg.password:
LOGGER.warning("No auth config for domain=%s, keeping existing session", domain)
return {**existing_session, "expired": True}
payload: dict[str, Any] = dict(cfg.extra_payload)
payload[cfg.username_field] = cfg.username
payload[cfg.password_field] = cfg.password
timeout = aiohttp.ClientTimeout(total=self.timeout_seconds)
async with aiohttp.ClientSession(timeout=timeout) as session:
if cfg.submit_type.lower() == "json":
async with session.post(cfg.login_url, json=payload) as response:
body_text = await response.text()
if response.status not in set(cfg.success_status_codes):
raise RuntimeError(f"Login failed for {domain}: {response.status} {body_text[:180]}")
else:
async with session.post(cfg.login_url, data=payload) as response:
body_text = await response.text()
if response.status not in set(cfg.success_status_codes):
raise RuntimeError(f"Login failed for {domain}: {response.status} {body_text[:180]}")
cookie_pairs = [f"{k}={v.value}" for k, v in session.cookie_jar.filter_cookies(cfg.login_url).items()]
cookie_header = "; ".join(cookie_pairs)
if not cookie_header:
# Fallback: keep old cookies if target does not set cookies in login response.
cookie_header = str(existing_session.get("cookie", ""))
return {
**existing_session,
"expired": False,
"cookie": cookie_header,
"updated_by": "ConfigurableAuthProvider",
}
+9
View File
@@ -0,0 +1,9 @@
from __future__ import annotations
import random
def exponential_backoff(attempt: int, base_seconds: float, max_seconds: float) -> float:
delay = min(max_seconds, base_seconds * (2 ** max(attempt, 0)))
jitter = random.uniform(0, delay * 0.2 if delay > 0 else 0)
return delay + jitter
+49
View File
@@ -0,0 +1,49 @@
from __future__ import annotations
import math
from hashlib import md5, sha1
from redis.asyncio import Redis
class RedisBloomFilter:
def __init__(self, redis: Redis, key: str, bits: int = 8_000_000, hashes: int = 6) -> None:
self.redis = redis
self.key = key
self.bits = max(int(bits), 1)
self.hashes = max(int(hashes), 1)
@classmethod
def from_capacity(
cls,
redis: Redis,
key: str,
*,
capacity: int,
false_positive_rate: float = 0.01,
hashes: int | None = None,
) -> "RedisBloomFilter":
capacity = max(int(capacity), 1)
false_positive_rate = min(max(float(false_positive_rate), 1e-9), 0.999999)
bits = math.ceil(-(capacity * math.log(false_positive_rate)) / (math.log(2) ** 2))
optimal_hashes = max(1, round((bits / capacity) * math.log(2)))
return cls(redis, key, bits=bits, hashes=hashes or optimal_hashes)
def _positions(self, value: str) -> list[int]:
digest_a = int(md5(value.encode("utf-8")).hexdigest(), 16)
digest_b = int(sha1(value.encode("utf-8")).hexdigest(), 16)
return [((digest_a + index * digest_b) % self.bits) for index in range(self.hashes)]
async def contains_or_add(self, value: str) -> bool:
positions = self._positions(value)
pipeline = self.redis.pipeline()
for position in positions:
pipeline.getbit(self.key, position)
existing = await pipeline.execute()
seen = all(bit == 1 for bit in existing)
pipeline = self.redis.pipeline()
for position in positions:
pipeline.setbit(self.key, position, 1)
await pipeline.execute()
return seen
@@ -0,0 +1,67 @@
from __future__ import annotations
from dataclasses import dataclass
from enum import Enum
from time import monotonic
from .config import CircuitBreakerSettings
class CircuitState(str, Enum):
CLOSED = "closed"
OPEN = "open"
HALF_OPEN = "half_open"
@dataclass
class Circuit:
state: CircuitState = CircuitState.CLOSED
failures: int = 0
half_open_successes: int = 0
opened_at: float = 0.0
class CircuitOpenError(RuntimeError):
pass
class DomainCircuitBreaker:
def __init__(self, settings: CircuitBreakerSettings) -> None:
self.settings = settings
self._circuits: dict[str, Circuit] = {}
def before_request(self, domain: str) -> None:
if not self.settings.enabled:
return
circuit = self._circuits.setdefault(domain, Circuit())
if circuit.state != CircuitState.OPEN:
return
elapsed = monotonic() - circuit.opened_at
if elapsed >= self.settings.recovery_timeout_seconds:
circuit.state = CircuitState.HALF_OPEN
circuit.half_open_successes = 0
return
raise CircuitOpenError(f"Circuit open for {domain}")
def record_success(self, domain: str) -> None:
if not self.settings.enabled:
return
circuit = self._circuits.setdefault(domain, Circuit())
if circuit.state == CircuitState.HALF_OPEN:
circuit.half_open_successes += 1
if circuit.half_open_successes < self.settings.half_open_success_threshold:
return
circuit.state = CircuitState.CLOSED
circuit.failures = 0
circuit.half_open_successes = 0
circuit.opened_at = 0.0
def record_failure(self, domain: str) -> None:
if not self.settings.enabled:
return
circuit = self._circuits.setdefault(domain, Circuit())
circuit.failures += 1
circuit.half_open_successes = 0
if circuit.failures >= self.settings.failure_threshold or circuit.state == CircuitState.HALF_OPEN:
circuit.state = CircuitState.OPEN
circuit.opened_at = monotonic()
+206
View File
@@ -0,0 +1,206 @@
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import yaml
@dataclass
class MasterSettings:
host: str = "127.0.0.1"
port: int = 8088
worker_ttl_seconds: int = 30
stale_task_seconds: int = 120
bloom_capacity: int = 1_000_000
bloom_false_positive_rate: float = 0.01
bloom_hashes: int | None = None
queue_name: str = "crawler"
result_queue_name: str = "crawler:results"
dead_letter_queue_name: str = "crawler:dead_letter"
default_spider: str = "generic_fetch"
@dataclass
class WorkerSettings:
worker_id: str = "worker-1"
concurrency: int = 100
heartbeat_interval_seconds: int = 5
request_timeout_seconds: int = 20
connect_timeout_seconds: int = 10
retries: int = 4
base_backoff_seconds: float = 1.0
max_backoff_seconds: float = 16.0
download_dir: str = "./runtime/downloads"
@dataclass
class StorageSettings:
backend: str = "jsonl"
jsonl_path: str = "./runtime/results/results.jsonl"
mongodb_dsn: str = "mongodb://127.0.0.1:27017"
mongodb_database: str = "crawler"
mongodb_collection: str = "results"
postgres_dsn: str = "postgresql://postgres:postgres@127.0.0.1:5432/crawler"
postgres_table: str = "crawl_results"
@dataclass
class ProxySettings:
enabled: bool = False
verify_url: str = "https://httpbin.org/ip"
verify_timeout_seconds: int = 8
refresh_interval_seconds: int = 300
failure_threshold: int = 3
cooldown_seconds: int = 60
static_proxies: list[str] = field(default_factory=list)
provider_api_url: str = ""
provider_auth_header: str = ""
@dataclass
class CircuitBreakerSettings:
enabled: bool = True
failure_threshold: int = 8
recovery_timeout_seconds: int = 30
half_open_success_threshold: int = 2
@dataclass
class SessionSettings:
enabled: bool = True
redis_prefix: str = "crawler:sessions"
local_backup_path: str = "./runtime/sessions/sessions.json"
@dataclass
class AuthDomainSettings:
login_url: str = ""
username: str = ""
password: str = ""
username_field: str = "username"
password_field: str = "password"
submit_type: str = "form"
extra_payload: dict[str, Any] = field(default_factory=dict)
success_status_codes: list[int] = field(default_factory=lambda: [200])
expire_status_codes: list[int] = field(default_factory=lambda: [401, 403])
expire_body_keywords: list[str] = field(default_factory=lambda: ["login required", "please login", "unauthorized"])
@dataclass
class AuthSettings:
enabled: bool = False
domains: dict[str, AuthDomainSettings] = field(default_factory=dict)
@dataclass
class DomainRateLimitSettings:
qps: float = 3.0
burst: int = 6
@dataclass
class RateLimitSettings:
default_qps: float = 3.0
default_burst: int = 6
per_domain: dict[str, DomainRateLimitSettings] = field(default_factory=dict)
@dataclass
class ScheduleTaskSettings:
name: str
trigger: str
task: dict[str, Any]
priority: str = "medium"
seconds: int | None = None
minutes: int | None = None
cron: str | None = None
@dataclass
class AppSettings:
app_name: str = "distributed-crawler"
redis_dsn: str = "redis://127.0.0.1:6379/0"
checkpoint_dir: str = "./runtime/checkpoints"
master: MasterSettings = field(default_factory=MasterSettings)
worker: WorkerSettings = field(default_factory=WorkerSettings)
storage: StorageSettings = field(default_factory=StorageSettings)
proxy: ProxySettings = field(default_factory=ProxySettings)
circuit_breaker: CircuitBreakerSettings = field(default_factory=CircuitBreakerSettings)
session: SessionSettings = field(default_factory=SessionSettings)
auth: AuthSettings = field(default_factory=AuthSettings)
rate_limit: RateLimitSettings = field(default_factory=RateLimitSettings)
schedules: list[ScheduleTaskSettings] = field(default_factory=list)
source_path: Path | None = None
@property
def base_dir(self) -> Path:
if self.source_path is None:
return Path.cwd()
return self.source_path.parent
def resolve_path(self, value: str) -> Path:
candidate = Path(value)
if candidate.is_absolute():
return candidate
return (self.base_dir / candidate).resolve()
def _load_domain_limits(raw_limits: dict[str, Any]) -> dict[str, DomainRateLimitSettings]:
result: dict[str, DomainRateLimitSettings] = {}
for domain, item in raw_limits.items():
result[domain] = DomainRateLimitSettings(
qps=float(item.get("qps", 3.0)),
burst=int(item.get("burst", 6)),
)
return result
def _load_auth_domains(raw_domains: dict[str, Any]) -> dict[str, AuthDomainSettings]:
result: dict[str, AuthDomainSettings] = {}
for domain, item in raw_domains.items():
if not isinstance(item, dict):
continue
result[domain] = AuthDomainSettings(
login_url=str(item.get("login_url", "")),
username=str(item.get("username", "")),
password=str(item.get("password", "")),
username_field=str(item.get("username_field", "username")),
password_field=str(item.get("password_field", "password")),
submit_type=str(item.get("submit_type", "form")),
extra_payload=dict(item.get("extra_payload", {})),
success_status_codes=[int(v) for v in item.get("success_status_codes", [200])],
expire_status_codes=[int(v) for v in item.get("expire_status_codes", [401, 403])],
expire_body_keywords=[str(v).lower() for v in item.get("expire_body_keywords", ["login required", "please login", "unauthorized"])],
)
return result
def load_settings(path: str | Path) -> AppSettings:
config_path = Path(path).resolve()
data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
settings = AppSettings(
app_name=str(data.get("app_name", "distributed-crawler")),
redis_dsn=str(data.get("redis_dsn", "redis://127.0.0.1:6379/0")),
checkpoint_dir=str(data.get("checkpoint_dir", "./runtime/checkpoints")),
master=MasterSettings(**(data.get("master") or {})),
worker=WorkerSettings(**(data.get("worker") or {})),
storage=StorageSettings(**(data.get("storage") or {})),
proxy=ProxySettings(**(data.get("proxy") or {})),
circuit_breaker=CircuitBreakerSettings(**(data.get("circuit_breaker") or {})),
session=SessionSettings(**(data.get("session") or {})),
auth=AuthSettings(
enabled=bool((data.get("auth") or {}).get("enabled", False)),
domains=_load_auth_domains((data.get("auth") or {}).get("domains", {})),
),
rate_limit=RateLimitSettings(
default_qps=float((data.get("rate_limit") or {}).get("default_qps", 3.0)),
default_burst=int((data.get("rate_limit") or {}).get("default_burst", 6)),
per_domain=_load_domain_limits((data.get("rate_limit") or {}).get("per_domain", {})),
),
schedules=[ScheduleTaskSettings(**item) for item in data.get("schedules", [])],
source_path=config_path,
)
return settings
+33
View File
@@ -0,0 +1,33 @@
from __future__ import annotations
from bisect import bisect
from hashlib import md5
class ConsistentHashRing:
def __init__(self, replicas: int = 100) -> None:
self.replicas = replicas
self._keys: list[int] = []
self._nodes: dict[int, str] = {}
def _hash(self, value: str) -> int:
return int(md5(value.encode("utf-8")).hexdigest(), 16)
def rebuild(self, nodes: list[str]) -> None:
self._keys.clear()
self._nodes.clear()
for node in sorted(set(nodes)):
for replica in range(self.replicas):
key = self._hash(f"{node}:{replica}")
self._keys.append(key)
self._nodes[key] = node
self._keys.sort()
def get_node(self, key: str) -> str | None:
if not self._keys:
return None
hashed = self._hash(key)
index = bisect(self._keys, hashed)
if index == len(self._keys):
index = 0
return self._nodes[self._keys[index]]
+180
View File
@@ -0,0 +1,180 @@
from __future__ import annotations
import json
import random
from pathlib import Path
from time import monotonic
from typing import Any
import aiohttp
from .auth import AuthProvider
from .circuit_breaker import DomainCircuitBreaker
from .config import AppSettings
from .models import CrawlTask
from .proxy import ProxyPool
from .rate_limit import DomainRateLimiter
from .sessions import DualSessionStore
from .signatures import SignatureProvider
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/126.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4) AppleWebKit/605.1.15 Version/17.4 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Edg/126.0.0.0 Safari/537.36",
]
class AsyncHttpClient:
def __init__(
self,
settings: AppSettings,
rate_limiter: DomainRateLimiter,
proxy_pool: ProxyPool,
session_store: DualSessionStore,
auth_provider: AuthProvider,
signature_provider: SignatureProvider,
) -> None:
self.settings = settings
self.rate_limiter = rate_limiter
self.proxy_pool = proxy_pool
self.session_store = session_store
self.auth_provider = auth_provider
self.signature_provider = signature_provider
self.session: aiohttp.ClientSession | None = None
self.circuit_breaker = DomainCircuitBreaker(settings.circuit_breaker)
async def start(self) -> None:
timeout = aiohttp.ClientTimeout(
total=self.settings.worker.request_timeout_seconds,
connect=self.settings.worker.connect_timeout_seconds,
)
connector = aiohttp.TCPConnector(limit=self.settings.worker.concurrency, ssl=False)
self.session = aiohttp.ClientSession(timeout=timeout, connector=connector)
async def close(self) -> None:
if self.session is not None:
await self.session.close()
async def fetch(self, task: CrawlTask) -> dict[str, Any]:
if self.session is None:
raise RuntimeError("HTTP client is not started")
await self.rate_limiter.acquire(task.domain)
self.circuit_breaker.before_request(task.domain)
proxy = await self.proxy_pool.acquire()
session_payload = await self.session_store.load(task.domain)
if self.settings.session.enabled and session_payload.get("expired"):
session_payload = await self.auth_provider.refresh_session(task.domain, session_payload)
await self.session_store.save(task.domain, session_payload)
headers = dict(task.headers)
headers.setdefault("User-Agent", random.choice(USER_AGENTS))
headers.setdefault("Accept", "*/*")
if task.metadata.get("referer"):
headers.setdefault("Referer", str(task.metadata["referer"]))
if task.metadata.get("origin"):
headers.setdefault("Origin", str(task.metadata["origin"]))
headers.setdefault("X-Task-Id", task.task_id)
if task.metadata.get("request_id"):
headers.setdefault("X-Request-Id", str(task.metadata["request_id"]))
if task.metadata.get("worker_id"):
headers.setdefault("X-Worker-Id", str(task.metadata["worker_id"]))
cookie_header = session_payload.get("cookie")
if cookie_header:
headers.setdefault("Cookie", str(cookie_header))
signature_headers = await self.signature_provider.sign(
{
"url": task.url,
"method": task.method,
"body": task.body,
"metadata": task.metadata,
}
)
headers.update(signature_headers)
started_at = monotonic()
try:
async with self.session.request(
task.method,
task.url,
headers=headers,
data=task.body.encode("utf-8") if isinstance(task.body, str) else None,
proxy=proxy.url if proxy else None,
) as response:
content_type = response.headers.get("Content-Type", "")
body_bytes = await response.read()
if "application/json" in content_type:
body: Any = json.loads(body_bytes.decode("utf-8", errors="ignore"))
else:
body = body_bytes.decode("utf-8", errors="ignore")
response_data = {
"status": response.status,
"headers": dict(response.headers),
"body": body,
"body_bytes": body_bytes,
"proxy": proxy.url if proxy else None,
}
if self.settings.session.enabled:
if self._is_session_expired(task.domain, response.status, body):
session_payload["expired"] = True
await self.session_store.save(task.domain, session_payload)
else:
if "Set-Cookie" in response.headers:
cookie_header = self._extract_cookie_header(response)
if cookie_header:
session_payload["cookie"] = cookie_header
session_payload["expired"] = False
await self.session_store.save(task.domain, session_payload)
if response.status >= 500:
self.circuit_breaker.record_failure(task.domain)
else:
self.circuit_breaker.record_success(task.domain)
await self.proxy_pool.mark_success(proxy, (monotonic() - started_at) * 1000)
return response_data
except Exception:
self.circuit_breaker.record_failure(task.domain)
await self.proxy_pool.mark_failed(proxy)
raise
async def save_binary(self, task: CrawlTask, file_name: str, body_bytes: bytes) -> str:
save_dir = self.settings.resolve_path(self.settings.worker.download_dir)
save_dir.mkdir(parents=True, exist_ok=True)
target = save_dir / file_name
target.write_bytes(body_bytes)
return str(target)
def _is_session_expired(self, domain: str, status_code: int, body: Any) -> bool:
if not self.settings.auth.enabled:
return False
domain_cfg = self.settings.auth.domains.get(domain)
if domain_cfg is None:
return False
if status_code in set(domain_cfg.expire_status_codes):
return True
if isinstance(body, str):
lower_body = body.lower()
return any(keyword in lower_body for keyword in domain_cfg.expire_body_keywords)
if isinstance(body, dict):
packed = json.dumps(body, ensure_ascii=False).lower()
return any(keyword in packed for keyword in domain_cfg.expire_body_keywords)
return False
def _extract_cookie_header(self, response: aiohttp.ClientResponse) -> str:
values = response.headers.getall("Set-Cookie", [])
cookie_parts: list[str] = []
for value in values:
head = value.split(";", 1)[0].strip()
if "=" in head:
cookie_parts.append(head)
# keep order and remove duplicates
seen: set[str] = set()
deduped: list[str] = []
for item in cookie_parts:
key = item.split("=", 1)[0].strip()
if key in seen:
continue
seen.add(key)
deduped.append(item)
return "; ".join(deduped)
+346
View File
@@ -0,0 +1,346 @@
from __future__ import annotations
import asyncio
import logging
from time import time
from typing import Any
from html import escape
from aiohttp import web
from redis.asyncio import Redis
from .bloom import RedisBloomFilter
from .config import AppSettings
from .hash_ring import ConsistentHashRing
from .models import CrawlResult, CrawlTask, DeadLetterEntry, TaskPriority, TaskStatus, WorkerHeartbeat
from .queue import RedisPriorityQueue
from .results import ResultRepository, build_result_repository
from .scheduler import ScheduleService
from .state import CheckpointManager
LOGGER = logging.getLogger(__name__)
class MasterNode:
def __init__(self, settings: AppSettings) -> None:
self.settings = settings
self.redis = Redis.from_url(settings.redis_dsn, decode_responses=True)
self.queue = RedisPriorityQueue(
self.redis,
settings.master.queue_name,
settings.master.result_queue_name,
settings.master.dead_letter_queue_name,
)
self.checkpoints = CheckpointManager(self.redis, settings.resolve_path(settings.checkpoint_dir), settings.master.queue_name)
self.bloom = RedisBloomFilter.from_capacity(
self.redis,
key=f"{settings.master.queue_name}:bloom",
capacity=settings.master.bloom_capacity,
false_positive_rate=settings.master.bloom_false_positive_rate,
hashes=settings.master.bloom_hashes,
)
self.hash_ring = ConsistentHashRing()
self.result_repo: ResultRepository = build_result_repository(
settings.storage,
settings.resolve_path(settings.storage.jsonl_path),
)
self.scheduler = ScheduleService(settings, self.submit_task)
self.workers: dict[str, WorkerHeartbeat] = {}
self.app = web.Application()
self.app.add_routes(
[
web.post("/tasks", self.handle_submit_task),
web.post("/heartbeat", self.handle_heartbeat),
web.get("/workers", self.handle_workers),
web.get("/stats", self.handle_stats),
web.get("/health", self.handle_health),
web.get("/dashboard", self.handle_dashboard),
]
)
self._background_tasks: list[asyncio.Task] = []
async def start(self) -> None:
await self.result_repo.setup()
await self._recover_local_checkpoints()
self.scheduler.start()
self._background_tasks = [
asyncio.create_task(self.consume_results_loop(), name="consume-results"),
asyncio.create_task(self.reap_stale_workers_loop(), name="reap-workers"),
]
async def close(self) -> None:
for task in self._background_tasks:
task.cancel()
await asyncio.gather(*self._background_tasks, return_exceptions=True)
await self.scheduler.shutdown()
await self.result_repo.close()
await self.redis.aclose()
async def submit_task(self, task: CrawlTask) -> bool:
duplicate = await self.bloom.contains_or_add(task.fingerprint)
if duplicate:
LOGGER.info("Skip duplicate task %s", task.url)
return False
worker_id = self._assign_worker(task)
task.assigned_worker = worker_id
task.status = TaskStatus.PENDING
await self.checkpoints.save(task)
await self.queue.enqueue_task(worker_id, task)
LOGGER.info("Submitted task %s to %s", task.task_id, worker_id)
return True
async def handle_submit_task(self, request: web.Request) -> web.Response:
payload = await request.json()
task = CrawlTask.create(
url=payload["url"],
spider=payload.get("spider", self.settings.master.default_spider),
method=payload.get("method", "GET"),
headers=payload.get("headers"),
metadata=payload.get("metadata"),
body=payload.get("body"),
priority=payload.get("priority", TaskPriority.MEDIUM.value),
max_retries=int(payload.get("max_retries", self.settings.worker.retries)),
)
accepted = await self.submit_task(task)
return web.json_response({"accepted": accepted, "task_id": task.task_id, "worker_id": task.assigned_worker})
async def handle_heartbeat(self, request: web.Request) -> web.Response:
payload = await request.json()
heartbeat = WorkerHeartbeat(
worker_id=str(payload["worker_id"]),
concurrency=int(payload["concurrency"]),
active_tasks=int(payload["active_tasks"]),
host=payload.get("host"),
tags=list(payload.get("tags", [])),
)
self.workers[heartbeat.worker_id] = heartbeat
self.hash_ring.rebuild(list(self.workers.keys()))
return web.json_response({"ok": True, "known_workers": len(self.workers)})
async def handle_workers(self, _: web.Request) -> web.Response:
return web.json_response({"workers": [item.to_dict() for item in self.workers.values()]})
async def handle_stats(self, _: web.Request) -> web.Response:
return web.json_response(
{
"workers": len(self.workers),
"queues": {
worker_id: {
priority.value: await self.redis.llen(self.queue._task_key(worker_id, priority))
for priority in TaskPriority
}
for worker_id in self.workers or {"default": None}
},
"dead_letters": await self.redis.llen(self.queue.dead_letter_queue_name),
}
)
async def handle_health(self, _: web.Request) -> web.Response:
pong = await self.redis.ping()
return web.json_response({"status": "ok", "redis": bool(pong), "workers": len(self.workers)})
async def handle_dashboard(self, _: web.Request) -> web.Response:
queue_stats = await self._collect_queue_stats()
session_count = await self._count_sessions()
dead_letter_count = await self.redis.llen(self.queue.dead_letter_queue_name)
html = self._render_dashboard_html(
queue_stats=queue_stats,
session_count=session_count,
dead_letter_count=dead_letter_count,
)
return web.Response(text=html, content_type="text/html")
async def consume_results_loop(self) -> None:
while True:
try:
result = await self.queue.consume_result(timeout=5)
if result is None:
continue
await self.process_result(result)
except asyncio.CancelledError:
raise
except Exception:
LOGGER.exception("Result consumer failed")
await asyncio.sleep(1)
async def process_result(self, result: CrawlResult) -> None:
task = await self.checkpoints.load(result.task_id)
if task is None:
LOGGER.warning("Result for unknown task %s", result.task_id)
return
task.status = result.status
await self.checkpoints.mark_status(task, result.status)
await self.result_repo.store(result)
if result.status == TaskStatus.FAILED:
await self.queue.publish_dead_letter(DeadLetterEntry.from_task_result(task, result))
LOGGER.warning(
"Dead-lettered task task_id=%s worker_id=%s request_id=%s reason=%s",
task.task_id,
result.worker_id,
result.data.get("request_id", task.metadata.get("request_id", task.task_id)),
result.error,
)
async def reap_stale_workers_loop(self) -> None:
while True:
try:
await asyncio.sleep(self.settings.master.worker_ttl_seconds)
now = time()
stale_workers = [
worker_id
for worker_id, heartbeat in self.workers.items()
if now - self._heartbeat_epoch(heartbeat.last_seen) > self.settings.master.worker_ttl_seconds
]
for worker_id in stale_workers:
LOGGER.warning("Worker %s is stale, recovering tasks", worker_id)
recovered = await self.checkpoints.recover_stale(worker_id, self.settings.master.stale_task_seconds)
for task in recovered:
task.retries += 1
if task.retries > task.max_retries:
task.status = TaskStatus.FAILED
await self.checkpoints.mark_status(task, TaskStatus.FAILED)
result = CrawlResult(
task_id=task.task_id,
worker_id=worker_id,
status=TaskStatus.FAILED,
url=task.url,
spider=task.spider,
error="Task exceeded retry budget during worker recovery",
)
await self.result_repo.store(result)
await self.queue.publish_dead_letter(DeadLetterEntry.from_task_result(task, result))
continue
task.assigned_worker = self._assign_worker(task, exclude=worker_id)
await self.checkpoints.save(task)
await self.queue.enqueue_task(task.assigned_worker, task)
self.workers.pop(worker_id, None)
self.hash_ring.rebuild(list(self.workers.keys()))
except asyncio.CancelledError:
raise
except Exception:
LOGGER.exception("Stale worker reaper failed")
async def _recover_local_checkpoints(self) -> None:
tasks = await self.checkpoints.recover_local()
for task in tasks:
task.assigned_worker = self._assign_worker(task)
task.status = TaskStatus.RETRY
await self.checkpoints.save(task)
await self.queue.enqueue_task(task.assigned_worker, task)
async def _collect_queue_stats(self) -> dict[str, dict[str, int]]:
worker_keys = list(self.workers.keys()) or ["default"]
result: dict[str, dict[str, int]] = {}
for worker_id in worker_keys:
result[worker_id] = {
priority.value: await self.redis.llen(self.queue._task_key(worker_id, priority))
for priority in TaskPriority
}
return result
async def _count_sessions(self) -> int:
pattern = f"{self.settings.session.redis_prefix}:*"
count = 0
async for _ in self.redis.scan_iter(match=pattern):
count += 1
return count
def _render_dashboard_html(self, queue_stats: dict[str, dict[str, int]], session_count: int, dead_letter_count: int) -> str:
worker_rows = []
for worker_id, hb in self.workers.items():
load = hb.active_tasks / max(hb.concurrency, 1)
worker_rows.append(
f"<tr><td>{escape(worker_id)}</td><td>{hb.active_tasks}</td><td>{hb.concurrency}</td><td>{load:.2f}</td><td>{escape(hb.last_seen)}</td></tr>"
)
if not worker_rows:
worker_rows.append("<tr><td colspan='5'>暂无在线 Worker</td></tr>")
queue_blocks = []
for worker_id, levels in queue_stats.items():
queue_blocks.append(
"<div class='card'><h3>Queue - "
+ escape(worker_id)
+ "</h3><p>high: "
+ str(levels.get("high", 0))
+ " | medium: "
+ str(levels.get("medium", 0))
+ " | low: "
+ str(levels.get("low", 0))
+ "</p></div>"
)
queue_html = "\n".join(queue_blocks)
enabled_auth_domains = ", ".join(sorted(self.settings.auth.domains.keys())) if self.settings.auth.domains else "-"
enabled_proxy = "enabled" if self.settings.proxy.enabled else "disabled"
enabled_session = "enabled" if self.settings.session.enabled else "disabled"
enabled_auth = "enabled" if self.settings.auth.enabled else "disabled"
return f"""<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<title>Distributed Crawler Dashboard</title>
<style>
body {{ font-family: Segoe UI, Arial, sans-serif; background:#f6f8fb; margin:0; padding:24px; color:#1f2d3d; }}
.grid {{ display:grid; grid-template-columns: repeat(auto-fill, minmax(280px,1fr)); gap:12px; margin-bottom:16px; }}
.card {{ background:#fff; border:1px solid #dce6f4; border-radius:10px; padding:12px 14px; }}
h1 {{ margin:0 0 14px 0; font-size:24px; }}
h3 {{ margin:0 0 8px 0; font-size:16px; }}
table {{ width:100%; border-collapse: collapse; background:#fff; border:1px solid #dce6f4; }}
th, td {{ text-align:left; padding:10px; border-bottom:1px solid #eef2f8; font-size:13px; }}
th {{ background:#f1f5fc; }}
.muted {{ color:#607086; font-size:12px; }}
code {{ background:#eef3fb; padding:2px 6px; border-radius:4px; }}
</style>
</head>
<body>
<h1>Distributed Crawler Dashboard</h1>
<div class="grid">
<div class="card"><h3>Workers</h3><p>{len(self.workers)}</p></div>
<div class="card"><h3>Session Keys</h3><p>{session_count}</p></div>
<div class="card"><h3>Dead Letters</h3><p>{dead_letter_count}</p></div>
<div class="card"><h3>Proxy</h3><p>{enabled_proxy}</p></div>
<div class="card"><h3>Auto Login</h3><p>{enabled_auth}</p></div>
<div class="card"><h3>Session Sharing</h3><p>{enabled_session}</p></div>
<div class="card"><h3>Auth Domains</h3><p>{escape(enabled_auth_domains)}</p></div>
</div>
<div class="grid">
{queue_html}
</div>
<div class="card">
<h3>Worker Runtime</h3>
<table>
<thead><tr><th>worker_id</th><th>active_tasks</th><th>concurrency</th><th>load</th><th>last_seen</th></tr></thead>
<tbody>
{''.join(worker_rows)}
</tbody>
</table>
</div>
<p class="muted">API endpoints: <code>/tasks</code> <code>/workers</code> <code>/stats</code> <code>/health</code></p>
</body>
</html>"""
def _assign_worker(self, task: CrawlTask, exclude: str | None = None) -> str:
candidates = [worker_id for worker_id in self.workers if worker_id != exclude]
if not candidates:
return "default"
self.hash_ring.rebuild(candidates)
preferred = self.hash_ring.get_node(task.url) or candidates[0]
preferred_heartbeat = self.workers.get(preferred)
if preferred_heartbeat is None:
return preferred
preferred_load = preferred_heartbeat.active_tasks / max(preferred_heartbeat.concurrency, 1)
if preferred_load < 0.9:
return preferred
least_loaded = min(
candidates,
key=lambda worker_id: self.workers[worker_id].active_tasks / max(self.workers[worker_id].concurrency, 1),
)
return least_loaded
def _heartbeat_epoch(self, timestamp: str) -> float:
from datetime import datetime
return datetime.fromisoformat(timestamp).timestamp()
+189
View File
@@ -0,0 +1,189 @@
from __future__ import annotations
from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime
from enum import Enum
from hashlib import sha1
from typing import Any
from urllib.parse import urlparse
from uuid import uuid4
def utc_now() -> str:
return datetime.now(UTC).isoformat()
def make_url_hash(url: str) -> str:
return sha1(url.encode("utf-8")).hexdigest()
class TaskPriority(str, Enum):
HIGH = "high"
MEDIUM = "medium"
LOW = "low"
class TaskStatus(str, Enum):
PENDING = "pending"
RUNNING = "running"
SUCCESS = "success"
FAILED = "failed"
RETRY = "retry"
@dataclass
class CrawlTask:
task_id: str
url: str
spider: str
method: str = "GET"
headers: dict[str, str] = field(default_factory=dict)
metadata: dict[str, Any] = field(default_factory=dict)
body: str | None = None
priority: TaskPriority = TaskPriority.MEDIUM
status: TaskStatus = TaskStatus.PENDING
retries: int = 0
max_retries: int = 4
assigned_worker: str | None = None
created_at: str = field(default_factory=utc_now)
updated_at: str = field(default_factory=utc_now)
schedule_name: str | None = None
@property
def domain(self) -> str:
return urlparse(self.url).netloc
@property
def fingerprint(self) -> str:
payload = f"{self.method}:{self.url}:{self.spider}:{self.body or ''}"
return sha1(payload.encode("utf-8")).hexdigest()
def to_dict(self) -> dict[str, Any]:
payload = asdict(self)
payload["priority"] = self.priority.value
payload["status"] = self.status.value
return payload
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> "CrawlTask":
return cls(
task_id=str(payload["task_id"]),
url=str(payload["url"]),
spider=str(payload["spider"]),
method=str(payload.get("method", "GET")),
headers=dict(payload.get("headers", {})),
metadata=dict(payload.get("metadata", {})),
body=payload.get("body"),
priority=TaskPriority(payload.get("priority", TaskPriority.MEDIUM.value)),
status=TaskStatus(payload.get("status", TaskStatus.PENDING.value)),
retries=int(payload.get("retries", 0)),
max_retries=int(payload.get("max_retries", 4)),
assigned_worker=payload.get("assigned_worker"),
created_at=str(payload.get("created_at", utc_now())),
updated_at=str(payload.get("updated_at", utc_now())),
schedule_name=payload.get("schedule_name"),
)
@classmethod
def create(
cls,
*,
url: str,
spider: str,
method: str = "GET",
headers: dict[str, str] | None = None,
metadata: dict[str, Any] | None = None,
body: str | None = None,
priority: str = "medium",
max_retries: int = 4,
schedule_name: str | None = None,
) -> "CrawlTask":
return cls(
task_id=uuid4().hex,
url=url,
spider=spider,
method=method.upper(),
headers=headers or {},
metadata=metadata or {},
body=body,
priority=TaskPriority(priority),
max_retries=max_retries,
schedule_name=schedule_name,
)
@dataclass
class CrawlResult:
task_id: str
worker_id: str
status: TaskStatus
url: str
spider: str
http_status: int | None = None
data: dict[str, Any] = field(default_factory=dict)
error: str | None = None
finished_at: str = field(default_factory=utc_now)
@property
def url_hash(self) -> str:
return make_url_hash(self.url)
def to_dict(self) -> dict[str, Any]:
payload = asdict(self)
payload["status"] = self.status.value
payload["url_hash"] = self.url_hash
return payload
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> "CrawlResult":
return cls(
task_id=str(payload["task_id"]),
worker_id=str(payload["worker_id"]),
status=TaskStatus(payload["status"]),
url=str(payload["url"]),
spider=str(payload["spider"]),
http_status=payload.get("http_status"),
data=dict(payload.get("data", {})),
error=payload.get("error"),
finished_at=str(payload.get("finished_at", utc_now())),
)
@dataclass
class DeadLetterEntry:
task: dict[str, Any]
result: dict[str, Any] | None = None
reason: str = ""
failed_at: str = field(default_factory=utc_now)
last_error: str | None = None
response_status: int | None = None
retry_count: int = 0
worker_id: str | None = None
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_task_result(cls, task: CrawlTask, result: CrawlResult, reason: str | None = None) -> "DeadLetterEntry":
return cls(
task=task.to_dict(),
result=result.to_dict(),
reason=reason or result.error or "Task failed",
last_error=result.error,
response_status=result.http_status,
retry_count=task.retries,
worker_id=result.worker_id,
)
@dataclass
class WorkerHeartbeat:
worker_id: str
concurrency: int
active_tasks: int
last_seen: str = field(default_factory=utc_now)
host: str | None = None
tags: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return asdict(self)
+162
View File
@@ -0,0 +1,162 @@
from __future__ import annotations
import asyncio
import random
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from time import monotonic
import aiohttp
from .config import ProxySettings
@dataclass
class ProxyEndpoint:
url: str
failures: int = 0
successes: int = 0
latency_ms: float | None = None
last_verified_at: float = 0.0
last_failed_at: float = 0.0
healthy: bool = True
metadata: dict[str, str] = field(default_factory=dict)
@property
def success_rate(self) -> float:
total = self.successes + self.failures
if total == 0:
return 1.0
return self.successes / total
@property
def score(self) -> float:
latency_penalty = 1.0
if self.latency_ms is not None:
latency_penalty = max(0.1, min(1.0, 1000.0 / max(self.latency_ms, 1.0)))
failure_penalty = 1.0 / (1.0 + self.failures)
return max(0.01, self.success_rate * latency_penalty * failure_penalty)
class ProxyProvider(ABC):
@abstractmethod
async def fetch(self) -> list[ProxyEndpoint]: ...
class StaticProxyProvider(ProxyProvider):
def __init__(self, proxies: list[str]) -> None:
self.proxies = proxies
async def fetch(self) -> list[ProxyEndpoint]:
return [ProxyEndpoint(url=value) for value in self.proxies]
class ApiProxyProvider(ProxyProvider):
def __init__(self, api_url: str, auth_header: str = "") -> None:
self.api_url = api_url
self.auth_header = auth_header
async def fetch(self) -> list[ProxyEndpoint]:
if not self.api_url:
return []
headers = {"Authorization": self.auth_header} if self.auth_header else {}
async with aiohttp.ClientSession() as session:
async with session.get(self.api_url, headers=headers) as response:
response.raise_for_status()
text = await response.text()
proxies: list[ProxyEndpoint] = []
for line in text.splitlines():
line = line.strip()
if line:
proxies.append(ProxyEndpoint(url=line))
return proxies
class ProxyPool:
def __init__(self, settings: ProxySettings) -> None:
self.settings = settings
self._proxies: list[ProxyEndpoint] = []
self._lock = asyncio.Lock()
self._next_index = 0
async def refresh(self) -> None:
providers: list[ProxyProvider] = [StaticProxyProvider(self.settings.static_proxies)]
if self.settings.provider_api_url:
providers.append(ApiProxyProvider(self.settings.provider_api_url, self.settings.provider_auth_header))
async with self._lock:
existing = {endpoint.url: endpoint for endpoint in self._proxies}
merged: dict[str, ProxyEndpoint] = {}
for provider in providers:
for endpoint in await provider.fetch():
merged.setdefault(endpoint.url, existing.get(endpoint.url, endpoint))
async with self._lock:
self._proxies = list(merged.values())
async def verify(self) -> None:
if not self.settings.enabled:
return
async with self._lock:
proxies = list(self._proxies)
if not proxies:
return
async with aiohttp.ClientSession() as session:
tasks = [self._verify_proxy(session, proxy) for proxy in proxies]
await asyncio.gather(*tasks, return_exceptions=True)
async def acquire(self) -> ProxyEndpoint | None:
if not self.settings.enabled:
return None
async with self._lock:
now = monotonic()
candidates = [
item
for item in self._proxies
if item.healthy or now - item.last_failed_at >= self.settings.cooldown_seconds
]
if not candidates:
return None
proxy = random.choices(candidates, weights=[item.score for item in candidates], k=1)[0]
self._next_index += 1
return proxy
async def mark_success(self, proxy: ProxyEndpoint | None, latency_ms: float | None = None) -> None:
if proxy is None:
return
async with self._lock:
proxy.successes += 1
proxy.failures = 0
proxy.healthy = True
if latency_ms is not None:
proxy.latency_ms = latency_ms
proxy.last_verified_at = monotonic()
async def mark_failed(self, proxy: ProxyEndpoint | None) -> None:
if proxy is None:
return
async with self._lock:
proxy.failures += 1
proxy.last_failed_at = monotonic()
if proxy.failures >= self.settings.failure_threshold:
proxy.healthy = False
async def healthy_count(self) -> int:
async with self._lock:
return len([item for item in self._proxies if item.healthy])
async def _verify_proxy(self, session: aiohttp.ClientSession, proxy: ProxyEndpoint) -> None:
try:
timeout = aiohttp.ClientTimeout(total=self.settings.verify_timeout_seconds)
started_at = monotonic()
async with session.get(self.settings.verify_url, proxy=proxy.url, timeout=timeout) as response:
response.raise_for_status()
await response.read()
proxy.healthy = True
proxy.failures = 0
proxy.successes += 1
proxy.latency_ms = (monotonic() - started_at) * 1000
proxy.last_verified_at = monotonic()
except Exception:
proxy.failures += 1
proxy.last_failed_at = monotonic()
if proxy.failures >= self.settings.failure_threshold:
proxy.healthy = False
+42
View File
@@ -0,0 +1,42 @@
from __future__ import annotations
import json
from redis.asyncio import Redis
from .models import CrawlResult, CrawlTask, DeadLetterEntry, TaskPriority
class RedisPriorityQueue:
def __init__(self, redis: Redis, namespace: str, result_queue_name: str, dead_letter_queue_name: str | None = None) -> None:
self.redis = redis
self.namespace = namespace
self.result_queue_name = result_queue_name
self.dead_letter_queue_name = dead_letter_queue_name or f"{namespace}:dead_letter"
def _task_key(self, worker_id: str, priority: TaskPriority) -> str:
return f"{self.namespace}:tasks:{worker_id}:{priority.value}"
async def enqueue_task(self, worker_id: str, task: CrawlTask) -> None:
await self.redis.rpush(self._task_key(worker_id, task.priority), json.dumps(task.to_dict()))
async def dequeue_task(self, worker_id: str, timeout: int = 5) -> CrawlTask | None:
keys = [self._task_key(worker_id, priority) for priority in (TaskPriority.HIGH, TaskPriority.MEDIUM, TaskPriority.LOW)]
result = await self.redis.blpop(keys, timeout=timeout)
if not result:
return None
_, payload = result
return CrawlTask.from_dict(json.loads(payload))
async def publish_result(self, result: CrawlResult) -> None:
await self.redis.rpush(self.result_queue_name, json.dumps(result.to_dict()))
async def consume_result(self, timeout: int = 5) -> CrawlResult | None:
result = await self.redis.blpop(self.result_queue_name, timeout=timeout)
if not result:
return None
_, payload = result
return CrawlResult.from_dict(json.loads(payload))
async def publish_dead_letter(self, entry: DeadLetterEntry) -> None:
await self.redis.rpush(self.dead_letter_queue_name, json.dumps(entry.to_dict(), ensure_ascii=False))
+42
View File
@@ -0,0 +1,42 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from time import monotonic
from .config import RateLimitSettings
@dataclass
class TokenBucket:
qps: float
burst: int
tokens: float
updated_at: float
class DomainRateLimiter:
def __init__(self, settings: RateLimitSettings) -> None:
self.settings = settings
self._buckets: dict[str, TokenBucket] = {}
self._locks: dict[str, asyncio.Lock] = {}
async def acquire(self, domain: str) -> None:
lock = self._locks.setdefault(domain, asyncio.Lock())
async with lock:
bucket = self._buckets.setdefault(domain, self._new_bucket(domain))
while True:
now = monotonic()
elapsed = max(0.0, now - bucket.updated_at)
bucket.tokens = min(bucket.burst, bucket.tokens + elapsed * bucket.qps)
bucket.updated_at = now
if bucket.tokens >= 1:
bucket.tokens -= 1
return
await asyncio.sleep(max(0.05, 1 / bucket.qps))
def _new_bucket(self, domain: str) -> TokenBucket:
domain_settings = self.settings.per_domain.get(domain)
qps = domain_settings.qps if domain_settings else self.settings.default_qps
burst = domain_settings.burst if domain_settings else self.settings.default_burst
return TokenBucket(qps=qps, burst=burst, tokens=float(burst), updated_at=monotonic())
+164
View File
@@ -0,0 +1,164 @@
from __future__ import annotations
import json
from abc import ABC, abstractmethod
from pathlib import Path
import asyncpg
from motor.motor_asyncio import AsyncIOMotorClient
from .config import StorageSettings
from .models import CrawlResult, make_url_hash
class ResultRepository(ABC):
@abstractmethod
async def setup(self) -> None: ...
@abstractmethod
async def store(self, result: CrawlResult) -> None: ...
@abstractmethod
async def close(self) -> None: ...
class JsonlResultRepository(ResultRepository):
def __init__(self, path: Path) -> None:
self.path = path
self.index_path = path.with_suffix(path.suffix + ".url_hashes")
self._seen_url_hashes: set[str] = set()
self.path.parent.mkdir(parents=True, exist_ok=True)
async def setup(self) -> None:
if self.index_path.exists():
self._seen_url_hashes = {
line.strip()
for line in self.index_path.read_text(encoding="utf-8").splitlines()
if line.strip()
}
return
if self.path.exists():
with self.path.open("r", encoding="utf-8") as file_handle:
for line in file_handle:
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
url_hash = payload.get("url_hash") or (make_url_hash(str(payload["url"])) if payload.get("url") else None)
if url_hash:
self._seen_url_hashes.add(str(url_hash))
if self._seen_url_hashes:
self.index_path.write_text("\n".join(sorted(self._seen_url_hashes)) + "\n", encoding="utf-8")
async def store(self, result: CrawlResult) -> None:
if result.url_hash in self._seen_url_hashes:
return
with self.path.open("a", encoding="utf-8") as file_handle:
file_handle.write(json.dumps(result.to_dict(), ensure_ascii=False) + "\n")
with self.index_path.open("a", encoding="utf-8") as file_handle:
file_handle.write(result.url_hash + "\n")
self._seen_url_hashes.add(result.url_hash)
async def close(self) -> None:
return None
class MongoResultRepository(ResultRepository):
def __init__(self, settings: StorageSettings) -> None:
self.client = AsyncIOMotorClient(settings.mongodb_dsn)
self.collection = self.client[settings.mongodb_database][settings.mongodb_collection]
async def setup(self) -> None:
await self.collection.create_index("task_id", unique=False)
await self.collection.create_index(
"url_hash",
unique=True,
partialFilterExpression={"url_hash": {"$exists": True}},
)
async def store(self, result: CrawlResult) -> None:
await self.collection.update_one(
{"url_hash": result.url_hash},
{"$set": result.to_dict()},
upsert=True,
)
async def close(self) -> None:
self.client.close()
class PostgresResultRepository(ResultRepository):
def __init__(self, settings: StorageSettings) -> None:
self.dsn = settings.postgres_dsn
self.table = settings.postgres_table
self.pool: asyncpg.Pool | None = None
async def setup(self) -> None:
self.pool = await asyncpg.create_pool(self.dsn)
async with self.pool.acquire() as connection:
await connection.execute(
f"""
CREATE TABLE IF NOT EXISTS {self.table} (
id BIGSERIAL PRIMARY KEY,
task_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
status TEXT NOT NULL,
url TEXT NOT NULL,
url_hash TEXT NOT NULL,
spider TEXT NOT NULL,
http_status INTEGER,
payload JSONB NOT NULL,
error TEXT,
finished_at TIMESTAMPTZ NOT NULL
)
"""
)
await connection.execute(f"ALTER TABLE {self.table} ADD COLUMN IF NOT EXISTS url_hash TEXT")
await connection.execute(
f"CREATE UNIQUE INDEX IF NOT EXISTS {self.table}_url_hash_uidx ON {self.table} (url_hash)"
)
async def store(self, result: CrawlResult) -> None:
if self.pool is None:
raise RuntimeError("Postgres repository is not initialized")
async with self.pool.acquire() as connection:
await connection.execute(
f"""
INSERT INTO {self.table}
(task_id, worker_id, status, url, url_hash, spider, http_status, payload, error, finished_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10::timestamptz)
ON CONFLICT (url_hash) DO UPDATE SET
task_id = EXCLUDED.task_id,
worker_id = EXCLUDED.worker_id,
status = EXCLUDED.status,
url = EXCLUDED.url,
spider = EXCLUDED.spider,
http_status = EXCLUDED.http_status,
payload = EXCLUDED.payload,
error = EXCLUDED.error,
finished_at = EXCLUDED.finished_at
""",
result.task_id,
result.worker_id,
result.status.value,
result.url,
result.url_hash,
result.spider,
result.http_status,
json.dumps(result.data, ensure_ascii=False),
result.error,
result.finished_at,
)
async def close(self) -> None:
if self.pool is not None:
await self.pool.close()
def build_result_repository(settings: StorageSettings, jsonl_path: Path) -> ResultRepository:
backend = settings.backend.lower()
if backend == "mongodb":
return MongoResultRepository(settings)
if backend == "postgresql":
return PostgresResultRepository(settings)
return JsonlResultRepository(jsonl_path)
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.interval import IntervalTrigger
from .config import AppSettings
from .models import CrawlTask
class ScheduleService:
def __init__(self, settings: AppSettings, submit_callback) -> None:
self.settings = settings
self.submit_callback = submit_callback
self.scheduler = AsyncIOScheduler()
def start(self) -> None:
for item in self.settings.schedules:
if item.trigger == "cron" and item.cron:
trigger = CronTrigger.from_crontab(item.cron)
else:
seconds = item.seconds or 0
minutes = item.minutes or 0
trigger = IntervalTrigger(seconds=seconds, minutes=minutes)
self.scheduler.add_job(self._submit_job, trigger=trigger, args=[item], id=item.name, replace_existing=True)
self.scheduler.start()
async def shutdown(self) -> None:
self.scheduler.shutdown(wait=False)
async def _submit_job(self, item) -> None:
payload = dict(item.task)
task = CrawlTask.create(
url=payload["url"],
spider=payload.get("spider", self.settings.master.default_spider),
method=payload.get("method", "GET"),
headers=payload.get("headers"),
metadata=payload.get("metadata"),
body=payload.get("body"),
priority=item.priority,
schedule_name=item.name,
)
await self.submit_callback(task)
+35
View File
@@ -0,0 +1,35 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from redis.asyncio import Redis
class DualSessionStore:
def __init__(self, redis: Redis, redis_prefix: str, backup_path: Path) -> None:
self.redis = redis
self.redis_prefix = redis_prefix
self.backup_path = backup_path
self.backup_path.parent.mkdir(parents=True, exist_ok=True)
def _key(self, domain: str) -> str:
return f"{self.redis_prefix}:{domain}"
async def load(self, domain: str) -> dict[str, Any]:
raw = await self.redis.get(self._key(domain))
if raw:
return json.loads(raw)
if self.backup_path.exists():
data = json.loads(self.backup_path.read_text(encoding="utf-8"))
return data.get(domain, {})
return {}
async def save(self, domain: str, payload: dict[str, Any]) -> None:
await self.redis.set(self._key(domain), json.dumps(payload))
snapshot: dict[str, Any] = {}
if self.backup_path.exists():
snapshot = json.loads(self.backup_path.read_text(encoding="utf-8"))
snapshot[domain] = payload
self.backup_path.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2), encoding="utf-8")
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
import aiohttp
class SignatureProvider(ABC):
@abstractmethod
async def sign(self, request_payload: dict[str, Any]) -> dict[str, Any]: ...
class NoopSignatureProvider(SignatureProvider):
async def sign(self, request_payload: dict[str, Any]) -> dict[str, Any]:
return {}
class RpcSignatureProvider(SignatureProvider):
def __init__(self, endpoint: str) -> None:
self.endpoint = endpoint
async def sign(self, request_payload: dict[str, Any]) -> dict[str, Any]:
if not self.endpoint:
return {}
async with aiohttp.ClientSession() as session:
async with session.post(self.endpoint, json=request_payload) as response:
response.raise_for_status()
data = await response.json()
return dict(data)
+89
View File
@@ -0,0 +1,89 @@
from __future__ import annotations
import json
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
from .http import AsyncHttpClient
from .models import CrawlResult, CrawlTask, TaskStatus
class BaseSpider(ABC):
name: str
@abstractmethod
async def execute(self, task: CrawlTask, client: AsyncHttpClient, worker_id: str) -> CrawlResult: ...
class GenericFetchSpider(BaseSpider):
name = "generic_fetch"
async def execute(self, task: CrawlTask, client: AsyncHttpClient, worker_id: str) -> CrawlResult:
response = await client.fetch(task)
data: dict[str, Any] = {
"headers": response["headers"],
"body": response["body"],
"proxy": response["proxy"],
"request_id": task.metadata.get("request_id"),
}
status = TaskStatus.SUCCESS if 200 <= response["status"] < 400 else TaskStatus.FAILED
return CrawlResult(
task_id=task.task_id,
worker_id=worker_id,
status=status,
url=task.url,
spider=self.name,
http_status=response["status"],
data=data,
error=None if status == TaskStatus.SUCCESS else f"Unexpected status {response['status']}",
)
class AssetDownloadSpider(BaseSpider):
name = "asset_download"
async def execute(self, task: CrawlTask, client: AsyncHttpClient, worker_id: str) -> CrawlResult:
response = await client.fetch(task)
parsed = urlparse(task.url)
file_name = task.metadata.get("file_name") or Path(parsed.path or "resource.bin").name or "resource.bin"
saved_to = await client.save_binary(task, file_name, response["body_bytes"])
return CrawlResult(
task_id=task.task_id,
worker_id=worker_id,
status=TaskStatus.SUCCESS if response["status"] < 400 else TaskStatus.FAILED,
url=task.url,
spider=self.name,
http_status=response["status"],
data={"saved_to": saved_to, "proxy": response["proxy"], "request_id": task.metadata.get("request_id")},
error=None if response["status"] < 400 else f"Unexpected status {response['status']}",
)
class JsonApiSpider(BaseSpider):
name = "json_api"
async def execute(self, task: CrawlTask, client: AsyncHttpClient, worker_id: str) -> CrawlResult:
response = await client.fetch(task)
body = response["body"]
if isinstance(body, str):
try:
body = json.loads(body)
except json.JSONDecodeError:
body = {"raw": body}
return CrawlResult(
task_id=task.task_id,
worker_id=worker_id,
status=TaskStatus.SUCCESS if response["status"] < 400 else TaskStatus.FAILED,
url=task.url,
spider=self.name,
http_status=response["status"],
data={"payload": body, "proxy": response["proxy"], "request_id": task.metadata.get("request_id")},
error=None if response["status"] < 400 else f"Unexpected status {response['status']}",
)
def build_spider_registry() -> dict[str, BaseSpider]:
spiders: list[BaseSpider] = [GenericFetchSpider(), AssetDownloadSpider(), JsonApiSpider()]
return {spider.name: spider for spider in spiders}
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
import json
from pathlib import Path
from redis.asyncio import Redis
from .models import CrawlTask, TaskStatus, utc_now
class CheckpointManager:
def __init__(self, redis: Redis, checkpoint_dir: Path, namespace: str) -> None:
self.redis = redis
self.checkpoint_dir = checkpoint_dir
self.namespace = namespace
self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
def _task_key(self, task_id: str) -> str:
return f"{self.namespace}:task:{task_id}"
def _running_key(self, worker_id: str) -> str:
return f"{self.namespace}:running:{worker_id}"
def _file_path(self, task_id: str) -> Path:
return self.checkpoint_dir / f"{task_id}.json"
async def save(self, task: CrawlTask) -> None:
task.updated_at = utc_now()
await self.redis.set(self._task_key(task.task_id), json.dumps(task.to_dict()))
await self._write_file(task)
async def mark_status(self, task: CrawlTask, status: TaskStatus) -> None:
task.status = status
await self.save(task)
if task.assigned_worker:
running_key = self._running_key(task.assigned_worker)
if status == TaskStatus.RUNNING:
await self.redis.zadd(running_key, {task.task_id: self._score_now()})
elif status in {TaskStatus.SUCCESS, TaskStatus.FAILED}:
await self.redis.zrem(running_key, task.task_id)
self._cleanup_file(task.task_id)
async def load(self, task_id: str) -> CrawlTask | None:
raw = await self.redis.get(self._task_key(task_id))
if raw:
return CrawlTask.from_dict(json.loads(raw))
file_path = self._file_path(task_id)
if file_path.exists():
return CrawlTask.from_dict(json.loads(file_path.read_text(encoding="utf-8")))
return None
async def recover_stale(self, worker_id: str, older_than_seconds: int) -> list[CrawlTask]:
threshold = self._score_now() - older_than_seconds
task_ids = await self.redis.zrangebyscore(self._running_key(worker_id), min=0, max=threshold)
tasks: list[CrawlTask] = []
for task_id in task_ids:
task = await self.load(task_id)
if task is None:
continue
if task.status == TaskStatus.RUNNING:
task.status = TaskStatus.RETRY
tasks.append(task)
return tasks
async def recover_local(self) -> list[CrawlTask]:
tasks: list[CrawlTask] = []
for file_path in self.checkpoint_dir.glob("*.json"):
payload = json.loads(file_path.read_text(encoding="utf-8"))
task = CrawlTask.from_dict(payload)
if task.status in {TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.RETRY}:
tasks.append(task)
return tasks
async def _write_file(self, task: CrawlTask) -> None:
self._file_path(task.task_id).write_text(json.dumps(task.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
def _cleanup_file(self, task_id: str) -> None:
file_path = self._file_path(task_id)
if file_path.exists():
file_path.unlink()
def _score_now(self) -> float:
from time import time
return time()
+186
View File
@@ -0,0 +1,186 @@
from __future__ import annotations
import asyncio
import logging
import socket
from pathlib import Path
from uuid import uuid4
import aiohttp
from redis.asyncio import Redis
from .auth import ConfigurableAuthProvider, ManualAuthProvider
from .backoff import exponential_backoff
from .config import AppSettings
from .http import AsyncHttpClient
from .models import CrawlResult, CrawlTask, TaskStatus, WorkerHeartbeat
from .proxy import ProxyPool
from .queue import RedisPriorityQueue
from .rate_limit import DomainRateLimiter
from .sessions import DualSessionStore
from .signatures import NoopSignatureProvider
from .spider import BaseSpider, build_spider_registry
from .state import CheckpointManager
LOGGER = logging.getLogger(__name__)
class WorkerNode:
def __init__(self, settings: AppSettings) -> None:
self.settings = settings
self.worker_id = settings.worker.worker_id
self.redis = Redis.from_url(settings.redis_dsn, decode_responses=True)
self.queue = RedisPriorityQueue(
self.redis,
settings.master.queue_name,
settings.master.result_queue_name,
settings.master.dead_letter_queue_name,
)
self.checkpoints = CheckpointManager(self.redis, settings.resolve_path(settings.checkpoint_dir), settings.master.queue_name)
self.rate_limiter = DomainRateLimiter(settings.rate_limit)
self.proxy_pool = ProxyPool(settings.proxy)
self.session_store = DualSessionStore(
self.redis,
settings.session.redis_prefix,
settings.resolve_path(settings.session.local_backup_path),
)
auth_provider = ConfigurableAuthProvider(settings.auth) if settings.auth.enabled else ManualAuthProvider()
self.client = AsyncHttpClient(
settings,
self.rate_limiter,
self.proxy_pool,
self.session_store,
auth_provider,
NoopSignatureProvider(),
)
self.spiders: dict[str, BaseSpider] = build_spider_registry()
self.active_tasks: set[str] = set()
self.semaphore = asyncio.Semaphore(settings.worker.concurrency)
self.master_base_url = f"http://{settings.master.host}:{settings.master.port}"
self._loops: list[asyncio.Task] = []
self._http_session: aiohttp.ClientSession | None = None
async def start(self) -> None:
self._http_session = aiohttp.ClientSession()
await self.client.start()
if self.settings.proxy.enabled:
await self.proxy_pool.refresh()
await self.proxy_pool.verify()
recovered = await self.checkpoints.recover_local()
for task in recovered:
if task.assigned_worker in {None, self.worker_id, "default"}:
await self.queue.enqueue_task(self.worker_id, task)
self._loops = [
asyncio.create_task(self.consume_loop(), name=f"{self.worker_id}-consume"),
asyncio.create_task(self.heartbeat_loop(), name=f"{self.worker_id}-heartbeat"),
asyncio.create_task(self.proxy_refresh_loop(), name=f"{self.worker_id}-proxy-refresh"),
]
async def close(self) -> None:
for task in self._loops:
task.cancel()
await asyncio.gather(*self._loops, return_exceptions=True)
await self.client.close()
if self._http_session is not None:
await self._http_session.close()
await self.redis.aclose()
async def consume_loop(self) -> None:
while True:
try:
task = await self.queue.dequeue_task(self.worker_id, timeout=5)
if task is None:
task = await self.queue.dequeue_task("default", timeout=1)
if task is None:
continue
await self.semaphore.acquire()
asyncio.create_task(self.execute_task(task))
except asyncio.CancelledError:
raise
except Exception:
LOGGER.exception("Worker %s consume loop failed", self.worker_id)
await asyncio.sleep(1)
async def execute_task(self, task: CrawlTask) -> None:
self.active_tasks.add(task.task_id)
try:
task.assigned_worker = self.worker_id
task.metadata.setdefault("task_id", task.task_id)
task.metadata["worker_id"] = self.worker_id
task.metadata["request_id"] = f"{task.task_id}-{task.retries}-{uuid4().hex[:8]}"
await self.checkpoints.mark_status(task, TaskStatus.RUNNING)
spider = self.spiders.get(task.spider)
if spider is None:
raise RuntimeError(f"Unknown spider: {task.spider}")
result = await spider.execute(task, self.client, self.worker_id)
if result.status == TaskStatus.FAILED and task.retries < task.max_retries:
await self._retry_task(task, result.error or "worker failure")
else:
await self.queue.publish_result(result)
except Exception as exc:
LOGGER.exception("Task %s failed", task.task_id)
if task.retries < task.max_retries:
await self._retry_task(task, str(exc))
else:
result = CrawlResult(
task_id=task.task_id,
worker_id=self.worker_id,
status=TaskStatus.FAILED,
url=task.url,
spider=task.spider,
error=str(exc),
)
await self.queue.publish_result(result)
finally:
self.active_tasks.discard(task.task_id)
self.semaphore.release()
async def heartbeat_loop(self) -> None:
while True:
try:
await self.send_heartbeat()
await asyncio.sleep(self.settings.worker.heartbeat_interval_seconds)
except asyncio.CancelledError:
raise
except Exception:
LOGGER.exception("Worker %s heartbeat failed", self.worker_id)
await asyncio.sleep(2)
async def send_heartbeat(self) -> None:
if self._http_session is None:
return
heartbeat = WorkerHeartbeat(
worker_id=self.worker_id,
concurrency=self.settings.worker.concurrency,
active_tasks=len(self.active_tasks),
host=socket.gethostname(),
)
async with self._http_session.post(f"{self.master_base_url}/heartbeat", json=heartbeat.to_dict()) as response:
response.raise_for_status()
async def proxy_refresh_loop(self) -> None:
if not self.settings.proxy.enabled:
return
while True:
try:
await asyncio.sleep(self.settings.proxy.refresh_interval_seconds)
await self.proxy_pool.refresh()
await self.proxy_pool.verify()
LOGGER.info("Worker %s healthy proxies: %s", self.worker_id, await self.proxy_pool.healthy_count())
except asyncio.CancelledError:
raise
except Exception:
LOGGER.exception("Proxy refresh loop failed")
async def _retry_task(self, task: CrawlTask, reason: str) -> None:
task.retries += 1
task.status = TaskStatus.RETRY
await self.checkpoints.save(task)
delay = exponential_backoff(
task.retries - 1,
self.settings.worker.base_backoff_seconds,
self.settings.worker.max_backoff_seconds,
)
LOGGER.warning("Retry task %s in %.2fs: %s", task.task_id, delay, reason)
await asyncio.sleep(delay)
await self.queue.enqueue_task(self.worker_id, task)
+6
View File
@@ -0,0 +1,6 @@
aiohttp>=3.10.10
redis>=5.0.8
APScheduler>=3.10.4
PyYAML>=6.0.2
motor>=3.6.0
asyncpg>=0.30.0
+38
View File
@@ -0,0 +1,38 @@
from __future__ import annotations
import argparse
import asyncio
import logging
from aiohttp import web
from crawler.config import load_settings
from crawler.master import MasterNode
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
settings = load_settings(args.config)
node = MasterNode(settings)
await node.start()
runner = web.AppRunner(node.app)
await runner.setup()
site = web.TCPSite(runner, settings.master.host, settings.master.port)
await site.start()
logging.getLogger(__name__).info("Master started at http://%s:%s", settings.master.host, settings.master.port)
try:
while True:
await asyncio.sleep(3600)
finally:
await runner.cleanup()
await node.close()
if __name__ == "__main__":
asyncio.run(main())
+34
View File
@@ -0,0 +1,34 @@
from __future__ import annotations
import argparse
import asyncio
import logging
from crawler.config import load_settings
from crawler.worker import WorkerNode
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
parser.add_argument("--worker-id", required=False)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
settings = load_settings(args.config)
if args.worker_id:
settings.worker.worker_id = args.worker_id
node = WorkerNode(settings)
await node.start()
logging.getLogger(__name__).info("Worker started: %s", settings.worker.worker_id)
try:
while True:
await asyncio.sleep(3600)
finally:
await node.close()
if __name__ == "__main__":
asyncio.run(main())
+280
View File
@@ -0,0 +1,280 @@
# APK 下拉刷新 / 删除后页面不刷新 修复
**日期**: 2026-06-22
**问题**: 删除作品后页面不刷新;下拉刷新不生效,无法重新请求当前页数据
---
## 根因分析
### 根因 1:`RefreshIndicator` 放错位置,嵌套在 `FutureBuilder` 内部
`RefreshIndicator` 被放在 `FutureBuilder` 的 `builder` 闭包内。每次 `setState` 更新 `future` → `FutureBuilder` 重建 → `RefreshIndicator` 被销毁再创建 → 下拉手势/动画状态丢失。
**正常工作**的 `feed_tab.dart` 结构:
```
RefreshIndicator → ListView → FutureBuilder(内部)
```
**异常结构**(修复前):
```
FutureBuilder → RefreshIndicator → CustomScrollView
```
### 根因 2:缺少 `AlwaysScrollableScrollPhysics`
`CustomScrollView` 未设置 `physics: AlwaysScrollableScrollPhysics()`。数据少于一屏时不可滚动 → `RefreshIndicator` 无法探测下拉手势。
### 根因 3(核心):API 客户端缓存导致的"假刷新"
`api_client.dart` 的 `gallery()` 方法有 **30 分钟 SharedPreferences 缓存**:
```dart
// 修复前 - 每次都先查缓存
final cacheKey = _cacheKey('gallery', params);
final cached = await _readCache<List<dynamic>>(cacheKey);
if (cached != null) return cached; // ← 始终返回旧数据
```
刷新链路全追踪:
```
_refresh()
→ ref.invalidate(apiClientProvider) ✅ 创建新 SakuraApiClient
→ _load()
→ gallery()
→ _cacheKey('gallery', params) ❌ token 没变 → cache key 一样
→ _readCache(cacheKey) ❌ 命中 30 分钟前的缓存
→ 返回删除/修改前的旧数据 ❌ 表现:刷新无效
```
Token 不变 → `_cacheKey` 内 `token.hashCode` 不变 → **缓存 key 不变** → 永远返回缓存。即使 `ref.invalidate` 创建了新客户端实例也无效,因为缓存存在 SharedPreferences 中,跨实例共享。
---
## 修改文件清单(3 个文件)
| 文件 | 改动 |
|------|------|
| `sakura_mobile/lib/src/core/api_client.dart` | `gallery()` 加 `bypassCache` 参数 |
| `sakura_mobile/lib/src/pages/tabs/gallery_tab.dart` | 结构调整 + 缓存穿透 + 删除刷新按钮 |
| `sakura_mobile/lib/src/pages/tabs/novels_tab.dart` | 结构调整 + 删除刷新按钮 |
---
## 改动详情
### 一、`sakura_mobile/lib/src/core/api_client.dart`
#### `gallery()` 加 `bypassCache` 参数
**改动前**:
```dart
Future<List<dynamic>> gallery({
String? search,
String searchScope = 'all',
bool? isFavorite,
String? contentType,
String? sort,
int page = 1,
int limit = 60,
bool includeTotal = true,
}) async {
...
final cacheKey = _cacheKey('gallery', params);
final cached = await _readCache<List<dynamic>>(cacheKey);
if (cached != null) return cached;
final response = await dio.get('/api/v1/gallery', queryParameters: params);
final data = response.data['data']['items'] as List<dynamic>;
await _writeCache(cacheKey, data);
return data;
}
```
**改动后**:
```dart
Future<List<dynamic>> gallery({
String? search,
String searchScope = 'all',
bool? isFavorite,
String? contentType,
String? sort,
int page = 1,
int limit = 60,
bool includeTotal = true,
bool bypassCache = false, // ← 新增
}) async {
...
final cacheKey = _cacheKey('gallery', params);
if (!bypassCache) { // ← 新增:仅在非刷新时读缓存
final cached = await _readCache<List<dynamic>>(cacheKey);
if (cached != null) return cached;
}
final response = await dio.get('/api/v1/gallery', queryParameters: params);
final data = response.data['data']['items'] as List<dynamic>;
await _writeCache(cacheKey, data);
return data;
}
```
---
### 二、`sakura_mobile/lib/src/pages/tabs/gallery_tab.dart`
#### 改动 1:`_load()` 加 `forceRefresh` 参数,传递到 `gallery()`
**改动前**:
```dart
Future<List<Artwork>> _load({bool reset = true}) async {
if (reset) {
_page = 1;
_hasMore = true;
_items.clear();
selectedIds.clear();
selectionMode = false;
}
final rows = await ref.read(apiClientProvider).gallery(
search: search.text.trim().isEmpty ? null : search.text.trim(),
searchScope: searchScope,
isFavorite: favoritesOnly ? true : null,
contentType: _effectiveContentType,
sort: recentUpdatesOnly ? 'updated' : null,
page: _page,
limit: _pageSize,
includeTotal: false,
);
```
**改动后**:
```dart
Future<List<Artwork>> _load({bool reset = true, bool forceRefresh = false}) async {
if (reset) {
_page = 1;
_hasMore = true;
_items.clear();
selectedIds.clear();
selectionMode = false;
}
final rows = await ref.read(apiClientProvider).gallery(
search: search.text.trim().isEmpty ? null : search.text.trim(),
searchScope: searchScope,
isFavorite: favoritesOnly ? true : null,
contentType: _effectiveContentType,
sort: recentUpdatesOnly ? 'updated' : null,
page: _page,
limit: _pageSize,
includeTotal: false,
bypassCache: forceRefresh, // ← 新增:刷新时绕过缓存
);
```
#### 改动 2:`_refresh()` 改为 async,传递 `forceRefresh: true`
**改动前**:
```dart
void _refresh() {
setState(() {
future = _load();
});
}
```
**改动后**:
```dart
Future<void> _refresh() async {
ref.invalidate(apiClientProvider);
final f = _load(forceRefresh: true); // ← 绕过缓存
setState(() {
future = f;
});
await f; // ← RefreshIndicator 等 future 完成才停止动画
}
```
#### 改动 3:`RefreshIndicator` 移到 `FutureBuilder` 外层
**改动前**:
```dart
return FutureBuilder<List<Artwork>>(
future: future,
builder: (context, snapshot) {
...
return RefreshIndicator(
onRefresh: _refresh,
child: CustomScrollView(
controller: _scrollController,
slivers: [...],
),
);
},
);
```
**改动后**:
```dart
return RefreshIndicator(
color: const Color(0xFFD95A83),
onRefresh: _refresh,
child: FutureBuilder<List<Artwork>>(
future: future,
builder: (context, snapshot) {
...
return CustomScrollView(
controller: _scrollController,
physics: const AlwaysScrollableScrollPhysics(), // ← 新增
slivers: [...],
);
},
),
);
```
#### 改动 4:删除刷新 IconButton
**改动前**:
```dart
trailing: [
...
IconButton(
tooltip: '刷新',
onPressed: _refresh,
icon: const Icon(Icons.refresh),
),
],
```
**改动后**:
```dart
trailing: [
... // 刷新按钮已移除,统一使用下拉刷新
],
```
---
### 三、`sakura_mobile/lib/src/pages/tabs/novels_tab.dart`
同 gallery_tab 的改动 3、4(结构调整 + 删除按钮)。novels API 本身无缓存,无需 `bypassCache`。
---
## 修复后的刷新链路
```
用户下拉 → RefreshIndicator.onRefresh → _refresh()
→ ref.invalidate(apiClientProvider) ✅ 创建新客户端
→ _load(forceRefresh: true)
→ gallery(bypassCache: true) ✅ 跳过缓存
→ dio.get('/api/v1/gallery') ✅ 真实 HTTP 请求
→ _writeCache(cacheKey, newData) ✅ 写入新缓存
→ 返回服务器最新数据 ✅
→ setState({future: f}) ✅ FutureBuilder 拿到新 future
→ await f ✅ 等待完成,停止下拉动画
→ UI 展示最新数据 ✅ 删除/修改后的数据正确
```
## 构建产物
| 产物 | 路径 | 大小 |
|------|------|------|
| APK | `release/sakura.apk` | 66.8 MB |
| EXE | `dist/sakuradownloader.exe` | 63 MB |
+310
View File
@@ -0,0 +1,310 @@
# 云同步 401 自动刷新 Token 修复
**日期**: 2026-06-22
**问题**: 桌面端 `POST /api/v1/sync/check` 持续返回 401,无重试/续期机制
**根因**: `CloudSyncConfig` 只存 `access_token`,无 `refresh_token` 和自动续期逻辑
---
## 修改文件清单
| 文件 | 改动类型 |
|------|----------|
| `pixiv_app/core/cloud_sync.py` | 核心:加 refresh_token、401 自动刷新、token 更新回调 |
| `pixiv_app/gui/app.py` | 存储/加载/传递 refresh_token,回调持久化 |
| `pixiv_app/core/gallery_service.py` | 传递 refresh_token |
| `pixiv_app/core/origin_follow_sync.py` | 传递 refresh_token |
| `pixiv_app/core/x_media_service.py` | 传递 refresh_token |
---
## 改动详情
### 1. `pixiv_app/core/cloud_sync.py`
#### 1.1 `CloudSyncConfig` — 加 `refresh_token` 字段
**改动前**:
```python
@dataclass
class CloudSyncConfig:
base_url: str
access_token: str
chunk_size: int = 5 * 1024 * 1024
timeout: float = 60.0
```
**改动后**:
```python
@dataclass
class CloudSyncConfig:
base_url: str
access_token: str
refresh_token: str = ""
chunk_size: int = 5 * 1024 * 1024
timeout: float = 60.0
```
#### 1.2 `SakuraCloudClient.__init__` — 加 `on_token_refreshed` 回调
**改动前**:
```python
class SakuraCloudClient:
def __init__(self, config: CloudSyncConfig) -> None:
self.config = config
self.base_url = config.base_url.rstrip("/")
self.session = requests.Session()
self.session.headers.update({"Authorization": f"Bearer {config.access_token}"})
```
**改动后**:
```python
class SakuraCloudClient:
def __init__(self, config: CloudSyncConfig, *, on_token_refreshed: callable | None = None) -> None:
self.config = config
self.base_url = config.base_url.rstrip("/")
self.session = requests.Session()
self.session.headers.update({"Authorization": f"Bearer {config.access_token}"})
self._on_token_refreshed = on_token_refreshed
self._refresh_lock = False # 防止并发刷新
```
#### 1.3 `_post` — 加 401 自动刷新 + 重试
**改动前**:
```python
def _post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]:
response = self.session.post(f"{self.base_url}{path}", json=payload, timeout=self.config.timeout)
return self._unwrap(response)
```
**改动后**:
```python
def _post(self, path: str, payload: dict[str, Any], *, _retry_on_401: bool = True) -> dict[str, Any]:
response = self.session.post(f"{self.base_url}{path}", json=payload, timeout=self.config.timeout)
if response.status_code == 401 and _retry_on_401:
refreshed = self._try_refresh_token()
if refreshed:
return self._post(path, payload, _retry_on_401=False)
return self._unwrap(response)
```
#### 1.4 新增 `_try_refresh_token` 方法
**新增代码**:
```python
def _try_refresh_token(self) -> bool:
"""用 refresh_token 换取新的 access_token。成功返回 True 并通知上层持久化。"""
refresh = self.config.refresh_token.strip()
if not refresh or self._refresh_lock:
return False
self._refresh_lock = True
try:
resp = self.session.post(
f"{self.base_url}/api/v1/auth/refresh",
json={"refresh_token": refresh},
timeout=self.config.timeout,
)
if resp.status_code != 200:
return False
body = resp.json()
data = body.get("data", body) if isinstance(body, dict) else {}
new_access = (data.get("access_token") or "").strip()
new_refresh = (data.get("refresh_token") or "").strip()
if not new_access:
return False
self.config.access_token = new_access
self.session.headers.update({"Authorization": f"Bearer {new_access}"})
if new_refresh:
self.config.refresh_token = new_refresh
if self._on_token_refreshed:
try:
self._on_token_refreshed(self.config.access_token, self.config.refresh_token)
except Exception:
pass # 持久化失败不阻塞业务
return True
except Exception:
return False
finally:
self._refresh_lock = False
```
---
### 2. `pixiv_app/gui/app.py`
#### 2.1 新增 `cloud_refresh_token` 实例变量
**改动前**:
```python
self.cloud_access_token = ""
```
**改动后**:
```python
self.cloud_access_token = ""
self.cloud_refresh_token = ""
```
#### 2.2 `get_cloud_sync_config` — 加 `refresh_token`
**改动前**:
```python
def get_cloud_sync_config(self) -> dict:
return {
"enabled": bool(self.cloud_sync_enabled_var.get()),
"base_url": (self.cloud_base_url or DEFAULT_CLOUD_API).strip(),
"access_token": self.cloud_access_token.strip(),
}
```
**改动后**:
```python
def get_cloud_sync_config(self) -> dict:
return {
"enabled": bool(self.cloud_sync_enabled_var.get()),
"base_url": (self.cloud_base_url or DEFAULT_CLOUD_API).strip(),
"access_token": self.cloud_access_token.strip(),
"refresh_token": self.cloud_refresh_token.strip(),
}
```
#### 2.3 登录成功时捕获 `refresh_token`
**改动前**:
```python
token = data.get("data", {}).get("access_token")
if token:
self.cloud_access_token = str(token).strip()
self.enqueue_ui(self.cloud_status_var.set, "登录成功,云同步凭证已自动保存")
```
**改动后**:
```python
token = data.get("data", {}).get("access_token")
if token:
self.cloud_access_token = str(token).strip()
refresh = (data.get("data", {}).get("refresh_token") or "")
if refresh:
self.cloud_refresh_token = str(refresh).strip()
self.enqueue_ui(self.cloud_status_var.set, "登录成功,云同步凭证已自动保存")
```
#### 2.4 `restore_session` 加载 refresh_token
**改动前**:
```python
self.cloud_access_token = str(cloud.get("access_token") or "")
if self.cloud_access_token:
```
**改动后**:
```python
self.cloud_access_token = str(cloud.get("access_token") or "")
self.cloud_refresh_token = str(cloud.get("refresh_token") or "")
if self.cloud_access_token:
```
#### 2.5 新增 `_on_cloud_token_refreshed` 回调 + 更新 `create_sync_manager`
**改动前**:
```python
def create_sync_manager(self) -> SyncManager | None:
config = self.get_cloud_sync_config()
if not (config.get("enabled") and config.get("base_url") and config.get("access_token")):
return None
return SyncManager(
SakuraCloudClient(
CloudSyncConfig(
base_url=str(config["base_url"]),
access_token=str(config["access_token"]),
)
)
)
```
**改动后**:
```python
def _on_cloud_token_refreshed(self, new_access: str, new_refresh: str) -> None:
"""Token 自动刷新后更新内存并持久化到本地会话文件。"""
self.cloud_access_token = new_access
if new_refresh:
self.cloud_refresh_token = new_refresh
self.enqueue_ui(self.cloud_status_var.set, "云同步 Token 已自动续期")
self.enqueue_ui(self.save_session)
def create_sync_manager(self) -> SyncManager | None:
config = self.get_cloud_sync_config()
if not (config.get("enabled") and config.get("base_url") and config.get("access_token")):
return None
return SyncManager(
SakuraCloudClient(
CloudSyncConfig(
base_url=str(config["base_url"]),
access_token=str(config["access_token"]),
refresh_token=str(config.get("refresh_token") or ""),
),
on_token_refreshed=self._on_cloud_token_refreshed,
)
)
```
---
### 3. `pixiv_app/core/gallery_service.py`
**改动前**:
```python
return SyncManager(
SakuraCloudClient(
CloudSyncConfig(
base_url=base_url,
access_token=access_token,
)
)
)
```
**改动后**:
```python
return SyncManager(
SakuraCloudClient(
CloudSyncConfig(
base_url=base_url,
access_token=access_token,
refresh_token=str(cloud.get("refresh_token") or ""),
)
)
)
```
---
### 4. `pixiv_app/core/origin_follow_sync.py`
同 gallery_service.py,仅加 `refresh_token` 行。
### 5. `pixiv_app/core/x_media_service.py`
同 gallery_service.py,仅加 `refresh_token` 行。
---
## 工作流程
```
POST /api/v1/sync/check
├── 200 → 正常返回
└── 401
├── refresh_token 为空? → 抛异常(旧 token 永久失效)
├── 正在刷新中? → 抛异常(防并发)
└── POST /api/v1/auth/refresh
├── 200 → 更新内存 token → 回调持久化 → 重试原请求
└── 非200 → 抛异常(refresh 也过期了,需重新登录)
```
## 兼容性
- `refresh_token` 默认值 `""` — 不传也能正常工作(旧行为)
- `on_token_refreshed` 可选 — 不传也不影响其他调用方
- 现有的 `ConfigCloudSync` 构造方式全兼容
File diff suppressed because it is too large. Load diff
+492
View File
@@ -0,0 +1,492 @@
# APP 端问题分析与解决方案
> 更新日期:2026-06-14
> 范围:Sakura Mobile (Flutter) + Sakura Cloud (FastAPI 后端) + Sakura Downloader (Python 桌面端)
> 说明:本文档仅提供问题分析和修改方案,不包含实际代码改动。
---
## 问题一:APP 莫名其妙跳转到 "dakura" 用户
### 问题现象
APP 在使用过程中会突然切换到 "dakura" 用户身份,导致看到的是其他用户的数据而并非自己的内容。
### 根因分析
经过对全部代码的排查,问题出在 **Session 持久化缺少用户隔离机制**。涉及以下关键位置:
#### 1. 移动端 Session 存储无用户绑定 (`sakura_mobile/lib/src/core/session.dart`)
**文件位置:** `sakura_mobile/lib/src/core/session.dart` 第 44-52 行
```dart
static const _baseUrlKey = 'sakura.baseUrl';
static const _accessTokenKey = 'sakura.accessToken';
Future<void> _load() async {
final prefs = await SharedPreferences.getInstance();
state = SessionState(
baseUrl: prefs.getString(_baseUrlKey) ?? defaultCloudApi,
accessToken: prefs.getString(_accessTokenKey) ?? '',
);
```
**问题:** `SharedPreferences` 使用固定 key `sakura.accessToken` 存储 token,不绑定任何用户标识。当存在以下情况时会发生串号:
- 同一设备上先后登录过不同账号(如 "dakura" 和当前用户)
- SharedPreferences 中的旧 token 未完全清除
- App 从后台恢复时读取了旧的/缓存的 token
- 多进程/多任务环境下 SharedPreferences 跨进程同步延迟
#### 2. 登录流程缺少旧 Session 清理 (`sakura_mobile/lib/src/core/session.dart` 第 70-86 行)
```dart
Future<void> login(String usernameOrEmail, String password) async {
state = state.copyWith(loading: true, error: null);
try {
final api = SakuraApiClient(baseUrl: state.baseUrl);
final tokens = await api.login(usernameOrEmail, password);
final accessToken = tokens['access_token']?.toString() ?? '';
final prefs = await SharedPreferences.getInstance();
await prefs.setString(_accessTokenKey, accessToken);
```
**问题:** 登录时直接覆盖 `_accessTokenKey`,但如果 SharedPreferences 写入失败或未完全刷盘,旧 token 仍然存在。没有先清除旧 token 再写入新 token 的原子操作。
#### 3. Web 端使用 sessionStorage 同样缺少隔离 (`sakura_admin_web/src/stores/session.ts` 第 15-30 行)
```typescript
const ACCESS_KEY = "sakura_access_token";
const REFRESH_KEY = "sakura_refresh_token";
export const authState = reactive({
accessToken: sessionStorage.getItem(ACCESS_KEY) || "",
refreshToken: sessionStorage.getItem(REFRESH_KEY) || "",
user: null as CurrentUser | null,
loading: false
});
```
**问题:** sessionStorage 的 key 是固定的,同一浏览器不同标签页可能共享或冲突。
#### 4. 桌面端 Session 文件共享 (`pixiv_app/core/paths.py` 和 `pixiv_app/core/auth.py`)
Session 文件 `pixiv_app_session.json` 保存的是单一的 cookie 和 cloud_sync 配置,没有对应用户上下文。如果云端切换了用户但本地 session 文件未同步更新,可能导致不一致。
### 解决方案
#### 方案 A:Token 绑定用户名校验(推荐)
**修改位置:**
| 文件 | 行号 | 修改内容 |
|------|------|----------|
| `sakura_mobile/lib/src/core/session.dart` | 第 44-52 行 | 在 `_load()` 后调用 `/api/v1/users/me` 校验 token 归属用户 |
| `sakura_mobile/lib/src/core/session.dart` | 第 70-86 行 | `login()` 时先清除旧 token,写入后立即验证 |
| `sakura_mobile/lib/src/core/session.dart` | 新增方法 | 添加 `_validateAndFixSession()` 方法,对比登录用户名与 token 归属 |
**具体做法:**
1. 在 `SessionController._load()` 中,加载 token 后立即调用 `api.me()` 获取当前 token 对应的用户名
2. 将用户名也存入 SharedPreferences,key 使用 `sakura.username`
3. 每次 `_load()` 后比对 `sakura.username` 与 `me()` 返回的 username,不一致则清除旧 token 强制重新登录
4. `login()` 成功后立即存储用户名,防止与旧数据混淆
#### 方案 B:增加 Token 失效主动检测
**修改位置:**
| 文件 | 行号 | 修改内容 |
|------|------|----------|
| `sakura_mobile/lib/src/core/api_client.dart` | 第 12-47 行 | 在 `SakuraApiClient` 中增加 401 拦截器 |
| `sakura_mobile/lib/src/core/session.dart` | 第 100-108 行 | `logout()` 时增加确认清理 SharedPreferences 全部 key |
**具体做法:**
1. 通过 Dio 拦截器全局捕获 401/403 响应,自动清除 token 并跳转登录页
2. 在 `logout()` 中不单删除 `_accessTokenKey`,同时清理所有 sakura 相关 key
3. 增加 `clearAllSakuraData()` 方法用于完全重置
#### 方案 C:服务端 Token 版本号机制
**修改位置:** 后端 `sakura_cloud/` (FastAPI) 相关认证模块
**具体做法:**
1. 在用户表增加 `token_version` 字段
2. 每次重新登录时 token_version +1
3. JWT payload 中嵌入 token_version
4. 验证 token 时比对版本号,旧版本 token 直接拒绝
5. 确保用户登出后旧 token 彻底失效
---
## 问题二:发现页查询过慢,需要加载缓冲或优化
### 问题现象
APP 打开"发现"页(FeedTab)时,整个页面长时间白屏/转圈,用户体验差。
### 根因分析
#### 1. 五个 API 串行阻塞全部内容 (`sakura_mobile/lib/src/pages/tabs/feed_tab.dart`)
**文件位置:** `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` 第 20-26 行
```dart
return FutureBuilder<List<Object?>>(
future: Future.wait([
api.galleryStat(), // 统计总数
api.galleryFeed(mode: 'created', limit: 36), // 今日新增
api.galleryFeed(mode: 'recent', limit: 24), // 最近更新
api.recentNovels(limit: 12), // 最近小说
api.unreadAlerts(limit: 12), // 未读提醒
]),
```
**问题:**
- `Future.wait` 等待所有 5 个请求完成才渲染页面
- 任一接口慢(尤其是 `galleryStat()` 和 `galleryFeed()`),整个发现页都会卡住
- `galleryStat()` 可能后端做了全表 COUNT,数据量大时耗时数秒
#### 2. 后端 gallery/feed 查询可能缺少优化
**文件位置:** 后端 API(`sakura_cloud/` 目录下的 gallery/feed 路由)
`galleryFeed` 接口通常对应:
- `mode: 'created'` → 按创建时间排序的最近作品
- `mode: 'recent'` → 按更新时间排序
如果后端 SQL 查询缺少合适的复合索引,或者做了全表扫描 + JOIN 多表,在数据量上千条后就会明显变慢。典型慢查询模式:
```sql
-- 可能的慢查询(缺少索引或 JOIN 过多)
SELECT a.*, ar.name, tc.thumbnail_path, ...
FROM artworks a
LEFT JOIN artists ar ON ...
LEFT JOIN thumbnail_cache tc ON ...
LEFT JOIN artwork_files af ON ...
LEFT JOIN favorites fav ON ...
ORDER BY a.created_at DESC
LIMIT 36
```
**对比本地库查询** (`pixiv_app/core/gallery_service.py` 第 96-134 行) 同样有类似的多表 LEFT JOIN + GROUP BY + 复合搜索条件,这在 SQLite 上尤其慢:
```python
total = library.connection.execute(
f"SELECT COUNT(DISTINCT a.artwork_id) AS total FROM artworks a {joins} {where_sql}",
params,
).fetchone()["total"]
```
先 COUNT 再 SELECT,等于两次大查询。
#### 3. 图片缩略图同步加载阻塞渲染
**文件位置:** `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` 第 385-389 行
```dart
CachedNetworkImage(
imageUrl: thumbUrl,
httpHeaders: api.authHeaders(),
fit: BoxFit.cover,
),
```
**问题:** 虽然使用了 `CachedNetworkImage` 做缓存,但如果后端 `thumbnail_url` 指向的是需要实时生成或大文件的原图,加载会很慢;且横向滚动的 `ListView` 会一次性构建多个图片 widget,造成网络拥塞。
### 解决方案
#### 方案 A:渐进式加载 + 骨架屏(推荐,前端改动最小)
**修改位置:**
| 文件 | 行号 | 修改内容 |
|------|------|----------|
| `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` | 第 20-26 行 | 将 `Future.wait` 拆分为独立请求,每个区块独立加载 |
| `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` | 第 62-137 行 | 为每个 `_ArtworkStrip`、`_NovelStrip`、`_AlertStrip` 增加独立的 loading/skeleton 状态 |
| `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` | 新增组件 | 增加 `_SkeletonStrip` 骨架屏组件 |
**具体做法:**
1. 不再使用单个 `FutureBuilder` + `Future.wait`,改为每个 section 独立使用 `FutureBuilder`
2. 统计卡片(`_TodayUpdateCard` 和 `_StatCard`)优先加载
3. 各内容条(插画、漫画、小说、提醒)各自独立加载,失败的只影响自己
4. 加载中的条显示 shimmer 骨架屏而非空白
5. `galleryStat()` 如果慢,可设置 3 秒超时,超时则降级显示 "--"
#### 方案 B:后端增加 feed 缓存表(推荐,解决根本问题)
**修改位置:** 后端 `sakura_cloud/`
**具体做法:**
1. 新增 `feed_cache` 表:
```sql
CREATE TABLE feed_cache (
user_id UUID NOT NULL,
feed_type VARCHAR(32) NOT NULL, -- 'created', 'recent', 'stat'
payload_json JSONB NOT NULL,
generated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (user_id, feed_type)
);
```
2. 在每次上传新作品、同步关注的定时任务完成后,异步刷新 feed_cache
3. `/api/v1/gallery/feed` 和 `/api/v1/gallery/stat` 优先从 `feed_cache` 读取
4. 缓存有效期设为 5-10 分钟,过期后后台异步刷新,当前请求仍返回旧缓存
5. 如果缓存不存在,降级为直接查询(首次慢但后续快)
#### 方案 C:本地 SQLite 缓存 + 增量同步(移动端离线方案)
**修改位置:**
| 文件 | 行号 | 修改内容 |
|------|------|----------|
| `sakura_mobile/lib/src/core/api_client.dart` | 新增方法 | 增加本地缓存读写方法 |
| `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` | 第 20 行 | 优先读本地缓存,后台静默刷新 |
**具体做法:**
1. 移动端使用 `sqflite` 或 `drift` 建立轻量本地库
2. 发现页数据缓存到本地,设置 30 分钟有效期
3. 打开发现页时先展示缓存数据(秒开),同时后台请求新数据
4. 新数据到达后平滑替换(`AnimatedSwitcher` 已就位)
5. 支持下拉强制刷新(`RefreshIndicator` 已就位)
#### 方案 D:优化后端 gallery/feed 查询语句
**修改位置:** 后端 `sakura_cloud/api/v1/` 中 gallery/feed 相关路由
**具体做法:**
1. 分析慢查询日志,定位具体慢的 SQL
2. 为 `artworks` 表增加复合索引:
```sql
CREATE INDEX idx_artworks_user_created ON artworks(user_id, created_at DESC);
CREATE INDEX idx_artworks_user_updated ON artworks(user_id, updated_at DESC);
```
3. 将 COUNT + SELECT 两次查询合并为一次(用窗口函数或子查询)
4. 限制 `galleryStat()` 的统计精度,超过 10000 条时显示 "10000+"
5. 对缩略图 URL 确保使用 CDN 或 Nginx 静态文件服务,而非 Django/FastAPI 流式响应
---
## 问题三:提醒中心用户更新检测问题
### 问题现象
1. **最近提醒显示的并非实际关注的作者名字**,而是作者 ID 数字
2. **已关注的作者有更新但未检测到**,提醒中心未生成新提醒
### 根因分析
#### 1. 提醒标题使用 author_id 而非 author_name
**文件位置:** `pixiv_app/core/gallery_service.py` 第 588-613 行
```python
def _create_cloud_pixiv_update_alerts(self, artist_id: int, work_ids: list[int]) -> int:
...
for work_id in work_ids:
try:
manager.client.create_alert(
{
"platform": "pixiv",
"creator_id": str(int(artist_id)),
"title": f"Pixiv 作者 {int(artist_id)} 更新了作品 {int(work_id)}",
# ↑↑↑ 问题在这里:用的是 author_id 数字,而非 author name
"body": "你关注的 Pixiv 作者有新作品了",
...
}
)
```
**问题详解:**
- `_create_cloud_pixiv_update_alerts` 没有接收 `artist_name` 参数
- 调用它的 `sync_followed_artists`(第 505-586 行)虽然能从 `library` 查到作者信息,但没传给 alert 创建函数
- 导致提醒标题变成 "Pixiv 作者 123456 更新了作品 789012",用户看到的是一串数字
- 移动端 `reminder_center_page.dart` 直接展示 `alert['title']`,所以显示的是数字 ID
#### 2. 关注作者有更新但未检测到
**文件位置:** `pixiv_app/core/gallery_service.py` 第 505-586 行
```python
def sync_followed_artists(self, *, cookie, max_new_per_artist=20, download=True, save_root="Sakura_Downloads"):
...
for row in followed:
artist_id = int(row["artist_id"])
# 获取已存在的作品 ID
existing = {
int(item["artwork_id"])
for item in library.connection.execute(
"SELECT artwork_id FROM artworks WHERE artist_id = ? AND type IN ('illust', 'manga')",
(artist_id,),
).fetchall()
}
# 获取远程作品 ID
_, work_ids = fetch_user_work_ids(user_id=artist_id, cookie=cookie)
# 只取本地没有的
new_ids = [work_id for work_id in work_ids if work_id not in existing][:max_new_per_artist]
```
**问题详解:**
- `fetch_user_work_ids` 需要有效的 Pixiv Cookie,如果 Cookie 已过期,API 调用静默失败
- 对比逻辑只看 `existing` 中的 artwork_id,但如果作品之前下载失败、入库未完成,可能漏掉
- `mark_artist_synced(artist_id, error=str(exc))` 在异常时记录了错误但未区分错误类型(Cookie 过期 vs 网络错误 vs 限流)
- 没有"最近检查时间"的限流机制 —— 如果短时间内多次同步同一个作者,每次都做全量对比
#### 3. 检查完成后未更新 `last_checked_at`
**文件位置:** `pixiv_app/core/library.py` 第 592-603 行
```python
def mark_artist_synced(self, artist_id: int, *, error: str = "") -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO followed_artists (artist_id, enabled, last_synced_at, last_error)
VALUES (?, 1, CURRENT_TIMESTAMP, ?)
ON CONFLICT(artist_id) DO UPDATE SET
last_synced_at = CURRENT_TIMESTAMP,
last_error = excluded.last_error
""",
(artist_id, error),
)
```
**问题:** `last_synced_at` 只在本地 SQLite 更新,**没有同步到云端** follows 表的 `last_checked_at`。移动端提醒中心展示的是云端 follows 数据:
**文件位置:** `sakura_mobile/lib/src/pages/reminder_center_page.dart` 第 22-27 行
```dart
Future<_ReminderCenterData> _load() async {
final api = ref.read(apiClientProvider);
final results = await Future.wait<Object?>([
api.me(),
api.follows(enabled: true), // ← 从云端读取 follows
api.alerts(limit: 50), // ← 从云端读取 alerts
]);
```
**问题详解:**
- 移动端显示的 `last_checked_at` 来自云端 follows 表
- 桌面端 `_upsert_cloud_follow` 只更新了 creator 基本信息,**没有更新 `last_checked_at`**
- 所以云端看到的"最近检查"时间永远是旧的甚至为空
- 用户无法判断哪个作者实际上被检查过
#### 4. Alert 创建没有去重
**文件位置:** `pixiv_app/core/gallery_service.py` 第 588-613 行
```python
for work_id in work_ids:
try:
manager.client.create_alert({...})
created += 1
except Exception:
continue
```
**问题:**
- 每次同步都会为同一个 work_id 创建新的 alert
- 如果多次同步同一批作品,会产生大量重复提醒
- 没有检查 "这个 work_id 是否已经发过 alert"
### 解决方案
#### 方案 A:修复 Alert 标题——传入作者名(推荐,最小改动)
**修改位置:**
| 文件 | 行号 | 修改内容 |
|------|------|----------|
| `pixiv_app/core/gallery_service.py` | 第 588 行 | `_create_cloud_pixiv_update_alerts` 增加 `artist_name` 参数 |
| `pixiv_app/core/gallery_service.py` | 第 562-565 行 | `sync_followed_artists` 传入作者名 |
| `pixiv_app/core/gallery_service.py` | 第 599 行 | 修改 title 使用作者名 |
**具体做法:**
在 `sync_followed_artists` 方法中,从 followed_artists 或 artists 表获取作者名:
```python
# 第 527-538 行之间,获取 followed 列表时同时获取 name
followed = library.connection.execute(
"SELECT fa.artist_id, COALESCE(NULLIF(fa.name,''), NULLIF(ar.name,''), '') AS name "
"FROM followed_artists fa "
"LEFT JOIN artists ar ON ar.artist_id = fa.artist_id "
"WHERE fa.enabled = 1 ORDER BY fa.artist_id"
).fetchall()
```
然后修改 `_create_cloud_pixiv_update_alerts` 签名:
```python
def _create_cloud_pixiv_update_alerts(
self, artist_id: int, work_ids: list[int], artist_name: str = ""
) -> int:
```
title 改为:
```python
"title": f"Pixiv 作者 {artist_name or str(artist_id)} 更新了 {len(work_ids)} 个新作品"
```
#### 方案 B:同步云端 follows 的检查状态
**修改位置:**
| 文件 | 行号 | 修改内容 |
|------|------|----------|
| `pixiv_app/core/gallery_service.py` | 第 556-559 行 | `mark_artist_synced` 之后同步更新云端 follows 的 `last_checked_at` |
| `pixiv_app/core/cloud_sync.py` | 第 167-168 行 | `upsert_follow` 增加 `last_checked_at` 和 `latest_published_at` 字段 |
| `pixiv_app/core/gallery_service.py` | 第 415-447 行 | `_upsert_cloud_follow` 增加更新时间戳参数 |
**具体做法:**
1. 在 `_upsert_cloud_follow` 方法中增加 `last_checked_at` 和 `latest_published_at` 参数
2. 每次同步完一个作者后,调用 `_upsert_cloud_follow` 更新云端状态
3. 在云端 follows API 的 upsert 逻辑中接受并存储这些时间字段
4. 移动端 `reminder_center_page.dart` 中 `_FollowTile` 就能显示正确的"最近检查"时间
#### 方案 C:Alert 去重 & 按作者聚合
**修改位置:**
| 文件 | 行号 | 修改内容 |
|------|------|----------|
| `pixiv_app/core/gallery_service.py` | 第 588-613 行 | 重构 `_create_cloud_pixiv_update_alerts` 逻辑 |
| 后端 `sakura_cloud/api/v1/follows.py` | alert 创建路由 | 增加去重逻辑 |
**具体做法:**
1. 将每个作者的多个新作品合并为**一条聚合 alert**,而非每个作品一条
2. Alert 的 `source_id` 使用 `f"{artist_id}:{min(work_ids)}:{max(work_ids)}"` 生成唯一标识
3. 创建 alert 前先检查是否存在相似 alert(同一作者 + 同一天内)
4. 如果存在则更新而非新建
5. title 格式改为:`"你关注的 {作者名} 更新了 {N} 个新作品"`
#### 方案 D:增加关注作者更新状态页面
**修改位置:**
| 文件 | 行号 | 修改内容 |
|------|------|----------|
| `sakura_mobile/lib/src/pages/reminder_center_page.dart` | 第 350-371 行 | 已关注作者列表增加"手动检查更新"按钮 |
| 后端新增接口 | - | `POST /api/v1/follows/{follow_id}/check` 触发单作者更新检测 |
**具体做法:**
1. 在已关注作者列表中为每个作者增加状态标识:
- 🟢 最近 24h 内检查过
- 🟡 24h-7d 内检查过
- 🔴 超过 7 天未检查
2. 增加"手动检查"按钮,调用后端触发单个作者的更新检测
3. 检查完成后自动刷新提醒列表
---
## 总结优先级
| 优先级 | 问题 | 推荐方案 | 改动范围 | 预期效果 |
|--------|------|----------|----------|----------|
| P0 紧急 | 跳转到 dakura 用户 | 方案 A:Token 绑定用户名 | 移动端 session.dart | 根除串号问题 |
| P0 紧急 | Alert 显示数字而非作者名 | 方案 A:传入作者名 | gallery_service.py | 提醒可读 |
| P1 高 | 发现页加载慢 | 方案 A + B:渐进加载 + feed_cache | feed_tab.dart + 后端 | 首屏从 3-5s 降到 <1s |
| P1 高 | 关注作者未检测更新 | 方案 B:同步检查状态到云端 | gallery_service.py + cloud_sync.py | 状态准确可追溯 |
| P2 中 | Alert 重复创建 | 方案 C:去重 + 聚合 | gallery_service.py + 后端 | 减少重复提醒 |
| P2 中 | 查询性能优化 | 方案 D:SQL 索引 | 后端数据库 | 查询耗时降低 50%+ |
@@ -0,0 +1,738 @@
# Legado 规则引擎重构 — 分阶段实施方案
> 日期:2026-06-21
> 基于:源码架构总结 + 截图问题诊断 + 方案文档反馈
> 涉及:`legado_rule_parser.dart`、`legado_engine.dart`、`legado_models.dart`
> 原则:不改 UI 层,只在 core 层动刀;前 4 步零新依赖。
---
## 一、问题根因定性与实施顺序纠偏
### 1.1 截图 Bug ↔ 真实根因对照
| 截图 | 现象 | 代码层面的真实原因 |
|------|------|-------------------|
| 截图3 | "这一章还没有找到漫画页面" | `fetchChapterContent()` → `contentRule` → `RuleParser` 只认 CSS/JSON/Regex 三种前缀,**包子漫画5 的 `contentRule` 使用了 `@tag.img@src` / `##` / `&&` 等链式语法**,`_safeSelect()` 返回空串,`rawHtml` 兜底也没触发。 |
| 截图4 | "未知作者" / 封面空白 | `bookInfoAuthorRule` / `bookInfoCoverRule` 同上,规则语法不被 `RuleParser` 支持。 |
| 截图1 | "书籍详情加载失败" | `fetchBookInfo()` 整体 try/catch 后只设 `infoError`,不区分"网络不通"还是"规则不支持"。 |
| 搜索 `unknown:none` | 错误消息原样展示 | `_shortError()` 已修好(源码已含完整中文映射),这个 bug 已不存在于当前源码。 |
### 1.2 原方案步骤的纠偏
原方案第 1 步是「支持 URL,{method,body,headers}」,但实际上:
- `_buildSearchRequest()` 已有 GET/POST 分支
- `_applySearchTemplate()` 已支持 12 种占位符变体
- `fromJson` 解析 `header` 已支持 Map / JSON 字符串 / 冒号分隔行三种格式
**请求层不是最优先的问题。最优先的是 RuleParser 不支持 Legado 链式规则语法。**
### 1.3 修正后的实施顺序
```
第1步:rawHtml 兜底扫描 + script 变量扫描 ← 零成本,立刻修复截图3
第2步:RuleParser 支持 @tag/@text/@html/@src 链式语法 ← 解锁包子漫画5等核心书源
第3步:支持 ## 替换、|| 备选、&& 拼接 ← 兼容更多规则变体
第4步:fetchBookInfo / fetchToc 失败时区分错误类型 ← 解决截图1
第5步:JS 特判(newImgs / imageList 等高频变量) ← 覆盖 60%+ JS 规则,零依赖
第6步:@put / @get 上下文变量 ← 跨规则传值
第7步:书源兼容度检测 UI ← 用户体验闭环
第8步:完整 JS 引擎(flutter_js / quickjs) ← 最后做,覆盖剩余 40%
```
**前 4 步不需要引入任何新依赖,只在 `legado_rule_parser.dart` 和 `legado_engine.dart` 里改。**
---
## 二、总体架构重构方案
### 2.1 不要继续往 RuleParser 里塞逻辑
当前 `RuleParser`(262 行)已经承担了 CSS / JSONPath / Regex 三种解析路径,再往里面加链式语法会导致它变成无法维护的千行怪物。
**建议拆成三层**:
```
请求层 ── LegadoRequestBuilder ── 构建 URL、headers、body、编码
规则层 ── LegadoRuleExecutor ── 链式规则执行引擎
检测层 ── SourceCompatChecker ── 导入时打兼容度标签
```
```
┌──────────────────────────────────────────────────┐
│ LegadoEngine │
│ (对外 API:search / fetchBookInfo / fetchToc / │
│ fetchChapterContent 不变) │
├──────────────────────────────────────────────────┤
│ │
│ ┌──────────────────┐ ┌─────────────────────┐ │
│ │LegadoRequestBuilder│ │ LegadoRuleExecutor │ │
│ │· buildSearchUrl │ │· select() │ │
│ │· buildRequest │ │· selectList() │ │
│ │· encodeKeyword │ │· execChain() 🆕 │ │
│ │· applyTemplate │ │· applyReplace() 🆕 │ │
│ └──────────────────┘ │· extractImages() 🆕 │ │
│ │· scanScriptVars() 🆕 │ │
│ └──────────┬──────────┘ │
│ │ │
│ ┌──────────▼──────────┐ │
│ │ RuleParser (保留) │ │
│ │ CSS / JSON / Regex │ │
│ │ 不改内部,只当底层 │ │
│ └─────────────────────┘ │
└──────────────────────────────────────────────────┘
```
### 2.2 核心原则
1. **LegadoEngine 对外 API 不动**。`search()` / `fetchBookInfo()` / `fetchToc()` / `fetchChapterContent()` 签名不变,UI 层零改动。
2. **RuleParser 降级为底层原子工具**。它继续支持 CSS / JSONPath / Regex 三种基础解析,但不理解链式语法。所有链式逻辑放在 `LegadoRuleExecutor` 里。
3. **新规则执行器按优先级链执行**。每条规则被解析为一条 **执行链 (execChain)**,按序执行,任意一步返回空就终止并返回空。
---
## 三、第 1 步:rawHtml 兜底扫描 + script 变量扫描
> 目标:不改 RuleParser,在 `fetchChapterContent()` 和 `fetchImageUrls()` 里补兜底逻辑。
### 3.1 问题
当 `contentRule` 为空或解析失败时,当前代码回到 `_htmlToText(body)` 或 `_extractImageUrls(body)` 暴力扫描 HTML。但如果漫画图片 URL 藏在 `<script>` 的 JS 变量里(如 `var newImgs = ["url1","url2"]`),暴力扫描 `<img>` 标签找不到任何东西。
### 3.2 修改位置
`legado_engine.dart` 中的 `fetchChapterContent()` 和 `_extractImageUrls()`。
### 3.3 逻辑
```
extractImagesWithFallback(String rawHtml, String baseUrl):
1. 先用 RuleParser 走 contentRule → 语法错误或结果为空 → 进入兜底
2. 兜底 1:扫描 HTML img 标签
RegExp: <img[^>]+(?:src|data-src|data-original|data-srcset)=["']([^"']+)["']
同时提取 srcset 中的备选 URL
3. 兜底 2:扫描常见 JS 变量
按优先级依次尝试:
newImgs = [...]
imageList = [...]
chapterImages = [...]
page_url = ...
pics = [...]
imgData = [...]
chapterData.images = [...]
km5_img_url = ...
result.imageList = ...
提取方式:
RegExp: var\s+变量名\s*=\s*(\[[^\]]*\])
RegExp: var\s+变量名\s*=\s*['"]([^'"]+)['"]
如果是 JSON 数组 → jsonDecode
如果是逗号分隔 → split(',').map(trim)
4. 兜底 3:扫描 data:image 内嵌图片
5. 全部 URL resolve 为绝对路径
```
### 3.4 改动量
- `_extractImageUrls()` 新增 ~40 行
- 不影响现有 `RuleParser` 任何逻辑
- 无需新依赖
### 3.5 预期效果
截图 3 「这一章还没有找到漫画页面」在包子漫画 5 上直接消除(即使 `contentRule` 语法不被支持,script 变量扫描也能抓到 `newImgs`)。
---
## 四、第 2 步:RuleParser 支持 @tag/@text/@html/@src 链式语法
> 这是最核心的一步。包子漫画 5、6漫画、WebToon 等主流漫画源全部依赖这种语法。
### 4.1 Legado 链式规则语法规范
Legado 规则的本质是**管道链式操作**,每一步的输出是下一步的输入。语法格式:
```
基础选择器@操作.子选择器@属性
```
**链式管道符**:
| 符号 | 含义 | 示例 |
|------|------|------|
| `@tag.xxx` | 在当前结果内找第一个 `<xxx>` 子元素 | `@tag.img` → 找 img 标签 |
| `@text` | 提取当前元素的 innerText | `.title@text` |
| `@html` | 提取当前元素的 innerHtml | `.content@html` |
| `@src` | 提取 src 属性值 | `img@src` |
| `@href` | 提取 href 属性值 | `a@href` |
| `@class` | 提取 class 属性值 | `div@class` |
| `@alt` / `@title` | 提取 alt / title 属性 | `img@alt` |
| `@data-xxx` | 提取 data-xxx 属性值 | `img@data-original` |
| `@css:.xxx` | 用 CSS 选择器在当前结果内再次选择 | `div.content@css:.chapter@text` |
**管道执行顺序**:
```
原始 HTML
→ CSS 选择器(第一段)→ 得到匹配元素集合
→ @tag.img → 在每个元素内找第一个 img
→ @src → 提取 src 属性值
→ 返回字符串列表
```
### 4.2 实现方式:在 RuleParser 内部新增链式分支
`select()` 和 `selectList()` 方法检测规则中是否包含 `@tag` / `@text` / `@html` / `@src` 等链式操作符。如果有,走新分支 `_selectChain()`。
```dart
List<String> _selectChain(String fragment, String rule) {
// 1. 解析 rule 为执行步列表
final steps = _parseChainSteps(rule);
// 2. 从 fragment 开始,逐步执行
var current = <String>[fragment];
for (final step in steps) {
current = _executeChainStep(current, step);
if (current.isEmpty) return const [];
}
return current;
}
List<_ChainStep> _parseChainSteps(String rule) {
// 规则示例: ".book-list li@tag.a@href"
// 拆分为: [('.book-list li', css), ('a', tag), ('href', attr)]
//
// 关键:@ 是管道分隔符,不是属性选择器的一部分
// "@tag.xxx" 是一个整体 token
// "@src" "@href" "@text" "@html" 是属性提取 token
// "@css:.xxx" 是嵌套 CSS token
}
List<String> _executeChainStep(List<String> fragments, _ChainStep step) {
switch (step.type) {
case _StepType.css:
return _cssSelect(fragments, step.value); // 降级到现有 CSS 路径
case _StepType.tag:
return _tagSelect(fragments, step.value); // fragments 中每个元素内找第一个 <tag>
case _StepType.attr:
return _attrExtract(fragments, step.value); // 提取属性值
case _StepType.text:
return _textExtract(fragments); // 提取 innerText
case _StepType.html:
return _htmlExtract(fragments); // 提取 innerHtml
case _StepType.nestedCss:
return _nestedCssSelect(fragments, step.value); // @css: 语法
}
}
```
### 4.3 链式步骤解析规则
```
规则字符串:
"div.image-area@tag.img@src"
解析产物:
[
{ type: css, value: "div.image-area" }, // @ 之前的部分 → CSS 选择器
{ type: tag, value: "img" }, // @tag.img → 找 img 子标签
{ type: attr, value: "src" }, // @src → 提取 src 属性
]
规则字符串:
"h1.title@text||h1.book-name@text"
解析产物(备选规则):
[
[ {css:"h1.title"}, {text} ], // 方案 A
[ {css:"h1.book-name"}, {text} ], // 方案 B → 先执行 A,A 为空则执行 B
]
规则字符串:
".chapter-content@html##去除广告正则##"
解析产物(带替换):
[
{ type: css, value: ".chapter-content" },
{ type: html, value: null },
{ type: regexReplace, value: "去除广告正则" },
]
```
### 4.4 改动量
- `legado_rule_parser.dart` 新增 `_selectChain()` 方法 ~120 行
- `_parseChainSteps()` ~40 行
- 辅助方法 `_tagSelect()` / `_attrExtract()` / `_textExtract()` / `_htmlExtract()` ~80 行
- `select()` 和 `selectList()` 入口处新增链式检测分支 ~10 行
- 总计 ~250 行
- **零新依赖**(tagSelect / attrExtract 用现有 html parser 或 RegExp 实现)
### 4.5 预期效果
包子漫画 5 的 `contentRule: "div.comic-content@tag.img@src"` 能正确执行:
1. CSS `div.comic-content` → 拿到内容区 DOM
2. `@tag.img` → 在内容区找所有 img 标签
3. `@src` → 提取每个 img 的 src
4. 返回图片 URL 列表
---
## 五、第 3 步:支持 ## 替换、|| 备选、&& 拼接
### 5.1 语法定义
| 符号 | 含义 | 示例 |
|------|------|------|
| `\|\|` | 备选规则:第一个成功就返回,否则尝试下一个 | `.title@text\|\|h1@text` |
| `&&` | 串联拼接:两个结果用空格或换行拼接 | `.title@text&&.author@text` |
| `##` | 正则替换/净化:对当前结果执行正则替换 | `@text##\s+\|\|##g` |
| `###` | 正则删除:删除匹配的部分 | `@html###<script[^>]*>.*?</script>###` |
### 5.2 实现位置
`LegadoRuleExecutor` 的 `execChain()` 方法。
### 5.3 执行顺序
```
解析 rawRule:
1. 按 || 拆分为备选组
2. 每个备选组按 && 拆分为串联组
3. 每个串联组解析为链式步骤(复用第2步的 _parseChainSteps)
4. 每个链式步骤末尾检测 ## 替换指令
执行:
for 备选组 in 备选组列表:
结果 = 执行 备选组
if 结果非空 → 返回结果
return 空
执行备选组:
parts = []
for 串联组 in 备选组:
part = 执行串联组
parts.add(part)
return parts.join('\n') // && 的结果用换行拼接
执行串联组:
result = 执行链式步骤(复用 _selectChain)
if 有 ## 替换指令:
for 替换 in 替换指令:
result = result.replaceAll(替换.regex, 替换.replacement)
return result
```
### 5.4 改动量
- `legado_rule_parser.dart` 新增 `_selectWithAlternatives()` ~50 行
- `##` 替换逻辑 ~30 行
- 总计 ~80 行
- **零新依赖**
---
## 六、第 4 步:fetchBookInfo / fetchToc 失败时区分错误类型
### 6.1 问题
当前 `fetchBookInfo()` 的 try/catch 结构:
```dart
try {
final value = await engine.fetchBookInfo(...);
setState(() { info = value; loadingInfo = false; });
} catch (error) {
setState(() { infoError = error; loadingInfo = false; }); // 所有错误一视同仁
}
```
用户看不到是"网络不通"还是"规则不支持",只能看到统一的"书籍详情加载失败"。
### 6.2 方案:定义错误类型,不新增异常类
```dart
enum LoadFailureReason {
network, // 网络不通 / DNS 失败 / 超时
httpError, // HTTP 4xx / 5xx
ruleNotSupported, // 规则语法不被当前引擎支持
ruleParseEmpty, // 规则执行成功但结果为空(可能是网站改版)
encoding, // 编码问题
unknown,
}
```
在 `fetchBookInfo()` 和 `fetchToc()` 内部区分异常类型,通过 `_shortError()` 返回带 reason 的错误消息(或通过一个简单的包装类)。
### 6.3 UI 差异化展示
```
网络不通:
"无法连接到目标网站,请检查网络或稍后重试。"
[重试]
HTTP 404:
"书源详情页已失效(404),该书源可能需要更新。"
[更换书源]
规则不支持:
"该书源的详情解析规则包含不被当前版本支持的语法(如 @js),更新 APP 后可能可用。"
[查看原始页面]
规则解析为空:
"目标网站页面结构已变化,书源规则可能已过时。"
[重试] [更换书源]
```
### 6.4 改动量
- `legado_engine.dart` 新增 `_classifyError()` 方法 ~30 行
- `_shortError()` 新增 reason 字段 ~10 行
- `online_book_detail_page.dart` 根据 reason 展示不同的 UI ~30 行
- 总计 ~70 行
- **零新依赖**
---
## 七、第 5 步:JS 特判(零依赖)
### 7.1 问题
很多漫画源的 `contentRule` 直接写 JS:
```json
"content": "<js>\nvar result = [];\nvar imgs = JSON.parse...\nnewImgs.map(...)\nresult.join('\\n')</js>"
```
或者写在 `ruleContent.content` 里用 `@js:` 前缀。
当前引擎对 `@js:` 和 `<js>` 完全无处理能力,规则执行到这一步直接抛异常。
### 7.2 第一阶段的 JS 特判策略
不引入完整 JS 引擎。只做模式匹配,覆盖最常见的漫画图片提取模式:
| 模式 | 真实 JS 代码 | 特判正则 |
|------|-------------|---------|
| `newImgs.map` | `newImgs.map(i => i.url)` | 先扫描 rawHtml 中的 `var newImgs = [...]` |
| `imageList` | `imageList[0].src` | 扫描 `var imageList = [...]` |
| `page_url` | `page_url + img` | 扫描 `var page_url = "..."` |
| `chapterImages` | `chapterImages.join` | 扫描 `var chapterImages = [...]` |
| `km5_img_url` | `km5_img_url` | 扫描 `var km5_img_url = "..."` |
| `base64Images` | 从 base64 解码 | 检测 `data:image` 前缀 |
| `eval(decrypt(...))` | 简单解密 | 正则提取被 eval 的字符串 |
### 7.3 实现方式
```dart
List<String> _jsFallbackImages(String rawHtml, String baseUrl) {
final patterns = [
// 1. 数组变量
_JsVarPattern('newImgs', arrayPattern: true),
_JsVarPattern('imageList', arrayPattern: true),
_JsVarPattern('chapterImages', arrayPattern: true),
_JsVarPattern('pics', arrayPattern: true),
_JsVarPattern('imgList', arrayPattern: true),
_JsVarPattern('pageImgs', arrayPattern: true),
// 2. 字符串拼接变量
_JsVarPattern('page_url', arrayPattern: false),
_JsVarPattern('imgDomain', arrayPattern: false),
// 3. data:image 内嵌
_JsVarPattern('data:image', base64Pattern: true),
];
for (final pattern in patterns) {
final urls = _tryExtractJsVar(rawHtml, pattern);
if (urls.isNotEmpty) return urls.map((u) => resolveUrl(baseUrl, u)).toList();
}
return const [];
}
```
### 7.4 改动量
- `legado_engine.dart` 新增 `_jsFallbackImages()` ~60 行
- `_tryExtractJsVar()` ~40 行
- 总计 ~100 行
- **零新依赖**
---
## 八、第 6 步:@put / @get 上下文变量
### 8.1 用途
Legado 高级规则中可以在同一次执行中跨步骤传值:
```
@put:{bookId:$.id}
// ... 在后续规则中使用 ...
/api/book/{{bookId}}/chapters
```
### 8.2 实现方式
在 `LegadoRuleExecutor` 中维护一个 `Map<String, String> _context`。
```dart
class LegadoRuleExecutor {
final Map<String, String> _context = {};
// 执行 @put:{key:value} 时写入 _context
// 解析 URL 模板时用 _context 替换 {{key}}
// @get:key → 从 _context 读取
}
```
### 8.3 改动量
- `legado_rule_parser.dart` 新增 `_context` Map ~5 行
- `@put` / `@get` 语法解析 ~30 行
- `_applySearchTemplate()` 新增 `{{$.key}}` 占位符支持 ~10 行
- 总计 ~45 行
- **零新依赖**
---
## 九、第 7 步:书源兼容度检测
### 9.1 目标
导入书源后,自动分析每个书源用到的规则语法,打上兼容度标签,在书源管理 UI 中展示。
### 9.2 兼容度标签定义
```dart
enum SourceCompatLevel {
full, // 完全兼容:所有规则语法已被引擎支持
partial, // 部分兼容:搜索可用,但正文/详情可能因缺少 JS 执行而不完整
needsJs, // 需要 JS:contentRule 或 bookInfoRule 依赖 @js 或 <js>
needsPost, // 需要 POST 搜索:searchMethod == POST 或 URL 包含 ,{
needsHeaders, // 需要自定义请求头
broken, // 已失效:基础 URL 不可达
unknown, // 尚未验证
}
class SourceCompatInfo {
final SourceCompatLevel level;
final bool searchOk; // 搜索规则可执行
final bool bookInfoOk; // 详情规则可执行
final bool tocOk; // 目录规则可执行
final bool contentOk; // 正文规则可执行
final bool hasJsRule; // 规则中包含 @js / <js>
final bool hasPutGet; // 规则中包含 @put / @get
final List<String> warnings; // 警告信息列表
}
```
### 9.3 检测逻辑(导入时运行)
```dart
SourceCompatInfo analyzeSource(LegadoSource source) {
final warnings = <String>[];
bool hasJs = false;
bool hasPutGet = false;
bool hasChainSyntax = false;
// 1. 检查所有规则字段是否包含 JS 语法
for (final rule in _allRuleFields(source)) {
if (rule == null) continue;
if (rule.contains('@js') || rule.contains('<js>')) hasJs = true;
if (rule.contains('@put') || rule.contains('@get')) hasPutGet = true;
if (rule.contains('@tag.') || rule.contains('@src') ||
rule.contains('@text') || rule.contains('@html')) {
hasChainSyntax = true;
}
}
// 2. 评估各模块可行性
final searchOk = source.searchUrl != null && source.searchListRule != null;
final bookInfoOk = source.bookInfoTocRule != null || source.bookInfoNameRule != null;
final tocOk = source.tocListRule != null || source.tocNameRule != null;
final contentOk = source.contentRule != null;
// 3. 确定级别
SourceCompatLevel level;
if (!searchOk && !contentOk) {
level = SourceCompatLevel.broken;
} else if (hasJs) {
level = SourceCompatLevel.needsJs;
} else if (hasPutGet) {
level = SourceCompatLevel.partial;
} else if (!bookInfoOk && !contentOk) {
level = SourceCompatLevel.partial; // 只能搜
} else {
level = SourceCompatLevel.full;
}
return SourceCompatInfo(level: level, ...);
}
```
### 9.4 UI 展示
在 `source_manager_page.dart` 的书源列表中,每行右侧显示兼容度标签:
```
包子漫画5 [🔴 需要 JS]
6漫画 [🔴 需要 JS]
全本玄幻小说 [🟢 完全兼容]
WebToon [🟡 需要 POST]
某已失效源 [⚫ 已失效]
```
### 9.5 改动量
- `legado_models.dart` 新增 `SourceCompatInfo` 类 ~30 行
- `source_manager.dart` 新增 `analyzeSource()` ~50 行
- `source_manager_page.dart` UI 展示 ~40 行
- 总计 ~120 行
- **零新依赖**
---
## 十、第 8 步:完整 JS 引擎
### 10.1 时机
前 7 步完成后,已有书源中约 60-70% 的漫画源可正常阅读(通过 JS 特判覆盖)。剩下的 30-40% 需要完整 JS 执行环境。
### 10.2 推荐方案
| 方案 | 优点 | 缺点 |
|------|------|------|
| **flutter_js** | 纯 Dart,不依赖 WebView | 性能一般,JS 互操作受限 |
| **quickjs_dart_bridge** | 轻量、快、支持 ES2020 | 需要 Native 桥接 |
| **flutter_js_engine** | 基于 JavaScriptCore/QuickJS | 体积较大 |
| **WebView 隐藏执行** | 兼容性最好 | 太重,不适合批量 |
**建议**:先用 **quickjs_dart_bridge**(已有成功集成到阅读器的先例)。
### 10.3 注入的 JS 运行环境
```js
// 注入变量
result; // 当前 HTML 或 RuleParser 结果
baseUrl; // 当前请求的 base URL
// 注入函数(兼容 Legado 的 java.xxx 调用风格)
java.get(key); // → 从 _context 读
java.put(key, value); // → 写入 _context
java.getElements(html, selector); // → querySelectorAll → 返回元素列表
java.base64Decode(str); // → base64 解码
java.ajax(url); // → HTTP GET → 返回响应文本
// 注入输出收集器
__result; // 脚本执行完毕后读取此变量作为输出
```
### 10.4 安全措施
- JS 沙箱:禁止 `XMLHttpRequest`、`fetch`、`localStorage`
- 超时限制:每条 JS 规则最多执行 3 秒
- 错误隔离:单条 JS 规则失败不影响其他规则
- Isolate 执行:防卡 UI(可选,复杂度较高)
### 10.5 改动量
- `pubspec.yaml` 新增 `quickjs_dart_bridge` 依赖
- 新建 `legado_js_runner.dart` ~150 行
- `LegadoRuleExecutor` 集成 JS 执行 ~30 行
- 总计 ~180 行
---
## 十一、完整实施路线图
```
Week 1 ████████░░░░░░░░░░░░
第1步: rawHtml 兜底 + script 变量扫描
第2步: @tag/@text/@html/@src 链式语法
Week 2 ████████░░░░░░░░░░░░
第3步: ##/||/&& 语法
第4步: 错误类型区分
Week 3 ████████████████░░░░
第5步: JS 特判
第6步: @put/@get
第7步: 书源兼容度检测
Week 4+ ████████████████████
第8步: 完整 JS 引擎
测试: 30+ 书源的回归测试
零新依赖 新依赖
├───────────────────┼──────────┤
第1步 第8步
第2步
第3步
第4步
第5步
第6步
第7步
```
---
## 十二、改动文件清单
| 文件 | 第1步 | 第2步 | 第3步 | 第4步 | 第5步 | 第6步 | 第7步 | 第8步 |
|------|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|
| `legado_rule_parser.dart` | | 🔧250 | 🔧80 | | | 🔧45 | | |
| `legado_engine.dart` | 🔧40 | | | 🔧70 | 🔧100 | 🔧10 | | 🔧30 |
| `legado_models.dart` | | | | | | | 🔧30 | |
| `source_manager.dart` | | | | | | | 🔧50 | |
| `source_manager_page.dart` | | | | | | | 🔧40 | |
| `online_book_detail_page.dart` | | | | 🔧30 | | | | |
| `legado_js_runner.dart` 🆕 | | | | | | | | 🆕150 |
| `pubspec.yaml` | | | | | | | | +1dep |
---
## 十三、总结
整个方案的核心理念是:
```
RuleParser 是对现有 CSS/JSON/Regex 的兼容 → 保留为底层原子工具
LegadoRuleExecutor 是新增的链式规则执行层 → 理解 @tag、##、||、&&
JS 分两级做 → 特判覆盖 60% + 引擎覆盖 40%
书源兼容度检测 → 导入时打标签,用户体验闭环
请求层不动 —— 它没问题
Bug 优先级由截图驱动 —— 第1步修截图3,第2步解锁主流漫画源,第4步修截图1
前4步零新依赖,直接在现有文件里改
```
**一句话**:不推翻现有代码,在 `RuleParser` 之上加一层**链式规则执行器**,让 `@tag.img@src` 和 `##替换` 能被正确理解,这是解锁包子漫画 5 等主流漫画源的关键。
---
## 附录:All.json 常见规则语法统计
基于对 30+ 个典型漫画/小说书源规则的手工分析:
| 语法 | 出现频率 | 典型场景 | 本方案覆盖 |
|------|:--------:|---------|:----------:|
| `@tag.xxx` | ★★★★★ | 漫画正文图片提取 | 第2步 |
| `@src` | ★★★★★ | 图片 URL 提取 | 第2步 |
| `@text` | ★★★★ | 书名/作者/章节名 | 第2步 |
| `@html` | ★★★ | 正文 HTML 提取 | 第2步 |
| `##` | ★★★★ | 广告/水印净化 | 第3步 |
| `\|\|` | ★★★★ | 备选规则 | 第3步 |
| `&&` | ★★★ | 多字段拼接 | 第3步 |
| `<js>` / `@js:` | ★★★ | 漫画图片解密 | 第5/8步 |
| `@put` / `@get` | ★★ | 跨步骤传递 ID | 第6步 |
| `@css:` | ★★★ | 嵌套 CSS 选择 | 第2步 |
| `@data-xxx` | ★★ | data-original 等 | 第2步 |
| `,{method,body}` | ★★ | POST 搜索 | 请求层(现有) |
File diff suppressed because it is too large. Load diff
Binary file not shown.
@@ -0,0 +1,393 @@
# 小说与漫画三级层级展示 —— 前端改造方案
## Context
当前 `sakura_admin_web` 的小说和漫画展示是 **2 级扁平结构**:列表页将同一部作品的章节折叠为一张"作品卡片",点击直接跳转到单章阅读器。用户无法浏览"一本书下面有哪些章节"这个中间层。
本方案将漫画和小说改为 **3 级层级导航**:
```
Level 1 (系列列表) → Level 2 (章节列表) → Level 3 (内容阅读)
```
## 后端 API 现状与数据模型
### Novel(小说)
- **有专用 series API**:`GET /novels/series`(列表)、`GET /novels/series/{series_id}`(单系列含全部章节)
- `NovelSeriesGroup` 包含:`series_id, series_title, author, chapter_count, cover_path, chapters[]`
- 每个 chapter(`NovelOut`)有:`id, title, chapter_title, chapter_order, updated_at`
- 当前前端调的是 `GET /me/novels`(扁平 rows),客户端自己 `groupNovels()`
### Manga/Comic(漫画)
- **无专用 series API**,漫画存储为 `Artwork` 行(`content_type="manga"`)
- 系列分组依赖 JSON 字段 `artwork_metadata.series_id` / `artwork_metadata.series_title`
- 当前前端调用 `GET /me/gallery?content_type=manga`,客户端 `groupManga()` 分组
- 需要在客户端缓存分组数据,供 Level 2 页面使用
---
## 关键设计决策(必读)
### 1. 漫画 seriesKey 稳定性规则
`seriesKey` 是 Level 1 跳 Level 2 的路由参数,也是缓存的索引,必须稳定且防冲突。
**构造规则(优先级从高到低):**
```
seriesKey = series_id
? `manga-${series_id}`
: `manga-title-${series_title || title}`
```
说明:
- 优先用 `artwork_metadata.series_id`,它最稳定
- `series_id` 缺失时用 `series_title || title` 回退,加 `manga-title-` 前缀避免与 `manga-` 冲突
- **所有来源的值都要 `trim()`**
- URL 传递时使用 `encodeURIComponent(seriesKey)`,接收端用 Vue Router 自动解码后的值
### 2. 小说 series_id 类型统一
后端可能返回数字或字符串类型的 `series_id`。**所有地方必须统一转成字符串:**
```
seriesId: String(item.series_id)
```
原因:Vue Router 的 `:seriesId` 参数总是字符串,如果缓存 key 或比较逻辑里混了数字类型,会导致匹配失败。
### 3. 漫画缓存缺失回退流程(明确)
当用户直接访问 `/app/manga/series/:seriesKey` 而缓存为空时,按以下步骤回退:
1. 调用 `GET /me/gallery?content_type=manga&limit=200`
2. 对返回的 rows 执行与 `UserListPage.groupManga()` **完全一致的分组逻辑**
3. 在所有分组结果中查找 `seriesKey` 匹配的组(匹配规则同构造规则)
4. 找到后填充页面
5. 未找到则显示错误:"未找到该系列的章节"
这样保证了不管是从 Level 1 正常导航进来,还是直接访问 URL,显示结果一致。
### 4. Level 3 返回逻辑的 fallback
阅读器中通过 `fromSeries` query 参数构建"返回章节列表"链接:
```
if (fromSeries) → /app/${kind}/series/${fromSeries}
else → /app/${kind} (返回系列列表)
```
面包屑中间的"章节列表"链接同理——没有 `fromSeries` 时不渲染该级面包屑,直接从 Level 1 跳到 Level 3。
### 5. 路由顺序严格要求
`/app` 下的漫画和小说子路由必须按以下顺序注册(**顺序错会导致路由匹配错误**):
```
1. manga ← Level 1
2. manga/series/:seriesKey ← Level 2(固定路径段优先)
3. manga/read/:id ← Level 3(固定路径段优先)
4. manga/:id → redirect ← 旧路由兼容,放最后
5. novels ← Level 1
6. novels/series/:seriesId ← Level 2(固定路径段优先)
7. novels/read/:id ← Level 3(固定路径段优先)
8. novels/:id → redirect ← 旧路由兼容,放最后
```
原理:`manga/series/:seriesKey` 有固定前缀 `series/`,如果放在 `manga/:id` 之后,Vue Router 会把 `series` 当成 `:id` 的值匹配进去。**固定路径段必须排在动态 `:id` 前面。**
### 6. 章节数据 normalize
不同来源的章节字段命名不一致(漫画在 `artwork_metadata` 里嵌套、小说是顶层字段),建议统一写一个 normalize 函数,不要在各处散着取字段。
**小说章节 normalize(输入:`NovelOut`):**
| 要取的字段 | 取值路径 |
|-----------|---------|
| `id` | `String(item.id)` |
| `title` | `item.chapter_title \|\| item.title \|\| "未命名章节"` |
| `order` | `Number(item.chapter_order \|\| 0)` |
| `updated_at` | `item.updated_at \|\| ""` |
**漫画章节 normalize(输入:`ArtworkOut` 单行):**
| 要取的字段 | 取值路径 |
|-----------|---------|
| `id` | `String(item.id)` |
| `title` | `item.title \|\| item.artwork_metadata?.chapter_title \|\| "未命名章节"` |
| `order` | `Number(item.artwork_metadata?.chapter_order \|\| 0)` |
| `page_count` | `Number(item.page_count \|\| 1)` |
| `thumbnail_url` | `item.thumbnail_url \|\| ""` |
| `updated_at` | `item.updated_at \|\| ""` |
在 `UserListPage.groupManga()` 和 `UserChapterListPage.loadMangaSeries()` 中都应使用同一套 normalize。
### 7. Level 2 的状态覆盖
`UserChapterListPage` 必须覆盖以下四种状态,否则直接访问 URL 时体验差:
| 状态 | 条件 | UI |
|------|------|----|
| **加载中** | `loading === true` | `"正在加载章节..."` |
| **加载失败** | `error !== ""` | 显示错误信息,带"重试"按钮 |
| **空章节** | `!loading && !error && chapters.length === 0` | `"暂无章节"` |
| **缓存失效重载** | manga 缓存命空 → 自动触发 API 回退 | `"正在重新加载..."`(过渡态) |
---
## 路由改造
### 新路由结构
**文件:** `src/router/index.ts` —— 修改
```
修改前(2 级) 修改后(3 级)
───────────────── ─────────────────
/app/novels /app/novels ← Level 1 系列列表
/app/novels/:id /app/novels/series/:seriesId ← Level 2 章节列表(新)
/app/novels/read/:id ← Level 3 阅读器(原 /novels/:id)
/app/manga /app/manga ← Level 1 系列列表
/app/manga/:id /app/manga/series/:seriesKey ← Level 2 章节列表(新)
/app/manga/read/:id ← Level 3 详情(原 /manga/:id)
```
**路由注册顺序(严格):**
```ts
// 漫画
{ path: "manga", component: UserListPage, props: { kind: "manga" } },
{ path: "manga/series/:seriesKey", component: UserChapterListPage, props: (route) => ({ kind: "manga", seriesKey: route.params.seriesKey }) },
{ path: "manga/read/:id", component: UserDetailPage, props: (route) => ({ kind: "manga", id: route.params.id }) },
// 小说
{ path: "novels", component: UserListPage, props: { kind: "novels" } },
{ path: "novels/series/:seriesId", component: UserChapterListPage, props: (route) => ({ kind: "novels", seriesId: route.params.seriesId }) },
{ path: "novels/read/:id", component: UserDetailPage, props: (route) => ({ kind: "novels", id: route.params.id }) },
// 旧路由重定向(必须放最后)
{ path: "manga/:id", redirect: (to) => `/app/manga/read/${to.params.id}` },
{ path: "novels/:id", redirect: (to) => `/app/novels/read/${to.params.id}` },
```
---
## 改造清单
### 一、新建文件
#### 1. `src/stores/seriesCache.ts` —— 漫画系列缓存
**为何需要:** 后端没有 manga series 端点。Level 1 页面做完客户端分组后,需要把分组数据传给 Level 2 页面。使用 Vue `reactive()` 对象做跨组件共享缓存。
```ts
// 核心结构
interface MangaChapterCacheEntry {
id: string // gallery item id
title: string // 章节标题(normalize 后)
page_count: number // 页数
thumbnail_url: string // 缩略图
updated_at: string // 更新时间
}
interface MangaSeriesCacheEntry {
seriesKey: string // 唯一标识(按"关键设计决策 1"构造)
seriesTitle: string // 系列标题
coverUrl: string // 封面图
author: string // 作者
chapters: MangaChapterCacheEntry[] // 该系列下所有章节(已 normalize 且按 order 排序)
}
// 导出 mangaSeriesCache: reactive<Record<string, MangaSeriesCacheEntry>>
// 导出 clearMangaCache(): void
```
#### 2. `src/views/app/UserChapterListPage.vue` —— 章节列表页(Level 2)
**复用项目 pattern:** 通过 `kind` prop 同时支持 novels 和 manga。
**Props:**
```ts
{
kind: "novels" | "manga"
seriesId?: string // novels: 后端 series_id(统一 String 类型)
seriesKey?: string // manga: 客户端分组 key
}
```
**模板结构:**
- 面包屑导航:`小说/漫画 > 系列标题`
- "← 返回" 按钮(`router.back()`)
- **四种状态覆盖:**
- 加载中 → `"正在加载章节..."`
- 加载失败 → `error` 文本 + 重试按钮
- 空章节 → `"暂无章节"`(加载成功但 chapters 为空)
- 缓存失效重载 → manga 缓存命中但 chapters 为空时,自动切到 API 回退
- 系列信息头:封面 + 标题 + 作者 + 章节数
- 章节列表:每行显示序号 | 章节标题 | 页数/更新时间 | →
**数据加载:**
- `kind="novels"`:调用 `GET /novels/series/{seriesId}`,对返回的 chapters 执行 normalize
- `kind="manga"`:
1. 优先读 `mangaSeriesCache[seriesKey]`
2. 缓存命中且 chapters 不为空 → 直接展示
3. 缓存未命中或 chapters 为空 → 按"关键设计决策 3"执行 API 回退
**跳转到 Level 3:** `/app/{kind}/read/{id}?fromSeries={encodedSeriesRef}`
---
### 二、修改现有文件
#### 3. `src/views/app/UserListPage.vue` —— 系列列表页(Level 1)
##### 3.1 novels 加载改为 `/novels/series`
当前 `load()` → `GET /me/novels` → `groupNovels()`。改为:
- `endpoint()` 对 novels 返回 `/novels/series`
- `params()` 对 novels 传 `{ include_chapters: false }`(后端返回轻量 series 摘要)
- `groupNovels()` 直接从 series 数据映射 WorkGroup,不再做客户端分组
- WorkGroup 新增 `seriesId: String(series.series_id)` 字段
##### 3.2 manga 加载保持 `/me/gallery`,增强分组逻辑
- 按照"关键设计决策 1"构造稳定的 `seriesKey`
- 构建 WorkGroup 时同步写入 `mangaSeriesCache[seriesKey]`
- 缓存中的 chapters 按"关键设计决策 6"的 normalize 规则统一格式,并已按 order 排序
- WorkGroup 新增 `seriesKey` 字段
##### 3.3 detailUrl 修改
```ts
// 修改前:直接跳阅读器
function detailUrl(id: string) { return `/app/${props.kind}/${id}` }
// 修改后:跳章节列表页
function detailUrl(group: WorkGroup): string {
if (kind === "novels") return `/app/novels/series/${group.seriesId}`
if (kind === "manga") return `/app/manga/series/${encodeURIComponent(group.seriesKey!)}`
...
}
// 为 gallery / alerts 保留原始逻辑
function detailUrlById(id: string): string {
return `/app/${props.kind}/${id}`
}
```
##### 3.4 WorkGroup 类型扩展
```ts
type WorkGroup = {
// ... 原有字段
seriesKey?: string // manga 用(稳定构造)
seriesId?: string // novels 用(统一 String)
}
```
##### 3.5 分页调整
novels 改用 `/novels/series` 后不再需要前端分页(后端一次返回所有系列),因此:
- `supportsPagination` computed 对 novels 返回 false
- 隐藏分页控件
#### 4. `src/views/app/UserDetailPage.vue` —— 阅读器页(Level 3)
- **新增面包屑导航**(仅 novels/manga):
- 始终显示:`小说/漫画 > 当前章节标题`
- `fromSeries` 存在时插入中间级:`小说/漫画 > [章节列表] > 当前章节标题`
- **新增返回按钮**:`"← 返回"`(`$router.back()`)
- **`fromSeries` 参数处理**(按"关键设计决策 4"):
```ts
const fromSeries = computed(() => String(route.query.fromSeries || ""))
const backToListRoute = computed(() => {
if (fromSeries.value) return `/app/${props.kind}/series/${fromSeries.value}`
return `/app/${props.kind}` // fallback 到系列列表
})
```
- 原有阅读器/详情逻辑不变
#### 5. `src/router/index.ts` —— 路由配置
- 新增 `UserChapterListPage` 导入
- 新增 4 条路由(novels/manga 的 series 和 read),严格按照"关键设计决策 5"的顺序排列
- 末尾添加 2 条旧路由重定向
---
### 三、CSS 新增样式
**文件:** `src/styles.css` —— 追加
| 样式类 | 用途 |
|---------|------|
| `.breadcrumb` | 面包屑导航容器(flex, 小字号 muted) |
| `.breadcrumb a` | 面包屑链接(pink-strong, hover opacity) |
| `.breadcrumb .current` | 当前页名(ink, bold) |
| `.series-header` | 章节列表页的系列信息头(flex, 封面+标题+作者+统计) |
| `.series-cover` | 系列封面图(110px 宽, 3:4 比例) |
| `.series-cover.placeholder` | 封面色块占位(粉色背景, 居中文字) |
| `.series-info` | 系列元信息容器 |
| `.series-info h2` | 系列标题(ellipsis 溢出省略) |
| `.chapter-list` | 章节列表网格(gap 8px) |
| `.chapter-row` | 单行章节(flex, hover 阴影+边框变色) |
| `.chapter-row .chapter-index` | 圆形粉色序号标识(36px 圆, number) |
| `.chapter-row .chapter-info` | 章节标题与元信息(flex 收缩, ellipsis) |
| `.chapter-row .chapter-info strong` | 章节标题 |
| `.chapter-row .chapter-info .muted` | 页数 / 更新时间的灰色副文本 |
---
## 数据流示意
### 小说
```
UserListPage ──GET /novels/series?include_chapters=false──→ [{ series_id, chapter_count, cover_path }]
│ seriesId = String(series_id)
│ 点击卡片 → /app/novels/series/:seriesId
▼
UserChapterListPage ──GET /novels/series/:id──→ { chapters: [{id, chapter_title, ...}] }
│ 对 chapters 执行 normalize
│ 点击章节 → /app/novels/read/:id?fromSeries=:seriesId
▼
UserDetailPage ──GET /novels/:id/content──→ 正文阅读
│ 面包屑:小说 > [章节列表] > 当前章名
│ fromSeries 缺失时省略中间级
```
### 漫画
```
UserListPage ──GET /me/gallery?content_type=manga──→ client groupManga()
│ seriesKey 按稳定规则构造(series_id 优先)
│ 章节按 normalize 统一格式、排序
│ └→ 写入 mangaSeriesCache[seriesKey]
│ 点击卡片 → /app/manga/series/:seriesKey(encoded)
▼
UserChapterListPage ──读 mangaSeriesCache[seriesKey]──→ 展示
│ 缓存未命中或 chapters 为空:
│ ① GET /me/gallery?content_type=manga&limit=200
│ ② 执行同一套 groupManga() 分组逻辑
│ ③ 查找匹配 seriesKey 的组
│ ④ 找到 → 填充;未找到 → 显示错误
│ 点击章节 → /app/manga/read/:id?fromSeries=:seriesKey(encoded)
▼
UserDetailPage ──GET /gallery/:id──→ 详情展示
│ 面包屑:漫画 > [章节列表] > 当前章名
│ fromSeries 缺失时省略中间级
```
---
## 验证方案
1. **Level 1 → Level 2 导航**:访问 `/app/novels`,点击任意作品卡片,确认跳转到 `/app/novels/series/:seriesId`,且显示正确的章节列表
2. **Level 2 → Level 3 导航**:在章节列表页点击某一章,确认跳转到 `/app/novels/read/:id?fromSeries=:seriesId`,正文正常显示
3. **面包屑返回**:在阅读器中点击面包屑"章节列表",确认返回 Level 2;点击"← 返回",确认能正常回退
4. **面包屑 fallback**:直接访问 `/app/novels/read/:id`(无 fromSeries),确认面包屑不显示中间级,返回按钮回到 `/app/novels`
5. **漫画缓存链路**:从 Level 1 → Level 2 → Level 3 完整走通漫画导航
6. **漫画缓存缺失回退**:清除缓存后直接访问 `/app/manga/series/:seriesKey`,确认自动回退到 API 拉取并正确展示
7. **漫画 series_id 缺失**:对没有 `artwork_metadata.series_id` 的漫画数据,确认 seriesKey 回退到 `manga-title-{series_title}` 且导航正常
8. **Level 2 空状态**:对章节数为 0 的系列(理论上不应出现,但做防御),确认显示"暂无章节"
9. **Level 2 错误状态**:模拟 API 失败,确认显示错误信息 + 重试按钮
10. **不受影响的功能**:图片(gallery)、提醒(alerts)、个人资料(profile)页面仍正常运作
11. **旧路由兼容**:访问 `/app/novels/旧id` 确认重定向到 `/app/novels/read/旧id`
12. **管理后台不受影响**:`/admin/novels`、`/admin/gallery` 表格正常
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,656 @@
# 两项核心改进 — TOC 分页规则 · 搜索健康分
> 日期:2026-06-22
> 范围:仅移动端 Dart 侧(`sakura_mobile/lib/src/core/`)
> 涉及文件:
>
> - `legado_models.dart` — 数据模型
> - `legado_rule_parser.dart` — 规则解析器
> - `legado_engine.dart` — 搜索引擎入口
> 原则:不改 Python 插件,不改 UI 层,只动 core 层。
---
## 一、改进 1:TOC 章节目录分页规则完整支持
### 1.1 问题
`fetchToc()` 只拉第一页目录。很多书源第一页只给 50~100 章,剩下的靠 **分页目录规则** 指向下一页。不支持分页规则 → 用户看到的目录永远不全。
### 1.2 兼容的字段名
书源 JSON 中,下一页 URL 的字段名可能是以下任意一种,按优先级遍历取第一个非空值:
```dart
// legado_models.dart — BookSource / TocRule 中新增字段解析
static const List<String> _tocNextPageCandidates = [
'tocNextPageRule', // Legado 标准名
'nextTocUrl', // 常见别名
'nextUrl', // 通用别名
'nextPage', // 页码式
'tocNextPage', // 变体
];
String? get tocNextPageRule {
for (final name in _tocNextPageCandidates) {
final v = rawJson[name];
if (v is String && v.trim().isNotEmpty) return v.trim();
}
return null;
}
```
### 1.3 分页规则语法全集
| 规则类型 | 语法示例 | 说明 |
| --------- | ----------------------------------- | ------------------ |
| JS 表达式 | `<js>nextUrl</js>` | 执行 JS 获取下一页 URL |
| JS 缩写 | `@js:nextUrl` | 同上,简写形式 |
| 模板拼接 | `https://host/list?page={{page}}` | `{{page}}` 从 1 自增 |
| 偏移模板 | `{{page+1}}` / `{{page-1}}` | 页号偏移 |
| CSS 选择器 | `text.下一页@href` | 从当前页 HTML 提取下一页链接 |
| class 选择器 | `class.next@href` | 同上 |
| id 选择器 | `id.pagenext@href` | 同上 |
| 属性选择器 | `@href` | 取当前元素 href |
| XPath | `//a[contains(text(),'下一页')]/@href` | XPath 提取 |
| 字符串切片 | `!0:-1` | 对当前 URL 切片(去掉末尾字符) |
| offset 翻页 | `offset=50` | 按 offset 递增翻页 |
| 纯文本 URL | `https://...` | 直接跳到指定 URL |
| 相对路径 | `/list/2.html` | 基于当前 URL 拼接 |
### 1.4 实现 — `TocPaginationResolver`
```dart
// 放在 legado_engine.dart 或新建 legado_toc_pagination.dart
class TocPaginationResolver {
final LegadoRuleExecutor _ruleExecutor;
final HttpClient _client;
TocPaginationResolver(this._ruleExecutor, this._client);
/// 拉取全部分页章节,合并去重。
Future<List<ChapterInfo>> fetchAllChapters({
required String firstPageUrl,
required String? tocNextPageRule,
required String tocRule,
required Map<String, String>? headers,
int maxPages = 20,
}) async {
final all = <ChapterInfo>[];
final seenPageUrls = <String>{};
final seenChapterUrls = <String>{};
String? url = firstPageUrl;
int page = 0;
while (url != null && page < maxPages) {
if (!seenPageUrls.add(url)) break; // 死循环保护
final html = await _client.get(url, headers: headers);
final batch = _ruleExecutor.selectList(html, url, rule: tocRule);
int added = 0;
for (final ch in batch) {
if (seenChapterUrls.add(ch.url)) {
all.add(ch);
added++;
}
}
if (added == 0) break; // 本页无新章节,终止翻页
url = _resolveNextPage(
currentUrl: url,
currentHtml: html,
rule: tocNextPageRule,
pageIndex: page,
);
page++;
}
return all;
}
// ---------- 下一页 URL 解析 ----------
String? _resolveNextPage({
required String currentUrl,
required String currentHtml,
required String? rule,
required int pageIndex,
}) {
if (rule == null || rule.trim().isEmpty) return null;
final r = rule.trim();
// 1) JS: <js>...</js> 或 @js:...
if (r.startsWith('<js>') || r.startsWith('@js:')) {
final code = r
.replaceFirst(RegExp(r'^<js>'), '')
.replaceFirst(RegExp(r'</js>$'), '')
.replaceFirst(RegExp(r'^@js:'), '');
return _ruleExecutor.evalJs(code, context: {
'result': currentHtml,
'baseUrl': currentUrl,
'page': pageIndex,
});
}
// 2) 切片: !start:end
if (r.startsWith('!')) {
return _applySlice(currentUrl, r.substring(1));
}
// 3) CSS 选择器 / XPath
if (_isSelector(r)) {
return _ruleExecutor.select(currentHtml, currentUrl, rule: r);
}
// 4) 模板替换 {{page}} / {{page+1}} / {{page-1}}
if (r.contains('{{page')) {
return _renderPageTemplate(r, pageIndex);
}
// 5) offset / start 参数
if (RegExp(r'[?&](offset|start)=').hasMatch(r)) {
return _renderOffsetTemplate(r, pageIndex);
}
// 6) 绝对 URL
if (r.startsWith('http://') || r.startsWith('https://')) {
return _renderPageTemplate(r, pageIndex);
}
// 7) 相对路径
return Uri.parse(currentUrl).resolve(r).toString();
}
// ---------- 规则判定 ----------
bool _isSelector(String s) {
return s.startsWith('text.') ||
s.startsWith('class.') ||
s.startsWith('id.') ||
s.startsWith('@') ||
s.startsWith('//'); // XPath
}
// ---------- 模板渲染 ----------
String _renderPageTemplate(String tpl, int page) {
return tpl
.replaceAll('{{page+1}}', '${page + 1}')
.replaceAll('{{page-1}}', '${page - 1}')
.replaceAll('{{page}}', '$page');
}
String _renderOffsetTemplate(String tpl, int page) {
return tpl
.replaceAllMapped(
RegExp(r'offset=(\d+)'),
(m) => 'offset=${int.parse(m.group(1)!) * page}',
)
.replaceAllMapped(
RegExp(r'start=(\d+)'),
(m) => 'start=${int.parse(m.group(1)!) * page}',
);
}
String? _applySlice(String url, String expr) {
final parts = expr.split(':');
if (parts.length != 2) return null;
final a = int.tryParse(parts[0]) ?? 0;
final b = int.tryParse(parts[1]);
final start = a < 0 ? url.length + a : a;
if (b != null) {
final end = b < 0 ? url.length + b : b;
return url.substring(start, end);
}
return url.substring(start);
}
}
```
### 1.5 集成到 `LegadoEngine.fetchToc`
```dart
// legado_engine.dart — 修改 fetchToc
Future<List<ChapterInfo>> fetchToc(BookSource source, String tocUrl) async {
final rule = source.tocRule;
if (rule == null || rule.isEmpty) return [];
final resolver = TocPaginationResolver(_ruleExecutor, _client);
return resolver.fetchAllChapters(
firstPageUrl: tocUrl,
tocNextPageRule: source.tocNextPageRule, // ← 新增
tocRule: rule,
headers: _buildHeaders(source),
maxPages: 20,
);
}
```
### 1.6 死循环防护(三层)
1. **`maxPages = 20`** — 单书最多翻 20 页目录
2. **`seenPageUrls`** — 同一 URL 不重复请求
3. **`added == 0` 终止** — 当前页没有新增章节则停止
---
## 二、改进 2:搜索源健康分 + 短超时 + 分阶段返回
### 2.1 问题
当前 `search()` 对所有启用源并发请求,每个源 timeout 20~30 秒。10 个源一批,坏源多时整批拖到 20+ 秒才出结果。
### 2.2 源健康分模型
```dart
// legado_models.dart — 新增 SourceHealth
class SourceHealth {
final String sourceId;
int successCount = 0;
int failCount = 0;
int timeoutCount = 0;
int totalLatencyMs = 0;
DateTime? lastSuccessAt;
DateTime? lastFailAt;
int _consecutiveFails = 0;
SourceHealth({required this.sourceId});
// ---- 判定 ----
/// 连续失败 ≥ 5 次 → 临时禁用
bool get isUsable {
if (_consecutiveFails >= 5) return false;
if (failCount >= 5 && successCount == 0) return false;
return true;
}
int get consecutiveFails => _consecutiveFails;
/// 健康分 0~100
double get score {
final total = successCount + failCount + timeoutCount;
if (total == 0) return 50.0; // 新源默认 50
double s = 0.0;
// 成功率 60%
s += 60.0 * (successCount / total);
// 平均延迟 25%
if (successCount > 0 && totalLatencyMs > 0) {
final avg = totalLatencyMs / successCount;
if (avg < 500) {
s += 25.0;
} else if (avg < 2000) {
s += 25.0 * (1.0 - (avg - 500) / 1500);
}
}
// 新鲜度 15%(24h 内有成功记录)
if (lastSuccessAt != null) {
final h = DateTime.now().difference(lastSuccessAt!).inHours;
if (h < 1) {
s += 15.0;
} else if (h < 24) {
s += 15.0 * (1.0 - h / 24);
}
}
return s.clamp(0.0, 100.0);
}
// ---- 记录 ----
void recordSuccess(int latencyMs) {
successCount++;
totalLatencyMs += latencyMs;
lastSuccessAt = DateTime.now();
_consecutiveFails = 0;
}
void recordFailure() {
failCount++;
lastFailAt = DateTime.now();
_consecutiveFails++;
}
void recordTimeout() {
timeoutCount++;
lastFailAt = DateTime.now();
_consecutiveFails++;
}
void resetConsecutiveFails() => _consecutiveFails = 0;
// ---- 序列化(持久化到 SharedPreferences)----
Map<String, dynamic> toJson() => {
'sourceId': sourceId,
'successCount': successCount,
'failCount': failCount,
'timeoutCount': timeoutCount,
'totalLatencyMs': totalLatencyMs,
'lastSuccessAt': lastSuccessAt?.toIso8601String(),
'lastFailAt': lastFailAt?.toIso8601String(),
'consecutiveFails': _consecutiveFails,
};
factory SourceHealth.fromJson(Map<String, dynamic> json) {
final h = SourceHealth(sourceId: json['sourceId']);
h.successCount = json['successCount'] ?? 0;
h.failCount = json['failCount'] ?? 0;
h.timeoutCount = json['timeoutCount'] ?? 0;
h.totalLatencyMs = json['totalLatencyMs'] ?? 0;
h.lastSuccessAt = json['lastSuccessAt'] != null
? DateTime.tryParse(json['lastSuccessAt'])
: null;
h.lastFailAt = json['lastFailAt'] != null
? DateTime.tryParse(json['lastFailAt'])
: null;
h._consecutiveFails = json['consecutiveFails'] ?? 0;
return h;
}
}
```
### 2.3 健康分持久化
```dart
// legado_engine.dart — SourceHealthStore
class SourceHealthStore {
static const _key = 'source_health_v1';
final Map<String, SourceHealth> _data = {};
Future<void> load() async {
final prefs = await SharedPreferences.getInstance();
final raw = prefs.getStringList(_key);
if (raw == null) return;
for (final s in raw) {
try {
final h = SourceHealth.fromJson(jsonDecode(s));
_data[h.sourceId] = h;
} catch (_) {}
}
}
Future<void> save() async {
final prefs = await SharedPreferences.getInstance();
final raw = _data.values.map((h) => jsonEncode(h.toJson())).toList();
await prefs.setStringList(_key, raw);
}
bool isUsable(String id) => _data[id]?.isUsable ?? true;
double score(String id) => _data[id]?.score ?? 50.0;
void _ensure(String id) {
_data.putIfAbsent(id, () => SourceHealth(sourceId: id));
}
void recordSuccess(String id, int latencyMs) {
_ensure(id);
_data[id]!.recordSuccess(latencyMs);
}
void recordFailure(String id) {
_ensure(id);
_data[id]!.recordFailure();
}
void recordTimeout(String id) {
_ensure(id);
_data[id]!.recordTimeout();
}
/// 用户手动重置所有源健康状态
void resetAll() {
for (final h in _data.values) {
h.resetConsecutiveFails();
}
}
}
```
### 2.4 分阶段搜索调度器
```dart
// legado_engine.dart — SearchScheduler
class SearchScheduler {
static const fastTimeout = Duration(seconds: 5);
static const slowTimeout = Duration(seconds: 8);
static const maxConcurrent = 6;
final SourceHealthStore _health;
final LegadoEngine _engine;
SearchScheduler(this._health, this._engine);
/// 返回 Stream,UI 层逐阶段渲染结果
Stream<SearchPhase> search(String keyword, List<BookSource> allSources) async* {
// 过滤 + 按健康分排序
final usable = allSources
.where((s) => _health.isUsable(s.id))
.toList()
..sort((a, b) => _health.score(b.id).compareTo(_health.score(a.id)));
if (usable.isEmpty) {
yield SearchPhase(name: 'error', message: '没有可用的搜索源');
return;
}
// 分三档
final fast = usable.where((s) => _health.score(s.id) >= 70).toList();
final normal = usable.where((s) {
final sc = _health.score(s.id);
return sc >= 40 && sc < 70;
}).toList();
final slow = usable.where((s) => _health.score(s.id) < 40).toList();
final results = <SearchItem>[];
final doneIds = <String>{};
final seen = <String>{};
// ---- 阶段 1:快速源,超时 5s ----
yield SearchPhase(name: 'searching', message: '正在搜索 ${usable.length} 个源...');
final r1 = await _batch(fast, keyword, timeout: fastTimeout, onDone: doneIds.add);
_merge(results, r1, seen);
yield SearchPhase(
name: 'partial',
items: List.of(results),
doneCount: doneIds.length,
totalCount: usable.length,
);
// ---- 阶段 2:普通源,超时 8s ----
final r2 = await _batch(normal, keyword, timeout: slowTimeout, onDone: doneIds.add);
_merge(results, r2, seen);
yield SearchPhase(
name: 'partial',
items: List.of(results),
doneCount: doneIds.length,
totalCount: usable.length,
);
// ---- 阶段 3:慢源后台继续,不阻塞 ----
if (slow.isNotEmpty) {
// 放入后台队列,不 await
_batch(slow, keyword, timeout: slowTimeout, onDone: doneIds.add)
.then((r3) {
_merge(results, r3, seen);
// TODO: EventBus 通知 UI 追加
});
yield SearchPhase(
name: 'complete_with_pending',
items: List.of(results),
doneCount: doneIds.length,
totalCount: usable.length,
message: '${slow.length} 个慢源将在后台继续搜索',
);
} else {
yield SearchPhase(
name: 'complete',
items: List.of(results),
doneCount: doneIds.length,
totalCount: usable.length,
);
}
}
// ---- 内部分批并发 ----
Future<List<SearchItem>> _batch(
List<BookSource> sources,
String keyword, {
required Duration timeout,
required void Function(String) onDone,
}) async {
final out = <SearchItem>[];
for (var i = 0; i < sources.length; i += maxConcurrent) {
final chunk = sources.skip(i).take(maxConcurrent).toList();
final futures = chunk.map((s) => _searchOne(s, keyword, timeout));
final results = await Future.wait(futures);
for (var j = 0; j < results.length; j++) {
if (results[j] != null) out.addAll(results[j]!);
onDone(chunk[j].id);
}
}
return out;
}
Future<List<SearchItem>?> _searchOne(
BookSource source, String keyword, Duration timeout,
) async {
final sw = Stopwatch()..start();
try {
final items = await _engine.search(source, keyword).timeout(timeout);
sw.stop();
_health.recordSuccess(source.id, sw.elapsedMilliseconds);
return items;
} on TimeoutException {
_health.recordTimeout(source.id);
return null;
} catch (_) {
_health.recordFailure(source.id);
return null;
}
}
void _merge(List<SearchItem> all, List<SearchItem> incoming, Set<String> seen) {
for (final item in incoming) {
if (seen.add(item.id)) all.add(item);
}
}
}
// ---- 阶段结果 ----
class SearchPhase {
final String name; // searching | partial | complete | complete_with_pending | error
final List<SearchItem> items;
final int doneCount;
final int totalCount;
final String? message;
SearchPhase({
required this.name,
this.items = const [],
this.doneCount = 0,
this.totalCount = 0,
this.message,
});
}
```
### 2.5 UI 增量渲染时序
```
t=0s → 显示 "正在搜索 N 个源..."
t=0~5s → 第 1 批结果到达 → 首屏渲染(最快的 5~10 条)
t=5~8s → 第 2 批结果到达 → 列表追加
t=8s → 停止等待 → 提示"慢源已跳过,后台补充中"
t=8~15s → 第 3 批结果到达 → 静默追加到列表末尾
```
### 2.6 健康分自动衰减
每次 `save()` 时做一次衰减:
```dart
// SourceHealthStore.save() 追加衰减逻辑
Future<void> save() async {
// 超过 7 天无任何记录 → 衰减 successCount 和 failCount 各减半
for (final h in _data.values) {
final latest = h.lastSuccessAt ?? h.lastFailAt;
if (latest != null &&
DateTime.now().difference(latest).inDays >= 7) {
h.successCount = (h.successCount * 0.5).round();
h.failCount = (h.failCount * 0.5).round();
h.timeoutCount = (h.timeoutCount * 0.5).round();
}
}
// ... 写入 SharedPreferences
}
```
---
## 三、两改进的协作关系
```
用户搜索一本书
│
├─ SearchScheduler.search(keyword)
│ ├─ 按健康分排序源
│ ├─ 快速源先搜(5s 超时)
│ ├─ 普通源补搜(8s 超时)
│ └─ 返回结果列表
│
├─ 用户点击某条结果 → fetchBookInfo → fetchToc
│ └─ TocPaginationResolver 拉全章节(支持分页规则)
│
└─ 用户阅读某章 → fetchChapterContent(按需加载)
```
改进 1 保证"搜到书后章节不断尾",改进 2 保证"搜索本身不卡"。
---
## 四、实施顺序
```
第 1 步:改进 2 的 SourceHealth + SourceHealthStore
→ 只加数据模型 + 持久化,不影响现有搜索流程
→ 可以先跑几天积累健康数据
第 2 步:改进 2 的 SearchScheduler + 短超时
→ 替换现有 search() 的并发逻辑
→ UI 层适配 Stream<SearchPhase>
第 3 步:改进 1 的 TocPaginationResolver
→ 依赖 RuleParser 链式语法的基础支持
→ 见《Legado 规则引擎重构方案》第 1~3 步
第 4 步:联调 + 回归
```
---
## 五、风险控制
| 风险 | 应对 |
| -------------------- | --------------------------------------------- |
| TOC 分页死循环 | `maxPages=20` + `seenUrls` 去重 + `added==0` 终止 |
| 健康分误杀快源 | 只降权不禁用;连续失败 ≥5 才标记不可用 + 手动可重置 |
| 短超时漏结果 | 慢源后台继续,event bus 异步追加,不丢数据 |
| 新源全部 50 分 | 第一轮搜索后就有真实分,冷启动无影响 |
| SharedPreferences 过大 | 只存 String list,每个源约 200 字节,100 个源 ≈ 20KB |
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+410
View File
@@ -0,0 +1,410 @@
# 代码审计报告
> 审计日期:2026-06-16
> 审计范围:全项目(Flutter 移动端 30 个文件 + Python 桌面端 16 个文件 + Cloud 后端 10 个文件 + 5 个插件)
> 发现总数:111 项(Critical 9 / High 20 / Medium 46 / Low 36)
---
## 目录
1. [Critical(紧急修复)](#一critical紧急修复)
2. [High(高危)](#二high高危)
3. [Medium(中危)](#三medium中危)
4. [Low(低危)](#四low低危)
5. [跨模块问题汇总](#五跨模块问题汇总)
6. [修复优先级建议](#六修复优先级建议)
---
## 一、Critical(紧急修复)
### C1. Cloud 后端:JWT 密钥使用已知默认值
- **文件:** `sakura_cloud/core/config.py:29`
- **代码:** `jwt_secret_key: str = Field(default="change-this-secret-in-production", min_length=16)`
- **问题:** 默认 JWT 密钥是公开的占位字符串。如果运维人员忘记设置 `JWT_SECRET_KEY` 环境变量,任何知道此默认值的人都可以伪造任意用户的 JWT token,获得完全未授权访问。
- **修复:** 生产环境启动时检测是否为默认值,若是则拒绝启动并输出明确错误信息。不使用带默认值的 `Field`,改用 `Field(default=None)` 并在启动时校验。
### C2. 笔趣阁插件:硬编码 AES 密钥和 IV
- **文件:** `plugins/biquge/plugin.py:60-67`
- **代码:**
```python
TOKEN_SECRET = "book@token.html"
TOKEN_MD5 = hashlib.md5(TOKEN_SECRET.encode("utf-8")).hexdigest()
TOKEN_IV = TOKEN_MD5[:16].encode("utf-8")
TOKEN_KEY = TOKEN_MD5[16:].encode("utf-8")
```
- **问题:** AES 密钥和 IV 从硬编码字符串经 MD5 派生,任何人看到源代码即可解密和伪造 API token。固定 IV 下的 CBC 模式使相同明文产生相同密文,可被已知明文攻击。这不是加密,是混淆。
- **修复:** 每次加密使用随机 IV(`os.urandom(16)`),密钥从用户配置或安全随机源派生。
### C3. 小红书插件:`--no-sandbox` 禁用 Chromium 安全沙箱
- **文件:** `plugins/xiaohongshu/plugin.py:361`
- **代码:** `"args": ["--disable-blink-features=AutomationControlled", "--no-sandbox"]`
- **问题:** `--no-sandbox` 完全禁用 Chromium 沙箱,浏览器进程以完整 OS 权限运行。小红书的页面是不受信任的外部内容——如果渲染引擎存在漏洞被利用,攻击者获得主机系统的无限制访问权。
- **修复:** 移除 `--no-sandbox`。如果必须在 Docker/root 环境运行,使用 `--cap-add=SYS_ADMIN` 或配置 seccomp profile,而非全局禁用沙箱。
### C4. Cloud 后端:数据库默认凭据 `sakura:sakura`
- **文件:** `sakura_cloud/core/config.py:17`
- **代码:** `database_url: str = "postgresql+psycopg://sakura:sakura@localhost:5432/sakura_cloud"`
- **问题:** 默认数据库用户名和密码都是 `sakura`。如果部署时未覆盖 `DATABASE_URL`,数据库对任何能访问 `localhost:5432` 的人开放。
- **修复:** 同 C1——生产环境启动时检测默认值并拒绝启动。
### C5. Cloud 后端:MinIO 默认凭据 `minioadmin:minioadmin`
- **文件:** `sakura_cloud/core/config.py:24-25`
- **代码:** `minio_access_key: str = "minioadmin"` / `minio_secret_key: str = "minioadmin"`
- **问题:** 这是 MinIO 官方默认凭据。如果 MinIO 端点暴露在网络上,任何人可以访问所有用户存储的文件。
- **修复:** 同上——启动时检测是否为默认值,若是则拒绝启动。
### C6. 桌面端:云同步 API 使用明文 HTTP
- **文件:** `pixiv_app/gui/app.py:54`
- **代码:** `DEFAULT_CLOUD_API = "http://api.kunkunxiaomao.top:8000"`
- **问题:** 使用 HTTP 而非 HTTPS。用户凭据(邮箱、密码、access token)和上传文件内容以明文在网络上传输。同一网络上的攻击者可通过被动嗅探截获所有认证凭据和文件数据。
- **修复:** 改为 `http://api.kunkunxiaomao.top`(如果服务器支持 HTTPS)。最差情况下至少做出显式警告。
### C7. Flutter:`_initialized` 标志在 Firebase 初始化成功前设置
- **文件:** `sakura_mobile/lib/src/core/push_notifications.dart:27-29`
- **代码:** `_initialized = true` 在 `await Firebase.initializeApp()` **之前**设置
- **问题:** 如果 `Firebase.initializeApp()` 抛出异常,`_initialized` 已经是 `true`,导致后续 `initialize()` 调用全部命中 `if (_initialized) return` 的 early return,Firebase 永远无法重试初始化。推送功能永久失效,无任何提示。
- **修复:** 将 `_initialized = true` 移到 `await Firebase.initializeApp()` 成功之后。
### C8. Flutter:`recentUpdatesOnly` 是死开关(dead UI switch)
- **文件:** `sakura_mobile/lib/src/pages/tabs/gallery_tab.dart:32,120`
- **问题:** `recentUpdatesOnly` 状态变量控制哪个 Chip 显示"已选中",但 `_load()` 方法从未将其作为查询参数传给 API。用户看到的"最近更新"筛选切换实际上没有任何效果。
- **修复:** 在 `_load()` 中将 `recentUpdatesOnly` 转换为对应的 API 参数,或者移除该 Chip。
### C9. 桌面端:ProxyPool 的多线程数据竞争
- **文件:** `pixiv_app/core/proxy_pool.py:193,272-273,278-279`
- **问题:**
1. `ProxyPool.current_index` 的读取(line 272)和递增(line 273)之间无锁保护,`ThreadPoolExecutor` 的多个 worker 同时调用 `get_proxy()` 时产生重复或跳跃的索引
2. `ProxyPool.working_proxies` 被 `mark_failed()` 的 `.remove()` 修改,而其他线程可能正在迭代它——可抛出 `ValueError` 或不一致迭代
3. `ProxyInfo.fail_count` 和 `success_count` 的非原子递增在 `ThreadPoolExecutor` 16 个 worker 并行调用 `verify_proxy()` 时静默丢失计数
- **修复:** 使用 `threading.Lock` 保护所有对 `working_proxies`、`current_index` 的写操作,`fail_count`/`success_count` 改用 `threading` 安全计数或原子操作。
---
## 二、High(高危)
### H1. Cloud 后端:用户登录无限流 → 可暴力破解
- **文件:** `sakura_cloud/api/v1/endpoints/auth.py:56-76`
- **问题:** `/login` 端点无限流(`admin_login_rate_limit_per_minute` 只用于 admin 端点)。攻击者可无限制尝试密码组合进行暴力破解。
- **修复:** 对 `/login`、`/register`、`/token/refresh` 增加速率限制(如每 IP 每分钟 5 次)。
### H2. Cloud 后端:用户注册无限流 + 信息泄露
- **文件:** `sakura_cloud/api/v1/endpoints/auth.py:24-53`
- **问题:** `/register` 端点无限流。409 响应明确区分"用户名已存在"和"邮箱已存在",可用于枚举已注册用户。
- **修复:** 加速率限制。错误消息统一为"注册失败,请检查输入"。
### H3. Cloud 后端:存储配额未强制执行
- **文件:** `sakura_cloud/api/v1/endpoints/gallery.py:223`、`sync.py:86-141`
- **问题:** `default_storage_limit_bytes` 在 config 中定义了(line 108-109),但在创建 artwork 和同步 metadata 时**从未检查** `current_user.storage_used` 是否超过配额。用户可以上传无限制的数据。
- **修复:** 在 `gallery.py` 和 `sync.py` 的写入路径中增加 `if current_user.storage_used + new_size > limit: raise HTTPException(413)`。
### H4. Cloud 后端:Logout 不撤销 token
- **文件:** `sakura_cloud/api/v1/endpoints/auth.py:79-81`
- **问题:** `/logout` 只返回 `{"ok": True}`,不做任何 token 撤销操作。access token 在其自然过期(最长 24 小时)前持续有效。用户登出后无法阻止 token 被盗用。
- **修复:** Logout 时递增 `user.token_version`,使当前所有 token 立即失效。
### H5. Cloud 后端:Refresh token 轮转不撤销旧 token
- **文件:** `sakura_cloud/api/v1/endpoints/auth.py:84-104`
- **问题:** 交换 refresh token 时签发新的 access + refresh token,但旧的 refresh token 未失效。如果 refresh token 被盗,攻击者和合法用户都可以持续刷新——直到某人重新登录(才会 bump `token_version`)。
- **修复:** 实现 refresh token 轮转——每次使用时废弃旧 token 并签发新的。
### H6. 桌面端 & Flutter:敏感凭据多处明文存储
|| 位置 | 凭据类型 | 存储方式 |
||------|---------|---------|
|| `pixiv_app/core/auth.py:36-41` | Pixiv cookie | JSON 文件明文 |
|| `plugins/pixiv/plugin.py:42-43` | Pixiv cookie | 内存明文 |
|| `plugins/x/plugin.py:76` | X/Twitter cookies | `x_cookies.json` 明文 |
|| `plugins/ao3/plugin.py:687-688` | AO3 cookie | 内存明文 |
|| `plugins/xiaohongshu/plugin.py:386-404` | 小红书 cookies | `xiaohongshu_cookies.json` 明文 |
|| `pixiv_app/core/proxy_pool.py:67-68` | Quake API key | `app_session_file()` JSON 明文 |
|| `sakura_mobile/lib/src/core/session.dart:44-47` | Access token | SharedPreferences 明文 |
- **修复:** 生产环境使用操作系统密钥链(Windows DPAPI / macOS Keychain / Android Keystore)。最低限度应对配置文件设置受限的文件权限。
### H7. Flutter:小说阅读页面翻页不节流 → API 洪泛
- **文件:** `sakura_mobile/lib/src/pages/novel_reader_page.dart:787-790`
- **问题:** `onPageChanged` 对**每次翻页**同步调用 `updateNovelProgress(currentNovel.id, page)`。快速翻页时数十上百个 API 请求瞬间发出,压垮服务器并浪费电量/带宽。
- **修复:** 增加 500ms 防抖,只发送最终位置。
### H8. Flutter:探索页从"插画"tab 点搜索 → 打开"小说"模式
- **文件:** `sakura_mobile/lib/src/pages/tabs/gallery_tab.dart:398-399`
- **问题:** 用户在"图片"(illust)tab 点击在线搜索时,代码 `widget.fixedContentType != 'manga'` 为真,传入 `UnifiedSearchMode.novel`。从图片页搜索却打开了小说搜索模式。
- **修复:** 将 explore mode 与 gallery content type 解耦。默认为 manga 模式(漫画源更多)。或增加一个独立的选择器。
### H9. Flutter:Session 构造器中异步工作不可 await
- **文件:** `sakura_mobile/lib/src/core/session.dart:44-47`
- **问题:** `SessionController()` 构造器中调用异步 `_load()`,但构造器不能 `async`。Provider 观察者先看到默认 loading 态再被 `_load()` 完成后的 `notifyListeners()` 更新。Provider 初始化异常无法被外部捕获。
- **修复:** 使用 Riverpod 的 `FutureProvider` 或 `AsyncNotifier`,或将初始化方法显式暴露给 widget tree 调用。
### H10. Flutter:`_validateStoredSession` 静默吞所有异常
- **文件:** `sakura_mobile/lib/src/core/session.dart:159`
- **问题:** `catch (_)` 捕获所有异常返回 `null`,不记录日志。网络故障或服务器宕机时用户被静默登出,无任何提示。
- **修复:** 最低限度 `debugPrint(e)`。更好:区分网络错误(可恢复)和认证错误(需重新登录),并显示对应 UI。
### H11. Cloud 后端:JWT 缺少 `aud`/`iss` 声明验证
- **文件:** `sakura_cloud/core/security.py:23-32,53-62`
- **问题:** Token 创建时不设置 `aud`(受众)和 `iss`(签发者),`decode_token` 也不验证。如果同一 JWT 密钥被意外共享,一个服务的 token 可在另一个服务中复用。
- **修复:** 创建 token 时设置 `aud="sakura-cloud"`,解码时验证。
### H12. 小红书插件:代理凭据暴露在 URL 字符串中
- **文件:** `plugins/xiaohongshu/plugin.py:107-110`
- **问题:** `ProxyConfig.url` 属性将明文用户名密码嵌入 URL:`"https://user:pass@host:port"`。如果这个 URL 字符串被记录到 Playwright 调试日志、异常堆栈或错误消息中,凭据泄露。
- **修复:** 通过 Playwright 的 `proxy` context 选项分别传入 `username`/`password`(lines 368-371 已部分正确),`.url` 属性不包含凭据。
### H13. Flutter:默认 API URL 用 HTTP 明文
- **文件:** `sakura_mobile/lib/src/core/api_client.dart:9`
- **代码:** `const defaultCloudApi = 'http://api.kunkunxiaomao.top:8000'`
- **问题:** 与桌面端 C6 相同——明文 HTTP 传输所有凭据和内容。
- **修复:** 改为 HTTPS。
### H14. Flutter:Push notification token refresh 监听器持有过期凭据
- **文件:** `sakura_mobile/lib/src/core/push_notifications.dart:66-83`
- **问题:** `onTokenRefresh.listen` 捕获注册时的 `baseUrl` 和 `accessToken`。如果用户后续登录其他服务器或换号,监听器仍用旧凭据,导致 push 注册静默失败(line 79 catch 吞错)。
- **修复:** 每次 token 刷新时从当前 provider 读取最新凭据。
### H15. 桌面端:SQLite 跨线程共享缺少显式锁
- **文件:** `pixiv_app/core/library.py:29`
- **代码:** `check_same_thread=False` 允许跨线程共享连接,"当有锁保护时"——但锁的保护由调用者负责且不一致。
- **问题:** `TaskQueue`(`enqueue.py:232`)将连接传给多个 `ThreadPoolExecutor` worker,没有外部锁。WAL 模式下 SQLite 通常能处理,但高并发写入时仍可能 `SQLITE_BUSY`。
- **修复:** 改用连接池(如 `sqlite3` 的 URI 模式 `file:path?mode=rw`),或每个 worker 使用独立连接。
### H16. 桌面端:画廊同步对每个文件行新开 DB 连接(O(N) 连接泛滥)
- **文件:** `pixiv_app/gui/app.py:2175-2184`
- **问题:** `_sync_pixiv_rows` 中**每个文件行**都调用 `sqlite3.connect(DEFAULT_LIBRARY_DB)` → 查询 → `close()`。100 个文件同步 = 100 次 DB 打开/关闭循环。性能低下且在高负载下可能触及 OS 文件描述符上限。
- **修复:** 复用单个连接或将查询批量化。
### H17. Cloud 后端:用户登录无限流(跨所有端点)
- **问题范围:** `/login`(auth.py:56)、`/register`(auth.py:24)、`/token/refresh`(auth.py:84)、所有 gallery 端点、所有 novel 端点、所有 sync 端点——除 admin 外**无一有限流**。
- **修复:** 全局应用 rate limiting 中间件(如 `slowapi` 或自定义中间件)。
### H18. Cloud 后端:`--no-sandbox` 同样存在于 X 插件
- **文件:** `plugins/x/plugin.py`(Playwright 启动参数)
- **问题:** 与 C3 相同风险。X/Twitter 页面同样是不受信任的外部内容。
- **修复:** 移除或至少评估必要性。
### H19. 桌面端:N+1 查询模式在关注作者同步中
- **文件:** `pixiv_app/core/gallery_service.py:547-561`
- **问题:** `sync_followed_artists` 对每个关注的作者创建 → 查询 → 关闭独立的 `PixivLibrary` 实例。50 个作者 = 100+ 次 DB 打开/关闭。
- **修复:** 使用池化或缓存的连接。
### H20. Flutter:多个页面存在静默吞错 → 空列表
|| 文件 | 位置 | 模式 |
||------|------|------|
|| `novels_tab.dart` | 63-67 | `.catchError((_) => <dynamic>[])` |
|| `api_client.dart` | 532 | `_readCache` catch 返回 null |
|| `providers.dart` | 17-21 | `initializeIfFirstRun()` 的 future 被丢弃 |
- **影响:** API 失败时用户看到空书架/空画廊/空收藏,无法区分"真的没数据"和"加载失败了"。
- **修复:** 区分 error 状态并在 UI 中显示错误提示和重试按钮。
---
## 三、Medium(中危)
### 后端
| # | 文件 | 行号 | 问题 |
|---|------|------|------|
| M1 | `sakura_cloud/main.py` | 130-131 | asyncio background task 无引用监控,异常静默死亡 |
| M2 | `sakura_cloud/api/middleware.py` | 77 | `img-src https: http:` 完全取消 CSP 图片保护 |
| M3 | `sakura_cloud/api/middleware.py` | 79 | `style-src 'unsafe-inline'` 可被 XSS 利用 |
| M4 | `sakura_cloud/core/config.py` | 31 | access token 有效期 24h 过长,token 被盗后窗口大 |
| M5 | `sakura_cloud/core/config.py` | 56-58 | FCM 默认启用但无有效凭据,运行时报错 |
| M6 | `sakura_cloud/core/config.py` | 18 | Redis 默认 URL 无密码认证 |
| M7 | `sakura_cloud/api/v1/endpoints/gallery.py` | 324-332 | 删除 artwork 时文件不清理 → 磁盘孤儿文件累积 |
| M8 | `sakura_cloud/api/v1/endpoints/novels.py` | 53-73 | GET 请求触发 DB 写操作(`_resolve_novel_storage_path`),违反 REST 幂等性 |
| M9 | `sakura_cloud/api/v1/endpoints/files.py` | 39 | 缩略图生成同步阻塞 FastAPI 事件循环 |
| M10 | `sakura_cloud/api/v1/endpoints/sync.py` | 38,100,138 | metadata dict 无大小限制 → 数据库 bloat 攻击 |
| M11 | `sakura_cloud/api/v1/endpoints/sync.py` | 37,99,138 | tags 列表无长度/元素限制 → 数据库 bloat 攻击 |
| M12 | `sakura_cloud/api/v1/endpoints/auth.py` | 9 | 密码复杂度仅 `min_length=8`,弱密码可过 |
| M13 | `pixiv_app/core/downloader.py` | 46-48 | 文件名未做长度限制,Windows MAX_PATH 260 字符溢出导致 OSError |
| M14 | `pixiv_app/core/downloader.py` | 476,502,537 | 大量 `except Exception: continue` 静默丢错,生产调试不可能 |
| M15 | `pixiv_app/core/downloader.py` | 690,901-945 | 每个作品创建新 `requests.Session()`(无 keep-alive),100 作品 = 100 次 TLS 握手 |
| M16 | `pixiv_app/core/gallery_service.py` | 399-414 | `_create_sync_manager` 在 3 个文件中重复实现 |
| M17 | `pixiv_app/core/origin_follow_sync.py` | 103 | `asyncio.run()` 在可能已有 event loop 的同步上下文中调用 |
| M18 | `pixiv_app/core/proxy_pool.py` | 213 | 代理测试时 `verify=False`,代理自身可 MITM 注入虚假成功结果 |
| M19 | `pixiv_app/core/auth.py` | 36-41 | `SessionStore.load()` 用裸 `except Exception` 吞所有异常(JSON 损坏、权限错误等) |
| M20 | `pixiv_app/core/gallery_service.py` | 79-81 | tag 筛选用 `JOIN`(非 `LEFT JOIN`),孤儿 `artwork_tags` 行导致作品静默遗漏 |
| M21 | `pixiv_app/core/downloader.py` | 698 | `int(detail.get("pageCount") or 0)` 遇非数字字符串抛 `ValueError` |
### 插件
| # | 文件 | 行号 | 问题 |
|---|------|------|------|
| M22 | `plugins/pixiv/plugin.py` | 243-244 | 文件名从 URL 提取,URL 编码的 `%2F` 可绕过 `rsplit` 做路径穿越 |
| M23 | `plugins/x/plugin.py` | 217-218,386 | 每次操作启动新 Playwright browser context,无复用 → 资源耗尽 |
| M24 | `plugins/biquge/plugin.py` | 845-867 | Playwright page 实例缓存跨操作复用,一次失败污染后续 |
| M25 | `plugins/ao3/plugin.py` | 503-506 | EPUB 解压无 zip bomb 防护,压缩炸弹可 OOM |
| M26 | `plugins/ao3/plugin.py` | 92-96 | `Retry-After` 头无上限,`Retry-After: 999999999` 可使插件休眠 ~31 年 |
| M27 | `plugins/xiaohongshu/plugin.py` | 377 | `headless=False` 在服务器/无 GUI 环境运行失败 |
| M28 | `plugins/xiaohongshu/plugin.py` | 480-588 | `page.evaluate()` 执行不受信任页面的 JS,结果未经域名验证 |
| M29 | `plugins/xiaohongshu/plugin.py` | 630-648 | 下载图片时转发页面所有 cookies 给图片 CDN(含第三方追踪 cookie) |
### Flutter
| # | 文件 | 行号 | 问题 |
|---|------|------|------|
| M30 | `api_client.dart` | 440-502 | `translateText()` 每次调用创建并销毁新的 Dio 实例 |
| M31 | `api_client.dart` | 518-535 | `_readCache<T>` 泛型类型不安全——声称返回任何 T 但 SharedPreferences 只存 JSON 基本类型 |
| M32 | `providers.dart` | 17-21 | `sourceManager.initializeIfFirstRun()` 是 async 但返回值被丢弃 |
| M33 | `app.dart` | 24 | 硬编码 Windows 字体 `Microsoft YaHei UI`,iOS/macOS/Linux 回退不一致 |
| M34 | `gallery_tab.dart` | 269-278 | 收藏/取消收藏触发全量列表重载(`_refresh()`),非单项更新 |
| M35 | `gallery_tab.dart` | 336-367 | 批量下载用 `for` + 顺序 `await`,无并发无进度 |
| M36 | `login_page.dart` | 141 | TabBarView 高度硬编码 430px,不同屏幕尺寸下可能裁剪或留白 |
| M37 | `novel_reader_page.dart` | 17-32 | 14 个可变实例字段——翻页设置(字号/行高/主题)在每次页面跳转时丢失 |
| M38 | `novel_detail_page.dart` | 67 | 硬编码 `430`px 滚动偏移到目录区,不随内容高度自适应 |
| M39 | `reminder_center_page.dart` | 22-35 | `results[0]! as Map` 等不安全类型强转,API 返回异常形状时抛 `TypeError` |
| M40 | `profile_tab.dart` | 403 | `logout()` 是 async 但 button onPressed 丢弃返回的 Future |
| M41 | `profile_tab.dart` + `reminder_center_page.dart` | 44-97 | 提醒时间设置逻辑完全重复两份 |
| M42 | `favorites_page.dart` | 51-54 | 每次点击 artwork 都重新执行 O(n) 的 `buildMangaCollections` |
| M43 | `novels_tab.dart` | 461 | epoch 0 作为时间戳兜底 → 显示"53 年前",无意义 |
| M44 | `manga_collections.dart` | 59 | 不同 null-author 作者的同名作品被错误合并为同一合集 |
### Cloud 后端(补充)
| # | 文件 | 行号 | 问题 |
|---|------|------|------|
| M45 | `sakura_cloud/main.py` | 123-126 | 启动时 digest 发送失败仅打日志,无告警 |
| M46 | `sakura_cloud/core/security.py` | 54-59 | `decode_token` 冗余手动检查 `exp`,且未设 clock skew 容差 |
---
## 四、Low(低危)
### 后端
| # | 文件 | 行号 | 问题 |
|---|------|------|------|
| L1 | `sakura_cloud/main.py` | 113-131 | startup 事件阻塞事件循环,无就绪探针/超时 |
| L2 | `sakura_cloud/api/middleware.py` | 32-42 | debug 模式 CSP `connect-src` 开放 `localhost:*`(所有本地端口) |
| L3 | `sakura_cloud/api/deps.py` | 29 | `str(user_id)` 对 None 产生字面字符串 `"None"` |
| L4 | `sakura_cloud/api/deps.py` | 34 | `token_version` 比较中,双方都为 0/None 时通过,旧 token 缺少 `tv` 声明不被拒绝 |
| L5 | `sakura_cloud/core/config.py` | 89-103 | `json.loads()` 在 field validator 中内联 import,不标准 |
| L6 | `sakura_cloud/api/v1/endpoints/auth.py` | 79-81 | logout 不递增 `token_version` |
| L7 | `sakura_cloud/api/v1/endpoints/gallery.py` | 51-97 | `ILIKE` + `plainto_tsquery` 用户输入无长度限制 → 可能触发慢查询 |
| L8 | `sakura_cloud/api/v1/endpoints/files.py` | 32 | `original_name` 用户可控 → 下载附件名可用于社工 |
| L9 | `sakura_cloud/api/v1/endpoints/sync.py` | 25 | SHA256 字段只验证长度不验证 hex 字符 |
| L10 | `sakura_cloud/api/v1/endpoints/sync.py` | 144-157 | 设备注册无每用户数量上限 |
| L11 | `pixiv_app/core/downloader.py` | 40 | `PixivRequestError` 继承 `RuntimeError` 而非 `Exception`(语义不当) |
| L12 | `pixiv_app/core/downloader.py` | 233 | `str(user_id).isdigit()` 中 `user_id` 可能为 None → `str(None)` = `"None"` |
| L13 | `pixiv_app/core/downloader.py` | 700,709,771 | 同一 artwork 三次调用 `ensure_artwork_stub`(`INSERT OR IGNORE`,无害但浪费) |
| L14 | `pixiv_app/core/gallery_service.py` | 588-613 | 同一 artwork 多次同步中重复创建 alert |
| L15 | `pixiv_app/core/x_media_service.py` | 522-523 | `datetime.now()` 无时区 → 跨时区对比可能偏差一天 |
| L16 | `pixiv_app/gui/app.py` | 1153-1157 | 小红书页缩进不一致(7 空格,PEP 8 违规) |
| L17 | `sakura_cloud/core/security.py` | 54-59 | `decode_token` 的 `exp` 手动检查与 `jwt.decode` 内建检查重复 |
### 插件
| # | 文件 | 行号 | 问题 |
|---|------|------|------|
| L18 | `plugins/x/plugin.py` | 414 | 下载缓存 JSON 暴露本地绝对路径结构 |
| L19 | `plugins/x/plugin.py` | 463-475 | 文件类型仅凭 URL 子串匹配判断,无 MIME 验证 |
| L20 | `plugins/biquge/plugin.py` | 135,360-361 | `requests.Session()` 无连接池限制 |
| L21 | `plugins/xiaohongshu/plugin.py` | 375-376 | 浏览器 profile 跨会话共享,可能积累旧 cookies 泄漏 |
### Flutter
| # | 文件 | 行号 | 问题 |
|---|------|------|------|
| L22 | `app.dart` | 11-16 | `ColorScheme.fromSeed()` 在 build 中创建(实际仅一次,低影响) |
| L23 | `artwork_detail_page.dart` | 264-265 | `showDialog` 异常抛出时 TextEditingController 泄漏 |
| L24 | `online_manga_reader_page.dart` | 101-103 | 源 headers 直接传给 `Image.network`,第三方源可能捕获凭据 |
| L25 | `novel_reader_page.dart` | 78,330 | `Future.microtask` 在 widget 可能已 dispose 的上下文中执行 |
| L26 | `novels_tab.dart` | 446,461 | 时间戳兜底使用 epoch 0 → 显示"53 年前" |
| L27 | `explore_page.dart` | 186-222 | `_openItem` 从 provider 同步读取 source,可能搜完源已被删除 |
| L28 | `source_manager.dart` | 86-96 | `initializeIfFirstRun()` 的 initialized flag 在可能失败的 `load()` 之后无条件设为 true |
| L29 | `source_manager.dart` | 242-260 | 从 JSON 导入时,源的 id 由 name+URL 的 codeUnit 求和生成——不同源可能碰撞 |
| L30 | `legado_engine.dart` | 167-200 | `_encodeKeyword` 对非 UTF-8/GBK 编码的平台通道调用无超时 |
---
## 五、跨模块问题汇总
| 问题类别 | 涉及模块 | 严重程度 |
|---------|---------|:--------:|
| **凭据明文存储** | 桌面端 + 5 个插件 + Flutter | Critical/High(7 处) |
| **HTTP 明文传输** | 桌面端 API + Flutter API | Critical/High(2 处) |
| **默认密码/密钥** | Cloud config(JWT、DB、MinIO、Redis) | Critical/High(4 处) |
| **无限流** | Cloud 全部用户端点 | High(17+ 端点) |
| **存储配额未执行** | Cloud gallery + sync | High(2 处) |
| **静默吞错 → 空列表** | Flutter + 桌面 downloader + session | High/Medium(10+ 处) |
| **线程安全** | 桌面 ProxyPool + SQLite + counter | Critical/Medium(3 处) |
| **资源泄漏** | 桌面 DB 连接 + Flutter Dio × translate + X 插件 browser | High/Medium(3 处) |
| **Token 生命周期** | Cloud auth(logout、refresh、expiry、version) | High/Medium(5 处) |
| **插件安全** | `--no-sandbox` + 硬编码密钥 + zip bomb + URL 路径穿越 | Critical/Medium(5 处) |
| **Flutter 状态管理** | FutureBuilder 模式 + async 构造器 + provider 初始化 | High/Medium(5 处) |
---
## 六、修复优先级建议
### 第一轮(立即修复,1–3 天)
| 顺序 | ID | 问题 | 工时 |
|:----:|:---|------|:----:|
| 1 | C1,C4,C5 | Cloud 默认凭据:JWT 密钥 + DB 密码 + MinIO 凭据 | 1h |
| 2 | C3,H18 | 移除 `--no-sandbox`(小红书 + X 插件) | 1h |
| 3 | C6,H13 | HTTP → HTTPS(桌面端 + Flutter) | 2h |
| 4 | C9 | ProxyPool 多线程数据竞争(加锁) | 2h |
| 5 | C7 | Firebase 初始化 flag 移到成功之后 | 10m |
| 6 | C2 | 笔趣阁 AES 密钥改为随机 IV + 安全随机密钥 | 2h |
| 7 | H1,H2,H17 | Cloud 端全局速率限制 | 4h |
| 8 | H3 | 存储配额强制执行 | 2h |
### 第二轮(本周内,3–5 天)
| 顺序 | ID | 问题 | 工时 |
|:----:|:---|------|:----:|
| 9 | H4,H5 | Token 撤销(logout + refresh rotation) | 4h |
| 10 | H16 | 云同步 O(N) DB 连接改为复用连接 | 2h |
| 11 | H15 | SQLite 跨线程连接池化 | 3h |
| 12 | H7 | 小说翻页 API 增加 500ms 防抖 | 1h |
| 13 | H8 | 探索页"插画→小说"模式修复 | 1h |
| 14 | C8 | gallery_tab `recentUpdatesOnly` 死开关修复 | 1h |
| 15 | H19 | 关注作者同步 N+1 查询优化 | 3h |
| 16 | H20 | 多处静默吞错 → 显式错误状态 UI | 4h |
| 17 | M8 | GET novel 路径解析触发的 DB 写操作改为 lazily 计算 | 2h |
### 第三轮(下一迭代)
| 顺序 | 类别 | 内容 | 工时 |
|:----:|------|------|:----:|
| 18 | 凭据加密 | 7 处明文凭据改为 OS 密钥链存储 | 8h |
| 19 | CSP | CSP 策略加固(nonce/hash 替代 unsafe-inline) | 4h |
| 20 | 测试 | 为 Critical/High 项编写回归测试 | 8h |
| 21 | Flutter 重构 | AsyncNotifier 替代 FutureBuilder(feed_tab, gallery_tab, novels_tab) | 8h |
| 22 | 插件审计 | 对全部 5 个插件的网络/文件访问权做沙箱化评估 | 4h |
---
> **审核范围确认:** Flutter 移动端 30 个 Dart 文件 × Python 桌面端 16 个文件 × Cloud 后端 10 个端点文件 × 5 个插件 × 中间件/配置/安全模块。合计约 65 个源文件,发现 111 个问题。
+507
View File
@@ -0,0 +1,507 @@
# 樱云下载器 — 优化方案与功能规划
> 版本:2.1.0-draft | 日期:2026-06-24
> 基于代码审计报告、产品文档、CHANGELOG 及全仓库源码分析制定
---
## 目录
1. [优先级定义](#0-优先级定义)
2. [P0 — 安全与基础设施(必须修)](#1-p0--安全与基础设施)
3. [P1 — 核心体验缺陷(尽快修)](#2-p1--核心体验缺陷)
4. [P2 — 功能完善与增强](#3-p2--功能完善与增强)
5. [P3 — 体验优化与锦上添花](#4-p3--体验优化与锦上添花)
6. [P4 — 长期架构演进](#5-p4--长期架构演进)
7. [实施路线图](#6-实施路线图)
---
## 0. 优先级定义
| 等级 | 含义 | 建议时间 |
|------|------|---------|
| **P0** | 安全漏洞 / 数据风险 / 系统崩溃 | 1 周内 |
| **P1** | 核心功能阻断或严重缺陷 | 2–4 周 |
| **P2** | 功能缺口 / 重要增强 | 1–2 月 |
| **P3** | 体验优化 / 锦上添花 | 按需排期 |
| **P4** | 架构演进 / 平台扩展 | 中长期规划 |
---
## 1. P0 — 安全与基础设施
> 来源:代码审计报告 Critical 项 + 其他发现
### 1.1 JWT 密钥强制生产配置
- **现状**:`sakura_cloud/core/config.py` 中 `jwt_secret_key` 默认值为 `"change-this-secret-in-production"`,任何人都可伪造 JWT
- **方案**:启动时检测是否为已知默认值,若是则拒绝启动并输出明确的错误信息,引导运维设置 `JWT_SECRET_KEY` 环境变量
- **影响范围**:`sakura_cloud/core/config.py`、`sakura_cloud/core/security.py`
### 1.2 数据库凭据与 MinIO 凭据硬编码
- **现状**:
- PostgreSQL 默认凭据 `sakura:sakura`
- MinIO 默认凭据 `minioadmin:minioadmin`
- **方案**:同 JWT 策略——启动时检测默认值,若未被覆盖则拒绝启动;为 MinIO 增加随机密码生成脚本
- **影响范围**:`sakura_cloud/core/config.py`、部署文档
### 1.3 桌面端 → 云端启用 HTTPS
- **现状**:`DEFAULT_CLOUD_API = "http://api.kunkunxiaomao.top:8000"` 使用明文 HTTP,用户凭据和文件明文传输
- **方案**:
- 短期:支持 HTTP 但增加显式警告弹窗("当前使用非加密连接,凭据可能被窃取")
- 长期:服务器配置 SSL 证书(Let's Encrypt / 自签名),客户端默认改为 HTTPS,初次连接时做证书校验引导
- **影响范围**:`pixiv_app/gui/app.py`、`pixiv_app/core/cloud_sync.py`、云端 nginx/Caddy 配置
### 1.4 登录接口限流
- **现状**:`/api/v1/auth/login` 无限流,可暴力破解
- **方案**:对 `/login`、`/register`、`/token/refresh` 增加 IP 级别速率限制(如每分钟 5 次),使用 Redis 或内存计数器实现
- **影响范围**:`sakura_cloud/api/v1/endpoints/auth.py`、新增限流中间件
### 1.5 笔趣阁插件 AES 密钥随机化
- **现状**:`plugins/biquge/plugin.py` 中 AES 密钥和 IV 从硬编码字符串经 MD5 派生,固定 IV 下的 CBC 模式可被已知明文攻击
- **方案**:
- 每次加密使用随机 IV(`os.urandom(16)`),与密文一起传输
- 密钥从用户配置或安全随机源派生,而非硬编码
- **影响范围**:`plugins/biquge/plugin.py`
### 1.6 Chrome `--no-sandbox` 移除
- **现状**:小红书插件使用 `--no-sandbox` 禁用 Chromium 沙箱,浏览器以完整 OS 权限运行不受信任的外部内容
- **方案**:移除 `--no-sandbox`。若在 Docker/root 环境下必须使用,改为配置 `--cap-add=SYS_ADMIN` 或 seccomp profile
- **影响范围**:`plugins/xiaohongshu/plugin.py`
### 1.7 Firebase 初始化竞态修复
- **现状**:`push_notifications.dart` 中 `_initialized = true` 在 `await Firebase.initializeApp()` **之前**设置,若初始化失败则永久不再重试
- **方案**:将 `_initialized = true` 移到 `await Firebase.initializeApp()` 成功之后
- **影响范围**:`sakura_mobile/lib/src/core/push_notifications.dart`
### 1.8 ProxyPool 多线程数据竞争
- **现状**:`current_index` 无锁读写、`working_proxies` 并发修改、`fail_count`/`success_count` 非原子递增
- **方案**:引入 `threading.Lock` 保护所有对共享数据的写操作;或将 `current_index` 改为 `itertools.cycle` + 原子计数器
- **影响范围**:`pixiv_app/core/proxy_pool.py`
### 1.9 敏感文件 `.gitignore` 完善
- **现状**:`.env.cloud.example` 已提供模板,但不清楚 `.env` 是否在 `.gitignore` 中;会话文件 `app_session.json` 可能被误提交
- **方案**:在 `.gitignore` 中显式添加 `*.env`(除 `.example`)、`app_session.json`、`runtime/`、`*.db`、`*.key`
- **影响范围**:`.gitignore`
---
## 2. P1 — 核心体验缺陷
> 来源:代码审计报告 High/Medium 项 + 已知 bug + 功能缺口
### 2.1 手机端 `recentUpdatesOnly` 死开关修复
- **现状**:`gallery_tab.dart` 中 `recentUpdatesOnly` 状态变量只控制 Chip 高亮,从未作为 API 参数传入 `_load()`,切换无实际效果
- **方案**:在 `_load()` 中将 `recentUpdatesOnly` 转换为 `order=updated_at` 或后端新增 `recent_only` 查询参数;或如果后端暂时不支持,先移除该 Chip
- **影响范围**:`sakura_mobile/lib/src/pages/tabs/gallery_tab.dart`
### 2.2 Legado 书源搜索引擎完善
- **现状**:6 个默认种子源 `searchUrl` / `searchListRule` 全为 null,CSS 选择器解析不支持 class/ID/属性选择器,GBK 编码无容错
- **方案**(分两步):
- **2.2a 引擎层**:扩展 CSS 选择器支持 class(`.classname`)、ID(`#id`)、属性(`[attr=val]`)、后代(`A B`)选择器;GBK 解码增加容错参数
- **2.2b 源数据层**:用真实可用的书源 JSON 替换默认种子源,或提供从 Legado 仓库一键导入的引导流程
- **影响范围**:`legado_rule_parser.dart`、`legado_engine.dart`、`source_manager.dart`
### 2.3 手机端发现页滚动卡死修复
- **现状**:发现页 Feed 向上滚动时可能卡在底部
- **方案**:排查 `feed_tab.dart` 中的 `ScrollController` 和分页加载逻辑,确保加载更多后正确更新 `hasMore` 状态,以及 `ListView` 的 `addPostFrameCallback` 不产生死循环
- **影响范围**:`sakura_mobile/lib/src/pages/tabs/feed_tab.dart`
### 2.4 Web 管理后台页面完善
- **现状**:普通用户 Web 端详情页已从占位改为真实加载,但管理后台可能仍有死角
- **方案**:
- 完成管理员端用户管理、内容审核、统计面板页面
- 普通用户端确保图片/漫画/小说三种内容类型均有完整的列表+详情页
- **影响范围**:`sakura_admin_web/`
### 2.5 云同步上传失败队列与断点续传
- **现状**:`SAKURA_CLOUD_V1_STATUS.md` 明确标注"上传失败队列与断点恢复细节"尚未完成
- **方案**:
- 桌面端新增 `sync_failures` 表记录上传失败的文件,支持自动/手动重试
- 上传分片支持断点续传(保存已上传分片索引)
- 云端 `upload/complete` 接口增加分片校验
- **影响范围**:`pixiv_app/core/cloud_sync.py`、`sakura_cloud/api/v1/endpoints/upload.py`
### 2.6 Alembic 数据库迁移完善
- **现状**:SAKURA_CLOUD_V1_STATUS 标注 Alembic 迁移尚未完成
- **方案**:补全所有模型的 Alembic migration 脚本,确保 `alembic upgrade head` 一键建库;新增 CI 中检查迁移一致性
- **影响范围**:`sakura_cloud/migrations/`
### 2.7 手机端 App 正式签名
- **现状**:APK 使用调试签名,不适合正式分发;README 和用户手册均强调此限制
- **方案**:
- 生成正式 keystore
- 更新 `sakura_mobile/android/app/build.gradle.kts` 签名配置
- 文档更新,区分 debug/release 构建流程
- **影响范围**:`sakura_mobile/android/`、构建脚本
### 2.8 桌面端 Cookie 过期自动检测与提醒
- **现状**:Pixiv Cookie 过期后抓取静默失败,用户需手动排查
- **方案**:
- 在每次抓取前自动检测会话有效性(已有「检测会话」按钮逻辑,改为自动调用)
- Cookie 将过期或已过期时弹窗提醒并引导重新导入
- **影响范围**:`pixiv_app/core/auth.py`、`pixiv_app/gui/app.py`
### 2.9 下载失败自动重试与错误分类
- **现状**:`runtime/executor.py` 已有基本重试与错误分类,但未区分限流(429)、临时故障(5xx)、权限错误(403/404)
- **方案**:
- 429 → 自动退避重试(指数退避 + 抖动)
- 5xx → 有限重试(最多 2 次)
- 403/404 → 不重试,直接标记失败
- 失败原因写入 `download_tasks` 表便于排查
- **影响范围**:`pixiv_app/runtime/executor.py`、`pixiv_app/tasks/models.py`
### 2.10 图库搜索性能优化
- **现状**:图库 SQLite 查询在大数据量下(数万条作品)可能变慢,标签模糊搜索使用 `LIKE '%keyword%'` 无法走索引
- **方案**:
- 对 `artworks.title`、`artists.name`、`tags.name` 建立索引
- 标签搜索改为 FTS5 全文索引
- 云端的 `gallery` 查询为 `include_total` 参数已有优化,进一步对 `artworks.created_at`、`artworks.artist_id` 建立联合索引
- **影响范围**:`pixiv_app/core/library.py`、`sakura_cloud/models/artwork.py`
---
## 3. P2 — 功能完善与增强
### 3.1 手机端推送通知(FCM)
- **现状**:Firebase 已集成但推送链路不完整;当前"每日提醒"仅是应用内摘要,非系统级推送
- **方案**:
- 完成 FCM token 注册与服务端对接
- 云端提醒生成后通过 FCM 发送系统通知
- 点击通知直接跳转到对应内容
- 用户可配置"仅 Wi-Fi 下推送"、"免打扰时段"
- **影响范围**:`sakura_mobile/lib/src/core/push_notifications.dart`、`sakura_cloud/services/push.py`
### 3.2 插件市场在线对接
- **现状**:`PluginMarketClient` 为占位类,仅支持本地文件系统发现插件
- **方案**:
- 云端提供 `/api/v1/plugins/` 端点(列表、下载、版本检查)
- 桌面端「插件管理」面板增加"在线市场"标签页
- 支持一键安装/更新/卸载
- 插件版本兼容性检查
- **影响范围**:`pixiv_app/core/plugin/market.py`、`pixiv_app/gui/plugin_panel.py`、`sakura_cloud/` 新增端点
### 3.3 视频下载与播放支持
- **现状**:X/Twitter 插件可抓取视频文件,但本地图库和缩略图墙未针对视频做优化;手机端无视频播放页
- **方案**:
- 缩略图墙增加视频缩略图(自动截取首帧或使用平台提供的 thumbnail)
- 手机端新增视频播放页(基于 `video_player` 插件)
- 图库列表支持视频类型筛选
- **影响范围**:`pixiv_app/core/gallery_service.py`、`sakura_mobile/` 新增页面
### 3.4 阅读进度跨设备同步
- **现状**:手机端小说阅读器记录阅读进度,但仅本地存储(`online_reading_history.dart`)
- **方案**:
- 云端新增 `/api/v1/reading-progress/` 端点(GET/PUT)
- 手机端在进度变更时自动上报云端
- 桌面端缩略图墙也可展示阅读进度
- 多设备间同账号自动同步阅读位置
- **影响范围**:`sakura_mobile/lib/src/core/online_reading_history.dart`、`sakura_cloud/` 新增端点
### 3.5 内容合集 / 专辑功能
- **现状**:作品可以按作者、标签归类,但没有用户自定义合集的能力
- **方案**:
- 桌面端和手机端均支持创建"合集",将任意作品(图片/漫画/小说)归入合集
- 合集支持排序、封面设置、备注
- 云端同步合集数据
- **影响范围**:`pixiv_app/core/library.py`(新增表)、`sakura_cloud/models/`(新增模型)、`sakura_mobile/` 新增页面
### 3.6 漫画连续阅读模式增强
- **现状**:手机端已有漫画分组与连续阅读,但体验可以更流畅
- **方案**:
- 左右滑动翻页(当前可能仅支持上下滚动)
- 预加载下一页/上一页(当前页 ± 2 页)
- 阅读进度条(当前页/总页数)
- 双击缩放、长按保存
- **影响范围**:`sakura_mobile/lib/src/pages/online_manga_reader_page.dart`
### 3.7 批量导出功能
- **现状**:用户可通过缩略图墙浏览,但无法批量导出原图
- **方案**:
- 缩略图墙增加"批量选择 → 导出原图"功能
- 支持按作者/标签/收藏筛选后全选导出
- 导出格式:原图 ZIP 包或保持目录结构
- **影响范围**:`pixiv_app/core/gallery_service.py`、缩略图墙前端模板
### 3.8 X (Twitter) 作者时间线监控
- **现状**:X 插件支持抓取作者媒体和单个推文,但缺少"监控作者新推文"的主动检测
- **方案**:
- 桌面端新增"X 监控"面板,可添加监控的 X 用户名
- 定时轮询作者时间线,检测新推文 → 自动下载 → 写入云端关注提醒
- 与现有每日提醒系统打通
- **影响范围**:`plugins/x/`、`pixiv_app/core/x_media_service.py`
### 3.9 下载内容去重增强
- **现状**:`download_tasks` 表有 `fingerprint` 唯一约束,`artwork_files.content_hash` 字段已预留但默认空字符串
- **方案**:
- 下载完成后计算文件 SHA256 并写入 `content_hash`
- 新下载前先检查 `content_hash` 是否已存在,跳过重复文件
- 缩略图墙可展示"重复文件"列表,支持清理
- **影响范围**:`pixiv_app/core/downloader.py`、`pixiv_app/core/library.py`
### 3.10 代理池健康度评分
- **现状**:代理有 `fail_count` / `success_count` 但仅用于简单标记失败
- **方案**:
- 引入代理健康度评分(成功率、响应延迟、可用性百分比)
- 代理按健康度排序,优先使用高分代理
- 健康度过低的代理自动移出池
- 代理池 UI 展示健康度统计
- **影响范围**:`pixiv_app/core/proxy_pool.py`
---
## 4. P3 — 体验优化与锦上添花
### 4.1 桌面端深色模式
- **现状**:GUI 锁定 `ctk.set_appearance_mode("light")`
- **方案**:增加"外观"设置项——浅色 / 深色 / 跟随系统;`customtkinter` 原生支持 `dark` 模式,改动极小
- **影响范围**:`pixiv_app/gui/app.py`
### 4.2 缩略图墙 UI 现代化
- **现状**:缩略图墙是本地 WebUI(Jinja2 模板),功能齐全但 UI 风格较为基础
- **方案**:
- 引入轻量 CSS 框架(如 Pico.css 或 Water.css)美化
- 支持网格/列表视图切换
- 图片懒加载(当前可能一次性加载所有缩略图)
- 虚拟滚动(大量作品时性能优化)
- **影响范围**:`pixiv_app/core/gallery_service.py`、模板文件
### 4.3 手机端首页个性化配置
- **现状**:发现页展示"最近浏览的图片/漫画/小说"和"更新提醒"
- **方案**:
- 用户可选择发现页展示哪些模块(图片/漫画/小说/提醒)
- 支持拖拽排序模块
- "继续阅读"模块(直接从上次阅读位置进入)
- **影响范围**:`sakura_mobile/lib/src/pages/tabs/feed_tab.dart`
### 4.4 下载速度与流量统计
- **现状**:桌面端有进度条但无历史统计
- **方案**:
- 记录每次下载的文件大小、耗时
- 主界面增加"本次会话下载量"和"总计下载量"统计
- 可查看每日/每周下载量趋势图(可选,使用 matplotlib)
- **影响范围**:`pixiv_app/gui/app.py`、`pixiv_app/core/library.py`
### 4.5 国际化(i18n)支持
- **现状**:GUI 和手机端 UI 均为中文硬编码
- **方案**:
- 桌面端:提取所有 UI 字符串到 JSON 语言文件,支持中文/英文/日文
- 手机端:使用 Flutter `intl` / `easy_localization` 实现多语言
- 优先完成英文翻译以扩大用户群
- **影响范围**:全局 UI
### 4.6 错误日志收集与诊断
- **现状**:日志分散在 `日志/` 目录和终端输出,排查问题需手动翻找
- **方案**:
- 桌面端增加"一键导出诊断包"按钮(日志 + 配置 + 数据库状态摘要,压缩为 ZIP)
- 手机端增加"上传诊断日志"功能
- 云端增加错误聚合面板(按错误类型/时间统计)
- **影响范围**:`pixiv_app/`、`sakura_mobile/`、`sakura_cloud/`
### 4.7 桌面端系统托盘与后台运行
- **现状**:关闭 GUI 窗口即退出程序
- **方案**:
- 最小化到系统托盘(使用 `pystray` 或 `customtkinter` 配合系统托盘库)
- 托盘菜单:显示/隐藏窗口、开始/停止抓取、退出
- 下载完成后托盘气泡提示
- **影响范围**:`pixiv_app/gui/app.py`
### 4.8 自动更新检查
- **现状**:用户需手动关注 releases 目录获取新版本
- **方案**:
- 桌面端启动时检查云端最新版本号
- 有新版本时提示下载,可选"自动下载 → 下次启动替换"
- 手机端同理(检查 APK 版本号)
- **影响范围**:`pixiv_app/gui/app.py`、`sakura_mobile/lib/src/app.dart`、`sakura_cloud/` 新增版本检查端点
---
## 5. P4 — 长期架构演进
### 5.1 桌面端从 CustomTkinter 迁移到更现代化的 GUI 框架
- **理由**:CustomTkinter 功能有限,深色模式、动画、富文本等支持较弱;Windows 11 下风格略显过时
- **候选方案**:
- **Tauri (Rust + Web 前端)**:体积小、性能好、现代 UI,但需重写全部 Python 逻辑为 Rust 或通过 sidecar 通信
- **Electron + Python 后端**:重用现有 Python 下载引擎,Electron 负责 UI,但体积大(~150MB+)
- **PySide6 / PyQt6**:纯 Python 方案,迁移成本最低,支持 QSS 样式和丰富控件
- **Flet (Flutter for Python)**:用 Python 写 Flutter UI,可与现有 Flutter 移动端共享设计语言
- **建议**:优先评估 **Flet**(与 Flutter 移动端风格统一、纯 Python)或 **PySide6**(迁移成本最低)。不宜急于重写,等 P0/P1 完成后评估
### 5.2 iOS 客户端
- **理由**:当前仅支持 Android,iOS 用户无法使用
- **方案**:
- Flutter 跨平台特性使得 iOS 适配成本较低(大部分 Dart 代码可复用)
- 需额外处理:iOS 文件系统权限、后台任务限制、APNs 推送替代 FCM
- 需要 Apple Developer Program 账号($99/年)用于签名分发
- **影响范围**:`sakura_mobile/ios/`(生成)、`sakura_mobile/lib/`(少量平台适配)
### 5.3 Web 端 PWA(渐进式 Web 应用)
- **理由**:桌面端 EXE 体积大(~68MB),部分用户只希望轻量浏览
- **方案**:
- 将缩略图墙升级为 PWA(Service Worker + 离线缓存 + 可安装到桌面)
- 或单独开发 `sakura_web` 前端项目替代当前 `sakura_admin_web`
- **影响范围**:`sakura_admin_web/` 或新建 `sakura_web/`
### 5.4 分布式爬虫完善
- **现状**:`distributed_crawler/` 目录有 Master-Worker 架构但尚不完整
- **方案**:
- 补全 Worker 节点注册、心跳、任务分配
- Master 面板(Web UI)展示各 Worker 状态和任务进度
- Docker Compose 一键部署 Master + N Workers
- **影响范围**:`distributed_crawler/`
### 5.5 AI 辅助功能
- **理由**:PaddleOCR 已集成,可在其基础上扩展更多 AI 能力
- **方案**:
- **AI 标签自动生成**:对下载的图片使用轻量视觉模型(如 MobileNet/EfficientNet)自动打标签
- **翻译增强**:手机端小说翻译当前仅支持"当前页翻译",可升级为整本翻译 + 双语对照
- **图片去重**:基于感知哈希(pHash/dHash)的相似图片检测
- **内容安全扫描**:对上传至云端的内容进行自动安全审核(NSFW 检测)
- **影响范围**:`pixiv_app/core/ocr.py`、`sakura_cloud/services/ocr.py`
### 5.6 多用户/家庭共享
- **现状**:云同步为单用户设计,无分享功能
- **方案**:
- 云端新增"家庭组"/"共享空间"概念
- 用户可邀请其他账号加入共享空间
- 共享空间内的内容(图库/小说/漫画)成员均可访问
- 权限控制:管理员/编辑/只读
- **影响范围**:`sakura_cloud/` 新增模型与端点、手机端新增分享页面
---
## 6. 实施路线图
### 第一阶段:安全加固(第 1–2 周)
```
P0-1 JWT 密钥强制生产配置 ████████░░░░░░░░ 1d
P0-2 数据库/MinIO 凭据强制覆盖 ████████░░░░░░░░ 1d
P0-3 HTTPS 支持 + 警告 ████████████░░░░ 1.5d
P0-4 登录接口限流 ████████░░░░░░░░ 1d
P0-5 笔趣阁 AES 随机化 ████░░░░░░░░░░░░ 0.5d
P0-6 Chrome --no-sandbox 移除 ████░░░░░░░░░░░░ 0.5d
P0-7 Firebase 初始化竞态修复 ████░░░░░░░░░░░░ 0.5d
P0-8 ProxyPool 线程安全 ████████░░░░░░░░ 1d
P0-9 .gitignore 完善 ████░░░░░░░░░░░░ 0.5d
────────────────────────────────────────────────────────
小计约 7.5 人天
```
### 第二阶段:体验修复(第 3–6 周)
```
P1-1 手机端死开关修复 ████░░░░░░░░░░░░ 0.5d
P1-2 Legado 搜索引擎完善 ████████████████ 4d
P1-3 发现页滚动卡死 ████████░░░░░░░░ 2d
P1-4 Web 管理后台完善 ████████████░░░░ 3d
P1-5 上传失败队列 + 断点续传 ████████████████ 4d
P1-6 Alembic 迁移 ████████░░░░░░░░ 2d
P1-7 正式签名 ████░░░░░░░░░░░░ 1d
P1-8 Cookie 过期自动检测 ████████░░░░░░░░ 2d
P1-9 下载失败重试优化 ████████░░░░░░░░ 2d
P1-10 图库搜索性能 ████████████░░░░ 3d
────────────────────────────────────────────────────────
小计约 23.5 人天
```
### 第三阶段:功能增强(第 7–14 周)
```
P2-1 FCM 推送通知 ████████████████ 5d
P2-2 插件市场在线对接 ████████████████ 5d
P2-3 视频下载与播放 ████████████░░░░ 4d
P2-4 阅读进度跨设备同步 ████████████░░░░ 4d
P2-5 内容合集/专辑 ████████████████ 5d
P2-6 漫画连续阅读增强 ████████░░░░░░░░ 3d
P2-7 批量导出 ████████████░░░░ 4d
P2-8 X 作者时间线监控 ████████████░░░░ 4d
P2-9 下载去重增强 ████████░░░░░░░░ 3d
P2-10 代理池健康度评分 ████████░░░░░░░░ 3d
────────────────────────────────────────────────────────
小计约 40 人天
```
### 第四阶段及以后(按需排期)
```
P3-1 深色模式 ████░░ 1d
P3-2 缩略图墙 UI 现代化 ██████ 2d
P3-3 首页个性化 ██████ 2d
P3-4 下载统计 ████░░ 1.5d
P3-5 国际化 ██████ 3d
P3-6 错误诊断包 ██████ 2d
P3-7 系统托盘 ██████ 2d
P3-8 自动更新 ██████ 2d
P4-X 长期架构演进 TBD
────────────────────────────────────────────────────────
小计约 15.5 人天
```
### 关键里程碑
| 里程碑 | 时间 | 交付物 |
|--------|------|--------|
| **M1: 安全基线** | 第 2 周末 | 所有 P0 项修复完成,安全审计复测通过 |
| **M2: 体验基线** | 第 6 周末 | 所有 P1 项修复完成,核心链路无阻断 |
| **M3: 功能补全** | 第 14 周末 | P2 主要项完成,推送/插件市场/跨设备进度同步可用 |
| **M4: 体验升级** | 第 20 周末 | P3 项完成,深色模式/国际化/自动更新上线 |
---
## 附录:已完成功能清单(便于对照)
| 模块 | 已具备能力 |
|------|-----------|
| **桌面端** | 多平台抓取、Cookie/密码登录、插件系统、本地图库 SQLite、缩略图墙 WebUI、云同步、代理池、任务队列 + 增量同步、OCR |
| **云端** | FastAPI REST API、JWT 认证、图库/小说/文件 CRUD、关注管理、每日提醒摘要、分片上传、健康检查 |
| **手机端** | 登录注册、发现/图片/漫画/书架/我的 五个标签页、小说阅读器(分页/收藏/翻译/进度)、漫画分组阅读、书源管理+Legado 引擎、在线搜索、FCM(部分) |
| **插件** | Pixiv、X/Twitter、小红书、笔趣阁、长佩文学、AO3 |
| **DevOps** | 一键构建 EXE/APK、Docker Compose 部署、Alembic 迁移 |
---
*本文档由代码审计报告、产品文档、CHANGELOG、SAKURA_CLOUD_V1_STATUS 及全仓库源码分析综合生成,供开发排期与需求评审参考。*
+745
View File
@@ -0,0 +1,745 @@
# 四大问题完整修复方案(终版)
> 更新日期:2026-06-16
> 范围:Sakura Mobile (Flutter) + Sakura Cloud (FastAPI) + Sakura Admin Web (Vue.js) + Desktop GUI (Python)
> 版本:v2.0 — 调整优先级为 P0/P1 两级、修正问题二的风险边界和工期、问题一拆为两层修复、问题四拆为开发/生产两套方案、问题三补搜索防抖
---
## 目录
1. [优先级总览](#零优先级总览)
2. [P0-1:无法搜索本地不存在的图库(源不可用)](#一p0-1无法搜索本地不存在的图库源不可用)
3. [P0-2:网页端显示 JSON 数据而非图片/小说/漫画](#二p0-2网页端显示-json-数据而非图片小说漫画)
4. [P1-1:APP 发现页向上滚动卡在底部](#三p1-1app-发现页向上滚动卡在底部)
5. [P1-2:导入源的位置排版有问题](#四p1-2导入源的位置排版有问题)
6. [实施计划](#五实施计划)
---
## 零、优先级总览
| 等级 | 问题 | 严重程度 | 工期 | 风险 |
|:----:|------|:--------:|:----:|:----:|
| **P0-1** | 书源搜索不可用 | 核心功能阻断 | 3–5 天 | **高** |
| **P0-2** | Web 端显示 JSON / 无法展示内容 | 入口不可用 | 1–2 天 | 低 |
| **P1-1** | 发现页 Feed 滚动卡死 | 高频场景阻断 | 1–2 天 | 低 |
| **P1-2** | 书源管理页排版和性能 | 管理体验差 | 0.5–1 天 | 低 |
**优先级判断依据:**
- 问题二是"在线探索"的整个功能链路不通,影响范围最大。
- 问题四是 Web 端入口整体不可用,新用户访问会直接流失。
- 问题一如果高频复现(每次打开发现页都卡),也接近阻断级,但只影响单页面。
- 问题三是管理后台的体验优化,不影响核心浏览链路。
---
## 一、P0-1:无法搜索本地不存在的图库(源不可用)
### 1.1 问题现象
"在线探索"(ExplorePage)搜索漫画/小说时,所有已启用的书源永远不返回结果。无论是否从仓库导入源,`legado_engine.search()` 都静默返回 `[]`,UI 显示"暂无搜索结果"。
### 1.2 根因分析
**审查文件(6 个):** `source_manager.dart`、`legado_engine.dart`、`legado_models.dart`、`legado_rule_parser.dart`、`unified_search.dart`、`explore_page.dart`
#### 根因 1(阻断级):默认种子源 URL = `example.invalid`,且 searchUrl/searchListRule 全空
**文件:** `source_manager.dart` 第 274–317 行
```dart
LegadoSource(
name: '笔趣阁',
url: 'https://example.invalid/novel/biquge', // IANA 保留测试域名
type: 0, enabled: true,
// searchUrl、searchListRule —— 全部 null
),
```
`LegadoEngine.search()` 入口(第 36–39 行):
```dart
if (searchUrl == null || listRule == null) {
return const <UnifiedSearchResult>[]; // 静默短路
}
```
六个默认种子源全都没有搜索规则。不论用户是否"启用",搜索入口直接返回空。这些种子源是从设计之初就存在的占位符,但从未被替换为可用源。
#### 根因 2(阻断级):CSS 选择器仅支持 `<tag>...</tag>` 简单正则,不支持 class/ID/属性/后代
**文件:** `legado_rule_parser.dart` 第 86–96 行
```dart
List<String> _cssFragments(String selector) {
final tag = _selectorTag(selector); // 只取开头 a-zA-Z 部分作 tag
if (tag == null) return const <String>[];
final pattern = RegExp('<$tag\\b[^>]*>.*?</$tag>', ...);
return pattern.allMatches(body)...;
}
```
Legado 源仓库中 95%+ 的规则使用 class 选择器(`.book-list > .item`),当前引擎完全无法解析。
#### 根因 3(阻断级):GBK 编码 HTML 抛 `FormatException`
**文件:** `legado_engine.dart` 第 216–220 行
```dart
if (charset == 'gbk' || charset == 'gb2312' || ...) {
return gbk.decode(bytes); // ← 无容错参数,遇非 GBK 字节直接抛异常
}
```
`utf8.decode` 使用了 `allowMalformed: true`,但 `gbk.decode` **没有容错**。大量声称 charset=gbk 的页面实际混入了非 GBK 字节(emoji、广告注入的特殊符号等),导致整个 `_fetchBody` 失败。
#### 根因 4:`_ruleType` 忽略传入的 `kind` 参数
**文件:** `legado_rule_parser.dart` 第 130–134 行
`selectList` 和 `select` 虽然接受 `kind` 参数,但传给 `_ruleType` 的逻辑有断裂——`_ruleType` 方法签名不接受 `kind`,只根据规则文本前缀判断类型。如果 `source.ruleKind == 'xpath'` 但规则文本不以 `xpath:` 开头,会错误地按 CSS 解析。
#### 根因 5:仓库 URL 在国内网络不可达 + 首次启动静默失败
`legado.aoaostar.com`、`gcore.jsdelivr.net`、`git.nyne.dev` 均可能被 DNS 污染或不可达。`initializeIfFirstRun()` 首次启动自动拉仓库,12 秒超时后静默失败,用户无法感知问题所在。
#### 根因 6:搜索结果 DOM 解析在 UI 线程
每条结果调一次 `html_parser.parse(fragment)`,搜索结果 20 条 × 6 源 = 120 次 DOM 解析。低端机上会造成 frame drop。
### 1.3 解决方案
**⚠️ 法律与风险声明:** 问题二的本质是半个解析引擎重构 + 第三方内容源集成。涉及版权法律风险、源站反爬策略、编码兼容性和解析性能。以下方案在技术层面给出修复路径,**实际实施前需要产品侧确认法律边界和使用条款**。
#### 方案 A:删除占位种子源,首次启动不做任何自动导入
**修改文件:** `source_manager.dart`
**策略——客户端不内置任何第三方书源仓库、书源 URL 或规则:**
```dart
Future<void> initializeIfFirstRun() async {
await load();
final prefs = await SharedPreferences.getInstance();
if (prefs.getBool(_initializedKey) == true) return;
// 不做任何自动导入。首次启动源列表为空。
await prefs.setBool(_initializedKey, true);
notifyListeners();
}
```
删除 `_seedDefaultSources()` 中的 6 个 `example.invalid` 占位源。首次启动后源列表为空,UI 显示引导入口:
> **当前没有可用书源。**
> 书源规则是第三方网站的内容解析配置,请从你信任的仓库导入,或手动添加你有使用权限的书源。
> [从内置仓库导入] [从 URL 导入]
**不内置任何仓库索引 JSON。** 仓库列表仅作为 UI 入口的快捷链接提供,不硬编码到客户端。
#### 方案 B:重写 CSS 选择器——引入 `html` 包 + Isolate 解析
**修改文件:** `legado_rule_parser.dart`、`pubspec.yaml`
```yaml
dependencies:
html: ^0.15.5
```
```dart
import 'package:html/parser.dart' as html_parser;
List<String> _cssFragments(String selector) {
try {
final doc = html_parser.parse(body);
final elements = doc.querySelectorAll(selector);
return elements.map((el) => el.outerHtml).where(_notEmpty).toList();
} on Object {
return _legacyCssFragments(selector); // 正则降级
}
}
String _cssText(String fragment, String selector) {
final attrMatch = RegExp(r'@([a-zA-Z0-9_-]+)$').firstMatch(selector);
final clean = selector.replaceFirst(RegExp(r'@([a-zA-Z0-9_-]+)$'), '').trim();
try {
final el = html_parser.parse(fragment).querySelector(clean);
if (el == null) return '';
if (attrMatch != null) return el.attributes[attrMatch.group(1)] ?? '';
return el.text.trim();
} on Object {
return _legacyCssText(fragment, selector);
}
}
```
对超过 20 条的搜索结果,在 `Isolate` 中批量解析:
```dart
import 'dart:isolate';
Future<List<UnifiedSearchResult>> _parseBatch(
List<_SearchRawResult> raw,
) async {
if (raw.length < 20) return _parseInline(raw);
return Isolate.run(() => _parseInIsolate(raw));
}
```
#### 方案 C:修复 GBK/GB18030 解码容错
**修改文件:** `legado_engine.dart` 第 202–223 行
```dart
String _decodeBody(List<int> bytes, {String? declaredCharset, ...}) {
if (bytes.isEmpty) return '';
final charset = (declaredCharset ?? _charsetFromContentType(contentType))
?.trim().toLowerCase();
if (charset == 'gbk' || charset == 'gb2312' || charset == 'gb18030' || charset == 'gbk-0') {
try {
return gbk.decode(bytes);
} on FormatException {
return latin1.decode(bytes, allowInvalid: true); // 降级,保留可读部分
}
}
if (charset == 'latin1' || charset == 'iso-8859-1' || charset == 'windows-1252') {
return latin1.decode(bytes, allowInvalid: true);
}
try {
return utf8.decode(bytes, allowMalformed: true);
} on FormatException {
return latin1.decode(bytes, allowInvalid: true);
}
}
```
#### 方案 D:修复 `_ruleType`——优先使用传入的 `kind`
```dart
String _resolveKind(String rule, String? explicitKind) {
if (explicitKind != null && explicitKind.isNotEmpty) return explicitKind;
final lower = rule.toLowerCase();
if (lower.startsWith('json:') || lower.startsWith(r'$.')) return 'json';
if (lower.startsWith('regex:')) return 'regex';
return 'css';
}
```
调用方改为 `_resolveKind(normalized, kind)` 替代原来的 `kind ?? _ruleType(normalized)`。
#### 方案 E:多源异常隔离——单源失败不拖垮全局
**修改文件:** `legado_engine.dart` `searchAll()` 方法
当前实现已经是 `.catchError` 静默跳过,但错误信息被丢弃。改为:
```dart
Future.wait(batch.map((source) =>
search(source, keyword, page: page)
.timeout(perSourceTimeout)
.then((items) => items)
.catchError((error) => <UnifiedSearchResult>[
UnifiedSearchResult.error(
sourceId: source.id,
sourceName: source.name,
errorMessage: _shortError(error),
),
]),
));
```
错误结果显示在 UI 中(如 `"笔趣阁 - 请求超时"`),用户可以看到哪些源失败了。
#### 方案 F:仓库镜像轮询 + 网络探测
为 5 个仓库 URL 各提供 2–3 个镜像域名。`importFromRepo()` 依次尝试,全部失败则提示用户手动输入 URL。
---
## 二、P0-2:网页端显示 JSON 数据而非图片/小说/漫画
### 2.1 问题现象
浏览器访问 Sakura Cloud 后端或 Admin Web 时,页面显示原始 JSON(`{"items": [...]}`),而非渲染好的图片网格、小说内容、漫画翻页器。
### 2.2 根因分析
**审查文件:** `sakura_cloud/main.py`、`sakura_cloud/api/middleware.py`、`sakura_admin_web/`、`pixiv_app/services/gallery_api.py`
#### 根因 1(阻断级):FastAPI 未注册 CORS 中间件
**文件:** `sakura_cloud/main.py`
```python
app = FastAPI(...)
# ← 缺少 app.add_middleware(CORSMiddleware, ...)
```
Admin Web 开发时运行在 `localhost:5174`,后端在 `localhost:8000`。浏览器跨域请求被拦截,前端连 API 都调不通。CSP 中间件(`middleware.py`)处理的是内容安全策略,**不替代 CORS**。
#### 根因 2(阻断级):Cloud 后端未挂载 SPA 静态文件
**文件:** `sakura_cloud/main.py` 第 61–78 行
`/admin` 路由返回的是硬编码占位 HTML,不是 `sakura_admin_web/dist/index.html` 构建产物。任何 `/app`、`/login` 路径返回 404 或触发认证检查。
#### 根因 3:Admin Web 缺少内容浏览 Vue 组件
- `UserDashboard.vue`:仅导航卡片,无实际内容页
- `UserDetailPage.vue`:占位组件——"详情数据由现有 API 按需加载"
- 路由没有注册 `/gallery`、`/novels` 等内容浏览路由
#### 根因 4:CSP `img-src 'self'` 阻止外部图片
API 返回的 `thumbnail_url` 指向外部域名时,浏览器 CSP 引擎直接拦截。
### 2.3 解决方案
#### ⸻ 开发环境 ⸻
#### 方案 A:注册 CORS 中间件
**修改文件:** `sakura_cloud/main.py`
```python
from fastapi.middleware.cors import CORSMiddleware
app.add_middleware(
CORSMiddleware,
allow_origins=[
"http://localhost:5174",
"http://127.0.0.1:5174",
"app://.",
],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
```
#### 方案 B:修复 CSP 中间件
**修改文件:** `sakura_cloud/api/middleware.py`
```python
"img-src 'self' data: blob: https: http:;"
"connect-src 'self' http://localhost:* ws://localhost:*;"
```
#### 方案 C:开发环境可选挂载 Vue dist
```python
# sakura_cloud/main.py — 开发环境可选
import os
from fastapi.staticfiles import StaticFiles
frontend_dist = os.path.join(os.path.dirname(__file__), "..", "sakura_admin_web", "dist")
if os.path.isdir(frontend_dist) and settings.debug:
assets = os.path.join(frontend_dist, "assets")
if os.path.isdir(assets):
app.mount("/assets", StaticFiles(directory=assets), name="assets")
@app.get("/app/{full_path:path}")
async def spa_fallback(full_path: str = ""):
index = os.path.join(frontend_dist, "index.html")
return FileResponse(index) if os.path.isfile(index) else HTMLResponse("Not built")
```
**注意:不添加 API 路径自动重定向的中间件。** 这会导致 APP 的 Dio/浏览器 Extension 发起的 API 请求被误重定向。API 路径就让它返回 JSON——它是 REST API 的正确行为。前端 SPA 从 `/app` 入口加载后用 JS fetch 数据。
#### ⸻ 生产环境 ⸻
#### 方案 D:Nginx 托管 SPA + 反向代理 API
**推荐的生产结构:**
```
Nginx
├── / → /var/www/sakura_admin_web/dist/index.html
├── /app → SPA fallback (try_files $uri /index.html)
├── /login → SPA fallback
├── /assets/ → 静态资源,expires 1y(Vite 产物带 hash)
└── /api/ → proxy_pass http://127.0.0.1:8000
```
**Nginx 配置示例:**
```nginx
server {
listen 80;
server_name admin.your-domain.com;
root /var/www/sakura_admin_web/dist;
index index.html;
location /api/ {
proxy_pass http://127.0.0.1:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
location /assets/ {
expires 1y;
add_header Cache-Control "public, immutable";
}
location / {
try_files $uri $uri/ /index.html;
}
}
```
FastAPI **不长期托管前端静态资源**,只负责 API。
#### 方案 E:新增内容浏览 Vue 组件
**新增文件:**
```
sakura_admin_web/src/views/app/
├── GalleryBrowsePage.vue # 画廊浏览:grid + 分页 + 搜索筛选
├── NovelReadPage.vue # 小说阅读:章节列表 + 正文渲染
└── MangaViewerPage.vue # 漫画翻页:PageView 横向翻页
```
路由注册:
```typescript
// router/index.ts
{ path: '/app/gallery', component: () => import('@/views/app/GalleryBrowsePage.vue') },
{ path: '/app/novels', component: () => import('@/views/app/NovelReadPage.vue') },
{ path: '/app/manga/:id', component: () => import('@/views/app/MangaViewerPage.vue') },
```
---
## 三、P1-1:APP 发现页向上滚动卡在底部
### 3.1 问题现象
发现页从底部向上回滚时卡住,须等待数据加载完或强制下拉刷新才能恢复。
### 3.2 根因分析
**审查文件:** `feed_tab.dart`(829 行)
| 根因 | 位置 | 严重程度 |
|------|------|:--------:|
| `_ensureFutures()` 在 `build()` 中反复调用 | 第 64–66 行 | 高 |
| `_refresh()` 用 `setState` 归零所有 future → 布局高度剧烈跳动 | 第 53–62 行 | **阻断** |
| `_StatCard` 在列表末尾加载完成后高度变化 | 第 182–186 行 | 中 |
| 内嵌横向 `ListView` 与纵向 `ListView` 手势竞争 | 第 443–507 行 | 中(部分设备) |
| `ScrollController` 未显式管理生命周期 | 缺 | 低 |
### 3.3 解决方案——拆为两层
#### 第一层:必须做(P1 范围)
**1. `AsyncNotifier` 替代 `FutureBuilder` + 状态统一管理**
当前 `_FeedTabState` 用 5 个散落的 `Future` + `FutureBuilder` 管理异步状态,刷新时 `setState(null)` 归零所有 future 再在下一帧重建——这个模式是滚动问题的根源。
改为用一个统一的 `FeedState` 对象:
```dart
class FeedState {
final bool loading;
final List<Artwork> todayArtworks;
final List<Artwork> recentArtworks;
final List<Novel> recentNovels;
final List<Map<String, dynamic>> alerts;
final Map<String, dynamic> stat;
final String? error;
}
```
用 Riverpod `AsyncNotifier` 管理整个页面的数据加载。`build()` 只读取 `ref.watch(feedProvider)`,不调用 `_ensureFutures()`,不在 build 中创建 future。
**2. 显式 `ScrollController`**
```dart
class _FeedTabState extends ConsumerState<FeedTab> {
final ScrollController _scrollController = ScrollController();
@override
void dispose() {
_scrollController.dispose();
super.dispose();
}
}
```
外层 `ListView` 绑定此 controller。切换 tab 时通过 `_scrollController.hasClients` 检查位置有效性。
**3. `_StatCard` 固定高度**
```dart
SizedBox(
height: 106,
child: _StatCard(stat: stat),
),
```
通过 `AnimatedSwitcher` 平滑过渡内容而非跳变布局。
**4. Refresh 状态统一管理**
`RefreshIndicator.onRefresh` 调用 provider 的 `ref.invalidate(feedProvider)`,由一个入口触发重载,不再散射到 5 个独立的 `setState(null)` 调用。
#### 第二层:如果仍有横向列表抢手势(专项修复)
仅当第一层修复后,在部分设备上仍出现"在对角线滑动时外层垂直滚动丢失"的问题时,才实施此层:
对横向 `_ArtworkStrip` 的 `ListView` 外套方向判定:
```dart
SizedBox(
height: 210,
child: RawGestureDetector(
gestures: {
HorizontalDragGestureRecognizer:
GestureRecognizerFactoryWithHandlers<HorizontalDragGestureRecognizer>(
() => HorizontalDragGestureRecognizer(),
(instance) {},
),
},
child: ListView.separated(
scrollDirection: Axis.horizontal,
physics: const ClampingScrollPhysics(),
...
),
),
),
```
`HorizontalDragGestureRecognizer` 在竞技场中只响应水平方向的拖拽,对角滑动时垂直分量不会被它吃掉,外层垂直 ListView 正常响应。
**不推荐 `PageStorageKey` 解决手势竞争。** `PageStorageKey` 的职责是页面切换后恢复滚动位置(`PageStorage` 机制),它不参与手势竞技场裁决。两项是不同层面的问题,不应混用。
---
## 四、P1-2:导入源的位置排版有问题
### 4.1 问题现象
书源管理页布局混乱:搜索栏在导入区底部(须滑过所有卡片才能搜索)、已导入 2200+ 条后 `for` 循环展开所有 Card 导致卡顿、5 个仓库全部展开占用过多空间。
### 4.2 根因分析
**审查文件:** `source_manager_page.dart`(465 行)
| 根因 | 位置 |
|------|------|
| 搜索栏放在仓库 + 导入卡片**之后** | 第 81–85 行 |
| `for` 循环展开全部源,无懒加载 | 第 102–129 行 |
| 5 个仓库全部展开,每个 ~120px | 第 376–413 行 |
| 缺少分段标题和视觉层次 | 全页 |
### 4.3 解决方案
#### 方案 A:CustomScrollView + SliverAppBar 吸顶搜索栏
```dart
@override
Widget build(BuildContext context) {
final manager = ref.watch(sourceManagerProvider);
final filteredSources = _filterSources(manager);
return Scaffold(
body: CustomScrollView(
slivers: [
// 搜索栏吸顶
SliverAppBar(
pinned: true,
title: const Text('书源管理'),
bottom: PreferredSize(
preferredSize: const Size.fromHeight(56),
child: Padding(
padding: const EdgeInsets.fromLTRB(16, 0, 16, 8),
child: SearchBar(
hintText: '搜索已导入的书源',
leading: const Icon(Icons.search),
onChanged: onSearchChanged, // 带防抖
),
),
),
),
// 统计 + 维护 + 仓库 + 导入 —— 各 SliverToBoxAdapter
SliverToBoxAdapter(child: _StatsCard(manager: manager)),
SliverToBoxAdapter(child: _MaintenanceCard(...)),
SliverToBoxAdapter(child: _CollapsibleRepoSection(...)),
SliverToBoxAdapter(child: _CustomImportCard(...)),
// 分段标题
SliverToBoxAdapter(
child: Padding(
padding: const EdgeInsets.fromLTRB(16, 8, 16, 4),
child: Text('已导入 (${filteredSources.length})', ...),
),
),
// 源列表 —— 懒加载
SliverList.builder(
itemCount: filteredSources.length,
itemBuilder: (context, index) {
final source = filteredSources[index];
return Card(
child: SwitchListTile(
value: source.enabled,
onChanged: (value) =>
ref.read(sourceManagerProvider).setEnabled(source.id, value),
title: Text(source.name),
subtitle: Text('${source.isManga ? '漫画' : '小说'} - ${source.url}'),
secondary: Icon(source.isManga
? Icons.menu_book_rounded
: Icons.auto_stories_rounded),
),
);
},
),
if (filteredSources.isEmpty)
SliverToBoxAdapter(
child: const Padding(
padding: EdgeInsets.all(28),
child: Center(child: Text('还没有导入书源,请先导入一个书源仓库。')),
),
),
const SliverToBoxAdapter(child: SizedBox(height: 28)),
],
),
);
}
```
**不使用 `shrinkWrap: true` ——** `SliverList.builder` 本身就是懒加载,不需要这个参数。
#### 方案 B:搜索防抖
```dart
Timer? _debounce;
void onSearchChanged(String value) {
_debounce?.cancel();
_debounce = Timer(const Duration(milliseconds: 180), () {
setState(() {
filter = value;
});
});
}
@override
void dispose() {
_debounce?.cancel();
customUrl.dispose();
super.dispose();
}
```
180ms 防抖确保用户快速输入时不会每按一个键就触发一次全文过滤。
#### 方案 C:仓库列表改用 ExpansionTile
```dart
class _CollapsibleRepoSection extends StatelessWidget {
@override
Widget build(BuildContext context) {
return Card(
child: ExpansionTile(
title: const Text('内置书源仓库'),
subtitle: Text('${SourceManager.builtinRepos.length} 个仓库可用'),
initiallyExpanded: false,
children: [
for (final repo in SourceManager.builtinRepos)
ListTile(
leading: const Icon(Icons.cloud_download_outlined),
title: Text(repo.name),
subtitle: Text(repo.description),
trailing: FilledButton.tonal(
onPressed: ...,
child: Text(repo.isSubscription ? '订阅' : '导入'),
),
),
],
),
);
}
}
```
---
## 五、实施计划
### 阶段 1:P0 可用性修复(3–6 天)
| 天数 | 任务 | 涉及文件 |
|:----:|------|----------|
| 1–3 | **问题二:书源搜索链路修复** | |
| | └ 方案 A:删除占位种子源,空列表首次启动 | `source_manager.dart` |
| | └ 方案 B:`html` 包重写 CSS 选择器 + Isolate | `legado_rule_parser.dart`、`pubspec.yaml` |
| | └ 方案 C:GBK/GB18030 编码异常处理 | `legado_engine.dart` |
| | └ 方案 D:`_ruleType` 优先使用 `kind` 参数 | `legado_rule_parser.dart` |
| | └ 方案 E:多源异常隔离(错误透传 UI) | `legado_engine.dart` |
| | └ 方案 F:仓库镜像轮询 + 网络探测 | `source_manager.dart` |
| | └ **测试:** 真实书源搜索回归(≥10 个不同源站点) | |
| 4–6 | **问题四:Web 端 JSON → 可视化内容** | |
| | └ 方案 A:CORS 中间件(10min) | `sakura_cloud/main.py` |
| | └ 方案 B:CSP 修复(10min) | `sakura_cloud/api/middleware.py` |
| | └ 方案 C:开发环境可选挂载 Vue dist | `sakura_cloud/main.py` |
| | └ 方案 D:Nginx 生产部署配置 | 新建 `sakura_admin_web/nginx.conf` |
| | └ 方案 E:GalleryBrowsePage + NovelReadPage + MangaViewerPage | `sakura_admin_web/src/views/app/` |
| | └ **测试:** 浏览器 `/app` → 图片网格 → 点击作品 → 大图 → 分页 |
### 阶段 2:P1 体验稳定性修复(2–4 天)
| 天数 | 任务 | 涉及文件 |
|:----:|------|----------|
| 1–2 | **问题一:Feed 滚动稳定性(第一层必做)** | |
| | └ AsyncNotifier 替代 FutureBuilder + 状态统一管理 | `feed_tab.dart` |
| | └ 显式 ScrollController + dispose | `feed_tab.dart` |
| | └ StatCard 固定高度 106px | `feed_tab.dart` |
| | └ Refresh 状态统一入口(Riverpod provider) | `feed_tab.dart` + `providers.dart` |
| | └ 第二层(按需):RawGestureDetector 手势方向判定 | `feed_tab.dart` |
| | └ **测试:** Android 真机快速上下滚动 × 50 次 | |
| 3–4 | **问题三:书源管理页 Sliver 重构** | |
| | └ CustomScrollView + SliverAppBar 吸顶搜索栏 | `source_manager_page.dart` |
| | └ SliverList.builder 懒加载 | `source_manager_page.dart` |
| | └ ExpansionTile 折叠仓库列表 | `source_manager_page.dart` |
| | └ 搜索防抖 180ms + dispose 清理 | `source_manager_page.dart` |
| | └ **测试:** 导入 2200+ 源后滚动 + 搜索 | |
### 阶段 3:回归测试(1–2 天)
| 测试项 | 方法 |
|--------|------|
| Android 真机滚动 | feed_tab 快速上下滑 × 50 次,观察有无卡顿 |
| 2200+ 书源压力测试 | 从"精选书源仓库"导入全量 → 搜索 → 启用/禁用单个源 → 滚动 |
| Web 端 `/app` | Chrome + Firefox 访问 → 图片网格加载 → 分页翻页 |
| Web 端 `/api` | 直接访问 API 路径 → 确认返回 JSON(正常行为) |
| Web 端图片加载 | 检查 CSP 日志 → 确认无 `img-src` 拦截 |
| 无书源首次启动 | 清除 SharedPreferences → 冷启动 APP → 确认显示引导 UI 而非假源 |
### 风险矩阵
| 风险 | 等级 | 缓解措施 |
|------|:----:|----------|
| 问题二中 CSS 选择器 `html` 包性能不足 | 中 | 保留正则降级路径,低端机自动切降级 |
| 问题二中真实书源测试覆盖不足 | 高 | 选择 ≥10 个已知稳定的 Legado 源作为回归集 |
| 问题一中 AsyncNotifier 迁移引入新问题 | 低 | 先写测试用例覆盖原有 5 种数据加载路径 |
| 问题四中 Nginx 配置与现有部署冲突 | 中 | 提供独立配置文件和部署检查清单 |
| 书源法律边界 | 高 | 客户端不内置任何仓库 URL 或规则,由用户自行导入 |
---
> **版本记录:**
> - v1.0 (2026-06-16):初始版本
> - v1.1 (2026-06-16):技术评审修订——修正延迟数字、shrinkWrap 陷阱、重定向误伤、补充 CORS/ScrollController/GBK 异常/Isolate
> - v2.0 (2026-06-16):终版——优先级改为 P0/P1、问题二风险升级为 3–5 天且不内置任何仓库、问题一拆为两层修复(第一层必做 + 第二层按需)、问题四拆为开发/生产两套方案、问题三补搜索防抖、三阶段实施计划
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+778
View File
@@ -0,0 +1,778 @@
下面是整合后的**最终完整方案**,包括:漫画解析、章节完整性、图片加载、调试能力、搜索性能,以及你刚刚提到的顶部搜索框 UI 问题。
Sakura 漫画模块完整改进方案
=================
一、改进目标
------
本轮改造解决 5 类问题:
1. 漫画章节目录不全
2. 漫画图片解析失败
3. 图片 URL 有数量但无法显示
4. 在线搜索速度慢
5. 漫画 / 图片页顶部搜索框布局异常,在线搜索入口位置不合理
* * *
二、整体优先级
=======
P0:顶部搜索框 UI 修复与在线搜索入口重构
P1:图片 URL 合法性验证
P2:PageImage + headers 拆分
P3:图片 HTTP 探测与调试能力
P4:TOC 分页抓全章节
P5:章节排序支持番外
P6:Script 图片变量提取
P7:漫画调试页
P8:搜索健康分与增量搜索
说明:
P0 先做,因为它是明显 UI 问题,改动小、收益快。
P1~P7 解决漫画“能不能看”。
P8 最后做,解决“搜得慢”。
* * *
P0:顶部搜索框 UI 修复与在线搜索入口重构
=======================
问题
--
截图中红框区域出现:
顶部搜索栏被状态栏 / 右上角按钮挤压
漫画页和图片页都出现同类问题
在线搜索入口放在右上角,不符合用户预期
目标
--
漫画页 / 图片页顶部布局稳定
搜索框不被状态栏遮挡
在线搜索入口集成到搜索框流程中
不再在右上角放在线搜索按钮
推荐交互
----
页面顶部只保留一个统一搜索框:
[ 搜索图标 ] 搜索标题、作者或标签 [筛选] [刷新]
点击搜索框进入统一搜索页:
SearchPage
├─ 搜索输入框
├─ 搜索范围切换:本地 / 在线 / 全部
└─ 结果分区:
├─ 本地漫画
├─ 在线漫画源
└─ 图片 / 插画
代码结构
----
新增统一组件:
class SakuraSearchBar extends StatelessWidget {
final String hintText;
final VoidCallback onTap;
final VoidCallback? onFilterTap;
final VoidCallback? onRefreshTap;
const SakuraSearchBar({
super.key,
required this.hintText,
required this.onTap,
this.onFilterTap,
this.onRefreshTap,
});
@override
Widget build(BuildContext context) {
return Material(
elevation: 6,
borderRadius: BorderRadius.circular(32),
color: Theme.of(context).colorScheme.surface,
child: InkWell(
borderRadius: BorderRadius.circular(32),
onTap: onTap,
child: SizedBox(
height: 56,
child: Row(
children: [
const SizedBox(width: 18),
const Icon(Icons.search),
const SizedBox(width: 12),
Expanded(
child: Text(
hintText,
maxLines: 1,
overflow: TextOverflow.ellipsis,
style: Theme.of(context).textTheme.titleMedium,
),
),
if (onFilterTap != null)
IconButton(
icon: const Icon(Icons.tune),
onPressed: onFilterTap,
),
if (onRefreshTap != null)
IconButton(
icon: const Icon(Icons.refresh),
onPressed: onRefreshTap,
),
const SizedBox(width: 8),
],
),
),
),
);
}
}
页面顶部必须用:
SafeArea(
bottom: false,
child: Padding(
padding: const EdgeInsets.fromLTRB(16, 8, 16, 8),
child: SakuraSearchBar(
hintText: '搜索标题、作者或标签',
onTap: () {
Navigator.push(
context,
MaterialPageRoute(
builder: (_) => UnifiedSearchPage(
initialMode: SearchMode.manga,
),
),
);
},
onFilterTap: _openFilter,
onRefreshTap: _refresh,
),
),
)
禁止再用:
Stack + Positioned 顶部硬定位
AppBar 右上角塞在线搜索入口
状态栏高度手动写死
* * *
P1:图片 URL 合法性验证
===============
目标
--
不要相信 `1/17` 这种页数。它可能是:
17 个真实图片
17 个空字符串
17 个 loading 图
17 个 placeholder
17 个错误 URL
新增模型:
class ImageCandidate {
final String raw;
final String? url;
final bool isValid;
final String? reason;
const ImageCandidate({
required this.raw,
required this.url,
required this.isValid,
this.reason,
});
}
验证逻辑:
ImageCandidate validateImageCandidate(String raw, String baseUrl) {
final value = raw.trim();
if (value.isEmpty) {
return ImageCandidate(
raw: raw,
url: null,
isValid: false,
reason: 'empty',
);
}
if (value.startsWith('data:')) {
return ImageCandidate(
raw: raw,
url: null,
isValid: false,
reason: 'data-uri',
);
}
final lower = value.toLowerCase();
if (lower.contains('loading') ||
lower.contains('placeholder') ||
lower.contains('logo') ||
lower.contains('icon')) {
return ImageCandidate(
raw: raw,
url: value,
isValid: false,
reason: 'placeholder',
);
}
final normalized = normalizeImageUrl(value, baseUrl);
final uri = Uri.tryParse(normalized);
if (uri == null || !uri.hasScheme || uri.host.isEmpty) {
return ImageCandidate(
raw: raw,
url: normalized,
isValid: false,
reason: 'invalid-url',
);
}
return ImageCandidate(
raw: raw,
url: normalized,
isValid: true,
);
}
URL 归一化:
String normalizeImageUrl(String raw, String baseUrl) {
var u = raw.trim()
.replaceAll(r'\/', '/')
.replaceAll(r'\u002F', '/');
if (u.startsWith('//')) {
u = 'https:$u';
}
return Uri.parse(baseUrl).resolve(u).toString();
}
* * *
P2:PageImage + headers 拆分
=========================
问题
--
Legado 漫画源经常返回:
https://xxx.jpg,{"headers":{"Referer":"https://site.com"}}
如果直接丢给:
Image.network(raw)
图片必炸。
新模型
---
class PageImage {
final int index;
final String raw;
final String url;
final Map<String, String> headers;
const PageImage({
required this.index,
required this.raw,
required this.url,
required this.headers,
});
}
拆分逻辑
----
class ParsedImageSpec {
final String url;
final Map<String, String> headers;
ParsedImageSpec({
required this.url,
required this.headers,
});
}
ParsedImageSpec splitLegadoImageSpec(String raw) {
final idx = raw.indexOf(',{');
if (idx < 0) {
return ParsedImageSpec(url: raw.trim(), headers: {});
}
final url = raw.substring(0, idx).trim();
final jsonText = raw.substring(idx + 1).trim();
final headers = <String, String>{};
try {
final decoded = jsonDecode(jsonText);
final h = decoded['headers'];
if (h is Map) {
h.forEach((k, v) {
headers[k.toString()] = v.toString();
});
} else if (h is String) {
final inner = jsonDecode(h);
if (inner is Map) {
inner.forEach((k, v) {
headers[k.toString()] = v.toString();
});
}
}
} catch (_) {}
return ParsedImageSpec(url: url, headers: headers);
}
生成 PageImage:
PageImage buildPageImage({
required String raw,
required String chapterUrl,
required LegadoSource source,
required int index,
}) {
final parsed = splitLegadoImageSpec(raw);
final headers = <String, String>{
...source.headers,
'User-Agent': source.userAgent ?? defaultUserAgent,
'Referer': chapterUrl,
...parsed.headers,
};
return PageImage(
index: index,
raw: raw,
url: normalizeImageUrl(parsed.url, chapterUrl),
headers: headers,
);
}
阅读器改成:
Image.network(
page.url,
headers: page.headers,
fit: BoxFit.contain,
errorBuilder: (_, error, stack) {
return const Icon(Icons.broken_image);
},
)
* * *
P3:图片 HTTP 探测
=============
目标
--
判断到底是:
URL 错
headers 错
反爬
图片损坏
Flutter 缓存问题
新增:
class ImageProbeResult {
final String url;
final int? statusCode;
final String? contentType;
final int contentLength;
final String first16BytesHex;
final bool looksLikeImage;
final String? error;
const ImageProbeResult({
required this.url,
this.statusCode,
this.contentType,
required this.contentLength,
required this.first16BytesHex,
required this.looksLikeImage,
this.error,
});
}
探测:
Future<ImageProbeResult> probeImage(Dio dio, PageImage image) async {
try {
final resp = await dio.get<List<int>>(
image.url,
options: Options(
responseType: ResponseType.bytes,
headers: image.headers,
validateStatus: (_) => true,
),
);
final bytes = resp.data ?? const <int>[];
final first16 = bytes
.take(16)
.map((b) => b.toRadixString(16).padLeft(2, '0'))
.join();
return ImageProbeResult(
url: image.url,
statusCode: resp.statusCode,
contentType: resp.headers.value('content-type'),
contentLength: bytes.length,
first16BytesHex: first16,
looksLikeImage: isImageMagic(bytes),
);
} catch (e) {
return ImageProbeResult(
url: image.url,
contentLength: 0,
first16BytesHex: '',
looksLikeImage: false,
error: e.toString(),
);
}
}
魔数判断:
bool isImageMagic(List<int> b) {
if (b.length < 4) return false;
// JPG
if (b[0] == 0xFF && b[1] == 0xD8 && b[2] == 0xFF) return true;
// PNG
if (b[0] == 0x89 && b[1] == 0x50 && b[2] == 0x4E && b[3] == 0x47) {
return true;
}
// WEBP
if (b.length >= 12 &&
b[0] == 0x52 &&
b[1] == 0x49 &&
b[2] == 0x46 &&
b[3] == 0x46 &&
b[8] == 0x57 &&
b[9] == 0x45 &&
b[10] == 0x42 &&
b[11] == 0x50) {
return true;
}
// GIF
if (b[0] == 0x47 && b[1] == 0x49 && b[2] == 0x46 && b[3] == 0x38) {
return true;
}
// BMP
if (b[0] == 0x42 && b[1] == 0x4D) return true;
return false;
}
诊断表:
403 / 444
→ Referer / User-Agent / Cookie 问题
404
→ 图片 URL 拼错或章节页已失效
200 + text/html
→ 反爬页 / 登录页 / 错误页
200 + image/* + contentLength < 1024
→ 可能是伪装错误图或破损图
200 + image/* + 魔数失败
→ 响应不是有效图片
200 + image/* + 魔数成功但 Image.network 不显示
→ headers 没传进 Image.network / 图片缓存污染 / Flutter 图片组件问题
* * *
P4:TOC 分页抓全章节
=============
问题
--
一人之下 600+ 话,但现在只显示几十章,说明目录分页没有抓全。
支持字段
----
tocNextPageRule
nextTocUrl
nextUrl
nextPage
tocNextPage
抓取流程
----
firstTocUrl
→ 请求当前页
→ 解析 chapterList
→ 解析 nextTocUrl
→ resolve 成绝对 URL
→ 合并章节并去重
→ 继续下一页
终止条件
----
nextUrl == null
seenPageUrls 已访问
连续 2 页新增章节为 0
pageCount >= 100
chapterCount >= 3000
建议参数:
const maxTocPages = 100;
const maxTocChapters = 3000;
const emptyPageDelay = Duration(milliseconds: 500);
* * *
P5:章节排序支持番外
===========
支持:
第660话
第660.5回
Ch.660
ch 660.5
660话
660.5 番外
章节号提取:
double? extractChapterNo(String title) {
var m = RegExp(r'第\s*(\d+(?:\.\d+)?)\s*[回话章节节]').firstMatch(title);
if (m != null) return double.tryParse(m.group(1)!);
m = RegExp(r'[Cc]h\.?\s*(\d+(?:\.\d+)?)').firstMatch(title);
if (m != null) return double.tryParse(m.group(1)!);
m = RegExp(r'^\s*(\d+(?:\.\d+)?)\s*(?:话|回|章|节)?').firstMatch(title);
if (m != null) return double.tryParse(m.group(1)!);
return null;
}
排序:
chapters.sort((a, b) {
final na = extractChapterNo(a.title);
final nb = extractChapterNo(b.title);
if (na != null && nb != null && na != nb) {
return na.compareTo(nb);
}
return a.index.compareTo(b.index);
});
* * *
P6:Script 图片变量提取
================
当 `ruleContent.content` 失败或候选图片全无效时,从 rawHtml 的 script 中提取图片。
支持变量:
newImgs
imageList
page_url
chapterImages
photosr
img_data
picTree
示例:
List<String> extractImagesFromScripts(String html) {
final out = <String>[];
final patterns = [
RegExp(r'newImgs\s*=\s*(\[[\s\S]*?\])'),
RegExp(r'imageList\s*=\s*(\[[\s\S]*?\])'),
RegExp(r'"page_url"\s*:\s*(\[[\s\S]*?\])'),
RegExp(r'chapterImages\s*=\s*(\[[\s\S]*?\])'),
RegExp(r'photosr\s*=\s*(\[[\s\S]*?\])'),
RegExp(r'img_data\s*=\s*(\[[\s\S]*?\])'),
RegExp(r'picTree\s*=\s*(\[[\s\S]*?\])'),
];
for (final p in patterns) {
final m = p.firstMatch(html);
if (m == null) continue;
final raw = m.group(1)!;
final list = tryDecodeStringList(raw);
out.addAll(list.map(sanitizeScriptUrl));
}
return out;
}
清洗:
String sanitizeScriptUrl(String url) {
return url
.replaceAll(r'\/', '/')
.replaceAll(r'\u002F', '/');
}
* * *
P7:漫画调试页
========
入口
--
书籍详情页 → 长按章节 → 调试本章
阅读器右上角菜单 → 调试当前章
展示内容
----
{
"sourceName": "包子漫画5",
"bookTitle": "一人之下",
"chapterTitle": "第652回",
"chapterUrl": "...",
"ruleContent": "...",
"rawHtmlLength": 123456,
"contentOutputLength": 2048,
"rawCandidateCount": 17,
"validCandidateCount": 0,
"pageImageCount": 0,
"firstCandidates": [
{
"raw": "...",
"normalized": "...",
"reason": "placeholder"
}
],
"probe": [
{
"url": "...",
"status": 200,
"contentType": "image/jpeg",
"contentLength": 312,
"first16Bytes": "ffd8ff...",
"looksLikeImage": false
}
]
}
操作按钮
----
复制调试信息
重新检测前 5 张图
切换 Referer:章节页 / 详情页 / 源首页
清除图片缓存后重试
打开原始章节 URL
* * *
P8:搜索健康分与增量搜索
=============
问题
--
搜索慢主要来自:
坏源太多
每个源 timeout 太长
所有源一起等结果
模型
--
class SourceHealth {
final String sourceId;
int successCount = 0;
int failCount = 0;
int timeoutCount = 0;
int totalLatencyMs = 0;
int consecutiveFails = 0;
DateTime? lastSuccessAt;
DateTime? lastFailAt;
}
策略
--
健康源优先
坏源降权
连续失败 >= 5 临时跳过
搜索超时 5~8 秒
快速源先返回
慢源后台补充
搜索阶段
----
0~5s:快速源结果
5~8s:普通源结果
8s 后:慢源后台补
* * *
三、最终落地顺序
========
第 1 阶段:UI 先止血
-------------
P0:顶部搜索框 UI 修复
P0:在线搜索入口放进统一搜索页
结果:
漫画页 / 图片页顶部不再错位
在线搜索不再占右上角
搜索体验统一
第 2 阶段:图片链路可观测
--------------
P1:图片 URL 合法性验证
P2:PageImage + headers 拆分
P3:图片 HTTP 探测
P7:漫画调试页
结果:
可以准确判断图片失败原因
不再靠猜
第 3 阶段:章节抓全
-----------
P4:TOC 分页
P5:章节排序
结果:
一人之下这类 600+ 话漫画可以抓全
章节顺序稳定
第 4 阶段:提升漫画源覆盖率
---------------
P6:Script 图片变量提取
结果:
更多漫画源能解析图片
第 5 阶段:优化搜索速度
-------------
P8:搜索健康分与增量搜索
结果:
坏源不拖慢搜索
结果更快出现
* * *
=======
@@ -0,0 +1,70 @@
# 视频云同步与内容合集设计方案
## 目标
让 X、小红书、抖音采集到的视频和图片一样进入 Sakura Cloud,并能在手机端与 Web 端查看。云端对每个用户的视频总占用做 30GB 上限,Web 端从“后台列表”调整为适合阅读、看视频、浏览插画的内容合集。
## 云端设计
- 复用现有 `upload / sync / gallery / files` 链路,不新增数据库表,不改变 API 返回结构。
- 视频仍作为 `Artwork` 保存,`content_type=video`,`mime_type=video/*`,文件实体复用 `FileBlob` 与 `UserFile`。
- 上传接口允许 `resource_type=video`,视频文件进入 `uploads/videos/{user_id}/{year}/{month}`。
- 新增配置:
- `MAX_VIDEO_UPLOAD_SIZE_MB=2048`:单个视频默认最大 2GB。
- `VIDEO_STORAGE_LIMIT_GB=30`:单用户视频总空间上限 30GB。
- 配额计算以当前登录用户为准,从 `UserFile -> FileBlob` 统计该用户所有 `mime_type LIKE 'video/%'` 的文件大小。命中上限时返回 `413`,不会创建上传会话。
- 已存在文件秒传时同样检查该用户的视频总配额,防止跨用户复用文件绕过限制。
## 桌面端采集同步设计
- `SyncManager` 根据 MIME 与 `content_type` 判断上传资源类型:
- 小说:`novel`
- 视频:`video`
- 其他:`image`
- X 手动预览下载和关注作者同步共用云端同步逻辑,视频文件下载成功后自动同步 metadata。
- WebUI 通用插件下载路径也接入云同步,因此小红书、抖音插件只要返回 `files=[{type: "video"}]`,就能自动上传云端。
- 小红书插件补充 DOM 视频解析,支持从 `video/source` 标签提取直接视频地址;搜索结果进入详情下载时也同时处理 `images + videos`。
- 抖音插件已有视频文件抽取与下载逻辑,本方案复用其 `Resource.files` 输出,不重写下载器。
## 手机端设计
- `Artwork` 模型补充 `mimeType`、`fileSize`、`isVideo` 与 `primaryMediaPath`。
- 图库页新增视频筛选,视频卡片显示播放叠层,不再伪装成普通插画。
- 视频详情页展示视频封面、标题、作者、MIME、大小,并提供:
- 播放视频
- 下载到本地
- 收藏
- 为避免新增播放器依赖导致构建不稳定,当前采用“受保护下载到本地后调用系统播放器”的方式。
- Android 通过 FileProvider 暴露 Sakura 下载目录中的临时读取 URI,不暴露任意路径。
## Web 端内容合集设计
- 普通用户首页改为内容合集:
- 顶部概览:用户、空间、视频额度、最近内容。
- 视频区:16:9 视频卡片,播放标记,显示作者与大小。
- 插画区:缩略图网格。
- 漫画区:阅读列表入口。
- 小说区:阅读列表入口。
- 在线搜索与提醒中心作为主操作入口。
- 页面复用现有 API:
- `/me/gallery?content_type=video`
- `/me/gallery?content_type=illust`
- `/me/gallery?content_type=manga`
- `/novels/series`
- CSS 使用现有粉白主题、8px 圆角和响应式网格,手机浏览时自动改为单列。
## 安全与边界
- 不上传第三方平台 Cookie、Token 或浏览器登录态。
- 云端只保存视频文件与作品 metadata。
- 视频配额按当前登录用户统计,避免水平越权。
- 文件读取继续走现有 `/files` 和 gallery ownership 校验。
- Android FileProvider 只给外部播放器一次性读取 URI,不暴露数据库或任意文件系统路径。
## 验收点
- X、小红书、抖音插件下载视频后,云同步开启时视频可进入 cloud gallery。
- 超过 30GB 视频总配额时,上传初始化返回 413。
- 手机端图库可以筛选视频,视频详情可以下载并调用系统播放器查看。
- Web 普通用户首页能看到视频、插画、漫画、小说四类内容入口。
- 现有图片、漫画、小说同步与阅读入口保持可用。
+322
View File
@@ -0,0 +1,322 @@
# Sakura Downloader(樱云下载器)项目方案书
> 版本:v1.0 | 日期:2026-06-18 | 状态:已交付
---
## 一、项目概述
### 1.1 项目名称
**Sakura Downloader(樱云下载器)** — 个人媒体采集、本地图库管理与跨设备云同步系统。
### 1.2 项目定位
一个面向个人用户的**多媒体内容采集-整理-阅读一体化平台**,解决"从多个内容平台采集插画/漫画/小说 → 本地归档整理 → 在手机/桌面等多设备上随时浏览阅读"的完整链路。
### 1.3 核心价值
| 痛点 | 解决方案 |
|------|----------|
| 多平台内容分散(Pixiv、X、小红书、AO3、笔趣阁等) | 统一插件架构,一个入口采集所有平台 |
| 下载后文件散乱、难以检索 | 本地图库 + SQLite 索引 + 缩略图墙 + 标签/作者搜索 |
| 桌面下载了,手机上没法看 | Sakura Cloud 云端同步,手机端实时访问 |
| 关注的作者更新了不知道 | 云端每日摘要提醒,手机端查看 |
| 跨平台小说阅读体验差 | Flutter 移动端专为阅读优化:自动分页、进度记忆、翻译、字体调节 |
---
## 二、系统架构
### 2.1 整体拓扑
```text
┌─────────────────────────────────────────────────────────┐
│ Sakura Cloud (FastAPI) │
│ PostgreSQL + JWT Auth │
│ api.kunkunxiaomao.top:8000 │
└──────┬──────────────────────┬───────────────────────────┘
│ REST API │ REST API
▼ ▼
┌──────────────┐ ┌──────────────────┐
│ Windows 桌面端 │ │ Android 移动端 │
│ Python + │ │ Flutter + │
│ CustomTkinter │ │ Riverpod + Dio │
│ (采集/整理中枢) │ │ (阅读/提醒终端) │
└──────┬─────────┘ └──────────────────┘
│ 本地 SQLite │
▼ ▼
┌──────────────┐ ┌──────────────┐
│ 本地图库文件 │ │ 本地缓存/下载 │
│ (图片/漫画/小说)│ │ (离线阅读) │
└──────────────┘ └──────────────┘
```
### 2.2 技术栈总览
| 层级 | 技术选型 |
|------|----------|
| **Windows 桌面端** | Python 3.8+ · CustomTkinter · SQLite · Playwright · Pillow |
| **云端 API** | FastAPI · PostgreSQL · SQLAlchemy · Alembic · JWT · Docker |
| **Android 移动端** | Flutter 3.x · Dart · Riverpod · Dio · shared_preferences |
| **管理后台 Web** | Vite · TypeScript · Vue/React · Nginx |
| **插件系统** | Python 动态加载 · manifest.json · 统一 PluginBase 接口 |
| **打包分发** | PyInstaller (EXE) · Flutter build (APK) |
---
## 三、客户端种类与职责
### 3.1 Windows 桌面端(`pixiv_app/` + `main.py`)
**角色:采集与整理中枢**
| 功能模块 | 说明 |
|----------|------|
| **多平台采集** | 通过插件系统支持 Pixiv、X (Twitter)、小红书、AO3、笔趣阁等 |
| **Cookie/密码登录** | 支持 Cookie 导入(推荐)和账号密码登录 |
| **灵活抓取模式** | 作者全部作品、单作品/漫画、小说/系列、关键词搜索 |
| **本地图库** | SQLite 索引 + 文件系统存储,支持缩略图生成 |
| **缩略图墙** | 本地 Web 服务(127.0.0.1:8765),浏览器中浏览、搜索、筛选、收藏 |
| **关注同步** | Pixiv 关注作者管理 + X 关注同步 |
| **云同步** | 一键上传到 Sakura Cloud,自动同步新下载内容 |
| **代理池** | 浏览器代理池 + HTTP 代理,规避反爬 |
| **Playwright 集成** | 无头浏览器渲染 JS 页面(X 插件依赖) |
| **指纹伪装** | 浏览器指纹随机化,降低封禁风险 |
**打包产物**:`sakuradownloader.exe` (~68 MB, PyInstaller 单文件)
### 3.2 Sakura Cloud 云端(`sakura_cloud/`)
**角色:数据与账号中心**
| 功能模块 | 说明 |
|----------|------|
| **用户系统** | 注册/登录/邮箱验证/JWT Token 鉴权 |
| **媒体存储** | 图片/漫画/小说上传、存储、缩略图生成 |
| **收藏系统** | 跨设备收藏同步 |
| **关注与提醒** | 关注作者管理 + 每日更新摘要提醒 |
| **阅读进度** | 小说阅读进度云端存储,跨设备同步 |
| **管理后台** | SPA 管理控制台(可选) |
| **自动运维** | 启动自动 migrate、定时清理过期上传、每 5 分钟检查提醒摘要 |
**部署方式**:Docker Compose(FastAPI + PostgreSQL)
### 3.3 Android 移动端(`sakura_mobile/`)
**角色:跨设备阅读与提醒终端**
| 功能模块 | 说明 |
|----------|------|
| **发现页** | 最近浏览 + 更新提醒摘要 |
| **图片页** | 缩略图列表 · 详情 · 收藏 · 下载 · 编辑 · 删除 |
| **漫画页** | 按漫画分组浏览,整组连续阅读,整组下载 |
| **书架页** | 三列彩色方块风格,系列小说按书名归档,章节选择 |
| **小说阅读器** | 自动分页 · 阅读进度记忆 · 收藏 · 当前页翻译 · 原文/译文切换 · 字体/行高/背景色调节 · 章节切换 |
| **我的页** | 用户资料 · 云端空间 · 收藏管理 · 每日提醒开关/时间 · 提醒中心 |
| **搜索** | 图片/漫画/小说全局搜索 |
**打包产物**:`app-release.apk` (~54 MB, 调试签名)
### 3.4 管理后台 Web(`sakura_admin_web/`)
**角色:云端管理控制台**
- 基于 Vite + TypeScript 的 SPA 单页应用
- 通过 Sakura Cloud 的 `/admin` 路由挂载
- 提供可视化的数据和用户管理界面
- 可选部署(`admin_frontend_enabled` 配置项)
---
## 四、插件体系
### 4.1 插件架构
```text
plugins/
├── pixiv/ # Pixiv 插画/漫画/小说
│ ├── manifest.json
│ └── plugin.py
├── x/ # X (Twitter) 媒体
│ ├── manifest.json
│ ├── plugin.py
│ ├── playwright_client.py
│ └── x_login.py
├── xiaohongshu/ # 小红书
├── ao3/ # Archive of Our Own
└── biquge/ # 笔趣阁(小说)
```
### 4.2 插件接口规范
每个插件只需实现 `PluginBase` 抽象类,包含:
- `platform_name` — 平台标识
- `supports_modes` — 支持的抓取模式列表
- `search()` — 搜索
- `fetch_work()` — 获取作品详情
- `fetch_author_works()` — 获取作者全部作品
- `download()` — 下载媒体文件
通过 `manifest.json` 声明元数据,桌面端启动时自动发现并加载。
---
## 五、核心数据流
### 5.1 采集下载流程
```text
用户输入(ID/链接/关键词)
│
▼
任务解析器 (parser.py)
│
▼
任务队列 (queue.py)
│
▼
插件调度 → 对应平台 plugin.py
│
▼
HTTP/Playwright 抓取 → 解析作品列表
│
▼
下载器 (downloader.py) → 保存到本地图库
│
▼
SQLite 索引写入 + 缩略图生成
│
▼
(可选) 云同步上传到 Sakura Cloud
```
### 5.2 云同步流程
```text
桌面端下载完成
│
▼
检查"启用云同步"开关
│
▼
cloud_sync.py → POST /api/v1/media/upload
│
▼
Sakura Cloud 存储 + 生成缩略图
│
▼
移动端刷新 → GET /api/v1/media → 展示
```
### 5.3 每日提醒流程
```text
桌面端同步关注作者的新作品到云端
│
▼
云端写入 follow + update_alert 记录
│
▼
reminder_digest_loop (每5分钟检查)
│
▼
到达用户设定的提醒时间 → 生成每日摘要
│
▼
移动端"发现"页展示更新提醒
```
---
## 六、用法概览
### 6.1 桌面端典型工作流
1. **登录** → 导入 Pixiv Cookie(推荐)或账号密码登录
2. **抓取** → 选择平台 → 选择模式 → 输入目标 → 开始抓取
3. **浏览** → 打开缩略图墙 → 搜索/筛选/收藏
4. **关注** → 在缩略图墙中关注作者 → 后续同步关注作者更新
5. **云同步** → 登录 Sakura Cloud → 开启云同步 → 自动上传
### 6.2 移动端典型工作流
1. **安装** → 安装 APK → 允许未知来源
2. **登录** → 使用与桌面端相同的 Sakura Cloud 账号
3. **浏览** → 底部 5 个页签切换(发现/图片/漫画/书架/我的)
4. **阅读小说** → 自动分页 · 调节字体 · 翻译 · 章节切换
5. **提醒** → 我的 → 每日更新提醒 → 开启并设置时间
### 6.3 云端部署
```bash
cd sakura_cloud
docker-compose up -d
# FastAPI :8000 + PostgreSQL :5432
```
### 6.4 构建打包
```bash
# 桌面 EXE
scripts\build_desktop_exe.bat
# Android APK
scripts\build_mobile_android.bat
# 复制到 releases/
scripts\copy_release_artifacts.bat
```
---
## 七、项目交付物
| 交付物 | 路径 | 说明 |
|--------|------|------|
| Windows EXE | `releases/sakuradownloader.exe` | 双击即用,~68 MB |
| Android APK | `releases/app-release.apk` | 调试签名,~54 MB |
| 用户使用说明 | `使用说明-桌面版与安卓版.md` | 终端用户操作手册 |
| 项目方案书 | `docs/项目方案书.md` | 本文档 |
| 开发文档 | `项目文件/BUILD.md` | 构建与排错 |
| 构建脚本 | `scripts/` | 一键打包脚本 |
| 云端 API | `sakura_cloud/` | Docker 部署 |
| 源码 | 全部 Python/Dart/TypeScript | 可二次开发 |
---
## 八、技术亮点
1. **插件化架构** — 新平台接入只需新增一个插件目录,不改动核心代码
2. **多反爬策略** — Cookie 管理 · Playwright 无头浏览器 · 代理池 · 指纹伪装 · 限速退避
3. **跨设备云同步** — 桌面采集 → 云端存储 → 手机阅读,全链路打通
4. **沉浸式阅读体验** — 自动分页 · 进度记忆 · 即时翻译 · 个性化排版
5. **智能提醒** — 关注作者更新云端汇总,非本地闹钟,静默不打扰
6. **本地优先** — 所有数据本地 SQLite 存储,云端为可选项,离线也能用
---
## 九、后续演进方向
| 方向 | 说明 |
|------|------|
| iOS 客户端 | Flutter 已预留 iOS 平台支持,补充苹果开发者账号与签名 |
| 正式应用签名 | APK 替换调试签名为正式 keystore,支持上架分发 |
| 更多平台插件 | 持续扩充插件库(如 Fantia、Fanbox、DeviantArt 等) |
| AI 能力增强 | OCR 文字识别(代码已预留)· AI 翻译优化 · 智能标签 |
| 分布式爬取 | `distributed_crawler/` 目录已有 Master-Worker 架构原型 |
| PWA 移动端 | 缩略图墙升级为 PWA,支持手机浏览器直接访问 |
---
## 十、许可与免责
- 本项目仅供个人学习、研究及合法使用。
- 使用 X 插件需自行承担对应平台的使用条款风险。
- APK 当前为调试签名,仅供个人测试,不宜正式分发。
- 请遵守各内容平台的用户协议,合理使用爬取功能。
---
*文档生成于 2026-06-18 · Sakura Downloader Project*
+10
View File
@@ -0,0 +1,10 @@
from pixiv_app.gui.app import SakuraDownloaderGUI
def main() -> None:
app = SakuraDownloaderGUI()
app.run()
if __name__ == "__main__":
main()
+1
View File
@@ -0,0 +1 @@
"""Sakura downloader application package."""
+1
View File
@@ -0,0 +1 @@
"""Core business logic for Sakura downloader."""
+167
View File
@@ -0,0 +1,167 @@
from __future__ import annotations
import html
import json
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import urlencode
import requests
from pixiv_app.core.downloader import USER_AGENT, build_headers
LOGIN_PAGE_URL = "https://accounts.pixiv.net/login"
LOGIN_AJAX_URL = "https://accounts.pixiv.net/ajax/login?lang=zh"
VALIDATE_URL = "https://www.pixiv.net/setting_user.php"
@dataclass
class AuthResult:
success: bool
message: str
cookie: str = ""
requires_verification: bool = False
user_name: str = ""
user_id: str = ""
class SessionStore:
def __init__(self, path: str | Path = "pixiv_app_session.json") -> None:
self.path = Path(path)
def load(self) -> dict[str, Any]:
if not self.path.exists():
return {}
try:
return json.loads(self.path.read_text(encoding="utf-8"))
except Exception:
return {}
def save(self, payload: dict[str, Any]) -> None:
self.path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
class PixivAuthClient:
def __init__(self) -> None:
self.timeout = 20
def validate_cookie(self, cookie: str) -> AuthResult:
cookie = cookie.strip()
if not cookie:
return AuthResult(False, "Cookie 为空,请先填写或登录。")
session = requests.Session()
session.headers.update(build_headers(cookie=cookie, referer="https://www.pixiv.net/"))
try:
response = session.get(VALIDATE_URL, allow_redirects=True, timeout=self.timeout)
final_url = response.url
if "return_to=" in final_url or "accounts.pixiv.net" in final_url:
return AuthResult(False, "当前 Cookie 已失效或未登录,请更新会话。")
user_id, user_name = self._extract_user_info(response.text)
return AuthResult(True, "Cookie 有效,可直接开始抓取。", cookie=cookie, user_name=user_name, user_id=user_id)
except Exception as exc:
return AuthResult(False, f"Cookie 检测失败: {exc}")
def login_with_password(self, login_id: str, password: str) -> AuthResult:
login_id = login_id.strip()
password = password.strip()
if not login_id or not password:
return AuthResult(False, "账号和密码都不能为空。")
session = requests.Session()
session.headers.update(
{
"User-Agent": USER_AGENT,
"Accept": "application/json, text/plain, */*",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Origin": "https://accounts.pixiv.net",
"Referer": LOGIN_PAGE_URL,
"X-Requested-With": "XMLHttpRequest",
}
)
try:
login_page = session.get(LOGIN_PAGE_URL, timeout=self.timeout)
login_page.raise_for_status()
tt = self._extract_tt(login_page.text)
payload = {
"login_id": login_id,
"password": password,
"source": "accounts",
"app_ios": "0",
"ref": "",
"return_to": "https://www.pixiv.net/",
"g_recaptcha_response": "",
"recaptcha_enterprise_score_token": "",
}
headers = dict(session.headers)
if tt:
headers["X-CSRF-Token"] = tt
response = session.post(
LOGIN_AJAX_URL,
data=urlencode(payload),
headers={**headers, "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"},
timeout=self.timeout,
)
response.raise_for_status()
body = response.json()
if isinstance(body, dict) and body.get("body", {}).get("success"):
cookie = self._cookies_to_header(session.cookies)
validation = self.validate_cookie(cookie)
if validation.success:
validation.message = "账号密码登录成功,会话已可用。"
return validation
return AuthResult(True, "登录接口返回成功,但会话校验未完全通过,请手动检测。", cookie=cookie)
body_payload = body.get("body", {}) if isinstance(body, dict) else {}
errors = body_payload.get("errors", {}) if isinstance(body_payload, dict) else {}
if errors.get("recaptcha") or body_payload.get("requireExtraVerification") or body_payload.get("requireTwoFactorAuthentication"):
return AuthResult(
False,
"Pixiv 要求额外验证或二步验证,程序无法静默完成。请改用 Cookie 登录,或先在浏览器登录后导出 Cookie。",
requires_verification=True,
)
message = self._flatten_error_message(errors) or "账号密码登录失败,请检查账号、密码或改用 Cookie。"
return AuthResult(False, message)
except Exception as exc:
return AuthResult(False, f"账号密码登录失败: {exc}")
def _extract_tt(self, html_text: str) -> str:
match = re.search(r'"pixivAccount\.tt":"([^"]+)"', html_text)
return match.group(1) if match else ""
def _extract_user_info(self, html_text: str) -> tuple[str, str]:
user_id = ""
user_name = ""
patterns = [
r'"userData":\{"id":"(\d+)","pixivId":"[^"]*","name":"([^"]+)"',
r'"user_id":"(\d+)".*?"name":"([^"]+)"',
]
for pattern in patterns:
match = re.search(pattern, html_text)
if match:
user_id = match.group(1)
user_name = html.unescape(match.group(2))
break
return user_id, user_name
def _flatten_error_message(self, errors: dict[str, Any]) -> str:
messages: list[str] = []
for key, value in errors.items():
if isinstance(value, str) and value.strip():
messages.append(f"{key}: {value}")
elif isinstance(value, list):
text_items = [str(item).strip() for item in value if str(item).strip()]
if text_items:
messages.append(f"{key}: {'; '.join(text_items)}")
return " | ".join(messages)
def _cookies_to_header(self, jar: requests.cookies.RequestsCookieJar) -> str:
parts = [f"{cookie.name}={cookie.value}" for cookie in jar]
return "; ".join(parts)
+353
View File
@@ -0,0 +1,353 @@
"""
Playwright 浏览器代理池 - 支持动态换 IP
功能:
- 多代理轮换(轮询/随机/会话粘性)
- 健康检查(自动剔除失效代理)
- 延迟统计(加权选择最优代理)
- 线程安全
"""
import random
import threading
import time
from dataclasses import dataclass, field
from typing import Optional, List, Dict, Any
from enum import Enum
class RotationStrategy(Enum):
"""轮换策略"""
ROUND_ROBIN = "round_robin" # 轮询
RANDOM = "random" # 随机
WEIGHTED = "weighted" # 加权(按成功率+延迟)
SESSION_STICKY = "session_sticky" # 会话粘性(同会话同IP)
@dataclass
class ProxyInfo:
"""代理信息"""
server: str # http://ip:port 或 socks5://ip:port
username: str = ""
password: str = ""
# 统计数据
success_count: int = 0
fail_count: int = 0
total_latency_ms: float = 0.0
avg_latency_ms: float = 0.0
last_check_time: float = 0.0
last_fail_time: float = 0.0
# 状态
healthy: bool = True
cooldown_until: float = 0.0 # 冷却结束时间
@property
def proxy_url(self) -> str:
"""获取完整的代理URL(含认证)"""
if self.username and self.password:
# http://user:pass@ip:port
protocol = self.server.split("://")[0]
rest = self.server.split("://")[1]
return f"{protocol}://{self.username}:{self.password}@{rest}"
return self.server
def to_requests_proxies(self) -> dict[str, str]:
return {"http": self.proxy_url, "https": self.proxy_url}
def to_playwright_proxy(self) -> dict[str, str]:
data = {"server": self.server}
if self.username:
data["username"] = self.username
data["password"] = self.password
return data
@property
def success_rate(self) -> float:
"""成功率"""
total = self.success_count + self.fail_count
if total == 0:
return 1.0
return self.success_count / total
@property
def weight(self) -> float:
"""
计算代理权重(用于加权随机选择)
考虑因素:成功率、延迟、冷却状态
"""
if not self.healthy:
return 0.0
# 冷却中
if self.cooldown_until > time.time():
return 0.0
# 成功率权重
success_weight = self.success_rate
# 延迟权重(延迟越低权重越高)
if self.avg_latency_ms > 0:
latency_weight = max(0.1, 1000.0 / max(self.avg_latency_ms, 1))
else:
latency_weight = 1.0
# 失败惩罚
fail_penalty = 1.0 / (1.0 + self.fail_count * 0.5)
return success_weight * latency_weight * fail_penalty
class ProxyPool:
"""代理池 - 管理多个代理的获取和轮换"""
def __init__(
self,
strategy: RotationStrategy = RotationStrategy.WEIGHTED,
health_check_url: str = "https://httpbin.org/ip",
health_check_timeout: int = 8,
max_failures: int = 3,
cooldown_seconds: int = 60,
auto_health_check: bool = True,
):
"""
Args:
strategy: 轮换策略
health_check_url: 健康检查URL
health_check_timeout: 健康检查超时(秒)
max_failures: 最大失败次数,超过后标记为不健康
cooldown_seconds: 不健康代理的冷却时间(秒)
auto_health_check: 是否自动健康检查
"""
self.strategy = strategy
self.health_check_url = health_check_url
self.health_check_timeout = health_check_timeout
self.max_failures = max_failures
self.cooldown_seconds = cooldown_seconds
self.auto_health_check = auto_health_check
self._proxies: List[ProxyInfo] = []
self._session_proxy_map: Dict[str, ProxyInfo] = {} # 会话粘性用
self._round_robin_index = 0
self._lock = threading.Lock()
def add_proxy(self, server: str, username: str = "", password: str = "") -> None:
"""添加单个代理"""
with self._lock:
# 去重
existing = {p.server for p in self._proxies}
if server not in existing:
self._proxies.append(ProxyInfo(
server=server,
username=username,
password=password,
))
def add_proxy_url(self, value: str) -> None:
parsed = parse_proxy_url(value)
if parsed is not None:
self.add_proxy(parsed.server, parsed.username, parsed.password)
def add_proxies(self, proxies: List[dict]) -> None:
"""批量添加代理
Args:
proxies: [{"server": "http://1.2.3.4:8080", "username": "", "password": ""}, ...]
"""
for p in proxies:
self.add_proxy(
server=p["server"],
username=p.get("username", ""),
password=p.get("password", ""),
)
def add_proxy_text(self, text: str) -> int:
count = 0
for line in text.splitlines():
parsed = parse_proxy_url(line)
if parsed is None:
continue
self.add_proxy(parsed.server, parsed.username, parsed.password)
count += 1
return count
def remove_proxy(self, server: str) -> None:
"""移除代理"""
with self._lock:
self._proxies = [p for p in self._proxies if p.server != server]
def get_proxy(self, session_id: str = None) -> Optional[ProxyInfo]:
"""
获取一个代理
Args:
session_id: 会话ID,用于SESSION_STICKY策略
Returns:
代理信息,如果没有可用代理返回None
"""
with self._lock:
# 过滤出健康的代理
now = time.time()
healthy = [p for p in self._proxies if p.healthy and p.cooldown_until <= now]
if not healthy:
# 尝试从冷却中恢复
now = time.time()
recovered = []
for p in self._proxies:
if not p.healthy and p.cooldown_until <= now:
p.healthy = True
p.fail_count = 0
recovered.append(p)
if recovered:
healthy = recovered
if not healthy:
return None
# 根据策略选择代理
if self.strategy == RotationStrategy.RANDOM:
return random.choice(healthy)
elif self.strategy == RotationStrategy.WEIGHTED:
# 加权随机选择
weights = [p.weight for p in healthy]
total_weight = sum(weights)
if total_weight <= 0:
return random.choice(healthy)
r = random.uniform(0, total_weight)
cumsum = 0
for i, w in enumerate(weights):
cumsum += w
if r <= cumsum:
return healthy[i]
return healthy[0]
elif self.strategy == RotationStrategy.SESSION_STICKY:
if session_id and session_id in self._session_proxy_map:
proxy = self._session_proxy_map[session_id]
if proxy in healthy:
return proxy
# 新会话,分配一个代理
proxy = self._get_round_robin(healthy)
if session_id:
self._session_proxy_map[session_id] = proxy
return proxy
else: # ROUND_ROBIN
return self._get_round_robin(healthy)
def _get_round_robin(self, proxies: List[ProxyInfo]) -> ProxyInfo:
"""轮询获取代理"""
proxy = proxies[self._round_robin_index % len(proxies)]
self._round_robin_index += 1
return proxy
def mark_success(self, proxy: ProxyInfo, latency_ms: float = 0) -> None:
"""标记代理请求成功"""
if proxy is None:
return
with self._lock:
proxy.success_count += 1
proxy.fail_count = 0
proxy.healthy = True
if latency_ms > 0:
# 更新平均延迟(指数移动平均)
if proxy.avg_latency_ms == 0:
proxy.avg_latency_ms = latency_ms
else:
proxy.avg_latency_ms = proxy.avg_latency_ms * 0.7 + latency_ms * 0.3
proxy.last_check_time = time.time()
def mark_failed(self, proxy: ProxyInfo) -> None:
"""标记代理请求失败"""
if proxy is None:
return
with self._lock:
proxy.fail_count += 1
if proxy.fail_count >= self.max_failures:
proxy.healthy = False
proxy.cooldown_until = time.time() + self.cooldown_seconds
def health_check(self, proxy: ProxyInfo = None) -> bool:
"""
健康检查单个或所有代理
Returns:
如果检查通过返回True
"""
import requests
proxies_to_check = [proxy] if proxy else self._proxies
for p in proxies_to_check:
if p is None:
continue
try:
start = time.time()
resp = requests.get(
self.health_check_url,
proxies={"http": p.proxy_url, "https": p.proxy_url},
timeout=self.health_check_timeout,
)
if resp.status_code == 200:
latency_ms = (time.time() - start) * 1000
self.mark_success(p, latency_ms)
else:
self.mark_failed(p)
except Exception:
self.mark_failed(p)
return proxy.healthy if proxy else len(self.get_healthy_proxies()) > 0
def get_healthy_proxies(self) -> List[ProxyInfo]:
"""获取所有健康代理"""
with self._lock:
return [p for p in self._proxies if p.healthy]
def get_stats(self) -> dict:
"""获取代理池统计信息"""
with self._lock:
healthy = [p for p in self._proxies if p.healthy]
return {
"total": len(self._proxies),
"healthy": len(healthy),
"unhealthy": len(self._proxies) - len(healthy),
"proxies": [
{
"server": p.server,
"healthy": p.healthy,
"success_rate": p.success_rate,
"avg_latency_ms": round(p.avg_latency_ms, 2),
"fail_count": p.fail_count,
}
for p in self._proxies
]
}
def release_session(self, session_id: str) -> None:
"""释放会话绑定的代理"""
with self._lock:
self._session_proxy_map.pop(session_id, None)
def parse_proxy_url(value: str) -> Optional[ProxyInfo]:
text = value.strip()
if not text or text.startswith("#"):
return None
if "://" not in text:
text = "http://" + text
import re
match = re.match(r"^(https?|socks5)://(?:(?P<user>[^:@/]+):(?P<pwd>[^@/]+)@)?(?P<host>[\w.\-]+):(?P<port>\d{2,5})$", text, re.I)
if not match:
return None
scheme = text.split("://", 1)[0].lower()
host = match.group("host")
port = match.group("port")
user = match.group("user") or ""
pwd = match.group("pwd") or ""
return ProxyInfo(server=f"{scheme}://{host}:{port}", username=user, password=pwd)
+602
View File
@@ -0,0 +1,602 @@
from __future__ import annotations
import base64
import hashlib
import json
import logging
import threading
import time
from collections.abc import Callable
from dataclasses import asdict
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import requests
from pixiv_app.core.paths import runtime_path
logger = logging.getLogger(__name__)
def file_sha256(path: str | Path) -> str:
digest = hashlib.sha256()
with Path(path).open("rb") as fp:
for chunk in iter(lambda: fp.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def persist_cloud_session_tokens(access_token: str, refresh_token: str = "") -> None:
"""将刷新后的 Token 写回桌面会话文件,供 GUI 与 Gallery API 共用。"""
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.paths import app_session_file
store = SessionStore(app_session_file())
payload = store.load()
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
cloud["access_token"] = access_token
if refresh_token:
cloud["refresh_token"] = refresh_token
payload["cloud_sync"] = cloud
store.save(payload)
def _decode_jwt_exp(token: str) -> float:
"""解码 JWT 的 exp 字段,返回 Unix 时间戳;解码失败返回 0。"""
if not token or "." not in token:
return 0.0
try:
payload = token.split(".")[1]
payload += "=" * (4 - len(payload) % 4)
decoded = json.loads(base64.urlsafe_b64decode(payload))
return float(decoded.get("exp", 0))
except Exception:
return 0.0
class CloudSession:
"""全局单例,统一管理云同步 Token 的读写、刷新与持久化。
所有需要访问 Sakura Cloud API 的模块(GUI、Gallery、FollowSync、XMedia)
都通过此单例获取配置和创建客户端,确保 token 刷新互不冲突、变更全局可见。
"""
_instance: CloudSession | None = None
_instance_lock = threading.Lock()
def __new__(cls) -> CloudSession:
if cls._instance is None:
with cls._instance_lock:
if cls._instance is None:
instance = super().__new__(cls)
instance._base_url = ""
instance._access_token = ""
instance._refresh_token = ""
instance._enabled = False
instance._refresh_lock = threading.Lock()
instance._listeners: list[Callable[[str, str], None]] = []
instance._last_refresh_time = 0.0
cls._instance = instance
return cls._instance
# ------------------------------------------------------------------
# 配置入口
# ------------------------------------------------------------------
def configure(
self,
*,
base_url: str = "",
access_token: str = "",
refresh_token: str = "",
enabled: bool = False,
) -> None:
"""GUI 启动 / 登录 / 保存会话时调用,同步最新 token 到全局单例。"""
self._base_url = (base_url or "").strip()
self._access_token = (access_token or "").strip()
self._refresh_token = (refresh_token or "").strip()
self._enabled = bool(enabled)
def load_from_disk(self) -> dict[str, Any]:
"""从磁盘会话文件读取云端配置并应用到单例,返回原始 payload。"""
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.paths import app_session_file
store = SessionStore(app_session_file())
payload = store.load()
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
self.configure(
base_url=str(cloud.get("base_url") or ""),
access_token=str(cloud.get("access_token") or ""),
refresh_token=str(cloud.get("refresh_token") or ""),
enabled=bool(cloud.get("enabled", False)),
)
return payload
@property
def enabled(self) -> bool:
return self._enabled and bool(self._base_url) and bool(self._access_token)
# ------------------------------------------------------------------
# Config 快照
# ------------------------------------------------------------------
def get_config(self) -> CloudSyncConfig:
"""返回当前 token 的 CloudSyncConfig 快照。
调用时机:每次创建 SakuraCloudClient 前。
若 access_token 即将过期(< 5 分钟),主动触发一次刷新。
"""
if self._access_token and self._is_expired_or_near(threshold_seconds=300):
self.try_refresh()
return CloudSyncConfig(
base_url=self._base_url,
access_token=self._access_token,
refresh_token=self._refresh_token,
)
# ------------------------------------------------------------------
# 监听者管理
# ------------------------------------------------------------------
def add_listener(self, callback: Callable[[str, str], None]) -> None:
"""注册 token 刷新回调(如 GUI 的 _on_cloud_token_refreshed)。"""
if callback not in self._listeners:
self._listeners.append(callback)
def remove_listener(self, callback: Callable[[str, str], None]) -> None:
try:
self._listeners.remove(callback)
except ValueError:
pass
def notify_listeners(self, access_token: str, refresh_token: str) -> None:
"""通知所有监听者 token 已更新(成功时为新的 token 对,失败时为空字符串)。"""
for callback in self._listeners:
try:
callback(access_token, refresh_token)
except Exception:
logger.exception("CloudSession listener %s failed", getattr(callback, "__name__", callback))
# ------------------------------------------------------------------
# Token 刷新(核心)
# ------------------------------------------------------------------
def try_refresh(self) -> bool:
"""统一刷新入口。使用全局锁防止并发重复刷新。
Returns:
True: 刷新成功,access_token 已更新
False: 刷新失败,token 已被清空,监听者已收到空 token 通知
"""
refresh = self._refresh_token
if not refresh:
return False
with self._refresh_lock:
# 二次检查:可能其他线程已刷新
refresh = self._refresh_token
if not refresh:
return False
# 距上次刷新不足 5 秒,跳过(防抖)
elapsed = time.time() - self._last_refresh_time
if elapsed < 5.0 and self._access_token:
return True
try:
resp = requests.post(
f"{self._base_url}/api/v1/auth/refresh",
json={"refresh_token": refresh},
timeout=30.0,
)
if resp.status_code != 200:
logger.warning(
"CloudSession refresh failed HTTP %s: %s",
resp.status_code,
resp.text[:200],
)
return False
body = resp.json()
data = body.get("data", body) if isinstance(body, dict) else {}
if not isinstance(data, dict):
self._on_refresh_failed()
return False
new_access = (data.get("access_token") or "").strip()
new_refresh = (data.get("refresh_token") or "").strip()
if not new_access:
self._on_refresh_failed()
return False
# 更新内存
self._access_token = new_access
if new_refresh:
self._refresh_token = new_refresh
self._last_refresh_time = time.time()
# 写盘
self._persist()
# 通知监听者
self.notify_listeners(self._access_token, self._refresh_token)
logger.info("CloudSession token refreshed successfully")
return True
except Exception:
logger.exception("CloudSession refresh error")
self._on_refresh_failed()
return False
def _on_refresh_failed(self) -> None:
"""刷新失败:清空 token + 写盘 + 通知监听者。"""
self._access_token = ""
self._refresh_token = ""
self._last_refresh_time = time.time()
self._persist()
self.notify_listeners("", "")
def _persist(self) -> None:
"""将当前 token 写入磁盘会话文件。异常记录日志但不抛出。"""
try:
persist_cloud_session_tokens(self._access_token, self._refresh_token)
except Exception:
logger.exception("CloudSession persist to disk failed")
# ------------------------------------------------------------------
# JWT 辅助
# ------------------------------------------------------------------
def _is_expired_or_near(self, threshold_seconds: int = 300) -> bool:
exp = _decode_jwt_exp(self._access_token)
if exp <= 0:
return False
return (exp - threshold_seconds) <= time.time()
@dataclass
class CloudSyncConfig:
base_url: str
access_token: str
refresh_token: str = ""
chunk_size: int = 5 * 1024 * 1024
timeout: float = 60.0
@dataclass
class SyncMetadata:
resource_type: str = "artwork"
source_platform: str = "pixiv"
source_id: str | None = None
content_type: str | None = None
title: str = ""
author: str | None = None
author_id: str | None = None
tags: list[str] = field(default_factory=list)
metadata: dict[str, Any] = field(default_factory=dict)
page_count: int = 1
thumbnail_url: str | None = None
description: str | None = None
word_count: int = 0
series_id: str | None = None
series_title: str | None = None
chapter_title: str | None = None
chapter_order: int = 0
class SyncFailureQueue:
def __init__(self, path: str | Path | None = None) -> None:
self.path = Path(path) if path else runtime_path("cloud_sync_failures.json")
def load(self) -> list[dict[str, Any]]:
if not self.path.is_file():
return []
try:
data = json.loads(self.path.read_text(encoding="utf-8"))
return data if isinstance(data, list) else []
except Exception:
return []
def save(self, items: list[dict[str, Any]]) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8")
def add(self, file_path: str | Path, metadata: SyncMetadata, mime_type: str | None, error: str) -> None:
now = time.time()
file_path = str(Path(file_path))
items = self.load()
key = (file_path, metadata.source_platform, metadata.source_id)
for item in items:
item_key = (
str(item.get("file_path") or ""),
str(item.get("metadata", {}).get("source_platform") or ""),
item.get("metadata", {}).get("source_id"),
)
if item_key == key:
attempts = int(item.get("attempts") or 0)
item.update(
{
"metadata": asdict(metadata),
"mime_type": mime_type,
"attempts": attempts,
"last_error": error,
"updated_at": now,
"next_retry_at": now + min(3600, 60 * (2 ** min(attempts, 5))),
}
)
self.save(items)
return
items.append(
{
"file_path": file_path,
"metadata": asdict(metadata),
"mime_type": mime_type,
"attempts": 0,
"last_error": error,
"created_at": now,
"updated_at": now,
"next_retry_at": now + 60,
}
)
self.save(items)
class SakuraCloudClient:
def __init__(
self,
config: CloudSyncConfig,
*,
on_token_refreshed: Callable[[str, str], None] | None = None,
cloud_session: CloudSession | None = None,
) -> None:
self.config = config
self.base_url = config.base_url.rstrip("/")
self.session = requests.Session()
self.session.headers.update({"Authorization": f"Bearer {config.access_token}"})
self._cloud_session = cloud_session # 全局单例引用
# 仅在没有 cloud_session 时使用独立锁和回调(兼容旧路径)
if cloud_session is None:
self._on_token_refreshed = on_token_refreshed or persist_cloud_session_tokens
self._refresh_lock = threading.Lock()
else:
self._on_token_refreshed = None
self._refresh_lock = None
def check_file(self, sha256: str, file_size: int) -> dict[str, Any]:
return self._post("/api/v1/sync/check", {"sha256": sha256, "file_size": file_size})
def init_upload(
self,
*,
sha256: str,
file_name: str,
file_size: int,
mime_type: str | None,
resource_type: str,
) -> dict[str, Any]:
return self._post(
"/api/v1/upload/init",
{
"sha256": sha256,
"file_name": file_name,
"file_size": file_size,
"mime_type": mime_type,
"resource_type": resource_type,
},
)
def upload_chunk(self, upload_id: str, chunk: bytes) -> dict[str, Any]:
response = self._request(
"POST",
"/api/v1/upload/chunk",
data={"upload_id": upload_id},
files={"chunk": ("chunk.bin", chunk)},
)
return self._unwrap(response)
def complete_upload(self, upload_id: str) -> dict[str, Any]:
return self._post("/api/v1/upload/complete", {"upload_id": upload_id})
def sync_metadata(self, file_id: str, metadata: SyncMetadata) -> dict[str, Any]:
payload = {
"file_id": file_id,
"resource_type": metadata.resource_type,
"source_platform": metadata.source_platform,
"source_id": metadata.source_id,
"content_type": metadata.content_type,
"title": metadata.title,
"author": metadata.author,
"author_id": metadata.author_id,
"tags": metadata.tags,
"metadata": metadata.metadata,
"page_count": metadata.page_count,
"thumbnail_url": metadata.thumbnail_url,
"description": metadata.description,
"word_count": metadata.word_count,
"series_id": metadata.series_id,
"series_title": metadata.series_title,
"chapter_title": metadata.chapter_title,
"chapter_order": metadata.chapter_order,
}
return self._post("/api/v1/sync/metadata", payload)
def upsert_follow(self, payload: dict[str, Any]) -> dict[str, Any]:
return self._post("/api/v1/follows", payload)
def bulk_upsert_follows(self, items: list[dict[str, Any]]) -> dict[str, Any]:
return self._post("/api/v1/follows/bulk-upsert", {"items": items})
def create_alert(self, payload: dict[str, Any]) -> dict[str, Any]:
return self._post("/api/v1/follows/alerts", payload)
def _request(
self,
method: str,
path: str,
*,
json: dict[str, Any] | None = None,
data: dict[str, Any] | None = None,
files: dict[str, Any] | None = None,
_retry_on_401: bool = True,
) -> requests.Response:
url = f"{self.base_url}{path}"
response = self.session.request(
method,
url,
json=json,
data=data,
files=files,
timeout=self.config.timeout,
)
if response.status_code == 401 and _retry_on_401 and self._try_refresh_token():
response = self.session.request(
method,
url,
json=json,
data=data,
files=files,
timeout=self.config.timeout,
)
return response
def _post(self, path: str, payload: dict[str, Any], *, _retry_on_401: bool = True) -> dict[str, Any]:
response = self._request("POST", path, json=payload, _retry_on_401=_retry_on_401)
return self._unwrap(response)
def _try_refresh_token(self) -> bool:
# --- 新路径:委托给全局 CloudSession ---
if self._cloud_session is not None:
ok = self._cloud_session.try_refresh()
if ok:
# 刷新成功后同步更新本地 config 和 session headers
new_config = self._cloud_session.get_config()
self.config.access_token = new_config.access_token
self.config.refresh_token = new_config.refresh_token
self.session.headers.update({"Authorization": f"Bearer {new_config.access_token}"})
return ok
# --- 旧路径(向后兼容,无 cloud_session 时使用) ---
refresh = self.config.refresh_token.strip()
if not refresh:
return False
with self._refresh_lock:
refresh = self.config.refresh_token.strip()
if not refresh:
return False
try:
resp = requests.post(
f"{self.base_url}/api/v1/auth/refresh",
json={"refresh_token": refresh},
timeout=self.config.timeout,
)
if resp.status_code != 200:
return False
body = resp.json()
data = body.get("data", body) if isinstance(body, dict) else {}
if not isinstance(data, dict):
return False
new_access = (data.get("access_token") or "").strip()
new_refresh = (data.get("refresh_token") or "").strip()
if not new_access:
return False
self.config.access_token = new_access
self.session.headers.update({"Authorization": f"Bearer {new_access}"})
if new_refresh:
self.config.refresh_token = new_refresh
try:
self._on_token_refreshed(self.config.access_token, self.config.refresh_token)
except Exception:
logger.exception("SakuraCloudClient._on_token_refreshed failed")
return True
except Exception:
logger.exception("SakuraCloudClient._try_refresh_token error")
return False
@staticmethod
def _unwrap(response: requests.Response) -> dict[str, Any]:
response.raise_for_status()
body = response.json()
if isinstance(body, dict) and "data" in body:
return body["data"]
return body
class SyncManager:
def __init__(self, client: SakuraCloudClient, failure_queue: SyncFailureQueue | None = None) -> None:
self.client = client
self.failure_queue = failure_queue or SyncFailureQueue()
def sync_file(self, file_path: str | Path, metadata: SyncMetadata, *, mime_type: str | None = None) -> dict[str, Any]:
try:
return self._sync_file_once(file_path, metadata, mime_type=mime_type)
except Exception as exc:
self.failure_queue.add(file_path, metadata, mime_type, str(exc))
raise
def _sync_file_once(self, file_path: str | Path, metadata: SyncMetadata, *, mime_type: str | None = None) -> dict[str, Any]:
path = Path(file_path)
sha256 = file_sha256(path)
file_size = path.stat().st_size
checked = self.client.check_file(sha256, file_size)
file_id = checked.get("file_id") if checked.get("owned_by_user") else None
if not file_id:
init = self.client.init_upload(
sha256=sha256,
file_name=path.name,
file_size=file_size,
mime_type=mime_type,
resource_type=self._upload_resource_type(metadata, mime_type),
)
if init.get("exists") and init.get("file_id"):
file_id = str(init["file_id"])
else:
upload_id = str(init["upload_id"])
with path.open("rb") as fp:
for chunk in iter(lambda: fp.read(self.client.config.chunk_size), b""):
self.client.upload_chunk(upload_id, chunk)
completed = self.client.complete_upload(upload_id)
file_id = str(completed["file_id"])
return self.client.sync_metadata(str(file_id), metadata)
@staticmethod
def _upload_resource_type(metadata: SyncMetadata, mime_type: str | None) -> str:
if metadata.resource_type == "novel":
return "novel"
if (mime_type or "").lower().startswith("video/") or (metadata.content_type or "").lower() == "video":
return "video"
return "image"
def retry_failed(self, *, limit: int = 5) -> dict[str, int]:
now = time.time()
items = self.failure_queue.load()
kept: list[dict[str, Any]] = []
retried = 0
succeeded = 0
failed = 0
for item in items:
if retried >= limit or float(item.get("next_retry_at") or 0) > now:
kept.append(item)
continue
path = Path(str(item.get("file_path") or ""))
if not path.is_file():
failed += 1
continue
try:
metadata = SyncMetadata(**dict(item.get("metadata") or {}))
self._sync_file_once(path, metadata, mime_type=item.get("mime_type"))
succeeded += 1
except Exception as exc:
attempts = int(item.get("attempts") or 0) + 1
item["attempts"] = attempts
item["last_error"] = str(exc)
item["updated_at"] = now
item["next_retry_at"] = now + min(3600, 60 * (2 ** min(attempts, 5)))
kept.append(item)
failed += 1
retried += 1
self.failure_queue.save(kept)
return {"retried": retried, "succeeded": succeeded, "failed": failed, "remaining": len(kept)}
+175
View File
@@ -0,0 +1,175 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
Cookie = dict[str, Any]
def parse_cookie_text(text: str) -> list[Cookie]:
"""Parse JSON cookie exports or Netscape cookies.txt content."""
text = text.strip()
if not text:
raise ValueError("Cookie 文件为空。")
try:
payload = json.loads(text)
except json.JSONDecodeError:
payload = None
if payload is not None:
raw_items = payload.get("cookies", payload) if isinstance(payload, dict) else payload
if not isinstance(raw_items, list):
raise ValueError("Cookie JSON 必须是数组,或包含 cookies 数组。")
cookies = [_normalize_json_cookie(item) for item in raw_items if isinstance(item, dict)]
else:
cookies = _parse_netscape_cookie_text(text)
cookies = [item for item in cookies if item.get("name") and item.get("value") is not None]
if not cookies:
raise ValueError("没有解析到有效 Cookie。")
return cookies
def filter_cookies_for_domains(cookies: list[Cookie], domains: tuple[str, ...]) -> list[Cookie]:
result: list[Cookie] = []
for cookie in cookies:
domain = str(cookie.get("domain", "")).lower().lstrip(".")
if any(domain == item or domain.endswith("." + item) for item in domains):
result.append(cookie)
return result
def cookie_domains(cookies: list[Cookie]) -> list[str]:
domains = sorted({str(item.get("domain", "")).lower().lstrip(".") for item in cookies if item.get("domain")})
return [item for item in domains if item]
def has_cookie_domain(cookies: list[Cookie], domains: tuple[str, ...]) -> bool:
return bool(filter_cookies_for_domains(cookies, domains))
def cookies_to_header(cookies: list[Cookie], domains: tuple[str, ...] | None = None) -> str:
selected = filter_cookies_for_domains(cookies, domains) if domains else cookies
pairs: list[str] = []
seen: set[str] = set()
for cookie in selected:
name = str(cookie.get("name", "")).strip()
value = str(cookie.get("value", ""))
if not name or name in seen:
continue
seen.add(name)
pairs.append(f"{name}={value}")
if not pairs:
raise ValueError("没有找到匹配站点的 Cookie。")
return "; ".join(pairs)
def cookies_to_playwright(cookies: list[Cookie], domains: tuple[str, ...] | None = None) -> list[Cookie]:
selected = filter_cookies_for_domains(cookies, domains) if domains else cookies
output_by_key: dict[tuple[str, str, str], Cookie] = {}
for cookie in selected:
name = str(cookie.get("name") or "").strip()
domain = str(cookie.get("domain") or "").strip()
if not name or not domain:
continue
item: Cookie = {
"name": name,
"value": str(cookie.get("value", "")),
"domain": domain,
"path": str(cookie.get("path") or "/"),
"secure": bool(cookie.get("secure", False)),
"httpOnly": bool(cookie.get("httpOnly", False)),
}
expires = cookie.get("expires")
if expires is not None:
try:
expires_value = int(float(expires))
if expires_value > 0:
item["expires"] = expires_value
except (TypeError, ValueError):
pass
same_site = str(cookie.get("sameSite", "") or "").strip()
if same_site in {"Strict", "Lax", "None"}:
item["sameSite"] = same_site
key = (str(item["domain"]).lower(), str(item["path"]), name)
output_by_key[key] = item
output = list(output_by_key.values())
if not output:
raise ValueError("没有找到匹配站点的 Cookie。")
return output
def save_playwright_cookies(cookies: list[Cookie], path: str | Path) -> Path:
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(cookies, ensure_ascii=False, indent=2), encoding="utf-8")
return target
def cookie_summary(cookies: list[Cookie]) -> str:
domains = sorted({str(item.get("domain", "")).lstrip(".") or "-" for item in cookies})
shown = ", ".join(domains[:5])
suffix = "..." if len(domains) > 5 else ""
return f"{len(cookies)} 个 Cookie;域名:{shown}{suffix}"
def _parse_netscape_cookie_text(text: str) -> list[Cookie]:
cookies: list[Cookie] = []
for raw_line in text.splitlines():
line = raw_line.strip()
if not line or line.startswith("#HttpOnly_"):
http_only = line.startswith("#HttpOnly_")
line = line.removeprefix("#HttpOnly_")
elif line.startswith("#"):
continue
else:
http_only = False
parts = line.split("\t")
if len(parts) < 7:
parts = line.split()
if len(parts) < 7:
continue
domain, _include_subdomains, path, secure, expires, name = parts[:6]
value = "\t".join(parts[6:])
cookies.append(
{
"domain": domain,
"path": path or "/",
"secure": secure.upper() == "TRUE",
"expires": int(expires) if str(expires).isdigit() else -1,
"name": name,
"value": value,
"httpOnly": http_only,
"sameSite": "Lax",
}
)
return cookies
def _normalize_json_cookie(item: Cookie) -> Cookie:
expires = item.get("expires", item.get("expirationDate", item.get("expiry")))
same_site = _normalize_same_site(item.get("sameSite"))
return {
"domain": str(item.get("domain") or item.get("host") or ""),
"path": str(item.get("path") or "/"),
"secure": bool(item.get("secure", False)),
"expires": expires,
"name": str(item.get("name") or ""),
"value": str(item.get("value") or ""),
"httpOnly": bool(item.get("httpOnly", item.get("httponly", False))),
"sameSite": same_site,
}
def _normalize_same_site(value: Any) -> str:
text = str(value or "").strip().lower()
if text in {"strict", "samesitestrict"}:
return "Strict"
if text in {"none", "no_restriction", "samesitenone"}:
return "None"
return "Lax"
File diff suppressed because it is too large. Load diff
+159
View File
@@ -0,0 +1,159 @@
from __future__ import annotations
import copy
import os
import random
import threading
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import Any
class DeviceType(Enum):
WINDOWS_CHROME = "windows_chrome"
WINDOWS_EDGE = "windows_edge"
MAC_CHROME = "mac_chrome"
@dataclass
class BrowserProfile:
"""Stable browser context settings for one logical session.
This module intentionally avoids Canvas/WebGL spoofing or large fingerprint rotation.
The goal is consistency inside a session, not impersonating many different users.
"""
user_agent: str
platform: str = "Win32"
viewport: dict[str, int] = field(default_factory=lambda: {"width": 1920, "height": 1080})
locale: str = "zh-CN"
timezone_id: str = "Asia/Shanghai"
color_scheme: str = "light"
reduced_motion: str = "no-preference"
device_scale_factor: float = 1.0
is_mobile: bool = False
has_touch: bool = False
def to_playwright_context_options(self) -> dict[str, Any]:
return {
"user_agent": self.user_agent,
"viewport": dict(self.viewport),
"locale": self.locale,
"timezone_id": self.timezone_id,
"color_scheme": self.color_scheme,
"reduced_motion": self.reduced_motion,
"device_scale_factor": self.device_scale_factor,
"is_mobile": self.is_mobile,
"has_touch": self.has_touch,
}
def headers(self) -> dict[str, str]:
return {
"User-Agent": self.user_agent,
"Accept-Language": f"{self.locale},zh;q=0.9,en;q=0.8",
}
class BrowserProfileDatabase:
@staticmethod
def windows_chrome() -> BrowserProfile:
return BrowserProfile(
user_agent=real_browser_user_agent()
or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
platform="Win32",
viewport={"width": 1920, "height": 1080},
)
@staticmethod
def windows_edge() -> BrowserProfile:
return BrowserProfile(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
platform="Win32",
viewport={"width": 1536, "height": 864},
device_scale_factor=1.25,
)
@staticmethod
def mac_chrome() -> BrowserProfile:
return BrowserProfile(
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
platform="MacIntel",
viewport={"width": 1512, "height": 982},
device_scale_factor=2.0,
)
@staticmethod
def all_desktop() -> list[BrowserProfile]:
return [
BrowserProfileDatabase.windows_chrome(),
BrowserProfileDatabase.windows_edge(),
BrowserProfileDatabase.mac_chrome(),
]
@staticmethod
def by_device_type(device_type: DeviceType) -> BrowserProfile:
mapping = {
DeviceType.WINDOWS_CHROME: BrowserProfileDatabase.windows_chrome,
DeviceType.WINDOWS_EDGE: BrowserProfileDatabase.windows_edge,
DeviceType.MAC_CHROME: BrowserProfileDatabase.mac_chrome,
}
return mapping.get(device_type, BrowserProfileDatabase.windows_chrome)()
class SessionFingerprintManager:
"""Session-sticky browser profile manager."""
def __init__(self, profiles: list[BrowserProfile] | None = None, *, strategy: str = "sticky") -> None:
self.profiles = profiles or BrowserProfileDatabase.all_desktop()
self.strategy = strategy
self._session_map: dict[str, BrowserProfile] = {}
self._index = 0
self._lock = threading.Lock()
def get_fingerprint(self, session_id: str = "default") -> BrowserProfile:
with self._lock:
if self.strategy == "sticky" and session_id in self._session_map:
return copy.deepcopy(self._session_map[session_id])
if self.strategy == "round_robin":
profile = self.profiles[self._index % len(self.profiles)]
self._index += 1
elif self.strategy == "random":
profile = random.choice(self.profiles)
else:
stable_index = abs(hash(session_id)) % len(self.profiles)
profile = self.profiles[stable_index]
self._session_map[session_id] = copy.deepcopy(profile)
return copy.deepcopy(profile)
def get_stats(self) -> dict[str, Any]:
return {
"strategy": self.strategy,
"total_profiles": len(self.profiles),
"sticky_sessions": len(self._session_map),
}
Fingerprint = BrowserProfile
FingerprintManager = SessionFingerprintManager
FingerprintDatabase = BrowserProfileDatabase
def real_browser_user_agent() -> str:
"""Load a user-provided real browser UA, if configured."""
env_value = os.environ.get("PIXIV_REAL_USER_AGENT", "").strip()
if env_value:
return env_value
path_value = os.environ.get("PIXIV_REAL_USER_AGENT_FILE", "").strip()
if path_value:
path = Path(path_value)
if path.is_file():
return path.read_text(encoding="utf-8").strip()
return ""
def default_profile(session_id: str = "default") -> BrowserProfile:
return SessionFingerprintManager().get_fingerprint(session_id)
+597
View File
@@ -0,0 +1,597 @@
from __future__ import annotations
import re
from pathlib import Path
from typing import Any
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.downloader import download_user_works, fetch_user_profile, fetch_user_work_ids
from pixiv_app.core.library_importer import LegacyDownloadImporter
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
from pixiv_app.core.paths import app_session_file
from pixiv_app.core.thumbnails import DEFAULT_THUMBNAIL_DIR, ThumbnailCache
class LocalGalleryService:
def __init__(
self,
*,
db_path: str | Path = DEFAULT_LIBRARY_DB,
thumbnail_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
) -> None:
self.db_path = Path(db_path)
library = PixivLibrary(self.db_path)
library.close()
self.thumbnail_cache = ThumbnailCache(db_path=self.db_path, cache_dir=thumbnail_dir)
def list_artworks(
self,
*,
query: str = "",
search_scope: str = "all",
tag: str = "",
artist_id: int | None = None,
artwork_type: str = "",
include_restricted: bool = True,
limit: int = 60,
offset: int = 0,
ensure_thumbnails: bool = True,
) -> dict[str, Any]:
limit = max(1, min(int(limit), 200))
offset = max(0, int(offset))
library = PixivLibrary(self.db_path)
try:
where: list[str] = []
params: list[Any] = []
joins = """
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
LEFT JOIN thumbnail_cache tc ON tc.artwork_id = a.artwork_id AND tc.page_index = 0
LEFT JOIN artwork_files af ON af.artwork_id = a.artwork_id AND af.page_index = 0
LEFT JOIN artwork_ocr_texts ocr ON ocr.artwork_id = a.artwork_id
"""
if query:
like = f"%{query}%"
if search_scope == "ocr":
where.append("(ocr.status = 'done' AND ocr.ocr_text LIKE ?)")
params.append(like)
elif search_scope == "title":
where.append("(a.title LIKE ? OR a.caption LIKE ? OR ar.name LIKE ?)")
params.extend([like, like, like])
elif search_scope == "tags":
joins += """
LEFT JOIN artwork_tags at_search ON at_search.artwork_id = a.artwork_id
LEFT JOIN tags t_search ON t_search.tag_id = at_search.tag_id
"""
where.append("t_search.name LIKE ?")
params.append(like)
else:
joins += """
LEFT JOIN artwork_tags at_search ON at_search.artwork_id = a.artwork_id
LEFT JOIN tags t_search ON t_search.tag_id = at_search.tag_id
"""
where.append(
"(a.title LIKE ? OR a.caption LIKE ? OR ar.name LIKE ? OR t_search.name LIKE ? "
"OR (ocr.status = 'done' AND ocr.ocr_text LIKE ?))"
)
params.extend([like, like, like, like, like])
if tag:
joins += """
JOIN artwork_tags at_filter ON at_filter.artwork_id = a.artwork_id
JOIN tags t_filter ON t_filter.tag_id = at_filter.tag_id
"""
where.append("t_filter.name = ?")
params.append(tag)
if artist_id is not None:
where.append("a.artist_id = ?")
params.append(int(artist_id))
if artwork_type:
where.append("a.type = ?")
params.append(artwork_type)
else:
where.append("a.type IN ('illust', 'manga')")
if not include_restricted:
where.append("a.x_restrict = 0")
where_sql = "WHERE " + " AND ".join(where) if where else ""
total = library.connection.execute(
f"SELECT COUNT(DISTINCT a.artwork_id) AS total FROM artworks a {joins} {where_sql}",
params,
).fetchone()["total"]
rows = library.connection.execute(
f"""
SELECT
a.artwork_id,
a.type,
a.title,
a.artist_id,
ar.name AS artist_name,
ar.account AS artist_account,
a.page_count,
a.bookmark_count,
a.like_count,
a.view_count,
a.x_restrict,
a.ai_type,
a.thumbnail_url,
CASE WHEN fav.artwork_id IS NULL THEN 0 ELSE 1 END AS is_favorite,
CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed,
COALESCE(tc.thumbnail_path, '') AS thumbnail_path,
COALESCE(af.file_path, '') AS primary_file_path,
CASE
WHEN ocr.status = 'done' AND ocr.ocr_text LIKE ? THEN 'ocr'
ELSE ''
END AS matched_by,
a.updated_at
FROM artworks a
{joins}
LEFT JOIN favorites fav ON fav.artwork_id = a.artwork_id
{where_sql}
GROUP BY a.artwork_id
ORDER BY a.updated_at DESC, a.artwork_id DESC
LIMIT ? OFFSET ?
""",
[f"%{query}%" if query else "", *params, limit, offset],
).fetchall()
items = [dict(row) for row in rows]
if ensure_thumbnails:
self._ensure_list_thumbnails(items)
items = self._refresh_thumbnail_paths(library, items)
return {"total": int(total), "limit": limit, "offset": offset, "items": items}
finally:
library.close()
def get_artwork(self, artwork_id: int, *, ensure_thumbnails: bool = True) -> dict[str, Any] | None:
if ensure_thumbnails:
self.thumbnail_cache.ensure_for_artwork(artwork_id)
library = PixivLibrary(self.db_path)
try:
row = library.connection.execute(
"""
SELECT a.*, ar.name AS artist_name, ar.account AS artist_account
, CASE WHEN fav.artwork_id IS NULL THEN 0 ELSE 1 END AS is_favorite
, CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed
FROM artworks a
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
LEFT JOIN favorites fav ON fav.artwork_id = a.artwork_id
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
WHERE a.artwork_id = ?
""",
(artwork_id,),
).fetchone()
if row is None:
return None
files = library.connection.execute(
"""
SELECT f.*, tc.thumbnail_path, tc.width AS thumbnail_width, tc.height AS thumbnail_height
FROM artwork_files f
LEFT JOIN thumbnail_cache tc
ON tc.artwork_id = f.artwork_id AND tc.page_index = f.page_index
WHERE f.artwork_id = ?
ORDER BY f.page_index ASC
""",
(artwork_id,),
).fetchall()
tags = library.connection.execute(
"""
SELECT t.name, t.translated_name
FROM tags t
JOIN artwork_tags at ON at.tag_id = t.tag_id
WHERE at.artwork_id = ?
ORDER BY t.name ASC
""",
(artwork_id,),
).fetchall()
history = library.connection.execute(
"""
SELECT status, downloaded_pages, skipped_pages, failed_pages, message, finished_at
FROM download_history
WHERE artwork_id = ?
ORDER BY finished_at DESC
LIMIT 20
""",
(artwork_id,),
).fetchall()
return {
"artwork": dict(row),
"files": [dict(item) for item in files],
"tags": [dict(item) for item in tags],
"history": [dict(item) for item in history],
}
finally:
library.close()
def list_tags(self, *, query: str = "", limit: int = 100) -> list[dict[str, Any]]:
library = PixivLibrary(self.db_path)
try:
params: list[Any] = []
where = ""
if query:
where = "WHERE t.name LIKE ? OR t.translated_name LIKE ?"
params.extend([f"%{query}%", f"%{query}%"])
rows = library.connection.execute(
f"""
SELECT t.name, t.translated_name, COUNT(at.artwork_id) AS artwork_count
FROM tags t
LEFT JOIN artwork_tags at ON at.tag_id = t.tag_id
{where}
GROUP BY t.tag_id
ORDER BY artwork_count DESC, t.name ASC
LIMIT ?
""",
[*params, max(1, min(int(limit), 500))],
).fetchall()
return [dict(row) for row in rows]
finally:
library.close()
def list_artists(self, *, query: str = "", limit: int = 100) -> list[dict[str, Any]]:
library = PixivLibrary(self.db_path)
try:
params: list[Any] = []
where = ""
if query:
where = "WHERE ar.name LIKE ? OR ar.account LIKE ?"
params.extend([f"%{query}%", f"%{query}%"])
rows = library.connection.execute(
f"""
SELECT ar.artist_id, ar.name, ar.account, COUNT(a.artwork_id) AS artwork_count
FROM artists ar
LEFT JOIN artworks a ON a.artist_id = ar.artist_id
{where}
GROUP BY ar.artist_id
ORDER BY
CASE WHEN COALESCE(NULLIF(ar.name, ''), NULLIF(ar.account, '')) = '' THEN 1 ELSE 0 END,
COALESCE(NULLIF(ar.name, ''), NULLIF(ar.account, ''), CAST(ar.artist_id AS TEXT)) ASC,
artwork_count DESC
LIMIT ?
""",
[*params, max(1, min(int(limit), 500))],
).fetchall()
return [dict(row) for row in rows]
finally:
library.close()
def get_artist(self, artist_id: int, *, limit: int = 60, offset: int = 0) -> dict[str, Any] | None:
library = PixivLibrary(self.db_path)
try:
row = library.connection.execute(
"""
SELECT ar.*, COUNT(a.artwork_id) AS artwork_count,
CASE WHEN fa.enabled = 1 THEN 1 ELSE 0 END AS is_followed,
fa.last_synced_at, fa.last_error
FROM artists ar
LEFT JOIN artworks a ON a.artist_id = ar.artist_id
LEFT JOIN followed_artists fa ON fa.artist_id = ar.artist_id
WHERE ar.artist_id = ?
GROUP BY ar.artist_id
""",
(artist_id,),
).fetchone()
if row is None:
return None
return {
"artist": dict(row),
"artworks": self.list_artworks(artist_id=artist_id, limit=limit, offset=offset),
}
finally:
library.close()
def list_favorites(self, *, limit: int = 60, offset: int = 0) -> dict[str, Any]:
library = PixivLibrary(self.db_path)
try:
total = library.connection.execute(
"""
SELECT COUNT(*) AS total
FROM favorites fav
JOIN artworks a ON a.artwork_id = fav.artwork_id
WHERE a.type IN ('illust', 'manga')
"""
).fetchone()["total"]
rows = library.connection.execute(
"""
SELECT
a.artwork_id, a.type, a.title, a.artist_id, ar.name AS artist_name,
ar.account AS artist_account,
a.page_count, a.bookmark_count, a.like_count, a.view_count,
a.x_restrict, a.ai_type, a.thumbnail_url,
1 AS is_favorite,
CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed,
COALESCE(tc.thumbnail_path, '') AS thumbnail_path,
COALESCE(af.file_path, '') AS primary_file_path,
fav.created_at AS favorite_at
FROM favorites fav
JOIN artworks a ON a.artwork_id = fav.artwork_id
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
LEFT JOIN thumbnail_cache tc ON tc.artwork_id = a.artwork_id AND tc.page_index = 0
LEFT JOIN artwork_files af ON af.artwork_id = a.artwork_id AND af.page_index = 0
WHERE a.type IN ('illust', 'manga')
ORDER BY fav.created_at DESC
LIMIT ? OFFSET ?
""",
(max(1, min(int(limit), 200)), max(0, int(offset))),
).fetchall()
items = [dict(row) for row in rows]
self._ensure_list_thumbnails(items)
items = self._refresh_thumbnail_paths(library, items)
return {"total": int(total), "limit": limit, "offset": offset, "items": items}
finally:
library.close()
def set_favorite(self, artwork_id: int, favorite: bool, note: str = "") -> dict[str, Any]:
library = PixivLibrary(self.db_path)
try:
library.set_favorite(artwork_id, favorite, note)
return {"artwork_id": artwork_id, "favorite": favorite}
finally:
library.close()
def list_followed_artists(self) -> list[dict[str, Any]]:
library = PixivLibrary(self.db_path)
try:
rows = library.connection.execute(
"""
SELECT fa.artist_id, COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), '') AS name,
COALESCE(NULLIF(ar.account, ''), '') AS account,
fa.enabled, fa.last_synced_at, fa.last_error,
COUNT(a.artwork_id) AS artwork_count
FROM followed_artists fa
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
LEFT JOIN artworks a ON a.artist_id = fa.artist_id
WHERE fa.enabled = 1
GROUP BY fa.artist_id
ORDER BY
CASE WHEN COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, '')) = '' THEN 1 ELSE 0 END,
COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, ''), CAST(fa.artist_id AS TEXT)) ASC,
fa.artist_id ASC
"""
).fetchall()
return [dict(row) for row in rows]
finally:
library.close()
def set_followed_artist(self, artist_ref: int | str, followed: bool, name: str = "") -> dict[str, Any]:
artist_id = self._resolve_artist_id(artist_ref)
clean_name = self._clean_name(name)
account = ""
if followed and not clean_name:
try:
profile = fetch_user_profile(
artist_id,
cookie=str(SessionStore(app_session_file()).load().get("cookie", "")).strip(),
)
profile_info = profile.get("profile") if isinstance(profile.get("profile"), dict) else {}
clean_name = self._clean_name(
profile.get("name")
or profile.get("userName")
or profile.get("displayName")
or profile_info.get("name")
)
account = self._clean_name(profile.get("account") or profile.get("userAccount"))
except Exception:
clean_name = ""
library = PixivLibrary(self.db_path)
try:
if clean_name or account:
library.upsert_artist_profile(artist_id, name=clean_name, account=account)
library.set_followed_artist(artist_id, followed, clean_name)
return {
"artist_id": artist_id,
"followed": followed,
"name": clean_name,
"account": account,
"cloud_synced": False,
}
finally:
library.close()
def _resolve_artist_id(self, artist_ref: int | str) -> int:
text = str(artist_ref).strip()
if not text:
raise ValueError("Please enter a Pixiv artist ID, profile URL, or synced artist name.")
if text.lower().startswith("artist_") and text[7:].isdigit():
return int(text[7:])
match = re.search(r"/users/(\d+)", text)
if match:
return int(match.group(1))
if text.isdigit():
return int(text)
lookup = text.lstrip("@")
library = PixivLibrary(self.db_path)
try:
rows = [
dict(row)
for row in library.connection.execute(
"""
SELECT artist_id, name, account
FROM artists
WHERE lower(name) = lower(?) OR lower(account) = lower(?)
ORDER BY
CASE
WHEN lower(account) = lower(?) THEN 0
WHEN lower(name) = lower(?) THEN 1
ELSE 2
END,
artist_id ASC
""",
(lookup, lookup, lookup, lookup),
).fetchall()
]
finally:
library.close()
if len(rows) == 1:
return int(rows[0]["artist_id"])
if not rows:
raise ValueError("Could not resolve Pixiv artist ID from that name. Enter an ID/profile URL or sync the artist first.")
choices = ", ".join(str(int(row["artist_id"])) for row in rows[:5])
raise ValueError(f"Artist name matched multiple Pixiv users; use an ID instead: {choices}")
@staticmethod
def _clean_name(value: Any) -> str:
text = str(value or "").strip()
return "" if text.lower() in {"none", "null", "undefined"} else text
def scan_legacy_downloads(self, *, root: str | Path = "Sakura_Downloads") -> dict[str, Any]:
summary = LegacyDownloadImporter(db_path=self.db_path).scan(root)
self.thumbnail_cache.build_missing(limit=500)
return summary.__dict__
def sync_followed_artists(
self,
*,
cookie: str = "",
max_new_per_artist: int = 20,
download: bool = True,
save_root: str | Path = "Sakura_Downloads",
) -> dict[str, Any]:
if not cookie:
cookie = str(SessionStore(app_session_file()).load().get("cookie", "")).strip()
if not cookie:
raise ValueError("No Pixiv cookie available. Save a session in the GUI first or pass cookie explicitly.")
library = PixivLibrary(self.db_path)
try:
followed = library.connection.execute(
"""
SELECT fa.artist_id,
COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, ''), '') AS name,
COALESCE(NULLIF(ar.account, ''), '') AS account
FROM followed_artists fa
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
WHERE fa.enabled = 1
ORDER BY fa.artist_id
"""
).fetchall()
finally:
library.close()
results: list[dict[str, Any]] = []
for row in followed:
artist_id = int(row["artist_id"])
artist_name = self._clean_name(row["name"])
artist_account = self._clean_name(row["account"])
library = PixivLibrary(self.db_path)
try:
existing = {
int(item["artwork_id"])
for item in library.connection.execute(
"SELECT artwork_id FROM artworks WHERE artist_id = ? AND type IN ('illust', 'manga')",
(artist_id,),
).fetchall()
}
finally:
library.close()
try:
_, work_ids = fetch_user_work_ids(user_id=artist_id, cookie=cookie)
new_ids = [work_id for work_id in work_ids if work_id not in existing][: max(0, int(max_new_per_artist))]
downloaded = 0
if download and new_ids:
_, download_results = download_user_works(
user_id=artist_id,
cookie=cookie,
max_workers=3,
request_delay=0.2,
save_root=save_root,
work_ids=new_ids,
)
downloaded = len([item for item in download_results if item.ok])
library = PixivLibrary(self.db_path)
try:
library.mark_artist_synced(artist_id)
finally:
library.close()
cloud_synced = False
cloud_alerts = 0
results.append(
{
"artist_id": artist_id,
"remote_count": len(work_ids),
"new_count": len(new_ids),
"downloaded": downloaded,
"cloud_synced": cloud_synced,
"alerts": cloud_alerts,
"error": "",
}
)
except Exception as exc:
library = PixivLibrary(self.db_path)
try:
library.mark_artist_synced(artist_id, error=str(exc))
finally:
library.close()
results.append({"artist_id": artist_id, "remote_count": 0, "new_count": 0, "downloaded": 0, "alerts": 0, "error": str(exc)})
return {"artists": results}
def build_missing_thumbnails(self, *, limit: int = 200) -> list[dict[str, Any]]:
return [item.__dict__ for item in self.thumbnail_cache.build_missing(limit=limit)]
def get_thumbnail_path(self, artwork_id: int, *, page_index: int = 0) -> Path | None:
self.thumbnail_cache.ensure_for_artwork(artwork_id)
library = PixivLibrary(self.db_path)
try:
row = library.connection.execute(
"""
SELECT thumbnail_path
FROM thumbnail_cache
WHERE artwork_id = ? AND page_index = ?
""",
(artwork_id, page_index),
).fetchone()
if row is None:
return None
path = Path(str(row["thumbnail_path"]))
return path if path.exists() else None
finally:
library.close()
def get_source_file_path(self, artwork_id: int, *, page_index: int = 0) -> Path | None:
library = PixivLibrary(self.db_path)
try:
row = library.connection.execute(
"""
SELECT file_path
FROM artwork_files
WHERE artwork_id = ? AND page_index = ?
""",
(artwork_id, page_index),
).fetchone()
if row is None:
return None
path = Path(str(row["file_path"]))
return path if path.exists() else None
finally:
library.close()
def _ensure_list_thumbnails(self, items: list[dict[str, Any]]) -> None:
for item in items:
if item.get("thumbnail_path"):
continue
if not item.get("primary_file_path"):
continue
self.thumbnail_cache.ensure_for_artwork(int(item["artwork_id"]))
def _refresh_thumbnail_paths(self, library: PixivLibrary, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
artwork_ids = [int(item["artwork_id"]) for item in items]
if not artwork_ids:
return items
placeholders = ",".join("?" for _ in artwork_ids)
rows = library.connection.execute(
f"""
SELECT artwork_id, thumbnail_path
FROM thumbnail_cache
WHERE page_index = 0 AND artwork_id IN ({placeholders})
""",
artwork_ids,
).fetchall()
thumbs = {int(row["artwork_id"]): str(row["thumbnail_path"]) for row in rows}
for item in items:
item["thumbnail_path"] = thumbs.get(int(item["artwork_id"]), item.get("thumbnail_path", ""))
return items
+703
View File
@@ -0,0 +1,703 @@
from __future__ import annotations
import json
import re
import sqlite3
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from pixiv_app.core.paths import app_library_file
DEFAULT_LIBRARY_DB = app_library_file()
@dataclass
class DownloadFileRecord:
page_index: int
file_path: str
source_url: str
status: str
size_bytes: int = 0
content_hash: str = ""
class PixivLibrary:
def __init__(self, db_path: str | Path = DEFAULT_LIBRARY_DB) -> None:
self.db_path = Path(db_path)
self.db_path.parent.mkdir(parents=True, exist_ok=True) if self.db_path.parent != Path(".") else None
# Allow sharing one connection across threads when guarded by a lock (e.g. TaskQueue).
self.connection = sqlite3.connect(self.db_path, timeout=30, check_same_thread=False)
self.connection.row_factory = sqlite3.Row
self.connection.execute("PRAGMA journal_mode=WAL")
self.connection.execute("PRAGMA foreign_keys=ON")
self.setup()
def close(self) -> None:
self.connection.close()
def setup(self) -> None:
with self.connection:
self.connection.executescript(
"""
CREATE TABLE IF NOT EXISTS artists (
artist_id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
account TEXT,
avatar_url TEXT,
metadata_json TEXT NOT NULL DEFAULT '{}',
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS artworks (
artwork_id INTEGER PRIMARY KEY,
artist_id INTEGER,
type TEXT NOT NULL DEFAULT 'illust',
title TEXT NOT NULL DEFAULT '',
caption TEXT NOT NULL DEFAULT '',
page_count INTEGER NOT NULL DEFAULT 0,
bookmark_count INTEGER NOT NULL DEFAULT 0,
like_count INTEGER NOT NULL DEFAULT 0,
view_count INTEGER NOT NULL DEFAULT 0,
x_restrict INTEGER NOT NULL DEFAULT 0,
ai_type INTEGER NOT NULL DEFAULT 0,
thumbnail_url TEXT,
metadata_json TEXT NOT NULL DEFAULT '{}',
created_at TEXT,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (artist_id) REFERENCES artists(artist_id)
);
CREATE TABLE IF NOT EXISTS tags (
tag_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE,
translated_name TEXT
);
CREATE TABLE IF NOT EXISTS artwork_tags (
artwork_id INTEGER NOT NULL,
tag_id INTEGER NOT NULL,
PRIMARY KEY (artwork_id, tag_id),
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE,
FOREIGN KEY (tag_id) REFERENCES tags(tag_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS artwork_files (
artwork_id INTEGER NOT NULL,
page_index INTEGER NOT NULL,
file_path TEXT NOT NULL,
source_url TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'unknown',
size_bytes INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (artwork_id, page_index),
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS download_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
artwork_id INTEGER NOT NULL,
status TEXT NOT NULL,
downloaded_pages INTEGER NOT NULL DEFAULT 0,
skipped_pages INTEGER NOT NULL DEFAULT 0,
failed_pages INTEGER NOT NULL DEFAULT 0,
message TEXT NOT NULL DEFAULT '',
started_at TEXT,
finished_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS sync_state (
key TEXT PRIMARY KEY,
value_json TEXT NOT NULL DEFAULT '{}',
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS thumbnail_cache (
artwork_id INTEGER NOT NULL,
page_index INTEGER NOT NULL,
source_path TEXT NOT NULL,
thumbnail_path TEXT NOT NULL,
width INTEGER NOT NULL DEFAULT 0,
height INTEGER NOT NULL DEFAULT 0,
source_mtime REAL NOT NULL DEFAULT 0,
source_size INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (artwork_id, page_index),
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS artwork_ocr_texts (
artwork_id INTEGER PRIMARY KEY,
language TEXT NOT NULL DEFAULT 'mixed',
ocr_text TEXT NOT NULL DEFAULT '',
confidence REAL,
engine TEXT NOT NULL DEFAULT 'paddleocr',
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS favorites (
artwork_id INTEGER PRIMARY KEY,
note TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS followed_artists (
artist_id INTEGER PRIMARY KEY,
name TEXT NOT NULL DEFAULT '',
enabled INTEGER NOT NULL DEFAULT 1,
last_synced_at TEXT,
last_error TEXT,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_artworks_artist_id ON artworks(artist_id);
CREATE INDEX IF NOT EXISTS idx_artworks_type ON artworks(type);
CREATE INDEX IF NOT EXISTS idx_download_history_artwork_id ON download_history(artwork_id);
CREATE INDEX IF NOT EXISTS idx_artwork_files_path ON artwork_files(file_path);
CREATE INDEX IF NOT EXISTS idx_thumbnail_cache_path ON thumbnail_cache(thumbnail_path);
CREATE INDEX IF NOT EXISTS idx_artwork_ocr_texts_status ON artwork_ocr_texts(status);
CREATE INDEX IF NOT EXISTS idx_artwork_ocr_texts_text ON artwork_ocr_texts(ocr_text);
CREATE INDEX IF NOT EXISTS idx_favorites_created_at ON favorites(created_at);
CREATE INDEX IF NOT EXISTS idx_followed_artists_enabled ON followed_artists(enabled);
CREATE TABLE IF NOT EXISTS download_tasks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
fingerprint TEXT NOT NULL UNIQUE,
target_type TEXT NOT NULL,
target_id INTEGER,
page_index INTEGER,
task_kind TEXT NOT NULL,
priority INTEGER NOT NULL DEFAULT 2,
status TEXT NOT NULL DEFAULT 'pending',
payload_json TEXT NOT NULL DEFAULT '{}',
attempts INTEGER NOT NULL DEFAULT 0,
leased_until REAL,
worker_id TEXT,
last_error TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_download_tasks_status_pri_id
ON download_tasks(status, priority DESC, id ASC);
CREATE TABLE IF NOT EXISTS artist_sync_watermark (
artist_id INTEGER PRIMARY KEY,
last_synced_illust_id INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);
"""
)
self._apply_schema_migrations()
def _apply_schema_migrations(self) -> None:
"""Incremental ALTER for existing databases created before new columns/tables."""
with self.connection:
cols = {row[1] for row in self.connection.execute("PRAGMA table_info(artwork_files)")}
if "content_hash" not in cols:
self.connection.execute(
"ALTER TABLE artwork_files ADD COLUMN content_hash TEXT NOT NULL DEFAULT ''"
)
# Drop legacy placeholder names like artist_12345 so UI can fall back cleanly.
self.connection.execute(
"UPDATE artists SET name = '' WHERE name = 'artist_' || artist_id"
)
self.connection.execute(
"UPDATE followed_artists SET name = '' WHERE name = 'artist_' || artist_id"
)
self.connection.execute(
"UPDATE artists SET name = '' WHERE lower(name) IN ('none', 'null', 'undefined')"
)
self.connection.execute(
"UPDATE followed_artists SET name = '' WHERE lower(name) IN ('none', 'null', 'undefined')"
)
rows = self.connection.execute(
"SELECT artwork_id, metadata_json FROM artworks WHERE artist_id IS NULL OR artist_id = 0"
).fetchall()
for row in rows:
try:
metadata = json.loads(str(row["metadata_json"] or "{}"))
except Exception:
continue
artist_id = _as_int(metadata.get("userId") or metadata.get("author_id") or metadata.get("artistId"))
if artist_id is None:
continue
self.ensure_artist_stub(artist_id, name=str(metadata.get("userName") or metadata.get("artistName") or ""))
self.connection.execute(
"UPDATE artworks SET artist_id = ?, updated_at = CURRENT_TIMESTAMP WHERE artwork_id = ?",
(artist_id, int(row["artwork_id"])),
)
def upsert_artwork(self, detail: dict[str, Any], *, artwork_type: str = "illust") -> None:
source_id = _as_int(detail.get("illustId") or detail.get("id") or detail.get("novelId"))
if source_id is None:
return
source_platform = str(detail.get("source_platform") or detail.get("source") or "")
artwork_id = library_artwork_id(source_id, artwork_type, source_platform=source_platform)
artist_id = _as_int(detail.get("userId"))
if artist_id is not None:
self._upsert_artist(detail, artist_id)
tags = _extract_tags(detail)
urls = detail.get("urls") if isinstance(detail.get("urls"), dict) else {}
thumbnail_url = (
detail.get("url")
or detail.get("coverUrl")
or urls.get("small")
or urls.get("thumb")
or urls.get("regular")
or urls.get("original")
)
with self.connection:
self.connection.execute(
"""
INSERT INTO artworks (
artwork_id, artist_id, type, title, caption, page_count,
bookmark_count, like_count, view_count, x_restrict, ai_type,
thumbnail_url, metadata_json, created_at, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id) DO UPDATE SET
artist_id = excluded.artist_id,
type = excluded.type,
title = excluded.title,
caption = excluded.caption,
page_count = excluded.page_count,
bookmark_count = excluded.bookmark_count,
like_count = excluded.like_count,
view_count = excluded.view_count,
x_restrict = excluded.x_restrict,
ai_type = excluded.ai_type,
thumbnail_url = excluded.thumbnail_url,
metadata_json = excluded.metadata_json,
created_at = COALESCE(excluded.created_at, artworks.created_at),
updated_at = CURRENT_TIMESTAMP
""",
(
artwork_id,
artist_id,
artwork_type,
str(detail.get("title") or ""),
str(detail.get("description") or detail.get("caption") or ""),
_as_int(detail.get("pageCount")) or 0,
_as_int(detail.get("bookmarkCount")) or 0,
_as_int(detail.get("likeCount")) or 0,
_as_int(detail.get("viewCount")) or 0,
_as_int(detail.get("xRestrict")) or 0,
_as_int(detail.get("aiType")) or 0,
str(thumbnail_url) if thumbnail_url else None,
json.dumps({**detail, "sourceId": source_id}, ensure_ascii=False),
str(detail.get("createDate") or detail.get("uploadDate") or "") or None,
),
)
self.connection.execute("DELETE FROM artwork_tags WHERE artwork_id = ?", (artwork_id,))
for name, translated_name in tags:
tag_id = self._upsert_tag(name, translated_name)
self.connection.execute(
"INSERT OR IGNORE INTO artwork_tags (artwork_id, tag_id) VALUES (?, ?)",
(artwork_id, tag_id),
)
def ensure_artwork_stub(self, artwork_id: int, *, artwork_type: str = "illust", title: str = "", source_platform: str = "") -> int:
db_artwork_id = library_artwork_id(artwork_id, artwork_type, source_platform=source_platform)
with self.connection:
self.connection.execute(
"""
INSERT OR IGNORE INTO artworks (artwork_id, type, title, updated_at)
VALUES (?, ?, ?, CURRENT_TIMESTAMP)
""",
(db_artwork_id, artwork_type, title),
)
return db_artwork_id
def ensure_artist_stub(self, artist_id: int, *, name: str = "") -> None:
with self.connection:
self.connection.execute(
"""
INSERT OR IGNORE INTO artists (artist_id, name, updated_at)
VALUES (?, ?, CURRENT_TIMESTAMP)
""",
(artist_id, name or ""),
)
def upsert_artist_profile(self, artist_id: int, *, name: str = "", account: str = "", avatar_url: str = "") -> None:
clean_name = _clean_optional_text(name)
clean_account = _clean_optional_text(account)
clean_avatar = _clean_optional_text(avatar_url)
with self.connection:
self.connection.execute(
"""
INSERT INTO artists (artist_id, name, account, avatar_url, metadata_json, updated_at)
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artist_id) DO UPDATE SET
name = COALESCE(NULLIF(excluded.name, ''), artists.name),
account = COALESCE(NULLIF(excluded.account, ''), artists.account),
avatar_url = COALESCE(NULLIF(excluded.avatar_url, ''), artists.avatar_url),
metadata_json = CASE
WHEN excluded.metadata_json != '{}' THEN excluded.metadata_json
ELSE artists.metadata_json
END,
updated_at = CURRENT_TIMESTAMP
""",
(
int(artist_id),
clean_name,
clean_account,
clean_avatar,
json.dumps(
{
"userId": int(artist_id),
"userName": clean_name,
"userAccount": clean_account,
},
ensure_ascii=False,
),
),
)
def set_artwork_artist(self, artwork_id: int, artist_id: int) -> None:
with self.connection:
self.ensure_artist_stub(artist_id)
self.connection.execute(
"""
UPDATE artworks
SET artist_id = ?, updated_at = CURRENT_TIMESTAMP
WHERE artwork_id = ?
""",
(artist_id, artwork_id),
)
def record_files(self, artwork_id: int, files: list[DownloadFileRecord]) -> None:
with self.connection:
for item in files:
self.connection.execute(
"""
INSERT INTO artwork_files (
artwork_id, page_index, file_path, source_url, status, size_bytes,
content_hash, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id, page_index) DO UPDATE SET
file_path = excluded.file_path,
source_url = excluded.source_url,
status = excluded.status,
size_bytes = excluded.size_bytes,
content_hash = CASE
WHEN excluded.content_hash != '' THEN excluded.content_hash
ELSE artwork_files.content_hash
END,
updated_at = CURRENT_TIMESTAMP
""",
(
artwork_id,
item.page_index,
item.file_path,
item.source_url,
item.status,
item.size_bytes,
item.content_hash or "",
),
)
def record_download_history(
self,
*,
artwork_id: int,
status: str,
downloaded_pages: int,
skipped_pages: int,
failed_pages: int,
message: str,
started_at: str | None = None,
) -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO download_history (
artwork_id, status, downloaded_pages, skipped_pages, failed_pages, message, started_at
)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(artwork_id, status, downloaded_pages, skipped_pages, failed_pages, message, started_at),
)
def update_sync_state(self, key: str, value: dict[str, Any]) -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO sync_state (key, value_json, updated_at)
VALUES (?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(key) DO UPDATE SET
value_json = excluded.value_json,
updated_at = CURRENT_TIMESTAMP
""",
(key, json.dumps(value, ensure_ascii=False)),
)
def upsert_thumbnail(
self,
*,
artwork_id: int,
page_index: int,
source_path: str,
thumbnail_path: str,
width: int,
height: int,
source_mtime: float,
source_size: int,
) -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO thumbnail_cache (
artwork_id, page_index, source_path, thumbnail_path, width, height,
source_mtime, source_size, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id, page_index) DO UPDATE SET
source_path = excluded.source_path,
thumbnail_path = excluded.thumbnail_path,
width = excluded.width,
height = excluded.height,
source_mtime = excluded.source_mtime,
source_size = excluded.source_size,
updated_at = CURRENT_TIMESTAMP
""",
(artwork_id, page_index, source_path, thumbnail_path, width, height, source_mtime, source_size),
)
def upsert_ocr_text(
self,
artwork_id: int,
*,
text: str,
language: str = "mixed",
confidence: float | None = None,
engine: str = "paddleocr",
status: str = "done",
attempts: int = 1,
) -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO artwork_ocr_texts (
artwork_id, language, ocr_text, confidence, engine, status, attempts, updated_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id) DO UPDATE SET
language = excluded.language,
ocr_text = excluded.ocr_text,
confidence = excluded.confidence,
engine = excluded.engine,
status = excluded.status,
attempts = excluded.attempts,
updated_at = CURRENT_TIMESTAMP
""",
(int(artwork_id), language, text, confidence, engine, status, attempts),
)
def get_ocr_text(self, artwork_id: int) -> sqlite3.Row | None:
return self.connection.execute(
"SELECT * FROM artwork_ocr_texts WHERE artwork_id = ?",
(int(artwork_id),),
).fetchone()
def set_favorite(self, artwork_id: int, favorite: bool, note: str = "") -> None:
with self.connection:
if favorite:
self.connection.execute(
"""
INSERT INTO favorites (artwork_id, note, created_at)
VALUES (?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artwork_id) DO UPDATE SET note = excluded.note
""",
(artwork_id, note),
)
else:
self.connection.execute("DELETE FROM favorites WHERE artwork_id = ?", (artwork_id,))
def set_followed_artist(self, artist_id: int, followed: bool, name: str = "") -> None:
with self.connection:
self.ensure_artist_stub(artist_id, name=name)
if followed:
self.connection.execute(
"""
INSERT INTO followed_artists (artist_id, name, enabled, created_at)
VALUES (?, ?, 1, CURRENT_TIMESTAMP)
ON CONFLICT(artist_id) DO UPDATE SET
name = COALESCE(NULLIF(excluded.name, ''), followed_artists.name),
enabled = 1
""",
(artist_id, name),
)
else:
self.connection.execute(
"UPDATE followed_artists SET enabled = 0 WHERE artist_id = ?",
(artist_id,),
)
def upsert_origin_followed_artist(
self,
artist_id: int,
*,
name: str = "",
account: str = "",
avatar_url: str = "",
) -> str:
clean_name = _clean_optional_text(name)
before = self.connection.execute(
"""
SELECT fa.enabled, fa.name AS follow_name, ar.name AS artist_name,
ar.account, ar.avatar_url
FROM followed_artists fa
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
WHERE fa.artist_id = ?
""",
(int(artist_id),),
).fetchone()
status = "created"
if before is not None:
changed = (
not bool(before["enabled"])
or (clean_name and clean_name != str(before["follow_name"] or before["artist_name"] or ""))
or (account and account != str(before["account"] or ""))
or (avatar_url and avatar_url != str(before["avatar_url"] or ""))
)
status = "updated" if changed else "skipped"
with self.connection:
self.upsert_artist_profile(
int(artist_id),
name=clean_name,
account=account,
avatar_url=avatar_url,
)
self.connection.execute(
"""
INSERT INTO followed_artists (artist_id, name, enabled, last_synced_at, last_error, created_at)
VALUES (?, ?, 1, CURRENT_TIMESTAMP, '', CURRENT_TIMESTAMP)
ON CONFLICT(artist_id) DO UPDATE SET
name = COALESCE(NULLIF(excluded.name, ''), followed_artists.name),
enabled = 1,
last_synced_at = CURRENT_TIMESTAMP,
last_error = ''
""",
(int(artist_id), clean_name),
)
return status
def mark_artist_synced(self, artist_id: int, *, error: str = "") -> None:
with self.connection:
self.connection.execute(
"""
INSERT INTO followed_artists (artist_id, enabled, last_synced_at, last_error)
VALUES (?, 1, CURRENT_TIMESTAMP, ?)
ON CONFLICT(artist_id) DO UPDATE SET
last_synced_at = CURRENT_TIMESTAMP,
last_error = excluded.last_error
""",
(artist_id, error),
)
def _upsert_artist(self, detail: dict[str, Any], artist_id: int) -> None:
self.connection.execute(
"""
INSERT INTO artists (artist_id, name, account, avatar_url, metadata_json, updated_at)
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artist_id) DO UPDATE SET
name = excluded.name,
account = COALESCE(excluded.account, artists.account),
avatar_url = COALESCE(excluded.avatar_url, artists.avatar_url),
metadata_json = excluded.metadata_json,
updated_at = CURRENT_TIMESTAMP
""",
(
artist_id,
str(detail.get("userName") or detail.get("artistName") or ""),
str(detail.get("userAccount") or "") or None,
str(detail.get("profileImageUrl") or detail.get("userImage") or "") or None,
json.dumps(
{
"userId": artist_id,
"userName": detail.get("userName"),
"userAccount": detail.get("userAccount"),
},
ensure_ascii=False,
),
),
)
def _upsert_tag(self, name: str, translated_name: str | None) -> int:
self.connection.execute(
"""
INSERT INTO tags (name, translated_name)
VALUES (?, ?)
ON CONFLICT(name) DO UPDATE SET
translated_name = COALESCE(excluded.translated_name, tags.translated_name)
""",
(name, translated_name),
)
row = self.connection.execute("SELECT tag_id FROM tags WHERE name = ?", (name,)).fetchone()
return int(row["tag_id"])
def _as_int(value: Any) -> int | None:
if value is None:
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _clean_optional_text(value: Any) -> str:
text = str(value or "").strip()
return "" if text.lower() in {"none", "null", "undefined"} else text
def library_artwork_id(source_id: int, artwork_type: str = "illust", source_platform: str = "") -> int:
"""Map a source-native ID into the unified library artwork_id namespace.
Pixiv novels use negative source_id; AO3 works use a negative offset
range to avoid collisions with Pixiv novel IDs.
"""
if artwork_type == "novel":
if source_platform == "ao3":
# AO3: use negative offset range -(200M + work_id) to stay clear
# of Pixiv novel IDs (currently -1 .. -199_999_999).
AO3_OFFSET = 200_000_000
return -(AO3_OFFSET + abs(int(source_id)))
return -abs(int(source_id))
return int(source_id)
def _extract_tags(detail: dict[str, Any]) -> list[tuple[str, str | None]]:
raw_tags = detail.get("tags")
if isinstance(raw_tags, dict):
raw_tags = raw_tags.get("tags")
if not isinstance(raw_tags, list):
return []
result: list[tuple[str, str | None]] = []
for item in raw_tags:
if isinstance(item, str):
result.append((item, None))
continue
if not isinstance(item, dict):
continue
name = item.get("tag") or item.get("name")
if not name:
continue
translation = item.get("translation")
translated_name = None
if isinstance(translation, dict):
translated_name = translation.get("en") or translation.get("zh") or translation.get("romaji")
elif isinstance(translation, str):
translated_name = translation
result.append((str(name), str(translated_name) if translated_name else None))
return result
+76
View File
@@ -0,0 +1,76 @@
from __future__ import annotations
import re
from dataclasses import dataclass
from pathlib import Path
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, DownloadFileRecord, PixivLibrary
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
ILLUST_FILE_PATTERN = re.compile(r"(?P<artwork_id>\d+)_p(?P<page_index>\d+)", re.IGNORECASE)
@dataclass
class ImportSummary:
scanned_files: int = 0
imported_files: int = 0
skipped_files: int = 0
imported_artworks: int = 0
class LegacyDownloadImporter:
def __init__(self, *, db_path: str | Path = DEFAULT_LIBRARY_DB) -> None:
self.db_path = Path(db_path)
def scan(self, root: str | Path = "Sakura_Downloads") -> ImportSummary:
root_path = Path(root)
summary = ImportSummary()
if not root_path.exists():
return summary
library = PixivLibrary(self.db_path)
imported_artworks: set[int] = set()
try:
for file_path in root_path.rglob("*"):
if not file_path.is_file() or file_path.suffix.lower() not in IMAGE_SUFFIXES:
continue
summary.scanned_files += 1
match = ILLUST_FILE_PATTERN.search(file_path.stem)
if match is None:
summary.skipped_files += 1
continue
artwork_id = int(match.group("artwork_id"))
page_index = int(match.group("page_index"))
library.ensure_artwork_stub(artwork_id, artwork_type="illust", title=str(artwork_id))
artist_id = self._infer_artist_id(root_path, file_path)
if artist_id is not None:
library.set_artwork_artist(artwork_id, artist_id)
library.record_files(
artwork_id,
[
DownloadFileRecord(
page_index=page_index,
file_path=str(file_path.resolve()),
source_url="",
status="imported",
size_bytes=file_path.stat().st_size,
)
],
)
imported_artworks.add(artwork_id)
summary.imported_files += 1
summary.imported_artworks = len(imported_artworks)
return summary
finally:
library.close()
def _infer_artist_id(self, root_path: Path, file_path: Path) -> int | None:
try:
relative = file_path.relative_to(root_path)
except ValueError:
return None
if not relative.parts:
return None
first = relative.parts[0]
return int(first) if first.isdigit() else None
+128
View File
@@ -0,0 +1,128 @@
from __future__ import annotations
import logging
import os
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Callable
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
OcrExtractor = Callable[[Path], tuple[str, float | None, str]]
@dataclass
class LocalOcrResult:
artwork_id: int
status: str
text_length: int = 0
confidence: float | None = None
error: str | None = None
class LocalOcrIndexer:
def __init__(
self,
*,
db_path: str | Path = DEFAULT_LIBRARY_DB,
enabled: bool | None = None,
max_dimension: int | None = None,
language: str | None = None,
max_concurrency: int | None = None,
) -> None:
self.db_path = Path(db_path)
self.enabled = enabled if enabled is not None else _env_bool("SAKURA_LOCAL_OCR_ENABLED", False)
self.max_dimension = max_dimension or int(os.getenv("SAKURA_LOCAL_OCR_MAX_DIMENSION", "1800"))
self.language = language or os.getenv("SAKURA_LOCAL_OCR_LANGUAGE", "ch")
self._semaphore = threading.BoundedSemaphore(
max_concurrency or int(os.getenv("SAKURA_LOCAL_OCR_MAX_CONCURRENCY", "1"))
)
def index_artwork(self, artwork_id: int, *, extractor: OcrExtractor | None = None) -> LocalOcrResult:
if not self.enabled and extractor is None:
return LocalOcrResult(artwork_id=artwork_id, status="disabled")
if not self._semaphore.acquire(blocking=False):
return LocalOcrResult(artwork_id=artwork_id, status="busy")
library = PixivLibrary(self.db_path)
try:
existing = library.get_ocr_text(artwork_id)
if existing is not None and str(existing["status"]) == "done" and str(existing["ocr_text"]).strip():
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(str(existing["ocr_text"])))
row = library.connection.execute(
"""
SELECT file_path
FROM artwork_files
WHERE artwork_id = ?
ORDER BY page_index ASC
LIMIT 1
""",
(int(artwork_id),),
).fetchone()
if row is None:
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="file_not_found")
path = Path(str(row["file_path"]))
if path.suffix.lower() not in {".jpg", ".jpeg", ".png", ".webp"}:
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="unsupported_file_type")
text, confidence, language = (extractor or self._paddle_ocr_extract)(path)
library.upsert_ocr_text(
artwork_id,
text=text.strip(),
confidence=confidence,
language=language or self.language,
status="done",
attempts=1,
)
logging.info("Local OCR done artwork=%s text_len=%s confidence=%s", artwork_id, len(text.strip()), confidence)
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(text.strip()), confidence=confidence)
except Exception as exc:
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
logging.warning("Local OCR failed artwork=%s error=%s", artwork_id, exc.__class__.__name__)
return LocalOcrResult(artwork_id=artwork_id, status="failed", error=exc.__class__.__name__)
finally:
library.close()
self._semaphore.release()
def _paddle_ocr_extract(self, path: Path) -> tuple[str, float | None, str]:
from PIL import Image
from paddleocr import PaddleOCR
work_path = path
with Image.open(path) as image:
original_size = image.size
image.thumbnail((self.max_dimension, self.max_dimension))
if image.size != original_size:
tmp = path.with_suffix(".local-ocr-resized.jpg")
image.convert("RGB").save(tmp, quality=90)
work_path = tmp
try:
ocr = PaddleOCR(use_angle_cls=True, lang=self.language, use_gpu=False)
result = ocr.ocr(str(work_path), cls=True)
texts: list[str] = []
scores: list[float] = []
for page in result or []:
for line in page or []:
if len(line) >= 2 and isinstance(line[1], (list, tuple)):
text = str(line[1][0]).strip()
if text:
texts.append(text)
try:
scores.append(float(line[1][1]))
except (TypeError, ValueError, IndexError):
pass
confidence = sum(scores) / len(scores) if scores else None
return "\n".join(texts), confidence, self.language
finally:
if work_path != path:
work_path.unlink(missing_ok=True)
def _env_bool(name: str, default: bool) -> bool:
value = os.getenv(name)
if value is None:
return default
return value.strip().lower() in {"1", "true", "yes", "on"}
+50
View File
@@ -0,0 +1,50 @@
"""
Bridge URL-level plugin parsing with the rest of the app (tasks/GUI logging).
Full TaskQueue integration can map Resource → DownloadTaskSpec per site.
"""
from __future__ import annotations
from dataclasses import dataclass
import logging
from pathlib import Path
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
from pixiv_app.core.plugin.manager import PluginManager
logger = logging.getLogger(__name__)
@dataclass
class OrchestrationResult:
plugin_name: str
resources: list[Resource]
class PluginTaskOrchestrator:
"""Resolve plugins and run parse/download without replacing the legacy Pixiv GUI flow."""
def __init__(self, plugin_manager: PluginManager) -> None:
self.plugin_manager = plugin_manager
def get_plugin_for_url(self, url: str) -> BasePlugin | None:
return self.plugin_manager.get_plugin_for_url(url)
def parse_url(self, url: str) -> OrchestrationResult:
plugin = self.get_plugin_for_url(url)
if plugin is None:
raise PluginParseError(f"No plugin registered for: {url}")
resources = plugin.parse(url)
return OrchestrationResult(plugin_name=plugin.name, resources=resources)
def download_resource(
self,
plugin_name: str,
resource: Resource,
save_path: Path,
) -> list[Path]:
plugin = self.plugin_manager.plugins.get(plugin_name)
if plugin is None:
raise PluginParseError(f"Plugin not loaded: {plugin_name}")
return plugin.download(resource, save_path)
+423
View File
@@ -0,0 +1,423 @@
from __future__ import annotations
import asyncio
import re
from dataclasses import dataclass, field
from typing import Any, Callable
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.downloader import create_session, request_json
from pixiv_app.core.gallery_service import LocalGalleryService
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
from pixiv_app.core.paths import app_session_file
from pixiv_app.core.x_media_service import XMediaService, _normalize_username
ProgressCallback = Callable[[dict[str, Any]], None]
@dataclass
class OriginFollow:
platform: str
external_id: str
name: str = ""
handle: str = ""
avatar_url: str = ""
profile_url: str = ""
@dataclass
class OriginFollowSyncResult:
platform: str
created: int = 0
updated: int = 0
skipped: int = 0
failed: int = 0
cloud_created: int = 0
cloud_updated: int = 0
cloud_skipped: int = 0
pages: int = 0
errors: list[dict[str, str]] = field(default_factory=list)
progress: list[dict[str, Any]] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"platform": self.platform,
"created": self.created,
"updated": self.updated,
"skipped": self.skipped,
"failed": self.failed,
"cloud_created": self.cloud_created,
"cloud_updated": self.cloud_updated,
"cloud_skipped": self.cloud_skipped,
"pages": self.pages,
"errors": self.errors,
"progress": self.progress,
}
class OriginFollowSyncService:
def __init__(
self,
*,
db_path: str = DEFAULT_LIBRARY_DB,
gallery_service: LocalGalleryService | None = None,
x_media_service: XMediaService | None = None,
) -> None:
self.db_path = db_path
self.gallery_service = gallery_service or LocalGalleryService(db_path=db_path)
self.x_media_service = x_media_service or XMediaService()
def sync(self, platform: str, *, progress: ProgressCallback | None = None) -> dict[str, Any]:
normalized = platform.strip().lower()
if normalized == "pixiv":
return self.sync_pixiv(progress=progress).to_dict()
if normalized == "x":
return self.sync_x(progress=progress).to_dict()
raise ValueError("请选择 Pixiv 或 X。")
def sync_pixiv(self, *, progress: ProgressCallback | None = None) -> OriginFollowSyncResult:
cookie = str(SessionStore(app_session_file()).load().get("cookie", "")).strip()
if not cookie:
raise ValueError("Pixiv Cookie 不存在,请先导入或保存 Pixiv 登录态。")
result = OriginFollowSyncResult(platform="pixiv")
session = create_session(cookie)
try:
user_id = self._fetch_pixiv_current_user_id(session, cookie)
follows = self._fetch_pixiv_following(session, cookie, user_id, result, progress=progress)
finally:
session.close()
self._upsert_pixiv_local(follows, result)
return result
def sync_x(self, *, progress: ProgressCallback | None = None) -> OriginFollowSyncResult:
plugin = self.x_media_service._get_plugin()
if not plugin.validate():
raise ValueError("X Playwright 不可用,请先安装 Playwright 并完成 X 登录。")
result = OriginFollowSyncResult(platform="x")
with plugin._playwright_lock:
follows = asyncio.run(self._fetch_x_following(plugin, result, progress=progress))
self._upsert_x_local(follows, result)
return result
def _emit(
self,
result: OriginFollowSyncResult,
progress: ProgressCallback | None,
payload: dict[str, Any],
) -> None:
result.progress.append(payload)
if progress is not None:
progress(payload)
def _fetch_pixiv_current_user_id(self, session: Any, cookie: str) -> str:
for url in (
"https://www.pixiv.net/ajax/user/self?lang=zh",
"https://www.pixiv.net/ajax/user/self",
):
try:
payload = request_json(session, url, cookie=cookie, context="Pixiv 当前账号")
body = payload.get("body") if isinstance(payload, dict) else {}
user_id = _first_text(body, "userId", "id")
if user_id:
return user_id
except Exception:
continue
try:
response = session.get("https://www.pixiv.net/", timeout=30)
response.raise_for_status()
match = re.search(r'"userData"\s*:\s*\{[^}]*"id"\s*:\s*"?(?P<id>\d+)"?', response.text)
if match:
return match.group("id")
except Exception:
pass
raise ValueError("Pixiv Cookie 可能已失效:无法识别当前登录账号。")
def _fetch_pixiv_following(
self,
session: Any,
cookie: str,
user_id: str,
result: OriginFollowSyncResult,
*,
progress: ProgressCallback | None = None,
) -> list[OriginFollow]:
follows: list[OriginFollow] = []
seen: set[str] = set()
limit = 100
offset = 0
page = 1
while True:
self._emit(result, progress, {"platform": "pixiv", "page": page, "message": f"正在同步第 {page} 页"})
url = (
f"https://www.pixiv.net/ajax/user/{user_id}/following"
f"?offset={offset}&limit={limit}&rest=show&tag=&acceptingRequests=0&lang=zh"
)
try:
payload = request_json(
session,
url,
cookie=cookie,
referer=f"https://www.pixiv.net/users/{user_id}/following",
context=f"Pixiv 关注列表第 {page} 页",
)
except Exception as exc:
if page == 1:
raise ValueError("Pixiv Cookie 可能已失效:无法读取关注列表。") from exc
result.failed += 1
result.errors.append({"id": f"page:{page}", "reason": str(exc)})
break
users = _extract_pixiv_users(payload)
result.pages = page
if not users:
break
for item in users:
follow = _pixiv_follow_from_payload(item)
if follow is None:
result.failed += 1
result.errors.append({"id": "unknown", "reason": "Pixiv 作者记录缺少 userId"})
continue
if follow.external_id in seen:
result.skipped += 1
continue
seen.add(follow.external_id)
follows.append(follow)
if len(users) < limit:
break
offset += limit
page += 1
return follows
async def _fetch_x_following(
self,
plugin: Any,
result: OriginFollowSyncResult,
*,
progress: ProgressCallback | None = None,
) -> list[OriginFollow]:
from plugins.x import selectors
from plugins.x.playwright_client import XPlaywrightClient
client = XPlaywrightClient(plugin.config)
follows: list[OriginFollow] = []
seen: set[str] = set()
try:
await client.start()
page = await client.new_page_goto("https://x.com/home")
try:
await client.raise_if_guest_wall(page)
profile_href = await page.locator(selectors.PROFILE_LINK).first.get_attribute("href", timeout=8000)
username = _x_username_from_href(profile_href or "")
if not username:
raise ValueError("X 未登录:请先在插件管理里完成 X 登录。")
await page.goto(f"https://x.com/{username}{selectors.FOLLOWING_URL_SUFFIX}", wait_until="domcontentloaded")
await client.raise_if_guest_wall(page)
stale_rounds = 0
page_no = 1
while stale_rounds < 4 and page_no <= 80:
self._emit(result, progress, {"platform": "x", "page": page_no, "message": f"正在同步第 {page_no} 页"})
added = await _collect_x_user_cells(page, seen, follows)
result.pages = page_no
stale_rounds = stale_rounds + 1 if added == 0 else 0
await client.human_scroll_page(page)
page_no += 1
if not follows:
raise ValueError("X 未读取到 following 列表,请确认账号已登录且 following 页面可访问。")
finally:
await page.close()
finally:
await client.close()
return follows
def _upsert_pixiv_local(self, follows: list[OriginFollow], result: OriginFollowSyncResult) -> None:
library = PixivLibrary(self.db_path)
try:
for follow in follows:
try:
status = library.upsert_origin_followed_artist(
int(follow.external_id),
name=follow.name,
account=follow.handle,
avatar_url=follow.avatar_url,
)
_apply_local_status(result, status)
except Exception as exc:
result.failed += 1
result.errors.append({"id": follow.external_id, "reason": str(exc)})
finally:
library.close()
def _upsert_x_local(self, follows: list[OriginFollow], result: OriginFollowSyncResult) -> None:
rows = self.x_media_service._load_follow_rows()
now = _now_text()
by_handle = {str(item.get("username", "")).lower(): item for item in rows}
for follow in follows:
handle = _normalize_username(follow.handle or follow.external_id)
if not handle:
result.failed += 1
result.errors.append({"id": follow.external_id, "reason": "X 用户名为空"})
continue
row = by_handle.get(handle.lower())
status = "created"
if row is None:
row = {
"username": handle,
"creator_name": follow.name or handle,
"enabled": True,
"profile_url": follow.profile_url or f"https://x.com/{handle}",
"avatar_url": follow.avatar_url,
"latest_tweet_id": "",
"latest_published_at": "",
"last_checked_at": now,
"last_error": "",
"seen_tweet_ids": [],
"created_at": now,
"updated_at": now,
}
rows.append(row)
by_handle[handle.lower()] = row
else:
changed = (
not bool(row.get("enabled", True))
or (follow.name and follow.name != str(row.get("creator_name") or ""))
or (follow.avatar_url and follow.avatar_url != str(row.get("avatar_url") or ""))
)
status = "updated" if changed else "skipped"
row["username"] = handle
row["creator_name"] = follow.name or str(row.get("creator_name") or handle)
row["enabled"] = True
row["profile_url"] = follow.profile_url or f"https://x.com/{handle}"
row["avatar_url"] = follow.avatar_url or str(row.get("avatar_url") or "")
row["last_checked_at"] = now
row["last_error"] = ""
row["updated_at"] = now
_apply_local_status(result, status)
self.x_media_service._save_follow_rows(rows)
def _apply_local_status(result: OriginFollowSyncResult, status: str) -> None:
if status == "created":
result.created += 1
elif status == "updated":
result.updated += 1
else:
result.skipped += 1
def _extract_pixiv_users(payload: dict[str, Any]) -> list[dict[str, Any]]:
body = payload.get("body") if isinstance(payload, dict) else payload
if isinstance(body, list):
return [item for item in body if isinstance(item, dict)]
if not isinstance(body, dict):
return []
candidates = (
body.get("users"),
body.get("followUsers"),
body.get("userPreviews"),
body.get("thumbnails", {}).get("user") if isinstance(body.get("thumbnails"), dict) else None,
)
for candidate in candidates:
if isinstance(candidate, list):
return [item for item in candidate if isinstance(item, dict)]
return []
def _pixiv_follow_from_payload(item: dict[str, Any]) -> OriginFollow | None:
user_id = _first_text(item, "userId", "user_id", "id")
if not user_id:
user = item.get("user") if isinstance(item.get("user"), dict) else {}
user_id = _first_text(user, "userId", "user_id", "id")
item = {**item, **user}
if not user_id:
return None
name = _first_text(item, "name", "userName", "displayName")
handle = _first_text(item, "userAccount", "account", "screenName")
avatar = _first_text(item, "imageBig", "image", "profileImageUrl", "profile_image_url")
return OriginFollow(
platform="pixiv",
external_id=str(user_id),
name=name,
handle=handle,
avatar_url=avatar,
profile_url=f"https://www.pixiv.net/users/{user_id}",
)
async def _collect_x_user_cells(page: Any, seen: set[str], follows: list[OriginFollow]) -> int:
from plugins.x import selectors
added = 0
cells = await page.locator(selectors.USER_CELL).all()
for cell in cells:
try:
data = await cell.evaluate(
"""(node) => {
const text = node.innerText || "";
const links = Array.from(node.querySelectorAll('a[href^="/"]')).map(a => a.getAttribute("href") || "");
const img = node.querySelector('img[src*="profile_images"]');
return { text, links, avatar: img ? img.getAttribute("src") : "" };
}"""
)
handle = _x_handle_from_cell(data)
if not handle or handle.lower() in seen:
continue
seen.add(handle.lower())
follows.append(
OriginFollow(
platform="x",
external_id=handle,
name=_x_display_name_from_cell(data, handle),
handle=handle,
avatar_url=str(data.get("avatar") or ""),
profile_url=f"https://x.com/{handle}",
)
)
added += 1
except Exception:
continue
return added
def _x_handle_from_cell(data: dict[str, Any]) -> str:
for href in data.get("links") or []:
username = _x_username_from_href(str(href))
if username:
return username
text = str(data.get("text") or "")
match = re.search(r"@([A-Za-z0-9_]{1,15})", text)
return match.group(1) if match else ""
def _x_display_name_from_cell(data: dict[str, Any], handle: str) -> str:
text = str(data.get("text") or "")
for line in [item.strip() for item in text.splitlines() if item.strip()]:
if not line.startswith("@") and line.lower() not in {"follow", "following"}:
return line
return handle
def _x_username_from_href(href: str) -> str:
match = re.match(r"^/?([A-Za-z0-9_]{1,15})(?:/)?$", href.strip())
if not match:
return ""
username = match.group(1)
if username.lower() in {"home", "explore", "notifications", "messages", "settings", "i"}:
return ""
return username
def _first_text(payload: Any, *keys: str) -> str:
if not isinstance(payload, dict):
return ""
for key in keys:
value = payload.get(key)
if value is not None and str(value).strip():
return str(value).strip()
return ""
def _now_text() -> str:
from datetime import datetime, timezone
return datetime.now(timezone.utc).isoformat()
+113
View File
@@ -0,0 +1,113 @@
from __future__ import annotations
import os
import shutil
import sys
from pathlib import Path
APP_NAME = "SakuraDownloader"
LEGACY_APP_NAME = "PixivDownloader"
def is_frozen() -> bool:
return bool(getattr(sys, "frozen", False))
def resource_root() -> Path:
if is_frozen():
return Path(getattr(sys, "_MEIPASS")).resolve()
return Path(__file__).resolve().parents[2]
def executable_dir() -> Path:
if is_frozen():
return Path(sys.executable).resolve().parent
return resource_root()
def data_root() -> Path:
override = os.environ.get("SAKURA_APP_DATA_DIR") or os.environ.get("PIXIV_APP_DATA_DIR")
if override:
root = Path(override)
elif is_frozen():
local_appdata = os.environ.get("LOCALAPPDATA")
root = Path(local_appdata) / APP_NAME if local_appdata else executable_dir() / "data"
legacy_root = Path(local_appdata) / LEGACY_APP_NAME if local_appdata else None
if legacy_root and legacy_root.exists() and not root.exists():
root.mkdir(parents=True, exist_ok=True)
for name in ("pixiv_app_session.json", "runtime", "profiles", "plugins"):
src = legacy_root / name
dst = root / name
if src.exists() and not dst.exists():
if src.is_dir():
import shutil
shutil.copytree(src, dst)
else:
dst.write_bytes(src.read_bytes())
else:
root = resource_root()
root.mkdir(parents=True, exist_ok=True)
return root
def runtime_path(*parts: str) -> Path:
path = data_root() / "runtime"
path.mkdir(parents=True, exist_ok=True)
return path.joinpath(*parts)
def profiles_root() -> Path:
path = data_root() / "profiles"
path.mkdir(parents=True, exist_ok=True)
return path
def plugins_root() -> Path:
return resource_root() / "plugins"
def external_plugins_root() -> Path:
"""Plugins shipped beside the executable, outside the PyInstaller bundle."""
return executable_dir() / "plugins"
def user_plugins_root() -> Path:
path = data_root() / "plugins"
path.mkdir(parents=True, exist_ok=True)
return path
def plugin_roots() -> list[Path]:
roots = [plugins_root(), external_plugins_root(), user_plugins_root()]
result: list[Path] = []
for root in roots:
if root not in result:
result.append(root)
return result
def webui_root() -> Path:
return resource_root() / "pixiv_app" / "webui"
def app_session_file() -> Path:
return data_root() / "pixiv_app_session.json"
def app_library_file() -> Path:
"""Persistent local library DB, writable even when the EXE directory is read-only."""
target = data_root() / "pixiv_app_library.db"
if is_frozen() and not target.exists():
legacy = executable_dir() / "pixiv_app_library.db"
if legacy.is_file():
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(legacy, target)
return target
def downloads_root() -> Path:
path = executable_dir() / "Sakura_Downloads"
path.mkdir(parents=True, exist_ok=True)
return path
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
import os
from pathlib import Path
def playwright_browsers_root() -> Path | None:
local_appdata = os.environ.get("LOCALAPPDATA", "").strip()
if not local_appdata:
return None
return Path(local_appdata) / "ms-playwright"
def configure_playwright_runtime_env() -> Path | None:
root = playwright_browsers_root()
if root is None:
return None
if root.exists() and "PLAYWRIGHT_BROWSERS_PATH" not in os.environ:
os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(root)
return root
def preferred_chromium_executable() -> Path | None:
root = configure_playwright_runtime_env()
if root is None or not root.exists():
return None
candidates: list[tuple[int, float, Path]] = []
for browser_dir in root.glob("chromium-*"):
if not browser_dir.is_dir():
continue
revision = _parse_revision(browser_dir.name)
for relative in (
Path("chrome-win64") / "chrome.exe",
Path("chrome-win") / "chrome.exe",
):
executable = browser_dir / relative
if executable.is_file():
try:
mtime = executable.stat().st_mtime
except OSError:
mtime = 0.0
candidates.append((revision, mtime, executable))
break
if not candidates:
return None
candidates.sort(key=lambda item: (item[0], item[1]), reverse=True)
return candidates[0][2]
def explain_missing_browser(exc: Exception, *, feature_name: str) -> str:
text = str(exc)
lowered = text.lower()
if "executable doesn't exist" not in lowered and "browsertype.launch_persistent_context" not in lowered:
return text
return (
f"{feature_name} 需要 Playwright Chromium 浏览器,但当前系统里没有可用的浏览器文件。\n"
"请先运行同目录下的 install_playwright_chromium.bat,安装完成后重新打开程序再试。"
)
def _parse_revision(name: str) -> int:
try:
return int(name.rsplit("-", 1)[-1])
except (TypeError, ValueError):
return 0
+24
View File
@@ -0,0 +1,24 @@
"""Plugin system: base types, manager, generator, optional market client."""
from pixiv_app.core.plugin.base import (
BasePlugin,
PluginAuthError,
PluginDownloadError,
PluginError,
PluginManifest,
PluginParseError,
Resource,
)
from pixiv_app.core.plugin.manager import PluginEvent, PluginManager
__all__ = [
"BasePlugin",
"PluginAuthError",
"PluginDownloadError",
"PluginError",
"PluginEvent",
"PluginManager",
"PluginManifest",
"PluginParseError",
"Resource",
]
+119
View File
@@ -0,0 +1,119 @@
"""
Multi-site plugin core abstractions.
解析与下载采用同步接口,便于与现有 requests / 线程模型集成;
若站点 SDK 仅提供 asyncio,可在插件内部使用 asyncio.run 包装。
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
@dataclass
class Resource:
"""Unified resource model returned by plugins."""
id: str
url: str
title: str
author: str
author_id: str
files: list[dict[str, Any]]
metadata: dict[str, Any] = field(default_factory=dict)
thumbnail: str | None = None
created_at: str | None = None
@dataclass
class PluginManifest:
"""Optional manifest shipped as manifest.json next to plugin.py."""
name: str
version: str
domain: str
author: str = ""
description: str = ""
requires_auth: bool = False
min_platform_version: str = "1.0.0"
tags: list[str] = field(default_factory=list)
dependencies: list[str] = field(default_factory=list)
class PluginError(Exception):
"""Base class for plugin failures."""
class PluginAuthError(PluginError):
"""Authentication or authorization failed."""
class PluginParseError(PluginError):
"""URL or API response could not be parsed."""
class PluginDownloadError(PluginError):
"""Download or filesystem error."""
class BasePlugin(ABC):
"""Interface implemented by every site plugin."""
@property
@abstractmethod
def domain(self) -> str:
"""Primary domain, e.g. pixiv.net."""
@property
@abstractmethod
def name(self) -> str:
"""Human-readable plugin name."""
@property
@abstractmethod
def version(self) -> str:
"""Semantic version string."""
@abstractmethod
def can_handle(self, url: str) -> bool:
"""Return True if this plugin should process the URL or raw token."""
@abstractmethod
def parse(self, url: str) -> list[Resource]:
"""Resolve URL into one or more downloadable resources."""
@abstractmethod
def download(self, resource: Resource, save_path: Path) -> list[Path]:
"""Download files for ``resource`` under ``save_path``."""
@abstractmethod
def get_headers(self) -> dict[str, str]:
"""HTTP headers for requests (Cookie, Bearer token, etc.)."""
def validate(self) -> bool:
"""Optional self-check (credentials present, deps installed)."""
return True
def get_config_schema(self) -> dict[str, Any]:
"""Optional JSON-schema-like dict for dynamic GUI forms."""
return {}
def load_config(self) -> dict[str, Any]:
"""Override to load persisted plugin settings."""
return {}
def save_config(self, data: dict[str, Any]) -> None:
"""Override to persist plugin settings."""
__all__ = [
"BasePlugin",
"PluginAuthError",
"PluginDownloadError",
"PluginError",
"PluginManifest",
"PluginParseError",
"Resource",
]
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
import re
from datetime import datetime
from pathlib import Path
from typing import Any
from jinja2 import Environment, FileSystemLoader, select_autoescape
def _class_name_from_plugin_name(name: str) -> str:
parts = re.split(r"[\s_]+", name.strip())
stem = "".join(p[:1].upper() + p[1:].lower() if p else "" for p in parts if p)
return f"{stem}Plugin" if stem else "GeneratedPlugin"
class PluginGenerator:
"""Generate a skeleton plugin directory from Jinja2 templates."""
def __init__(self, template_dir: str | Path | None = None) -> None:
base = Path(__file__).resolve().parent / "templates"
self.template_dir = Path(template_dir) if template_dir else base
self.env = Environment(
loader=FileSystemLoader(str(self.template_dir)),
autoescape=select_autoescape(enabled_extensions=()),
)
def generate(
self,
name: str,
domain: str,
author: str,
description: str,
*,
output_root: str | Path = "plugins",
) -> Path:
slug = name.strip().lower().replace(" ", "_")
plugin_dir = Path(output_root) / slug
plugin_dir.mkdir(parents=True, exist_ok=True)
context: dict[str, Any] = {
"name": name.strip(),
"domain": domain.strip(),
"author": author.strip(),
"description": description.strip(),
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"class_name": _class_name_from_plugin_name(name),
}
py_template = self.env.get_template("base_plugin.py.jinja")
(plugin_dir / "plugin.py").write_text(py_template.render(**context), encoding="utf-8")
manifest_template = self.env.get_template("manifest.json.jinja")
(plugin_dir / "manifest.json").write_text(manifest_template.render(**context), encoding="utf-8")
init_file = plugin_dir / "__init__.py"
if not init_file.exists():
init_file.write_text('"""Generated plugin."""\n', encoding="utf-8")
return plugin_dir.resolve()
def generate_from_gui(self, form_data: dict[str, str], *, output_root: str | Path = "plugins") -> Path:
return self.generate(
name=form_data["name"],
domain=form_data["domain"],
author=form_data["author"],
description=form_data["description"],
output_root=output_root,
)
+194
View File
@@ -0,0 +1,194 @@
from __future__ import annotations
import importlib.util
import json
import logging
import sys
from collections import defaultdict
from enum import Enum
from pathlib import Path
from typing import Any, Callable
from pixiv_app.core.plugin.base import BasePlugin, PluginManifest
logger = logging.getLogger(__name__)
class PluginEvent(str, Enum):
BEFORE_LOAD = "before_load"
AFTER_LOAD = "after_load"
BEFORE_UNLOAD = "before_unload"
AFTER_UNLOAD = "after_unload"
ERROR = "error"
class PluginManager:
"""Discover and load plugins from filesystem directories."""
def __init__(self, plugin_dirs: list[str | Path]) -> None:
self.plugin_dirs = [Path(d).resolve() for d in plugin_dirs]
self.plugins: dict[str, BasePlugin] = {}
self.manifests: dict[str, PluginManifest] = {}
self.loaded_modules: dict[str, Any] = {}
self._paths: dict[str, Path] = {}
self.load_errors: dict[Path, str] = {}
self.event_handlers: dict[PluginEvent, list[Callable[..., None]]] = defaultdict(list)
def on_event(self, event: PluginEvent, callback: Callable[..., None]) -> None:
self.event_handlers[event].append(callback)
def _emit(self, event: PluginEvent, *args: Any, **kwargs: Any) -> None:
for cb in self.event_handlers.get(event, []):
try:
cb(*args, **kwargs)
except Exception as exc:
logger.exception("Plugin event handler failed: %s", exc)
def discover(self) -> list[Path]:
"""Return paths to plugin.py files."""
found: list[Path] = []
for root in self.plugin_dirs:
if not root.is_dir():
continue
for subdir in sorted(root.iterdir()):
if not subdir.is_dir():
continue
candidate = subdir / "plugin.py"
if candidate.is_file():
found.append(candidate)
return found
def _load_manifest(self, plugin_dir: Path) -> PluginManifest | None:
manifest_path = plugin_dir / "manifest.json"
if not manifest_path.is_file():
return None
try:
data = json.loads(manifest_path.read_text(encoding="utf-8"))
return PluginManifest(
name=str(data.get("name", plugin_dir.name)),
version=str(data.get("version", "0.0.0")),
domain=str(data.get("domain", "")),
author=str(data.get("author", "")),
description=str(data.get("description", "")),
requires_auth=bool(data.get("requires_auth", False)),
min_platform_version=str(data.get("min_platform_version", "1.0.0")),
tags=list(data.get("tags") or []),
dependencies=list(data.get("dependencies") or []),
)
except Exception as exc:
logger.warning("Invalid manifest %s: %s", manifest_path, exc)
return None
def load_plugin(self, plugin_path: Path) -> BasePlugin | None:
"""Load a single plugin from ``.../plugin.py``."""
plugin_path = plugin_path.resolve()
plugin_dir = plugin_path.parent
module_name = f"sakura_dynamic_plugin_{plugin_dir.name}"
module_stem = module_name.rsplit(".", 1)[0]
for loaded_name in list(sys.modules):
if loaded_name == module_name or loaded_name.startswith(f"{module_name}."):
del sys.modules[loaded_name]
self.load_errors.pop(plugin_path, None)
self._emit(PluginEvent.BEFORE_LOAD, plugin_path)
spec = importlib.util.spec_from_file_location(
module_name,
plugin_path,
submodule_search_locations=[str(plugin_dir)],
)
if spec is None or spec.loader is None:
self._emit(PluginEvent.ERROR, plugin_path, "invalid spec")
self.load_errors[plugin_path] = "invalid spec"
return None
module = importlib.util.module_from_spec(spec)
module.__package__ = module_stem
module.__file__ = str(plugin_path)
sys.modules[module_name] = module
try:
spec.loader.exec_module(module)
except Exception as exc:
logger.exception("Failed to load %s", plugin_path)
self._emit(PluginEvent.ERROR, plugin_path, exc)
self.load_errors[plugin_path] = f"{type(exc).__name__}: {exc}"
return None
plugin_class = getattr(module, "plugin_class", None)
if plugin_class is None:
logger.error("Module %s has no plugin_class export", plugin_path)
self._emit(PluginEvent.ERROR, plugin_path, "missing plugin_class")
self.load_errors[plugin_path] = "missing plugin_class"
return None
try:
plugin = plugin_class()
except Exception as exc:
logger.exception("Failed to instantiate plugin from %s", plugin_path)
self._emit(PluginEvent.ERROR, plugin_path, exc)
self.load_errors[plugin_path] = f"{type(exc).__name__}: {exc}"
return None
if not isinstance(plugin, BasePlugin):
logger.error("plugin_class is not BasePlugin: %s", plugin_path)
self.load_errors[plugin_path] = "plugin_class is not BasePlugin"
return None
key = plugin.name
manifest = self._load_manifest(plugin_dir)
if manifest:
self.manifests[key] = manifest
self.plugins[key] = plugin
self.loaded_modules[key] = module
self._paths[key] = plugin_path
self.load_errors.pop(plugin_path, None)
self._emit(PluginEvent.AFTER_LOAD, plugin)
return plugin
def load_all(self) -> dict[str, BasePlugin]:
for path in self.discover():
try:
self.load_plugin(path)
except Exception as exc:
logger.exception("Failed during load_all %s: %s", path, exc)
return self.plugins
def reload_all_from_disk(self) -> None:
"""Unload every plugin entry then scan directories again."""
for name in list(self.plugins.keys()):
self.unload_plugin(name)
self.load_errors.clear()
self.load_all()
def reload_plugin(self, name: str) -> bool:
path = self._paths.get(name)
if path is None:
return False
self.unload_plugin(name)
return self.load_plugin(path) is not None
def get_plugin_path(self, name: str) -> Path | None:
return self._paths.get(name)
def unload_plugin(self, name: str) -> None:
self._emit(PluginEvent.BEFORE_UNLOAD, name)
self.plugins.pop(name, None)
mod = self.loaded_modules.pop(name, None)
self.manifests.pop(name, None)
self._paths.pop(name, None)
if mod is not None and getattr(mod, "__name__", None) in sys.modules:
del sys.modules[mod.__name__]
self._emit(PluginEvent.AFTER_UNLOAD, name)
def get_plugin_for_url(self, url: str) -> BasePlugin | None:
for plugin in self.plugins.values():
if plugin.can_handle(url):
return plugin
return None
def check_dependencies(self, manifest: PluginManifest) -> bool:
for dep in manifest.dependencies:
if dep not in self.plugins:
return False
return True
+29
View File
@@ -0,0 +1,29 @@
"""
Optional plugin market client — stub for future HTTP catalog integration.
"""
from __future__ import annotations
from typing import Any
import requests
class PluginMarketClient:
def __init__(self, api_url: str = "") -> None:
self.api_url = api_url.rstrip("/")
def list_plugins(self) -> list[dict[str, Any]]:
if not self.api_url:
return []
response = requests.get(f"{self.api_url}/plugins", timeout=15)
response.raise_for_status()
data = response.json()
return list(data.get("plugins", []))
def install_plugin(self, _plugin_name: str) -> bool:
"""Download zip and extract into plugins/ — not implemented."""
return False
def check_updates(self) -> list[dict[str, Any]]:
return []
@@ -0,0 +1,31 @@
"""
{{ name }} plugin — auto-generated at {{ generated_at }}.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from pixiv_app.core.plugin.base import BasePlugin, Resource
class {{ class_name }}(BasePlugin):
domain = "{{ domain }}"
name = "{{ name }}"
version = "1.0.0"
def can_handle(self, url: str) -> bool:
return "{{ domain }}" in url.lower()
def parse(self, url: str) -> list[Resource]:
raise NotImplementedError("Implement URL parsing and build Resource list.")
def download(self, resource: Resource, save_path: Path) -> list[Path]:
raise NotImplementedError("Download files for resource into save_path.")
def get_headers(self) -> dict[str, str]:
return {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
plugin_class = {{ class_name }}
@@ -0,0 +1,11 @@
{
"name": "{{ name }}",
"version": "1.0.0",
"domain": "{{ domain }}",
"author": "{{ author }}",
"description": "{{ description }}",
"requires_auth": false,
"tags": ["general"],
"min_platform_version": "1.0.0",
"dependencies": []
}
+292
View File
@@ -0,0 +1,292 @@
from __future__ import annotations
import random
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from typing import Any, Optional
import requests
@dataclass
class ProxyInfo:
ip: str
port: int
protocol: str = "http"
source: str = "manual"
location: str = ""
org: str = ""
last_check: float = 0.0
success_count: int = 0
fail_count: int = 0
@property
def proxy_url(self) -> str:
return f"{self.protocol}://{self.ip}:{self.port}"
@property
def is_alive(self) -> bool:
return self.success_count > 0 or self.fail_count < 3
def parse_proxy_text(text: str) -> list[ProxyInfo]:
proxies: list[ProxyInfo] = []
seen: set[tuple[str, int, str]] = set()
for raw_line in text.splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
if "://" not in line:
line = f"http://{line}"
match = re.match(r"^(https?)://([\w\.\-]+):(\d{2,5})$", line, re.IGNORECASE)
if not match:
continue
protocol, ip, port_text = match.groups()
key = (ip, int(port_text), protocol.lower())
if key in seen:
continue
seen.add(key)
proxies.append(ProxyInfo(ip=ip, port=int(port_text), protocol=protocol.lower(), source="manual"))
return proxies
class QuakeClient:
def __init__(
self,
api_key: str = "",
cookie: str = "",
mode: str = "api_v3",
request_kwargs: Optional[dict[str, Any]] = None,
):
self.api_key = api_key.strip()
self.cookie = cookie.strip()
self.mode = mode
self.request_kwargs = dict(request_kwargs or {})
self.base_url = "https://quake.360.net/api/v3"
self.web_assoc_url = "https://quake.360.net/api/search/field/association/quake_service"
def _api_headers(self) -> dict[str, str]:
return {
"X-QuakeToken": self.api_key,
"Content-Type": "application/json",
}
def _web_headers(self) -> dict[str, str]:
headers = {
"Accept": "application/json, text/plain, */*",
"Content-Type": "application/json",
"Origin": "https://quake.360.net",
"Referer": "https://quake.360.net/quake/",
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
),
}
if self.cookie:
headers["Cookie"] = self.cookie
return headers
def search_ips_api(self, query: str, size: int = 100, start: int = 0) -> list[dict[str, Any]]:
if not self.api_key:
return []
url = f"{self.base_url}/search/quake_service"
payload = {"query": query, "start": start, "size": min(size, 500)}
kwargs = dict(self.request_kwargs)
kwargs.setdefault("timeout", 30)
try:
response = requests.post(url, headers=self._api_headers(), json=payload, **kwargs)
if response.status_code != 200:
return []
body = response.json()
return body.get("data", []) if isinstance(body, dict) else []
except Exception:
return []
def search_ips_web_assoc(self, search_content: str) -> list[dict[str, Any]]:
if not self.cookie:
return []
payload = {
"search_content": search_content,
"device": {
"device_type": "PC",
"os": "Windows",
"os_version": "10.0",
"language": "zh_CN",
"network": "4g",
"browser_info": "Chrome",
"fingerprint": "pixiv-proxy-pool",
"user_agent": self._web_headers()["User-Agent"],
"date": time.strftime("%Y/%m/%d %H:%M:%S"),
"UUID": "pixiv-proxy-association",
},
}
kwargs = dict(self.request_kwargs)
kwargs.setdefault("timeout", 30)
try:
response = requests.post(
self.web_assoc_url,
headers=self._web_headers(),
json=payload,
**kwargs,
)
if response.status_code != 200:
return []
text = response.text
ips = sorted(set(re.findall(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", text)))
return [{"ip": ip, "port": 80, "organization": "", "location": {}} for ip in ips]
except Exception:
return []
def get_foreign_proxies(
self,
size: int = 120,
countries: Optional[list[str]] = None,
) -> list[ProxyInfo]:
proxy_ports = [80, 443, 8080, 3128, 8888, 1080, 8118]
port_query = " OR ".join([f"port:{port}" for port in proxy_ports])
country_codes = countries or ["US", "JP", "KR", "SG", "HK", "TW", "DE", "FR", "GB", "CA", "AU"]
country_query = " OR ".join([f'country:"{code}"' for code in country_codes])
query = f"({port_query}) AND ({country_query}) AND response:ok"
if self.mode == "web_assoc":
data = self.search_ips_web_assoc(query)
else:
data = self.search_ips_api(query=query, size=size)
proxies: list[ProxyInfo] = []
seen: set[tuple[str, int, str]] = set()
for item in data:
if not isinstance(item, dict):
continue
ip = item.get("ip")
port = item.get("port")
if not ip or not isinstance(port, int):
continue
protocol = "https" if port in (443, 8443) else "http"
key = (ip, port, protocol)
if key in seen:
continue
seen.add(key)
location_obj = item.get("location", {}) if isinstance(item.get("location"), dict) else {}
location = "/".join(
[str(location_obj.get("country", "")).strip(), str(location_obj.get("province", "")).strip()]
).strip("/")
proxies.append(
ProxyInfo(
ip=ip,
port=port,
protocol=protocol,
source=f"quake:{self.mode}",
location=location,
org=str(item.get("organization", "") or item.get("org", "")),
)
)
return proxies
class ProxyPool:
def __init__(self, test_url: str = "https://httpbin.org/ip", test_timeout: int = 6):
self.proxies: list[ProxyInfo] = []
self.working_proxies: list[ProxyInfo] = []
self.current_index = 0
self.test_url = test_url
self.test_timeout = test_timeout
self.session = requests.Session()
self.session.headers.update({"User-Agent": "SakuraDownloaderProxyPool/1.0"})
self._lock = threading.Lock()
def add_proxies(self, proxies: list[ProxyInfo]) -> None:
with self._lock:
existing = {(p.ip, p.port, p.protocol) for p in self.proxies}
for proxy in proxies:
key = (proxy.ip, proxy.port, proxy.protocol)
if key not in existing:
existing.add(key)
self.proxies.append(proxy)
def verify_proxy(self, proxy: ProxyInfo) -> bool:
target_proxies = {"http": proxy.proxy_url, "https": proxy.proxy_url}
try:
response = self.session.get(
self.test_url,
proxies=target_proxies,
timeout=self.test_timeout,
verify=False,
)
if response.status_code == 200:
with self._lock:
proxy.last_check = time.time()
proxy.success_count += 1
proxy.fail_count = 0
return True
except Exception:
pass
with self._lock:
proxy.fail_count += 1
return False
def verify_all(self, max_workers: int = 16, max_proxies: int = 80) -> list[ProxyInfo]:
with self._lock:
to_verify = list(self.proxies[:max_proxies])
self.working_proxies = []
if not to_verify:
return self.working_proxies
with ThreadPoolExecutor(max_workers=max_workers) as executor:
future_map = {executor.submit(self.verify_proxy, proxy): proxy for proxy in to_verify}
for future in as_completed(future_map):
proxy = future_map[future]
try:
if future.result():
with self._lock:
self.working_proxies.append(proxy)
except Exception:
continue
with self._lock:
return list(self.working_proxies)
def collect_until_target(
self,
quake_client: QuakeClient,
target_working: int = 5,
stable_rounds: int = 2,
fetch_batch_size: int = 60,
verify_batch_size: int = 30,
max_rounds: int = 6,
max_workers: int = 16,
countries: Optional[list[str]] = None,
) -> list[ProxyInfo]:
stable_hits = 0
for _ in range(max_rounds):
if stable_hits >= stable_rounds:
break
new_items = quake_client.get_foreign_proxies(size=fetch_batch_size, countries=countries)
if new_items:
self.add_proxies(new_items)
self.verify_all(max_workers=max_workers, max_proxies=verify_batch_size)
with self._lock:
working_count = len(self.working_proxies)
if working_count >= target_working:
stable_hits += 1
else:
stable_hits = 0
with self._lock:
return list(self.working_proxies)
def get_proxy(self, rotate: bool = True) -> Optional[ProxyInfo]:
with self._lock:
if not self.working_proxies:
return None
if not rotate:
return random.choice(self.working_proxies)
proxy = self.working_proxies[self.current_index % len(self.working_proxies)]
self.current_index += 1
return proxy
def mark_failed(self, proxy: ProxyInfo) -> None:
with self._lock:
proxy.fail_count += 1
if proxy.fail_count >= 3 and proxy in self.working_proxies:
self.working_proxies.remove(proxy)
+145
View File
@@ -0,0 +1,145 @@
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
from pixiv_app.core.paths import runtime_path
DEFAULT_THUMBNAIL_DIR = runtime_path("thumbnails")
SUPPORTED_IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
@dataclass
class ThumbnailResult:
artwork_id: int
page_index: int
source_path: str
thumbnail_path: str
width: int
height: int
generated: bool
class ThumbnailCache:
def __init__(
self,
*,
db_path: str | Path = DEFAULT_LIBRARY_DB,
cache_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
size: tuple[int, int] = (360, 360),
quality: int = 86,
) -> None:
self.db_path = Path(db_path)
self.cache_dir = Path(cache_dir)
self.size = size
self.quality = quality
self.cache_dir.mkdir(parents=True, exist_ok=True)
def ensure_for_artwork(self, artwork_id: int) -> list[ThumbnailResult]:
library = PixivLibrary(self.db_path)
try:
rows = library.connection.execute(
"""
SELECT artwork_id, page_index, file_path
FROM artwork_files
WHERE artwork_id = ?
ORDER BY page_index ASC
""",
(artwork_id,),
).fetchall()
return [result for row in rows if (result := self.ensure_for_file(library, row)) is not None]
finally:
library.close()
def build_missing(self, *, limit: int = 200) -> list[ThumbnailResult]:
library = PixivLibrary(self.db_path)
try:
rows = library.connection.execute(
"""
SELECT f.artwork_id, f.page_index, f.file_path
FROM artwork_files f
LEFT JOIN thumbnail_cache t
ON t.artwork_id = f.artwork_id AND t.page_index = f.page_index
WHERE t.thumbnail_path IS NULL
OR t.source_path != f.file_path
ORDER BY f.updated_at DESC
LIMIT ?
""",
(limit,),
).fetchall()
return [result for row in rows if (result := self.ensure_for_file(library, row)) is not None]
finally:
library.close()
def ensure_for_file(self, library: PixivLibrary, row) -> ThumbnailResult | None:
source_path = Path(str(row["file_path"]))
if source_path.suffix.lower() not in SUPPORTED_IMAGE_SUFFIXES or not source_path.exists():
return None
stat = source_path.stat()
thumbnail_path = self._thumbnail_path(int(row["artwork_id"]), int(row["page_index"]), source_path)
cached = library.connection.execute(
"""
SELECT thumbnail_path, width, height, source_mtime, source_size
FROM thumbnail_cache
WHERE artwork_id = ? AND page_index = ?
""",
(int(row["artwork_id"]), int(row["page_index"])),
).fetchone()
if (
cached
and Path(str(cached["thumbnail_path"])).exists()
and float(cached["source_mtime"]) == stat.st_mtime
and int(cached["source_size"]) == stat.st_size
):
return ThumbnailResult(
artwork_id=int(row["artwork_id"]),
page_index=int(row["page_index"]),
source_path=str(source_path),
thumbnail_path=str(cached["thumbnail_path"]),
width=int(cached["width"]),
height=int(cached["height"]),
generated=False,
)
width, height = self._generate(source_path, thumbnail_path)
library.upsert_thumbnail(
artwork_id=int(row["artwork_id"]),
page_index=int(row["page_index"]),
source_path=str(source_path),
thumbnail_path=str(thumbnail_path),
width=width,
height=height,
source_mtime=stat.st_mtime,
source_size=stat.st_size,
)
return ThumbnailResult(
artwork_id=int(row["artwork_id"]),
page_index=int(row["page_index"]),
source_path=str(source_path),
thumbnail_path=str(thumbnail_path),
width=width,
height=height,
generated=True,
)
def _generate(self, source_path: Path, thumbnail_path: Path) -> tuple[int, int]:
try:
from PIL import Image, ImageOps
except ImportError as exc:
raise RuntimeError("Pillow is required to generate thumbnails. Install it with: pip install Pillow") from exc
thumbnail_path.parent.mkdir(parents=True, exist_ok=True)
with Image.open(source_path) as image:
image = ImageOps.exif_transpose(image)
image.thumbnail(self.size, Image.Resampling.LANCZOS)
if image.mode not in {"RGB", "L"}:
image = image.convert("RGB")
image.save(thumbnail_path, format="JPEG", quality=self.quality, optimize=True)
return image.size
def _thumbnail_path(self, artwork_id: int, page_index: int, source_path: Path) -> Path:
bucket = str(abs(artwork_id) % 1000).zfill(3)
return self.cache_dir / bucket / f"{artwork_id}_{page_index}_{source_path.stem}.jpg"
+405
View File
@@ -0,0 +1,405 @@
from __future__ import annotations
import json
import time
import uuid
from dataclasses import asdict
from datetime import datetime
from pathlib import Path
from typing import Any
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.cookie_import import (
cookie_summary,
cookies_to_playwright,
parse_cookie_text,
save_playwright_cookies,
)
from pixiv_app.core.paths import (
app_session_file,
downloads_root,
plugin_roots,
plugins_root as app_plugins_root,
runtime_path,
)
from pixiv_app.core.plugin.base import Resource
from pixiv_app.core.plugin.manager import PluginManager
def project_root() -> Path:
return app_plugins_root().parent
class XMediaService:
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
VIDEO_EXTS = {".mp4", ".mov", ".webm", ".m4v"}
def __init__(self, *, plugins_root: str | Path | None = None, save_root: str | Path | None = None) -> None:
self.plugins_root = Path(plugins_root) if plugins_root else app_plugins_root()
self.save_root = Path(save_root) if save_root else downloads_root() / "x"
self.cookie_path = runtime_path("x_cookies.json")
self.follow_store_path = runtime_path("x_followed_authors.json")
self._manager: PluginManager | None = None
self._preview_cache: dict[str, list[Resource]] = {}
def import_cookie_text(self, text: str) -> dict[str, Any]:
cookies = parse_cookie_text(text)
x_cookies = cookies_to_playwright(cookies, ("x.com", "twitter.com"))
path = save_playwright_cookies(x_cookies, self.cookie_path)
plugin = self._get_plugin()
if hasattr(plugin, "reset_playwright_session"):
plugin.reset_playwright_session()
return {"count": len(x_cookies), "path": str(path), "summary": cookie_summary(x_cookies)}
def preview_author_media(self, username: str, *, media_type: str = "all", max_items: int = 120) -> dict[str, Any]:
handle = _normalize_username(username)
media_type = _normalize_media_type(media_type)
max_items = max(1, min(int(max_items), 500))
url = f"https://x.com/{handle}/media"
plugin = self._get_plugin()
if not plugin.validate():
raise ValueError("X plugin is unavailable. Install playwright/httpx and run playwright install chromium first.")
resources = plugin.parse(url)
resources = _filter_resources(resources, media_type)[:max_items]
preview_id = uuid.uuid4().hex
self._preview_cache[preview_id] = resources
return {
"preview_id": preview_id,
"username": handle,
"media_type": media_type,
"count": len(resources),
"items": [_resource_to_json(item, i) for i, item in enumerate(resources)],
"created_at": int(time.time()),
}
def download_preview(self, preview_id: str, *, selected_ids: list[str] | None = None) -> dict[str, Any]:
resources = self._preview_cache.get(preview_id)
if resources is None:
raise ValueError("Preview expired. Re-run the X media preview.")
selected = set(selected_ids or [])
if selected:
resources = [item for item in resources if item.id in selected]
if not resources:
raise ValueError("No X media items selected.")
plugin = self._get_plugin()
self.save_root.mkdir(parents=True, exist_ok=True)
results: list[dict[str, Any]] = []
downloaded = 0
failed = 0
for resource in resources:
try:
paths = plugin.download(resource, self.save_root)
downloaded += len(paths)
results.append(
{
"id": resource.id,
"ok": bool(paths),
"paths": [str(path) for path in paths],
"title": resource.title,
"synced_files": 0,
}
)
if not paths:
failed += 1
except Exception as exc: # Keep batch downloads moving.
failed += 1
results.append({"id": resource.id, "ok": False, "error": str(exc), "title": resource.title})
return {
"requested": len(resources),
"downloaded_files": downloaded,
"synced_files": 0,
"failed_items": failed,
"items": results,
}
def list_local_media(self, *, limit: int = 160, offset: int = 0) -> dict[str, Any]:
self.save_root.mkdir(parents=True, exist_ok=True)
limit = max(1, min(int(limit), 500))
offset = max(0, int(offset))
files = [
path
for path in self.save_root.rglob("*")
if path.is_file() and path.suffix.lower() in self.IMAGE_EXTS | self.VIDEO_EXTS
]
files.sort(key=lambda path: path.stat().st_mtime, reverse=True)
page = files[offset : offset + limit]
return {
"total": len(files),
"items": [self._local_media_to_json(path) for path in page],
}
def delete_local_media(self, media_ids: list[str]) -> dict[str, Any]:
deleted: list[str] = []
failed: list[dict[str, str]] = []
for media_id in media_ids:
try:
path = self.resolve_local_media(media_id)
path.unlink()
deleted.append(media_id)
except Exception as exc:
failed.append({"id": media_id, "error": str(exc)})
return {"deleted": deleted, "failed": failed}
def resolve_local_media(self, media_id: str) -> Path:
rel_text = _decode_media_id(media_id)
target = (self.save_root / rel_text).resolve()
root = self.save_root.resolve()
if root != target and root not in target.parents:
raise ValueError("Illegal media path.")
if not target.is_file():
raise ValueError("Local media file does not exist.")
return target
def list_followed_authors(self) -> list[dict[str, Any]]:
rows = [item for item in self._load_follow_rows() if bool(item.get("enabled", True))]
rows.sort(key=lambda item: ((str(item.get("creator_name") or "") or str(item.get("username") or "")).lower()))
return rows
def set_followed_author(self, username: str, *, followed: bool = True, creator_name: str = "") -> dict[str, Any]:
handle = _normalize_username(username)
rows = self._load_follow_rows()
row = next((item for item in rows if str(item.get("username", "")).lower() == handle.lower()), None)
now = _now_text()
if row is None:
row = {
"username": handle,
"creator_name": creator_name.strip() or handle,
"enabled": True,
"profile_url": f"https://x.com/{handle}",
"latest_tweet_id": "",
"latest_published_at": "",
"last_checked_at": "",
"last_error": "",
"seen_tweet_ids": [],
"created_at": now,
}
rows.append(row)
row["username"] = handle
row["creator_name"] = creator_name.strip() or str(row.get("creator_name") or handle)
row["enabled"] = bool(followed)
row["profile_url"] = f"https://x.com/{handle}"
row["updated_at"] = now
if not followed:
row["last_error"] = ""
self._save_follow_rows(rows)
return dict(row)
def sync_followed_authors(self, *, download: bool = True, max_new_per_author: int = 20) -> dict[str, Any]:
followed = self.list_followed_authors()
if not followed:
return {"authors": []}
plugin = self._get_plugin()
if not plugin.validate():
raise ValueError("X plugin is unavailable. Import a valid X cookie and ensure Playwright is installed.")
results: list[dict[str, Any]] = []
rows = self._load_follow_rows()
for row in rows:
if not bool(row.get("enabled", True)):
continue
handle = str(row.get("username") or "").strip()
if not handle:
continue
try:
parse_for_sync = getattr(plugin, "parse_for_sync", plugin.parse)
resources = parse_for_sync(f"https://x.com/{handle}")
grouped = _group_resources_by_tweet(resources)
seen_ids = [str(item) for item in row.get("seen_tweet_ids", []) if str(item).strip()]
seen_set = set(seen_ids)
new_groups = [item for item in grouped if item["tweet_id"] and item["tweet_id"] not in seen_set]
new_groups = new_groups[: max(0, int(max_new_per_author))]
downloaded_files = 0
synced_files = 0
created_alerts = 0
new_ids: list[str] = []
for group in new_groups:
representative = group["resource"]
new_ids.append(group["tweet_id"])
if download:
for resource in group["resources"]:
paths = plugin.download(resource, self.save_root)
downloaded_files += len(paths)
latest_group = grouped[0] if grouped else None
if latest_group is not None:
representative = latest_group["resource"]
row["creator_name"] = representative.author or str(row.get("creator_name") or handle)
row["latest_tweet_id"] = latest_group["tweet_id"]
row["latest_published_at"] = representative.created_at or str(row.get("latest_published_at") or "")
row["last_checked_at"] = _now_text()
row["last_error"] = ""
row["updated_at"] = _now_text()
row["seen_tweet_ids"] = _merge_seen_ids(new_ids, seen_ids)
results.append(
{
"username": handle,
"creator_name": row.get("creator_name") or handle,
"new_posts": len(new_groups),
"downloaded_files": downloaded_files,
"synced_files": synced_files,
"alerts": created_alerts,
"error": "",
}
)
except Exception as exc:
row["last_checked_at"] = _now_text()
row["last_error"] = str(exc)
row["updated_at"] = _now_text()
results.append(
{
"username": handle,
"creator_name": row.get("creator_name") or handle,
"new_posts": 0,
"downloaded_files": 0,
"synced_files": 0,
"alerts": 0,
"error": str(exc),
}
)
self._save_follow_rows(rows)
return {"authors": results}
def _get_plugin(self):
if self._manager is None:
self._manager = PluginManager(plugin_roots() if self.plugins_root == app_plugins_root() else [self.plugins_root])
self._manager.load_all()
plugin = self._manager.plugins.get("X")
if plugin is None:
raise ValueError("X plugin is missing. Confirm plugins/x exists.")
return plugin
def _load_follow_rows(self) -> list[dict[str, Any]]:
if not self.follow_store_path.is_file():
return []
try:
payload = json.loads(self.follow_store_path.read_text(encoding="utf-8"))
except Exception:
return []
if isinstance(payload, list):
return [dict(item) for item in payload if isinstance(item, dict)]
return []
def _save_follow_rows(self, rows: list[dict[str, Any]]) -> None:
self.follow_store_path.parent.mkdir(parents=True, exist_ok=True)
self.follow_store_path.write_text(
json.dumps(rows, ensure_ascii=False, indent=2),
encoding="utf-8",
)
def _session_payload(self) -> dict[str, Any]:
return SessionStore(app_session_file()).load()
def _local_media_to_json(self, path: Path) -> dict[str, Any]:
rel = path.relative_to(self.save_root)
stat = path.stat()
kind = "video" if path.suffix.lower() in self.VIDEO_EXTS else "image"
media_id = _encode_media_id(str(rel).replace("\\", "/"))
return {
"id": media_id,
"name": path.name,
"relative_path": str(rel).replace("\\", "/"),
"kind": kind,
"size": stat.st_size,
"modified_at": int(stat.st_mtime),
"url": f"/media/x-local/{media_id}",
}
def _normalize_username(username: str) -> str:
value = username.strip()
if value.startswith("@"):
value = value[1:]
if "/" in value:
parts = [item for item in value.replace("https://", "").replace("http://", "").split("/") if item]
value = parts[1] if parts and parts[0].lower() in {"x.com", "twitter.com", "www.x.com", "www.twitter.com"} else parts[0]
if not value:
raise ValueError("Please enter an X username.")
if len(value) > 15 or not value.replace("_", "").isalnum():
raise ValueError("Invalid X username format.")
return value
def _normalize_media_type(media_type: str) -> str:
value = (media_type or "all").strip().lower()
if value not in {"all", "image", "video"}:
raise ValueError("media_type must be all, image, or video.")
return value
def _filter_resources(resources: list[Resource], media_type: str) -> list[Resource]:
if media_type == "all":
return resources
return [item for item in resources if any(file.get("type", "image") == media_type for file in item.files)]
def _resource_to_json(resource: Resource, index: int) -> dict[str, Any]:
payload = asdict(resource)
payload["index"] = index
payload["kind"] = "video" if any(file.get("type") == "video" for file in resource.files) else "image"
payload["file_count"] = len(resource.files)
return payload
def _group_resources_by_tweet(resources: list[Resource]) -> list[dict[str, Any]]:
grouped: dict[str, dict[str, Any]] = {}
ordered: list[str] = []
for resource in resources:
tweet_id = str(resource.metadata.get("tweet_id") or resource.id or "")
key = tweet_id or resource.id
if key not in grouped:
grouped[key] = {
"tweet_id": tweet_id,
"resource": resource,
"resources": [resource],
}
ordered.append(key)
else:
grouped[key]["resources"].append(resource)
return [grouped[key] for key in ordered]
def _merge_seen_ids(new_ids: list[str], existing_ids: list[str], *, limit: int = 200) -> list[str]:
merged: list[str] = []
seen: set[str] = set()
for item in [*new_ids, *existing_ids]:
value = str(item).strip()
if not value or value in seen:
continue
seen.add(value)
merged.append(value)
if len(merged) >= limit:
break
return merged
def _encode_media_id(value: str) -> str:
from base64 import urlsafe_b64encode
return urlsafe_b64encode(value.encode("utf-8")).decode("ascii").rstrip("=")
def _decode_media_id(value: str) -> str:
from base64 import urlsafe_b64decode
padding = "=" * (-len(value) % 4)
return urlsafe_b64decode((value + padding).encode("ascii")).decode("utf-8")
def _now_text() -> str:
return datetime.now().isoformat(timespec="seconds")
+1
View File
@@ -0,0 +1 @@
"""GUI layer for Sakura downloader."""
+2794
View File
File diff suppressed because it is too large. Load diff
+343
View File
@@ -0,0 +1,343 @@
from __future__ import annotations
import re
import shutil
from collections.abc import Callable
from datetime import datetime
from pathlib import Path
from tkinter import filedialog, messagebox
import customtkinter as ctk
from pixiv_app.core.paths import plugin_roots, user_plugins_root
from pixiv_app.core.plugin.base import BasePlugin
from pixiv_app.core.plugin.generator import PluginGenerator
from pixiv_app.core.plugin.manager import PluginManager
def _slugify_plugin_name(value: str) -> str:
text = re.sub(r"[^0-9A-Za-z_\-\u4e00-\u9fff]+", "_", value.strip())
text = text.strip("._-")
return text or "imported_plugin"
class PluginPanel(ctk.CTkFrame):
"""Generic plugin list, import, reload, and template tools."""
def __init__(
self,
parent,
plugin_manager: PluginManager,
*,
user_root: Path | None = None,
text_color: str = "#1e3a5f",
muted_color: str = "#6f8dac",
border_color: str = "#d6e8fb",
danger_color: str = "#eb7d7d",
) -> None:
super().__init__(parent, fg_color="transparent")
self.manager = plugin_manager
self.user_root = user_root or user_plugins_root()
self._text_color = text_color
self._muted_color = muted_color
self._border_color = border_color
self._danger_color = danger_color
title_frame = ctk.CTkFrame(self, fg_color="transparent")
title_frame.pack(fill="x", padx=12, pady=(10, 6))
ctk.CTkLabel(
title_frame,
text="插件管理",
font=ctk.CTkFont(size=18, weight="bold"),
text_color=text_color,
).pack(side="left")
btn_row = ctk.CTkFrame(title_frame, fg_color="transparent")
btn_row.pack(side="right")
ctk.CTkButton(btn_row, text="导入 .py", width=88, command=self.import_python_plugin).pack(side="left", padx=4)
ctk.CTkButton(btn_row, text="生成模板", width=88, command=self.open_create_dialog).pack(side="left", padx=4)
ctk.CTkButton(btn_row, text="重新加载", width=88, command=self.reload_all).pack(side="left", padx=4)
hint = (
"导入一个 Python 插件文件即可扩展站点;插件需导出 plugin_class,"
"并继承 pixiv_app.core.plugin.base.BasePlugin。"
)
ctk.CTkLabel(
self,
text=hint,
text_color=muted_color,
font=ctk.CTkFont(size=12),
wraplength=760,
justify="left",
).pack(anchor="w", padx=14, pady=(0, 8))
self.summary_var = ctk.StringVar(value="正在读取插件状态...")
ctk.CTkLabel(
self,
textvariable=self.summary_var,
text_color=text_color,
font=ctk.CTkFont(size=12, weight="bold"),
anchor="w",
).pack(fill="x", padx=14, pady=(0, 8))
path_row = ctk.CTkFrame(self, fg_color="transparent")
path_row.pack(fill="x", padx=12, pady=(0, 8))
ctk.CTkLabel(path_row, text="用户插件目录:", text_color=text_color, font=ctk.CTkFont(size=12, weight="bold")).pack(
side="left"
)
ctk.CTkLabel(path_row, text=str(self.user_root), text_color=muted_color, font=ctk.CTkFont(size=11)).pack(
side="left", fill="x", expand=True
)
self.container = ctk.CTkScrollableFrame(self, fg_color="transparent")
self.container.pack(fill="both", expand=True, padx=8, pady=(0, 8))
self.error_container = ctk.CTkFrame(self, fg_color="transparent")
self.error_container.pack(fill="x", padx=8, pady=(0, 8))
self.log_text = ctk.CTkTextbox(self, height=120, font=ctk.CTkFont(family="Consolas", size=11))
self.log_text.pack(fill="x", padx=12, pady=(0, 12))
self.refresh()
def log(self, message: str) -> None:
stamp = datetime.now().strftime("%H:%M:%S")
self.log_text.insert("end", f"[{stamp}] {message}\n")
self.log_text.see("end")
def refresh(self) -> None:
for child in self.container.winfo_children():
child.destroy()
for child in self.error_container.winfo_children():
child.destroy()
if not self.manager.plugins:
ctk.CTkLabel(
self.container,
text="没有发现插件。请点击「导入 .py」或「生成模板」。",
text_color=self._muted_color,
justify="left",
).pack(anchor="w", pady=8)
for name, plugin in sorted(self.manager.plugins.items(), key=lambda x: x[0].lower()):
self._add_plugin_card(name, plugin)
ready_count = sum(1 for plugin in self.manager.plugins.values() if plugin.validate())
self.summary_var.set(
f"已加载 {len(self.manager.plugins)} 个插件 · 可用 {ready_count} 个 · "
f"加载失败 {len(self.manager.load_errors)} 个"
)
self._add_error_cards()
def _add_error_cards(self) -> None:
if not self.manager.load_errors:
return
title = ctk.CTkLabel(
self.error_container,
text="加载失败的插件",
font=ctk.CTkFont(size=14, weight="bold"),
text_color=self._danger_color,
)
title.pack(anchor="w", padx=4, pady=(8, 4))
for path, error in sorted(self.manager.load_errors.items(), key=lambda x: str(x[0]).lower()):
card = ctk.CTkFrame(self.error_container, corner_radius=10, border_width=1, border_color=self._danger_color)
card.pack(fill="x", pady=4)
ctk.CTkLabel(
card,
text=str(path),
font=ctk.CTkFont(size=11),
text_color=self._text_color,
wraplength=760,
justify="left",
).pack(anchor="w", padx=10, pady=(8, 2))
ctk.CTkLabel(
card,
text=error,
font=ctk.CTkFont(size=11),
text_color=self._danger_color,
wraplength=760,
justify="left",
).pack(anchor="w", padx=10, pady=(0, 8))
def _add_plugin_card(self, name: str, plugin: BasePlugin) -> None:
card = ctk.CTkFrame(self.container, corner_radius=12, border_width=1, border_color=self._border_color)
card.pack(fill="x", pady=6)
top = ctk.CTkFrame(card, fg_color="transparent")
top.pack(fill="x", padx=12, pady=(10, 4))
ctk.CTkLabel(top, text=plugin.name, font=ctk.CTkFont(size=15, weight="bold"), text_color=self._text_color).pack(
side="left"
)
ctk.CTkLabel(top, text=f"v{plugin.version}", font=ctk.CTkFont(size=12), text_color=self._muted_color).pack(
side="left", padx=10
)
is_ready = plugin.validate()
st = "可用" if is_ready else "需配置"
ctk.CTkLabel(
top,
text=st,
font=ctk.CTkFont(size=12, weight="bold"),
text_color="#3f9b72" if is_ready else self._danger_color,
).pack(side="right")
path = self.manager.get_plugin_path(name)
source = "用户插件" if path and self.user_root in path.parents else "内置插件"
ctk.CTkLabel(
card,
text=f"域名 {plugin.domain} 来源 {source}",
font=ctk.CTkFont(size=12),
text_color=self._muted_color,
).pack(anchor="w", padx=12, pady=(0, 4))
if path:
ctk.CTkLabel(
card,
text=str(path),
font=ctk.CTkFont(size=11),
text_color=self._muted_color,
wraplength=760,
justify="left",
).pack(anchor="w", padx=12, pady=(0, 8))
actions = ctk.CTkFrame(card, fg_color="transparent")
actions.pack(fill="x", padx=12, pady=(0, 10))
ctk.CTkButton(actions, text="重载", width=72, command=lambda n=name: self.reload_one(n)).pack(side="left", padx=(0, 6))
if path and self.user_root in path.parents:
ctk.CTkButton(
actions,
text="删除文件",
width=86,
fg_color=self._danger_color,
command=lambda n=name: self.delete_user_plugin(n),
).pack(side="left")
else:
ctk.CTkButton(actions, text="从内存卸载", width=92, command=lambda n=name: self.unload_one(n)).pack(side="left")
def import_python_plugin(self) -> None:
path = filedialog.askopenfilename(
title="导入 Python 插件",
filetypes=[("Python plugin", "*.py"), ("All files", "*.*")],
)
if not path:
return
src = Path(path)
if not src.is_file():
messagebox.showerror("导入失败", "文件不存在。")
return
plugin_dir = self.user_root / _slugify_plugin_name(src.stem)
index = 2
while plugin_dir.exists() and (plugin_dir / "plugin.py").resolve() != src.resolve():
plugin_dir = self.user_root / f"{_slugify_plugin_name(src.stem)}_{index}"
index += 1
plugin_dir.mkdir(parents=True, exist_ok=True)
dst = plugin_dir / "plugin.py"
try:
shutil.copy2(src, dst)
plugin = self.manager.load_plugin(dst)
except Exception as exc:
messagebox.showerror("导入失败", str(exc))
return
if plugin is None:
messagebox.showerror("导入失败", "插件未能加载。请确认文件中导出了 plugin_class。")
return
self.refresh()
self.log(f"已导入插件: {plugin.name} -> {dst}")
messagebox.showinfo("导入完成", f"已导入插件:{plugin.name}\n\n{dst}")
def reload_all(self) -> None:
self.manager.reload_all_from_disk()
self.refresh()
self.log("已重新加载全部插件。")
def reload_one(self, name: str) -> None:
if self.manager.reload_plugin(name):
self.refresh()
self.log(f"已重载插件: {name}")
else:
self.log(f"重载失败: {name}")
def unload_one(self, name: str) -> None:
self.manager.unload_plugin(name)
self.refresh()
self.log(f"已从内存卸载: {name}")
def delete_user_plugin(self, name: str) -> None:
path = self.manager.get_plugin_path(name)
if path is None or self.user_root not in path.parents:
messagebox.showinfo("提示", "只能删除用户导入的插件文件。")
return
if not messagebox.askyesno("确认删除", f"将删除插件目录:\n{path.parent}\n\n是否继续?"):
return
self.manager.unload_plugin(name)
shutil.rmtree(path.parent, ignore_errors=True)
self.manager.reload_all_from_disk()
self.refresh()
self.log(f"已删除用户插件: {name}")
def open_create_dialog(self) -> None:
CreatePluginDialog(self, self.user_root, on_created=lambda: (self.reload_all(), self.refresh()))
class CreatePluginDialog(ctk.CTkToplevel):
def __init__(self, parent, plugins_root: Path, *, on_created: Callable[[], None] | None = None) -> None:
super().__init__(parent)
self.title("生成插件模板")
self.geometry("520x420")
self.plugins_root = plugins_root
self._on_created = on_created
self.generator = PluginGenerator()
pad = {"padx": 16, "pady": 8}
ctk.CTkLabel(self, text="名称").pack(anchor="w", **pad)
self.name_entry = ctk.CTkEntry(self, placeholder_text="例如 MySite")
self.name_entry.pack(fill="x", **pad)
ctk.CTkLabel(self, text="域名").pack(anchor="w", **pad)
self.domain_entry = ctk.CTkEntry(self, placeholder_text="例如 example.com")
self.domain_entry.pack(fill="x", **pad)
ctk.CTkLabel(self, text="作者").pack(anchor="w", **pad)
self.author_entry = ctk.CTkEntry(self)
self.author_entry.pack(fill="x", **pad)
ctk.CTkLabel(self, text="描述").pack(anchor="w", **pad)
self.desc_text = ctk.CTkTextbox(self, height=100)
self.desc_text.pack(fill="x", **pad)
ctk.CTkButton(self, text="生成到用户插件目录", command=self._generate).pack(pady=16)
def _generate(self) -> None:
data = {
"name": self.name_entry.get().strip(),
"domain": self.domain_entry.get().strip(),
"author": self.author_entry.get().strip(),
"description": self.desc_text.get("1.0", "end").strip(),
}
if not data["name"] or not data["domain"]:
messagebox.showerror("校验失败", "请填写名称与域名。")
return
try:
path = self.generator.generate_from_gui(data, output_root=self.plugins_root)
except Exception as exc:
messagebox.showerror("失败", str(exc))
return
messagebox.showinfo("完成", f"已生成: {path}")
self.destroy()
if self._on_created:
self._on_created()
class PluginPanelWindow(ctk.CTkToplevel):
"""Standalone window hosting PluginPanel."""
def __init__(self, parent, *, cookie: str = "") -> None:
super().__init__(parent)
self.title("插件管理")
self.geometry("760x620")
user_root = user_plugins_root()
self.manager = PluginManager(plugin_roots())
self.manager.load_all()
pixiv = self.manager.plugins.get("Pixiv")
if pixiv is not None and hasattr(pixiv, "set_auth"):
pixiv.set_auth(cookie)
panel = PluginPanel(self, self.manager, user_root=user_root)
panel.pack(fill="both", expand=True)
+160
View File
@@ -0,0 +1,160 @@
from __future__ import annotations
import threading
from typing import Any, Callable, Optional
import customtkinter as ctk
class ProxyDialog(ctk.CTkToplevel):
def __init__(
self,
master,
initial_config: dict,
on_save: Callable[[dict], None],
on_action: Optional[Callable[[str, dict], str | dict[str, Any]]] = None,
):
super().__init__(master)
self.title("代理池设置")
self.geometry("760x700")
self.minsize(700, 640)
self.on_save = on_save
self.on_action = on_action
self.initial_config = initial_config
self.use_proxy_var = ctk.BooleanVar(value=bool(initial_config.get("enabled", False)))
self.use_quake_var = ctk.BooleanVar(value=bool(initial_config.get("use_quake", False)))
self.quake_mode_var = ctk.StringVar(value=str(initial_config.get("quake_mode", "api_v3")))
self.status_var = ctk.StringVar(value="准备就绪")
self._build_ui()
self.transient(master)
self.grab_set()
def _build_ui(self) -> None:
container = ctk.CTkFrame(self, fg_color="transparent")
container.pack(fill="both", expand=True, padx=16, pady=16)
ctk.CTkLabel(container, text="代理池设置", font=ctk.CTkFont(size=24, weight="bold")).pack(anchor="w", pady=(0, 6))
ctk.CTkLabel(
container,
text="预校验通过的代理会自动回填到下面的代理列表里,保存后下次可直接使用。",
text_color="#6f8dac",
font=ctk.CTkFont(size=12),
).pack(anchor="w", pady=(0, 12))
toggle_row = ctk.CTkFrame(container, fg_color="#f7fbff", corner_radius=12)
toggle_row.pack(fill="x", pady=(0, 10), padx=2)
ctk.CTkSwitch(toggle_row, text="启用代理池", variable=self.use_proxy_var).pack(anchor="w", padx=12, pady=(8, 6))
ctk.CTkSwitch(toggle_row, text="启用 Quake 自动爬取", variable=self.use_quake_var).pack(anchor="w", padx=12, pady=(0, 10))
mode_row = ctk.CTkFrame(container, fg_color="transparent")
mode_row.pack(fill="x", pady=(0, 8))
ctk.CTkLabel(mode_row, text="Quake 模式:", width=120).pack(side="left")
ctk.CTkOptionMenu(mode_row, values=["api_v3", "web_assoc"], variable=self.quake_mode_var, width=220).pack(side="left")
key_row = ctk.CTkFrame(container, fg_color="transparent")
key_row.pack(fill="x", pady=(0, 8))
ctk.CTkLabel(key_row, text="Quake API Key:", width=120).pack(side="left")
self.api_key_entry = ctk.CTkEntry(key_row, placeholder_text="输入 Quake API Key")
self.api_key_entry.pack(side="left", fill="x", expand=True)
self.api_key_entry.insert(0, str(self.initial_config.get("quake_api_key", "")))
cookie_row = ctk.CTkFrame(container, fg_color="transparent")
cookie_row.pack(fill="x", pady=(0, 8))
ctk.CTkLabel(cookie_row, text="Quake Cookie:", width=120).pack(side="left")
self.cookie_entry = ctk.CTkEntry(cookie_row, placeholder_text="输入 Quake Cookie(web_assoc 模式需要)")
self.cookie_entry.pack(side="left", fill="x", expand=True)
self.cookie_entry.insert(0, str(self.initial_config.get("quake_cookie", "")))
country_row = ctk.CTkFrame(container, fg_color="transparent")
country_row.pack(fill="x", pady=(0, 8))
ctk.CTkLabel(country_row, text="海外国家代码:", width=120).pack(side="left")
self.country_entry = ctk.CTkEntry(country_row, placeholder_text="US,JP,KR,SG,HK,TW,DE,FR,GB,CA,AU")
self.country_entry.pack(side="left", fill="x", expand=True)
self.country_entry.insert(0, str(self.initial_config.get("countries_text", "US,JP,KR,SG,HK,TW,DE,FR,GB,CA,AU")))
tool_row = ctk.CTkFrame(container, fg_color="transparent")
tool_row.pack(fill="x", pady=(8, 6))
self.manual_input = ctk.CTkEntry(tool_row, placeholder_text="快速添加单个代理,如 1.2.3.4:8080")
self.manual_input.pack(side="left", fill="x", expand=True)
ctk.CTkButton(tool_row, text="手动添加", width=110, command=self._add_manual_proxy).pack(side="left", padx=(8, 0))
self.fetch_button = ctk.CTkButton(tool_row, text="开始爬取并预校验", width=170, command=self._trigger_fetch)
self.fetch_button.pack(side="left", padx=(8, 0))
ctk.CTkLabel(
container,
text="代理列表(每行一个,支持 ip:port / http://ip:port / https://ip:port)",
anchor="w",
).pack(fill="x", pady=(8, 0))
self.proxy_text = ctk.CTkTextbox(container, height=360)
self.proxy_text.pack(fill="both", expand=True, pady=(6, 12))
self.proxy_text.insert("1.0", str(self.initial_config.get("manual_text", "")))
status_row = ctk.CTkFrame(container, fg_color="#f7fbff", corner_radius=12)
status_row.pack(fill="x", pady=(0, 10))
ctk.CTkLabel(status_row, textvariable=self.status_var, text_color="#365b85", font=ctk.CTkFont(size=12)).pack(
anchor="w", padx=12, pady=8
)
button_row = ctk.CTkFrame(container, fg_color="transparent")
button_row.pack(fill="x")
ctk.CTkButton(button_row, text="取消", command=self.destroy, width=120).pack(side="right", padx=(8, 0))
ctk.CTkButton(button_row, text="保存", command=self._save, width=120).pack(side="right")
def _collect_config(self) -> dict:
return {
"enabled": bool(self.use_proxy_var.get()),
"use_quake": bool(self.use_quake_var.get()),
"quake_mode": self.quake_mode_var.get().strip() or "api_v3",
"quake_api_key": self.api_key_entry.get().strip(),
"quake_cookie": self.cookie_entry.get().strip(),
"manual_text": self.proxy_text.get("1.0", "end").strip(),
"countries_text": self.country_entry.get().strip(),
}
def _add_manual_proxy(self) -> None:
value = self.manual_input.get().strip()
if not value:
self.status_var.set("请输入代理地址后再添加。")
return
old_text = self.proxy_text.get("1.0", "end").strip()
new_text = f"{old_text}\n{value}".strip()
self.proxy_text.delete("1.0", "end")
self.proxy_text.insert("1.0", new_text)
self.manual_input.delete(0, "end")
self.status_var.set("已加入手动代理列表。")
def _trigger_fetch(self) -> None:
if not self.on_action:
self.status_var.set("当前版本未启用预校验动作。")
return
self.fetch_button.configure(state="disabled", text="处理中...")
self.status_var.set("正在爬取并校验代理,请稍候...")
config = self._collect_config()
def worker() -> None:
try:
result = self.on_action("fetch_preview", config)
except Exception as exc:
result = {"status": f"预校验失败: {exc}"}
self.after(0, lambda: self._finish_fetch(result))
threading.Thread(target=worker, daemon=True).start()
def _finish_fetch(self, result: str | dict[str, Any]) -> None:
self.fetch_button.configure(state="normal", text="开始爬取并预校验")
if isinstance(result, dict):
manual_text = result.get("manual_text")
if isinstance(manual_text, str):
self.proxy_text.delete("1.0", "end")
self.proxy_text.insert("1.0", manual_text)
status = str(result.get("status", "预校验完成"))
self.status_var.set(status)
return
self.status_var.set(str(result))
def _save(self) -> None:
self.on_save(self._collect_config())
self.destroy()
+1
View File
@@ -0,0 +1 @@
"""Local service entry points for Sakura downloader."""
+500
View File
@@ -0,0 +1,500 @@
from __future__ import annotations
import json
import mimetypes
import time
import uuid
from dataclasses import asdict
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import parse_qs, urlparse
from pixiv_app.core.auth import SessionStore
from pixiv_app.core.cookie_import import cookie_summary, cookies_to_header, cookies_to_playwright, parse_cookie_text
from pixiv_app.core.gallery_service import LocalGalleryService
from pixiv_app.core.library import DEFAULT_LIBRARY_DB
from pixiv_app.core.origin_follow_sync import OriginFollowSyncService
from pixiv_app.core.paths import app_session_file, downloads_root, plugin_roots, webui_root
from pixiv_app.core.plugin.manager import PluginManager
from pixiv_app.core.thumbnails import DEFAULT_THUMBNAIL_DIR
from pixiv_app.core.x_media_service import XMediaService
class GalleryApiServer:
def __init__(
self,
*,
host: str = "127.0.0.1",
port: int = 8765,
db_path: str | Path = DEFAULT_LIBRARY_DB,
thumbnail_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
web_root: str | Path | None = None,
) -> None:
self.host = host
self.port = port
self.service = LocalGalleryService(db_path=db_path, thumbnail_dir=thumbnail_dir)
self.x_media_service = XMediaService()
self.origin_follow_sync_service = OriginFollowSyncService(
db_path=str(db_path),
gallery_service=self.service,
x_media_service=self.x_media_service,
)
self.plugin_manager = PluginManager(plugin_roots())
self.plugin_manager.load_all()
self.plugin_preview_cache: dict[str, tuple[str, list]] = {}
self.web_root = Path(web_root) if web_root else webui_root()
def serve_forever(self) -> None:
service = self.service
x_media_service = self.x_media_service
origin_follow_sync_service = self.origin_follow_sync_service
plugin_manager = self.plugin_manager
plugin_preview_cache = self.plugin_preview_cache
web_root = self.web_root
Handler = type(
"GalleryApiHandlerForServer",
(GalleryApiHandler,),
{
"gallery_service": service,
"x_media_service": x_media_service,
"origin_follow_sync_service": origin_follow_sync_service,
"plugin_manager": plugin_manager,
"plugin_preview_cache": plugin_preview_cache,
"web_root": web_root,
},
)
server = ThreadingHTTPServer((self.host, self.port), Handler)
try:
server.serve_forever()
finally:
server.server_close()
class GalleryApiHandler(BaseHTTPRequestHandler):
gallery_service: LocalGalleryService
x_media_service: XMediaService
origin_follow_sync_service: OriginFollowSyncService
plugin_manager: PluginManager
plugin_preview_cache: dict[str, tuple[str, list]]
web_root: Path
def do_GET(self) -> None:
parsed = urlparse(self.path)
query = parse_qs(parsed.query)
try:
if parsed.path in {"/", "/index.html"}:
self._send_static_file(self.web_root / "index.html")
elif parsed.path.startswith("/assets/"):
self._send_static_file(self.web_root / parsed.path.removeprefix("/assets/"))
elif parsed.path == "/api/health":
self._send_json({"status": "ok"})
elif parsed.path == "/api/session/status":
self._send_json(self._handle_session_status())
elif parsed.path == "/api/plugins/pages":
self._send_json(self._handle_plugin_pages())
elif parsed.path == "/api/artworks":
self._send_json(self._handle_artworks(query))
elif parsed.path == "/api/favorites":
self._send_json(self.gallery_service.list_favorites(limit=_int(query, "limit", 60), offset=_int(query, "offset", 0)))
elif parsed.path.startswith("/api/artworks/"):
self._send_json(self._handle_artwork_detail(parsed.path))
elif parsed.path == "/api/tags":
self._send_json(self.gallery_service.list_tags(query=_first(query, "q"), limit=_int(query, "limit", 100)))
elif parsed.path == "/api/artists":
self._send_json(self.gallery_service.list_artists(query=_first(query, "q"), limit=_int(query, "limit", 100)))
elif parsed.path.startswith("/api/artists/"):
self._send_json(self._handle_artist_detail(parsed.path, query))
elif parsed.path == "/api/followed-artists":
self._send_json(self.gallery_service.list_followed_artists())
elif parsed.path == "/api/thumbnails/build":
self._send_json({"items": self.gallery_service.build_missing_thumbnails(limit=_int(query, "limit", 200))})
elif parsed.path == "/api/import/legacy":
self._send_json(self.gallery_service.scan_legacy_downloads(root=_first(query, "root", "Sakura_Downloads")))
elif parsed.path == "/api/sync/followed":
self._send_json(
self.gallery_service.sync_followed_artists(
max_new_per_artist=_int(query, "max_new", 20),
download=_bool(query, "download", True),
)
)
elif parsed.path == "/api/x/local-media":
self._send_json(
self.x_media_service.list_local_media(
limit=_int(query, "limit", 160),
offset=_int(query, "offset", 0),
)
)
elif parsed.path == "/api/x/follows":
self._send_json(self.x_media_service.list_followed_authors())
elif parsed.path.startswith("/media/thumbnail/"):
self._send_file(self._resolve_media_path(parsed.path, thumbnail=True))
elif parsed.path.startswith("/media/source/"):
self._send_file(self._resolve_media_path(parsed.path, thumbnail=False))
elif parsed.path.startswith("/media/x-local/"):
self._send_file(self.x_media_service.resolve_local_media(parsed.path.rsplit("/", 1)[-1]))
else:
self._send_json({"error": "not found"}, status=404)
except ValueError as exc:
self._send_json({"error": str(exc)}, status=400)
except Exception as exc:
self._send_json({"error": str(exc)}, status=500)
def do_POST(self) -> None:
parsed = urlparse(self.path)
try:
payload = self._read_json_body()
if parsed.path == "/api/favorites":
self._send_json(
self.gallery_service.set_favorite(
int(payload["artwork_id"]),
bool(payload.get("favorite", True)),
str(payload.get("note", "")),
)
)
elif parsed.path == "/api/followed-artists":
self._send_json(
self.gallery_service.set_followed_artist(
payload.get("artist_id", ""),
bool(payload.get("followed", True)),
str(payload.get("name", "")),
)
)
elif parsed.path == "/api/import/legacy":
self._send_json(self.gallery_service.scan_legacy_downloads(root=str(payload.get("root", "Sakura_Downloads"))))
elif parsed.path == "/api/sync/followed":
self._send_json(
self.gallery_service.sync_followed_artists(
cookie=str(payload.get("cookie", "")),
max_new_per_artist=int(payload.get("max_new", 20)),
download=bool(payload.get("download", True)),
)
)
elif parsed.path == "/api/cookies/import":
self._send_json(self._handle_cookie_import(payload))
elif parsed.path == "/api/origin-follows/sync":
self._send_json(
self.origin_follow_sync_service.sync(
str(payload.get("platform", "")),
)
)
elif parsed.path == "/api/x/media/preview":
self._send_json(
self.x_media_service.preview_author_media(
str(payload.get("username", "")),
media_type=str(payload.get("media_type", "all")),
max_items=int(payload.get("max_items", 120)),
)
)
elif parsed.path == "/api/x/media/download":
selected_ids = payload.get("selected_ids")
if selected_ids is not None and not isinstance(selected_ids, list):
raise ValueError("selected_ids 必须是数组。")
self._send_json(
self.x_media_service.download_preview(
str(payload.get("preview_id", "")),
selected_ids=[str(item) for item in selected_ids] if selected_ids is not None else None,
)
)
elif parsed.path == "/api/x/local-media/delete":
media_ids = payload.get("ids", [])
if not isinstance(media_ids, list):
raise ValueError("ids 必须是数组。")
self._send_json(self.x_media_service.delete_local_media([str(item) for item in media_ids]))
elif parsed.path == "/api/x/follows":
self._send_json(
self.x_media_service.set_followed_author(
str(payload.get("username", "")),
followed=bool(payload.get("followed", True)),
creator_name=str(payload.get("name", "")),
)
)
elif parsed.path == "/api/x/follows/sync":
self._send_json(
self.x_media_service.sync_followed_authors(
download=bool(payload.get("download", True)),
max_new_per_author=int(payload.get("max_new", 20)),
)
)
elif parsed.path == "/api/xiaohongshu/preview":
self._send_json(self._handle_plugin_preview("小红书", payload))
elif parsed.path == "/api/xiaohongshu/download":
self._send_json(self._handle_plugin_download(payload))
else:
self._send_json({"error": "not found"}, status=404)
except ValueError as exc:
self._send_json({"error": str(exc)}, status=400)
except Exception as exc:
self._send_json({"error": str(exc)}, status=500)
def do_OPTIONS(self) -> None:
self.send_response(204)
self._send_cors_headers()
self.end_headers()
def log_message(self, format: str, *args) -> None:
return None
def _handle_artworks(self, query: dict[str, list[str]]) -> dict:
artist_id = _first(query, "artist_id")
return self.gallery_service.list_artworks(
query=_first(query, "q"),
search_scope=_first(query, "search_scope") or "all",
tag=_first(query, "tag"),
artist_id=int(artist_id) if artist_id else None,
artwork_type=_first(query, "type"),
include_restricted=_bool(query, "include_restricted", True),
limit=_int(query, "limit", 60),
offset=_int(query, "offset", 0),
ensure_thumbnails=_bool(query, "ensure_thumbnails", True),
)
def _handle_plugin_pages(self) -> dict:
self.plugin_manager.reload_all_from_disk()
items = []
if "X" in self.plugin_manager.plugins:
items.append(
{
"id": "x-media",
"view": "xMedia",
"title": "X 作者媒体",
"description": "按作者名称解析图片或视频并选择下载",
}
)
if "小红书" in self.plugin_manager.plugins:
plugin = self.plugin_manager.plugins["小红书"]
items.append(
{
"id": "xiaohongshu",
"view": "xiaohongshu",
"title": "小红书",
"description": "搜索关键词或解析笔记链接,预览后下载",
"available": bool(plugin.validate()),
}
)
return {"items": items}
def _handle_plugin_preview(self, plugin_name: str, payload: dict) -> dict:
plugin = self.plugin_manager.plugins.get(plugin_name)
if plugin is None:
raise ValueError(f"未安装插件: {plugin_name}")
if not plugin.validate():
raise ValueError(f"{plugin_name} 插件依赖不可用,请先安装 Playwright。")
target = str(payload.get("target", "")).strip()
if not target:
raise ValueError("请输入解析目标。")
limit = max(1, min(int(payload.get("limit", 20)), 50))
resources = plugin.parse(target)[:limit]
preview_id = uuid.uuid4().hex
self.plugin_preview_cache[preview_id] = (plugin_name, resources)
return {
"preview_id": preview_id,
"count": len(resources),
"items": [_resource_to_json(item, i) for i, item in enumerate(resources)],
}
def _handle_plugin_download(self, payload: dict) -> dict:
preview_id = str(payload.get("preview_id", ""))
cached = self.plugin_preview_cache.get(preview_id)
if cached is None:
raise ValueError("预览结果已过期,请重新解析。")
plugin_name, resources = cached
plugin = self.plugin_manager.plugins.get(plugin_name)
if plugin is None:
raise ValueError(f"插件已卸载: {plugin_name}")
selected_ids = payload.get("selected_ids")
selected = set(str(item) for item in selected_ids) if isinstance(selected_ids, list) else set()
if selected:
resources = [item for item in resources if item.id in selected]
save_root = downloads_root()
results = []
downloaded = 0
failed = 0
for resource in resources:
try:
paths = plugin.download(resource, save_root)
downloaded += len(paths)
results.append({
"id": resource.id,
"ok": bool(paths),
"paths": [str(path) for path in paths],
"synced_files": 0,
})
except Exception as exc:
failed += 1
results.append({"id": resource.id, "ok": False, "error": str(exc)})
return {
"requested": len(resources),
"downloaded_files": downloaded,
"synced_files": 0,
"failed_items": failed,
"items": results,
}
def _handle_artwork_detail(self, path: str) -> dict:
artwork_id = int(path.rsplit("/", 1)[-1])
item = self.gallery_service.get_artwork(artwork_id)
if item is None:
raise ValueError(f"Artwork not found: {artwork_id}")
return item
def _handle_artist_detail(self, path: str, query: dict[str, list[str]]) -> dict:
artist_id = int(path.rsplit("/", 1)[-1])
item = self.gallery_service.get_artist(
artist_id,
limit=_int(query, "limit", 60),
offset=_int(query, "offset", 0),
)
if item is None:
raise ValueError(f"Artist not found: {artist_id}")
return item
def _resolve_media_path(self, path: str, *, thumbnail: bool) -> Path:
parts = path.strip("/").split("/")
if len(parts) != 4:
raise ValueError("Expected /media/{thumbnail|source}/{artwork_id}/{page_index}")
artwork_id = int(parts[2])
page_index = int(parts[3])
media_path = (
self.gallery_service.get_thumbnail_path(artwork_id, page_index=page_index)
if thumbnail
else self.gallery_service.get_source_file_path(artwork_id, page_index=page_index)
)
if media_path is None:
raise ValueError("Media file not found")
return media_path
def _handle_cookie_import(self, payload: dict) -> dict:
platform = str(payload.get("platform", "")).strip().lower()
text = str(payload.get("text", ""))
if platform in {"pixiv", "px"}:
cookies = parse_cookie_text(text)
pixiv_cookies = cookies_to_playwright(cookies, ("pixiv.net",))
header = cookies_to_header(pixiv_cookies, ("pixiv.net",))
store = SessionStore(app_session_file())
session_payload = store.load()
session_payload.update(
{
"login_mode": "cookie",
"cookie": header,
"cookie_json": pixiv_cookies,
"login_id": "",
"saved_at": time.strftime("%Y-%m-%d %H:%M:%S"),
}
)
store.save(session_payload)
return {"platform": "pixiv", "count": len(pixiv_cookies), "summary": cookie_summary(pixiv_cookies)}
if platform in {"x", "twitter"}:
result = self.x_media_service.import_cookie_text(text)
result["platform"] = "x"
return result
raise ValueError("platform 必须是 pixiv 或 x。")
def _handle_session_status(self) -> dict:
payload = SessionStore(app_session_file()).load()
pixiv_cookie = str(payload.get("cookie") or "").strip()
cookie_json = payload.get("cookie_json")
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
return {
"pixiv": {
"has_cookie": bool(pixiv_cookie or cookie_json),
"saved_at": str(payload.get("saved_at") or ""),
"login_mode": str(payload.get("login_mode") or ""),
},
"cloud": {
"enabled": bool(cloud.get("enabled")),
"has_token": bool(str(cloud.get("access_token") or "").strip()),
"base_url": str(cloud.get("base_url") or "").strip(),
},
}
def _send_json(self, payload: dict | list, *, status: int = 200) -> None:
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json; charset=utf-8")
self._send_cors_headers()
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _send_file(self, path: Path) -> None:
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
body = path.read_bytes()
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _send_static_file(self, path: Path) -> None:
if not path.exists() or not path.is_file():
self._send_json({"error": "not found"}, status=404)
return
self._send_file(path)
def _send_cors_headers(self) -> None:
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
self.send_header("Access-Control-Allow-Headers", "Content-Type")
def _read_json_body(self) -> dict:
length = int(self.headers.get("Content-Length", "0") or "0")
if length == 0:
return {}
body = self.rfile.read(length)
payload = json.loads(body.decode("utf-8"))
if not isinstance(payload, dict):
raise ValueError("JSON body must be an object")
return payload
def _first(query: dict[str, list[str]], key: str, default: str = "") -> str:
values = query.get(key)
if not values:
return default
return values[0]
def _int(query: dict[str, list[str]], key: str, default: int) -> int:
value = _first(query, key)
if not value:
return default
return int(value)
def _bool(query: dict[str, list[str]], key: str, default: bool) -> bool:
value = _first(query, key)
if not value:
return default
return value.lower() in {"1", "true", "yes", "on"}
def _resource_to_json(resource, index: int) -> dict:
payload = asdict(resource)
payload["index"] = index
payload["kind"] = "video" if any(file.get("type") == "video" for file in resource.files) else "image"
payload["file_count"] = len(resource.files)
return payload
def _plugin_platform_slug(plugin_name: str) -> str:
lowered = plugin_name.lower()
if "douyin" in lowered or "抖音" in plugin_name:
return "douyin"
if "xiaohongshu" in lowered or "小红书" in plugin_name:
return "xiaohongshu"
if lowered == "x":
return "x"
return lowered.replace(" ", "_")[:20] or "plugin"
def _resource_file_type(resource, index: int, path: Path) -> str:
if index < len(resource.files):
file_type = str(resource.files[index].get("type") or "").lower()
if file_type == "video":
return "video"
mime_type = mimetypes.guess_type(str(path))[0] or ""
return "video" if mime_type.startswith("video/") else "illust"
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
import argparse
import logging
from pixiv_app.services.gallery_api import GalleryApiServer
def main() -> None:
parser = argparse.ArgumentParser(description="Run local Sakura gallery API")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8765)
parser.add_argument("--db", default="pixiv_app_library.db")
parser.add_argument("--thumbnail-dir", default="runtime/thumbnails")
parser.add_argument("--web-root", default="")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logging.info("Local gallery API listening on http://%s:%s", args.host, args.port)
GalleryApiServer(
host=args.host,
port=args.port,
db_path=args.db,
thumbnail_dir=args.thumbnail_dir,
web_root=args.web_root or None,
).serve_forever()
if __name__ == "__main__":
main()
+356
View File
@@ -0,0 +1,356 @@
"""
X (Twitter) authentication helper — Playwright **persistent context** only.
- First-time login is always manual (including email verification); no credential stuffing.
- Session persistence is Chromium's user_data_dir (cookies + localStorage live inside it).
- Optional JSON cookie export into ``profiles/x/<profile>/cookies/`` for backup only.
Directory layout (created automatically)::
profiles/
x/
<profile_key>/
chromium_profile/ # user_data_dir → persistent session (real storage)
cookies/ # optional Playwright cookie exports (JSON)
localStorage/ # reserved / notes (Chromium keeps LS inside chromium_profile)
Usage::
auth = XAuthManager(profile_key="default")
await auth.start_session()
page = await auth.get_page()
await page.goto("https://x.com/i/flow/login")
# ... user completes login manually ...
if await auth.is_logged_in():
...
await auth.close()
"""
from __future__ import annotations
import asyncio
import json
import logging
from pathlib import Path
from typing import Any
from playwright.async_api import BrowserContext, Page, Playwright, async_playwright
from pixiv_app.core.paths import profiles_root
from pixiv_app.core.playwright_runtime import configure_playwright_runtime_env, preferred_chromium_executable
logger = logging.getLogger(__name__)
def _default_profiles_root() -> Path:
return profiles_root()
class XAuthManager:
"""
Manage a long-lived Playwright Chromium **persistent** profile for x.com.
Not thread-safe: use one instance per logical browser session, or guard externally.
"""
HOME_URL = "https://x.com/home"
LOGIN_FLOW_HINT = "https://x.com/i/flow/login"
def __init__(
self,
*,
profile_key: str = "default",
profiles_root: Path | None = None,
headless: bool = True,
auto_reconnect: bool = True,
max_health_retries: int = 2,
) -> None:
"""
:param profile_key: Subdirectory under ``profiles/x/`` for multi-account switching.
:param profiles_root: Defaults to ``<project>/profiles``.
:param headless: Headless for automated checks; use ``False`` for first manual login.
:param auto_reconnect: If ``start_session`` finds a dead context, restart Playwright.
:param max_health_retries: Retries when validating session after transient failures.
"""
self._profile_key = profile_key
root = profiles_root or _default_profiles_root()
self._profile_home = root / "x" / profile_key
self._user_data_dir = self._profile_home / "chromium_profile"
self._cookies_dir = self._profile_home / "cookies"
self._localstorage_dir = self._profile_home / "localStorage"
self._headless = headless
self._auto_reconnect = auto_reconnect
self._max_health_retries = max(1, max_health_retries)
self._pw: Playwright | None = None
self._context: BrowserContext | None = None
self._primary_page: Page | None = None
self._ensure_storage_layout()
def _ensure_storage_layout(self) -> None:
"""Create profile dirs. Real cookie/localStorage persistence is under ``chromium_profile``."""
self._user_data_dir.mkdir(parents=True, exist_ok=True)
self._cookies_dir.mkdir(parents=True, exist_ok=True)
self._localstorage_dir.mkdir(parents=True, exist_ok=True)
@property
def user_data_dir(self) -> Path:
"""Playwright ``user_data_dir`` passed to ``launch_persistent_context``."""
return self._user_data_dir
def _context_alive(self) -> bool:
if self._context is None:
return False
try:
_ = self._context.pages
return True
except Exception:
return False
async def start_session(self, *, force_new: bool = False) -> BrowserContext:
"""
Launch or reuse persistent Chromium. Session is restored from disk automatically.
:param force_new: Close existing context and open a fresh one (same profile dir).
"""
if self._context_alive() and not force_new:
return self._context # type: ignore[return-value]
if self._context is not None:
await self.close()
self._pw = await async_playwright().start()
try:
self._context = await self._launch_persistent(self._pw)
except Exception:
await self._pw.stop()
self._pw = None
raise
self._primary_page = None
logger.info("XAuthManager: persistent context ready (%s)", self._user_data_dir)
return self._context
async def _launch_persistent(self, pw: Playwright) -> BrowserContext:
"""Single place for launch_persistent_context configuration."""
self._user_data_dir.mkdir(parents=True, exist_ok=True)
configure_playwright_runtime_env()
base_kw: dict[str, Any] = {
"user_data_dir": str(self._user_data_dir),
"headless": self._headless,
"viewport": {"width": 1280, "height": 800},
"locale": "zh-CN",
"ignore_default_args": ["--enable-automation"],
"args": ["--disable-blink-features=AutomationControlled"],
"user_agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
),
}
import os
env_ch = os.environ.get("X_PLAYWRIGHT_CHANNEL", "").strip()
local_executable = preferred_chromium_executable()
attempts: list[dict[str, Any]] = []
seen: set[str] = set()
for extra in (
{"channel": env_ch} if env_ch else {},
{"executable_path": str(local_executable)} if local_executable else {},
{"channel": "chrome"},
{},
):
key = repr(extra)
if key in seen:
continue
seen.add(key)
attempts.append(extra)
last_err: Exception | None = None
for extra in attempts:
kw = dict(base_kw)
kw.update(extra)
try:
ctx = await pw.chromium.launch_persistent_context(**kw)
await ctx.add_init_script(
"""
try {
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
} catch (e) {}
"""
)
return ctx
except Exception as exc:
last_err = exc
logger.warning("launch_persistent_context failed (%s): %s", extra, exc)
continue
assert last_err is not None
raise last_err
async def get_context(self) -> BrowserContext:
if not self._context_alive():
if self._auto_reconnect:
return await self.start_session()
raise RuntimeError("No browser session; call start_session() first.")
return self._context # type: ignore[return-value]
async def get_page(self, *, new: bool = False) -> Page:
"""
Return a page for navigation. Default reuses one tab; set ``new=True`` for a fresh tab.
"""
ctx = await self.get_context()
if new:
return await ctx.new_page()
if self._primary_page is not None:
try:
_ = self._primary_page.url
return self._primary_page
except Exception:
self._primary_page = None
if ctx.pages:
self._primary_page = ctx.pages[0]
return self._primary_page
self._primary_page = await ctx.new_page()
return self._primary_page
async def is_logged_in(self, *, page: Page | None = None, navigate: bool = True) -> bool:
"""
Heuristic login detection (DOM + URL). Not guaranteed if X changes UI.
- ``False`` if still on login / signup flow URLs or login UI is visible.
- ``True`` if home timeline / primary column or account UI is present.
"""
p = page if page is not None else await self.get_page()
if navigate:
await p.goto(self.HOME_URL, wait_until="domcontentloaded", timeout=45_000)
await asyncio.sleep(0.6)
url = (p.url or "").lower()
if any(
x in url
for x in (
"/i/flow/login",
"/i/flow/signup",
"/login",
)
):
return False
# Logged-in indicators (best-effort; X changes selectors periodically)
logged_in_selectors = (
'[data-testid="SideNav_AccountSwitcher_Button"]',
'[data-testid="AppTabBar_Home_Link"]',
'[data-testid="primaryColumn"]',
'[data-testid="tweet"]',
)
for sel in logged_in_selectors:
try:
el = await p.wait_for_selector(sel, timeout=2500, state="visible")
if el:
return True
except Exception:
continue
# Login / gate modal text (guest wall)
try:
guest = p.get_by_text(
"Sign in to X",
exact=False,
)
if await guest.count() and await guest.first.is_visible(timeout=800):
return False
except Exception:
pass
return False
async def session_healthy(self) -> bool:
"""True if browser context responds and ``is_logged_in`` succeeds within retries."""
for attempt in range(self._max_health_retries):
try:
if not self._context_alive():
return False
return await self.is_logged_in(navigate=True)
except Exception as exc:
logger.debug("session_healthy attempt %s: %s", attempt + 1, exc)
await asyncio.sleep(1.0)
return False
async def export_playwright_cookies(self, filename: str = "playwright_cookies.json") -> Path:
"""Optional backup: write context cookies to ``profiles/x/<key>/cookies/``."""
ctx = await self.get_context()
cookies = await ctx.cookies()
path = self._cookies_dir / filename
path.write_text(json.dumps(cookies, indent=2), encoding="utf-8")
return path
async def close(self) -> None:
"""Close Playwright; persistent data remains on disk for next run."""
if self._primary_page is not None:
try:
await self._primary_page.close()
except Exception:
pass
self._primary_page = None
if self._context is not None:
try:
await self._context.close()
except Exception:
pass
self._context = None
if self._pw is not None:
try:
await self._pw.stop()
except Exception:
pass
self._pw = None
logger.info("XAuthManager: closed (%s)", self._profile_key)
# --- sync helpers for Tkinter / scripts without asyncio ---
def start_session_sync(self, *, force_new: bool = False) -> BrowserContext:
return asyncio.run(self.start_session(force_new=force_new))
def get_page_sync(self, *, new: bool = False) -> Page:
return asyncio.run(self.get_page(new=new))
def is_logged_in_sync(self, *, navigate: bool = True) -> bool:
return asyncio.run(self.is_logged_in(navigate=navigate))
def close_sync(self) -> None:
asyncio.run(self.close())
async def _demo() -> None:
logging.basicConfig(level=logging.INFO)
auth = XAuthManager(profile_key="default", headless=False)
try:
await auth.start_session()
page = await auth.get_page()
await page.goto(XAuthManager.LOGIN_FLOW_HINT, wait_until="domcontentloaded")
print(
"浏览器已打开登录流程。请在窗口中手动完成登录(含邮箱验证),完成后回到终端按 Enter…",
)
await asyncio.to_thread(input)
ok = await auth.is_logged_in()
print("is_logged_in:", ok)
if ok:
path = await auth.export_playwright_cookies()
print("Cookies 备份:", path)
finally:
await auth.close()
if __name__ == "__main__":
asyncio.run(_demo())
+156
View File
@@ -0,0 +1,156 @@
"""
Helpers for X (Twitter) web login flow in Playwright.
X often opens a combined sheet titled 「注册 X」 even from /i/flow/login; users must switch to
the actual sign-in path or they loop after entering email. These helpers only automate UI clicks
(no credential stuffing, no bypass).
"""
from __future__ import annotations
import asyncio
import re
from playwright.async_api import Page
LOGIN_URL_X = "https://x.com/i/flow/login"
LOGIN_URL_TWITTER = "https://twitter.com/i/flow/login"
# When X shows this, login inside Playwright often cannot proceed (rate limit / bot heuristics).
X_FLOW_ERROR_RECOVERY_HINT_CN = (
"这是 X 端返回的通用错误,常见于自动化浏览器或风控,并不代表本程序损坏。\n\n"
"可行做法(任选):\n"
"1)点击弹窗「重试」后等待几分钟再试,或更换网络(如手机热点);\n"
"2)在插件管理中「清除本 Profile 浏览器数据」后重新「手动登录」;\n"
"3)用本机普通 Chrome 登录 x.com,导出 Cookie 为 JSON,放到 runtime/x_cookies.json,再点「同步到抓取」。\n\n"
"我们无法绕过 X/Google 的风控策略。"
)
async def detect_x_generic_flow_error(page: Page) -> bool:
"""True if the grey modal 「出错了 / Something went wrong」 is likely visible."""
try:
dlg = page.locator('[role="dialog"]')
if await dlg.count() > 0:
text = (await dlg.first.inner_text())[:1200]
low = text.lower()
if ("出错了" in text or "something went wrong" in low) and (
"重试" in text or "reload" in low or "retry" in low or "重新加载" in text
):
return True
except Exception:
pass
return False
async def prefer_login_over_signup(page: Page, *, max_attempts: int = 5) -> None:
"""
If the modal looks like sign-up (注册 X / Sign up for X), try to switch to login.
Safe to call multiple times; no-op when login sheet is already shown.
"""
for _ in range(max_attempts):
await asyncio.sleep(0.35)
# Already looks like login-oriented sheet (best-effort)
try:
login_heading = page.get_by_role("heading", name=re.compile(r"登录.*X|Sign in to X|登录或注册", re.I))
if await login_heading.count() > 0 and await login_heading.first.is_visible(timeout=400):
return
except Exception:
pass
signup_visible = False
try:
su = page.get_by_text(re.compile(r"注册 X|Sign up for X"), exact=False)
if await su.count() > 0:
signup_visible = await su.first.is_visible(timeout=500)
except Exception:
pass
if not signup_visible:
try:
dlg = page.locator('[role="dialog"]')
if await dlg.count() > 0:
t = await dlg.first.inner_text()
if "注册 X" in t or "Sign up for X" in t:
signup_visible = True
except Exception:
pass
if not signup_visible:
return
clicked = await _try_click_login_switch(page)
if not clicked:
break
async def _try_click_login_switch(page: Page) -> bool:
"""Return True if a qualifying control was clicked."""
# Known Twitter/X test ids (may change)
for tid in ("signupSheetSwitchToLogin", "switch_to_login"):
try:
loc = page.locator(f'[data-testid="{tid}"]')
if await loc.count() > 0 and await loc.first.is_visible(timeout=600):
await loc.first.click(timeout=3000)
await asyncio.sleep(0.8)
return True
except Exception:
continue
# Footer / inverse of 「还没有账号?注册」 → often 「登录」 link nearby
try:
inv = page.get_by_text(re.compile(r"已有账号|已有帐户|Already have an account", re.I))
if await inv.count() > 0:
await inv.first.click(timeout=3500)
await asyncio.sleep(0.85)
return True
except Exception:
pass
try:
plain = page.get_by_role("link", name=re.compile(r"^(登录|Sign in)$", re.I))
if await plain.count() > 0:
await plain.first.click(timeout=3500)
await asyncio.sleep(0.85)
return True
except Exception:
pass
try:
href_login = page.locator('a[href*="login"]').filter(has_not_text=re.compile(r"Google|Apple", re.I))
if await href_login.count() > 0 and await href_login.first.is_visible(timeout=600):
await href_login.first.click(timeout=3500)
await asyncio.sleep(0.85)
return True
except Exception:
pass
try:
link = page.locator('a[href="/login"]').first
if await link.is_visible(timeout=600):
await link.click(timeout=3500)
await asyncio.sleep(0.85)
return True
except Exception:
pass
return False
async def goto_x_login_entry(page: Page, *, timeout: float = 90_000) -> None:
"""Navigate to official login flow and try to leave sign-up shell."""
await page.goto(LOGIN_URL_X, wait_until="domcontentloaded", timeout=timeout)
await prefer_login_over_signup(page)
# Some locales keep showing 「注册 X」 on x.com; twitter.com entry occasionally differs.
try:
su = page.get_by_text(re.compile(r"注册 X|Sign up for X"), exact=False)
if await su.count() > 0 and await su.first.is_visible(timeout=700):
await page.goto(LOGIN_URL_TWITTER, wait_until="domcontentloaded", timeout=timeout)
await prefer_login_over_signup(page)
except Exception:
pass
+19
View File
@@ -0,0 +1,19 @@
"""Task models, SQLite-backed queue, batch parser, and watermark helpers."""
from pixiv_app.tasks.models import DownloadTaskSpec, ParsePreview, ParsedLine, TaskPriority
from pixiv_app.tasks.parser import parse_batch_for_mode, split_input_blob
from pixiv_app.tasks.queue import TaskQueue
from pixiv_app.tasks.watermark import advance_watermark, filter_newer_than_watermark, get_last_synced_illust_id
__all__ = [
"DownloadTaskSpec",
"ParsePreview",
"ParsedLine",
"TaskPriority",
"TaskQueue",
"advance_watermark",
"filter_newer_than_watermark",
"get_last_synced_illust_id",
"parse_batch_for_mode",
"split_input_blob",
]
+50
View File
@@ -0,0 +1,50 @@
from __future__ import annotations
from dataclasses import dataclass, field
from enum import IntEnum
from typing import Any, Literal
class TaskPriority(IntEnum):
LOW = 1
MEDIUM = 2
HIGH = 3
TaskStatus = Literal["pending", "running", "done", "failed"]
@dataclass(frozen=True)
class DownloadTaskSpec:
"""Serializable unit stored in SQLite via TaskQueue (payload omits secrets)."""
target_type: Literal["illust", "novel"]
target_id: int
task_kind: str
priority: TaskPriority
page_index: int | None = None
payload: dict[str, Any] = field(default_factory=dict)
def fingerprint(self) -> str:
page_part = self.page_index if self.page_index is not None else -1
return f"{self.task_kind}:{self.target_type}:{self.target_id}:{page_part}"
@dataclass
class ParsedLine:
"""One logical input token after splitting (before expansion to illust ids)."""
category: Literal["user", "illust", "novel", "tag", "keyword"]
id_value: int
tag: str = ""
keyword_text: str = ""
raw_token: str = ""
pixiv_kind: str = "user"
@dataclass
class ParsePreview:
"""Statistics for GUI preview."""
total_lines: int
by_category: dict[str, int]
+89
View File
@@ -0,0 +1,89 @@
from __future__ import annotations
import re
from typing import Literal
from pixiv_app.core.downloader import parse_pixiv_target
from pixiv_app.tasks.models import ParsePreview, ParsedLine
def split_input_blob(blob: str) -> list[str]:
parts = re.split(r"[\s,,;;\n\r]+", blob.strip())
return [p for p in parts if p]
def parse_batch_for_mode(
raw: str,
*,
ui_mode: Literal["user", "illust", "manga", "novel", "keyword"],
) -> tuple[list[ParsedLine], ParsePreview]:
"""
Parse multi-line / comma / space separated input into structured intents.
ui_mode matches GUI combo selection so validation stays aligned with existing rules.
"""
tokens = split_input_blob(raw)
lines: list[ParsedLine] = []
stats: dict[str, int] = {}
def bump(key: str) -> None:
stats[key] = stats.get(key, 0) + 1
if ui_mode == "keyword":
kw = " ".join(tokens) if tokens else raw.strip()
if kw.startswith("#"):
kw = kw[1:].strip()
if kw:
lines.append(ParsedLine(category="keyword", id_value=0, keyword_text=kw, raw_token=raw.strip()))
bump("keyword")
return lines, ParsePreview(total_lines=len(lines), by_category=stats)
for token in tokens:
t = token.strip()
if not t:
continue
if t.startswith("#"):
tag = t[1:].strip()
if tag:
lines.append(ParsedLine(category="tag", id_value=0, tag=tag, raw_token=t))
bump("tag")
continue
try:
kind, num_id = parse_pixiv_target(t)
except ValueError:
continue
if ui_mode == "novel":
if kind == "novel":
lines.append(ParsedLine(category="novel", id_value=num_id, raw_token=t, pixiv_kind=kind))
bump("novel")
elif kind == "novel_series":
lines.append(ParsedLine(category="novel", id_value=num_id, raw_token=t, pixiv_kind=kind))
bump("novel")
elif kind == "unknown":
lines.append(ParsedLine(category="novel", id_value=num_id, raw_token=t, pixiv_kind=kind))
bump("novel")
continue
if ui_mode in {"illust", "manga"}:
if kind == "illust":
lines.append(ParsedLine(category="illust", id_value=num_id, raw_token=t, pixiv_kind=kind))
bump("illust")
elif kind == "unknown":
lines.append(ParsedLine(category="illust", id_value=num_id, raw_token=t, pixiv_kind=kind))
bump("illust")
continue
# ui_mode == "user"
if kind == "user":
lines.append(ParsedLine(category="user", id_value=num_id, raw_token=t, pixiv_kind=kind))
bump("user")
elif kind == "illust":
lines.append(ParsedLine(category="user", id_value=num_id, raw_token=t, pixiv_kind=kind))
bump("user")
elif kind == "unknown":
lines.append(ParsedLine(category="user", id_value=num_id, raw_token=t, pixiv_kind=kind))
bump("user")
return lines, ParsePreview(total_lines=len(lines), by_category=stats)
+238
View File
@@ -0,0 +1,238 @@
from __future__ import annotations
import json
import sqlite3
import threading
import time
from pixiv_app.tasks.models import DownloadTaskSpec
class TaskQueue:
"""SQLite-backed queue with lease/reclaim; thread-safe for concurrent workers."""
def __init__(self, conn: sqlite3.Connection) -> None:
self._conn = conn
self._lock = threading.Lock()
def enqueue(self, tasks: list[DownloadTaskSpec]) -> tuple[int, int]:
inserted = 0
with self._lock:
with self._conn:
for t in tasks:
fp = t.fingerprint()
cur = self._conn.execute(
"""
INSERT OR IGNORE INTO download_tasks (
fingerprint, target_type, target_id, page_index, task_kind,
priority, status, payload_json
)
VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)
""",
(
fp,
t.target_type,
t.target_id,
t.page_index,
t.task_kind,
int(t.priority),
json.dumps(t.payload, ensure_ascii=False),
),
)
inserted += cur.rowcount
skipped = len(tasks) - inserted
return inserted, skipped
def requeue_failed_by_fingerprints(self, fingerprints: list[str]) -> int:
"""Reset `failed` rows to `pending` so a retry can pick them up."""
if not fingerprints:
return 0
placeholders = ",".join("?" * len(fingerprints))
with self._lock:
with self._conn:
cur = self._conn.execute(
f"""
UPDATE download_tasks
SET status = 'pending',
leased_until = NULL,
worker_id = NULL,
last_error = '',
updated_at = CURRENT_TIMESTAMP
WHERE fingerprint IN ({placeholders}) AND status = 'failed'
""",
fingerprints,
)
return int(cur.rowcount or 0)
def requeue_done_by_fingerprints(self, fingerprints: list[str]) -> int:
"""Reset completed rows to `pending` so the user can rerun the same task batch."""
if not fingerprints:
return 0
placeholders = ",".join("?" * len(fingerprints))
with self._lock:
with self._conn:
cur = self._conn.execute(
f"""
UPDATE download_tasks
SET status = 'pending',
attempts = 0,
leased_until = NULL,
worker_id = NULL,
last_error = '',
updated_at = CURRENT_TIMESTAMP
WHERE fingerprint IN ({placeholders}) AND status = 'done'
""",
fingerprints,
)
return int(cur.rowcount or 0)
def count_incomplete_for_fingerprints(self, fingerprints: list[str]) -> int:
"""Rows for these fingerprints that are not successfully finished (`done`)."""
if not fingerprints:
return 0
placeholders = ",".join("?" * len(fingerprints))
with self._lock:
row = self._conn.execute(
f"""
SELECT COUNT(*) FROM download_tasks
WHERE fingerprint IN ({placeholders}) AND status != 'done'
""",
fingerprints,
).fetchone()
return int(row[0]) if row else 0
def lease(
self,
worker_id: str,
*,
limit: int = 4,
lease_seconds: float = 180.0,
) -> list[int]:
"""Grab up to `limit` tasks; reclaims expired leases."""
now = time.time()
deadline = now + lease_seconds
ids: list[int] = []
with self._lock:
with self._conn:
for _ in range(limit):
cur = self._conn.execute(
"""
WITH picked AS (
SELECT id FROM download_tasks
WHERE status = 'pending'
OR (status = 'running' AND leased_until IS NOT NULL AND leased_until < ?)
ORDER BY priority DESC, id ASC
LIMIT 1
)
UPDATE download_tasks
SET status = 'running',
worker_id = ?,
leased_until = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id IN (SELECT id FROM picked)
RETURNING id
""",
(now, worker_id, deadline),
)
row = cur.fetchone()
if row is None:
break
ids.append(int(row[0]))
return ids
def load_task(self, task_id: int) -> sqlite3.Row | None:
with self._lock:
row = self._conn.execute(
"SELECT * FROM download_tasks WHERE id = ?",
(task_id,),
).fetchone()
return row
def mark_done(self, task_id: int) -> None:
with self._lock:
with self._conn:
self._conn.execute(
"""
UPDATE download_tasks
SET status = 'done',
leased_until = NULL,
worker_id = NULL,
last_error = '',
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(task_id,),
)
def mark_failed_final(self, task_id: int, message: str) -> None:
with self._lock:
with self._conn:
self._conn.execute(
"""
UPDATE download_tasks
SET status = 'failed',
leased_until = NULL,
worker_id = NULL,
last_error = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(message[:2000], task_id),
)
def requeue_retry(self, task_id: int, message: str, max_attempts: int = 4) -> bool:
"""Return True if scheduled for another attempt."""
with self._lock:
with self._conn:
row = self._conn.execute(
"SELECT attempts FROM download_tasks WHERE id = ?",
(task_id,),
).fetchone()
if row is None:
return False
attempts = int(row[0]) + 1
if attempts >= max_attempts:
self._conn.execute(
"""
UPDATE download_tasks
SET status = 'failed',
attempts = ?,
leased_until = NULL,
worker_id = NULL,
last_error = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(attempts, message[:2000], task_id),
)
return False
self._conn.execute(
"""
UPDATE download_tasks
SET status = 'pending',
attempts = ?,
leased_until = NULL,
worker_id = NULL,
last_error = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(attempts, message[:2000], task_id),
)
return True
def count_by_status(self, status: str) -> int:
with self._lock:
row = self._conn.execute(
"SELECT COUNT(*) FROM download_tasks WHERE status = ?",
(status,),
).fetchone()
return int(row[0]) if row else 0
def snapshot_counts(self) -> dict[str, int]:
with self._lock:
rows = self._conn.execute(
"""
SELECT status, COUNT(*) FROM download_tasks GROUP BY status
"""
).fetchall()
return {str(r[0]): int(r[1]) for r in rows}
+34
View File
@@ -0,0 +1,34 @@
from __future__ import annotations
import sqlite3
def get_last_synced_illust_id(conn: sqlite3.Connection, artist_id: int) -> int:
row = conn.execute(
"SELECT last_synced_illust_id FROM artist_sync_watermark WHERE artist_id = ?",
(artist_id,),
).fetchone()
return int(row[0]) if row else 0
def advance_watermark(conn: sqlite3.Connection, artist_id: int, illust_id: int) -> None:
"""Raise watermark to at least illust_id for incremental author sync."""
with conn:
conn.execute(
"""
INSERT INTO artist_sync_watermark (artist_id, last_synced_illust_id, updated_at)
VALUES (?, ?, CURRENT_TIMESTAMP)
ON CONFLICT(artist_id) DO UPDATE SET
last_synced_illust_id = MAX(
artist_sync_watermark.last_synced_illust_id,
excluded.last_synced_illust_id
),
updated_at = CURRENT_TIMESTAMP
""",
(artist_id, illust_id),
)
def filter_newer_than_watermark(work_ids: list[int], watermark: int) -> list[int]:
"""Keep illust ids strictly greater than last synced watermark (Pixiv ids grow over time)."""
return [wid for wid in work_ids if wid > watermark]
+947
View File
@@ -0,0 +1,947 @@
const state = {
view: "home",
query: "",
tag: "",
type: "",
artistId: "",
offset: 0,
limit: 48,
total: 0,
xPreviewId: "",
xPreviewUsername: "",
xPreviewAuthor: "",
xhsPreviewId: "",
};
const els = {
search: document.querySelector("#searchInput"),
type: document.querySelector("#typeFilter"),
status: document.querySelector("#statusLine"),
grid: document.querySelector("#artworkGrid"),
tags: document.querySelector("#tagList"),
artists: document.querySelector("#artistList"),
followed: document.querySelector("#followedList"),
xFollowed: document.querySelector("#xFollowedList"),
title: document.querySelector("#viewTitle"),
subtitle: document.querySelector("#viewSubtitle"),
pageInfo: document.querySelector("#pageInfo"),
dialog: document.querySelector("#detailDialog"),
detail: document.querySelector("#detailContent"),
pixivCookieFile: document.querySelector("#pixivCookieFile"),
pixivSessionStatus: document.querySelector("#pixivSessionStatus"),
refreshSessionButton: document.querySelector("#refreshSessionButton"),
xCookieFile: document.querySelector("#xCookieFile"),
xUsername: document.querySelector("#xUsernameInput"),
xMediaType: document.querySelector("#xMediaType"),
xMaxItems: document.querySelector("#xMaxItems"),
xMediaGrid: document.querySelector("#xMediaGrid"),
xMediaSummary: document.querySelector("#xMediaSummary"),
xSelectAll: document.querySelector("#xSelectAll"),
xFollowUsername: document.querySelector("#xFollowUsername"),
xFollowName: document.querySelector("#xFollowName"),
xLocalGrid: document.querySelector("#xLocalGrid"),
xLocalSummary: document.querySelector("#xLocalSummary"),
xLocalSelectAll: document.querySelector("#xLocalSelectAll"),
xhsNavButton: document.querySelector("#xiaohongshuNavButton"),
xhsTarget: document.querySelector("#xhsTargetInput"),
xhsLimit: document.querySelector("#xhsLimitInput"),
xhsGrid: document.querySelector("#xhsGrid"),
xhsSummary: document.querySelector("#xhsSummary"),
xhsSelectAll: document.querySelector("#xhsSelectAll"),
originFollowSyncDialog: document.querySelector("#originFollowSyncDialog"),
originFollowSyncResult: document.querySelector("#originFollowSyncResult"),
};
function setStatus(text) {
els.status.textContent = text;
}
function isPlaceholderPixivName(name, artistId) {
const value = String(name || "").trim();
if (!value) {
return true;
}
const idText = String(artistId || "").trim();
return Boolean(idText) && value.toLowerCase() === `artist_${idText}`.toLowerCase();
}
function pixivArtistStoredName(item) {
const artistId = item?.artist_id ?? item?.artistId ?? item?.creator_id ?? "";
const name = String(item?.name || item?.artist_name || item?.creator_name || "").trim();
return isPlaceholderPixivName(name, artistId) ? "" : name;
}
function pixivArtistDisplayName(item) {
const artistId = item?.artist_id ?? item?.artistId ?? item?.creator_id ?? "";
const name = pixivArtistStoredName(item);
if (name) {
return name;
}
const account = String(item?.account || item?.artist_account || item?.creator_handle || "").trim().replace(/^@/, "");
if (account) {
return `@${account}`;
}
return artistId ? `Pixiv ID ${artistId}` : "Pixiv 作者";
}
window.addEventListener("unhandledrejection", (event) => {
setStatus(event.reason?.message || "操作失败,请查看后端日志。");
});
async function api(path, options = {}) {
const response = await fetch(path, {
headers: { "Content-Type": "application/json" },
...options,
});
const payload = await response.json();
if (!response.ok || payload.error) {
throw new Error(payload.error || `HTTP ${response.status}`);
}
return payload;
}
async function loadSessionStatus() {
if (!els.pixivSessionStatus) {
return null;
}
try {
const payload = await api("/api/session/status");
const pixiv = payload.pixiv || {};
const cookieText = pixiv.has_cookie
? `已共享桌面端 Pixiv 会话${pixiv.saved_at ? `(${pixiv.saved_at})` : ""}`
: "未检测到桌面端 Pixiv 会话,请先在 EXE 登录或导入 Cookie。";
els.pixivSessionStatus.textContent = cookieText;
return payload;
} catch (error) {
els.pixivSessionStatus.textContent = `会话状态读取失败:${error.message || error}`;
return null;
}
}
function artworkUrl(item, pageIndex = 0, kind = "thumbnail") {
return `/media/${kind}/${item.artwork_id}/${pageIndex}`;
}
function renderArtworkGrid(payload) {
state.total = payload.total || 0;
els.grid.innerHTML = "";
if (!payload.items.length) {
els.grid.innerHTML = `<div class="empty">还没有作品。可以先下载作品,或点击“扫描旧目录”导入已有文件。</div>`;
} else {
els.grid.append(...payload.items.map(renderArtworkCard));
}
const currentPage = Math.floor(state.offset / state.limit) + 1;
const totalPages = Math.max(1, Math.ceil(state.total / state.limit));
els.pageInfo.textContent = `${currentPage} / ${totalPages}`;
}
function renderArtworkCard(item) {
const card = document.createElement("article");
card.className = "artwork-card";
const thumb = item.thumbnail_path
? `<img loading="lazy" src="${artworkUrl(item)}" alt="${escapeHtml(item.title || String(item.artwork_id))}">`
: `<div class="thumb-placeholder">${item.type === "novel" ? "小说" : "暂无缩略图"}</div>`;
card.innerHTML = `
<button class="thumb-button" data-open="${item.artwork_id}">${thumb}</button>
<div class="artwork-meta">
<div class="artwork-title">${escapeHtml(item.title || String(item.artwork_id))}</div>
<div class="artwork-sub">${escapeHtml(item.artist_name || "未知作者")} · ${escapeHtml(item.type || "")}</div>
<div class="card-actions">
<button class="small-button ${item.is_favorite ? "active" : ""}" data-fav="${item.artwork_id}" data-on="${item.is_favorite ? 1 : 0}">
${item.is_favorite ? "已收藏" : "收藏"}
</button>
${item.artist_id ? `<button class="small-button" data-artist="${item.artist_id}">作者</button>` : ""}
${item.artist_id ? `<button class="small-button ${Number(item.is_followed || 0) ? "active" : ""}" data-follow="${item.artist_id}" data-name="${escapeHtml(pixivArtistStoredName(item))}" data-followed="${Number(item.is_followed || 0)}">${Number(item.is_followed || 0) ? "取消关注" : "关注作者"}</button>` : ""}
</div>
</div>
`;
const artworkSub = card.querySelector(".artwork-sub");
if (artworkSub) {
artworkSub.textContent = `${pixivArtistDisplayName(item)} · ${item.type || ""}`;
}
const artworkFollow = card.querySelector("[data-follow]");
if (artworkFollow) {
artworkFollow.dataset.name = pixivArtistStoredName(item);
}
return card;
}
async function loadArtworks() {
const params = new URLSearchParams({
limit: state.limit,
offset: state.offset,
ensure_thumbnails: "true",
});
if (state.query) params.set("q", state.query);
if (state.tag) params.set("tag", state.tag);
if (state.type) params.set("type", state.type);
if (state.artistId) params.set("artist_id", state.artistId);
setStatus("正在加载图库...");
const payload = await api(`/api/artworks?${params}`);
renderArtworkGrid(payload);
setStatus(`已加载 ${payload.items.length} / ${payload.total} 个作品`);
}
async function loadFavorites() {
setStatus("正在加载收藏...");
const params = new URLSearchParams({ limit: state.limit, offset: state.offset });
const payload = await api(`/api/favorites?${params}`);
renderArtworkGrid(payload);
setStatus(`收藏 ${payload.total} 个作品`);
}
async function loadTags() {
const tags = await api("/api/tags?limit=80");
els.tags.innerHTML = "";
if (!tags.length) {
els.tags.innerHTML = `<span class="tag-chip">暂无标签</span>`;
return;
}
for (const tag of tags) {
const button = document.createElement("button");
button.className = `tag-chip ${state.tag === tag.name ? "active" : ""}`;
button.textContent = `${tag.name} ${tag.artwork_count}`;
button.dataset.tag = tag.name;
els.tags.append(button);
}
}
async function loadArtists() {
setStatus("正在加载作者...");
const artists = await api("/api/artists?limit=200");
els.artists.innerHTML = "";
if (!artists.length) {
els.artists.innerHTML = `<div class="empty">暂无作者数据。</div>`;
} else {
els.artists.append(...artists.map(renderArtistRow));
}
setStatus(`已加载 ${artists.length} 位作者`);
}
async function loadFollowed() {
const followed = await api("/api/followed-artists");
els.followed.innerHTML = "";
if (!followed.length) {
els.followed.innerHTML = `<div class="empty">还没有关注作者。</div>`;
} else {
els.followed.append(...followed.map(renderArtistRow));
}
}
async function loadXFollowed() {
const followed = await api("/api/x/follows");
els.xFollowed.innerHTML = "";
if (!followed.length) {
els.xFollowed.innerHTML = `<div class="empty">还没有 X 关注作者。</div>`;
} else {
els.xFollowed.append(...followed.map(renderXFollowRow));
}
}
function renderArtistRow(artist) {
const row = document.createElement("div");
row.className = "artist-row";
const isFollowed = Number(artist.enabled ?? artist.is_followed ?? 0);
row.innerHTML = `
<div>
<strong>${escapeHtml(artist.name || `artist_${artist.artist_id}`)}</strong>
<span>ID ${artist.artist_id} · ${artist.artwork_count || 0} 个作品${artist.last_synced_at ? ` · 上次同步 ${artist.last_synced_at}` : ""}</span>
</div>
<div class="card-actions">
<button class="small-button" data-artist="${artist.artist_id}">详情</button>
<button class="small-button ${isFollowed ? "active" : ""}" data-follow="${artist.artist_id}" data-name="${escapeHtml(artist.name || "")}" data-followed="${isFollowed ? 1 : 0}">${isFollowed ? "取消关注" : "关注"}</button>
</div>
`;
const artistTitle = row.querySelector("strong");
if (artistTitle) {
artistTitle.textContent = pixivArtistDisplayName(artist);
}
const artistFollow = row.querySelector("[data-follow]");
if (artistFollow) {
artistFollow.dataset.name = pixivArtistStoredName(artist);
}
return row;
}
function renderXFollowRow(artist) {
const row = document.createElement("div");
row.className = "artist-row";
const handle = artist.username || artist.creator_handle || "";
const isFollowed = Number(artist.enabled ?? 1);
const latest = artist.latest_published_at ? ` · 最新 ${artist.latest_published_at}` : "";
const checked = artist.last_checked_at ? ` · 检查 ${artist.last_checked_at}` : "";
const error = artist.last_error ? ` · ${escapeHtml(artist.last_error)}` : "";
row.innerHTML = `
<div>
<strong>${escapeHtml(artist.creator_name || handle || "x_author")}</strong>
<span>@${escapeHtml(handle)}${latest}${checked}${error}</span>
</div>
<div class="card-actions">
<button class="small-button" data-open-x="${escapeHtml(handle)}">去预览</button>
<button class="small-button ${isFollowed ? "active" : ""}" data-follow-x="${escapeHtml(handle)}" data-name="${escapeHtml(artist.creator_name || "")}" data-followed-x="${isFollowed ? 1 : 0}">${isFollowed ? "取消关注" : "关注"}</button>
</div>
`;
return row;
}
async function openArtwork(artworkId) {
const payload = await api(`/api/artworks/${artworkId}`);
const artwork = payload.artwork;
const files = payload.files || [];
const tags = payload.tags || [];
const firstFile = files[0];
const image = firstFile
? `<img src="/media/source/${artwork.artwork_id}/${firstFile.page_index}" alt="${escapeHtml(artwork.title)}">`
: `<div class="thumb-placeholder">没有本地文件</div>`;
els.detail.innerHTML = `
<div class="detail-layout">
<div class="detail-media">${image}</div>
<div class="detail-info">
<h2>${escapeHtml(artwork.title || String(artwork.artwork_id))}</h2>
<p>${escapeHtml(artwork.artist_name || "未知作者")} · ${escapeHtml(artwork.type)}</p>
<div class="card-actions">
<button class="small-button ${artwork.is_favorite ? "active" : ""}" data-fav="${artwork.artwork_id}" data-on="${artwork.is_favorite ? 1 : 0}">
${artwork.is_favorite ? "已收藏" : "收藏"}
</button>
${artwork.artist_id ? `<button class="small-button ${Number(artwork.is_followed || 0) ? "active" : ""}" data-follow="${artwork.artist_id}" data-name="${escapeHtml(artwork.artist_name || "")}" data-followed="${Number(artwork.is_followed || 0)}">${Number(artwork.is_followed || 0) ? "取消关注" : "关注作者"}</button>` : ""}
</div>
<div class="detail-tags">
${tags.map((tag) => `<button class="tag-chip" data-tag="${escapeHtml(tag.name)}">${escapeHtml(tag.name)}</button>`).join("")}
</div>
<p>${escapeHtml(stripHtml(artwork.caption || ""))}</p>
<p>页数 ${files.length || artwork.page_count || 0} · 收藏 ${artwork.bookmark_count || 0} · 浏览 ${artwork.view_count || 0}</p>
</div>
</div>
`;
const detailArtist = els.detail.querySelector(".detail-info p");
if (detailArtist) {
detailArtist.textContent = `${pixivArtistDisplayName(artwork)} · ${artwork.type}`;
}
const detailFollow = els.detail.querySelector("[data-follow]");
if (detailFollow) {
detailFollow.dataset.name = pixivArtistStoredName(artwork);
}
els.dialog.showModal();
}
async function showArtist(artistId) {
setActiveView("gallery");
state.artistId = String(artistId);
state.tag = "";
state.offset = 0;
const detail = await api(`/api/artists/${artistId}?limit=1`);
els.title.textContent = detail.artist.name || `作者 ${artistId}`;
els.subtitle.textContent = `作者 ID ${artistId} · ${detail.artist.artwork_count || 0} 个本地作品`;
await loadArtworks();
els.title.textContent = pixivArtistDisplayName(detail.artist);
}
async function toggleFavorite(artworkId, currentOn) {
await api("/api/favorites", {
method: "POST",
body: JSON.stringify({ artwork_id: Number(artworkId), favorite: !Number(currentOn) }),
});
await reloadCurrent();
}
async function followArtist(artistId, name = "") {
await setPixivFollow(artistId, name, true);
}
async function setPixivFollow(artistId, name = "", followed = true) {
await api("/api/followed-artists", {
method: "POST",
body: JSON.stringify({ artist_id: String(artistId).trim(), name, followed }),
});
setStatus(followed ? `已关注作者 ${artistId}` : `已取消关注作者 ${artistId}`);
await Promise.all([loadFollowed(), reloadCurrent()]);
}
async function followXAuthor(username, name = "") {
await setXFollow(username, name, true);
}
async function setXFollow(username, name = "", followed = true) {
const handle = username.trim().replace(/^@/, "");
if (!handle) {
return;
}
await api("/api/x/follows", {
method: "POST",
body: JSON.stringify({ username: handle, name, followed }),
});
setStatus(followed ? `已关注 X 作者 @${handle}` : `已取消关注 X 作者 @${handle}`);
await loadXFollowed();
}
async function scanLegacy() {
setStatus("正在扫描旧下载目录...");
const result = await api("/api/import/legacy", { method: "POST", body: JSON.stringify({ root: "Sakura_Downloads" }) });
setStatus(`扫描完成:导入 ${result.imported_files} 个文件,${result.imported_artworks} 个作品`);
await Promise.all([loadTags(), reloadCurrent()]);
}
async function syncFollowed() {
setStatus("正在同步关注作者...");
try {
const result = await api("/api/sync/followed", {
method: "POST",
body: JSON.stringify({ download: true, max_new: 20 }),
});
const downloaded = result.artists.reduce((sum, item) => sum + (item.downloaded || 0), 0);
setStatus(`同步完成:下载 ${downloaded} 个新作品`);
await Promise.all([loadFollowed(), loadTags(), reloadCurrent()]);
} catch (error) {
setStatus(error.message || "同步关注作者失败");
}
}
async function syncXFollowed() {
setStatus("正在同步 X 关注...");
const result = await api("/api/x/follows/sync", {
method: "POST",
body: JSON.stringify({ download: true, max_new: 20 }),
});
const total = (result.authors || []).reduce((sum, item) => sum + (item.new_posts || 0), 0);
setStatus(`X 同步完成:发现 ${total} 个新动态`);
await loadXFollowed();
}
function openOriginFollowSyncDialog() {
els.originFollowSyncResult.textContent = "请选择要同步的平台。";
els.originFollowSyncDialog.showModal();
}
async function syncOriginFollows(platform) {
const label = platform === "pixiv" ? "Pixiv" : "X";
const button = platform === "pixiv" ? document.querySelector("#syncOriginPixivButton") : document.querySelector("#syncOriginXButton");
if (button?.disabled) {
return;
}
if (button) {
button.disabled = true;
}
setStatus(`正在同步 ${label} 原站关注...`);
els.originFollowSyncResult.textContent = `正在同步 ${label} 原站关注,请稍候...`;
try {
if (platform === "pixiv") {
const session = await loadSessionStatus();
if (session && !session.pixiv?.has_cookie) {
throw new Error("未检测到桌面端 Pixiv 会话,请先在 EXE 登录或导入 Cookie。");
}
}
const result = await api("/api/origin-follows/sync", {
method: "POST",
body: JSON.stringify({ platform }),
});
const lines = [
`${label} 原站关注同步完成`,
`同步页数:${result.pages || 0}`,
`新增:${result.created || 0} 位`,
`更新:${result.updated || 0} 位`,
`跳过:${result.skipped || 0} 位`,
`失败:${result.failed || 0} 位`,
];
if (Array.isArray(result.progress) && result.progress.length) {
lines.push("", "进度:");
for (const item of result.progress.slice(-20)) {
lines.push(item.message || `正在同步第 ${item.page || "?"} 页`);
}
}
if (Array.isArray(result.errors) && result.errors.length) {
lines.push("", "失败原因:");
for (const item of result.errors.slice(0, 10)) {
lines.push(`${item.id || "-"}:${item.reason || ""}`);
}
}
els.originFollowSyncResult.textContent = lines.join("\n");
setStatus(`${label} 原站关注同步完成:新增 ${result.created || 0},更新 ${result.updated || 0},跳过 ${result.skipped || 0},失败 ${result.failed || 0}`);
if (platform === "pixiv") {
await loadFollowed();
} else {
await loadXFollowed();
}
} catch (error) {
const message = error.message || `${label} 原站关注同步失败`;
els.originFollowSyncResult.textContent = message;
setStatus(message);
} finally {
if (button) {
button.disabled = false;
}
}
}
async function followCurrentX() {
const username = els.xUsername.value.trim();
if (!username) {
setStatus("请先输入 X 作者名。");
return;
}
const handle = username.replace(/^@/, "");
const creatorName = state.xPreviewUsername === handle ? state.xPreviewAuthor : "";
await followXAuthor(username, creatorName || handle);
}
async function buildThumbnails() {
setStatus("正在生成缩略图...");
const result = await api("/api/thumbnails/build?limit=500");
setStatus(`缩略图完成:处理 ${result.items.length} 个文件`);
await reloadCurrent();
}
async function loadPluginPages() {
const payload = await api("/api/plugins/pages");
const pages = payload.items || [];
const hasXhs = pages.some((item) => item.view === "xiaohongshu");
els.xhsNavButton.classList.toggle("hidden", !hasXhs);
}
async function importCookieFile(platform, fileInput) {
const file = fileInput.files?.[0];
if (!file) {
setStatus("请先选择 cookies.txt 或 json 文件。");
return;
}
setStatus(`正在导入 ${platform} Cookie...`);
const text = await file.text();
const result = await api("/api/cookies/import", {
method: "POST",
body: JSON.stringify({ platform, text }),
});
setStatus(`Cookie 导入完成:${result.summary || `${result.count || 0} 个 Cookie`}`);
}
async function previewXMedia(event) {
event.preventDefault();
const username = els.xUsername.value.trim();
if (!username) {
setStatus("请输入 X 作者名称。");
return;
}
els.xMediaGrid.innerHTML = `<div class="empty">正在解析 X 媒体页,首次启动 Playwright 可能需要一些时间。</div>`;
els.xMediaSummary.textContent = "解析中...";
const submitButton = event.submitter || document.querySelector("#xMediaForm button[type='submit']");
if (submitButton) {
submitButton.disabled = true;
}
setStatus(`正在解析 X 作者 @${username.replace(/^@/, "")}...`);
try {
const payload = await api("/api/x/media/preview", {
method: "POST",
body: JSON.stringify({
username,
media_type: els.xMediaType.value,
max_items: Number(els.xMaxItems.value || 120),
}),
});
state.xPreviewId = payload.preview_id;
renderXMedia(payload);
setStatus(`X 解析完成:${payload.count} 个媒体项`);
} catch (error) {
state.xPreviewId = "";
els.xMediaSummary.textContent = "解析失败";
els.xMediaGrid.innerHTML = `<div class="empty">${escapeHtml(error.message || "X 解析失败")}</div>`;
setStatus(error.message || "X 解析失败");
} finally {
if (submitButton) {
submitButton.disabled = false;
}
}
}
function renderXMedia(payload) {
els.xMediaGrid.innerHTML = "";
els.xMediaSummary.textContent = `${payload.username} · ${payload.count} 个媒体项`;
if (!payload.items.length) {
els.xMediaGrid.innerHTML = `<div class="empty">没有解析到媒体。可能需要先导入 X Cookie,或该作者媒体页为空。</div>`;
return;
}
els.xMediaGrid.append(...payload.items.map(renderXMediaCard));
els.xSelectAll.checked = true;
}
function renderXMediaCard(item) {
const card = document.createElement("article");
card.className = "x-media-card";
const preview = item.thumbnail || item.files?.[0]?.url || "";
const media = item.kind === "video"
? `<div class="video-tile">VIDEO</div>`
: `<img loading="lazy" referrerpolicy="no-referrer" src="${escapeHtml(preview)}" alt="${escapeHtml(item.title || item.id)}">`;
card.innerHTML = `
<label class="x-check">
<input type="checkbox" data-x-id="${escapeHtml(item.id)}" checked />
<span>${item.kind === "video" ? "视频" : "图片"}</span>
</label>
<div class="x-thumb">${media}</div>
<div class="artwork-meta">
<div class="artwork-title">${escapeHtml(item.title || item.id)}</div>
<div class="artwork-sub">@${escapeHtml(item.author_id || "")} · ${item.file_count || 1} 个文件</div>
</div>
`;
return card;
}
async function downloadXSelected() {
if (!state.xPreviewId) {
setStatus("请先解析 X 作者媒体。");
return;
}
const selected = [...document.querySelectorAll("[data-x-id]:checked")].map((item) => item.dataset.xId);
if (!selected.length) {
setStatus("没有选中的媒体。");
return;
}
setStatus(`正在下载 ${selected.length} 个 X 媒体项...`);
const result = await api("/api/x/media/download", {
method: "POST",
body: JSON.stringify({ preview_id: state.xPreviewId, selected_ids: selected }),
});
setStatus(`X 下载完成:保存 ${result.downloaded_files} 个文件,失败 ${result.failed_items} 项`);
await loadXLocalMedia();
}
async function loadXLocalMedia() {
const payload = await api("/api/x/local-media?limit=160");
els.xLocalGrid.innerHTML = "";
els.xLocalSummary.textContent = `本地 ${payload.total || 0} 个文件`;
if (!payload.items.length) {
els.xLocalGrid.innerHTML = `<div class="empty">还没有本地 X 图片或视频。</div>`;
return;
}
els.xLocalGrid.append(...payload.items.map(renderXLocalCard));
}
function renderXLocalCard(item) {
const card = document.createElement("article");
card.className = "x-media-card";
const media = item.kind === "video"
? `<video src="${escapeHtml(item.url)}" muted preload="metadata" controls></video>`
: `<img loading="lazy" src="${escapeHtml(item.url)}" alt="${escapeHtml(item.name)}">`;
card.innerHTML = `
<label class="x-check">
<input type="checkbox" data-x-local-id="${escapeHtml(item.id)}" />
<span>${item.kind === "video" ? "视频" : "图片"}</span>
</label>
<div class="x-thumb">${media}</div>
<div class="artwork-meta">
<div class="artwork-title">${escapeHtml(item.name)}</div>
<div class="artwork-sub">${escapeHtml(item.relative_path)} · ${formatBytes(item.size)}</div>
</div>
`;
return card;
}
async function deleteXLocalSelected() {
const selected = [...document.querySelectorAll("[data-x-local-id]:checked")].map((item) => item.dataset.xLocalId);
if (!selected.length) {
setStatus("没有选中的本地 X 文件。");
return;
}
setStatus(`正在删除 ${selected.length} 个本地 X 文件...`);
const result = await api("/api/x/local-media/delete", {
method: "POST",
body: JSON.stringify({ ids: selected }),
});
setStatus(`删除完成:${result.deleted.length} 个,失败 ${result.failed.length} 个`);
els.xLocalSelectAll.checked = false;
await loadXLocalMedia();
}
async function previewXhs(event) {
event.preventDefault();
const target = els.xhsTarget.value.trim();
if (!target) {
setStatus("请输入小红书关键词或笔记链接。");
return;
}
els.xhsGrid.innerHTML = `<div class="empty">正在解析小红书内容...</div>`;
els.xhsSummary.textContent = "解析中...";
const payload = await api("/api/xiaohongshu/preview", {
method: "POST",
body: JSON.stringify({ target, limit: Number(els.xhsLimit.value || 20) }),
});
state.xhsPreviewId = payload.preview_id;
renderXhs(payload);
setStatus(`小红书解析完成:${payload.count} 个项目`);
}
function renderXhs(payload) {
els.xhsGrid.innerHTML = "";
els.xhsSummary.textContent = `${payload.count} 个项目`;
if (!payload.items.length) {
els.xhsGrid.innerHTML = `<div class="empty">没有解析到内容,可能需要先登录或换一个关键词。</div>`;
return;
}
els.xhsGrid.append(...payload.items.map(renderXhsCard));
els.xhsSelectAll.checked = true;
}
function renderXhsCard(item) {
const card = document.createElement("article");
card.className = "x-media-card";
const preview = item.thumbnail || item.metadata?.cover_url || "";
const media = preview
? `<img loading="lazy" referrerpolicy="no-referrer" src="${escapeHtml(preview)}" alt="${escapeHtml(item.title || item.id)}">`
: `<div class="thumb-placeholder">暂无预览</div>`;
card.innerHTML = `
<label class="x-check">
<input type="checkbox" data-xhs-id="${escapeHtml(item.id)}" checked />
<span>笔记</span>
</label>
<div class="x-thumb">${media}</div>
<div class="artwork-meta">
<div class="artwork-title">${escapeHtml(item.title || item.id)}</div>
<div class="artwork-sub">${escapeHtml(item.author || "")} · ${item.file_count || 0} 个文件</div>
</div>
`;
return card;
}
async function downloadXhsSelected() {
if (!state.xhsPreviewId) {
setStatus("请先解析小红书内容。");
return;
}
const selected = [...document.querySelectorAll("[data-xhs-id]:checked")].map((item) => item.dataset.xhsId);
if (!selected.length) {
setStatus("没有选中的小红书项目。");
return;
}
setStatus(`正在下载 ${selected.length} 个小红书项目...`);
const result = await api("/api/xiaohongshu/download", {
method: "POST",
body: JSON.stringify({ preview_id: state.xhsPreviewId, selected_ids: selected }),
});
setStatus(`小红书下载完成:保存 ${result.downloaded_files} 个文件,失败 ${result.failed_items} 项`);
}
function setActiveView(view) {
state.view = view;
document.querySelectorAll(".nav-button").forEach((button) => {
button.classList.toggle("active", button.dataset.view === view);
});
document.querySelectorAll(".view").forEach((section) => section.classList.remove("active"));
if (view === "home") {
document.querySelector("#homeView").classList.add("active");
} else if (view === "artists") {
document.querySelector("#artistsView").classList.add("active");
} else if (view === "sync") {
document.querySelector("#syncView").classList.add("active");
} else if (view === "xMedia") {
document.querySelector("#xMediaView").classList.add("active");
} else if (view === "xiaohongshu") {
document.querySelector("#xiaohongshuView").classList.add("active");
} else {
document.querySelector("#galleryView").classList.add("active");
}
}
async function reloadCurrent() {
if (state.view === "home") {
setStatus("首页已就绪。选择一个入口开始整理或同步内容。");
} else if (state.view === "favorites") {
els.title.textContent = "本地收藏";
els.subtitle.textContent = "你标记收藏的作品";
await loadFavorites();
} else if (state.view === "artists") {
await loadArtists();
} else if (state.view === "sync") {
await Promise.all([loadFollowed(), loadXFollowed()]);
} else if (state.view === "xMedia") {
await loadXLocalMedia();
setStatus("X 作者媒体页已就绪。");
} else if (state.view === "xiaohongshu") {
setStatus("小红书插件页已就绪。");
} else {
if (!state.artistId) {
els.title.textContent = state.tag ? `标签:${state.tag}` : "全部作品";
els.subtitle.textContent = state.query ? `搜索:${state.query}` : "本地已入库作品";
}
await loadArtworks();
}
}
function clearFilters() {
state.query = "";
state.tag = "";
state.type = "";
state.artistId = "";
state.offset = 0;
els.search.value = "";
els.type.value = "";
}
function escapeHtml(value) {
return String(value ?? "").replace(/[&<>"']/g, (char) => ({
"&": "&amp;",
"<": "&lt;",
">": "&gt;",
'"': "&quot;",
"'": "&#39;",
})[char]);
}
function stripHtml(value) {
const tmp = document.createElement("div");
tmp.innerHTML = value;
return tmp.textContent || tmp.innerText || "";
}
function formatBytes(value) {
const size = Number(value || 0);
if (size < 1024) return `${size} B`;
if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KB`;
return `${(size / 1024 / 1024).toFixed(1)} MB`;
}
document.querySelectorAll(".nav-button").forEach((button) => {
button.addEventListener("click", async () => {
setActiveView(button.dataset.view);
state.offset = 0;
if (state.view !== "gallery") {
state.artistId = "";
}
await reloadCurrent();
});
});
els.search.addEventListener("input", debounce(async () => {
state.query = els.search.value.trim();
state.offset = 0;
setActiveView("gallery");
await reloadCurrent();
}, 280));
els.type.addEventListener("change", async () => {
state.type = els.type.value;
state.offset = 0;
setActiveView("gallery");
await reloadCurrent();
});
document.addEventListener("click", async (event) => {
const target = event.target.closest("button");
if (!target) return;
if (target.dataset.viewAction) {
setActiveView(target.dataset.viewAction);
state.offset = 0;
await reloadCurrent();
return;
}
if (target.dataset.open) await openArtwork(target.dataset.open);
if (target.dataset.fav) await toggleFavorite(target.dataset.fav, target.dataset.on);
if (target.dataset.artist) await showArtist(target.dataset.artist);
if (target.dataset.tag) {
state.tag = target.dataset.tag;
state.artistId = "";
state.offset = 0;
setActiveView("gallery");
await loadTags();
await reloadCurrent();
}
if (target.dataset.follow) {
const nextFollowed = target.dataset.followed === "1" ? false : true;
await setPixivFollow(target.dataset.follow, target.dataset.name || "", nextFollowed);
}
if (target.dataset.followX) {
const nextFollowed = target.dataset.followedX === "1" ? false : true;
await setXFollow(target.dataset.followX, target.dataset.name || "", nextFollowed);
}
if (target.dataset.openX) {
els.xUsername.value = `@${target.dataset.openX}`;
setActiveView("xMedia");
setStatus(`已填入 @${target.dataset.openX},可以直接解析最新媒体。`);
}
});
document.querySelector("#clearFiltersButton").addEventListener("click", async () => {
clearFilters();
await loadTags();
await reloadCurrent();
});
document.querySelector("#prevPageButton").addEventListener("click", async () => {
state.offset = Math.max(0, state.offset - state.limit);
await reloadCurrent();
});
document.querySelector("#nextPageButton").addEventListener("click", async () => {
if (state.offset + state.limit < state.total) {
state.offset += state.limit;
await reloadCurrent();
}
});
document.querySelector("#scanButton").addEventListener("click", scanLegacy);
document.querySelector("#homeScanButton")?.addEventListener("click", scanLegacy);
document.querySelector("#originFollowSyncButton").addEventListener("click", openOriginFollowSyncDialog);
document.querySelector("#closeOriginFollowSyncDialogButton").addEventListener("click", () => els.originFollowSyncDialog.close());
document.querySelector("#syncOriginPixivButton").addEventListener("click", () => syncOriginFollows("pixiv"));
document.querySelector("#syncOriginXButton").addEventListener("click", () => syncOriginFollows("x"));
document.querySelector("#syncButton").addEventListener("click", syncFollowed);
document.querySelector("#syncXButton").addEventListener("click", syncXFollowed);
document.querySelector("#buildThumbsButton").addEventListener("click", buildThumbnails);
document.querySelector("#homeThumbButton")?.addEventListener("click", buildThumbnails);
document.querySelector("#closeDialogButton").addEventListener("click", () => els.dialog.close());
document.querySelector("#importPixivCookieButton")?.addEventListener("click", () => importCookieFile("pixiv", els.pixivCookieFile));
els.refreshSessionButton?.addEventListener("click", loadSessionStatus);
document.querySelector("#importXCookieButton").addEventListener("click", () => importCookieFile("x", els.xCookieFile));
document.querySelector("#xMediaForm").addEventListener("submit", previewXMedia);
document.querySelector("#followCurrentXButton").addEventListener("click", followCurrentX);
document.querySelector("#downloadXSelectedButton").addEventListener("click", downloadXSelected);
document.querySelector("#refreshXLocalButton").addEventListener("click", loadXLocalMedia);
document.querySelector("#deleteXLocalButton").addEventListener("click", deleteXLocalSelected);
document.querySelector("#xhsForm").addEventListener("submit", previewXhs);
document.querySelector("#downloadXhsSelectedButton").addEventListener("click", downloadXhsSelected);
els.xSelectAll.addEventListener("change", () => {
document.querySelectorAll("[data-x-id]").forEach((item) => {
item.checked = els.xSelectAll.checked;
});
});
els.xLocalSelectAll.addEventListener("change", () => {
document.querySelectorAll("[data-x-local-id]").forEach((item) => {
item.checked = els.xLocalSelectAll.checked;
});
});
els.xhsSelectAll.addEventListener("change", () => {
document.querySelectorAll("[data-xhs-id]").forEach((item) => {
item.checked = els.xhsSelectAll.checked;
});
});
document.querySelector("#followForm").addEventListener("submit", async (event) => {
event.preventDefault();
const id = document.querySelector("#followArtistId").value.trim();
const name = document.querySelector("#followArtistName").value.trim();
if (!id) return;
await followArtist(id, name);
event.target.reset();
});
document.querySelector("#xFollowForm").addEventListener("submit", async (event) => {
event.preventDefault();
const username = els.xFollowUsername.value.trim();
const name = els.xFollowName.value.trim();
if (!username) return;
await followXAuthor(username, name);
event.target.reset();
});
function debounce(fn, delay) {
let timer = 0;
return (...args) => {
clearTimeout(timer);
timer = setTimeout(() => fn(...args), delay);
};
}
async function boot() {
try {
await Promise.all([loadPluginPages(), loadSessionStatus(), loadTags(), loadArtworks(), loadFollowed(), loadXFollowed()]);
} catch (error) {
setStatus(error.message);
}
}
boot();
+284
View File
@@ -0,0 +1,284 @@
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Sakura 本地媒体库</title>
<link rel="stylesheet" href="/assets/styles.css" />
</head>
<body>
<aside class="sidebar">
<div class="brand">
<div class="brand-mark">SK</div>
<div>
<h1>Sakura</h1>
<p>本地媒体库</p>
</div>
</div>
<nav class="nav">
<button class="nav-button active" data-view="home">首页</button>
<button class="nav-button" data-view="gallery">图库</button>
<button class="nav-button" data-view="favorites">收藏</button>
<button class="nav-button" data-view="artists">作者</button>
<button class="nav-button" data-view="sync">关注同步</button>
</nav>
<section class="panel plugin-panel">
<h2>插件页面</h2>
<div id="pluginNav" class="nav compact-nav">
<button class="nav-button" data-view="xMedia">X 作者媒体</button>
<button id="xiaohongshuNavButton" class="nav-button hidden" data-view="xiaohongshu">小红书</button>
</div>
</section>
<section class="panel">
<h2>标签</h2>
<div id="tagList" class="tag-list"></div>
</section>
</aside>
<main class="main">
<header class="toolbar">
<div class="search-wrap">
<input id="searchInput" type="search" placeholder="搜索标题、作者、简介或标签" />
<select id="typeFilter" aria-label="类型筛选">
<option value="">全部类型</option>
<option value="illust">插画</option>
<option value="manga">漫画</option>
<option value="novel">小说</option>
<option value="video">视频</option>
</select>
</div>
<div class="toolbar-actions">
<button id="buildThumbsButton" class="ghost-button">生成缩略图</button>
<button id="scanButton" class="primary-button">扫描旧目录</button>
</div>
</header>
<section id="statusLine" class="status-line">准备就绪</section>
<section id="homeView" class="view active">
<div class="welcome-card">
<div>
<p class="eyebrow">本地优先</p>
<h2>今天想看什么?</h2>
<p>插画、漫画、小说和视频都保存在本机。你可以直接浏览、搜索和收藏。</p>
</div>
<button class="primary-button" data-view-action="gallery">打开图库</button>
</div>
<div class="quick-grid">
<button class="quick-card" data-view-action="gallery">
<strong>查看本地图库</strong>
<span>按标题、作者、类型和标签筛选已经入库的作品。</span>
</button>
<button class="quick-card" data-view-action="sync">
<strong>同步关注作者</strong>
<span>读取 Pixiv / X 关注列表,把新作品下载到本地媒体库。</span>
</button>
<button class="quick-card" id="homeScanButton" type="button">
<strong>导入旧下载目录</strong>
<span>把 Sakura_Downloads 里的旧文件重新扫描进图库。</span>
</button>
<button class="quick-card" id="homeThumbButton" type="button">
<strong>修复缩略图</strong>
<span>图片不显示时,重新生成缺失的预览图。</span>
</button>
</div>
<div class="help-steps">
<article>
<b>1</b>
<div>
<strong>在 EXE 主界面导入 Cookie</strong>
<span>Pixiv、X、小红书等登录态会保存到本地运行目录。</span>
</div>
</article>
<article>
<b>2</b>
<div>
<strong>选择平台和目标后开始抓取</strong>
<span>作者、单作品、关键词、插件链接都可以从主界面发起。</span>
</div>
</article>
<article>
<b>3</b>
<div>
<strong>回到图库整理内容</strong>
<span>收藏、关注作者、生成缩略图和查看详情都在这里完成。</span>
</div>
</article>
</div>
</section>
<section id="galleryView" class="view">
<div class="section-head">
<div>
<h2 id="viewTitle">全部作品</h2>
<p id="viewSubtitle">本地已入库作品</p>
</div>
<button id="clearFiltersButton" class="ghost-button">清除筛选</button>
</div>
<div id="artworkGrid" class="artwork-grid"></div>
<div class="pager">
<button id="prevPageButton" class="ghost-button">上一页</button>
<span id="pageInfo">0 / 0</span>
<button id="nextPageButton" class="ghost-button">下一页</button>
</div>
</section>
<section id="artistsView" class="view">
<div class="section-head">
<div>
<h2>作者</h2>
<p>按本地作品数量排序</p>
</div>
</div>
<div id="artistList" class="artist-list"></div>
</section>
<section id="syncView" class="view">
<div class="import-strip">
<div>
<strong>同步原站关注</strong>
<span>读取当前 Pixiv / X 登录账号的关注列表,保存到本地。</span>
</div>
<button id="originFollowSyncButton" class="primary-button" type="button">同步原站关注</button>
</div>
<div class="section-head">
<div>
<h2>关注同步</h2>
<p>使用已保存 Cookie 同步关注作者的新作品</p>
</div>
<button id="syncButton" class="primary-button">同步关注作者</button>
</div>
<div class="import-strip session-strip">
<div>
<strong>Pixiv 登录态</strong>
<span id="pixivSessionStatus">正在读取桌面端已保存的 Pixiv 会话...</span>
</div>
<button id="refreshSessionButton" class="ghost-button" type="button">刷新会话状态</button>
</div>
<form id="followForm" class="follow-form">
<input id="followArtistId" inputmode="text" placeholder="作者 ID、作者名或主页链接" />
<input id="followArtistName" placeholder="备注名称,可选" />
<button class="ghost-button" type="submit">添加关注</button>
</form>
<div id="followedList" class="artist-list"></div>
<div class="section-head">
<div>
<h2>X 关注</h2>
<p>按时间线检查 X 作者的最新媒体推文,并保存到本地媒体库。</p>
</div>
<button id="syncXButton" class="primary-button" type="button">同步 X 关注</button>
</div>
<form id="xFollowForm" class="follow-form">
<input id="xFollowUsername" placeholder="X 用户名,例如 @nasa" />
<input id="xFollowName" placeholder="备注名称,可选" />
<button class="ghost-button" type="submit">添加 X 关注</button>
</form>
<div id="xFollowedList" class="artist-list"></div>
</section>
<section id="xMediaView" class="view">
<div class="section-head">
<div>
<h2>X 作者媒体</h2>
<p>输入作者名称,解析媒体页中的图片或视频,勾选后下载到本地。</p>
</div>
</div>
<div class="import-strip">
<div>
<strong>X Cookie</strong>
<span>导入 Get cookies.txt 导出的 txt/json,会写入 runtime/x_cookies.json。</span>
</div>
<label class="file-button">
选择 txt
<input id="xCookieFile" type="file" accept=".txt,.json" />
</label>
<button id="importXCookieButton" class="ghost-button" type="button">导入 X Cookie</button>
</div>
<form id="xMediaForm" class="x-media-form">
<input id="xUsernameInput" placeholder="作者名称,例如 nasa 或 @nasa" autocomplete="off" />
<select id="xMediaType">
<option value="all">图片和视频</option>
<option value="image">只看图片</option>
<option value="video">只看视频</option>
</select>
<input id="xMaxItems" type="number" min="1" max="500" value="120" />
<button class="primary-button" type="submit">解析媒体</button>
</form>
<div class="x-actions">
<label><input id="xSelectAll" type="checkbox" checked /> 全选</label>
<button id="followCurrentXButton" class="ghost-button" type="button">关注当前作者</button>
<button id="downloadXSelectedButton" class="primary-button" type="button">下载选中</button>
<span id="xMediaSummary">尚未解析</span>
</div>
<div id="xMediaGrid" class="x-media-grid"></div>
<div class="section-head x-local-head">
<div>
<h2>本地 X 媒体</h2>
<p>查看已下载的 X 图片和视频,可删除本地文件。</p>
</div>
<div class="toolbar-actions">
<button id="refreshXLocalButton" class="ghost-button" type="button">刷新</button>
<button id="deleteXLocalButton" class="ghost-button danger-button" type="button">删除选中</button>
</div>
</div>
<div class="x-actions">
<label><input id="xLocalSelectAll" type="checkbox" /> 全选本地文件</label>
<span id="xLocalSummary">等待加载</span>
</div>
<div id="xLocalGrid" class="x-media-grid"></div>
</section>
<section id="xiaohongshuView" class="view">
<div class="section-head">
<div>
<h2>小红书</h2>
<p>输入关键词或笔记链接,先预览再勾选下载。</p>
</div>
</div>
<form id="xhsForm" class="x-media-form">
<input id="xhsTargetInput" placeholder="关键词,或 xiaohongshu.com/explore/... 链接" autocomplete="off" />
<input id="xhsLimitInput" type="number" min="1" max="50" value="20" />
<button class="primary-button" type="submit">解析预览</button>
</form>
<div class="x-actions">
<label><input id="xhsSelectAll" type="checkbox" checked /> 全选</label>
<button id="downloadXhsSelectedButton" class="primary-button" type="button">下载选中</button>
<span id="xhsSummary">等待解析</span>
</div>
<div id="xhsGrid" class="x-media-grid"></div>
</section>
</main>
<dialog id="originFollowSyncDialog" class="detail-dialog">
<div class="dialog-head">
<div>
<strong>同步原站关注</strong>
<span>选择平台后,会读取当前登录账号的关注列表并保存到本地。</span>
</div>
<button id="closeOriginFollowSyncDialogButton" class="ghost-button" type="button">关闭</button>
</div>
<div class="dialog-actions">
<button id="syncOriginPixivButton" class="primary-button" type="button">Pixiv</button>
<button id="syncOriginXButton" class="primary-button" type="button">X</button>
</div>
<pre id="originFollowSyncResult" class="sync-result"></pre>
</dialog>
<dialog id="detailDialog" class="detail-dialog">
<button id="closeDialogButton" class="close-button" aria-label="关闭">×</button>
<div id="detailContent"></div>
</dialog>
<script src="/assets/app.js"></script>
</body>
</html>
+756
View File
@@ -0,0 +1,756 @@
:root {
color-scheme: light;
--bg: #f6f7f9;
--surface: #ffffff;
--surface-soft: #faf5f7;
--line: #e5dce1;
--text: #242229;
--muted: #706b73;
--primary: #cc5f91;
--primary-strong: #af477a;
--accent: #277d78;
--ok: #3f9b72;
--shadow: 0 10px 28px rgba(48, 39, 44, 0.09);
}
* {
box-sizing: border-box;
}
body {
margin: 0;
min-height: 100vh;
display: grid;
grid-template-columns: 248px minmax(0, 1fr);
background: var(--bg);
color: var(--text);
font-family: "Segoe UI", "Microsoft YaHei", Arial, sans-serif;
letter-spacing: 0;
}
button,
input,
select {
font: inherit;
}
button {
cursor: pointer;
}
.sidebar {
min-height: 100vh;
border-right: 1px solid var(--line);
background: #ffffff;
padding: 22px 18px;
position: sticky;
top: 0;
overflow-y: auto;
}
.brand {
display: flex;
align-items: center;
gap: 12px;
margin-bottom: 24px;
}
.brand-mark {
width: 46px;
height: 46px;
display: grid;
place-items: center;
background: var(--primary);
color: white;
border-radius: 8px;
font-weight: 700;
}
.brand h1,
.brand p,
.panel h2,
.section-head h2,
.section-head p {
margin: 0;
}
.brand h1 {
font-size: 19px;
font-weight: 750;
}
.brand p,
.section-head p {
color: var(--muted);
font-size: 13px;
margin-top: 3px;
}
.nav {
display: grid;
gap: 8px;
margin-bottom: 22px;
}
.nav-button,
.ghost-button,
.primary-button {
border: 1px solid var(--line);
border-radius: 10px;
min-height: 40px;
padding: 0 14px;
background: var(--surface);
color: var(--text);
}
.nav-button {
text-align: left;
font-weight: 650;
}
.nav-button.active,
.primary-button {
background: var(--primary);
border-color: var(--primary);
color: white;
box-shadow: 0 8px 18px rgba(204, 95, 145, 0.2);
}
.primary-button:hover {
background: var(--primary-strong);
}
.ghost-button:hover,
.nav-button:hover {
border-color: var(--primary);
color: var(--primary);
}
.danger-button:hover {
border-color: var(--accent);
color: var(--accent);
}
.panel {
border-top: 1px solid var(--line);
padding-top: 18px;
}
.plugin-panel {
margin-bottom: 18px;
}
.panel h2 {
font-size: 14px;
margin-bottom: 12px;
}
.compact-nav {
margin-bottom: 0;
}
.tag-list {
display: flex;
flex-wrap: wrap;
gap: 8px;
}
.tag-chip {
border: 1px solid var(--line);
background: #fff7fb;
border-radius: 999px;
padding: 6px 10px;
color: var(--muted);
max-width: 100%;
}
.tag-chip.active {
color: white;
background: var(--accent);
border-color: var(--accent);
}
.main {
min-width: 0;
padding: 24px 30px 40px;
}
.welcome-card {
border: 1px solid var(--line);
border-radius: 8px;
background: #ffffff;
box-shadow: var(--shadow);
padding: 28px 30px;
display: flex;
align-items: center;
justify-content: space-between;
gap: 20px;
margin-bottom: 18px;
}
.welcome-card h2 {
margin: 0 0 8px;
font-size: 28px;
line-height: 1.22;
}
.welcome-card p {
margin: 0;
color: var(--muted);
max-width: 660px;
line-height: 1.7;
}
.eyebrow {
color: var(--primary) !important;
font-size: 13px;
font-weight: 750;
margin-bottom: 6px !important;
}
.quick-grid {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
gap: 14px;
margin-bottom: 18px;
}
.quick-card {
border: 1px solid var(--line);
border-radius: 8px;
background: var(--surface);
min-height: 132px;
padding: 18px;
text-align: left;
box-shadow: 0 8px 22px rgba(130, 66, 98, 0.08);
}
.quick-card:hover {
border-color: var(--primary);
transform: translateY(-1px);
}
.quick-card strong,
.quick-card span {
display: block;
}
.quick-card strong {
font-size: 17px;
margin-bottom: 8px;
}
.quick-card span {
color: var(--muted);
font-size: 13px;
line-height: 1.55;
}
.help-steps {
display: grid;
gap: 12px;
}
.help-steps article {
border: 1px solid var(--line);
border-radius: 8px;
background: rgba(255, 255, 255, 0.86);
padding: 16px;
display: flex;
gap: 14px;
align-items: flex-start;
}
.help-steps b {
width: 30px;
height: 30px;
border-radius: 999px;
background: #ffe3ef;
color: var(--primary);
display: grid;
place-items: center;
}
.help-steps strong,
.help-steps span {
display: block;
}
.help-steps span {
color: var(--muted);
margin-top: 4px;
}
.toolbar {
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
margin-bottom: 14px;
}
.search-wrap {
display: grid;
grid-template-columns: minmax(220px, 420px) 150px;
gap: 10px;
width: min(100%, 610px);
}
.search-wrap input,
.search-wrap select,
.follow-form input,
.x-media-form input,
.x-media-form select {
height: 42px;
border: 1px solid var(--line);
border-radius: 8px;
background: var(--surface);
color: var(--text);
padding: 0 12px;
min-width: 0;
}
.toolbar-actions {
display: flex;
gap: 10px;
}
.status-line {
min-height: 32px;
color: var(--muted);
font-size: 13px;
background: rgba(255, 255, 255, 0.72);
border: 1px solid rgba(240, 213, 226, 0.8);
border-radius: 8px;
padding: 8px 12px;
margin-bottom: 14px;
}
.view {
display: none;
}
.view.active {
display: block;
}
.hidden {
display: none !important;
}
.section-head {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
margin-bottom: 16px;
}
.section-head h2 {
font-size: 24px;
}
.artwork-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(168px, 1fr));
gap: 16px;
}
.artwork-card {
border: 1px solid var(--line);
border-radius: 12px;
background: var(--surface);
overflow: hidden;
box-shadow: 0 8px 22px rgba(130, 66, 98, 0.08);
}
.thumb-button {
border: 0;
padding: 0;
width: 100%;
aspect-ratio: 1 / 1;
background: #dde5ec;
display: block;
}
.thumb-button img {
width: 100%;
height: 100%;
object-fit: cover;
display: block;
}
.thumb-placeholder {
height: 100%;
display: grid;
place-items: center;
color: var(--muted);
padding: 16px;
text-align: center;
}
.artwork-meta {
padding: 10px;
display: grid;
gap: 7px;
}
.artwork-title {
font-weight: 650;
line-height: 1.35;
min-height: 38px;
}
.artwork-sub {
color: var(--muted);
font-size: 12px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.card-actions {
display: flex;
gap: 8px;
}
.small-button {
border: 1px solid var(--line);
border-radius: 7px;
background: var(--surface-soft);
min-height: 30px;
padding: 0 9px;
color: var(--text);
}
.small-button.active {
background: var(--accent);
border-color: var(--accent);
color: white;
}
.pager {
display: flex;
align-items: center;
justify-content: center;
gap: 12px;
margin: 22px 0 0;
}
.artist-list {
display: grid;
gap: 10px;
}
.artist-row {
border: 1px solid var(--line);
border-radius: 12px;
background: var(--surface);
min-height: 66px;
padding: 12px 14px;
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
}
.artist-row strong {
display: block;
}
.artist-row span {
color: var(--muted);
font-size: 13px;
}
.follow-form {
display: grid;
grid-template-columns: 160px minmax(180px, 320px) 110px;
gap: 10px;
margin-bottom: 16px;
}
.import-strip {
border: 1px solid var(--line);
border-radius: 12px;
background: var(--surface);
padding: 12px;
display: grid;
grid-template-columns: minmax(220px, 1fr) auto auto;
gap: 10px;
align-items: center;
margin-bottom: 16px;
}
.import-strip strong,
.import-strip span {
display: block;
}
.import-strip span,
.x-actions {
color: var(--muted);
font-size: 13px;
}
.file-button {
border: 1px solid var(--line);
border-radius: 8px;
min-height: 38px;
padding: 0 14px;
display: inline-grid;
place-items: center;
background: var(--surface-soft);
color: var(--text);
cursor: pointer;
}
.file-button input {
display: none;
}
.x-media-form {
display: grid;
grid-template-columns: minmax(220px, 1fr) 150px 120px;
gap: 10px;
margin-bottom: 14px;
}
#xMediaForm {
grid-template-columns: minmax(220px, 1fr) 150px 110px 120px;
}
.x-actions {
min-height: 38px;
display: flex;
align-items: center;
gap: 14px;
margin-bottom: 14px;
}
.x-actions label {
display: flex;
align-items: center;
gap: 6px;
}
.x-media-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(180px, 1fr));
gap: 14px;
}
.x-media-card {
border: 1px solid var(--line);
border-radius: 8px;
background: var(--surface);
overflow: hidden;
position: relative;
}
.x-local-head {
border-top: 1px solid var(--line);
padding-top: 22px;
margin-top: 28px;
}
.x-check {
position: absolute;
z-index: 2;
left: 8px;
top: 8px;
min-height: 30px;
display: flex;
align-items: center;
gap: 6px;
border-radius: 8px;
background: rgba(255, 255, 255, 0.92);
padding: 4px 8px;
font-size: 12px;
}
.x-thumb {
width: 100%;
aspect-ratio: 1 / 1;
background: #101820;
display: grid;
place-items: center;
}
.x-thumb img {
width: 100%;
height: 100%;
object-fit: cover;
display: block;
}
.x-thumb video {
width: 100%;
height: 100%;
object-fit: contain;
background: #101820;
display: block;
}
.video-tile {
width: 100%;
height: 100%;
display: grid;
place-items: center;
color: white;
background: #223140;
font-weight: 700;
}
.detail-dialog {
width: min(980px, calc(100vw - 36px));
max-height: calc(100vh - 36px);
border: 0;
border-radius: 14px;
padding: 0;
box-shadow: var(--shadow);
}
.detail-dialog::backdrop {
background: rgba(9, 18, 28, 0.5);
}
.dialog-head {
display: flex;
gap: 16px;
align-items: flex-start;
justify-content: space-between;
padding: 20px 20px 0;
}
.dialog-head strong,
.dialog-head span {
display: block;
}
.dialog-head span {
color: var(--muted);
margin-top: 4px;
}
.dialog-actions {
display: flex;
flex-wrap: wrap;
gap: 10px;
padding: 18px 20px 0;
}
.sync-result {
background: #fff7fa;
border: 1px solid var(--line);
border-radius: 8px;
color: var(--text);
margin: 18px 20px 20px;
max-height: 260px;
overflow: auto;
padding: 12px;
white-space: pre-wrap;
}
.close-button {
position: absolute;
right: 12px;
top: 12px;
width: 34px;
height: 34px;
border: 1px solid var(--line);
border-radius: 50%;
background: var(--surface);
z-index: 2;
}
.detail-layout {
display: grid;
grid-template-columns: minmax(260px, 52%) minmax(260px, 1fr);
min-height: 520px;
}
.detail-media {
background: #101820;
display: grid;
place-items: center;
padding: 20px;
}
.detail-media img {
max-width: 100%;
max-height: 72vh;
object-fit: contain;
}
.detail-info {
padding: 28px;
overflow: auto;
}
.detail-info h2 {
margin: 0 0 8px;
font-size: 26px;
}
.detail-info p {
color: var(--muted);
}
.detail-tags {
display: flex;
flex-wrap: wrap;
gap: 8px;
margin: 16px 0;
}
.empty {
border: 1px dashed var(--line);
border-radius: 14px;
padding: 34px;
background: var(--surface);
color: var(--muted);
text-align: center;
}
@media (max-width: 860px) {
body {
grid-template-columns: 1fr;
}
.sidebar {
position: static;
min-height: auto;
}
.quick-grid {
grid-template-columns: 1fr;
}
.welcome-card {
align-items: stretch;
flex-direction: column;
}
.toolbar,
.section-head,
.artist-row {
align-items: stretch;
flex-direction: column;
}
.search-wrap,
.follow-form,
.import-strip,
.x-media-form {
grid-template-columns: 1fr;
}
.toolbar-actions {
flex-wrap: wrap;
}
.detail-layout {
grid-template-columns: 1fr;
}
}
+1
View File
@@ -0,0 +1 @@
"""Bundled AO3 (Archive of Our Own) plugin package."""
+11
View File
@@ -0,0 +1,11 @@
{
"name": "AO3",
"version": "1.0.0",
"domain": "archiveofourown.org",
"author": "SakuraHub",
"description": "AO3 fanfiction archive novel plugin. Supports work URL parsing, metadata extraction, EPUB/HTML download, cookie-based auth, and update checking.",
"requires_auth": false,
"min_platform_version": "1.0.0",
"tags": ["novel", "fanfiction", "ao3"],
"dependencies": []
}
File diff suppressed because it is too large. Load diff
+11
View File
@@ -0,0 +1,11 @@
{
"name": "笔趣阁",
"version": "3.0.0",
"domain": "www.bqg551.cc",
"author": "Sakura Downloader",
"description": "基于 bqg551 阅读站公开 API 的小说搜索、目录解析、章节抓取与 TXT 导出。",
"requires_auth": false,
"tags": ["novel", "bqg551", "biquge", "bookshelf"],
"min_platform_version": "1.0.0",
"dependencies": []
}
File diff suppressed because it is too large. Load diff
Loaded 100 of 412 files, more files were not shown because too many files have changed in this diff. Show more