commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,42 @@
|
||||
# Sakura Cloud example config. Copy to .env for local backend development.
|
||||
|
||||
APP_NAME=Sakura Cloud
|
||||
APP_VERSION=0.1.0
|
||||
API_PREFIX=/api/v1
|
||||
DEBUG=true
|
||||
|
||||
DATABASE_URL=postgresql+psycopg://sakura:sakura@localhost:5432/sakura_cloud
|
||||
REDIS_URL=redis://localhost:6379/0
|
||||
|
||||
STORAGE_ROOT=data/sakura
|
||||
LOG_DIR=data/sakura/logs
|
||||
|
||||
JWT_SECRET_KEY=change-this-secret-in-production
|
||||
JWT_ALGORITHM=HS256
|
||||
JWT_ACCESS_TOKEN_EXPIRE_HOURS=24
|
||||
JWT_REFRESH_TOKEN_EXPIRE_DAYS=30
|
||||
|
||||
DEFAULT_STORAGE_LIMIT_GB=5
|
||||
MAX_UPLOAD_SIZE_MB=100
|
||||
MAX_VIDEO_UPLOAD_SIZE_MB=2048
|
||||
VIDEO_STORAGE_LIMIT_GB=30
|
||||
MAX_BATCH_SIZE_MB=500
|
||||
UPLOAD_CHUNK_SIZE=5242880
|
||||
TEMP_UPLOAD_TTL_HOURS=24
|
||||
THUMBNAIL_MAX_SIZE=640
|
||||
THUMBNAIL_QUALITY=82
|
||||
|
||||
FCM_ENABLED=true
|
||||
FIREBASE_CREDENTIALS_PATH=
|
||||
FIREBASE_CREDENTIALS_JSON=
|
||||
|
||||
OCR_ENABLED=false
|
||||
OCR_LANGUAGE=ch
|
||||
OCR_MAX_IMAGE_DIMENSION=1800
|
||||
OCR_MAX_ATTEMPTS=3
|
||||
OCR_MAX_CONCURRENCY=1
|
||||
|
||||
ADMIN_LOGIN_RATE_LIMIT_PER_MINUTE=5
|
||||
ADMIN_FRONTEND_ENABLED=true
|
||||
|
||||
CORS_ORIGINS=["http://localhost:5173","http://localhost:3000"]
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
# Python cache and env
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.venv/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# Local toolchains (SDKs, caches) — keep only scripts in tools/
|
||||
tools/android-sdk/
|
||||
tools/android-commandlinetools.zip
|
||||
tools/flutter/
|
||||
tools/pub-cache/
|
||||
tools/python314/
|
||||
|
||||
# Build artifacts
|
||||
build/
|
||||
dist/
|
||||
release/
|
||||
releases/*
|
||||
!releases/.gitkeep
|
||||
*.spec
|
||||
!build_assets/PixivDownloader.spec
|
||||
!build_assets/SakuraDownloader.spec
|
||||
|
||||
# Runtime and sensitive local data
|
||||
runtime/
|
||||
profiles/
|
||||
Pixiv_Downloads/
|
||||
Sakura_Downloads/
|
||||
data/
|
||||
pixiv_app_session.json
|
||||
pixiv_app_library.db*
|
||||
.env
|
||||
.env.*
|
||||
!.env.cloud.example
|
||||
firebase-service-account*.json
|
||||
sakura_mobile/android/app/google-services.json
|
||||
|
||||
# Flutter generated files
|
||||
sakura_mobile/.dart_tool/
|
||||
sakura_mobile/build/
|
||||
sakura_mobile/.flutter-plugins
|
||||
sakura_mobile/.flutter-plugins-dependencies
|
||||
|
||||
# Local project docs and dev logs (not code)
|
||||
项目文件/
|
||||
日志/
|
||||
|
||||
# AI assistant / debug local files
|
||||
.claude/
|
||||
.chatgpt/
|
||||
*.log
|
||||
debug/
|
||||
|
||||
# Web frontend generated files
|
||||
sakura_admin_web/node_modules/
|
||||
sakura_admin_web/dist/
|
||||
sakura_admin_web/.npm-cache/
|
||||
@@ -0,0 +1,104 @@
|
||||
# Sakura Downloader(樱云下载器)
|
||||
|
||||
个人媒体采集、本地图库管理与跨设备云同步系统。
|
||||
|
||||
| 端 | 角色 |
|
||||
|----|------|
|
||||
| Windows 桌面端 | 采集与整理中枢 |
|
||||
| Sakura Cloud | 数据与账号中心 |
|
||||
| Android 客户端 | 跨设备阅读与提醒 |
|
||||
|
||||
## 成品文件(`releases/`)
|
||||
|
||||
| 文件 | 说明 | 大小(约) |
|
||||
|------|------|------------|
|
||||
| `releases/sakuradownloader.exe` | Windows 桌面端,双击即用 | ~68 MB |
|
||||
| `releases/app-release.apk` | Android 客户端(调试签名,适合自测) | ~54 MB |
|
||||
|
||||
构建产物来源:
|
||||
|
||||
- EXE:`dist/sakuradownloader.exe` → 复制到 `releases/`
|
||||
- APK:`sakura_mobile/build/app/outputs/flutter-apk/app-release.apk` → 复制到 `releases/`
|
||||
|
||||
## 文档索引
|
||||
|
||||
| 文档 | 面向读者 | 内容 |
|
||||
|------|----------|------|
|
||||
| [使用说明-桌面版与安卓版.md](使用说明-桌面版与安卓版.md) | 终端用户 | 登录、下载、云同步、手机端操作 |
|
||||
| [docs/项目功能交付书.md](docs/项目功能交付书.md) | 产品 / 验收 | 功能模块、架构、API、代码索引 |
|
||||
| [docs/项目文档.md](docs/项目文档.md) | 开发者 | 环境、目录、构建、部署、开发流程 |
|
||||
| [项目文件/BUILD.md](项目文件/BUILD.md) | 开发者 | EXE / APK 打包细节与排错 |
|
||||
|
||||
## 快速开始(源码)
|
||||
|
||||
### 桌面端
|
||||
|
||||
```powershell
|
||||
python -m venv .venv
|
||||
.\.venv\Scripts\activate
|
||||
pip install -r requirements.txt
|
||||
python main.py
|
||||
```
|
||||
|
||||
### 云端(可选)
|
||||
|
||||
```powershell
|
||||
pip install -r sakura_cloud/requirements.txt
|
||||
uvicorn sakura_cloud.main:app --reload
|
||||
```
|
||||
|
||||
### 移动端(开发)
|
||||
|
||||
```powershell
|
||||
cd sakura_mobile
|
||||
flutter pub get
|
||||
flutter run
|
||||
```
|
||||
|
||||
## 一键打包
|
||||
|
||||
```powershell
|
||||
# 桌面 EXE
|
||||
scripts\build_desktop_exe.bat
|
||||
|
||||
# Android APK(需 JDK 17+,仓库内 tools\flutter 与 tools\android-sdk)
|
||||
scripts\build_mobile_android.bat
|
||||
|
||||
# 打包后复制到 releases/
|
||||
scripts\copy_release_artifacts.bat
|
||||
|
||||
# 或依次执行 EXE + APK + 复制
|
||||
scripts\build_all.bat
|
||||
```
|
||||
|
||||
## 技术栈摘要
|
||||
|
||||
- 桌面:Python 3.8+、CustomTkinter、SQLite、Playwright(X 插件)
|
||||
- 云端:FastAPI、PostgreSQL、JWT
|
||||
- 移动:Flutter、Riverpod、Dio
|
||||
|
||||
## 目录结构(精简)
|
||||
|
||||
```text
|
||||
Pixiv/
|
||||
├── main.py # 桌面入口
|
||||
├── pixiv_app/ # 桌面核心(GUI、下载、图库、云同步)
|
||||
├── plugins/ # Pixiv / X 等插件
|
||||
├── sakura_cloud/ # 云端 API、docker-compose、Alembic
|
||||
├── sakura_mobile/ # Flutter 安卓端
|
||||
├── releases/ # 可分发成品(EXE / APK)
|
||||
├── docs/ # 交付书、开发文档
|
||||
├── scripts/ # 构建与依赖安装脚本
|
||||
├── tools/ # 辅助脚本(如代码统计)
|
||||
└── 项目文件/ # 迭代与专题说明
|
||||
```
|
||||
|
||||
## 许可与说明
|
||||
|
||||
- APK 当前为 **调试签名**,仅供个人测试,不宜直接上架分发。
|
||||
- 使用 X 插件前需安装 Playwright Chromium(见 `项目文件/BUILD.md`)。
|
||||
- 会话与图库数据默认保存在 `%LOCALAPPDATA%\PixivDownloader`(打包版)。
|
||||
|
||||
---
|
||||
|
||||
*最后构建:2026-06-05*
|
||||
@@ -0,0 +1,115 @@
|
||||
# -*- mode: python ; coding: utf-8 -*-
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
block_cipher = None
|
||||
spec_path = Path(SPECPATH).resolve()
|
||||
root = spec_path.parent.parent if spec_path.suffix == ".spec" else spec_path.parent
|
||||
|
||||
|
||||
def tree_datas(source: str, dest: str, suffixes: tuple[str, ...] | None = None):
|
||||
base = root / source
|
||||
items = []
|
||||
if not base.exists():
|
||||
return items
|
||||
for path in base.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
parts = set(path.parts)
|
||||
if "__pycache__" in parts:
|
||||
continue
|
||||
if path.suffix.lower() in {".pyc", ".pyo"}:
|
||||
continue
|
||||
if suffixes and path.suffix.lower() not in suffixes:
|
||||
continue
|
||||
rel_parent = path.parent.relative_to(base)
|
||||
items.append((str(path), str(Path(dest) / rel_parent)))
|
||||
return items
|
||||
|
||||
|
||||
datas = []
|
||||
datas += tree_datas("pixiv_app/webui", "pixiv_app/webui", (".html", ".css", ".js", ".png", ".jpg", ".jpeg", ".webp", ".svg", ".ico"))
|
||||
datas += [
|
||||
(src, dst)
|
||||
for src, dst in tree_datas("plugins", "plugins", (".py", ".json", ".md", ".txt"))
|
||||
if not any(
|
||||
marker in src
|
||||
for marker in (
|
||||
"plugins\\ao3",
|
||||
"plugins/ao3",
|
||||
"plugins\\changpei",
|
||||
"plugins/changpei",
|
||||
"plugins\\taobao",
|
||||
"plugins/taobao",
|
||||
)
|
||||
)
|
||||
]
|
||||
datas += [(str(root / "build_assets" / "app_icon.ico"), "build_assets")]
|
||||
|
||||
|
||||
hiddenimports = [
|
||||
"pixiv_app.core.fingerprint",
|
||||
"pixiv_app.core.browser_proxy_pool",
|
||||
"PIL",
|
||||
"PIL.Image",
|
||||
"cryptography",
|
||||
"cryptography.hazmat.primitives",
|
||||
"cryptography.hazmat.primitives.padding",
|
||||
"cryptography.hazmat.primitives.ciphers",
|
||||
"customtkinter",
|
||||
"requests",
|
||||
"httpx",
|
||||
"playwright",
|
||||
"playwright.async_api",
|
||||
"playwright.sync_api",
|
||||
"pixiv_app.core.playwright_runtime",
|
||||
"pixiv_app.services.x_login_flow",
|
||||
]
|
||||
|
||||
|
||||
a = Analysis(
|
||||
[str(root / "main.py")],
|
||||
pathex=[str(root)],
|
||||
binaries=[],
|
||||
datas=datas,
|
||||
hiddenimports=hiddenimports,
|
||||
hookspath=[],
|
||||
hooksconfig={},
|
||||
runtime_hooks=[],
|
||||
excludes=[
|
||||
"pytest",
|
||||
"unittest",
|
||||
"tkinter.test",
|
||||
],
|
||||
win_no_prefer_redirects=False,
|
||||
win_private_assemblies=False,
|
||||
cipher=block_cipher,
|
||||
noarchive=False,
|
||||
)
|
||||
pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
[],
|
||||
name="sakuradownloader",
|
||||
debug=False,
|
||||
bootloader_ignore_signals=False,
|
||||
strip=False,
|
||||
upx=True,
|
||||
upx_exclude=[],
|
||||
runtime_tmpdir=None,
|
||||
console=False,
|
||||
disable_windowed_traceback=False,
|
||||
argv_emulation=False,
|
||||
target_arch=None,
|
||||
codesign_identity=None,
|
||||
entitlements_file=None,
|
||||
icon=str(root / "build_assets" / "app_icon.ico"),
|
||||
)
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 176 KiB |
@@ -0,0 +1,116 @@
|
||||
# Distributed Crawler
|
||||
|
||||
An enterprise-oriented distributed crawler scaffold built around `aiohttp` and `asyncio`.
|
||||
|
||||
## Included capabilities
|
||||
|
||||
- Master + Worker topology
|
||||
- Redis-backed priority queues
|
||||
- Consistent-hash worker assignment
|
||||
- Bloom-filter task dedupe
|
||||
- Dead-letter queue for exhausted failures
|
||||
- Checkpoint + state recovery
|
||||
- Task/request/worker trace headers
|
||||
- Exponential backoff retries
|
||||
- Domain-aware rate limiting
|
||||
- Domain-level circuit breaker
|
||||
- Proxy pool with health verification
|
||||
- Proxy scoring with weighted selection
|
||||
- Shared session store with Redis + local backup
|
||||
- Result aggregation with JSONL, MongoDB, or PostgreSQL sinks and URL-hash dedupe
|
||||
- APScheduler-driven periodic task submission
|
||||
|
||||
## Safety notes
|
||||
|
||||
This project intentionally keeps high-risk account automation, captcha bypass, and anti-protection evasion as extension points only.
|
||||
Use it only against systems you are authorized to access and collect from.
|
||||
|
||||
## Directory layout
|
||||
|
||||
```text
|
||||
distributed_crawler/
|
||||
config.example.yaml
|
||||
requirements.txt
|
||||
crawler/
|
||||
auth.py
|
||||
backoff.py
|
||||
bloom.py
|
||||
config.py
|
||||
hash_ring.py
|
||||
http.py
|
||||
master.py
|
||||
models.py
|
||||
proxy.py
|
||||
queue.py
|
||||
rate_limit.py
|
||||
results.py
|
||||
scheduler.py
|
||||
sessions.py
|
||||
signatures.py
|
||||
spider.py
|
||||
state.py
|
||||
worker.py
|
||||
scripts/
|
||||
run_master.py
|
||||
run_worker.py
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
1. Copy `config.example.yaml` to `config.yaml`.
|
||||
2. Start Redis.
|
||||
3. Install dependencies from `requirements.txt`.
|
||||
4. Start the master:
|
||||
|
||||
```bash
|
||||
python scripts/run_master.py --config config.yaml
|
||||
```
|
||||
|
||||
5. Start one or more workers:
|
||||
|
||||
```bash
|
||||
python scripts/run_worker.py --config config.yaml --worker-id worker-1
|
||||
python scripts/run_worker.py --config config.yaml --worker-id worker-2
|
||||
```
|
||||
|
||||
6. Submit a task:
|
||||
|
||||
```bash
|
||||
curl -X POST http://127.0.0.1:8088/tasks ^
|
||||
-H "Content-Type: application/json" ^
|
||||
-d "{\"url\":\"https://httpbin.org/get\",\"spider\":\"generic_fetch\"}"
|
||||
```
|
||||
|
||||
## Master endpoints
|
||||
|
||||
- `POST /tasks`
|
||||
- `POST /heartbeat`
|
||||
- `GET /workers`
|
||||
- `GET /stats`
|
||||
- `GET /health`
|
||||
- `GET /dashboard` (feature and runtime panel)
|
||||
|
||||
## Failure handling and tracing
|
||||
|
||||
- Bloom sizing uses `master.bloom_capacity` and `master.bloom_false_positive_rate` to calculate bit count and hash count automatically. Set `master.bloom_hashes` only when you need to override the computed optimum.
|
||||
- Final failed tasks are appended to `master.dead_letter_queue_name` with the original task, final result, retry count, worker ID, error, and HTTP status.
|
||||
- Worker requests include `X-Task-Id`, `X-Request-Id`, and `X-Worker-Id` headers. Result payloads also include `request_id` for easier log/result correlation.
|
||||
|
||||
## Stability controls
|
||||
|
||||
- Proxy endpoints track success count, failure count, latency, cooldown, and a computed score. Workers choose healthy or cooled-down proxies with weighted random selection.
|
||||
- `circuit_breaker` protects each domain independently. After repeated transport errors or 5xx responses, the domain opens briefly, then moves to half-open for recovery probing.
|
||||
- Result storage writes a stable `url_hash`. MongoDB and PostgreSQL enforce uniqueness with upsert behavior; JSONL keeps a sidecar `.url_hashes` index.
|
||||
|
||||
## Auto login and shared sessions
|
||||
|
||||
- Enable `auth.enabled: true` and configure per-domain credentials in `config.yaml`.
|
||||
- Worker marks sessions expired based on configured status codes/body keywords.
|
||||
- When expired, worker triggers `AuthProvider.refresh_session()` and writes refreshed cookie to Redis + local backup via `DualSessionStore`.
|
||||
- All workers read the same session keys (`session.redis_prefix`), so login state is shared.
|
||||
|
||||
## Extension points
|
||||
|
||||
- `crawler.auth.AuthProvider`
|
||||
- `crawler.signatures.SignatureProvider`
|
||||
- `crawler.spider.BaseSpider`
|
||||
@@ -0,0 +1,93 @@
|
||||
app_name: distributed-crawler
|
||||
redis_dsn: redis://127.0.0.1:6379/0
|
||||
checkpoint_dir: ./runtime/checkpoints
|
||||
|
||||
master:
|
||||
host: 127.0.0.1
|
||||
port: 8088
|
||||
worker_ttl_seconds: 30
|
||||
stale_task_seconds: 120
|
||||
bloom_capacity: 1000000
|
||||
bloom_false_positive_rate: 0.01
|
||||
bloom_hashes:
|
||||
queue_name: crawler
|
||||
result_queue_name: crawler:results
|
||||
dead_letter_queue_name: crawler:dead_letter
|
||||
default_spider: generic_fetch
|
||||
|
||||
worker:
|
||||
worker_id: worker-1
|
||||
concurrency: 100
|
||||
heartbeat_interval_seconds: 5
|
||||
request_timeout_seconds: 20
|
||||
connect_timeout_seconds: 10
|
||||
retries: 4
|
||||
base_backoff_seconds: 1
|
||||
max_backoff_seconds: 16
|
||||
download_dir: ./runtime/downloads
|
||||
|
||||
storage:
|
||||
backend: jsonl
|
||||
jsonl_path: ./runtime/results/results.jsonl
|
||||
mongodb_dsn: mongodb://127.0.0.1:27017
|
||||
mongodb_database: crawler
|
||||
mongodb_collection: results
|
||||
postgres_dsn: postgresql://postgres:postgres@127.0.0.1:5432/crawler
|
||||
postgres_table: crawl_results
|
||||
|
||||
proxy:
|
||||
enabled: false
|
||||
verify_url: https://httpbin.org/ip
|
||||
verify_timeout_seconds: 8
|
||||
refresh_interval_seconds: 300
|
||||
failure_threshold: 3
|
||||
cooldown_seconds: 60
|
||||
static_proxies: []
|
||||
provider_api_url: ""
|
||||
provider_auth_header: ""
|
||||
|
||||
circuit_breaker:
|
||||
enabled: true
|
||||
failure_threshold: 8
|
||||
recovery_timeout_seconds: 30
|
||||
half_open_success_threshold: 2
|
||||
|
||||
session:
|
||||
enabled: true
|
||||
redis_prefix: crawler:sessions
|
||||
local_backup_path: ./runtime/sessions/sessions.json
|
||||
|
||||
auth:
|
||||
enabled: false
|
||||
domains:
|
||||
example.com:
|
||||
login_url: https://example.com/api/login
|
||||
username: your_username
|
||||
password: your_password
|
||||
username_field: username
|
||||
password_field: password
|
||||
submit_type: json # json or form
|
||||
extra_payload: {}
|
||||
success_status_codes: [200]
|
||||
expire_status_codes: [401, 403]
|
||||
expire_body_keywords: ["login required", "please login", "unauthorized"]
|
||||
|
||||
rate_limit:
|
||||
default_qps: 3.0
|
||||
default_burst: 6
|
||||
per_domain:
|
||||
httpbin.org:
|
||||
qps: 2.0
|
||||
burst: 4
|
||||
|
||||
schedules:
|
||||
- name: sample-httpbin
|
||||
trigger: interval
|
||||
seconds: 300
|
||||
priority: medium
|
||||
task:
|
||||
spider: generic_fetch
|
||||
url: https://httpbin.org/get
|
||||
method: GET
|
||||
metadata:
|
||||
source: schedule
|
||||
@@ -0,0 +1,5 @@
|
||||
from .config import AppSettings, load_settings
|
||||
from .master import MasterNode
|
||||
from .worker import WorkerNode
|
||||
|
||||
__all__ = ["AppSettings", "MasterNode", "WorkerNode", "load_settings"]
|
||||
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
|
||||
from .config import AuthSettings
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AuthProvider(ABC):
|
||||
@abstractmethod
|
||||
async def refresh_session(self, domain: str, existing_session: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Provide an authorized and compliant session refresh implementation for the target system.
|
||||
"""
|
||||
|
||||
|
||||
class ManualAuthProvider(AuthProvider):
|
||||
async def refresh_session(self, domain: str, existing_session: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError(
|
||||
f"Session refresh for {domain} must be implemented with authorized credentials and target-specific logic."
|
||||
)
|
||||
|
||||
|
||||
class ConfigurableAuthProvider(AuthProvider):
|
||||
"""
|
||||
Generic username/password login provider.
|
||||
This intentionally keeps site-specific anti-bot behavior out of scope.
|
||||
"""
|
||||
|
||||
def __init__(self, settings: AuthSettings, timeout_seconds: int = 20) -> None:
|
||||
self.settings = settings
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
async def refresh_session(self, domain: str, existing_session: dict[str, Any]) -> dict[str, Any]:
|
||||
if not self.settings.enabled:
|
||||
return {**existing_session, "expired": False}
|
||||
cfg = self.settings.domains.get(domain)
|
||||
if cfg is None or not cfg.login_url or not cfg.username or not cfg.password:
|
||||
LOGGER.warning("No auth config for domain=%s, keeping existing session", domain)
|
||||
return {**existing_session, "expired": True}
|
||||
|
||||
payload: dict[str, Any] = dict(cfg.extra_payload)
|
||||
payload[cfg.username_field] = cfg.username
|
||||
payload[cfg.password_field] = cfg.password
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=self.timeout_seconds)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
if cfg.submit_type.lower() == "json":
|
||||
async with session.post(cfg.login_url, json=payload) as response:
|
||||
body_text = await response.text()
|
||||
if response.status not in set(cfg.success_status_codes):
|
||||
raise RuntimeError(f"Login failed for {domain}: {response.status} {body_text[:180]}")
|
||||
else:
|
||||
async with session.post(cfg.login_url, data=payload) as response:
|
||||
body_text = await response.text()
|
||||
if response.status not in set(cfg.success_status_codes):
|
||||
raise RuntimeError(f"Login failed for {domain}: {response.status} {body_text[:180]}")
|
||||
|
||||
cookie_pairs = [f"{k}={v.value}" for k, v in session.cookie_jar.filter_cookies(cfg.login_url).items()]
|
||||
cookie_header = "; ".join(cookie_pairs)
|
||||
if not cookie_header:
|
||||
# Fallback: keep old cookies if target does not set cookies in login response.
|
||||
cookie_header = str(existing_session.get("cookie", ""))
|
||||
|
||||
return {
|
||||
**existing_session,
|
||||
"expired": False,
|
||||
"cookie": cookie_header,
|
||||
"updated_by": "ConfigurableAuthProvider",
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
|
||||
|
||||
def exponential_backoff(attempt: int, base_seconds: float, max_seconds: float) -> float:
|
||||
delay = min(max_seconds, base_seconds * (2 ** max(attempt, 0)))
|
||||
jitter = random.uniform(0, delay * 0.2 if delay > 0 else 0)
|
||||
return delay + jitter
|
||||
@@ -0,0 +1,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from hashlib import md5, sha1
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
|
||||
class RedisBloomFilter:
|
||||
def __init__(self, redis: Redis, key: str, bits: int = 8_000_000, hashes: int = 6) -> None:
|
||||
self.redis = redis
|
||||
self.key = key
|
||||
self.bits = max(int(bits), 1)
|
||||
self.hashes = max(int(hashes), 1)
|
||||
|
||||
@classmethod
|
||||
def from_capacity(
|
||||
cls,
|
||||
redis: Redis,
|
||||
key: str,
|
||||
*,
|
||||
capacity: int,
|
||||
false_positive_rate: float = 0.01,
|
||||
hashes: int | None = None,
|
||||
) -> "RedisBloomFilter":
|
||||
capacity = max(int(capacity), 1)
|
||||
false_positive_rate = min(max(float(false_positive_rate), 1e-9), 0.999999)
|
||||
bits = math.ceil(-(capacity * math.log(false_positive_rate)) / (math.log(2) ** 2))
|
||||
optimal_hashes = max(1, round((bits / capacity) * math.log(2)))
|
||||
return cls(redis, key, bits=bits, hashes=hashes or optimal_hashes)
|
||||
|
||||
def _positions(self, value: str) -> list[int]:
|
||||
digest_a = int(md5(value.encode("utf-8")).hexdigest(), 16)
|
||||
digest_b = int(sha1(value.encode("utf-8")).hexdigest(), 16)
|
||||
return [((digest_a + index * digest_b) % self.bits) for index in range(self.hashes)]
|
||||
|
||||
async def contains_or_add(self, value: str) -> bool:
|
||||
positions = self._positions(value)
|
||||
pipeline = self.redis.pipeline()
|
||||
for position in positions:
|
||||
pipeline.getbit(self.key, position)
|
||||
existing = await pipeline.execute()
|
||||
seen = all(bit == 1 for bit in existing)
|
||||
|
||||
pipeline = self.redis.pipeline()
|
||||
for position in positions:
|
||||
pipeline.setbit(self.key, position, 1)
|
||||
await pipeline.execute()
|
||||
return seen
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from time import monotonic
|
||||
|
||||
from .config import CircuitBreakerSettings
|
||||
|
||||
|
||||
class CircuitState(str, Enum):
|
||||
CLOSED = "closed"
|
||||
OPEN = "open"
|
||||
HALF_OPEN = "half_open"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Circuit:
|
||||
state: CircuitState = CircuitState.CLOSED
|
||||
failures: int = 0
|
||||
half_open_successes: int = 0
|
||||
opened_at: float = 0.0
|
||||
|
||||
|
||||
class CircuitOpenError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class DomainCircuitBreaker:
|
||||
def __init__(self, settings: CircuitBreakerSettings) -> None:
|
||||
self.settings = settings
|
||||
self._circuits: dict[str, Circuit] = {}
|
||||
|
||||
def before_request(self, domain: str) -> None:
|
||||
if not self.settings.enabled:
|
||||
return
|
||||
circuit = self._circuits.setdefault(domain, Circuit())
|
||||
if circuit.state != CircuitState.OPEN:
|
||||
return
|
||||
elapsed = monotonic() - circuit.opened_at
|
||||
if elapsed >= self.settings.recovery_timeout_seconds:
|
||||
circuit.state = CircuitState.HALF_OPEN
|
||||
circuit.half_open_successes = 0
|
||||
return
|
||||
raise CircuitOpenError(f"Circuit open for {domain}")
|
||||
|
||||
def record_success(self, domain: str) -> None:
|
||||
if not self.settings.enabled:
|
||||
return
|
||||
circuit = self._circuits.setdefault(domain, Circuit())
|
||||
if circuit.state == CircuitState.HALF_OPEN:
|
||||
circuit.half_open_successes += 1
|
||||
if circuit.half_open_successes < self.settings.half_open_success_threshold:
|
||||
return
|
||||
circuit.state = CircuitState.CLOSED
|
||||
circuit.failures = 0
|
||||
circuit.half_open_successes = 0
|
||||
circuit.opened_at = 0.0
|
||||
|
||||
def record_failure(self, domain: str) -> None:
|
||||
if not self.settings.enabled:
|
||||
return
|
||||
circuit = self._circuits.setdefault(domain, Circuit())
|
||||
circuit.failures += 1
|
||||
circuit.half_open_successes = 0
|
||||
if circuit.failures >= self.settings.failure_threshold or circuit.state == CircuitState.HALF_OPEN:
|
||||
circuit.state = CircuitState.OPEN
|
||||
circuit.opened_at = monotonic()
|
||||
@@ -0,0 +1,206 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
@dataclass
|
||||
class MasterSettings:
|
||||
host: str = "127.0.0.1"
|
||||
port: int = 8088
|
||||
worker_ttl_seconds: int = 30
|
||||
stale_task_seconds: int = 120
|
||||
bloom_capacity: int = 1_000_000
|
||||
bloom_false_positive_rate: float = 0.01
|
||||
bloom_hashes: int | None = None
|
||||
queue_name: str = "crawler"
|
||||
result_queue_name: str = "crawler:results"
|
||||
dead_letter_queue_name: str = "crawler:dead_letter"
|
||||
default_spider: str = "generic_fetch"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkerSettings:
|
||||
worker_id: str = "worker-1"
|
||||
concurrency: int = 100
|
||||
heartbeat_interval_seconds: int = 5
|
||||
request_timeout_seconds: int = 20
|
||||
connect_timeout_seconds: int = 10
|
||||
retries: int = 4
|
||||
base_backoff_seconds: float = 1.0
|
||||
max_backoff_seconds: float = 16.0
|
||||
download_dir: str = "./runtime/downloads"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StorageSettings:
|
||||
backend: str = "jsonl"
|
||||
jsonl_path: str = "./runtime/results/results.jsonl"
|
||||
mongodb_dsn: str = "mongodb://127.0.0.1:27017"
|
||||
mongodb_database: str = "crawler"
|
||||
mongodb_collection: str = "results"
|
||||
postgres_dsn: str = "postgresql://postgres:postgres@127.0.0.1:5432/crawler"
|
||||
postgres_table: str = "crawl_results"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProxySettings:
|
||||
enabled: bool = False
|
||||
verify_url: str = "https://httpbin.org/ip"
|
||||
verify_timeout_seconds: int = 8
|
||||
refresh_interval_seconds: int = 300
|
||||
failure_threshold: int = 3
|
||||
cooldown_seconds: int = 60
|
||||
static_proxies: list[str] = field(default_factory=list)
|
||||
provider_api_url: str = ""
|
||||
provider_auth_header: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class CircuitBreakerSettings:
|
||||
enabled: bool = True
|
||||
failure_threshold: int = 8
|
||||
recovery_timeout_seconds: int = 30
|
||||
half_open_success_threshold: int = 2
|
||||
|
||||
|
||||
@dataclass
|
||||
class SessionSettings:
|
||||
enabled: bool = True
|
||||
redis_prefix: str = "crawler:sessions"
|
||||
local_backup_path: str = "./runtime/sessions/sessions.json"
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuthDomainSettings:
|
||||
login_url: str = ""
|
||||
username: str = ""
|
||||
password: str = ""
|
||||
username_field: str = "username"
|
||||
password_field: str = "password"
|
||||
submit_type: str = "form"
|
||||
extra_payload: dict[str, Any] = field(default_factory=dict)
|
||||
success_status_codes: list[int] = field(default_factory=lambda: [200])
|
||||
expire_status_codes: list[int] = field(default_factory=lambda: [401, 403])
|
||||
expire_body_keywords: list[str] = field(default_factory=lambda: ["login required", "please login", "unauthorized"])
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuthSettings:
|
||||
enabled: bool = False
|
||||
domains: dict[str, AuthDomainSettings] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DomainRateLimitSettings:
|
||||
qps: float = 3.0
|
||||
burst: int = 6
|
||||
|
||||
|
||||
@dataclass
|
||||
class RateLimitSettings:
|
||||
default_qps: float = 3.0
|
||||
default_burst: int = 6
|
||||
per_domain: dict[str, DomainRateLimitSettings] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScheduleTaskSettings:
|
||||
name: str
|
||||
trigger: str
|
||||
task: dict[str, Any]
|
||||
priority: str = "medium"
|
||||
seconds: int | None = None
|
||||
minutes: int | None = None
|
||||
cron: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AppSettings:
|
||||
app_name: str = "distributed-crawler"
|
||||
redis_dsn: str = "redis://127.0.0.1:6379/0"
|
||||
checkpoint_dir: str = "./runtime/checkpoints"
|
||||
master: MasterSettings = field(default_factory=MasterSettings)
|
||||
worker: WorkerSettings = field(default_factory=WorkerSettings)
|
||||
storage: StorageSettings = field(default_factory=StorageSettings)
|
||||
proxy: ProxySettings = field(default_factory=ProxySettings)
|
||||
circuit_breaker: CircuitBreakerSettings = field(default_factory=CircuitBreakerSettings)
|
||||
session: SessionSettings = field(default_factory=SessionSettings)
|
||||
auth: AuthSettings = field(default_factory=AuthSettings)
|
||||
rate_limit: RateLimitSettings = field(default_factory=RateLimitSettings)
|
||||
schedules: list[ScheduleTaskSettings] = field(default_factory=list)
|
||||
source_path: Path | None = None
|
||||
|
||||
@property
|
||||
def base_dir(self) -> Path:
|
||||
if self.source_path is None:
|
||||
return Path.cwd()
|
||||
return self.source_path.parent
|
||||
|
||||
def resolve_path(self, value: str) -> Path:
|
||||
candidate = Path(value)
|
||||
if candidate.is_absolute():
|
||||
return candidate
|
||||
return (self.base_dir / candidate).resolve()
|
||||
|
||||
|
||||
def _load_domain_limits(raw_limits: dict[str, Any]) -> dict[str, DomainRateLimitSettings]:
|
||||
result: dict[str, DomainRateLimitSettings] = {}
|
||||
for domain, item in raw_limits.items():
|
||||
result[domain] = DomainRateLimitSettings(
|
||||
qps=float(item.get("qps", 3.0)),
|
||||
burst=int(item.get("burst", 6)),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _load_auth_domains(raw_domains: dict[str, Any]) -> dict[str, AuthDomainSettings]:
|
||||
result: dict[str, AuthDomainSettings] = {}
|
||||
for domain, item in raw_domains.items():
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
result[domain] = AuthDomainSettings(
|
||||
login_url=str(item.get("login_url", "")),
|
||||
username=str(item.get("username", "")),
|
||||
password=str(item.get("password", "")),
|
||||
username_field=str(item.get("username_field", "username")),
|
||||
password_field=str(item.get("password_field", "password")),
|
||||
submit_type=str(item.get("submit_type", "form")),
|
||||
extra_payload=dict(item.get("extra_payload", {})),
|
||||
success_status_codes=[int(v) for v in item.get("success_status_codes", [200])],
|
||||
expire_status_codes=[int(v) for v in item.get("expire_status_codes", [401, 403])],
|
||||
expire_body_keywords=[str(v).lower() for v in item.get("expire_body_keywords", ["login required", "please login", "unauthorized"])],
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def load_settings(path: str | Path) -> AppSettings:
|
||||
config_path = Path(path).resolve()
|
||||
data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
|
||||
|
||||
settings = AppSettings(
|
||||
app_name=str(data.get("app_name", "distributed-crawler")),
|
||||
redis_dsn=str(data.get("redis_dsn", "redis://127.0.0.1:6379/0")),
|
||||
checkpoint_dir=str(data.get("checkpoint_dir", "./runtime/checkpoints")),
|
||||
master=MasterSettings(**(data.get("master") or {})),
|
||||
worker=WorkerSettings(**(data.get("worker") or {})),
|
||||
storage=StorageSettings(**(data.get("storage") or {})),
|
||||
proxy=ProxySettings(**(data.get("proxy") or {})),
|
||||
circuit_breaker=CircuitBreakerSettings(**(data.get("circuit_breaker") or {})),
|
||||
session=SessionSettings(**(data.get("session") or {})),
|
||||
auth=AuthSettings(
|
||||
enabled=bool((data.get("auth") or {}).get("enabled", False)),
|
||||
domains=_load_auth_domains((data.get("auth") or {}).get("domains", {})),
|
||||
),
|
||||
rate_limit=RateLimitSettings(
|
||||
default_qps=float((data.get("rate_limit") or {}).get("default_qps", 3.0)),
|
||||
default_burst=int((data.get("rate_limit") or {}).get("default_burst", 6)),
|
||||
per_domain=_load_domain_limits((data.get("rate_limit") or {}).get("per_domain", {})),
|
||||
),
|
||||
schedules=[ScheduleTaskSettings(**item) for item in data.get("schedules", [])],
|
||||
source_path=config_path,
|
||||
)
|
||||
return settings
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from bisect import bisect
|
||||
from hashlib import md5
|
||||
|
||||
|
||||
class ConsistentHashRing:
|
||||
def __init__(self, replicas: int = 100) -> None:
|
||||
self.replicas = replicas
|
||||
self._keys: list[int] = []
|
||||
self._nodes: dict[int, str] = {}
|
||||
|
||||
def _hash(self, value: str) -> int:
|
||||
return int(md5(value.encode("utf-8")).hexdigest(), 16)
|
||||
|
||||
def rebuild(self, nodes: list[str]) -> None:
|
||||
self._keys.clear()
|
||||
self._nodes.clear()
|
||||
for node in sorted(set(nodes)):
|
||||
for replica in range(self.replicas):
|
||||
key = self._hash(f"{node}:{replica}")
|
||||
self._keys.append(key)
|
||||
self._nodes[key] = node
|
||||
self._keys.sort()
|
||||
|
||||
def get_node(self, key: str) -> str | None:
|
||||
if not self._keys:
|
||||
return None
|
||||
hashed = self._hash(key)
|
||||
index = bisect(self._keys, hashed)
|
||||
if index == len(self._keys):
|
||||
index = 0
|
||||
return self._nodes[self._keys[index]]
|
||||
@@ -0,0 +1,180 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
from pathlib import Path
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
|
||||
from .auth import AuthProvider
|
||||
from .circuit_breaker import DomainCircuitBreaker
|
||||
from .config import AppSettings
|
||||
from .models import CrawlTask
|
||||
from .proxy import ProxyPool
|
||||
from .rate_limit import DomainRateLimiter
|
||||
from .sessions import DualSessionStore
|
||||
from .signatures import SignatureProvider
|
||||
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/126.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4) AppleWebKit/605.1.15 Version/17.4 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Edg/126.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
|
||||
class AsyncHttpClient:
|
||||
def __init__(
|
||||
self,
|
||||
settings: AppSettings,
|
||||
rate_limiter: DomainRateLimiter,
|
||||
proxy_pool: ProxyPool,
|
||||
session_store: DualSessionStore,
|
||||
auth_provider: AuthProvider,
|
||||
signature_provider: SignatureProvider,
|
||||
) -> None:
|
||||
self.settings = settings
|
||||
self.rate_limiter = rate_limiter
|
||||
self.proxy_pool = proxy_pool
|
||||
self.session_store = session_store
|
||||
self.auth_provider = auth_provider
|
||||
self.signature_provider = signature_provider
|
||||
self.session: aiohttp.ClientSession | None = None
|
||||
self.circuit_breaker = DomainCircuitBreaker(settings.circuit_breaker)
|
||||
|
||||
async def start(self) -> None:
|
||||
timeout = aiohttp.ClientTimeout(
|
||||
total=self.settings.worker.request_timeout_seconds,
|
||||
connect=self.settings.worker.connect_timeout_seconds,
|
||||
)
|
||||
connector = aiohttp.TCPConnector(limit=self.settings.worker.concurrency, ssl=False)
|
||||
self.session = aiohttp.ClientSession(timeout=timeout, connector=connector)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.session is not None:
|
||||
await self.session.close()
|
||||
|
||||
async def fetch(self, task: CrawlTask) -> dict[str, Any]:
|
||||
if self.session is None:
|
||||
raise RuntimeError("HTTP client is not started")
|
||||
|
||||
await self.rate_limiter.acquire(task.domain)
|
||||
self.circuit_breaker.before_request(task.domain)
|
||||
proxy = await self.proxy_pool.acquire()
|
||||
session_payload = await self.session_store.load(task.domain)
|
||||
if self.settings.session.enabled and session_payload.get("expired"):
|
||||
session_payload = await self.auth_provider.refresh_session(task.domain, session_payload)
|
||||
await self.session_store.save(task.domain, session_payload)
|
||||
|
||||
headers = dict(task.headers)
|
||||
headers.setdefault("User-Agent", random.choice(USER_AGENTS))
|
||||
headers.setdefault("Accept", "*/*")
|
||||
if task.metadata.get("referer"):
|
||||
headers.setdefault("Referer", str(task.metadata["referer"]))
|
||||
if task.metadata.get("origin"):
|
||||
headers.setdefault("Origin", str(task.metadata["origin"]))
|
||||
headers.setdefault("X-Task-Id", task.task_id)
|
||||
if task.metadata.get("request_id"):
|
||||
headers.setdefault("X-Request-Id", str(task.metadata["request_id"]))
|
||||
if task.metadata.get("worker_id"):
|
||||
headers.setdefault("X-Worker-Id", str(task.metadata["worker_id"]))
|
||||
cookie_header = session_payload.get("cookie")
|
||||
if cookie_header:
|
||||
headers.setdefault("Cookie", str(cookie_header))
|
||||
|
||||
signature_headers = await self.signature_provider.sign(
|
||||
{
|
||||
"url": task.url,
|
||||
"method": task.method,
|
||||
"body": task.body,
|
||||
"metadata": task.metadata,
|
||||
}
|
||||
)
|
||||
headers.update(signature_headers)
|
||||
|
||||
started_at = monotonic()
|
||||
try:
|
||||
async with self.session.request(
|
||||
task.method,
|
||||
task.url,
|
||||
headers=headers,
|
||||
data=task.body.encode("utf-8") if isinstance(task.body, str) else None,
|
||||
proxy=proxy.url if proxy else None,
|
||||
) as response:
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
body_bytes = await response.read()
|
||||
if "application/json" in content_type:
|
||||
body: Any = json.loads(body_bytes.decode("utf-8", errors="ignore"))
|
||||
else:
|
||||
body = body_bytes.decode("utf-8", errors="ignore")
|
||||
response_data = {
|
||||
"status": response.status,
|
||||
"headers": dict(response.headers),
|
||||
"body": body,
|
||||
"body_bytes": body_bytes,
|
||||
"proxy": proxy.url if proxy else None,
|
||||
}
|
||||
if self.settings.session.enabled:
|
||||
if self._is_session_expired(task.domain, response.status, body):
|
||||
session_payload["expired"] = True
|
||||
await self.session_store.save(task.domain, session_payload)
|
||||
else:
|
||||
if "Set-Cookie" in response.headers:
|
||||
cookie_header = self._extract_cookie_header(response)
|
||||
if cookie_header:
|
||||
session_payload["cookie"] = cookie_header
|
||||
session_payload["expired"] = False
|
||||
await self.session_store.save(task.domain, session_payload)
|
||||
if response.status >= 500:
|
||||
self.circuit_breaker.record_failure(task.domain)
|
||||
else:
|
||||
self.circuit_breaker.record_success(task.domain)
|
||||
await self.proxy_pool.mark_success(proxy, (monotonic() - started_at) * 1000)
|
||||
return response_data
|
||||
except Exception:
|
||||
self.circuit_breaker.record_failure(task.domain)
|
||||
await self.proxy_pool.mark_failed(proxy)
|
||||
raise
|
||||
|
||||
async def save_binary(self, task: CrawlTask, file_name: str, body_bytes: bytes) -> str:
|
||||
save_dir = self.settings.resolve_path(self.settings.worker.download_dir)
|
||||
save_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = save_dir / file_name
|
||||
target.write_bytes(body_bytes)
|
||||
return str(target)
|
||||
|
||||
def _is_session_expired(self, domain: str, status_code: int, body: Any) -> bool:
|
||||
if not self.settings.auth.enabled:
|
||||
return False
|
||||
domain_cfg = self.settings.auth.domains.get(domain)
|
||||
if domain_cfg is None:
|
||||
return False
|
||||
if status_code in set(domain_cfg.expire_status_codes):
|
||||
return True
|
||||
if isinstance(body, str):
|
||||
lower_body = body.lower()
|
||||
return any(keyword in lower_body for keyword in domain_cfg.expire_body_keywords)
|
||||
if isinstance(body, dict):
|
||||
packed = json.dumps(body, ensure_ascii=False).lower()
|
||||
return any(keyword in packed for keyword in domain_cfg.expire_body_keywords)
|
||||
return False
|
||||
|
||||
def _extract_cookie_header(self, response: aiohttp.ClientResponse) -> str:
|
||||
values = response.headers.getall("Set-Cookie", [])
|
||||
cookie_parts: list[str] = []
|
||||
for value in values:
|
||||
head = value.split(";", 1)[0].strip()
|
||||
if "=" in head:
|
||||
cookie_parts.append(head)
|
||||
# keep order and remove duplicates
|
||||
seen: set[str] = set()
|
||||
deduped: list[str] = []
|
||||
for item in cookie_parts:
|
||||
key = item.split("=", 1)[0].strip()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped.append(item)
|
||||
return "; ".join(deduped)
|
||||
@@ -0,0 +1,346 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from time import time
|
||||
from typing import Any
|
||||
from html import escape
|
||||
|
||||
from aiohttp import web
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from .bloom import RedisBloomFilter
|
||||
from .config import AppSettings
|
||||
from .hash_ring import ConsistentHashRing
|
||||
from .models import CrawlResult, CrawlTask, DeadLetterEntry, TaskPriority, TaskStatus, WorkerHeartbeat
|
||||
from .queue import RedisPriorityQueue
|
||||
from .results import ResultRepository, build_result_repository
|
||||
from .scheduler import ScheduleService
|
||||
from .state import CheckpointManager
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MasterNode:
|
||||
def __init__(self, settings: AppSettings) -> None:
|
||||
self.settings = settings
|
||||
self.redis = Redis.from_url(settings.redis_dsn, decode_responses=True)
|
||||
self.queue = RedisPriorityQueue(
|
||||
self.redis,
|
||||
settings.master.queue_name,
|
||||
settings.master.result_queue_name,
|
||||
settings.master.dead_letter_queue_name,
|
||||
)
|
||||
self.checkpoints = CheckpointManager(self.redis, settings.resolve_path(settings.checkpoint_dir), settings.master.queue_name)
|
||||
self.bloom = RedisBloomFilter.from_capacity(
|
||||
self.redis,
|
||||
key=f"{settings.master.queue_name}:bloom",
|
||||
capacity=settings.master.bloom_capacity,
|
||||
false_positive_rate=settings.master.bloom_false_positive_rate,
|
||||
hashes=settings.master.bloom_hashes,
|
||||
)
|
||||
self.hash_ring = ConsistentHashRing()
|
||||
self.result_repo: ResultRepository = build_result_repository(
|
||||
settings.storage,
|
||||
settings.resolve_path(settings.storage.jsonl_path),
|
||||
)
|
||||
self.scheduler = ScheduleService(settings, self.submit_task)
|
||||
self.workers: dict[str, WorkerHeartbeat] = {}
|
||||
self.app = web.Application()
|
||||
self.app.add_routes(
|
||||
[
|
||||
web.post("/tasks", self.handle_submit_task),
|
||||
web.post("/heartbeat", self.handle_heartbeat),
|
||||
web.get("/workers", self.handle_workers),
|
||||
web.get("/stats", self.handle_stats),
|
||||
web.get("/health", self.handle_health),
|
||||
web.get("/dashboard", self.handle_dashboard),
|
||||
]
|
||||
)
|
||||
self._background_tasks: list[asyncio.Task] = []
|
||||
|
||||
async def start(self) -> None:
|
||||
await self.result_repo.setup()
|
||||
await self._recover_local_checkpoints()
|
||||
self.scheduler.start()
|
||||
self._background_tasks = [
|
||||
asyncio.create_task(self.consume_results_loop(), name="consume-results"),
|
||||
asyncio.create_task(self.reap_stale_workers_loop(), name="reap-workers"),
|
||||
]
|
||||
|
||||
async def close(self) -> None:
|
||||
for task in self._background_tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*self._background_tasks, return_exceptions=True)
|
||||
await self.scheduler.shutdown()
|
||||
await self.result_repo.close()
|
||||
await self.redis.aclose()
|
||||
|
||||
async def submit_task(self, task: CrawlTask) -> bool:
|
||||
duplicate = await self.bloom.contains_or_add(task.fingerprint)
|
||||
if duplicate:
|
||||
LOGGER.info("Skip duplicate task %s", task.url)
|
||||
return False
|
||||
worker_id = self._assign_worker(task)
|
||||
task.assigned_worker = worker_id
|
||||
task.status = TaskStatus.PENDING
|
||||
await self.checkpoints.save(task)
|
||||
await self.queue.enqueue_task(worker_id, task)
|
||||
LOGGER.info("Submitted task %s to %s", task.task_id, worker_id)
|
||||
return True
|
||||
|
||||
async def handle_submit_task(self, request: web.Request) -> web.Response:
|
||||
payload = await request.json()
|
||||
task = CrawlTask.create(
|
||||
url=payload["url"],
|
||||
spider=payload.get("spider", self.settings.master.default_spider),
|
||||
method=payload.get("method", "GET"),
|
||||
headers=payload.get("headers"),
|
||||
metadata=payload.get("metadata"),
|
||||
body=payload.get("body"),
|
||||
priority=payload.get("priority", TaskPriority.MEDIUM.value),
|
||||
max_retries=int(payload.get("max_retries", self.settings.worker.retries)),
|
||||
)
|
||||
accepted = await self.submit_task(task)
|
||||
return web.json_response({"accepted": accepted, "task_id": task.task_id, "worker_id": task.assigned_worker})
|
||||
|
||||
async def handle_heartbeat(self, request: web.Request) -> web.Response:
|
||||
payload = await request.json()
|
||||
heartbeat = WorkerHeartbeat(
|
||||
worker_id=str(payload["worker_id"]),
|
||||
concurrency=int(payload["concurrency"]),
|
||||
active_tasks=int(payload["active_tasks"]),
|
||||
host=payload.get("host"),
|
||||
tags=list(payload.get("tags", [])),
|
||||
)
|
||||
self.workers[heartbeat.worker_id] = heartbeat
|
||||
self.hash_ring.rebuild(list(self.workers.keys()))
|
||||
return web.json_response({"ok": True, "known_workers": len(self.workers)})
|
||||
|
||||
async def handle_workers(self, _: web.Request) -> web.Response:
|
||||
return web.json_response({"workers": [item.to_dict() for item in self.workers.values()]})
|
||||
|
||||
async def handle_stats(self, _: web.Request) -> web.Response:
|
||||
return web.json_response(
|
||||
{
|
||||
"workers": len(self.workers),
|
||||
"queues": {
|
||||
worker_id: {
|
||||
priority.value: await self.redis.llen(self.queue._task_key(worker_id, priority))
|
||||
for priority in TaskPriority
|
||||
}
|
||||
for worker_id in self.workers or {"default": None}
|
||||
},
|
||||
"dead_letters": await self.redis.llen(self.queue.dead_letter_queue_name),
|
||||
}
|
||||
)
|
||||
|
||||
async def handle_health(self, _: web.Request) -> web.Response:
|
||||
pong = await self.redis.ping()
|
||||
return web.json_response({"status": "ok", "redis": bool(pong), "workers": len(self.workers)})
|
||||
|
||||
async def handle_dashboard(self, _: web.Request) -> web.Response:
|
||||
queue_stats = await self._collect_queue_stats()
|
||||
session_count = await self._count_sessions()
|
||||
dead_letter_count = await self.redis.llen(self.queue.dead_letter_queue_name)
|
||||
html = self._render_dashboard_html(
|
||||
queue_stats=queue_stats,
|
||||
session_count=session_count,
|
||||
dead_letter_count=dead_letter_count,
|
||||
)
|
||||
return web.Response(text=html, content_type="text/html")
|
||||
|
||||
async def consume_results_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
result = await self.queue.consume_result(timeout=5)
|
||||
if result is None:
|
||||
continue
|
||||
await self.process_result(result)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
LOGGER.exception("Result consumer failed")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def process_result(self, result: CrawlResult) -> None:
|
||||
task = await self.checkpoints.load(result.task_id)
|
||||
if task is None:
|
||||
LOGGER.warning("Result for unknown task %s", result.task_id)
|
||||
return
|
||||
task.status = result.status
|
||||
await self.checkpoints.mark_status(task, result.status)
|
||||
await self.result_repo.store(result)
|
||||
if result.status == TaskStatus.FAILED:
|
||||
await self.queue.publish_dead_letter(DeadLetterEntry.from_task_result(task, result))
|
||||
LOGGER.warning(
|
||||
"Dead-lettered task task_id=%s worker_id=%s request_id=%s reason=%s",
|
||||
task.task_id,
|
||||
result.worker_id,
|
||||
result.data.get("request_id", task.metadata.get("request_id", task.task_id)),
|
||||
result.error,
|
||||
)
|
||||
|
||||
async def reap_stale_workers_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(self.settings.master.worker_ttl_seconds)
|
||||
now = time()
|
||||
stale_workers = [
|
||||
worker_id
|
||||
for worker_id, heartbeat in self.workers.items()
|
||||
if now - self._heartbeat_epoch(heartbeat.last_seen) > self.settings.master.worker_ttl_seconds
|
||||
]
|
||||
for worker_id in stale_workers:
|
||||
LOGGER.warning("Worker %s is stale, recovering tasks", worker_id)
|
||||
recovered = await self.checkpoints.recover_stale(worker_id, self.settings.master.stale_task_seconds)
|
||||
for task in recovered:
|
||||
task.retries += 1
|
||||
if task.retries > task.max_retries:
|
||||
task.status = TaskStatus.FAILED
|
||||
await self.checkpoints.mark_status(task, TaskStatus.FAILED)
|
||||
result = CrawlResult(
|
||||
task_id=task.task_id,
|
||||
worker_id=worker_id,
|
||||
status=TaskStatus.FAILED,
|
||||
url=task.url,
|
||||
spider=task.spider,
|
||||
error="Task exceeded retry budget during worker recovery",
|
||||
)
|
||||
await self.result_repo.store(result)
|
||||
await self.queue.publish_dead_letter(DeadLetterEntry.from_task_result(task, result))
|
||||
continue
|
||||
task.assigned_worker = self._assign_worker(task, exclude=worker_id)
|
||||
await self.checkpoints.save(task)
|
||||
await self.queue.enqueue_task(task.assigned_worker, task)
|
||||
self.workers.pop(worker_id, None)
|
||||
self.hash_ring.rebuild(list(self.workers.keys()))
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
LOGGER.exception("Stale worker reaper failed")
|
||||
|
||||
async def _recover_local_checkpoints(self) -> None:
|
||||
tasks = await self.checkpoints.recover_local()
|
||||
for task in tasks:
|
||||
task.assigned_worker = self._assign_worker(task)
|
||||
task.status = TaskStatus.RETRY
|
||||
await self.checkpoints.save(task)
|
||||
await self.queue.enqueue_task(task.assigned_worker, task)
|
||||
|
||||
async def _collect_queue_stats(self) -> dict[str, dict[str, int]]:
|
||||
worker_keys = list(self.workers.keys()) or ["default"]
|
||||
result: dict[str, dict[str, int]] = {}
|
||||
for worker_id in worker_keys:
|
||||
result[worker_id] = {
|
||||
priority.value: await self.redis.llen(self.queue._task_key(worker_id, priority))
|
||||
for priority in TaskPriority
|
||||
}
|
||||
return result
|
||||
|
||||
async def _count_sessions(self) -> int:
|
||||
pattern = f"{self.settings.session.redis_prefix}:*"
|
||||
count = 0
|
||||
async for _ in self.redis.scan_iter(match=pattern):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def _render_dashboard_html(self, queue_stats: dict[str, dict[str, int]], session_count: int, dead_letter_count: int) -> str:
|
||||
worker_rows = []
|
||||
for worker_id, hb in self.workers.items():
|
||||
load = hb.active_tasks / max(hb.concurrency, 1)
|
||||
worker_rows.append(
|
||||
f"<tr><td>{escape(worker_id)}</td><td>{hb.active_tasks}</td><td>{hb.concurrency}</td><td>{load:.2f}</td><td>{escape(hb.last_seen)}</td></tr>"
|
||||
)
|
||||
if not worker_rows:
|
||||
worker_rows.append("<tr><td colspan='5'>暂无在线 Worker</td></tr>")
|
||||
|
||||
queue_blocks = []
|
||||
for worker_id, levels in queue_stats.items():
|
||||
queue_blocks.append(
|
||||
"<div class='card'><h3>Queue - "
|
||||
+ escape(worker_id)
|
||||
+ "</h3><p>high: "
|
||||
+ str(levels.get("high", 0))
|
||||
+ " | medium: "
|
||||
+ str(levels.get("medium", 0))
|
||||
+ " | low: "
|
||||
+ str(levels.get("low", 0))
|
||||
+ "</p></div>"
|
||||
)
|
||||
queue_html = "\n".join(queue_blocks)
|
||||
|
||||
enabled_auth_domains = ", ".join(sorted(self.settings.auth.domains.keys())) if self.settings.auth.domains else "-"
|
||||
enabled_proxy = "enabled" if self.settings.proxy.enabled else "disabled"
|
||||
enabled_session = "enabled" if self.settings.session.enabled else "disabled"
|
||||
enabled_auth = "enabled" if self.settings.auth.enabled else "disabled"
|
||||
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Distributed Crawler Dashboard</title>
|
||||
<style>
|
||||
body {{ font-family: Segoe UI, Arial, sans-serif; background:#f6f8fb; margin:0; padding:24px; color:#1f2d3d; }}
|
||||
.grid {{ display:grid; grid-template-columns: repeat(auto-fill, minmax(280px,1fr)); gap:12px; margin-bottom:16px; }}
|
||||
.card {{ background:#fff; border:1px solid #dce6f4; border-radius:10px; padding:12px 14px; }}
|
||||
h1 {{ margin:0 0 14px 0; font-size:24px; }}
|
||||
h3 {{ margin:0 0 8px 0; font-size:16px; }}
|
||||
table {{ width:100%; border-collapse: collapse; background:#fff; border:1px solid #dce6f4; }}
|
||||
th, td {{ text-align:left; padding:10px; border-bottom:1px solid #eef2f8; font-size:13px; }}
|
||||
th {{ background:#f1f5fc; }}
|
||||
.muted {{ color:#607086; font-size:12px; }}
|
||||
code {{ background:#eef3fb; padding:2px 6px; border-radius:4px; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Distributed Crawler Dashboard</h1>
|
||||
<div class="grid">
|
||||
<div class="card"><h3>Workers</h3><p>{len(self.workers)}</p></div>
|
||||
<div class="card"><h3>Session Keys</h3><p>{session_count}</p></div>
|
||||
<div class="card"><h3>Dead Letters</h3><p>{dead_letter_count}</p></div>
|
||||
<div class="card"><h3>Proxy</h3><p>{enabled_proxy}</p></div>
|
||||
<div class="card"><h3>Auto Login</h3><p>{enabled_auth}</p></div>
|
||||
<div class="card"><h3>Session Sharing</h3><p>{enabled_session}</p></div>
|
||||
<div class="card"><h3>Auth Domains</h3><p>{escape(enabled_auth_domains)}</p></div>
|
||||
</div>
|
||||
<div class="grid">
|
||||
{queue_html}
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3>Worker Runtime</h3>
|
||||
<table>
|
||||
<thead><tr><th>worker_id</th><th>active_tasks</th><th>concurrency</th><th>load</th><th>last_seen</th></tr></thead>
|
||||
<tbody>
|
||||
{''.join(worker_rows)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p class="muted">API endpoints: <code>/tasks</code> <code>/workers</code> <code>/stats</code> <code>/health</code></p>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
def _assign_worker(self, task: CrawlTask, exclude: str | None = None) -> str:
|
||||
candidates = [worker_id for worker_id in self.workers if worker_id != exclude]
|
||||
if not candidates:
|
||||
return "default"
|
||||
self.hash_ring.rebuild(candidates)
|
||||
preferred = self.hash_ring.get_node(task.url) or candidates[0]
|
||||
preferred_heartbeat = self.workers.get(preferred)
|
||||
if preferred_heartbeat is None:
|
||||
return preferred
|
||||
|
||||
preferred_load = preferred_heartbeat.active_tasks / max(preferred_heartbeat.concurrency, 1)
|
||||
if preferred_load < 0.9:
|
||||
return preferred
|
||||
|
||||
least_loaded = min(
|
||||
candidates,
|
||||
key=lambda worker_id: self.workers[worker_id].active_tasks / max(self.workers[worker_id].concurrency, 1),
|
||||
)
|
||||
return least_loaded
|
||||
|
||||
def _heartbeat_epoch(self, timestamp: str) -> float:
|
||||
from datetime import datetime
|
||||
|
||||
return datetime.fromisoformat(timestamp).timestamp()
|
||||
@@ -0,0 +1,189 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from enum import Enum
|
||||
from hashlib import sha1
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def make_url_hash(url: str) -> str:
|
||||
return sha1(url.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class TaskPriority(str, Enum):
|
||||
HIGH = "high"
|
||||
MEDIUM = "medium"
|
||||
LOW = "low"
|
||||
|
||||
|
||||
class TaskStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
SUCCESS = "success"
|
||||
FAILED = "failed"
|
||||
RETRY = "retry"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlTask:
|
||||
task_id: str
|
||||
url: str
|
||||
spider: str
|
||||
method: str = "GET"
|
||||
headers: dict[str, str] = field(default_factory=dict)
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
body: str | None = None
|
||||
priority: TaskPriority = TaskPriority.MEDIUM
|
||||
status: TaskStatus = TaskStatus.PENDING
|
||||
retries: int = 0
|
||||
max_retries: int = 4
|
||||
assigned_worker: str | None = None
|
||||
created_at: str = field(default_factory=utc_now)
|
||||
updated_at: str = field(default_factory=utc_now)
|
||||
schedule_name: str | None = None
|
||||
|
||||
@property
|
||||
def domain(self) -> str:
|
||||
return urlparse(self.url).netloc
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
payload = f"{self.method}:{self.url}:{self.spider}:{self.body or ''}"
|
||||
return sha1(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
payload = asdict(self)
|
||||
payload["priority"] = self.priority.value
|
||||
payload["status"] = self.status.value
|
||||
return payload
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, payload: dict[str, Any]) -> "CrawlTask":
|
||||
return cls(
|
||||
task_id=str(payload["task_id"]),
|
||||
url=str(payload["url"]),
|
||||
spider=str(payload["spider"]),
|
||||
method=str(payload.get("method", "GET")),
|
||||
headers=dict(payload.get("headers", {})),
|
||||
metadata=dict(payload.get("metadata", {})),
|
||||
body=payload.get("body"),
|
||||
priority=TaskPriority(payload.get("priority", TaskPriority.MEDIUM.value)),
|
||||
status=TaskStatus(payload.get("status", TaskStatus.PENDING.value)),
|
||||
retries=int(payload.get("retries", 0)),
|
||||
max_retries=int(payload.get("max_retries", 4)),
|
||||
assigned_worker=payload.get("assigned_worker"),
|
||||
created_at=str(payload.get("created_at", utc_now())),
|
||||
updated_at=str(payload.get("updated_at", utc_now())),
|
||||
schedule_name=payload.get("schedule_name"),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
*,
|
||||
url: str,
|
||||
spider: str,
|
||||
method: str = "GET",
|
||||
headers: dict[str, str] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
body: str | None = None,
|
||||
priority: str = "medium",
|
||||
max_retries: int = 4,
|
||||
schedule_name: str | None = None,
|
||||
) -> "CrawlTask":
|
||||
return cls(
|
||||
task_id=uuid4().hex,
|
||||
url=url,
|
||||
spider=spider,
|
||||
method=method.upper(),
|
||||
headers=headers or {},
|
||||
metadata=metadata or {},
|
||||
body=body,
|
||||
priority=TaskPriority(priority),
|
||||
max_retries=max_retries,
|
||||
schedule_name=schedule_name,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlResult:
|
||||
task_id: str
|
||||
worker_id: str
|
||||
status: TaskStatus
|
||||
url: str
|
||||
spider: str
|
||||
http_status: int | None = None
|
||||
data: dict[str, Any] = field(default_factory=dict)
|
||||
error: str | None = None
|
||||
finished_at: str = field(default_factory=utc_now)
|
||||
|
||||
@property
|
||||
def url_hash(self) -> str:
|
||||
return make_url_hash(self.url)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
payload = asdict(self)
|
||||
payload["status"] = self.status.value
|
||||
payload["url_hash"] = self.url_hash
|
||||
return payload
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, payload: dict[str, Any]) -> "CrawlResult":
|
||||
return cls(
|
||||
task_id=str(payload["task_id"]),
|
||||
worker_id=str(payload["worker_id"]),
|
||||
status=TaskStatus(payload["status"]),
|
||||
url=str(payload["url"]),
|
||||
spider=str(payload["spider"]),
|
||||
http_status=payload.get("http_status"),
|
||||
data=dict(payload.get("data", {})),
|
||||
error=payload.get("error"),
|
||||
finished_at=str(payload.get("finished_at", utc_now())),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeadLetterEntry:
|
||||
task: dict[str, Any]
|
||||
result: dict[str, Any] | None = None
|
||||
reason: str = ""
|
||||
failed_at: str = field(default_factory=utc_now)
|
||||
last_error: str | None = None
|
||||
response_status: int | None = None
|
||||
retry_count: int = 0
|
||||
worker_id: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_task_result(cls, task: CrawlTask, result: CrawlResult, reason: str | None = None) -> "DeadLetterEntry":
|
||||
return cls(
|
||||
task=task.to_dict(),
|
||||
result=result.to_dict(),
|
||||
reason=reason or result.error or "Task failed",
|
||||
last_error=result.error,
|
||||
response_status=result.http_status,
|
||||
retry_count=task.retries,
|
||||
worker_id=result.worker_id,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkerHeartbeat:
|
||||
worker_id: str
|
||||
concurrency: int
|
||||
active_tasks: int
|
||||
last_seen: str = field(default_factory=utc_now)
|
||||
host: str | None = None
|
||||
tags: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from time import monotonic
|
||||
|
||||
import aiohttp
|
||||
|
||||
from .config import ProxySettings
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProxyEndpoint:
|
||||
url: str
|
||||
failures: int = 0
|
||||
successes: int = 0
|
||||
latency_ms: float | None = None
|
||||
last_verified_at: float = 0.0
|
||||
last_failed_at: float = 0.0
|
||||
healthy: bool = True
|
||||
metadata: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def success_rate(self) -> float:
|
||||
total = self.successes + self.failures
|
||||
if total == 0:
|
||||
return 1.0
|
||||
return self.successes / total
|
||||
|
||||
@property
|
||||
def score(self) -> float:
|
||||
latency_penalty = 1.0
|
||||
if self.latency_ms is not None:
|
||||
latency_penalty = max(0.1, min(1.0, 1000.0 / max(self.latency_ms, 1.0)))
|
||||
failure_penalty = 1.0 / (1.0 + self.failures)
|
||||
return max(0.01, self.success_rate * latency_penalty * failure_penalty)
|
||||
|
||||
|
||||
class ProxyProvider(ABC):
|
||||
@abstractmethod
|
||||
async def fetch(self) -> list[ProxyEndpoint]: ...
|
||||
|
||||
|
||||
class StaticProxyProvider(ProxyProvider):
|
||||
def __init__(self, proxies: list[str]) -> None:
|
||||
self.proxies = proxies
|
||||
|
||||
async def fetch(self) -> list[ProxyEndpoint]:
|
||||
return [ProxyEndpoint(url=value) for value in self.proxies]
|
||||
|
||||
|
||||
class ApiProxyProvider(ProxyProvider):
|
||||
def __init__(self, api_url: str, auth_header: str = "") -> None:
|
||||
self.api_url = api_url
|
||||
self.auth_header = auth_header
|
||||
|
||||
async def fetch(self) -> list[ProxyEndpoint]:
|
||||
if not self.api_url:
|
||||
return []
|
||||
headers = {"Authorization": self.auth_header} if self.auth_header else {}
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(self.api_url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
text = await response.text()
|
||||
proxies: list[ProxyEndpoint] = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
proxies.append(ProxyEndpoint(url=line))
|
||||
return proxies
|
||||
|
||||
|
||||
class ProxyPool:
|
||||
def __init__(self, settings: ProxySettings) -> None:
|
||||
self.settings = settings
|
||||
self._proxies: list[ProxyEndpoint] = []
|
||||
self._lock = asyncio.Lock()
|
||||
self._next_index = 0
|
||||
|
||||
async def refresh(self) -> None:
|
||||
providers: list[ProxyProvider] = [StaticProxyProvider(self.settings.static_proxies)]
|
||||
if self.settings.provider_api_url:
|
||||
providers.append(ApiProxyProvider(self.settings.provider_api_url, self.settings.provider_auth_header))
|
||||
async with self._lock:
|
||||
existing = {endpoint.url: endpoint for endpoint in self._proxies}
|
||||
merged: dict[str, ProxyEndpoint] = {}
|
||||
for provider in providers:
|
||||
for endpoint in await provider.fetch():
|
||||
merged.setdefault(endpoint.url, existing.get(endpoint.url, endpoint))
|
||||
async with self._lock:
|
||||
self._proxies = list(merged.values())
|
||||
|
||||
async def verify(self) -> None:
|
||||
if not self.settings.enabled:
|
||||
return
|
||||
async with self._lock:
|
||||
proxies = list(self._proxies)
|
||||
if not proxies:
|
||||
return
|
||||
async with aiohttp.ClientSession() as session:
|
||||
tasks = [self._verify_proxy(session, proxy) for proxy in proxies]
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
async def acquire(self) -> ProxyEndpoint | None:
|
||||
if not self.settings.enabled:
|
||||
return None
|
||||
async with self._lock:
|
||||
now = monotonic()
|
||||
candidates = [
|
||||
item
|
||||
for item in self._proxies
|
||||
if item.healthy or now - item.last_failed_at >= self.settings.cooldown_seconds
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
proxy = random.choices(candidates, weights=[item.score for item in candidates], k=1)[0]
|
||||
self._next_index += 1
|
||||
return proxy
|
||||
|
||||
async def mark_success(self, proxy: ProxyEndpoint | None, latency_ms: float | None = None) -> None:
|
||||
if proxy is None:
|
||||
return
|
||||
async with self._lock:
|
||||
proxy.successes += 1
|
||||
proxy.failures = 0
|
||||
proxy.healthy = True
|
||||
if latency_ms is not None:
|
||||
proxy.latency_ms = latency_ms
|
||||
proxy.last_verified_at = monotonic()
|
||||
|
||||
async def mark_failed(self, proxy: ProxyEndpoint | None) -> None:
|
||||
if proxy is None:
|
||||
return
|
||||
async with self._lock:
|
||||
proxy.failures += 1
|
||||
proxy.last_failed_at = monotonic()
|
||||
if proxy.failures >= self.settings.failure_threshold:
|
||||
proxy.healthy = False
|
||||
|
||||
async def healthy_count(self) -> int:
|
||||
async with self._lock:
|
||||
return len([item for item in self._proxies if item.healthy])
|
||||
|
||||
async def _verify_proxy(self, session: aiohttp.ClientSession, proxy: ProxyEndpoint) -> None:
|
||||
try:
|
||||
timeout = aiohttp.ClientTimeout(total=self.settings.verify_timeout_seconds)
|
||||
started_at = monotonic()
|
||||
async with session.get(self.settings.verify_url, proxy=proxy.url, timeout=timeout) as response:
|
||||
response.raise_for_status()
|
||||
await response.read()
|
||||
proxy.healthy = True
|
||||
proxy.failures = 0
|
||||
proxy.successes += 1
|
||||
proxy.latency_ms = (monotonic() - started_at) * 1000
|
||||
proxy.last_verified_at = monotonic()
|
||||
except Exception:
|
||||
proxy.failures += 1
|
||||
proxy.last_failed_at = monotonic()
|
||||
if proxy.failures >= self.settings.failure_threshold:
|
||||
proxy.healthy = False
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from .models import CrawlResult, CrawlTask, DeadLetterEntry, TaskPriority
|
||||
|
||||
|
||||
class RedisPriorityQueue:
|
||||
def __init__(self, redis: Redis, namespace: str, result_queue_name: str, dead_letter_queue_name: str | None = None) -> None:
|
||||
self.redis = redis
|
||||
self.namespace = namespace
|
||||
self.result_queue_name = result_queue_name
|
||||
self.dead_letter_queue_name = dead_letter_queue_name or f"{namespace}:dead_letter"
|
||||
|
||||
def _task_key(self, worker_id: str, priority: TaskPriority) -> str:
|
||||
return f"{self.namespace}:tasks:{worker_id}:{priority.value}"
|
||||
|
||||
async def enqueue_task(self, worker_id: str, task: CrawlTask) -> None:
|
||||
await self.redis.rpush(self._task_key(worker_id, task.priority), json.dumps(task.to_dict()))
|
||||
|
||||
async def dequeue_task(self, worker_id: str, timeout: int = 5) -> CrawlTask | None:
|
||||
keys = [self._task_key(worker_id, priority) for priority in (TaskPriority.HIGH, TaskPriority.MEDIUM, TaskPriority.LOW)]
|
||||
result = await self.redis.blpop(keys, timeout=timeout)
|
||||
if not result:
|
||||
return None
|
||||
_, payload = result
|
||||
return CrawlTask.from_dict(json.loads(payload))
|
||||
|
||||
async def publish_result(self, result: CrawlResult) -> None:
|
||||
await self.redis.rpush(self.result_queue_name, json.dumps(result.to_dict()))
|
||||
|
||||
async def consume_result(self, timeout: int = 5) -> CrawlResult | None:
|
||||
result = await self.redis.blpop(self.result_queue_name, timeout=timeout)
|
||||
if not result:
|
||||
return None
|
||||
_, payload = result
|
||||
return CrawlResult.from_dict(json.loads(payload))
|
||||
|
||||
async def publish_dead_letter(self, entry: DeadLetterEntry) -> None:
|
||||
await self.redis.rpush(self.dead_letter_queue_name, json.dumps(entry.to_dict(), ensure_ascii=False))
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from time import monotonic
|
||||
|
||||
from .config import RateLimitSettings
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenBucket:
|
||||
qps: float
|
||||
burst: int
|
||||
tokens: float
|
||||
updated_at: float
|
||||
|
||||
|
||||
class DomainRateLimiter:
|
||||
def __init__(self, settings: RateLimitSettings) -> None:
|
||||
self.settings = settings
|
||||
self._buckets: dict[str, TokenBucket] = {}
|
||||
self._locks: dict[str, asyncio.Lock] = {}
|
||||
|
||||
async def acquire(self, domain: str) -> None:
|
||||
lock = self._locks.setdefault(domain, asyncio.Lock())
|
||||
async with lock:
|
||||
bucket = self._buckets.setdefault(domain, self._new_bucket(domain))
|
||||
while True:
|
||||
now = monotonic()
|
||||
elapsed = max(0.0, now - bucket.updated_at)
|
||||
bucket.tokens = min(bucket.burst, bucket.tokens + elapsed * bucket.qps)
|
||||
bucket.updated_at = now
|
||||
if bucket.tokens >= 1:
|
||||
bucket.tokens -= 1
|
||||
return
|
||||
await asyncio.sleep(max(0.05, 1 / bucket.qps))
|
||||
|
||||
def _new_bucket(self, domain: str) -> TokenBucket:
|
||||
domain_settings = self.settings.per_domain.get(domain)
|
||||
qps = domain_settings.qps if domain_settings else self.settings.default_qps
|
||||
burst = domain_settings.burst if domain_settings else self.settings.default_burst
|
||||
return TokenBucket(qps=qps, burst=burst, tokens=float(burst), updated_at=monotonic())
|
||||
@@ -0,0 +1,164 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
from motor.motor_asyncio import AsyncIOMotorClient
|
||||
|
||||
from .config import StorageSettings
|
||||
from .models import CrawlResult, make_url_hash
|
||||
|
||||
|
||||
class ResultRepository(ABC):
|
||||
@abstractmethod
|
||||
async def setup(self) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def store(self, result: CrawlResult) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def close(self) -> None: ...
|
||||
|
||||
|
||||
class JsonlResultRepository(ResultRepository):
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self.index_path = path.with_suffix(path.suffix + ".url_hashes")
|
||||
self._seen_url_hashes: set[str] = set()
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async def setup(self) -> None:
|
||||
if self.index_path.exists():
|
||||
self._seen_url_hashes = {
|
||||
line.strip()
|
||||
for line in self.index_path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
}
|
||||
return
|
||||
if self.path.exists():
|
||||
with self.path.open("r", encoding="utf-8") as file_handle:
|
||||
for line in file_handle:
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
url_hash = payload.get("url_hash") or (make_url_hash(str(payload["url"])) if payload.get("url") else None)
|
||||
if url_hash:
|
||||
self._seen_url_hashes.add(str(url_hash))
|
||||
if self._seen_url_hashes:
|
||||
self.index_path.write_text("\n".join(sorted(self._seen_url_hashes)) + "\n", encoding="utf-8")
|
||||
|
||||
async def store(self, result: CrawlResult) -> None:
|
||||
if result.url_hash in self._seen_url_hashes:
|
||||
return
|
||||
with self.path.open("a", encoding="utf-8") as file_handle:
|
||||
file_handle.write(json.dumps(result.to_dict(), ensure_ascii=False) + "\n")
|
||||
with self.index_path.open("a", encoding="utf-8") as file_handle:
|
||||
file_handle.write(result.url_hash + "\n")
|
||||
self._seen_url_hashes.add(result.url_hash)
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class MongoResultRepository(ResultRepository):
|
||||
def __init__(self, settings: StorageSettings) -> None:
|
||||
self.client = AsyncIOMotorClient(settings.mongodb_dsn)
|
||||
self.collection = self.client[settings.mongodb_database][settings.mongodb_collection]
|
||||
|
||||
async def setup(self) -> None:
|
||||
await self.collection.create_index("task_id", unique=False)
|
||||
await self.collection.create_index(
|
||||
"url_hash",
|
||||
unique=True,
|
||||
partialFilterExpression={"url_hash": {"$exists": True}},
|
||||
)
|
||||
|
||||
async def store(self, result: CrawlResult) -> None:
|
||||
await self.collection.update_one(
|
||||
{"url_hash": result.url_hash},
|
||||
{"$set": result.to_dict()},
|
||||
upsert=True,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
self.client.close()
|
||||
|
||||
|
||||
class PostgresResultRepository(ResultRepository):
|
||||
def __init__(self, settings: StorageSettings) -> None:
|
||||
self.dsn = settings.postgres_dsn
|
||||
self.table = settings.postgres_table
|
||||
self.pool: asyncpg.Pool | None = None
|
||||
|
||||
async def setup(self) -> None:
|
||||
self.pool = await asyncpg.create_pool(self.dsn)
|
||||
async with self.pool.acquire() as connection:
|
||||
await connection.execute(
|
||||
f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.table} (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
task_id TEXT NOT NULL,
|
||||
worker_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
url_hash TEXT NOT NULL,
|
||||
spider TEXT NOT NULL,
|
||||
http_status INTEGER,
|
||||
payload JSONB NOT NULL,
|
||||
error TEXT,
|
||||
finished_at TIMESTAMPTZ NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
await connection.execute(f"ALTER TABLE {self.table} ADD COLUMN IF NOT EXISTS url_hash TEXT")
|
||||
await connection.execute(
|
||||
f"CREATE UNIQUE INDEX IF NOT EXISTS {self.table}_url_hash_uidx ON {self.table} (url_hash)"
|
||||
)
|
||||
|
||||
async def store(self, result: CrawlResult) -> None:
|
||||
if self.pool is None:
|
||||
raise RuntimeError("Postgres repository is not initialized")
|
||||
async with self.pool.acquire() as connection:
|
||||
await connection.execute(
|
||||
f"""
|
||||
INSERT INTO {self.table}
|
||||
(task_id, worker_id, status, url, url_hash, spider, http_status, payload, error, finished_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10::timestamptz)
|
||||
ON CONFLICT (url_hash) DO UPDATE SET
|
||||
task_id = EXCLUDED.task_id,
|
||||
worker_id = EXCLUDED.worker_id,
|
||||
status = EXCLUDED.status,
|
||||
url = EXCLUDED.url,
|
||||
spider = EXCLUDED.spider,
|
||||
http_status = EXCLUDED.http_status,
|
||||
payload = EXCLUDED.payload,
|
||||
error = EXCLUDED.error,
|
||||
finished_at = EXCLUDED.finished_at
|
||||
""",
|
||||
result.task_id,
|
||||
result.worker_id,
|
||||
result.status.value,
|
||||
result.url,
|
||||
result.url_hash,
|
||||
result.spider,
|
||||
result.http_status,
|
||||
json.dumps(result.data, ensure_ascii=False),
|
||||
result.error,
|
||||
result.finished_at,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.pool is not None:
|
||||
await self.pool.close()
|
||||
|
||||
|
||||
def build_result_repository(settings: StorageSettings, jsonl_path: Path) -> ResultRepository:
|
||||
backend = settings.backend.lower()
|
||||
if backend == "mongodb":
|
||||
return MongoResultRepository(settings)
|
||||
if backend == "postgresql":
|
||||
return PostgresResultRepository(settings)
|
||||
return JsonlResultRepository(jsonl_path)
|
||||
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
from .config import AppSettings
|
||||
from .models import CrawlTask
|
||||
|
||||
|
||||
class ScheduleService:
|
||||
def __init__(self, settings: AppSettings, submit_callback) -> None:
|
||||
self.settings = settings
|
||||
self.submit_callback = submit_callback
|
||||
self.scheduler = AsyncIOScheduler()
|
||||
|
||||
def start(self) -> None:
|
||||
for item in self.settings.schedules:
|
||||
if item.trigger == "cron" and item.cron:
|
||||
trigger = CronTrigger.from_crontab(item.cron)
|
||||
else:
|
||||
seconds = item.seconds or 0
|
||||
minutes = item.minutes or 0
|
||||
trigger = IntervalTrigger(seconds=seconds, minutes=minutes)
|
||||
self.scheduler.add_job(self._submit_job, trigger=trigger, args=[item], id=item.name, replace_existing=True)
|
||||
self.scheduler.start()
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
self.scheduler.shutdown(wait=False)
|
||||
|
||||
async def _submit_job(self, item) -> None:
|
||||
payload = dict(item.task)
|
||||
task = CrawlTask.create(
|
||||
url=payload["url"],
|
||||
spider=payload.get("spider", self.settings.master.default_spider),
|
||||
method=payload.get("method", "GET"),
|
||||
headers=payload.get("headers"),
|
||||
metadata=payload.get("metadata"),
|
||||
body=payload.get("body"),
|
||||
priority=item.priority,
|
||||
schedule_name=item.name,
|
||||
)
|
||||
await self.submit_callback(task)
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
|
||||
class DualSessionStore:
|
||||
def __init__(self, redis: Redis, redis_prefix: str, backup_path: Path) -> None:
|
||||
self.redis = redis
|
||||
self.redis_prefix = redis_prefix
|
||||
self.backup_path = backup_path
|
||||
self.backup_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _key(self, domain: str) -> str:
|
||||
return f"{self.redis_prefix}:{domain}"
|
||||
|
||||
async def load(self, domain: str) -> dict[str, Any]:
|
||||
raw = await self.redis.get(self._key(domain))
|
||||
if raw:
|
||||
return json.loads(raw)
|
||||
if self.backup_path.exists():
|
||||
data = json.loads(self.backup_path.read_text(encoding="utf-8"))
|
||||
return data.get(domain, {})
|
||||
return {}
|
||||
|
||||
async def save(self, domain: str, payload: dict[str, Any]) -> None:
|
||||
await self.redis.set(self._key(domain), json.dumps(payload))
|
||||
snapshot: dict[str, Any] = {}
|
||||
if self.backup_path.exists():
|
||||
snapshot = json.loads(self.backup_path.read_text(encoding="utf-8"))
|
||||
snapshot[domain] = payload
|
||||
self.backup_path.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
|
||||
|
||||
class SignatureProvider(ABC):
|
||||
@abstractmethod
|
||||
async def sign(self, request_payload: dict[str, Any]) -> dict[str, Any]: ...
|
||||
|
||||
|
||||
class NoopSignatureProvider(SignatureProvider):
|
||||
async def sign(self, request_payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
class RpcSignatureProvider(SignatureProvider):
|
||||
def __init__(self, endpoint: str) -> None:
|
||||
self.endpoint = endpoint
|
||||
|
||||
async def sign(self, request_payload: dict[str, Any]) -> dict[str, Any]:
|
||||
if not self.endpoint:
|
||||
return {}
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(self.endpoint, json=request_payload) as response:
|
||||
response.raise_for_status()
|
||||
data = await response.json()
|
||||
return dict(data)
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from .http import AsyncHttpClient
|
||||
from .models import CrawlResult, CrawlTask, TaskStatus
|
||||
|
||||
|
||||
class BaseSpider(ABC):
|
||||
name: str
|
||||
|
||||
@abstractmethod
|
||||
async def execute(self, task: CrawlTask, client: AsyncHttpClient, worker_id: str) -> CrawlResult: ...
|
||||
|
||||
|
||||
class GenericFetchSpider(BaseSpider):
|
||||
name = "generic_fetch"
|
||||
|
||||
async def execute(self, task: CrawlTask, client: AsyncHttpClient, worker_id: str) -> CrawlResult:
|
||||
response = await client.fetch(task)
|
||||
data: dict[str, Any] = {
|
||||
"headers": response["headers"],
|
||||
"body": response["body"],
|
||||
"proxy": response["proxy"],
|
||||
"request_id": task.metadata.get("request_id"),
|
||||
}
|
||||
status = TaskStatus.SUCCESS if 200 <= response["status"] < 400 else TaskStatus.FAILED
|
||||
return CrawlResult(
|
||||
task_id=task.task_id,
|
||||
worker_id=worker_id,
|
||||
status=status,
|
||||
url=task.url,
|
||||
spider=self.name,
|
||||
http_status=response["status"],
|
||||
data=data,
|
||||
error=None if status == TaskStatus.SUCCESS else f"Unexpected status {response['status']}",
|
||||
)
|
||||
|
||||
|
||||
class AssetDownloadSpider(BaseSpider):
|
||||
name = "asset_download"
|
||||
|
||||
async def execute(self, task: CrawlTask, client: AsyncHttpClient, worker_id: str) -> CrawlResult:
|
||||
response = await client.fetch(task)
|
||||
parsed = urlparse(task.url)
|
||||
file_name = task.metadata.get("file_name") or Path(parsed.path or "resource.bin").name or "resource.bin"
|
||||
saved_to = await client.save_binary(task, file_name, response["body_bytes"])
|
||||
return CrawlResult(
|
||||
task_id=task.task_id,
|
||||
worker_id=worker_id,
|
||||
status=TaskStatus.SUCCESS if response["status"] < 400 else TaskStatus.FAILED,
|
||||
url=task.url,
|
||||
spider=self.name,
|
||||
http_status=response["status"],
|
||||
data={"saved_to": saved_to, "proxy": response["proxy"], "request_id": task.metadata.get("request_id")},
|
||||
error=None if response["status"] < 400 else f"Unexpected status {response['status']}",
|
||||
)
|
||||
|
||||
|
||||
class JsonApiSpider(BaseSpider):
|
||||
name = "json_api"
|
||||
|
||||
async def execute(self, task: CrawlTask, client: AsyncHttpClient, worker_id: str) -> CrawlResult:
|
||||
response = await client.fetch(task)
|
||||
body = response["body"]
|
||||
if isinstance(body, str):
|
||||
try:
|
||||
body = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
body = {"raw": body}
|
||||
return CrawlResult(
|
||||
task_id=task.task_id,
|
||||
worker_id=worker_id,
|
||||
status=TaskStatus.SUCCESS if response["status"] < 400 else TaskStatus.FAILED,
|
||||
url=task.url,
|
||||
spider=self.name,
|
||||
http_status=response["status"],
|
||||
data={"payload": body, "proxy": response["proxy"], "request_id": task.metadata.get("request_id")},
|
||||
error=None if response["status"] < 400 else f"Unexpected status {response['status']}",
|
||||
)
|
||||
|
||||
|
||||
def build_spider_registry() -> dict[str, BaseSpider]:
|
||||
spiders: list[BaseSpider] = [GenericFetchSpider(), AssetDownloadSpider(), JsonApiSpider()]
|
||||
return {spider.name: spider for spider in spiders}
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from .models import CrawlTask, TaskStatus, utc_now
|
||||
|
||||
|
||||
class CheckpointManager:
|
||||
def __init__(self, redis: Redis, checkpoint_dir: Path, namespace: str) -> None:
|
||||
self.redis = redis
|
||||
self.checkpoint_dir = checkpoint_dir
|
||||
self.namespace = namespace
|
||||
self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _task_key(self, task_id: str) -> str:
|
||||
return f"{self.namespace}:task:{task_id}"
|
||||
|
||||
def _running_key(self, worker_id: str) -> str:
|
||||
return f"{self.namespace}:running:{worker_id}"
|
||||
|
||||
def _file_path(self, task_id: str) -> Path:
|
||||
return self.checkpoint_dir / f"{task_id}.json"
|
||||
|
||||
async def save(self, task: CrawlTask) -> None:
|
||||
task.updated_at = utc_now()
|
||||
await self.redis.set(self._task_key(task.task_id), json.dumps(task.to_dict()))
|
||||
await self._write_file(task)
|
||||
|
||||
async def mark_status(self, task: CrawlTask, status: TaskStatus) -> None:
|
||||
task.status = status
|
||||
await self.save(task)
|
||||
if task.assigned_worker:
|
||||
running_key = self._running_key(task.assigned_worker)
|
||||
if status == TaskStatus.RUNNING:
|
||||
await self.redis.zadd(running_key, {task.task_id: self._score_now()})
|
||||
elif status in {TaskStatus.SUCCESS, TaskStatus.FAILED}:
|
||||
await self.redis.zrem(running_key, task.task_id)
|
||||
self._cleanup_file(task.task_id)
|
||||
|
||||
async def load(self, task_id: str) -> CrawlTask | None:
|
||||
raw = await self.redis.get(self._task_key(task_id))
|
||||
if raw:
|
||||
return CrawlTask.from_dict(json.loads(raw))
|
||||
file_path = self._file_path(task_id)
|
||||
if file_path.exists():
|
||||
return CrawlTask.from_dict(json.loads(file_path.read_text(encoding="utf-8")))
|
||||
return None
|
||||
|
||||
async def recover_stale(self, worker_id: str, older_than_seconds: int) -> list[CrawlTask]:
|
||||
threshold = self._score_now() - older_than_seconds
|
||||
task_ids = await self.redis.zrangebyscore(self._running_key(worker_id), min=0, max=threshold)
|
||||
tasks: list[CrawlTask] = []
|
||||
for task_id in task_ids:
|
||||
task = await self.load(task_id)
|
||||
if task is None:
|
||||
continue
|
||||
if task.status == TaskStatus.RUNNING:
|
||||
task.status = TaskStatus.RETRY
|
||||
tasks.append(task)
|
||||
return tasks
|
||||
|
||||
async def recover_local(self) -> list[CrawlTask]:
|
||||
tasks: list[CrawlTask] = []
|
||||
for file_path in self.checkpoint_dir.glob("*.json"):
|
||||
payload = json.loads(file_path.read_text(encoding="utf-8"))
|
||||
task = CrawlTask.from_dict(payload)
|
||||
if task.status in {TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.RETRY}:
|
||||
tasks.append(task)
|
||||
return tasks
|
||||
|
||||
async def _write_file(self, task: CrawlTask) -> None:
|
||||
self._file_path(task.task_id).write_text(json.dumps(task.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
def _cleanup_file(self, task_id: str) -> None:
|
||||
file_path = self._file_path(task_id)
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
|
||||
def _score_now(self) -> float:
|
||||
from time import time
|
||||
|
||||
return time()
|
||||
@@ -0,0 +1,186 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import aiohttp
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from .auth import ConfigurableAuthProvider, ManualAuthProvider
|
||||
from .backoff import exponential_backoff
|
||||
from .config import AppSettings
|
||||
from .http import AsyncHttpClient
|
||||
from .models import CrawlResult, CrawlTask, TaskStatus, WorkerHeartbeat
|
||||
from .proxy import ProxyPool
|
||||
from .queue import RedisPriorityQueue
|
||||
from .rate_limit import DomainRateLimiter
|
||||
from .sessions import DualSessionStore
|
||||
from .signatures import NoopSignatureProvider
|
||||
from .spider import BaseSpider, build_spider_registry
|
||||
from .state import CheckpointManager
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WorkerNode:
|
||||
def __init__(self, settings: AppSettings) -> None:
|
||||
self.settings = settings
|
||||
self.worker_id = settings.worker.worker_id
|
||||
self.redis = Redis.from_url(settings.redis_dsn, decode_responses=True)
|
||||
self.queue = RedisPriorityQueue(
|
||||
self.redis,
|
||||
settings.master.queue_name,
|
||||
settings.master.result_queue_name,
|
||||
settings.master.dead_letter_queue_name,
|
||||
)
|
||||
self.checkpoints = CheckpointManager(self.redis, settings.resolve_path(settings.checkpoint_dir), settings.master.queue_name)
|
||||
self.rate_limiter = DomainRateLimiter(settings.rate_limit)
|
||||
self.proxy_pool = ProxyPool(settings.proxy)
|
||||
self.session_store = DualSessionStore(
|
||||
self.redis,
|
||||
settings.session.redis_prefix,
|
||||
settings.resolve_path(settings.session.local_backup_path),
|
||||
)
|
||||
auth_provider = ConfigurableAuthProvider(settings.auth) if settings.auth.enabled else ManualAuthProvider()
|
||||
self.client = AsyncHttpClient(
|
||||
settings,
|
||||
self.rate_limiter,
|
||||
self.proxy_pool,
|
||||
self.session_store,
|
||||
auth_provider,
|
||||
NoopSignatureProvider(),
|
||||
)
|
||||
self.spiders: dict[str, BaseSpider] = build_spider_registry()
|
||||
self.active_tasks: set[str] = set()
|
||||
self.semaphore = asyncio.Semaphore(settings.worker.concurrency)
|
||||
self.master_base_url = f"http://{settings.master.host}:{settings.master.port}"
|
||||
self._loops: list[asyncio.Task] = []
|
||||
self._http_session: aiohttp.ClientSession | None = None
|
||||
|
||||
async def start(self) -> None:
|
||||
self._http_session = aiohttp.ClientSession()
|
||||
await self.client.start()
|
||||
if self.settings.proxy.enabled:
|
||||
await self.proxy_pool.refresh()
|
||||
await self.proxy_pool.verify()
|
||||
recovered = await self.checkpoints.recover_local()
|
||||
for task in recovered:
|
||||
if task.assigned_worker in {None, self.worker_id, "default"}:
|
||||
await self.queue.enqueue_task(self.worker_id, task)
|
||||
self._loops = [
|
||||
asyncio.create_task(self.consume_loop(), name=f"{self.worker_id}-consume"),
|
||||
asyncio.create_task(self.heartbeat_loop(), name=f"{self.worker_id}-heartbeat"),
|
||||
asyncio.create_task(self.proxy_refresh_loop(), name=f"{self.worker_id}-proxy-refresh"),
|
||||
]
|
||||
|
||||
async def close(self) -> None:
|
||||
for task in self._loops:
|
||||
task.cancel()
|
||||
await asyncio.gather(*self._loops, return_exceptions=True)
|
||||
await self.client.close()
|
||||
if self._http_session is not None:
|
||||
await self._http_session.close()
|
||||
await self.redis.aclose()
|
||||
|
||||
async def consume_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
task = await self.queue.dequeue_task(self.worker_id, timeout=5)
|
||||
if task is None:
|
||||
task = await self.queue.dequeue_task("default", timeout=1)
|
||||
if task is None:
|
||||
continue
|
||||
await self.semaphore.acquire()
|
||||
asyncio.create_task(self.execute_task(task))
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
LOGGER.exception("Worker %s consume loop failed", self.worker_id)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
async def execute_task(self, task: CrawlTask) -> None:
|
||||
self.active_tasks.add(task.task_id)
|
||||
try:
|
||||
task.assigned_worker = self.worker_id
|
||||
task.metadata.setdefault("task_id", task.task_id)
|
||||
task.metadata["worker_id"] = self.worker_id
|
||||
task.metadata["request_id"] = f"{task.task_id}-{task.retries}-{uuid4().hex[:8]}"
|
||||
await self.checkpoints.mark_status(task, TaskStatus.RUNNING)
|
||||
spider = self.spiders.get(task.spider)
|
||||
if spider is None:
|
||||
raise RuntimeError(f"Unknown spider: {task.spider}")
|
||||
result = await spider.execute(task, self.client, self.worker_id)
|
||||
if result.status == TaskStatus.FAILED and task.retries < task.max_retries:
|
||||
await self._retry_task(task, result.error or "worker failure")
|
||||
else:
|
||||
await self.queue.publish_result(result)
|
||||
except Exception as exc:
|
||||
LOGGER.exception("Task %s failed", task.task_id)
|
||||
if task.retries < task.max_retries:
|
||||
await self._retry_task(task, str(exc))
|
||||
else:
|
||||
result = CrawlResult(
|
||||
task_id=task.task_id,
|
||||
worker_id=self.worker_id,
|
||||
status=TaskStatus.FAILED,
|
||||
url=task.url,
|
||||
spider=task.spider,
|
||||
error=str(exc),
|
||||
)
|
||||
await self.queue.publish_result(result)
|
||||
finally:
|
||||
self.active_tasks.discard(task.task_id)
|
||||
self.semaphore.release()
|
||||
|
||||
async def heartbeat_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self.send_heartbeat()
|
||||
await asyncio.sleep(self.settings.worker.heartbeat_interval_seconds)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
LOGGER.exception("Worker %s heartbeat failed", self.worker_id)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def send_heartbeat(self) -> None:
|
||||
if self._http_session is None:
|
||||
return
|
||||
heartbeat = WorkerHeartbeat(
|
||||
worker_id=self.worker_id,
|
||||
concurrency=self.settings.worker.concurrency,
|
||||
active_tasks=len(self.active_tasks),
|
||||
host=socket.gethostname(),
|
||||
)
|
||||
async with self._http_session.post(f"{self.master_base_url}/heartbeat", json=heartbeat.to_dict()) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
async def proxy_refresh_loop(self) -> None:
|
||||
if not self.settings.proxy.enabled:
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(self.settings.proxy.refresh_interval_seconds)
|
||||
await self.proxy_pool.refresh()
|
||||
await self.proxy_pool.verify()
|
||||
LOGGER.info("Worker %s healthy proxies: %s", self.worker_id, await self.proxy_pool.healthy_count())
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
LOGGER.exception("Proxy refresh loop failed")
|
||||
|
||||
async def _retry_task(self, task: CrawlTask, reason: str) -> None:
|
||||
task.retries += 1
|
||||
task.status = TaskStatus.RETRY
|
||||
await self.checkpoints.save(task)
|
||||
delay = exponential_backoff(
|
||||
task.retries - 1,
|
||||
self.settings.worker.base_backoff_seconds,
|
||||
self.settings.worker.max_backoff_seconds,
|
||||
)
|
||||
LOGGER.warning("Retry task %s in %.2fs: %s", task.task_id, delay, reason)
|
||||
await asyncio.sleep(delay)
|
||||
await self.queue.enqueue_task(self.worker_id, task)
|
||||
@@ -0,0 +1,6 @@
|
||||
aiohttp>=3.10.10
|
||||
redis>=5.0.8
|
||||
APScheduler>=3.10.4
|
||||
PyYAML>=6.0.2
|
||||
motor>=3.6.0
|
||||
asyncpg>=0.30.0
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from crawler.config import load_settings
|
||||
from crawler.master import MasterNode
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
settings = load_settings(args.config)
|
||||
node = MasterNode(settings)
|
||||
await node.start()
|
||||
|
||||
runner = web.AppRunner(node.app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, settings.master.host, settings.master.port)
|
||||
await site.start()
|
||||
logging.getLogger(__name__).info("Master started at http://%s:%s", settings.master.host, settings.master.port)
|
||||
|
||||
try:
|
||||
while True:
|
||||
await asyncio.sleep(3600)
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
await node.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from crawler.config import load_settings
|
||||
from crawler.worker import WorkerNode
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", required=True)
|
||||
parser.add_argument("--worker-id", required=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
settings = load_settings(args.config)
|
||||
if args.worker_id:
|
||||
settings.worker.worker_id = args.worker_id
|
||||
|
||||
node = WorkerNode(settings)
|
||||
await node.start()
|
||||
logging.getLogger(__name__).info("Worker started: %s", settings.worker.worker_id)
|
||||
|
||||
try:
|
||||
while True:
|
||||
await asyncio.sleep(3600)
|
||||
finally:
|
||||
await node.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,280 @@
|
||||
# APK 下拉刷新 / 删除后页面不刷新 修复
|
||||
|
||||
**日期**: 2026-06-22
|
||||
**问题**: 删除作品后页面不刷新;下拉刷新不生效,无法重新请求当前页数据
|
||||
|
||||
---
|
||||
|
||||
## 根因分析
|
||||
|
||||
### 根因 1:`RefreshIndicator` 放错位置,嵌套在 `FutureBuilder` 内部
|
||||
|
||||
`RefreshIndicator` 被放在 `FutureBuilder` 的 `builder` 闭包内。每次 `setState` 更新 `future` → `FutureBuilder` 重建 → `RefreshIndicator` 被销毁再创建 → 下拉手势/动画状态丢失。
|
||||
|
||||
**正常工作**的 `feed_tab.dart` 结构:
|
||||
```
|
||||
RefreshIndicator → ListView → FutureBuilder(内部)
|
||||
```
|
||||
|
||||
**异常结构**(修复前):
|
||||
```
|
||||
FutureBuilder → RefreshIndicator → CustomScrollView
|
||||
```
|
||||
|
||||
### 根因 2:缺少 `AlwaysScrollableScrollPhysics`
|
||||
|
||||
`CustomScrollView` 未设置 `physics: AlwaysScrollableScrollPhysics()`。数据少于一屏时不可滚动 → `RefreshIndicator` 无法探测下拉手势。
|
||||
|
||||
### 根因 3(核心):API 客户端缓存导致的"假刷新"
|
||||
|
||||
`api_client.dart` 的 `gallery()` 方法有 **30 分钟 SharedPreferences 缓存**:
|
||||
|
||||
```dart
|
||||
// 修复前 - 每次都先查缓存
|
||||
final cacheKey = _cacheKey('gallery', params);
|
||||
final cached = await _readCache<List<dynamic>>(cacheKey);
|
||||
if (cached != null) return cached; // ← 始终返回旧数据
|
||||
```
|
||||
|
||||
刷新链路全追踪:
|
||||
```
|
||||
_refresh()
|
||||
→ ref.invalidate(apiClientProvider) ✅ 创建新 SakuraApiClient
|
||||
→ _load()
|
||||
→ gallery()
|
||||
→ _cacheKey('gallery', params) ❌ token 没变 → cache key 一样
|
||||
→ _readCache(cacheKey) ❌ 命中 30 分钟前的缓存
|
||||
→ 返回删除/修改前的旧数据 ❌ 表现:刷新无效
|
||||
```
|
||||
|
||||
Token 不变 → `_cacheKey` 内 `token.hashCode` 不变 → **缓存 key 不变** → 永远返回缓存。即使 `ref.invalidate` 创建了新客户端实例也无效,因为缓存存在 SharedPreferences 中,跨实例共享。
|
||||
|
||||
---
|
||||
|
||||
## 修改文件清单(3 个文件)
|
||||
|
||||
| 文件 | 改动 |
|
||||
|------|------|
|
||||
| `sakura_mobile/lib/src/core/api_client.dart` | `gallery()` 加 `bypassCache` 参数 |
|
||||
| `sakura_mobile/lib/src/pages/tabs/gallery_tab.dart` | 结构调整 + 缓存穿透 + 删除刷新按钮 |
|
||||
| `sakura_mobile/lib/src/pages/tabs/novels_tab.dart` | 结构调整 + 删除刷新按钮 |
|
||||
|
||||
---
|
||||
|
||||
## 改动详情
|
||||
|
||||
### 一、`sakura_mobile/lib/src/core/api_client.dart`
|
||||
|
||||
#### `gallery()` 加 `bypassCache` 参数
|
||||
|
||||
**改动前**:
|
||||
```dart
|
||||
Future<List<dynamic>> gallery({
|
||||
String? search,
|
||||
String searchScope = 'all',
|
||||
bool? isFavorite,
|
||||
String? contentType,
|
||||
String? sort,
|
||||
int page = 1,
|
||||
int limit = 60,
|
||||
bool includeTotal = true,
|
||||
}) async {
|
||||
...
|
||||
final cacheKey = _cacheKey('gallery', params);
|
||||
final cached = await _readCache<List<dynamic>>(cacheKey);
|
||||
if (cached != null) return cached;
|
||||
final response = await dio.get('/api/v1/gallery', queryParameters: params);
|
||||
final data = response.data['data']['items'] as List<dynamic>;
|
||||
await _writeCache(cacheKey, data);
|
||||
return data;
|
||||
}
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```dart
|
||||
Future<List<dynamic>> gallery({
|
||||
String? search,
|
||||
String searchScope = 'all',
|
||||
bool? isFavorite,
|
||||
String? contentType,
|
||||
String? sort,
|
||||
int page = 1,
|
||||
int limit = 60,
|
||||
bool includeTotal = true,
|
||||
bool bypassCache = false, // ← 新增
|
||||
}) async {
|
||||
...
|
||||
final cacheKey = _cacheKey('gallery', params);
|
||||
if (!bypassCache) { // ← 新增:仅在非刷新时读缓存
|
||||
final cached = await _readCache<List<dynamic>>(cacheKey);
|
||||
if (cached != null) return cached;
|
||||
}
|
||||
final response = await dio.get('/api/v1/gallery', queryParameters: params);
|
||||
final data = response.data['data']['items'] as List<dynamic>;
|
||||
await _writeCache(cacheKey, data);
|
||||
return data;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 二、`sakura_mobile/lib/src/pages/tabs/gallery_tab.dart`
|
||||
|
||||
#### 改动 1:`_load()` 加 `forceRefresh` 参数,传递到 `gallery()`
|
||||
|
||||
**改动前**:
|
||||
```dart
|
||||
Future<List<Artwork>> _load({bool reset = true}) async {
|
||||
if (reset) {
|
||||
_page = 1;
|
||||
_hasMore = true;
|
||||
_items.clear();
|
||||
selectedIds.clear();
|
||||
selectionMode = false;
|
||||
}
|
||||
final rows = await ref.read(apiClientProvider).gallery(
|
||||
search: search.text.trim().isEmpty ? null : search.text.trim(),
|
||||
searchScope: searchScope,
|
||||
isFavorite: favoritesOnly ? true : null,
|
||||
contentType: _effectiveContentType,
|
||||
sort: recentUpdatesOnly ? 'updated' : null,
|
||||
page: _page,
|
||||
limit: _pageSize,
|
||||
includeTotal: false,
|
||||
);
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```dart
|
||||
Future<List<Artwork>> _load({bool reset = true, bool forceRefresh = false}) async {
|
||||
if (reset) {
|
||||
_page = 1;
|
||||
_hasMore = true;
|
||||
_items.clear();
|
||||
selectedIds.clear();
|
||||
selectionMode = false;
|
||||
}
|
||||
final rows = await ref.read(apiClientProvider).gallery(
|
||||
search: search.text.trim().isEmpty ? null : search.text.trim(),
|
||||
searchScope: searchScope,
|
||||
isFavorite: favoritesOnly ? true : null,
|
||||
contentType: _effectiveContentType,
|
||||
sort: recentUpdatesOnly ? 'updated' : null,
|
||||
page: _page,
|
||||
limit: _pageSize,
|
||||
includeTotal: false,
|
||||
bypassCache: forceRefresh, // ← 新增:刷新时绕过缓存
|
||||
);
|
||||
```
|
||||
|
||||
#### 改动 2:`_refresh()` 改为 async,传递 `forceRefresh: true`
|
||||
|
||||
**改动前**:
|
||||
```dart
|
||||
void _refresh() {
|
||||
setState(() {
|
||||
future = _load();
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```dart
|
||||
Future<void> _refresh() async {
|
||||
ref.invalidate(apiClientProvider);
|
||||
final f = _load(forceRefresh: true); // ← 绕过缓存
|
||||
setState(() {
|
||||
future = f;
|
||||
});
|
||||
await f; // ← RefreshIndicator 等 future 完成才停止动画
|
||||
}
|
||||
```
|
||||
|
||||
#### 改动 3:`RefreshIndicator` 移到 `FutureBuilder` 外层
|
||||
|
||||
**改动前**:
|
||||
```dart
|
||||
return FutureBuilder<List<Artwork>>(
|
||||
future: future,
|
||||
builder: (context, snapshot) {
|
||||
...
|
||||
return RefreshIndicator(
|
||||
onRefresh: _refresh,
|
||||
child: CustomScrollView(
|
||||
controller: _scrollController,
|
||||
slivers: [...],
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```dart
|
||||
return RefreshIndicator(
|
||||
color: const Color(0xFFD95A83),
|
||||
onRefresh: _refresh,
|
||||
child: FutureBuilder<List<Artwork>>(
|
||||
future: future,
|
||||
builder: (context, snapshot) {
|
||||
...
|
||||
return CustomScrollView(
|
||||
controller: _scrollController,
|
||||
physics: const AlwaysScrollableScrollPhysics(), // ← 新增
|
||||
slivers: [...],
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
```
|
||||
|
||||
#### 改动 4:删除刷新 IconButton
|
||||
|
||||
**改动前**:
|
||||
```dart
|
||||
trailing: [
|
||||
...
|
||||
IconButton(
|
||||
tooltip: '刷新',
|
||||
onPressed: _refresh,
|
||||
icon: const Icon(Icons.refresh),
|
||||
),
|
||||
],
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```dart
|
||||
trailing: [
|
||||
... // 刷新按钮已移除,统一使用下拉刷新
|
||||
],
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 三、`sakura_mobile/lib/src/pages/tabs/novels_tab.dart`
|
||||
|
||||
同 gallery_tab 的改动 3、4(结构调整 + 删除按钮)。novels API 本身无缓存,无需 `bypassCache`。
|
||||
|
||||
---
|
||||
|
||||
## 修复后的刷新链路
|
||||
|
||||
```
|
||||
用户下拉 → RefreshIndicator.onRefresh → _refresh()
|
||||
→ ref.invalidate(apiClientProvider) ✅ 创建新客户端
|
||||
→ _load(forceRefresh: true)
|
||||
→ gallery(bypassCache: true) ✅ 跳过缓存
|
||||
→ dio.get('/api/v1/gallery') ✅ 真实 HTTP 请求
|
||||
→ _writeCache(cacheKey, newData) ✅ 写入新缓存
|
||||
→ 返回服务器最新数据 ✅
|
||||
→ setState({future: f}) ✅ FutureBuilder 拿到新 future
|
||||
→ await f ✅ 等待完成,停止下拉动画
|
||||
→ UI 展示最新数据 ✅ 删除/修改后的数据正确
|
||||
```
|
||||
|
||||
## 构建产物
|
||||
|
||||
| 产物 | 路径 | 大小 |
|
||||
|------|------|------|
|
||||
| APK | `release/sakura.apk` | 66.8 MB |
|
||||
| EXE | `dist/sakuradownloader.exe` | 63 MB |
|
||||
@@ -0,0 +1,310 @@
|
||||
# 云同步 401 自动刷新 Token 修复
|
||||
|
||||
**日期**: 2026-06-22
|
||||
**问题**: 桌面端 `POST /api/v1/sync/check` 持续返回 401,无重试/续期机制
|
||||
**根因**: `CloudSyncConfig` 只存 `access_token`,无 `refresh_token` 和自动续期逻辑
|
||||
|
||||
---
|
||||
|
||||
## 修改文件清单
|
||||
|
||||
| 文件 | 改动类型 |
|
||||
|------|----------|
|
||||
| `pixiv_app/core/cloud_sync.py` | 核心:加 refresh_token、401 自动刷新、token 更新回调 |
|
||||
| `pixiv_app/gui/app.py` | 存储/加载/传递 refresh_token,回调持久化 |
|
||||
| `pixiv_app/core/gallery_service.py` | 传递 refresh_token |
|
||||
| `pixiv_app/core/origin_follow_sync.py` | 传递 refresh_token |
|
||||
| `pixiv_app/core/x_media_service.py` | 传递 refresh_token |
|
||||
|
||||
---
|
||||
|
||||
## 改动详情
|
||||
|
||||
### 1. `pixiv_app/core/cloud_sync.py`
|
||||
|
||||
#### 1.1 `CloudSyncConfig` — 加 `refresh_token` 字段
|
||||
|
||||
**改动前**:
|
||||
```python
|
||||
@dataclass
|
||||
class CloudSyncConfig:
|
||||
base_url: str
|
||||
access_token: str
|
||||
chunk_size: int = 5 * 1024 * 1024
|
||||
timeout: float = 60.0
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```python
|
||||
@dataclass
|
||||
class CloudSyncConfig:
|
||||
base_url: str
|
||||
access_token: str
|
||||
refresh_token: str = ""
|
||||
chunk_size: int = 5 * 1024 * 1024
|
||||
timeout: float = 60.0
|
||||
```
|
||||
|
||||
#### 1.2 `SakuraCloudClient.__init__` — 加 `on_token_refreshed` 回调
|
||||
|
||||
**改动前**:
|
||||
```python
|
||||
class SakuraCloudClient:
|
||||
def __init__(self, config: CloudSyncConfig) -> None:
|
||||
self.config = config
|
||||
self.base_url = config.base_url.rstrip("/")
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({"Authorization": f"Bearer {config.access_token}"})
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```python
|
||||
class SakuraCloudClient:
|
||||
def __init__(self, config: CloudSyncConfig, *, on_token_refreshed: callable | None = None) -> None:
|
||||
self.config = config
|
||||
self.base_url = config.base_url.rstrip("/")
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({"Authorization": f"Bearer {config.access_token}"})
|
||||
self._on_token_refreshed = on_token_refreshed
|
||||
self._refresh_lock = False # 防止并发刷新
|
||||
```
|
||||
|
||||
#### 1.3 `_post` — 加 401 自动刷新 + 重试
|
||||
|
||||
**改动前**:
|
||||
```python
|
||||
def _post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
response = self.session.post(f"{self.base_url}{path}", json=payload, timeout=self.config.timeout)
|
||||
return self._unwrap(response)
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```python
|
||||
def _post(self, path: str, payload: dict[str, Any], *, _retry_on_401: bool = True) -> dict[str, Any]:
|
||||
response = self.session.post(f"{self.base_url}{path}", json=payload, timeout=self.config.timeout)
|
||||
if response.status_code == 401 and _retry_on_401:
|
||||
refreshed = self._try_refresh_token()
|
||||
if refreshed:
|
||||
return self._post(path, payload, _retry_on_401=False)
|
||||
return self._unwrap(response)
|
||||
```
|
||||
|
||||
#### 1.4 新增 `_try_refresh_token` 方法
|
||||
|
||||
**新增代码**:
|
||||
```python
|
||||
def _try_refresh_token(self) -> bool:
|
||||
"""用 refresh_token 换取新的 access_token。成功返回 True 并通知上层持久化。"""
|
||||
refresh = self.config.refresh_token.strip()
|
||||
if not refresh or self._refresh_lock:
|
||||
return False
|
||||
self._refresh_lock = True
|
||||
try:
|
||||
resp = self.session.post(
|
||||
f"{self.base_url}/api/v1/auth/refresh",
|
||||
json={"refresh_token": refresh},
|
||||
timeout=self.config.timeout,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return False
|
||||
body = resp.json()
|
||||
data = body.get("data", body) if isinstance(body, dict) else {}
|
||||
new_access = (data.get("access_token") or "").strip()
|
||||
new_refresh = (data.get("refresh_token") or "").strip()
|
||||
if not new_access:
|
||||
return False
|
||||
self.config.access_token = new_access
|
||||
self.session.headers.update({"Authorization": f"Bearer {new_access}"})
|
||||
if new_refresh:
|
||||
self.config.refresh_token = new_refresh
|
||||
if self._on_token_refreshed:
|
||||
try:
|
||||
self._on_token_refreshed(self.config.access_token, self.config.refresh_token)
|
||||
except Exception:
|
||||
pass # 持久化失败不阻塞业务
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
self._refresh_lock = False
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. `pixiv_app/gui/app.py`
|
||||
|
||||
#### 2.1 新增 `cloud_refresh_token` 实例变量
|
||||
|
||||
**改动前**:
|
||||
```python
|
||||
self.cloud_access_token = ""
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```python
|
||||
self.cloud_access_token = ""
|
||||
self.cloud_refresh_token = ""
|
||||
```
|
||||
|
||||
#### 2.2 `get_cloud_sync_config` — 加 `refresh_token`
|
||||
|
||||
**改动前**:
|
||||
```python
|
||||
def get_cloud_sync_config(self) -> dict:
|
||||
return {
|
||||
"enabled": bool(self.cloud_sync_enabled_var.get()),
|
||||
"base_url": (self.cloud_base_url or DEFAULT_CLOUD_API).strip(),
|
||||
"access_token": self.cloud_access_token.strip(),
|
||||
}
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```python
|
||||
def get_cloud_sync_config(self) -> dict:
|
||||
return {
|
||||
"enabled": bool(self.cloud_sync_enabled_var.get()),
|
||||
"base_url": (self.cloud_base_url or DEFAULT_CLOUD_API).strip(),
|
||||
"access_token": self.cloud_access_token.strip(),
|
||||
"refresh_token": self.cloud_refresh_token.strip(),
|
||||
}
|
||||
```
|
||||
|
||||
#### 2.3 登录成功时捕获 `refresh_token`
|
||||
|
||||
**改动前**:
|
||||
```python
|
||||
token = data.get("data", {}).get("access_token")
|
||||
if token:
|
||||
self.cloud_access_token = str(token).strip()
|
||||
self.enqueue_ui(self.cloud_status_var.set, "登录成功,云同步凭证已自动保存")
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```python
|
||||
token = data.get("data", {}).get("access_token")
|
||||
if token:
|
||||
self.cloud_access_token = str(token).strip()
|
||||
refresh = (data.get("data", {}).get("refresh_token") or "")
|
||||
if refresh:
|
||||
self.cloud_refresh_token = str(refresh).strip()
|
||||
self.enqueue_ui(self.cloud_status_var.set, "登录成功,云同步凭证已自动保存")
|
||||
```
|
||||
|
||||
#### 2.4 `restore_session` 加载 refresh_token
|
||||
|
||||
**改动前**:
|
||||
```python
|
||||
self.cloud_access_token = str(cloud.get("access_token") or "")
|
||||
if self.cloud_access_token:
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```python
|
||||
self.cloud_access_token = str(cloud.get("access_token") or "")
|
||||
self.cloud_refresh_token = str(cloud.get("refresh_token") or "")
|
||||
if self.cloud_access_token:
|
||||
```
|
||||
|
||||
#### 2.5 新增 `_on_cloud_token_refreshed` 回调 + 更新 `create_sync_manager`
|
||||
|
||||
**改动前**:
|
||||
```python
|
||||
def create_sync_manager(self) -> SyncManager | None:
|
||||
config = self.get_cloud_sync_config()
|
||||
if not (config.get("enabled") and config.get("base_url") and config.get("access_token")):
|
||||
return None
|
||||
return SyncManager(
|
||||
SakuraCloudClient(
|
||||
CloudSyncConfig(
|
||||
base_url=str(config["base_url"]),
|
||||
access_token=str(config["access_token"]),
|
||||
)
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```python
|
||||
def _on_cloud_token_refreshed(self, new_access: str, new_refresh: str) -> None:
|
||||
"""Token 自动刷新后更新内存并持久化到本地会话文件。"""
|
||||
self.cloud_access_token = new_access
|
||||
if new_refresh:
|
||||
self.cloud_refresh_token = new_refresh
|
||||
self.enqueue_ui(self.cloud_status_var.set, "云同步 Token 已自动续期")
|
||||
self.enqueue_ui(self.save_session)
|
||||
|
||||
def create_sync_manager(self) -> SyncManager | None:
|
||||
config = self.get_cloud_sync_config()
|
||||
if not (config.get("enabled") and config.get("base_url") and config.get("access_token")):
|
||||
return None
|
||||
return SyncManager(
|
||||
SakuraCloudClient(
|
||||
CloudSyncConfig(
|
||||
base_url=str(config["base_url"]),
|
||||
access_token=str(config["access_token"]),
|
||||
refresh_token=str(config.get("refresh_token") or ""),
|
||||
),
|
||||
on_token_refreshed=self._on_cloud_token_refreshed,
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3. `pixiv_app/core/gallery_service.py`
|
||||
|
||||
**改动前**:
|
||||
```python
|
||||
return SyncManager(
|
||||
SakuraCloudClient(
|
||||
CloudSyncConfig(
|
||||
base_url=base_url,
|
||||
access_token=access_token,
|
||||
)
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
**改动后**:
|
||||
```python
|
||||
return SyncManager(
|
||||
SakuraCloudClient(
|
||||
CloudSyncConfig(
|
||||
base_url=base_url,
|
||||
access_token=access_token,
|
||||
refresh_token=str(cloud.get("refresh_token") or ""),
|
||||
)
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. `pixiv_app/core/origin_follow_sync.py`
|
||||
|
||||
同 gallery_service.py,仅加 `refresh_token` 行。
|
||||
|
||||
### 5. `pixiv_app/core/x_media_service.py`
|
||||
|
||||
同 gallery_service.py,仅加 `refresh_token` 行。
|
||||
|
||||
---
|
||||
|
||||
## 工作流程
|
||||
|
||||
```
|
||||
POST /api/v1/sync/check
|
||||
├── 200 → 正常返回
|
||||
└── 401
|
||||
├── refresh_token 为空? → 抛异常(旧 token 永久失效)
|
||||
├── 正在刷新中? → 抛异常(防并发)
|
||||
└── POST /api/v1/auth/refresh
|
||||
├── 200 → 更新内存 token → 回调持久化 → 重试原请求
|
||||
└── 非200 → 抛异常(refresh 也过期了,需重新登录)
|
||||
```
|
||||
|
||||
## 兼容性
|
||||
|
||||
- `refresh_token` 默认值 `""` — 不传也能正常工作(旧行为)
|
||||
- `on_token_refreshed` 可选 — 不传也不影响其他调用方
|
||||
- 现有的 `ConfigCloudSync` 构造方式全兼容
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,492 @@
|
||||
# APP 端问题分析与解决方案
|
||||
|
||||
> 更新日期:2026-06-14
|
||||
> 范围:Sakura Mobile (Flutter) + Sakura Cloud (FastAPI 后端) + Sakura Downloader (Python 桌面端)
|
||||
> 说明:本文档仅提供问题分析和修改方案,不包含实际代码改动。
|
||||
|
||||
---
|
||||
|
||||
## 问题一:APP 莫名其妙跳转到 "dakura" 用户
|
||||
|
||||
### 问题现象
|
||||
|
||||
APP 在使用过程中会突然切换到 "dakura" 用户身份,导致看到的是其他用户的数据而并非自己的内容。
|
||||
|
||||
### 根因分析
|
||||
|
||||
经过对全部代码的排查,问题出在 **Session 持久化缺少用户隔离机制**。涉及以下关键位置:
|
||||
|
||||
#### 1. 移动端 Session 存储无用户绑定 (`sakura_mobile/lib/src/core/session.dart`)
|
||||
|
||||
**文件位置:** `sakura_mobile/lib/src/core/session.dart` 第 44-52 行
|
||||
|
||||
```dart
|
||||
static const _baseUrlKey = 'sakura.baseUrl';
|
||||
static const _accessTokenKey = 'sakura.accessToken';
|
||||
|
||||
Future<void> _load() async {
|
||||
final prefs = await SharedPreferences.getInstance();
|
||||
state = SessionState(
|
||||
baseUrl: prefs.getString(_baseUrlKey) ?? defaultCloudApi,
|
||||
accessToken: prefs.getString(_accessTokenKey) ?? '',
|
||||
);
|
||||
```
|
||||
|
||||
**问题:** `SharedPreferences` 使用固定 key `sakura.accessToken` 存储 token,不绑定任何用户标识。当存在以下情况时会发生串号:
|
||||
- 同一设备上先后登录过不同账号(如 "dakura" 和当前用户)
|
||||
- SharedPreferences 中的旧 token 未完全清除
|
||||
- App 从后台恢复时读取了旧的/缓存的 token
|
||||
- 多进程/多任务环境下 SharedPreferences 跨进程同步延迟
|
||||
|
||||
#### 2. 登录流程缺少旧 Session 清理 (`sakura_mobile/lib/src/core/session.dart` 第 70-86 行)
|
||||
|
||||
```dart
|
||||
Future<void> login(String usernameOrEmail, String password) async {
|
||||
state = state.copyWith(loading: true, error: null);
|
||||
try {
|
||||
final api = SakuraApiClient(baseUrl: state.baseUrl);
|
||||
final tokens = await api.login(usernameOrEmail, password);
|
||||
final accessToken = tokens['access_token']?.toString() ?? '';
|
||||
final prefs = await SharedPreferences.getInstance();
|
||||
await prefs.setString(_accessTokenKey, accessToken);
|
||||
```
|
||||
|
||||
**问题:** 登录时直接覆盖 `_accessTokenKey`,但如果 SharedPreferences 写入失败或未完全刷盘,旧 token 仍然存在。没有先清除旧 token 再写入新 token 的原子操作。
|
||||
|
||||
#### 3. Web 端使用 sessionStorage 同样缺少隔离 (`sakura_admin_web/src/stores/session.ts` 第 15-30 行)
|
||||
|
||||
```typescript
|
||||
const ACCESS_KEY = "sakura_access_token";
|
||||
const REFRESH_KEY = "sakura_refresh_token";
|
||||
|
||||
export const authState = reactive({
|
||||
accessToken: sessionStorage.getItem(ACCESS_KEY) || "",
|
||||
refreshToken: sessionStorage.getItem(REFRESH_KEY) || "",
|
||||
user: null as CurrentUser | null,
|
||||
loading: false
|
||||
});
|
||||
```
|
||||
|
||||
**问题:** sessionStorage 的 key 是固定的,同一浏览器不同标签页可能共享或冲突。
|
||||
|
||||
#### 4. 桌面端 Session 文件共享 (`pixiv_app/core/paths.py` 和 `pixiv_app/core/auth.py`)
|
||||
|
||||
Session 文件 `pixiv_app_session.json` 保存的是单一的 cookie 和 cloud_sync 配置,没有对应用户上下文。如果云端切换了用户但本地 session 文件未同步更新,可能导致不一致。
|
||||
|
||||
### 解决方案
|
||||
|
||||
#### 方案 A:Token 绑定用户名校验(推荐)
|
||||
|
||||
**修改位置:**
|
||||
|
||||
| 文件 | 行号 | 修改内容 |
|
||||
|------|------|----------|
|
||||
| `sakura_mobile/lib/src/core/session.dart` | 第 44-52 行 | 在 `_load()` 后调用 `/api/v1/users/me` 校验 token 归属用户 |
|
||||
| `sakura_mobile/lib/src/core/session.dart` | 第 70-86 行 | `login()` 时先清除旧 token,写入后立即验证 |
|
||||
| `sakura_mobile/lib/src/core/session.dart` | 新增方法 | 添加 `_validateAndFixSession()` 方法,对比登录用户名与 token 归属 |
|
||||
|
||||
**具体做法:**
|
||||
1. 在 `SessionController._load()` 中,加载 token 后立即调用 `api.me()` 获取当前 token 对应的用户名
|
||||
2. 将用户名也存入 SharedPreferences,key 使用 `sakura.username`
|
||||
3. 每次 `_load()` 后比对 `sakura.username` 与 `me()` 返回的 username,不一致则清除旧 token 强制重新登录
|
||||
4. `login()` 成功后立即存储用户名,防止与旧数据混淆
|
||||
|
||||
#### 方案 B:增加 Token 失效主动检测
|
||||
|
||||
**修改位置:**
|
||||
|
||||
| 文件 | 行号 | 修改内容 |
|
||||
|------|------|----------|
|
||||
| `sakura_mobile/lib/src/core/api_client.dart` | 第 12-47 行 | 在 `SakuraApiClient` 中增加 401 拦截器 |
|
||||
| `sakura_mobile/lib/src/core/session.dart` | 第 100-108 行 | `logout()` 时增加确认清理 SharedPreferences 全部 key |
|
||||
|
||||
**具体做法:**
|
||||
1. 通过 Dio 拦截器全局捕获 401/403 响应,自动清除 token 并跳转登录页
|
||||
2. 在 `logout()` 中不单删除 `_accessTokenKey`,同时清理所有 sakura 相关 key
|
||||
3. 增加 `clearAllSakuraData()` 方法用于完全重置
|
||||
|
||||
#### 方案 C:服务端 Token 版本号机制
|
||||
|
||||
**修改位置:** 后端 `sakura_cloud/` (FastAPI) 相关认证模块
|
||||
|
||||
**具体做法:**
|
||||
1. 在用户表增加 `token_version` 字段
|
||||
2. 每次重新登录时 token_version +1
|
||||
3. JWT payload 中嵌入 token_version
|
||||
4. 验证 token 时比对版本号,旧版本 token 直接拒绝
|
||||
5. 确保用户登出后旧 token 彻底失效
|
||||
|
||||
---
|
||||
|
||||
## 问题二:发现页查询过慢,需要加载缓冲或优化
|
||||
|
||||
### 问题现象
|
||||
|
||||
APP 打开"发现"页(FeedTab)时,整个页面长时间白屏/转圈,用户体验差。
|
||||
|
||||
### 根因分析
|
||||
|
||||
#### 1. 五个 API 串行阻塞全部内容 (`sakura_mobile/lib/src/pages/tabs/feed_tab.dart`)
|
||||
|
||||
**文件位置:** `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` 第 20-26 行
|
||||
|
||||
```dart
|
||||
return FutureBuilder<List<Object?>>(
|
||||
future: Future.wait([
|
||||
api.galleryStat(), // 统计总数
|
||||
api.galleryFeed(mode: 'created', limit: 36), // 今日新增
|
||||
api.galleryFeed(mode: 'recent', limit: 24), // 最近更新
|
||||
api.recentNovels(limit: 12), // 最近小说
|
||||
api.unreadAlerts(limit: 12), // 未读提醒
|
||||
]),
|
||||
```
|
||||
|
||||
**问题:**
|
||||
- `Future.wait` 等待所有 5 个请求完成才渲染页面
|
||||
- 任一接口慢(尤其是 `galleryStat()` 和 `galleryFeed()`),整个发现页都会卡住
|
||||
- `galleryStat()` 可能后端做了全表 COUNT,数据量大时耗时数秒
|
||||
|
||||
#### 2. 后端 gallery/feed 查询可能缺少优化
|
||||
|
||||
**文件位置:** 后端 API(`sakura_cloud/` 目录下的 gallery/feed 路由)
|
||||
|
||||
`galleryFeed` 接口通常对应:
|
||||
- `mode: 'created'` → 按创建时间排序的最近作品
|
||||
- `mode: 'recent'` → 按更新时间排序
|
||||
|
||||
如果后端 SQL 查询缺少合适的复合索引,或者做了全表扫描 + JOIN 多表,在数据量上千条后就会明显变慢。典型慢查询模式:
|
||||
|
||||
```sql
|
||||
-- 可能的慢查询(缺少索引或 JOIN 过多)
|
||||
SELECT a.*, ar.name, tc.thumbnail_path, ...
|
||||
FROM artworks a
|
||||
LEFT JOIN artists ar ON ...
|
||||
LEFT JOIN thumbnail_cache tc ON ...
|
||||
LEFT JOIN artwork_files af ON ...
|
||||
LEFT JOIN favorites fav ON ...
|
||||
ORDER BY a.created_at DESC
|
||||
LIMIT 36
|
||||
```
|
||||
|
||||
**对比本地库查询** (`pixiv_app/core/gallery_service.py` 第 96-134 行) 同样有类似的多表 LEFT JOIN + GROUP BY + 复合搜索条件,这在 SQLite 上尤其慢:
|
||||
|
||||
```python
|
||||
total = library.connection.execute(
|
||||
f"SELECT COUNT(DISTINCT a.artwork_id) AS total FROM artworks a {joins} {where_sql}",
|
||||
params,
|
||||
).fetchone()["total"]
|
||||
```
|
||||
|
||||
先 COUNT 再 SELECT,等于两次大查询。
|
||||
|
||||
#### 3. 图片缩略图同步加载阻塞渲染
|
||||
|
||||
**文件位置:** `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` 第 385-389 行
|
||||
|
||||
```dart
|
||||
CachedNetworkImage(
|
||||
imageUrl: thumbUrl,
|
||||
httpHeaders: api.authHeaders(),
|
||||
fit: BoxFit.cover,
|
||||
),
|
||||
```
|
||||
|
||||
**问题:** 虽然使用了 `CachedNetworkImage` 做缓存,但如果后端 `thumbnail_url` 指向的是需要实时生成或大文件的原图,加载会很慢;且横向滚动的 `ListView` 会一次性构建多个图片 widget,造成网络拥塞。
|
||||
|
||||
### 解决方案
|
||||
|
||||
#### 方案 A:渐进式加载 + 骨架屏(推荐,前端改动最小)
|
||||
|
||||
**修改位置:**
|
||||
|
||||
| 文件 | 行号 | 修改内容 |
|
||||
|------|------|----------|
|
||||
| `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` | 第 20-26 行 | 将 `Future.wait` 拆分为独立请求,每个区块独立加载 |
|
||||
| `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` | 第 62-137 行 | 为每个 `_ArtworkStrip`、`_NovelStrip`、`_AlertStrip` 增加独立的 loading/skeleton 状态 |
|
||||
| `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` | 新增组件 | 增加 `_SkeletonStrip` 骨架屏组件 |
|
||||
|
||||
**具体做法:**
|
||||
1. 不再使用单个 `FutureBuilder` + `Future.wait`,改为每个 section 独立使用 `FutureBuilder`
|
||||
2. 统计卡片(`_TodayUpdateCard` 和 `_StatCard`)优先加载
|
||||
3. 各内容条(插画、漫画、小说、提醒)各自独立加载,失败的只影响自己
|
||||
4. 加载中的条显示 shimmer 骨架屏而非空白
|
||||
5. `galleryStat()` 如果慢,可设置 3 秒超时,超时则降级显示 "--"
|
||||
|
||||
#### 方案 B:后端增加 feed 缓存表(推荐,解决根本问题)
|
||||
|
||||
**修改位置:** 后端 `sakura_cloud/`
|
||||
|
||||
**具体做法:**
|
||||
1. 新增 `feed_cache` 表:
|
||||
|
||||
```sql
|
||||
CREATE TABLE feed_cache (
|
||||
user_id UUID NOT NULL,
|
||||
feed_type VARCHAR(32) NOT NULL, -- 'created', 'recent', 'stat'
|
||||
payload_json JSONB NOT NULL,
|
||||
generated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (user_id, feed_type)
|
||||
);
|
||||
```
|
||||
|
||||
2. 在每次上传新作品、同步关注的定时任务完成后,异步刷新 feed_cache
|
||||
3. `/api/v1/gallery/feed` 和 `/api/v1/gallery/stat` 优先从 `feed_cache` 读取
|
||||
4. 缓存有效期设为 5-10 分钟,过期后后台异步刷新,当前请求仍返回旧缓存
|
||||
5. 如果缓存不存在,降级为直接查询(首次慢但后续快)
|
||||
|
||||
#### 方案 C:本地 SQLite 缓存 + 增量同步(移动端离线方案)
|
||||
|
||||
**修改位置:**
|
||||
|
||||
| 文件 | 行号 | 修改内容 |
|
||||
|------|------|----------|
|
||||
| `sakura_mobile/lib/src/core/api_client.dart` | 新增方法 | 增加本地缓存读写方法 |
|
||||
| `sakura_mobile/lib/src/pages/tabs/feed_tab.dart` | 第 20 行 | 优先读本地缓存,后台静默刷新 |
|
||||
|
||||
**具体做法:**
|
||||
1. 移动端使用 `sqflite` 或 `drift` 建立轻量本地库
|
||||
2. 发现页数据缓存到本地,设置 30 分钟有效期
|
||||
3. 打开发现页时先展示缓存数据(秒开),同时后台请求新数据
|
||||
4. 新数据到达后平滑替换(`AnimatedSwitcher` 已就位)
|
||||
5. 支持下拉强制刷新(`RefreshIndicator` 已就位)
|
||||
|
||||
#### 方案 D:优化后端 gallery/feed 查询语句
|
||||
|
||||
**修改位置:** 后端 `sakura_cloud/api/v1/` 中 gallery/feed 相关路由
|
||||
|
||||
**具体做法:**
|
||||
1. 分析慢查询日志,定位具体慢的 SQL
|
||||
2. 为 `artworks` 表增加复合索引:
|
||||
```sql
|
||||
CREATE INDEX idx_artworks_user_created ON artworks(user_id, created_at DESC);
|
||||
CREATE INDEX idx_artworks_user_updated ON artworks(user_id, updated_at DESC);
|
||||
```
|
||||
3. 将 COUNT + SELECT 两次查询合并为一次(用窗口函数或子查询)
|
||||
4. 限制 `galleryStat()` 的统计精度,超过 10000 条时显示 "10000+"
|
||||
5. 对缩略图 URL 确保使用 CDN 或 Nginx 静态文件服务,而非 Django/FastAPI 流式响应
|
||||
|
||||
---
|
||||
|
||||
## 问题三:提醒中心用户更新检测问题
|
||||
|
||||
### 问题现象
|
||||
|
||||
1. **最近提醒显示的并非实际关注的作者名字**,而是作者 ID 数字
|
||||
2. **已关注的作者有更新但未检测到**,提醒中心未生成新提醒
|
||||
|
||||
### 根因分析
|
||||
|
||||
#### 1. 提醒标题使用 author_id 而非 author_name
|
||||
|
||||
**文件位置:** `pixiv_app/core/gallery_service.py` 第 588-613 行
|
||||
|
||||
```python
|
||||
def _create_cloud_pixiv_update_alerts(self, artist_id: int, work_ids: list[int]) -> int:
|
||||
...
|
||||
for work_id in work_ids:
|
||||
try:
|
||||
manager.client.create_alert(
|
||||
{
|
||||
"platform": "pixiv",
|
||||
"creator_id": str(int(artist_id)),
|
||||
"title": f"Pixiv 作者 {int(artist_id)} 更新了作品 {int(work_id)}",
|
||||
# ↑↑↑ 问题在这里:用的是 author_id 数字,而非 author name
|
||||
"body": "你关注的 Pixiv 作者有新作品了",
|
||||
...
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
**问题详解:**
|
||||
- `_create_cloud_pixiv_update_alerts` 没有接收 `artist_name` 参数
|
||||
- 调用它的 `sync_followed_artists`(第 505-586 行)虽然能从 `library` 查到作者信息,但没传给 alert 创建函数
|
||||
- 导致提醒标题变成 "Pixiv 作者 123456 更新了作品 789012",用户看到的是一串数字
|
||||
- 移动端 `reminder_center_page.dart` 直接展示 `alert['title']`,所以显示的是数字 ID
|
||||
|
||||
#### 2. 关注作者有更新但未检测到
|
||||
|
||||
**文件位置:** `pixiv_app/core/gallery_service.py` 第 505-586 行
|
||||
|
||||
```python
|
||||
def sync_followed_artists(self, *, cookie, max_new_per_artist=20, download=True, save_root="Sakura_Downloads"):
|
||||
...
|
||||
for row in followed:
|
||||
artist_id = int(row["artist_id"])
|
||||
# 获取已存在的作品 ID
|
||||
existing = {
|
||||
int(item["artwork_id"])
|
||||
for item in library.connection.execute(
|
||||
"SELECT artwork_id FROM artworks WHERE artist_id = ? AND type IN ('illust', 'manga')",
|
||||
(artist_id,),
|
||||
).fetchall()
|
||||
}
|
||||
# 获取远程作品 ID
|
||||
_, work_ids = fetch_user_work_ids(user_id=artist_id, cookie=cookie)
|
||||
# 只取本地没有的
|
||||
new_ids = [work_id for work_id in work_ids if work_id not in existing][:max_new_per_artist]
|
||||
```
|
||||
|
||||
**问题详解:**
|
||||
- `fetch_user_work_ids` 需要有效的 Pixiv Cookie,如果 Cookie 已过期,API 调用静默失败
|
||||
- 对比逻辑只看 `existing` 中的 artwork_id,但如果作品之前下载失败、入库未完成,可能漏掉
|
||||
- `mark_artist_synced(artist_id, error=str(exc))` 在异常时记录了错误但未区分错误类型(Cookie 过期 vs 网络错误 vs 限流)
|
||||
- 没有"最近检查时间"的限流机制 —— 如果短时间内多次同步同一个作者,每次都做全量对比
|
||||
|
||||
#### 3. 检查完成后未更新 `last_checked_at`
|
||||
|
||||
**文件位置:** `pixiv_app/core/library.py` 第 592-603 行
|
||||
|
||||
```python
|
||||
def mark_artist_synced(self, artist_id: int, *, error: str = "") -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO followed_artists (artist_id, enabled, last_synced_at, last_error)
|
||||
VALUES (?, 1, CURRENT_TIMESTAMP, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
last_synced_at = CURRENT_TIMESTAMP,
|
||||
last_error = excluded.last_error
|
||||
""",
|
||||
(artist_id, error),
|
||||
)
|
||||
```
|
||||
|
||||
**问题:** `last_synced_at` 只在本地 SQLite 更新,**没有同步到云端** follows 表的 `last_checked_at`。移动端提醒中心展示的是云端 follows 数据:
|
||||
|
||||
**文件位置:** `sakura_mobile/lib/src/pages/reminder_center_page.dart` 第 22-27 行
|
||||
|
||||
```dart
|
||||
Future<_ReminderCenterData> _load() async {
|
||||
final api = ref.read(apiClientProvider);
|
||||
final results = await Future.wait<Object?>([
|
||||
api.me(),
|
||||
api.follows(enabled: true), // ← 从云端读取 follows
|
||||
api.alerts(limit: 50), // ← 从云端读取 alerts
|
||||
]);
|
||||
```
|
||||
|
||||
**问题详解:**
|
||||
- 移动端显示的 `last_checked_at` 来自云端 follows 表
|
||||
- 桌面端 `_upsert_cloud_follow` 只更新了 creator 基本信息,**没有更新 `last_checked_at`**
|
||||
- 所以云端看到的"最近检查"时间永远是旧的甚至为空
|
||||
- 用户无法判断哪个作者实际上被检查过
|
||||
|
||||
#### 4. Alert 创建没有去重
|
||||
|
||||
**文件位置:** `pixiv_app/core/gallery_service.py` 第 588-613 行
|
||||
|
||||
```python
|
||||
for work_id in work_ids:
|
||||
try:
|
||||
manager.client.create_alert({...})
|
||||
created += 1
|
||||
except Exception:
|
||||
continue
|
||||
```
|
||||
|
||||
**问题:**
|
||||
- 每次同步都会为同一个 work_id 创建新的 alert
|
||||
- 如果多次同步同一批作品,会产生大量重复提醒
|
||||
- 没有检查 "这个 work_id 是否已经发过 alert"
|
||||
|
||||
### 解决方案
|
||||
|
||||
#### 方案 A:修复 Alert 标题——传入作者名(推荐,最小改动)
|
||||
|
||||
**修改位置:**
|
||||
|
||||
| 文件 | 行号 | 修改内容 |
|
||||
|------|------|----------|
|
||||
| `pixiv_app/core/gallery_service.py` | 第 588 行 | `_create_cloud_pixiv_update_alerts` 增加 `artist_name` 参数 |
|
||||
| `pixiv_app/core/gallery_service.py` | 第 562-565 行 | `sync_followed_artists` 传入作者名 |
|
||||
| `pixiv_app/core/gallery_service.py` | 第 599 行 | 修改 title 使用作者名 |
|
||||
|
||||
**具体做法:**
|
||||
|
||||
在 `sync_followed_artists` 方法中,从 followed_artists 或 artists 表获取作者名:
|
||||
|
||||
```python
|
||||
# 第 527-538 行之间,获取 followed 列表时同时获取 name
|
||||
followed = library.connection.execute(
|
||||
"SELECT fa.artist_id, COALESCE(NULLIF(fa.name,''), NULLIF(ar.name,''), '') AS name "
|
||||
"FROM followed_artists fa "
|
||||
"LEFT JOIN artists ar ON ar.artist_id = fa.artist_id "
|
||||
"WHERE fa.enabled = 1 ORDER BY fa.artist_id"
|
||||
).fetchall()
|
||||
```
|
||||
|
||||
然后修改 `_create_cloud_pixiv_update_alerts` 签名:
|
||||
|
||||
```python
|
||||
def _create_cloud_pixiv_update_alerts(
|
||||
self, artist_id: int, work_ids: list[int], artist_name: str = ""
|
||||
) -> int:
|
||||
```
|
||||
|
||||
title 改为:
|
||||
|
||||
```python
|
||||
"title": f"Pixiv 作者 {artist_name or str(artist_id)} 更新了 {len(work_ids)} 个新作品"
|
||||
```
|
||||
|
||||
#### 方案 B:同步云端 follows 的检查状态
|
||||
|
||||
**修改位置:**
|
||||
|
||||
| 文件 | 行号 | 修改内容 |
|
||||
|------|------|----------|
|
||||
| `pixiv_app/core/gallery_service.py` | 第 556-559 行 | `mark_artist_synced` 之后同步更新云端 follows 的 `last_checked_at` |
|
||||
| `pixiv_app/core/cloud_sync.py` | 第 167-168 行 | `upsert_follow` 增加 `last_checked_at` 和 `latest_published_at` 字段 |
|
||||
| `pixiv_app/core/gallery_service.py` | 第 415-447 行 | `_upsert_cloud_follow` 增加更新时间戳参数 |
|
||||
|
||||
**具体做法:**
|
||||
1. 在 `_upsert_cloud_follow` 方法中增加 `last_checked_at` 和 `latest_published_at` 参数
|
||||
2. 每次同步完一个作者后,调用 `_upsert_cloud_follow` 更新云端状态
|
||||
3. 在云端 follows API 的 upsert 逻辑中接受并存储这些时间字段
|
||||
4. 移动端 `reminder_center_page.dart` 中 `_FollowTile` 就能显示正确的"最近检查"时间
|
||||
|
||||
#### 方案 C:Alert 去重 & 按作者聚合
|
||||
|
||||
**修改位置:**
|
||||
|
||||
| 文件 | 行号 | 修改内容 |
|
||||
|------|------|----------|
|
||||
| `pixiv_app/core/gallery_service.py` | 第 588-613 行 | 重构 `_create_cloud_pixiv_update_alerts` 逻辑 |
|
||||
| 后端 `sakura_cloud/api/v1/follows.py` | alert 创建路由 | 增加去重逻辑 |
|
||||
|
||||
**具体做法:**
|
||||
1. 将每个作者的多个新作品合并为**一条聚合 alert**,而非每个作品一条
|
||||
2. Alert 的 `source_id` 使用 `f"{artist_id}:{min(work_ids)}:{max(work_ids)}"` 生成唯一标识
|
||||
3. 创建 alert 前先检查是否存在相似 alert(同一作者 + 同一天内)
|
||||
4. 如果存在则更新而非新建
|
||||
5. title 格式改为:`"你关注的 {作者名} 更新了 {N} 个新作品"`
|
||||
|
||||
#### 方案 D:增加关注作者更新状态页面
|
||||
|
||||
**修改位置:**
|
||||
|
||||
| 文件 | 行号 | 修改内容 |
|
||||
|------|------|----------|
|
||||
| `sakura_mobile/lib/src/pages/reminder_center_page.dart` | 第 350-371 行 | 已关注作者列表增加"手动检查更新"按钮 |
|
||||
| 后端新增接口 | - | `POST /api/v1/follows/{follow_id}/check` 触发单作者更新检测 |
|
||||
|
||||
**具体做法:**
|
||||
1. 在已关注作者列表中为每个作者增加状态标识:
|
||||
- 🟢 最近 24h 内检查过
|
||||
- 🟡 24h-7d 内检查过
|
||||
- 🔴 超过 7 天未检查
|
||||
2. 增加"手动检查"按钮,调用后端触发单个作者的更新检测
|
||||
3. 检查完成后自动刷新提醒列表
|
||||
|
||||
---
|
||||
|
||||
## 总结优先级
|
||||
|
||||
| 优先级 | 问题 | 推荐方案 | 改动范围 | 预期效果 |
|
||||
|--------|------|----------|----------|----------|
|
||||
| P0 紧急 | 跳转到 dakura 用户 | 方案 A:Token 绑定用户名 | 移动端 session.dart | 根除串号问题 |
|
||||
| P0 紧急 | Alert 显示数字而非作者名 | 方案 A:传入作者名 | gallery_service.py | 提醒可读 |
|
||||
| P1 高 | 发现页加载慢 | 方案 A + B:渐进加载 + feed_cache | feed_tab.dart + 后端 | 首屏从 3-5s 降到 <1s |
|
||||
| P1 高 | 关注作者未检测更新 | 方案 B:同步检查状态到云端 | gallery_service.py + cloud_sync.py | 状态准确可追溯 |
|
||||
| P2 中 | Alert 重复创建 | 方案 C:去重 + 聚合 | gallery_service.py + 后端 | 减少重复提醒 |
|
||||
| P2 中 | 查询性能优化 | 方案 D:SQL 索引 | 后端数据库 | 查询耗时降低 50%+ |
|
||||
@@ -0,0 +1,738 @@
|
||||
# Legado 规则引擎重构 — 分阶段实施方案
|
||||
|
||||
> 日期:2026-06-21
|
||||
> 基于:源码架构总结 + 截图问题诊断 + 方案文档反馈
|
||||
> 涉及:`legado_rule_parser.dart`、`legado_engine.dart`、`legado_models.dart`
|
||||
> 原则:不改 UI 层,只在 core 层动刀;前 4 步零新依赖。
|
||||
|
||||
---
|
||||
|
||||
## 一、问题根因定性与实施顺序纠偏
|
||||
|
||||
### 1.1 截图 Bug ↔ 真实根因对照
|
||||
|
||||
| 截图 | 现象 | 代码层面的真实原因 |
|
||||
|------|------|-------------------|
|
||||
| 截图3 | "这一章还没有找到漫画页面" | `fetchChapterContent()` → `contentRule` → `RuleParser` 只认 CSS/JSON/Regex 三种前缀,**包子漫画5 的 `contentRule` 使用了 `@tag.img@src` / `##` / `&&` 等链式语法**,`_safeSelect()` 返回空串,`rawHtml` 兜底也没触发。 |
|
||||
| 截图4 | "未知作者" / 封面空白 | `bookInfoAuthorRule` / `bookInfoCoverRule` 同上,规则语法不被 `RuleParser` 支持。 |
|
||||
| 截图1 | "书籍详情加载失败" | `fetchBookInfo()` 整体 try/catch 后只设 `infoError`,不区分"网络不通"还是"规则不支持"。 |
|
||||
| 搜索 `unknown:none` | 错误消息原样展示 | `_shortError()` 已修好(源码已含完整中文映射),这个 bug 已不存在于当前源码。 |
|
||||
|
||||
### 1.2 原方案步骤的纠偏
|
||||
|
||||
原方案第 1 步是「支持 URL,{method,body,headers}」,但实际上:
|
||||
|
||||
- `_buildSearchRequest()` 已有 GET/POST 分支
|
||||
- `_applySearchTemplate()` 已支持 12 种占位符变体
|
||||
- `fromJson` 解析 `header` 已支持 Map / JSON 字符串 / 冒号分隔行三种格式
|
||||
|
||||
**请求层不是最优先的问题。最优先的是 RuleParser 不支持 Legado 链式规则语法。**
|
||||
|
||||
### 1.3 修正后的实施顺序
|
||||
|
||||
```
|
||||
第1步:rawHtml 兜底扫描 + script 变量扫描 ← 零成本,立刻修复截图3
|
||||
第2步:RuleParser 支持 @tag/@text/@html/@src 链式语法 ← 解锁包子漫画5等核心书源
|
||||
第3步:支持 ## 替换、|| 备选、&& 拼接 ← 兼容更多规则变体
|
||||
第4步:fetchBookInfo / fetchToc 失败时区分错误类型 ← 解决截图1
|
||||
第5步:JS 特判(newImgs / imageList 等高频变量) ← 覆盖 60%+ JS 规则,零依赖
|
||||
第6步:@put / @get 上下文变量 ← 跨规则传值
|
||||
第7步:书源兼容度检测 UI ← 用户体验闭环
|
||||
第8步:完整 JS 引擎(flutter_js / quickjs) ← 最后做,覆盖剩余 40%
|
||||
```
|
||||
|
||||
**前 4 步不需要引入任何新依赖,只在 `legado_rule_parser.dart` 和 `legado_engine.dart` 里改。**
|
||||
|
||||
---
|
||||
|
||||
## 二、总体架构重构方案
|
||||
|
||||
### 2.1 不要继续往 RuleParser 里塞逻辑
|
||||
|
||||
当前 `RuleParser`(262 行)已经承担了 CSS / JSONPath / Regex 三种解析路径,再往里面加链式语法会导致它变成无法维护的千行怪物。
|
||||
|
||||
**建议拆成三层**:
|
||||
|
||||
```
|
||||
请求层 ── LegadoRequestBuilder ── 构建 URL、headers、body、编码
|
||||
规则层 ── LegadoRuleExecutor ── 链式规则执行引擎
|
||||
检测层 ── SourceCompatChecker ── 导入时打兼容度标签
|
||||
```
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────┐
|
||||
│ LegadoEngine │
|
||||
│ (对外 API:search / fetchBookInfo / fetchToc / │
|
||||
│ fetchChapterContent 不变) │
|
||||
├──────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────────────┐ ┌─────────────────────┐ │
|
||||
│ │LegadoRequestBuilder│ │ LegadoRuleExecutor │ │
|
||||
│ │· buildSearchUrl │ │· select() │ │
|
||||
│ │· buildRequest │ │· selectList() │ │
|
||||
│ │· encodeKeyword │ │· execChain() 🆕 │ │
|
||||
│ │· applyTemplate │ │· applyReplace() 🆕 │ │
|
||||
│ └──────────────────┘ │· extractImages() 🆕 │ │
|
||||
│ │· scanScriptVars() 🆕 │ │
|
||||
│ └──────────┬──────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────▼──────────┐ │
|
||||
│ │ RuleParser (保留) │ │
|
||||
│ │ CSS / JSON / Regex │ │
|
||||
│ │ 不改内部,只当底层 │ │
|
||||
│ └─────────────────────┘ │
|
||||
└──────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2.2 核心原则
|
||||
|
||||
1. **LegadoEngine 对外 API 不动**。`search()` / `fetchBookInfo()` / `fetchToc()` / `fetchChapterContent()` 签名不变,UI 层零改动。
|
||||
|
||||
2. **RuleParser 降级为底层原子工具**。它继续支持 CSS / JSONPath / Regex 三种基础解析,但不理解链式语法。所有链式逻辑放在 `LegadoRuleExecutor` 里。
|
||||
|
||||
3. **新规则执行器按优先级链执行**。每条规则被解析为一条 **执行链 (execChain)**,按序执行,任意一步返回空就终止并返回空。
|
||||
|
||||
---
|
||||
|
||||
## 三、第 1 步:rawHtml 兜底扫描 + script 变量扫描
|
||||
|
||||
> 目标:不改 RuleParser,在 `fetchChapterContent()` 和 `fetchImageUrls()` 里补兜底逻辑。
|
||||
|
||||
### 3.1 问题
|
||||
|
||||
当 `contentRule` 为空或解析失败时,当前代码回到 `_htmlToText(body)` 或 `_extractImageUrls(body)` 暴力扫描 HTML。但如果漫画图片 URL 藏在 `<script>` 的 JS 变量里(如 `var newImgs = ["url1","url2"]`),暴力扫描 `<img>` 标签找不到任何东西。
|
||||
|
||||
### 3.2 修改位置
|
||||
|
||||
`legado_engine.dart` 中的 `fetchChapterContent()` 和 `_extractImageUrls()`。
|
||||
|
||||
### 3.3 逻辑
|
||||
|
||||
```
|
||||
extractImagesWithFallback(String rawHtml, String baseUrl):
|
||||
|
||||
1. 先用 RuleParser 走 contentRule → 语法错误或结果为空 → 进入兜底
|
||||
|
||||
2. 兜底 1:扫描 HTML img 标签
|
||||
RegExp: <img[^>]+(?:src|data-src|data-original|data-srcset)=["']([^"']+)["']
|
||||
同时提取 srcset 中的备选 URL
|
||||
|
||||
3. 兜底 2:扫描常见 JS 变量
|
||||
按优先级依次尝试:
|
||||
newImgs = [...]
|
||||
imageList = [...]
|
||||
chapterImages = [...]
|
||||
page_url = ...
|
||||
pics = [...]
|
||||
imgData = [...]
|
||||
chapterData.images = [...]
|
||||
km5_img_url = ...
|
||||
result.imageList = ...
|
||||
|
||||
提取方式:
|
||||
RegExp: var\s+变量名\s*=\s*(\[[^\]]*\])
|
||||
RegExp: var\s+变量名\s*=\s*['"]([^'"]+)['"]
|
||||
如果是 JSON 数组 → jsonDecode
|
||||
如果是逗号分隔 → split(',').map(trim)
|
||||
|
||||
4. 兜底 3:扫描 data:image 内嵌图片
|
||||
|
||||
5. 全部 URL resolve 为绝对路径
|
||||
```
|
||||
|
||||
### 3.4 改动量
|
||||
|
||||
- `_extractImageUrls()` 新增 ~40 行
|
||||
- 不影响现有 `RuleParser` 任何逻辑
|
||||
- 无需新依赖
|
||||
|
||||
### 3.5 预期效果
|
||||
|
||||
截图 3 「这一章还没有找到漫画页面」在包子漫画 5 上直接消除(即使 `contentRule` 语法不被支持,script 变量扫描也能抓到 `newImgs`)。
|
||||
|
||||
---
|
||||
|
||||
## 四、第 2 步:RuleParser 支持 @tag/@text/@html/@src 链式语法
|
||||
|
||||
> 这是最核心的一步。包子漫画 5、6漫画、WebToon 等主流漫画源全部依赖这种语法。
|
||||
|
||||
### 4.1 Legado 链式规则语法规范
|
||||
|
||||
Legado 规则的本质是**管道链式操作**,每一步的输出是下一步的输入。语法格式:
|
||||
|
||||
```
|
||||
基础选择器@操作.子选择器@属性
|
||||
```
|
||||
|
||||
**链式管道符**:
|
||||
|
||||
| 符号 | 含义 | 示例 |
|
||||
|------|------|------|
|
||||
| `@tag.xxx` | 在当前结果内找第一个 `<xxx>` 子元素 | `@tag.img` → 找 img 标签 |
|
||||
| `@text` | 提取当前元素的 innerText | `.title@text` |
|
||||
| `@html` | 提取当前元素的 innerHtml | `.content@html` |
|
||||
| `@src` | 提取 src 属性值 | `img@src` |
|
||||
| `@href` | 提取 href 属性值 | `a@href` |
|
||||
| `@class` | 提取 class 属性值 | `div@class` |
|
||||
| `@alt` / `@title` | 提取 alt / title 属性 | `img@alt` |
|
||||
| `@data-xxx` | 提取 data-xxx 属性值 | `img@data-original` |
|
||||
| `@css:.xxx` | 用 CSS 选择器在当前结果内再次选择 | `div.content@css:.chapter@text` |
|
||||
|
||||
**管道执行顺序**:
|
||||
|
||||
```
|
||||
原始 HTML
|
||||
→ CSS 选择器(第一段)→ 得到匹配元素集合
|
||||
→ @tag.img → 在每个元素内找第一个 img
|
||||
→ @src → 提取 src 属性值
|
||||
→ 返回字符串列表
|
||||
```
|
||||
|
||||
### 4.2 实现方式:在 RuleParser 内部新增链式分支
|
||||
|
||||
`select()` 和 `selectList()` 方法检测规则中是否包含 `@tag` / `@text` / `@html` / `@src` 等链式操作符。如果有,走新分支 `_selectChain()`。
|
||||
|
||||
```dart
|
||||
List<String> _selectChain(String fragment, String rule) {
|
||||
// 1. 解析 rule 为执行步列表
|
||||
final steps = _parseChainSteps(rule);
|
||||
// 2. 从 fragment 开始,逐步执行
|
||||
var current = <String>[fragment];
|
||||
for (final step in steps) {
|
||||
current = _executeChainStep(current, step);
|
||||
if (current.isEmpty) return const [];
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
List<_ChainStep> _parseChainSteps(String rule) {
|
||||
// 规则示例: ".book-list li@tag.a@href"
|
||||
// 拆分为: [('.book-list li', css), ('a', tag), ('href', attr)]
|
||||
//
|
||||
// 关键:@ 是管道分隔符,不是属性选择器的一部分
|
||||
// "@tag.xxx" 是一个整体 token
|
||||
// "@src" "@href" "@text" "@html" 是属性提取 token
|
||||
// "@css:.xxx" 是嵌套 CSS token
|
||||
}
|
||||
|
||||
List<String> _executeChainStep(List<String> fragments, _ChainStep step) {
|
||||
switch (step.type) {
|
||||
case _StepType.css:
|
||||
return _cssSelect(fragments, step.value); // 降级到现有 CSS 路径
|
||||
case _StepType.tag:
|
||||
return _tagSelect(fragments, step.value); // fragments 中每个元素内找第一个 <tag>
|
||||
case _StepType.attr:
|
||||
return _attrExtract(fragments, step.value); // 提取属性值
|
||||
case _StepType.text:
|
||||
return _textExtract(fragments); // 提取 innerText
|
||||
case _StepType.html:
|
||||
return _htmlExtract(fragments); // 提取 innerHtml
|
||||
case _StepType.nestedCss:
|
||||
return _nestedCssSelect(fragments, step.value); // @css: 语法
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4.3 链式步骤解析规则
|
||||
|
||||
```
|
||||
规则字符串:
|
||||
"div.image-area@tag.img@src"
|
||||
|
||||
解析产物:
|
||||
[
|
||||
{ type: css, value: "div.image-area" }, // @ 之前的部分 → CSS 选择器
|
||||
{ type: tag, value: "img" }, // @tag.img → 找 img 子标签
|
||||
{ type: attr, value: "src" }, // @src → 提取 src 属性
|
||||
]
|
||||
|
||||
规则字符串:
|
||||
"h1.title@text||h1.book-name@text"
|
||||
|
||||
解析产物(备选规则):
|
||||
[
|
||||
[ {css:"h1.title"}, {text} ], // 方案 A
|
||||
[ {css:"h1.book-name"}, {text} ], // 方案 B → 先执行 A,A 为空则执行 B
|
||||
]
|
||||
|
||||
规则字符串:
|
||||
".chapter-content@html##去除广告正则##"
|
||||
|
||||
解析产物(带替换):
|
||||
[
|
||||
{ type: css, value: ".chapter-content" },
|
||||
{ type: html, value: null },
|
||||
{ type: regexReplace, value: "去除广告正则" },
|
||||
]
|
||||
```
|
||||
|
||||
### 4.4 改动量
|
||||
|
||||
- `legado_rule_parser.dart` 新增 `_selectChain()` 方法 ~120 行
|
||||
- `_parseChainSteps()` ~40 行
|
||||
- 辅助方法 `_tagSelect()` / `_attrExtract()` / `_textExtract()` / `_htmlExtract()` ~80 行
|
||||
- `select()` 和 `selectList()` 入口处新增链式检测分支 ~10 行
|
||||
- 总计 ~250 行
|
||||
- **零新依赖**(tagSelect / attrExtract 用现有 html parser 或 RegExp 实现)
|
||||
|
||||
### 4.5 预期效果
|
||||
|
||||
包子漫画 5 的 `contentRule: "div.comic-content@tag.img@src"` 能正确执行:
|
||||
1. CSS `div.comic-content` → 拿到内容区 DOM
|
||||
2. `@tag.img` → 在内容区找所有 img 标签
|
||||
3. `@src` → 提取每个 img 的 src
|
||||
4. 返回图片 URL 列表
|
||||
|
||||
---
|
||||
|
||||
## 五、第 3 步:支持 ## 替换、|| 备选、&& 拼接
|
||||
|
||||
### 5.1 语法定义
|
||||
|
||||
| 符号 | 含义 | 示例 |
|
||||
|------|------|------|
|
||||
| `\|\|` | 备选规则:第一个成功就返回,否则尝试下一个 | `.title@text\|\|h1@text` |
|
||||
| `&&` | 串联拼接:两个结果用空格或换行拼接 | `.title@text&&.author@text` |
|
||||
| `##` | 正则替换/净化:对当前结果执行正则替换 | `@text##\s+\|\|##g` |
|
||||
| `###` | 正则删除:删除匹配的部分 | `@html###<script[^>]*>.*?</script>###` |
|
||||
|
||||
### 5.2 实现位置
|
||||
|
||||
`LegadoRuleExecutor` 的 `execChain()` 方法。
|
||||
|
||||
### 5.3 执行顺序
|
||||
|
||||
```
|
||||
解析 rawRule:
|
||||
1. 按 || 拆分为备选组
|
||||
2. 每个备选组按 && 拆分为串联组
|
||||
3. 每个串联组解析为链式步骤(复用第2步的 _parseChainSteps)
|
||||
4. 每个链式步骤末尾检测 ## 替换指令
|
||||
|
||||
执行:
|
||||
for 备选组 in 备选组列表:
|
||||
结果 = 执行 备选组
|
||||
if 结果非空 → 返回结果
|
||||
return 空
|
||||
|
||||
执行备选组:
|
||||
parts = []
|
||||
for 串联组 in 备选组:
|
||||
part = 执行串联组
|
||||
parts.add(part)
|
||||
return parts.join('\n') // && 的结果用换行拼接
|
||||
|
||||
执行串联组:
|
||||
result = 执行链式步骤(复用 _selectChain)
|
||||
if 有 ## 替换指令:
|
||||
for 替换 in 替换指令:
|
||||
result = result.replaceAll(替换.regex, 替换.replacement)
|
||||
return result
|
||||
```
|
||||
|
||||
### 5.4 改动量
|
||||
|
||||
- `legado_rule_parser.dart` 新增 `_selectWithAlternatives()` ~50 行
|
||||
- `##` 替换逻辑 ~30 行
|
||||
- 总计 ~80 行
|
||||
- **零新依赖**
|
||||
|
||||
---
|
||||
|
||||
## 六、第 4 步:fetchBookInfo / fetchToc 失败时区分错误类型
|
||||
|
||||
### 6.1 问题
|
||||
|
||||
当前 `fetchBookInfo()` 的 try/catch 结构:
|
||||
|
||||
```dart
|
||||
try {
|
||||
final value = await engine.fetchBookInfo(...);
|
||||
setState(() { info = value; loadingInfo = false; });
|
||||
} catch (error) {
|
||||
setState(() { infoError = error; loadingInfo = false; }); // 所有错误一视同仁
|
||||
}
|
||||
```
|
||||
|
||||
用户看不到是"网络不通"还是"规则不支持",只能看到统一的"书籍详情加载失败"。
|
||||
|
||||
### 6.2 方案:定义错误类型,不新增异常类
|
||||
|
||||
```dart
|
||||
enum LoadFailureReason {
|
||||
network, // 网络不通 / DNS 失败 / 超时
|
||||
httpError, // HTTP 4xx / 5xx
|
||||
ruleNotSupported, // 规则语法不被当前引擎支持
|
||||
ruleParseEmpty, // 规则执行成功但结果为空(可能是网站改版)
|
||||
encoding, // 编码问题
|
||||
unknown,
|
||||
}
|
||||
```
|
||||
|
||||
在 `fetchBookInfo()` 和 `fetchToc()` 内部区分异常类型,通过 `_shortError()` 返回带 reason 的错误消息(或通过一个简单的包装类)。
|
||||
|
||||
### 6.3 UI 差异化展示
|
||||
|
||||
```
|
||||
网络不通:
|
||||
"无法连接到目标网站,请检查网络或稍后重试。"
|
||||
[重试]
|
||||
|
||||
HTTP 404:
|
||||
"书源详情页已失效(404),该书源可能需要更新。"
|
||||
[更换书源]
|
||||
|
||||
规则不支持:
|
||||
"该书源的详情解析规则包含不被当前版本支持的语法(如 @js),更新 APP 后可能可用。"
|
||||
[查看原始页面]
|
||||
|
||||
规则解析为空:
|
||||
"目标网站页面结构已变化,书源规则可能已过时。"
|
||||
[重试] [更换书源]
|
||||
```
|
||||
|
||||
### 6.4 改动量
|
||||
|
||||
- `legado_engine.dart` 新增 `_classifyError()` 方法 ~30 行
|
||||
- `_shortError()` 新增 reason 字段 ~10 行
|
||||
- `online_book_detail_page.dart` 根据 reason 展示不同的 UI ~30 行
|
||||
- 总计 ~70 行
|
||||
- **零新依赖**
|
||||
|
||||
---
|
||||
|
||||
## 七、第 5 步:JS 特判(零依赖)
|
||||
|
||||
### 7.1 问题
|
||||
|
||||
很多漫画源的 `contentRule` 直接写 JS:
|
||||
|
||||
```json
|
||||
"content": "<js>\nvar result = [];\nvar imgs = JSON.parse...\nnewImgs.map(...)\nresult.join('\\n')</js>"
|
||||
```
|
||||
|
||||
或者写在 `ruleContent.content` 里用 `@js:` 前缀。
|
||||
|
||||
当前引擎对 `@js:` 和 `<js>` 完全无处理能力,规则执行到这一步直接抛异常。
|
||||
|
||||
### 7.2 第一阶段的 JS 特判策略
|
||||
|
||||
不引入完整 JS 引擎。只做模式匹配,覆盖最常见的漫画图片提取模式:
|
||||
|
||||
| 模式 | 真实 JS 代码 | 特判正则 |
|
||||
|------|-------------|---------|
|
||||
| `newImgs.map` | `newImgs.map(i => i.url)` | 先扫描 rawHtml 中的 `var newImgs = [...]` |
|
||||
| `imageList` | `imageList[0].src` | 扫描 `var imageList = [...]` |
|
||||
| `page_url` | `page_url + img` | 扫描 `var page_url = "..."` |
|
||||
| `chapterImages` | `chapterImages.join` | 扫描 `var chapterImages = [...]` |
|
||||
| `km5_img_url` | `km5_img_url` | 扫描 `var km5_img_url = "..."` |
|
||||
| `base64Images` | 从 base64 解码 | 检测 `data:image` 前缀 |
|
||||
| `eval(decrypt(...))` | 简单解密 | 正则提取被 eval 的字符串 |
|
||||
|
||||
### 7.3 实现方式
|
||||
|
||||
```dart
|
||||
List<String> _jsFallbackImages(String rawHtml, String baseUrl) {
|
||||
final patterns = [
|
||||
// 1. 数组变量
|
||||
_JsVarPattern('newImgs', arrayPattern: true),
|
||||
_JsVarPattern('imageList', arrayPattern: true),
|
||||
_JsVarPattern('chapterImages', arrayPattern: true),
|
||||
_JsVarPattern('pics', arrayPattern: true),
|
||||
_JsVarPattern('imgList', arrayPattern: true),
|
||||
_JsVarPattern('pageImgs', arrayPattern: true),
|
||||
|
||||
// 2. 字符串拼接变量
|
||||
_JsVarPattern('page_url', arrayPattern: false),
|
||||
_JsVarPattern('imgDomain', arrayPattern: false),
|
||||
|
||||
// 3. data:image 内嵌
|
||||
_JsVarPattern('data:image', base64Pattern: true),
|
||||
];
|
||||
|
||||
for (final pattern in patterns) {
|
||||
final urls = _tryExtractJsVar(rawHtml, pattern);
|
||||
if (urls.isNotEmpty) return urls.map((u) => resolveUrl(baseUrl, u)).toList();
|
||||
}
|
||||
return const [];
|
||||
}
|
||||
```
|
||||
|
||||
### 7.4 改动量
|
||||
|
||||
- `legado_engine.dart` 新增 `_jsFallbackImages()` ~60 行
|
||||
- `_tryExtractJsVar()` ~40 行
|
||||
- 总计 ~100 行
|
||||
- **零新依赖**
|
||||
|
||||
---
|
||||
|
||||
## 八、第 6 步:@put / @get 上下文变量
|
||||
|
||||
### 8.1 用途
|
||||
|
||||
Legado 高级规则中可以在同一次执行中跨步骤传值:
|
||||
|
||||
```
|
||||
@put:{bookId:$.id}
|
||||
// ... 在后续规则中使用 ...
|
||||
/api/book/{{bookId}}/chapters
|
||||
```
|
||||
|
||||
### 8.2 实现方式
|
||||
|
||||
在 `LegadoRuleExecutor` 中维护一个 `Map<String, String> _context`。
|
||||
|
||||
```dart
|
||||
class LegadoRuleExecutor {
|
||||
final Map<String, String> _context = {};
|
||||
|
||||
// 执行 @put:{key:value} 时写入 _context
|
||||
// 解析 URL 模板时用 _context 替换 {{key}}
|
||||
// @get:key → 从 _context 读取
|
||||
}
|
||||
```
|
||||
|
||||
### 8.3 改动量
|
||||
|
||||
- `legado_rule_parser.dart` 新增 `_context` Map ~5 行
|
||||
- `@put` / `@get` 语法解析 ~30 行
|
||||
- `_applySearchTemplate()` 新增 `{{$.key}}` 占位符支持 ~10 行
|
||||
- 总计 ~45 行
|
||||
- **零新依赖**
|
||||
|
||||
---
|
||||
|
||||
## 九、第 7 步:书源兼容度检测
|
||||
|
||||
### 9.1 目标
|
||||
|
||||
导入书源后,自动分析每个书源用到的规则语法,打上兼容度标签,在书源管理 UI 中展示。
|
||||
|
||||
### 9.2 兼容度标签定义
|
||||
|
||||
```dart
|
||||
enum SourceCompatLevel {
|
||||
full, // 完全兼容:所有规则语法已被引擎支持
|
||||
partial, // 部分兼容:搜索可用,但正文/详情可能因缺少 JS 执行而不完整
|
||||
needsJs, // 需要 JS:contentRule 或 bookInfoRule 依赖 @js 或 <js>
|
||||
needsPost, // 需要 POST 搜索:searchMethod == POST 或 URL 包含 ,{
|
||||
needsHeaders, // 需要自定义请求头
|
||||
broken, // 已失效:基础 URL 不可达
|
||||
unknown, // 尚未验证
|
||||
}
|
||||
|
||||
class SourceCompatInfo {
|
||||
final SourceCompatLevel level;
|
||||
final bool searchOk; // 搜索规则可执行
|
||||
final bool bookInfoOk; // 详情规则可执行
|
||||
final bool tocOk; // 目录规则可执行
|
||||
final bool contentOk; // 正文规则可执行
|
||||
final bool hasJsRule; // 规则中包含 @js / <js>
|
||||
final bool hasPutGet; // 规则中包含 @put / @get
|
||||
final List<String> warnings; // 警告信息列表
|
||||
}
|
||||
```
|
||||
|
||||
### 9.3 检测逻辑(导入时运行)
|
||||
|
||||
```dart
|
||||
SourceCompatInfo analyzeSource(LegadoSource source) {
|
||||
final warnings = <String>[];
|
||||
bool hasJs = false;
|
||||
bool hasPutGet = false;
|
||||
bool hasChainSyntax = false;
|
||||
|
||||
// 1. 检查所有规则字段是否包含 JS 语法
|
||||
for (final rule in _allRuleFields(source)) {
|
||||
if (rule == null) continue;
|
||||
if (rule.contains('@js') || rule.contains('<js>')) hasJs = true;
|
||||
if (rule.contains('@put') || rule.contains('@get')) hasPutGet = true;
|
||||
if (rule.contains('@tag.') || rule.contains('@src') ||
|
||||
rule.contains('@text') || rule.contains('@html')) {
|
||||
hasChainSyntax = true;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 评估各模块可行性
|
||||
final searchOk = source.searchUrl != null && source.searchListRule != null;
|
||||
final bookInfoOk = source.bookInfoTocRule != null || source.bookInfoNameRule != null;
|
||||
final tocOk = source.tocListRule != null || source.tocNameRule != null;
|
||||
final contentOk = source.contentRule != null;
|
||||
|
||||
// 3. 确定级别
|
||||
SourceCompatLevel level;
|
||||
if (!searchOk && !contentOk) {
|
||||
level = SourceCompatLevel.broken;
|
||||
} else if (hasJs) {
|
||||
level = SourceCompatLevel.needsJs;
|
||||
} else if (hasPutGet) {
|
||||
level = SourceCompatLevel.partial;
|
||||
} else if (!bookInfoOk && !contentOk) {
|
||||
level = SourceCompatLevel.partial; // 只能搜
|
||||
} else {
|
||||
level = SourceCompatLevel.full;
|
||||
}
|
||||
|
||||
return SourceCompatInfo(level: level, ...);
|
||||
}
|
||||
```
|
||||
|
||||
### 9.4 UI 展示
|
||||
|
||||
在 `source_manager_page.dart` 的书源列表中,每行右侧显示兼容度标签:
|
||||
|
||||
```
|
||||
包子漫画5 [🔴 需要 JS]
|
||||
6漫画 [🔴 需要 JS]
|
||||
全本玄幻小说 [🟢 完全兼容]
|
||||
WebToon [🟡 需要 POST]
|
||||
某已失效源 [⚫ 已失效]
|
||||
```
|
||||
|
||||
### 9.5 改动量
|
||||
|
||||
- `legado_models.dart` 新增 `SourceCompatInfo` 类 ~30 行
|
||||
- `source_manager.dart` 新增 `analyzeSource()` ~50 行
|
||||
- `source_manager_page.dart` UI 展示 ~40 行
|
||||
- 总计 ~120 行
|
||||
- **零新依赖**
|
||||
|
||||
---
|
||||
|
||||
## 十、第 8 步:完整 JS 引擎
|
||||
|
||||
### 10.1 时机
|
||||
|
||||
前 7 步完成后,已有书源中约 60-70% 的漫画源可正常阅读(通过 JS 特判覆盖)。剩下的 30-40% 需要完整 JS 执行环境。
|
||||
|
||||
### 10.2 推荐方案
|
||||
|
||||
| 方案 | 优点 | 缺点 |
|
||||
|------|------|------|
|
||||
| **flutter_js** | 纯 Dart,不依赖 WebView | 性能一般,JS 互操作受限 |
|
||||
| **quickjs_dart_bridge** | 轻量、快、支持 ES2020 | 需要 Native 桥接 |
|
||||
| **flutter_js_engine** | 基于 JavaScriptCore/QuickJS | 体积较大 |
|
||||
| **WebView 隐藏执行** | 兼容性最好 | 太重,不适合批量 |
|
||||
|
||||
**建议**:先用 **quickjs_dart_bridge**(已有成功集成到阅读器的先例)。
|
||||
|
||||
### 10.3 注入的 JS 运行环境
|
||||
|
||||
```js
|
||||
// 注入变量
|
||||
result; // 当前 HTML 或 RuleParser 结果
|
||||
baseUrl; // 当前请求的 base URL
|
||||
|
||||
// 注入函数(兼容 Legado 的 java.xxx 调用风格)
|
||||
java.get(key); // → 从 _context 读
|
||||
java.put(key, value); // → 写入 _context
|
||||
java.getElements(html, selector); // → querySelectorAll → 返回元素列表
|
||||
java.base64Decode(str); // → base64 解码
|
||||
java.ajax(url); // → HTTP GET → 返回响应文本
|
||||
|
||||
// 注入输出收集器
|
||||
__result; // 脚本执行完毕后读取此变量作为输出
|
||||
```
|
||||
|
||||
### 10.4 安全措施
|
||||
|
||||
- JS 沙箱:禁止 `XMLHttpRequest`、`fetch`、`localStorage`
|
||||
- 超时限制:每条 JS 规则最多执行 3 秒
|
||||
- 错误隔离:单条 JS 规则失败不影响其他规则
|
||||
- Isolate 执行:防卡 UI(可选,复杂度较高)
|
||||
|
||||
### 10.5 改动量
|
||||
|
||||
- `pubspec.yaml` 新增 `quickjs_dart_bridge` 依赖
|
||||
- 新建 `legado_js_runner.dart` ~150 行
|
||||
- `LegadoRuleExecutor` 集成 JS 执行 ~30 行
|
||||
- 总计 ~180 行
|
||||
|
||||
---
|
||||
|
||||
## 十一、完整实施路线图
|
||||
|
||||
```
|
||||
Week 1 ████████░░░░░░░░░░░░
|
||||
第1步: rawHtml 兜底 + script 变量扫描
|
||||
第2步: @tag/@text/@html/@src 链式语法
|
||||
|
||||
Week 2 ████████░░░░░░░░░░░░
|
||||
第3步: ##/||/&& 语法
|
||||
第4步: 错误类型区分
|
||||
|
||||
Week 3 ████████████████░░░░
|
||||
第5步: JS 特判
|
||||
第6步: @put/@get
|
||||
第7步: 书源兼容度检测
|
||||
|
||||
Week 4+ ████████████████████
|
||||
第8步: 完整 JS 引擎
|
||||
测试: 30+ 书源的回归测试
|
||||
|
||||
零新依赖 新依赖
|
||||
├───────────────────┼──────────┤
|
||||
第1步 第8步
|
||||
第2步
|
||||
第3步
|
||||
第4步
|
||||
第5步
|
||||
第6步
|
||||
第7步
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 十二、改动文件清单
|
||||
|
||||
| 文件 | 第1步 | 第2步 | 第3步 | 第4步 | 第5步 | 第6步 | 第7步 | 第8步 |
|
||||
|------|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|:-----:|
|
||||
| `legado_rule_parser.dart` | | 🔧250 | 🔧80 | | | 🔧45 | | |
|
||||
| `legado_engine.dart` | 🔧40 | | | 🔧70 | 🔧100 | 🔧10 | | 🔧30 |
|
||||
| `legado_models.dart` | | | | | | | 🔧30 | |
|
||||
| `source_manager.dart` | | | | | | | 🔧50 | |
|
||||
| `source_manager_page.dart` | | | | | | | 🔧40 | |
|
||||
| `online_book_detail_page.dart` | | | | 🔧30 | | | | |
|
||||
| `legado_js_runner.dart` 🆕 | | | | | | | | 🆕150 |
|
||||
| `pubspec.yaml` | | | | | | | | +1dep |
|
||||
|
||||
---
|
||||
|
||||
## 十三、总结
|
||||
|
||||
整个方案的核心理念是:
|
||||
|
||||
```
|
||||
RuleParser 是对现有 CSS/JSON/Regex 的兼容 → 保留为底层原子工具
|
||||
LegadoRuleExecutor 是新增的链式规则执行层 → 理解 @tag、##、||、&&
|
||||
JS 分两级做 → 特判覆盖 60% + 引擎覆盖 40%
|
||||
书源兼容度检测 → 导入时打标签,用户体验闭环
|
||||
|
||||
请求层不动 —— 它没问题
|
||||
Bug 优先级由截图驱动 —— 第1步修截图3,第2步解锁主流漫画源,第4步修截图1
|
||||
前4步零新依赖,直接在现有文件里改
|
||||
```
|
||||
|
||||
**一句话**:不推翻现有代码,在 `RuleParser` 之上加一层**链式规则执行器**,让 `@tag.img@src` 和 `##替换` 能被正确理解,这是解锁包子漫画 5 等主流漫画源的关键。
|
||||
|
||||
---
|
||||
|
||||
## 附录:All.json 常见规则语法统计
|
||||
|
||||
基于对 30+ 个典型漫画/小说书源规则的手工分析:
|
||||
|
||||
| 语法 | 出现频率 | 典型场景 | 本方案覆盖 |
|
||||
|------|:--------:|---------|:----------:|
|
||||
| `@tag.xxx` | ★★★★★ | 漫画正文图片提取 | 第2步 |
|
||||
| `@src` | ★★★★★ | 图片 URL 提取 | 第2步 |
|
||||
| `@text` | ★★★★ | 书名/作者/章节名 | 第2步 |
|
||||
| `@html` | ★★★ | 正文 HTML 提取 | 第2步 |
|
||||
| `##` | ★★★★ | 广告/水印净化 | 第3步 |
|
||||
| `\|\|` | ★★★★ | 备选规则 | 第3步 |
|
||||
| `&&` | ★★★ | 多字段拼接 | 第3步 |
|
||||
| `<js>` / `@js:` | ★★★ | 漫画图片解密 | 第5/8步 |
|
||||
| `@put` / `@get` | ★★ | 跨步骤传递 ID | 第6步 |
|
||||
| `@css:` | ★★★ | 嵌套 CSS 选择 | 第2步 |
|
||||
| `@data-xxx` | ★★ | data-original 等 | 第2步 |
|
||||
| `,{method,body}` | ★★ | POST 搜索 | 请求层(现有) |
|
||||
File diff suppressed because it is too large.
Load diff
Binary file not shown.
@@ -0,0 +1,393 @@
|
||||
# 小说与漫画三级层级展示 —— 前端改造方案
|
||||
|
||||
## Context
|
||||
|
||||
当前 `sakura_admin_web` 的小说和漫画展示是 **2 级扁平结构**:列表页将同一部作品的章节折叠为一张"作品卡片",点击直接跳转到单章阅读器。用户无法浏览"一本书下面有哪些章节"这个中间层。
|
||||
|
||||
本方案将漫画和小说改为 **3 级层级导航**:
|
||||
|
||||
```
|
||||
Level 1 (系列列表) → Level 2 (章节列表) → Level 3 (内容阅读)
|
||||
```
|
||||
|
||||
## 后端 API 现状与数据模型
|
||||
|
||||
### Novel(小说)
|
||||
- **有专用 series API**:`GET /novels/series`(列表)、`GET /novels/series/{series_id}`(单系列含全部章节)
|
||||
- `NovelSeriesGroup` 包含:`series_id, series_title, author, chapter_count, cover_path, chapters[]`
|
||||
- 每个 chapter(`NovelOut`)有:`id, title, chapter_title, chapter_order, updated_at`
|
||||
- 当前前端调的是 `GET /me/novels`(扁平 rows),客户端自己 `groupNovels()`
|
||||
|
||||
### Manga/Comic(漫画)
|
||||
- **无专用 series API**,漫画存储为 `Artwork` 行(`content_type="manga"`)
|
||||
- 系列分组依赖 JSON 字段 `artwork_metadata.series_id` / `artwork_metadata.series_title`
|
||||
- 当前前端调用 `GET /me/gallery?content_type=manga`,客户端 `groupManga()` 分组
|
||||
- 需要在客户端缓存分组数据,供 Level 2 页面使用
|
||||
|
||||
---
|
||||
|
||||
## 关键设计决策(必读)
|
||||
|
||||
### 1. 漫画 seriesKey 稳定性规则
|
||||
|
||||
`seriesKey` 是 Level 1 跳 Level 2 的路由参数,也是缓存的索引,必须稳定且防冲突。
|
||||
|
||||
**构造规则(优先级从高到低):**
|
||||
|
||||
```
|
||||
seriesKey = series_id
|
||||
? `manga-${series_id}`
|
||||
: `manga-title-${series_title || title}`
|
||||
```
|
||||
|
||||
说明:
|
||||
- 优先用 `artwork_metadata.series_id`,它最稳定
|
||||
- `series_id` 缺失时用 `series_title || title` 回退,加 `manga-title-` 前缀避免与 `manga-` 冲突
|
||||
- **所有来源的值都要 `trim()`**
|
||||
- URL 传递时使用 `encodeURIComponent(seriesKey)`,接收端用 Vue Router 自动解码后的值
|
||||
|
||||
### 2. 小说 series_id 类型统一
|
||||
|
||||
后端可能返回数字或字符串类型的 `series_id`。**所有地方必须统一转成字符串:**
|
||||
|
||||
```
|
||||
seriesId: String(item.series_id)
|
||||
```
|
||||
|
||||
原因:Vue Router 的 `:seriesId` 参数总是字符串,如果缓存 key 或比较逻辑里混了数字类型,会导致匹配失败。
|
||||
|
||||
### 3. 漫画缓存缺失回退流程(明确)
|
||||
|
||||
当用户直接访问 `/app/manga/series/:seriesKey` 而缓存为空时,按以下步骤回退:
|
||||
|
||||
1. 调用 `GET /me/gallery?content_type=manga&limit=200`
|
||||
2. 对返回的 rows 执行与 `UserListPage.groupManga()` **完全一致的分组逻辑**
|
||||
3. 在所有分组结果中查找 `seriesKey` 匹配的组(匹配规则同构造规则)
|
||||
4. 找到后填充页面
|
||||
5. 未找到则显示错误:"未找到该系列的章节"
|
||||
|
||||
这样保证了不管是从 Level 1 正常导航进来,还是直接访问 URL,显示结果一致。
|
||||
|
||||
### 4. Level 3 返回逻辑的 fallback
|
||||
|
||||
阅读器中通过 `fromSeries` query 参数构建"返回章节列表"链接:
|
||||
|
||||
```
|
||||
if (fromSeries) → /app/${kind}/series/${fromSeries}
|
||||
else → /app/${kind} (返回系列列表)
|
||||
```
|
||||
|
||||
面包屑中间的"章节列表"链接同理——没有 `fromSeries` 时不渲染该级面包屑,直接从 Level 1 跳到 Level 3。
|
||||
|
||||
### 5. 路由顺序严格要求
|
||||
|
||||
`/app` 下的漫画和小说子路由必须按以下顺序注册(**顺序错会导致路由匹配错误**):
|
||||
|
||||
```
|
||||
1. manga ← Level 1
|
||||
2. manga/series/:seriesKey ← Level 2(固定路径段优先)
|
||||
3. manga/read/:id ← Level 3(固定路径段优先)
|
||||
4. manga/:id → redirect ← 旧路由兼容,放最后
|
||||
5. novels ← Level 1
|
||||
6. novels/series/:seriesId ← Level 2(固定路径段优先)
|
||||
7. novels/read/:id ← Level 3(固定路径段优先)
|
||||
8. novels/:id → redirect ← 旧路由兼容,放最后
|
||||
```
|
||||
|
||||
原理:`manga/series/:seriesKey` 有固定前缀 `series/`,如果放在 `manga/:id` 之后,Vue Router 会把 `series` 当成 `:id` 的值匹配进去。**固定路径段必须排在动态 `:id` 前面。**
|
||||
|
||||
### 6. 章节数据 normalize
|
||||
|
||||
不同来源的章节字段命名不一致(漫画在 `artwork_metadata` 里嵌套、小说是顶层字段),建议统一写一个 normalize 函数,不要在各处散着取字段。
|
||||
|
||||
**小说章节 normalize(输入:`NovelOut`):**
|
||||
|
||||
| 要取的字段 | 取值路径 |
|
||||
|-----------|---------|
|
||||
| `id` | `String(item.id)` |
|
||||
| `title` | `item.chapter_title \|\| item.title \|\| "未命名章节"` |
|
||||
| `order` | `Number(item.chapter_order \|\| 0)` |
|
||||
| `updated_at` | `item.updated_at \|\| ""` |
|
||||
|
||||
**漫画章节 normalize(输入:`ArtworkOut` 单行):**
|
||||
|
||||
| 要取的字段 | 取值路径 |
|
||||
|-----------|---------|
|
||||
| `id` | `String(item.id)` |
|
||||
| `title` | `item.title \|\| item.artwork_metadata?.chapter_title \|\| "未命名章节"` |
|
||||
| `order` | `Number(item.artwork_metadata?.chapter_order \|\| 0)` |
|
||||
| `page_count` | `Number(item.page_count \|\| 1)` |
|
||||
| `thumbnail_url` | `item.thumbnail_url \|\| ""` |
|
||||
| `updated_at` | `item.updated_at \|\| ""` |
|
||||
|
||||
在 `UserListPage.groupManga()` 和 `UserChapterListPage.loadMangaSeries()` 中都应使用同一套 normalize。
|
||||
|
||||
### 7. Level 2 的状态覆盖
|
||||
|
||||
`UserChapterListPage` 必须覆盖以下四种状态,否则直接访问 URL 时体验差:
|
||||
|
||||
| 状态 | 条件 | UI |
|
||||
|------|------|----|
|
||||
| **加载中** | `loading === true` | `"正在加载章节..."` |
|
||||
| **加载失败** | `error !== ""` | 显示错误信息,带"重试"按钮 |
|
||||
| **空章节** | `!loading && !error && chapters.length === 0` | `"暂无章节"` |
|
||||
| **缓存失效重载** | manga 缓存命空 → 自动触发 API 回退 | `"正在重新加载..."`(过渡态) |
|
||||
|
||||
---
|
||||
|
||||
## 路由改造
|
||||
|
||||
### 新路由结构
|
||||
|
||||
**文件:** `src/router/index.ts` —— 修改
|
||||
|
||||
```
|
||||
修改前(2 级) 修改后(3 级)
|
||||
───────────────── ─────────────────
|
||||
/app/novels /app/novels ← Level 1 系列列表
|
||||
/app/novels/:id /app/novels/series/:seriesId ← Level 2 章节列表(新)
|
||||
/app/novels/read/:id ← Level 3 阅读器(原 /novels/:id)
|
||||
|
||||
/app/manga /app/manga ← Level 1 系列列表
|
||||
/app/manga/:id /app/manga/series/:seriesKey ← Level 2 章节列表(新)
|
||||
/app/manga/read/:id ← Level 3 详情(原 /manga/:id)
|
||||
```
|
||||
|
||||
**路由注册顺序(严格):**
|
||||
|
||||
```ts
|
||||
// 漫画
|
||||
{ path: "manga", component: UserListPage, props: { kind: "manga" } },
|
||||
{ path: "manga/series/:seriesKey", component: UserChapterListPage, props: (route) => ({ kind: "manga", seriesKey: route.params.seriesKey }) },
|
||||
{ path: "manga/read/:id", component: UserDetailPage, props: (route) => ({ kind: "manga", id: route.params.id }) },
|
||||
// 小说
|
||||
{ path: "novels", component: UserListPage, props: { kind: "novels" } },
|
||||
{ path: "novels/series/:seriesId", component: UserChapterListPage, props: (route) => ({ kind: "novels", seriesId: route.params.seriesId }) },
|
||||
{ path: "novels/read/:id", component: UserDetailPage, props: (route) => ({ kind: "novels", id: route.params.id }) },
|
||||
// 旧路由重定向(必须放最后)
|
||||
{ path: "manga/:id", redirect: (to) => `/app/manga/read/${to.params.id}` },
|
||||
{ path: "novels/:id", redirect: (to) => `/app/novels/read/${to.params.id}` },
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 改造清单
|
||||
|
||||
### 一、新建文件
|
||||
|
||||
#### 1. `src/stores/seriesCache.ts` —— 漫画系列缓存
|
||||
|
||||
**为何需要:** 后端没有 manga series 端点。Level 1 页面做完客户端分组后,需要把分组数据传给 Level 2 页面。使用 Vue `reactive()` 对象做跨组件共享缓存。
|
||||
|
||||
```ts
|
||||
// 核心结构
|
||||
interface MangaChapterCacheEntry {
|
||||
id: string // gallery item id
|
||||
title: string // 章节标题(normalize 后)
|
||||
page_count: number // 页数
|
||||
thumbnail_url: string // 缩略图
|
||||
updated_at: string // 更新时间
|
||||
}
|
||||
|
||||
interface MangaSeriesCacheEntry {
|
||||
seriesKey: string // 唯一标识(按"关键设计决策 1"构造)
|
||||
seriesTitle: string // 系列标题
|
||||
coverUrl: string // 封面图
|
||||
author: string // 作者
|
||||
chapters: MangaChapterCacheEntry[] // 该系列下所有章节(已 normalize 且按 order 排序)
|
||||
}
|
||||
// 导出 mangaSeriesCache: reactive<Record<string, MangaSeriesCacheEntry>>
|
||||
// 导出 clearMangaCache(): void
|
||||
```
|
||||
|
||||
#### 2. `src/views/app/UserChapterListPage.vue` —— 章节列表页(Level 2)
|
||||
|
||||
**复用项目 pattern:** 通过 `kind` prop 同时支持 novels 和 manga。
|
||||
|
||||
**Props:**
|
||||
|
||||
```ts
|
||||
{
|
||||
kind: "novels" | "manga"
|
||||
seriesId?: string // novels: 后端 series_id(统一 String 类型)
|
||||
seriesKey?: string // manga: 客户端分组 key
|
||||
}
|
||||
```
|
||||
|
||||
**模板结构:**
|
||||
- 面包屑导航:`小说/漫画 > 系列标题`
|
||||
- "← 返回" 按钮(`router.back()`)
|
||||
- **四种状态覆盖:**
|
||||
- 加载中 → `"正在加载章节..."`
|
||||
- 加载失败 → `error` 文本 + 重试按钮
|
||||
- 空章节 → `"暂无章节"`(加载成功但 chapters 为空)
|
||||
- 缓存失效重载 → manga 缓存命中但 chapters 为空时,自动切到 API 回退
|
||||
- 系列信息头:封面 + 标题 + 作者 + 章节数
|
||||
- 章节列表:每行显示序号 | 章节标题 | 页数/更新时间 | →
|
||||
|
||||
**数据加载:**
|
||||
- `kind="novels"`:调用 `GET /novels/series/{seriesId}`,对返回的 chapters 执行 normalize
|
||||
- `kind="manga"`:
|
||||
1. 优先读 `mangaSeriesCache[seriesKey]`
|
||||
2. 缓存命中且 chapters 不为空 → 直接展示
|
||||
3. 缓存未命中或 chapters 为空 → 按"关键设计决策 3"执行 API 回退
|
||||
|
||||
**跳转到 Level 3:** `/app/{kind}/read/{id}?fromSeries={encodedSeriesRef}`
|
||||
|
||||
---
|
||||
|
||||
### 二、修改现有文件
|
||||
|
||||
#### 3. `src/views/app/UserListPage.vue` —— 系列列表页(Level 1)
|
||||
|
||||
##### 3.1 novels 加载改为 `/novels/series`
|
||||
|
||||
当前 `load()` → `GET /me/novels` → `groupNovels()`。改为:
|
||||
- `endpoint()` 对 novels 返回 `/novels/series`
|
||||
- `params()` 对 novels 传 `{ include_chapters: false }`(后端返回轻量 series 摘要)
|
||||
- `groupNovels()` 直接从 series 数据映射 WorkGroup,不再做客户端分组
|
||||
- WorkGroup 新增 `seriesId: String(series.series_id)` 字段
|
||||
|
||||
##### 3.2 manga 加载保持 `/me/gallery`,增强分组逻辑
|
||||
|
||||
- 按照"关键设计决策 1"构造稳定的 `seriesKey`
|
||||
- 构建 WorkGroup 时同步写入 `mangaSeriesCache[seriesKey]`
|
||||
- 缓存中的 chapters 按"关键设计决策 6"的 normalize 规则统一格式,并已按 order 排序
|
||||
- WorkGroup 新增 `seriesKey` 字段
|
||||
|
||||
##### 3.3 detailUrl 修改
|
||||
|
||||
```ts
|
||||
// 修改前:直接跳阅读器
|
||||
function detailUrl(id: string) { return `/app/${props.kind}/${id}` }
|
||||
|
||||
// 修改后:跳章节列表页
|
||||
function detailUrl(group: WorkGroup): string {
|
||||
if (kind === "novels") return `/app/novels/series/${group.seriesId}`
|
||||
if (kind === "manga") return `/app/manga/series/${encodeURIComponent(group.seriesKey!)}`
|
||||
...
|
||||
}
|
||||
|
||||
// 为 gallery / alerts 保留原始逻辑
|
||||
function detailUrlById(id: string): string {
|
||||
return `/app/${props.kind}/${id}`
|
||||
}
|
||||
```
|
||||
|
||||
##### 3.4 WorkGroup 类型扩展
|
||||
|
||||
```ts
|
||||
type WorkGroup = {
|
||||
// ... 原有字段
|
||||
seriesKey?: string // manga 用(稳定构造)
|
||||
seriesId?: string // novels 用(统一 String)
|
||||
}
|
||||
```
|
||||
|
||||
##### 3.5 分页调整
|
||||
|
||||
novels 改用 `/novels/series` 后不再需要前端分页(后端一次返回所有系列),因此:
|
||||
- `supportsPagination` computed 对 novels 返回 false
|
||||
- 隐藏分页控件
|
||||
|
||||
#### 4. `src/views/app/UserDetailPage.vue` —— 阅读器页(Level 3)
|
||||
|
||||
- **新增面包屑导航**(仅 novels/manga):
|
||||
- 始终显示:`小说/漫画 > 当前章节标题`
|
||||
- `fromSeries` 存在时插入中间级:`小说/漫画 > [章节列表] > 当前章节标题`
|
||||
- **新增返回按钮**:`"← 返回"`(`$router.back()`)
|
||||
- **`fromSeries` 参数处理**(按"关键设计决策 4"):
|
||||
```ts
|
||||
const fromSeries = computed(() => String(route.query.fromSeries || ""))
|
||||
const backToListRoute = computed(() => {
|
||||
if (fromSeries.value) return `/app/${props.kind}/series/${fromSeries.value}`
|
||||
return `/app/${props.kind}` // fallback 到系列列表
|
||||
})
|
||||
```
|
||||
- 原有阅读器/详情逻辑不变
|
||||
|
||||
#### 5. `src/router/index.ts` —— 路由配置
|
||||
|
||||
- 新增 `UserChapterListPage` 导入
|
||||
- 新增 4 条路由(novels/manga 的 series 和 read),严格按照"关键设计决策 5"的顺序排列
|
||||
- 末尾添加 2 条旧路由重定向
|
||||
|
||||
---
|
||||
|
||||
### 三、CSS 新增样式
|
||||
|
||||
**文件:** `src/styles.css` —— 追加
|
||||
|
||||
| 样式类 | 用途 |
|
||||
|---------|------|
|
||||
| `.breadcrumb` | 面包屑导航容器(flex, 小字号 muted) |
|
||||
| `.breadcrumb a` | 面包屑链接(pink-strong, hover opacity) |
|
||||
| `.breadcrumb .current` | 当前页名(ink, bold) |
|
||||
| `.series-header` | 章节列表页的系列信息头(flex, 封面+标题+作者+统计) |
|
||||
| `.series-cover` | 系列封面图(110px 宽, 3:4 比例) |
|
||||
| `.series-cover.placeholder` | 封面色块占位(粉色背景, 居中文字) |
|
||||
| `.series-info` | 系列元信息容器 |
|
||||
| `.series-info h2` | 系列标题(ellipsis 溢出省略) |
|
||||
| `.chapter-list` | 章节列表网格(gap 8px) |
|
||||
| `.chapter-row` | 单行章节(flex, hover 阴影+边框变色) |
|
||||
| `.chapter-row .chapter-index` | 圆形粉色序号标识(36px 圆, number) |
|
||||
| `.chapter-row .chapter-info` | 章节标题与元信息(flex 收缩, ellipsis) |
|
||||
| `.chapter-row .chapter-info strong` | 章节标题 |
|
||||
| `.chapter-row .chapter-info .muted` | 页数 / 更新时间的灰色副文本 |
|
||||
|
||||
---
|
||||
|
||||
## 数据流示意
|
||||
|
||||
### 小说
|
||||
```
|
||||
UserListPage ──GET /novels/series?include_chapters=false──→ [{ series_id, chapter_count, cover_path }]
|
||||
│ seriesId = String(series_id)
|
||||
│ 点击卡片 → /app/novels/series/:seriesId
|
||||
▼
|
||||
UserChapterListPage ──GET /novels/series/:id──→ { chapters: [{id, chapter_title, ...}] }
|
||||
│ 对 chapters 执行 normalize
|
||||
│ 点击章节 → /app/novels/read/:id?fromSeries=:seriesId
|
||||
▼
|
||||
UserDetailPage ──GET /novels/:id/content──→ 正文阅读
|
||||
│ 面包屑:小说 > [章节列表] > 当前章名
|
||||
│ fromSeries 缺失时省略中间级
|
||||
```
|
||||
|
||||
### 漫画
|
||||
```
|
||||
UserListPage ──GET /me/gallery?content_type=manga──→ client groupManga()
|
||||
│ seriesKey 按稳定规则构造(series_id 优先)
|
||||
│ 章节按 normalize 统一格式、排序
|
||||
│ └→ 写入 mangaSeriesCache[seriesKey]
|
||||
│ 点击卡片 → /app/manga/series/:seriesKey(encoded)
|
||||
▼
|
||||
UserChapterListPage ──读 mangaSeriesCache[seriesKey]──→ 展示
|
||||
│ 缓存未命中或 chapters 为空:
|
||||
│ ① GET /me/gallery?content_type=manga&limit=200
|
||||
│ ② 执行同一套 groupManga() 分组逻辑
|
||||
│ ③ 查找匹配 seriesKey 的组
|
||||
│ ④ 找到 → 填充;未找到 → 显示错误
|
||||
│ 点击章节 → /app/manga/read/:id?fromSeries=:seriesKey(encoded)
|
||||
▼
|
||||
UserDetailPage ──GET /gallery/:id──→ 详情展示
|
||||
│ 面包屑:漫画 > [章节列表] > 当前章名
|
||||
│ fromSeries 缺失时省略中间级
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 验证方案
|
||||
|
||||
1. **Level 1 → Level 2 导航**:访问 `/app/novels`,点击任意作品卡片,确认跳转到 `/app/novels/series/:seriesId`,且显示正确的章节列表
|
||||
2. **Level 2 → Level 3 导航**:在章节列表页点击某一章,确认跳转到 `/app/novels/read/:id?fromSeries=:seriesId`,正文正常显示
|
||||
3. **面包屑返回**:在阅读器中点击面包屑"章节列表",确认返回 Level 2;点击"← 返回",确认能正常回退
|
||||
4. **面包屑 fallback**:直接访问 `/app/novels/read/:id`(无 fromSeries),确认面包屑不显示中间级,返回按钮回到 `/app/novels`
|
||||
5. **漫画缓存链路**:从 Level 1 → Level 2 → Level 3 完整走通漫画导航
|
||||
6. **漫画缓存缺失回退**:清除缓存后直接访问 `/app/manga/series/:seriesKey`,确认自动回退到 API 拉取并正确展示
|
||||
7. **漫画 series_id 缺失**:对没有 `artwork_metadata.series_id` 的漫画数据,确认 seriesKey 回退到 `manga-title-{series_title}` 且导航正常
|
||||
8. **Level 2 空状态**:对章节数为 0 的系列(理论上不应出现,但做防御),确认显示"暂无章节"
|
||||
9. **Level 2 错误状态**:模拟 API 失败,确认显示错误信息 + 重试按钮
|
||||
10. **不受影响的功能**:图片(gallery)、提醒(alerts)、个人资料(profile)页面仍正常运作
|
||||
11. **旧路由兼容**:访问 `/app/novels/旧id` 确认重定向到 `/app/novels/read/旧id`
|
||||
12. **管理后台不受影响**:`/admin/novels`、`/admin/gallery` 表格正常
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,656 @@
|
||||
# 两项核心改进 — TOC 分页规则 · 搜索健康分
|
||||
|
||||
> 日期:2026-06-22
|
||||
> 范围:仅移动端 Dart 侧(`sakura_mobile/lib/src/core/`)
|
||||
> 涉及文件:
|
||||
>
|
||||
> - `legado_models.dart` — 数据模型
|
||||
> - `legado_rule_parser.dart` — 规则解析器
|
||||
> - `legado_engine.dart` — 搜索引擎入口
|
||||
> 原则:不改 Python 插件,不改 UI 层,只动 core 层。
|
||||
|
||||
---
|
||||
|
||||
## 一、改进 1:TOC 章节目录分页规则完整支持
|
||||
|
||||
### 1.1 问题
|
||||
|
||||
`fetchToc()` 只拉第一页目录。很多书源第一页只给 50~100 章,剩下的靠 **分页目录规则** 指向下一页。不支持分页规则 → 用户看到的目录永远不全。
|
||||
|
||||
### 1.2 兼容的字段名
|
||||
|
||||
书源 JSON 中,下一页 URL 的字段名可能是以下任意一种,按优先级遍历取第一个非空值:
|
||||
|
||||
```dart
|
||||
// legado_models.dart — BookSource / TocRule 中新增字段解析
|
||||
|
||||
static const List<String> _tocNextPageCandidates = [
|
||||
'tocNextPageRule', // Legado 标准名
|
||||
'nextTocUrl', // 常见别名
|
||||
'nextUrl', // 通用别名
|
||||
'nextPage', // 页码式
|
||||
'tocNextPage', // 变体
|
||||
];
|
||||
|
||||
String? get tocNextPageRule {
|
||||
for (final name in _tocNextPageCandidates) {
|
||||
final v = rawJson[name];
|
||||
if (v is String && v.trim().isNotEmpty) return v.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
```
|
||||
|
||||
### 1.3 分页规则语法全集
|
||||
|
||||
| 规则类型 | 语法示例 | 说明 |
|
||||
| --------- | ----------------------------------- | ------------------ |
|
||||
| JS 表达式 | `<js>nextUrl</js>` | 执行 JS 获取下一页 URL |
|
||||
| JS 缩写 | `@js:nextUrl` | 同上,简写形式 |
|
||||
| 模板拼接 | `https://host/list?page={{page}}` | `{{page}}` 从 1 自增 |
|
||||
| 偏移模板 | `{{page+1}}` / `{{page-1}}` | 页号偏移 |
|
||||
| CSS 选择器 | `text.下一页@href` | 从当前页 HTML 提取下一页链接 |
|
||||
| class 选择器 | `class.next@href` | 同上 |
|
||||
| id 选择器 | `id.pagenext@href` | 同上 |
|
||||
| 属性选择器 | `@href` | 取当前元素 href |
|
||||
| XPath | `//a[contains(text(),'下一页')]/@href` | XPath 提取 |
|
||||
| 字符串切片 | `!0:-1` | 对当前 URL 切片(去掉末尾字符) |
|
||||
| offset 翻页 | `offset=50` | 按 offset 递增翻页 |
|
||||
| 纯文本 URL | `https://...` | 直接跳到指定 URL |
|
||||
| 相对路径 | `/list/2.html` | 基于当前 URL 拼接 |
|
||||
|
||||
### 1.4 实现 — `TocPaginationResolver`
|
||||
|
||||
```dart
|
||||
// 放在 legado_engine.dart 或新建 legado_toc_pagination.dart
|
||||
|
||||
class TocPaginationResolver {
|
||||
final LegadoRuleExecutor _ruleExecutor;
|
||||
final HttpClient _client;
|
||||
|
||||
TocPaginationResolver(this._ruleExecutor, this._client);
|
||||
|
||||
/// 拉取全部分页章节,合并去重。
|
||||
Future<List<ChapterInfo>> fetchAllChapters({
|
||||
required String firstPageUrl,
|
||||
required String? tocNextPageRule,
|
||||
required String tocRule,
|
||||
required Map<String, String>? headers,
|
||||
int maxPages = 20,
|
||||
}) async {
|
||||
final all = <ChapterInfo>[];
|
||||
final seenPageUrls = <String>{};
|
||||
final seenChapterUrls = <String>{};
|
||||
|
||||
String? url = firstPageUrl;
|
||||
int page = 0;
|
||||
|
||||
while (url != null && page < maxPages) {
|
||||
if (!seenPageUrls.add(url)) break; // 死循环保护
|
||||
|
||||
final html = await _client.get(url, headers: headers);
|
||||
final batch = _ruleExecutor.selectList(html, url, rule: tocRule);
|
||||
|
||||
int added = 0;
|
||||
for (final ch in batch) {
|
||||
if (seenChapterUrls.add(ch.url)) {
|
||||
all.add(ch);
|
||||
added++;
|
||||
}
|
||||
}
|
||||
|
||||
if (added == 0) break; // 本页无新章节,终止翻页
|
||||
|
||||
url = _resolveNextPage(
|
||||
currentUrl: url,
|
||||
currentHtml: html,
|
||||
rule: tocNextPageRule,
|
||||
pageIndex: page,
|
||||
);
|
||||
page++;
|
||||
}
|
||||
|
||||
return all;
|
||||
}
|
||||
|
||||
// ---------- 下一页 URL 解析 ----------
|
||||
|
||||
String? _resolveNextPage({
|
||||
required String currentUrl,
|
||||
required String currentHtml,
|
||||
required String? rule,
|
||||
required int pageIndex,
|
||||
}) {
|
||||
if (rule == null || rule.trim().isEmpty) return null;
|
||||
final r = rule.trim();
|
||||
|
||||
// 1) JS: <js>...</js> 或 @js:...
|
||||
if (r.startsWith('<js>') || r.startsWith('@js:')) {
|
||||
final code = r
|
||||
.replaceFirst(RegExp(r'^<js>'), '')
|
||||
.replaceFirst(RegExp(r'</js>$'), '')
|
||||
.replaceFirst(RegExp(r'^@js:'), '');
|
||||
return _ruleExecutor.evalJs(code, context: {
|
||||
'result': currentHtml,
|
||||
'baseUrl': currentUrl,
|
||||
'page': pageIndex,
|
||||
});
|
||||
}
|
||||
|
||||
// 2) 切片: !start:end
|
||||
if (r.startsWith('!')) {
|
||||
return _applySlice(currentUrl, r.substring(1));
|
||||
}
|
||||
|
||||
// 3) CSS 选择器 / XPath
|
||||
if (_isSelector(r)) {
|
||||
return _ruleExecutor.select(currentHtml, currentUrl, rule: r);
|
||||
}
|
||||
|
||||
// 4) 模板替换 {{page}} / {{page+1}} / {{page-1}}
|
||||
if (r.contains('{{page')) {
|
||||
return _renderPageTemplate(r, pageIndex);
|
||||
}
|
||||
|
||||
// 5) offset / start 参数
|
||||
if (RegExp(r'[?&](offset|start)=').hasMatch(r)) {
|
||||
return _renderOffsetTemplate(r, pageIndex);
|
||||
}
|
||||
|
||||
// 6) 绝对 URL
|
||||
if (r.startsWith('http://') || r.startsWith('https://')) {
|
||||
return _renderPageTemplate(r, pageIndex);
|
||||
}
|
||||
|
||||
// 7) 相对路径
|
||||
return Uri.parse(currentUrl).resolve(r).toString();
|
||||
}
|
||||
|
||||
// ---------- 规则判定 ----------
|
||||
|
||||
bool _isSelector(String s) {
|
||||
return s.startsWith('text.') ||
|
||||
s.startsWith('class.') ||
|
||||
s.startsWith('id.') ||
|
||||
s.startsWith('@') ||
|
||||
s.startsWith('//'); // XPath
|
||||
}
|
||||
|
||||
// ---------- 模板渲染 ----------
|
||||
|
||||
String _renderPageTemplate(String tpl, int page) {
|
||||
return tpl
|
||||
.replaceAll('{{page+1}}', '${page + 1}')
|
||||
.replaceAll('{{page-1}}', '${page - 1}')
|
||||
.replaceAll('{{page}}', '$page');
|
||||
}
|
||||
|
||||
String _renderOffsetTemplate(String tpl, int page) {
|
||||
return tpl
|
||||
.replaceAllMapped(
|
||||
RegExp(r'offset=(\d+)'),
|
||||
(m) => 'offset=${int.parse(m.group(1)!) * page}',
|
||||
)
|
||||
.replaceAllMapped(
|
||||
RegExp(r'start=(\d+)'),
|
||||
(m) => 'start=${int.parse(m.group(1)!) * page}',
|
||||
);
|
||||
}
|
||||
|
||||
String? _applySlice(String url, String expr) {
|
||||
final parts = expr.split(':');
|
||||
if (parts.length != 2) return null;
|
||||
final a = int.tryParse(parts[0]) ?? 0;
|
||||
final b = int.tryParse(parts[1]);
|
||||
final start = a < 0 ? url.length + a : a;
|
||||
if (b != null) {
|
||||
final end = b < 0 ? url.length + b : b;
|
||||
return url.substring(start, end);
|
||||
}
|
||||
return url.substring(start);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 1.5 集成到 `LegadoEngine.fetchToc`
|
||||
|
||||
```dart
|
||||
// legado_engine.dart — 修改 fetchToc
|
||||
|
||||
Future<List<ChapterInfo>> fetchToc(BookSource source, String tocUrl) async {
|
||||
final rule = source.tocRule;
|
||||
if (rule == null || rule.isEmpty) return [];
|
||||
|
||||
final resolver = TocPaginationResolver(_ruleExecutor, _client);
|
||||
return resolver.fetchAllChapters(
|
||||
firstPageUrl: tocUrl,
|
||||
tocNextPageRule: source.tocNextPageRule, // ← 新增
|
||||
tocRule: rule,
|
||||
headers: _buildHeaders(source),
|
||||
maxPages: 20,
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
### 1.6 死循环防护(三层)
|
||||
|
||||
1. **`maxPages = 20`** — 单书最多翻 20 页目录
|
||||
2. **`seenPageUrls`** — 同一 URL 不重复请求
|
||||
3. **`added == 0` 终止** — 当前页没有新增章节则停止
|
||||
|
||||
---
|
||||
|
||||
## 二、改进 2:搜索源健康分 + 短超时 + 分阶段返回
|
||||
|
||||
### 2.1 问题
|
||||
|
||||
当前 `search()` 对所有启用源并发请求,每个源 timeout 20~30 秒。10 个源一批,坏源多时整批拖到 20+ 秒才出结果。
|
||||
|
||||
### 2.2 源健康分模型
|
||||
|
||||
```dart
|
||||
// legado_models.dart — 新增 SourceHealth
|
||||
|
||||
class SourceHealth {
|
||||
final String sourceId;
|
||||
|
||||
int successCount = 0;
|
||||
int failCount = 0;
|
||||
int timeoutCount = 0;
|
||||
int totalLatencyMs = 0;
|
||||
DateTime? lastSuccessAt;
|
||||
DateTime? lastFailAt;
|
||||
int _consecutiveFails = 0;
|
||||
|
||||
SourceHealth({required this.sourceId});
|
||||
|
||||
// ---- 判定 ----
|
||||
|
||||
/// 连续失败 ≥ 5 次 → 临时禁用
|
||||
bool get isUsable {
|
||||
if (_consecutiveFails >= 5) return false;
|
||||
if (failCount >= 5 && successCount == 0) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
int get consecutiveFails => _consecutiveFails;
|
||||
|
||||
/// 健康分 0~100
|
||||
double get score {
|
||||
final total = successCount + failCount + timeoutCount;
|
||||
if (total == 0) return 50.0; // 新源默认 50
|
||||
|
||||
double s = 0.0;
|
||||
|
||||
// 成功率 60%
|
||||
s += 60.0 * (successCount / total);
|
||||
|
||||
// 平均延迟 25%
|
||||
if (successCount > 0 && totalLatencyMs > 0) {
|
||||
final avg = totalLatencyMs / successCount;
|
||||
if (avg < 500) {
|
||||
s += 25.0;
|
||||
} else if (avg < 2000) {
|
||||
s += 25.0 * (1.0 - (avg - 500) / 1500);
|
||||
}
|
||||
}
|
||||
|
||||
// 新鲜度 15%(24h 内有成功记录)
|
||||
if (lastSuccessAt != null) {
|
||||
final h = DateTime.now().difference(lastSuccessAt!).inHours;
|
||||
if (h < 1) {
|
||||
s += 15.0;
|
||||
} else if (h < 24) {
|
||||
s += 15.0 * (1.0 - h / 24);
|
||||
}
|
||||
}
|
||||
|
||||
return s.clamp(0.0, 100.0);
|
||||
}
|
||||
|
||||
// ---- 记录 ----
|
||||
|
||||
void recordSuccess(int latencyMs) {
|
||||
successCount++;
|
||||
totalLatencyMs += latencyMs;
|
||||
lastSuccessAt = DateTime.now();
|
||||
_consecutiveFails = 0;
|
||||
}
|
||||
|
||||
void recordFailure() {
|
||||
failCount++;
|
||||
lastFailAt = DateTime.now();
|
||||
_consecutiveFails++;
|
||||
}
|
||||
|
||||
void recordTimeout() {
|
||||
timeoutCount++;
|
||||
lastFailAt = DateTime.now();
|
||||
_consecutiveFails++;
|
||||
}
|
||||
|
||||
void resetConsecutiveFails() => _consecutiveFails = 0;
|
||||
|
||||
// ---- 序列化(持久化到 SharedPreferences)----
|
||||
|
||||
Map<String, dynamic> toJson() => {
|
||||
'sourceId': sourceId,
|
||||
'successCount': successCount,
|
||||
'failCount': failCount,
|
||||
'timeoutCount': timeoutCount,
|
||||
'totalLatencyMs': totalLatencyMs,
|
||||
'lastSuccessAt': lastSuccessAt?.toIso8601String(),
|
||||
'lastFailAt': lastFailAt?.toIso8601String(),
|
||||
'consecutiveFails': _consecutiveFails,
|
||||
};
|
||||
|
||||
factory SourceHealth.fromJson(Map<String, dynamic> json) {
|
||||
final h = SourceHealth(sourceId: json['sourceId']);
|
||||
h.successCount = json['successCount'] ?? 0;
|
||||
h.failCount = json['failCount'] ?? 0;
|
||||
h.timeoutCount = json['timeoutCount'] ?? 0;
|
||||
h.totalLatencyMs = json['totalLatencyMs'] ?? 0;
|
||||
h.lastSuccessAt = json['lastSuccessAt'] != null
|
||||
? DateTime.tryParse(json['lastSuccessAt'])
|
||||
: null;
|
||||
h.lastFailAt = json['lastFailAt'] != null
|
||||
? DateTime.tryParse(json['lastFailAt'])
|
||||
: null;
|
||||
h._consecutiveFails = json['consecutiveFails'] ?? 0;
|
||||
return h;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2.3 健康分持久化
|
||||
|
||||
```dart
|
||||
// legado_engine.dart — SourceHealthStore
|
||||
|
||||
class SourceHealthStore {
|
||||
static const _key = 'source_health_v1';
|
||||
final Map<String, SourceHealth> _data = {};
|
||||
|
||||
Future<void> load() async {
|
||||
final prefs = await SharedPreferences.getInstance();
|
||||
final raw = prefs.getStringList(_key);
|
||||
if (raw == null) return;
|
||||
for (final s in raw) {
|
||||
try {
|
||||
final h = SourceHealth.fromJson(jsonDecode(s));
|
||||
_data[h.sourceId] = h;
|
||||
} catch (_) {}
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> save() async {
|
||||
final prefs = await SharedPreferences.getInstance();
|
||||
final raw = _data.values.map((h) => jsonEncode(h.toJson())).toList();
|
||||
await prefs.setStringList(_key, raw);
|
||||
}
|
||||
|
||||
bool isUsable(String id) => _data[id]?.isUsable ?? true;
|
||||
double score(String id) => _data[id]?.score ?? 50.0;
|
||||
|
||||
void _ensure(String id) {
|
||||
_data.putIfAbsent(id, () => SourceHealth(sourceId: id));
|
||||
}
|
||||
|
||||
void recordSuccess(String id, int latencyMs) {
|
||||
_ensure(id);
|
||||
_data[id]!.recordSuccess(latencyMs);
|
||||
}
|
||||
|
||||
void recordFailure(String id) {
|
||||
_ensure(id);
|
||||
_data[id]!.recordFailure();
|
||||
}
|
||||
|
||||
void recordTimeout(String id) {
|
||||
_ensure(id);
|
||||
_data[id]!.recordTimeout();
|
||||
}
|
||||
|
||||
/// 用户手动重置所有源健康状态
|
||||
void resetAll() {
|
||||
for (final h in _data.values) {
|
||||
h.resetConsecutiveFails();
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2.4 分阶段搜索调度器
|
||||
|
||||
```dart
|
||||
// legado_engine.dart — SearchScheduler
|
||||
|
||||
class SearchScheduler {
|
||||
static const fastTimeout = Duration(seconds: 5);
|
||||
static const slowTimeout = Duration(seconds: 8);
|
||||
static const maxConcurrent = 6;
|
||||
|
||||
final SourceHealthStore _health;
|
||||
final LegadoEngine _engine;
|
||||
|
||||
SearchScheduler(this._health, this._engine);
|
||||
|
||||
/// 返回 Stream,UI 层逐阶段渲染结果
|
||||
Stream<SearchPhase> search(String keyword, List<BookSource> allSources) async* {
|
||||
// 过滤 + 按健康分排序
|
||||
final usable = allSources
|
||||
.where((s) => _health.isUsable(s.id))
|
||||
.toList()
|
||||
..sort((a, b) => _health.score(b.id).compareTo(_health.score(a.id)));
|
||||
|
||||
if (usable.isEmpty) {
|
||||
yield SearchPhase(name: 'error', message: '没有可用的搜索源');
|
||||
return;
|
||||
}
|
||||
|
||||
// 分三档
|
||||
final fast = usable.where((s) => _health.score(s.id) >= 70).toList();
|
||||
final normal = usable.where((s) {
|
||||
final sc = _health.score(s.id);
|
||||
return sc >= 40 && sc < 70;
|
||||
}).toList();
|
||||
final slow = usable.where((s) => _health.score(s.id) < 40).toList();
|
||||
|
||||
final results = <SearchItem>[];
|
||||
final doneIds = <String>{};
|
||||
final seen = <String>{};
|
||||
|
||||
// ---- 阶段 1:快速源,超时 5s ----
|
||||
yield SearchPhase(name: 'searching', message: '正在搜索 ${usable.length} 个源...');
|
||||
final r1 = await _batch(fast, keyword, timeout: fastTimeout, onDone: doneIds.add);
|
||||
_merge(results, r1, seen);
|
||||
yield SearchPhase(
|
||||
name: 'partial',
|
||||
items: List.of(results),
|
||||
doneCount: doneIds.length,
|
||||
totalCount: usable.length,
|
||||
);
|
||||
|
||||
// ---- 阶段 2:普通源,超时 8s ----
|
||||
final r2 = await _batch(normal, keyword, timeout: slowTimeout, onDone: doneIds.add);
|
||||
_merge(results, r2, seen);
|
||||
yield SearchPhase(
|
||||
name: 'partial',
|
||||
items: List.of(results),
|
||||
doneCount: doneIds.length,
|
||||
totalCount: usable.length,
|
||||
);
|
||||
|
||||
// ---- 阶段 3:慢源后台继续,不阻塞 ----
|
||||
if (slow.isNotEmpty) {
|
||||
// 放入后台队列,不 await
|
||||
_batch(slow, keyword, timeout: slowTimeout, onDone: doneIds.add)
|
||||
.then((r3) {
|
||||
_merge(results, r3, seen);
|
||||
// TODO: EventBus 通知 UI 追加
|
||||
});
|
||||
yield SearchPhase(
|
||||
name: 'complete_with_pending',
|
||||
items: List.of(results),
|
||||
doneCount: doneIds.length,
|
||||
totalCount: usable.length,
|
||||
message: '${slow.length} 个慢源将在后台继续搜索',
|
||||
);
|
||||
} else {
|
||||
yield SearchPhase(
|
||||
name: 'complete',
|
||||
items: List.of(results),
|
||||
doneCount: doneIds.length,
|
||||
totalCount: usable.length,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 内部分批并发 ----
|
||||
|
||||
Future<List<SearchItem>> _batch(
|
||||
List<BookSource> sources,
|
||||
String keyword, {
|
||||
required Duration timeout,
|
||||
required void Function(String) onDone,
|
||||
}) async {
|
||||
final out = <SearchItem>[];
|
||||
for (var i = 0; i < sources.length; i += maxConcurrent) {
|
||||
final chunk = sources.skip(i).take(maxConcurrent).toList();
|
||||
final futures = chunk.map((s) => _searchOne(s, keyword, timeout));
|
||||
final results = await Future.wait(futures);
|
||||
for (var j = 0; j < results.length; j++) {
|
||||
if (results[j] != null) out.addAll(results[j]!);
|
||||
onDone(chunk[j].id);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
Future<List<SearchItem>?> _searchOne(
|
||||
BookSource source, String keyword, Duration timeout,
|
||||
) async {
|
||||
final sw = Stopwatch()..start();
|
||||
try {
|
||||
final items = await _engine.search(source, keyword).timeout(timeout);
|
||||
sw.stop();
|
||||
_health.recordSuccess(source.id, sw.elapsedMilliseconds);
|
||||
return items;
|
||||
} on TimeoutException {
|
||||
_health.recordTimeout(source.id);
|
||||
return null;
|
||||
} catch (_) {
|
||||
_health.recordFailure(source.id);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
void _merge(List<SearchItem> all, List<SearchItem> incoming, Set<String> seen) {
|
||||
for (final item in incoming) {
|
||||
if (seen.add(item.id)) all.add(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 阶段结果 ----
|
||||
|
||||
class SearchPhase {
|
||||
final String name; // searching | partial | complete | complete_with_pending | error
|
||||
final List<SearchItem> items;
|
||||
final int doneCount;
|
||||
final int totalCount;
|
||||
final String? message;
|
||||
|
||||
SearchPhase({
|
||||
required this.name,
|
||||
this.items = const [],
|
||||
this.doneCount = 0,
|
||||
this.totalCount = 0,
|
||||
this.message,
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### 2.5 UI 增量渲染时序
|
||||
|
||||
```
|
||||
t=0s → 显示 "正在搜索 N 个源..."
|
||||
t=0~5s → 第 1 批结果到达 → 首屏渲染(最快的 5~10 条)
|
||||
t=5~8s → 第 2 批结果到达 → 列表追加
|
||||
t=8s → 停止等待 → 提示"慢源已跳过,后台补充中"
|
||||
t=8~15s → 第 3 批结果到达 → 静默追加到列表末尾
|
||||
```
|
||||
|
||||
### 2.6 健康分自动衰减
|
||||
|
||||
每次 `save()` 时做一次衰减:
|
||||
|
||||
```dart
|
||||
// SourceHealthStore.save() 追加衰减逻辑
|
||||
Future<void> save() async {
|
||||
// 超过 7 天无任何记录 → 衰减 successCount 和 failCount 各减半
|
||||
for (final h in _data.values) {
|
||||
final latest = h.lastSuccessAt ?? h.lastFailAt;
|
||||
if (latest != null &&
|
||||
DateTime.now().difference(latest).inDays >= 7) {
|
||||
h.successCount = (h.successCount * 0.5).round();
|
||||
h.failCount = (h.failCount * 0.5).round();
|
||||
h.timeoutCount = (h.timeoutCount * 0.5).round();
|
||||
}
|
||||
}
|
||||
// ... 写入 SharedPreferences
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 三、两改进的协作关系
|
||||
|
||||
```
|
||||
用户搜索一本书
|
||||
│
|
||||
├─ SearchScheduler.search(keyword)
|
||||
│ ├─ 按健康分排序源
|
||||
│ ├─ 快速源先搜(5s 超时)
|
||||
│ ├─ 普通源补搜(8s 超时)
|
||||
│ └─ 返回结果列表
|
||||
│
|
||||
├─ 用户点击某条结果 → fetchBookInfo → fetchToc
|
||||
│ └─ TocPaginationResolver 拉全章节(支持分页规则)
|
||||
│
|
||||
└─ 用户阅读某章 → fetchChapterContent(按需加载)
|
||||
```
|
||||
|
||||
改进 1 保证"搜到书后章节不断尾",改进 2 保证"搜索本身不卡"。
|
||||
|
||||
---
|
||||
|
||||
## 四、实施顺序
|
||||
|
||||
```
|
||||
第 1 步:改进 2 的 SourceHealth + SourceHealthStore
|
||||
→ 只加数据模型 + 持久化,不影响现有搜索流程
|
||||
→ 可以先跑几天积累健康数据
|
||||
|
||||
第 2 步:改进 2 的 SearchScheduler + 短超时
|
||||
→ 替换现有 search() 的并发逻辑
|
||||
→ UI 层适配 Stream<SearchPhase>
|
||||
|
||||
第 3 步:改进 1 的 TocPaginationResolver
|
||||
→ 依赖 RuleParser 链式语法的基础支持
|
||||
→ 见《Legado 规则引擎重构方案》第 1~3 步
|
||||
|
||||
第 4 步:联调 + 回归
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 五、风险控制
|
||||
|
||||
| 风险 | 应对 |
|
||||
| -------------------- | --------------------------------------------- |
|
||||
| TOC 分页死循环 | `maxPages=20` + `seenUrls` 去重 + `added==0` 终止 |
|
||||
| 健康分误杀快源 | 只降权不禁用;连续失败 ≥5 才标记不可用 + 手动可重置 |
|
||||
| 短超时漏结果 | 慢源后台继续,event bus 异步追加,不丢数据 |
|
||||
| 新源全部 50 分 | 第一轮搜索后就有真实分,冷启动无影响 |
|
||||
| SharedPreferences 过大 | 只存 String list,每个源约 200 字节,100 个源 ≈ 20KB |
|
||||
File diff suppressed because it is too large.
Load diff
+1187
File diff suppressed because it is too large.
Load diff
+410
@@ -0,0 +1,410 @@
|
||||
# 代码审计报告
|
||||
|
||||
> 审计日期:2026-06-16
|
||||
> 审计范围:全项目(Flutter 移动端 30 个文件 + Python 桌面端 16 个文件 + Cloud 后端 10 个文件 + 5 个插件)
|
||||
> 发现总数:111 项(Critical 9 / High 20 / Medium 46 / Low 36)
|
||||
|
||||
---
|
||||
|
||||
## 目录
|
||||
|
||||
1. [Critical(紧急修复)](#一critical紧急修复)
|
||||
2. [High(高危)](#二high高危)
|
||||
3. [Medium(中危)](#三medium中危)
|
||||
4. [Low(低危)](#四low低危)
|
||||
5. [跨模块问题汇总](#五跨模块问题汇总)
|
||||
6. [修复优先级建议](#六修复优先级建议)
|
||||
|
||||
---
|
||||
|
||||
## 一、Critical(紧急修复)
|
||||
|
||||
### C1. Cloud 后端:JWT 密钥使用已知默认值
|
||||
|
||||
- **文件:** `sakura_cloud/core/config.py:29`
|
||||
- **代码:** `jwt_secret_key: str = Field(default="change-this-secret-in-production", min_length=16)`
|
||||
- **问题:** 默认 JWT 密钥是公开的占位字符串。如果运维人员忘记设置 `JWT_SECRET_KEY` 环境变量,任何知道此默认值的人都可以伪造任意用户的 JWT token,获得完全未授权访问。
|
||||
- **修复:** 生产环境启动时检测是否为默认值,若是则拒绝启动并输出明确错误信息。不使用带默认值的 `Field`,改用 `Field(default=None)` 并在启动时校验。
|
||||
|
||||
### C2. 笔趣阁插件:硬编码 AES 密钥和 IV
|
||||
|
||||
- **文件:** `plugins/biquge/plugin.py:60-67`
|
||||
- **代码:**
|
||||
```python
|
||||
TOKEN_SECRET = "book@token.html"
|
||||
TOKEN_MD5 = hashlib.md5(TOKEN_SECRET.encode("utf-8")).hexdigest()
|
||||
TOKEN_IV = TOKEN_MD5[:16].encode("utf-8")
|
||||
TOKEN_KEY = TOKEN_MD5[16:].encode("utf-8")
|
||||
```
|
||||
- **问题:** AES 密钥和 IV 从硬编码字符串经 MD5 派生,任何人看到源代码即可解密和伪造 API token。固定 IV 下的 CBC 模式使相同明文产生相同密文,可被已知明文攻击。这不是加密,是混淆。
|
||||
- **修复:** 每次加密使用随机 IV(`os.urandom(16)`),密钥从用户配置或安全随机源派生。
|
||||
|
||||
### C3. 小红书插件:`--no-sandbox` 禁用 Chromium 安全沙箱
|
||||
|
||||
- **文件:** `plugins/xiaohongshu/plugin.py:361`
|
||||
- **代码:** `"args": ["--disable-blink-features=AutomationControlled", "--no-sandbox"]`
|
||||
- **问题:** `--no-sandbox` 完全禁用 Chromium 沙箱,浏览器进程以完整 OS 权限运行。小红书的页面是不受信任的外部内容——如果渲染引擎存在漏洞被利用,攻击者获得主机系统的无限制访问权。
|
||||
- **修复:** 移除 `--no-sandbox`。如果必须在 Docker/root 环境运行,使用 `--cap-add=SYS_ADMIN` 或配置 seccomp profile,而非全局禁用沙箱。
|
||||
|
||||
### C4. Cloud 后端:数据库默认凭据 `sakura:sakura`
|
||||
|
||||
- **文件:** `sakura_cloud/core/config.py:17`
|
||||
- **代码:** `database_url: str = "postgresql+psycopg://sakura:sakura@localhost:5432/sakura_cloud"`
|
||||
- **问题:** 默认数据库用户名和密码都是 `sakura`。如果部署时未覆盖 `DATABASE_URL`,数据库对任何能访问 `localhost:5432` 的人开放。
|
||||
- **修复:** 同 C1——生产环境启动时检测默认值并拒绝启动。
|
||||
|
||||
### C5. Cloud 后端:MinIO 默认凭据 `minioadmin:minioadmin`
|
||||
|
||||
- **文件:** `sakura_cloud/core/config.py:24-25`
|
||||
- **代码:** `minio_access_key: str = "minioadmin"` / `minio_secret_key: str = "minioadmin"`
|
||||
- **问题:** 这是 MinIO 官方默认凭据。如果 MinIO 端点暴露在网络上,任何人可以访问所有用户存储的文件。
|
||||
- **修复:** 同上——启动时检测是否为默认值,若是则拒绝启动。
|
||||
|
||||
### C6. 桌面端:云同步 API 使用明文 HTTP
|
||||
|
||||
- **文件:** `pixiv_app/gui/app.py:54`
|
||||
- **代码:** `DEFAULT_CLOUD_API = "http://api.kunkunxiaomao.top:8000"`
|
||||
- **问题:** 使用 HTTP 而非 HTTPS。用户凭据(邮箱、密码、access token)和上传文件内容以明文在网络上传输。同一网络上的攻击者可通过被动嗅探截获所有认证凭据和文件数据。
|
||||
- **修复:** 改为 `http://api.kunkunxiaomao.top`(如果服务器支持 HTTPS)。最差情况下至少做出显式警告。
|
||||
|
||||
### C7. Flutter:`_initialized` 标志在 Firebase 初始化成功前设置
|
||||
|
||||
- **文件:** `sakura_mobile/lib/src/core/push_notifications.dart:27-29`
|
||||
- **代码:** `_initialized = true` 在 `await Firebase.initializeApp()` **之前**设置
|
||||
- **问题:** 如果 `Firebase.initializeApp()` 抛出异常,`_initialized` 已经是 `true`,导致后续 `initialize()` 调用全部命中 `if (_initialized) return` 的 early return,Firebase 永远无法重试初始化。推送功能永久失效,无任何提示。
|
||||
- **修复:** 将 `_initialized = true` 移到 `await Firebase.initializeApp()` 成功之后。
|
||||
|
||||
### C8. Flutter:`recentUpdatesOnly` 是死开关(dead UI switch)
|
||||
|
||||
- **文件:** `sakura_mobile/lib/src/pages/tabs/gallery_tab.dart:32,120`
|
||||
- **问题:** `recentUpdatesOnly` 状态变量控制哪个 Chip 显示"已选中",但 `_load()` 方法从未将其作为查询参数传给 API。用户看到的"最近更新"筛选切换实际上没有任何效果。
|
||||
- **修复:** 在 `_load()` 中将 `recentUpdatesOnly` 转换为对应的 API 参数,或者移除该 Chip。
|
||||
|
||||
### C9. 桌面端:ProxyPool 的多线程数据竞争
|
||||
|
||||
- **文件:** `pixiv_app/core/proxy_pool.py:193,272-273,278-279`
|
||||
- **问题:**
|
||||
1. `ProxyPool.current_index` 的读取(line 272)和递增(line 273)之间无锁保护,`ThreadPoolExecutor` 的多个 worker 同时调用 `get_proxy()` 时产生重复或跳跃的索引
|
||||
2. `ProxyPool.working_proxies` 被 `mark_failed()` 的 `.remove()` 修改,而其他线程可能正在迭代它——可抛出 `ValueError` 或不一致迭代
|
||||
3. `ProxyInfo.fail_count` 和 `success_count` 的非原子递增在 `ThreadPoolExecutor` 16 个 worker 并行调用 `verify_proxy()` 时静默丢失计数
|
||||
- **修复:** 使用 `threading.Lock` 保护所有对 `working_proxies`、`current_index` 的写操作,`fail_count`/`success_count` 改用 `threading` 安全计数或原子操作。
|
||||
|
||||
---
|
||||
|
||||
## 二、High(高危)
|
||||
|
||||
### H1. Cloud 后端:用户登录无限流 → 可暴力破解
|
||||
|
||||
- **文件:** `sakura_cloud/api/v1/endpoints/auth.py:56-76`
|
||||
- **问题:** `/login` 端点无限流(`admin_login_rate_limit_per_minute` 只用于 admin 端点)。攻击者可无限制尝试密码组合进行暴力破解。
|
||||
- **修复:** 对 `/login`、`/register`、`/token/refresh` 增加速率限制(如每 IP 每分钟 5 次)。
|
||||
|
||||
### H2. Cloud 后端:用户注册无限流 + 信息泄露
|
||||
|
||||
- **文件:** `sakura_cloud/api/v1/endpoints/auth.py:24-53`
|
||||
- **问题:** `/register` 端点无限流。409 响应明确区分"用户名已存在"和"邮箱已存在",可用于枚举已注册用户。
|
||||
- **修复:** 加速率限制。错误消息统一为"注册失败,请检查输入"。
|
||||
|
||||
### H3. Cloud 后端:存储配额未强制执行
|
||||
|
||||
- **文件:** `sakura_cloud/api/v1/endpoints/gallery.py:223`、`sync.py:86-141`
|
||||
- **问题:** `default_storage_limit_bytes` 在 config 中定义了(line 108-109),但在创建 artwork 和同步 metadata 时**从未检查** `current_user.storage_used` 是否超过配额。用户可以上传无限制的数据。
|
||||
- **修复:** 在 `gallery.py` 和 `sync.py` 的写入路径中增加 `if current_user.storage_used + new_size > limit: raise HTTPException(413)`。
|
||||
|
||||
### H4. Cloud 后端:Logout 不撤销 token
|
||||
|
||||
- **文件:** `sakura_cloud/api/v1/endpoints/auth.py:79-81`
|
||||
- **问题:** `/logout` 只返回 `{"ok": True}`,不做任何 token 撤销操作。access token 在其自然过期(最长 24 小时)前持续有效。用户登出后无法阻止 token 被盗用。
|
||||
- **修复:** Logout 时递增 `user.token_version`,使当前所有 token 立即失效。
|
||||
|
||||
### H5. Cloud 后端:Refresh token 轮转不撤销旧 token
|
||||
|
||||
- **文件:** `sakura_cloud/api/v1/endpoints/auth.py:84-104`
|
||||
- **问题:** 交换 refresh token 时签发新的 access + refresh token,但旧的 refresh token 未失效。如果 refresh token 被盗,攻击者和合法用户都可以持续刷新——直到某人重新登录(才会 bump `token_version`)。
|
||||
- **修复:** 实现 refresh token 轮转——每次使用时废弃旧 token 并签发新的。
|
||||
|
||||
### H6. 桌面端 & Flutter:敏感凭据多处明文存储
|
||||
|
||||
|| 位置 | 凭据类型 | 存储方式 |
|
||||
||------|---------|---------|
|
||||
|| `pixiv_app/core/auth.py:36-41` | Pixiv cookie | JSON 文件明文 |
|
||||
|| `plugins/pixiv/plugin.py:42-43` | Pixiv cookie | 内存明文 |
|
||||
|| `plugins/x/plugin.py:76` | X/Twitter cookies | `x_cookies.json` 明文 |
|
||||
|| `plugins/ao3/plugin.py:687-688` | AO3 cookie | 内存明文 |
|
||||
|| `plugins/xiaohongshu/plugin.py:386-404` | 小红书 cookies | `xiaohongshu_cookies.json` 明文 |
|
||||
|| `pixiv_app/core/proxy_pool.py:67-68` | Quake API key | `app_session_file()` JSON 明文 |
|
||||
|| `sakura_mobile/lib/src/core/session.dart:44-47` | Access token | SharedPreferences 明文 |
|
||||
|
||||
- **修复:** 生产环境使用操作系统密钥链(Windows DPAPI / macOS Keychain / Android Keystore)。最低限度应对配置文件设置受限的文件权限。
|
||||
|
||||
### H7. Flutter:小说阅读页面翻页不节流 → API 洪泛
|
||||
|
||||
- **文件:** `sakura_mobile/lib/src/pages/novel_reader_page.dart:787-790`
|
||||
- **问题:** `onPageChanged` 对**每次翻页**同步调用 `updateNovelProgress(currentNovel.id, page)`。快速翻页时数十上百个 API 请求瞬间发出,压垮服务器并浪费电量/带宽。
|
||||
- **修复:** 增加 500ms 防抖,只发送最终位置。
|
||||
|
||||
### H8. Flutter:探索页从"插画"tab 点搜索 → 打开"小说"模式
|
||||
|
||||
- **文件:** `sakura_mobile/lib/src/pages/tabs/gallery_tab.dart:398-399`
|
||||
- **问题:** 用户在"图片"(illust)tab 点击在线搜索时,代码 `widget.fixedContentType != 'manga'` 为真,传入 `UnifiedSearchMode.novel`。从图片页搜索却打开了小说搜索模式。
|
||||
- **修复:** 将 explore mode 与 gallery content type 解耦。默认为 manga 模式(漫画源更多)。或增加一个独立的选择器。
|
||||
|
||||
### H9. Flutter:Session 构造器中异步工作不可 await
|
||||
|
||||
- **文件:** `sakura_mobile/lib/src/core/session.dart:44-47`
|
||||
- **问题:** `SessionController()` 构造器中调用异步 `_load()`,但构造器不能 `async`。Provider 观察者先看到默认 loading 态再被 `_load()` 完成后的 `notifyListeners()` 更新。Provider 初始化异常无法被外部捕获。
|
||||
- **修复:** 使用 Riverpod 的 `FutureProvider` 或 `AsyncNotifier`,或将初始化方法显式暴露给 widget tree 调用。
|
||||
|
||||
### H10. Flutter:`_validateStoredSession` 静默吞所有异常
|
||||
|
||||
- **文件:** `sakura_mobile/lib/src/core/session.dart:159`
|
||||
- **问题:** `catch (_)` 捕获所有异常返回 `null`,不记录日志。网络故障或服务器宕机时用户被静默登出,无任何提示。
|
||||
- **修复:** 最低限度 `debugPrint(e)`。更好:区分网络错误(可恢复)和认证错误(需重新登录),并显示对应 UI。
|
||||
|
||||
### H11. Cloud 后端:JWT 缺少 `aud`/`iss` 声明验证
|
||||
|
||||
- **文件:** `sakura_cloud/core/security.py:23-32,53-62`
|
||||
- **问题:** Token 创建时不设置 `aud`(受众)和 `iss`(签发者),`decode_token` 也不验证。如果同一 JWT 密钥被意外共享,一个服务的 token 可在另一个服务中复用。
|
||||
- **修复:** 创建 token 时设置 `aud="sakura-cloud"`,解码时验证。
|
||||
|
||||
### H12. 小红书插件:代理凭据暴露在 URL 字符串中
|
||||
|
||||
- **文件:** `plugins/xiaohongshu/plugin.py:107-110`
|
||||
- **问题:** `ProxyConfig.url` 属性将明文用户名密码嵌入 URL:`"https://user:pass@host:port"`。如果这个 URL 字符串被记录到 Playwright 调试日志、异常堆栈或错误消息中,凭据泄露。
|
||||
- **修复:** 通过 Playwright 的 `proxy` context 选项分别传入 `username`/`password`(lines 368-371 已部分正确),`.url` 属性不包含凭据。
|
||||
|
||||
### H13. Flutter:默认 API URL 用 HTTP 明文
|
||||
|
||||
- **文件:** `sakura_mobile/lib/src/core/api_client.dart:9`
|
||||
- **代码:** `const defaultCloudApi = 'http://api.kunkunxiaomao.top:8000'`
|
||||
- **问题:** 与桌面端 C6 相同——明文 HTTP 传输所有凭据和内容。
|
||||
- **修复:** 改为 HTTPS。
|
||||
|
||||
### H14. Flutter:Push notification token refresh 监听器持有过期凭据
|
||||
|
||||
- **文件:** `sakura_mobile/lib/src/core/push_notifications.dart:66-83`
|
||||
- **问题:** `onTokenRefresh.listen` 捕获注册时的 `baseUrl` 和 `accessToken`。如果用户后续登录其他服务器或换号,监听器仍用旧凭据,导致 push 注册静默失败(line 79 catch 吞错)。
|
||||
- **修复:** 每次 token 刷新时从当前 provider 读取最新凭据。
|
||||
|
||||
### H15. 桌面端:SQLite 跨线程共享缺少显式锁
|
||||
|
||||
- **文件:** `pixiv_app/core/library.py:29`
|
||||
- **代码:** `check_same_thread=False` 允许跨线程共享连接,"当有锁保护时"——但锁的保护由调用者负责且不一致。
|
||||
- **问题:** `TaskQueue`(`enqueue.py:232`)将连接传给多个 `ThreadPoolExecutor` worker,没有外部锁。WAL 模式下 SQLite 通常能处理,但高并发写入时仍可能 `SQLITE_BUSY`。
|
||||
- **修复:** 改用连接池(如 `sqlite3` 的 URI 模式 `file:path?mode=rw`),或每个 worker 使用独立连接。
|
||||
|
||||
### H16. 桌面端:画廊同步对每个文件行新开 DB 连接(O(N) 连接泛滥)
|
||||
|
||||
- **文件:** `pixiv_app/gui/app.py:2175-2184`
|
||||
- **问题:** `_sync_pixiv_rows` 中**每个文件行**都调用 `sqlite3.connect(DEFAULT_LIBRARY_DB)` → 查询 → `close()`。100 个文件同步 = 100 次 DB 打开/关闭循环。性能低下且在高负载下可能触及 OS 文件描述符上限。
|
||||
- **修复:** 复用单个连接或将查询批量化。
|
||||
|
||||
### H17. Cloud 后端:用户登录无限流(跨所有端点)
|
||||
|
||||
- **问题范围:** `/login`(auth.py:56)、`/register`(auth.py:24)、`/token/refresh`(auth.py:84)、所有 gallery 端点、所有 novel 端点、所有 sync 端点——除 admin 外**无一有限流**。
|
||||
- **修复:** 全局应用 rate limiting 中间件(如 `slowapi` 或自定义中间件)。
|
||||
|
||||
### H18. Cloud 后端:`--no-sandbox` 同样存在于 X 插件
|
||||
|
||||
- **文件:** `plugins/x/plugin.py`(Playwright 启动参数)
|
||||
- **问题:** 与 C3 相同风险。X/Twitter 页面同样是不受信任的外部内容。
|
||||
- **修复:** 移除或至少评估必要性。
|
||||
|
||||
### H19. 桌面端:N+1 查询模式在关注作者同步中
|
||||
|
||||
- **文件:** `pixiv_app/core/gallery_service.py:547-561`
|
||||
- **问题:** `sync_followed_artists` 对每个关注的作者创建 → 查询 → 关闭独立的 `PixivLibrary` 实例。50 个作者 = 100+ 次 DB 打开/关闭。
|
||||
- **修复:** 使用池化或缓存的连接。
|
||||
|
||||
### H20. Flutter:多个页面存在静默吞错 → 空列表
|
||||
|
||||
|| 文件 | 位置 | 模式 |
|
||||
||------|------|------|
|
||||
|| `novels_tab.dart` | 63-67 | `.catchError((_) => <dynamic>[])` |
|
||||
|| `api_client.dart` | 532 | `_readCache` catch 返回 null |
|
||||
|| `providers.dart` | 17-21 | `initializeIfFirstRun()` 的 future 被丢弃 |
|
||||
|
||||
- **影响:** API 失败时用户看到空书架/空画廊/空收藏,无法区分"真的没数据"和"加载失败了"。
|
||||
- **修复:** 区分 error 状态并在 UI 中显示错误提示和重试按钮。
|
||||
|
||||
---
|
||||
|
||||
## 三、Medium(中危)
|
||||
|
||||
### 后端
|
||||
|
||||
| # | 文件 | 行号 | 问题 |
|
||||
|---|------|------|------|
|
||||
| M1 | `sakura_cloud/main.py` | 130-131 | asyncio background task 无引用监控,异常静默死亡 |
|
||||
| M2 | `sakura_cloud/api/middleware.py` | 77 | `img-src https: http:` 完全取消 CSP 图片保护 |
|
||||
| M3 | `sakura_cloud/api/middleware.py` | 79 | `style-src 'unsafe-inline'` 可被 XSS 利用 |
|
||||
| M4 | `sakura_cloud/core/config.py` | 31 | access token 有效期 24h 过长,token 被盗后窗口大 |
|
||||
| M5 | `sakura_cloud/core/config.py` | 56-58 | FCM 默认启用但无有效凭据,运行时报错 |
|
||||
| M6 | `sakura_cloud/core/config.py` | 18 | Redis 默认 URL 无密码认证 |
|
||||
| M7 | `sakura_cloud/api/v1/endpoints/gallery.py` | 324-332 | 删除 artwork 时文件不清理 → 磁盘孤儿文件累积 |
|
||||
| M8 | `sakura_cloud/api/v1/endpoints/novels.py` | 53-73 | GET 请求触发 DB 写操作(`_resolve_novel_storage_path`),违反 REST 幂等性 |
|
||||
| M9 | `sakura_cloud/api/v1/endpoints/files.py` | 39 | 缩略图生成同步阻塞 FastAPI 事件循环 |
|
||||
| M10 | `sakura_cloud/api/v1/endpoints/sync.py` | 38,100,138 | metadata dict 无大小限制 → 数据库 bloat 攻击 |
|
||||
| M11 | `sakura_cloud/api/v1/endpoints/sync.py` | 37,99,138 | tags 列表无长度/元素限制 → 数据库 bloat 攻击 |
|
||||
| M12 | `sakura_cloud/api/v1/endpoints/auth.py` | 9 | 密码复杂度仅 `min_length=8`,弱密码可过 |
|
||||
| M13 | `pixiv_app/core/downloader.py` | 46-48 | 文件名未做长度限制,Windows MAX_PATH 260 字符溢出导致 OSError |
|
||||
| M14 | `pixiv_app/core/downloader.py` | 476,502,537 | 大量 `except Exception: continue` 静默丢错,生产调试不可能 |
|
||||
| M15 | `pixiv_app/core/downloader.py` | 690,901-945 | 每个作品创建新 `requests.Session()`(无 keep-alive),100 作品 = 100 次 TLS 握手 |
|
||||
| M16 | `pixiv_app/core/gallery_service.py` | 399-414 | `_create_sync_manager` 在 3 个文件中重复实现 |
|
||||
| M17 | `pixiv_app/core/origin_follow_sync.py` | 103 | `asyncio.run()` 在可能已有 event loop 的同步上下文中调用 |
|
||||
| M18 | `pixiv_app/core/proxy_pool.py` | 213 | 代理测试时 `verify=False`,代理自身可 MITM 注入虚假成功结果 |
|
||||
| M19 | `pixiv_app/core/auth.py` | 36-41 | `SessionStore.load()` 用裸 `except Exception` 吞所有异常(JSON 损坏、权限错误等) |
|
||||
| M20 | `pixiv_app/core/gallery_service.py` | 79-81 | tag 筛选用 `JOIN`(非 `LEFT JOIN`),孤儿 `artwork_tags` 行导致作品静默遗漏 |
|
||||
| M21 | `pixiv_app/core/downloader.py` | 698 | `int(detail.get("pageCount") or 0)` 遇非数字字符串抛 `ValueError` |
|
||||
|
||||
### 插件
|
||||
|
||||
| # | 文件 | 行号 | 问题 |
|
||||
|---|------|------|------|
|
||||
| M22 | `plugins/pixiv/plugin.py` | 243-244 | 文件名从 URL 提取,URL 编码的 `%2F` 可绕过 `rsplit` 做路径穿越 |
|
||||
| M23 | `plugins/x/plugin.py` | 217-218,386 | 每次操作启动新 Playwright browser context,无复用 → 资源耗尽 |
|
||||
| M24 | `plugins/biquge/plugin.py` | 845-867 | Playwright page 实例缓存跨操作复用,一次失败污染后续 |
|
||||
| M25 | `plugins/ao3/plugin.py` | 503-506 | EPUB 解压无 zip bomb 防护,压缩炸弹可 OOM |
|
||||
| M26 | `plugins/ao3/plugin.py` | 92-96 | `Retry-After` 头无上限,`Retry-After: 999999999` 可使插件休眠 ~31 年 |
|
||||
| M27 | `plugins/xiaohongshu/plugin.py` | 377 | `headless=False` 在服务器/无 GUI 环境运行失败 |
|
||||
| M28 | `plugins/xiaohongshu/plugin.py` | 480-588 | `page.evaluate()` 执行不受信任页面的 JS,结果未经域名验证 |
|
||||
| M29 | `plugins/xiaohongshu/plugin.py` | 630-648 | 下载图片时转发页面所有 cookies 给图片 CDN(含第三方追踪 cookie) |
|
||||
|
||||
### Flutter
|
||||
|
||||
| # | 文件 | 行号 | 问题 |
|
||||
|---|------|------|------|
|
||||
| M30 | `api_client.dart` | 440-502 | `translateText()` 每次调用创建并销毁新的 Dio 实例 |
|
||||
| M31 | `api_client.dart` | 518-535 | `_readCache<T>` 泛型类型不安全——声称返回任何 T 但 SharedPreferences 只存 JSON 基本类型 |
|
||||
| M32 | `providers.dart` | 17-21 | `sourceManager.initializeIfFirstRun()` 是 async 但返回值被丢弃 |
|
||||
| M33 | `app.dart` | 24 | 硬编码 Windows 字体 `Microsoft YaHei UI`,iOS/macOS/Linux 回退不一致 |
|
||||
| M34 | `gallery_tab.dart` | 269-278 | 收藏/取消收藏触发全量列表重载(`_refresh()`),非单项更新 |
|
||||
| M35 | `gallery_tab.dart` | 336-367 | 批量下载用 `for` + 顺序 `await`,无并发无进度 |
|
||||
| M36 | `login_page.dart` | 141 | TabBarView 高度硬编码 430px,不同屏幕尺寸下可能裁剪或留白 |
|
||||
| M37 | `novel_reader_page.dart` | 17-32 | 14 个可变实例字段——翻页设置(字号/行高/主题)在每次页面跳转时丢失 |
|
||||
| M38 | `novel_detail_page.dart` | 67 | 硬编码 `430`px 滚动偏移到目录区,不随内容高度自适应 |
|
||||
| M39 | `reminder_center_page.dart` | 22-35 | `results[0]! as Map` 等不安全类型强转,API 返回异常形状时抛 `TypeError` |
|
||||
| M40 | `profile_tab.dart` | 403 | `logout()` 是 async 但 button onPressed 丢弃返回的 Future |
|
||||
| M41 | `profile_tab.dart` + `reminder_center_page.dart` | 44-97 | 提醒时间设置逻辑完全重复两份 |
|
||||
| M42 | `favorites_page.dart` | 51-54 | 每次点击 artwork 都重新执行 O(n) 的 `buildMangaCollections` |
|
||||
| M43 | `novels_tab.dart` | 461 | epoch 0 作为时间戳兜底 → 显示"53 年前",无意义 |
|
||||
| M44 | `manga_collections.dart` | 59 | 不同 null-author 作者的同名作品被错误合并为同一合集 |
|
||||
|
||||
### Cloud 后端(补充)
|
||||
|
||||
| # | 文件 | 行号 | 问题 |
|
||||
|---|------|------|------|
|
||||
| M45 | `sakura_cloud/main.py` | 123-126 | 启动时 digest 发送失败仅打日志,无告警 |
|
||||
| M46 | `sakura_cloud/core/security.py` | 54-59 | `decode_token` 冗余手动检查 `exp`,且未设 clock skew 容差 |
|
||||
|
||||
---
|
||||
|
||||
## 四、Low(低危)
|
||||
|
||||
### 后端
|
||||
|
||||
| # | 文件 | 行号 | 问题 |
|
||||
|---|------|------|------|
|
||||
| L1 | `sakura_cloud/main.py` | 113-131 | startup 事件阻塞事件循环,无就绪探针/超时 |
|
||||
| L2 | `sakura_cloud/api/middleware.py` | 32-42 | debug 模式 CSP `connect-src` 开放 `localhost:*`(所有本地端口) |
|
||||
| L3 | `sakura_cloud/api/deps.py` | 29 | `str(user_id)` 对 None 产生字面字符串 `"None"` |
|
||||
| L4 | `sakura_cloud/api/deps.py` | 34 | `token_version` 比较中,双方都为 0/None 时通过,旧 token 缺少 `tv` 声明不被拒绝 |
|
||||
| L5 | `sakura_cloud/core/config.py` | 89-103 | `json.loads()` 在 field validator 中内联 import,不标准 |
|
||||
| L6 | `sakura_cloud/api/v1/endpoints/auth.py` | 79-81 | logout 不递增 `token_version` |
|
||||
| L7 | `sakura_cloud/api/v1/endpoints/gallery.py` | 51-97 | `ILIKE` + `plainto_tsquery` 用户输入无长度限制 → 可能触发慢查询 |
|
||||
| L8 | `sakura_cloud/api/v1/endpoints/files.py` | 32 | `original_name` 用户可控 → 下载附件名可用于社工 |
|
||||
| L9 | `sakura_cloud/api/v1/endpoints/sync.py` | 25 | SHA256 字段只验证长度不验证 hex 字符 |
|
||||
| L10 | `sakura_cloud/api/v1/endpoints/sync.py` | 144-157 | 设备注册无每用户数量上限 |
|
||||
| L11 | `pixiv_app/core/downloader.py` | 40 | `PixivRequestError` 继承 `RuntimeError` 而非 `Exception`(语义不当) |
|
||||
| L12 | `pixiv_app/core/downloader.py` | 233 | `str(user_id).isdigit()` 中 `user_id` 可能为 None → `str(None)` = `"None"` |
|
||||
| L13 | `pixiv_app/core/downloader.py` | 700,709,771 | 同一 artwork 三次调用 `ensure_artwork_stub`(`INSERT OR IGNORE`,无害但浪费) |
|
||||
| L14 | `pixiv_app/core/gallery_service.py` | 588-613 | 同一 artwork 多次同步中重复创建 alert |
|
||||
| L15 | `pixiv_app/core/x_media_service.py` | 522-523 | `datetime.now()` 无时区 → 跨时区对比可能偏差一天 |
|
||||
| L16 | `pixiv_app/gui/app.py` | 1153-1157 | 小红书页缩进不一致(7 空格,PEP 8 违规) |
|
||||
| L17 | `sakura_cloud/core/security.py` | 54-59 | `decode_token` 的 `exp` 手动检查与 `jwt.decode` 内建检查重复 |
|
||||
|
||||
### 插件
|
||||
|
||||
| # | 文件 | 行号 | 问题 |
|
||||
|---|------|------|------|
|
||||
| L18 | `plugins/x/plugin.py` | 414 | 下载缓存 JSON 暴露本地绝对路径结构 |
|
||||
| L19 | `plugins/x/plugin.py` | 463-475 | 文件类型仅凭 URL 子串匹配判断,无 MIME 验证 |
|
||||
| L20 | `plugins/biquge/plugin.py` | 135,360-361 | `requests.Session()` 无连接池限制 |
|
||||
| L21 | `plugins/xiaohongshu/plugin.py` | 375-376 | 浏览器 profile 跨会话共享,可能积累旧 cookies 泄漏 |
|
||||
|
||||
### Flutter
|
||||
|
||||
| # | 文件 | 行号 | 问题 |
|
||||
|---|------|------|------|
|
||||
| L22 | `app.dart` | 11-16 | `ColorScheme.fromSeed()` 在 build 中创建(实际仅一次,低影响) |
|
||||
| L23 | `artwork_detail_page.dart` | 264-265 | `showDialog` 异常抛出时 TextEditingController 泄漏 |
|
||||
| L24 | `online_manga_reader_page.dart` | 101-103 | 源 headers 直接传给 `Image.network`,第三方源可能捕获凭据 |
|
||||
| L25 | `novel_reader_page.dart` | 78,330 | `Future.microtask` 在 widget 可能已 dispose 的上下文中执行 |
|
||||
| L26 | `novels_tab.dart` | 446,461 | 时间戳兜底使用 epoch 0 → 显示"53 年前" |
|
||||
| L27 | `explore_page.dart` | 186-222 | `_openItem` 从 provider 同步读取 source,可能搜完源已被删除 |
|
||||
| L28 | `source_manager.dart` | 86-96 | `initializeIfFirstRun()` 的 initialized flag 在可能失败的 `load()` 之后无条件设为 true |
|
||||
| L29 | `source_manager.dart` | 242-260 | 从 JSON 导入时,源的 id 由 name+URL 的 codeUnit 求和生成——不同源可能碰撞 |
|
||||
| L30 | `legado_engine.dart` | 167-200 | `_encodeKeyword` 对非 UTF-8/GBK 编码的平台通道调用无超时 |
|
||||
|
||||
---
|
||||
|
||||
## 五、跨模块问题汇总
|
||||
|
||||
| 问题类别 | 涉及模块 | 严重程度 |
|
||||
|---------|---------|:--------:|
|
||||
| **凭据明文存储** | 桌面端 + 5 个插件 + Flutter | Critical/High(7 处) |
|
||||
| **HTTP 明文传输** | 桌面端 API + Flutter API | Critical/High(2 处) |
|
||||
| **默认密码/密钥** | Cloud config(JWT、DB、MinIO、Redis) | Critical/High(4 处) |
|
||||
| **无限流** | Cloud 全部用户端点 | High(17+ 端点) |
|
||||
| **存储配额未执行** | Cloud gallery + sync | High(2 处) |
|
||||
| **静默吞错 → 空列表** | Flutter + 桌面 downloader + session | High/Medium(10+ 处) |
|
||||
| **线程安全** | 桌面 ProxyPool + SQLite + counter | Critical/Medium(3 处) |
|
||||
| **资源泄漏** | 桌面 DB 连接 + Flutter Dio × translate + X 插件 browser | High/Medium(3 处) |
|
||||
| **Token 生命周期** | Cloud auth(logout、refresh、expiry、version) | High/Medium(5 处) |
|
||||
| **插件安全** | `--no-sandbox` + 硬编码密钥 + zip bomb + URL 路径穿越 | Critical/Medium(5 处) |
|
||||
| **Flutter 状态管理** | FutureBuilder 模式 + async 构造器 + provider 初始化 | High/Medium(5 处) |
|
||||
|
||||
---
|
||||
|
||||
## 六、修复优先级建议
|
||||
|
||||
### 第一轮(立即修复,1–3 天)
|
||||
|
||||
| 顺序 | ID | 问题 | 工时 |
|
||||
|:----:|:---|------|:----:|
|
||||
| 1 | C1,C4,C5 | Cloud 默认凭据:JWT 密钥 + DB 密码 + MinIO 凭据 | 1h |
|
||||
| 2 | C3,H18 | 移除 `--no-sandbox`(小红书 + X 插件) | 1h |
|
||||
| 3 | C6,H13 | HTTP → HTTPS(桌面端 + Flutter) | 2h |
|
||||
| 4 | C9 | ProxyPool 多线程数据竞争(加锁) | 2h |
|
||||
| 5 | C7 | Firebase 初始化 flag 移到成功之后 | 10m |
|
||||
| 6 | C2 | 笔趣阁 AES 密钥改为随机 IV + 安全随机密钥 | 2h |
|
||||
| 7 | H1,H2,H17 | Cloud 端全局速率限制 | 4h |
|
||||
| 8 | H3 | 存储配额强制执行 | 2h |
|
||||
|
||||
### 第二轮(本周内,3–5 天)
|
||||
|
||||
| 顺序 | ID | 问题 | 工时 |
|
||||
|:----:|:---|------|:----:|
|
||||
| 9 | H4,H5 | Token 撤销(logout + refresh rotation) | 4h |
|
||||
| 10 | H16 | 云同步 O(N) DB 连接改为复用连接 | 2h |
|
||||
| 11 | H15 | SQLite 跨线程连接池化 | 3h |
|
||||
| 12 | H7 | 小说翻页 API 增加 500ms 防抖 | 1h |
|
||||
| 13 | H8 | 探索页"插画→小说"模式修复 | 1h |
|
||||
| 14 | C8 | gallery_tab `recentUpdatesOnly` 死开关修复 | 1h |
|
||||
| 15 | H19 | 关注作者同步 N+1 查询优化 | 3h |
|
||||
| 16 | H20 | 多处静默吞错 → 显式错误状态 UI | 4h |
|
||||
| 17 | M8 | GET novel 路径解析触发的 DB 写操作改为 lazily 计算 | 2h |
|
||||
|
||||
### 第三轮(下一迭代)
|
||||
|
||||
| 顺序 | 类别 | 内容 | 工时 |
|
||||
|:----:|------|------|:----:|
|
||||
| 18 | 凭据加密 | 7 处明文凭据改为 OS 密钥链存储 | 8h |
|
||||
| 19 | CSP | CSP 策略加固(nonce/hash 替代 unsafe-inline) | 4h |
|
||||
| 20 | 测试 | 为 Critical/High 项编写回归测试 | 8h |
|
||||
| 21 | Flutter 重构 | AsyncNotifier 替代 FutureBuilder(feed_tab, gallery_tab, novels_tab) | 8h |
|
||||
| 22 | 插件审计 | 对全部 5 个插件的网络/文件访问权做沙箱化评估 | 4h |
|
||||
|
||||
---
|
||||
|
||||
> **审核范围确认:** Flutter 移动端 30 个 Dart 文件 × Python 桌面端 16 个文件 × Cloud 后端 10 个端点文件 × 5 个插件 × 中间件/配置/安全模块。合计约 65 个源文件,发现 111 个问题。
|
||||
@@ -0,0 +1,507 @@
|
||||
# 樱云下载器 — 优化方案与功能规划
|
||||
|
||||
> 版本:2.1.0-draft | 日期:2026-06-24
|
||||
> 基于代码审计报告、产品文档、CHANGELOG 及全仓库源码分析制定
|
||||
|
||||
---
|
||||
|
||||
## 目录
|
||||
|
||||
1. [优先级定义](#0-优先级定义)
|
||||
2. [P0 — 安全与基础设施(必须修)](#1-p0--安全与基础设施)
|
||||
3. [P1 — 核心体验缺陷(尽快修)](#2-p1--核心体验缺陷)
|
||||
4. [P2 — 功能完善与增强](#3-p2--功能完善与增强)
|
||||
5. [P3 — 体验优化与锦上添花](#4-p3--体验优化与锦上添花)
|
||||
6. [P4 — 长期架构演进](#5-p4--长期架构演进)
|
||||
7. [实施路线图](#6-实施路线图)
|
||||
|
||||
---
|
||||
|
||||
## 0. 优先级定义
|
||||
|
||||
| 等级 | 含义 | 建议时间 |
|
||||
|------|------|---------|
|
||||
| **P0** | 安全漏洞 / 数据风险 / 系统崩溃 | 1 周内 |
|
||||
| **P1** | 核心功能阻断或严重缺陷 | 2–4 周 |
|
||||
| **P2** | 功能缺口 / 重要增强 | 1–2 月 |
|
||||
| **P3** | 体验优化 / 锦上添花 | 按需排期 |
|
||||
| **P4** | 架构演进 / 平台扩展 | 中长期规划 |
|
||||
|
||||
---
|
||||
|
||||
## 1. P0 — 安全与基础设施
|
||||
|
||||
> 来源:代码审计报告 Critical 项 + 其他发现
|
||||
|
||||
### 1.1 JWT 密钥强制生产配置
|
||||
|
||||
- **现状**:`sakura_cloud/core/config.py` 中 `jwt_secret_key` 默认值为 `"change-this-secret-in-production"`,任何人都可伪造 JWT
|
||||
- **方案**:启动时检测是否为已知默认值,若是则拒绝启动并输出明确的错误信息,引导运维设置 `JWT_SECRET_KEY` 环境变量
|
||||
- **影响范围**:`sakura_cloud/core/config.py`、`sakura_cloud/core/security.py`
|
||||
|
||||
### 1.2 数据库凭据与 MinIO 凭据硬编码
|
||||
|
||||
- **现状**:
|
||||
- PostgreSQL 默认凭据 `sakura:sakura`
|
||||
- MinIO 默认凭据 `minioadmin:minioadmin`
|
||||
- **方案**:同 JWT 策略——启动时检测默认值,若未被覆盖则拒绝启动;为 MinIO 增加随机密码生成脚本
|
||||
- **影响范围**:`sakura_cloud/core/config.py`、部署文档
|
||||
|
||||
### 1.3 桌面端 → 云端启用 HTTPS
|
||||
|
||||
- **现状**:`DEFAULT_CLOUD_API = "http://api.kunkunxiaomao.top:8000"` 使用明文 HTTP,用户凭据和文件明文传输
|
||||
- **方案**:
|
||||
- 短期:支持 HTTP 但增加显式警告弹窗("当前使用非加密连接,凭据可能被窃取")
|
||||
- 长期:服务器配置 SSL 证书(Let's Encrypt / 自签名),客户端默认改为 HTTPS,初次连接时做证书校验引导
|
||||
- **影响范围**:`pixiv_app/gui/app.py`、`pixiv_app/core/cloud_sync.py`、云端 nginx/Caddy 配置
|
||||
|
||||
### 1.4 登录接口限流
|
||||
|
||||
- **现状**:`/api/v1/auth/login` 无限流,可暴力破解
|
||||
- **方案**:对 `/login`、`/register`、`/token/refresh` 增加 IP 级别速率限制(如每分钟 5 次),使用 Redis 或内存计数器实现
|
||||
- **影响范围**:`sakura_cloud/api/v1/endpoints/auth.py`、新增限流中间件
|
||||
|
||||
### 1.5 笔趣阁插件 AES 密钥随机化
|
||||
|
||||
- **现状**:`plugins/biquge/plugin.py` 中 AES 密钥和 IV 从硬编码字符串经 MD5 派生,固定 IV 下的 CBC 模式可被已知明文攻击
|
||||
- **方案**:
|
||||
- 每次加密使用随机 IV(`os.urandom(16)`),与密文一起传输
|
||||
- 密钥从用户配置或安全随机源派生,而非硬编码
|
||||
- **影响范围**:`plugins/biquge/plugin.py`
|
||||
|
||||
### 1.6 Chrome `--no-sandbox` 移除
|
||||
|
||||
- **现状**:小红书插件使用 `--no-sandbox` 禁用 Chromium 沙箱,浏览器以完整 OS 权限运行不受信任的外部内容
|
||||
- **方案**:移除 `--no-sandbox`。若在 Docker/root 环境下必须使用,改为配置 `--cap-add=SYS_ADMIN` 或 seccomp profile
|
||||
- **影响范围**:`plugins/xiaohongshu/plugin.py`
|
||||
|
||||
### 1.7 Firebase 初始化竞态修复
|
||||
|
||||
- **现状**:`push_notifications.dart` 中 `_initialized = true` 在 `await Firebase.initializeApp()` **之前**设置,若初始化失败则永久不再重试
|
||||
- **方案**:将 `_initialized = true` 移到 `await Firebase.initializeApp()` 成功之后
|
||||
- **影响范围**:`sakura_mobile/lib/src/core/push_notifications.dart`
|
||||
|
||||
### 1.8 ProxyPool 多线程数据竞争
|
||||
|
||||
- **现状**:`current_index` 无锁读写、`working_proxies` 并发修改、`fail_count`/`success_count` 非原子递增
|
||||
- **方案**:引入 `threading.Lock` 保护所有对共享数据的写操作;或将 `current_index` 改为 `itertools.cycle` + 原子计数器
|
||||
- **影响范围**:`pixiv_app/core/proxy_pool.py`
|
||||
|
||||
### 1.9 敏感文件 `.gitignore` 完善
|
||||
|
||||
- **现状**:`.env.cloud.example` 已提供模板,但不清楚 `.env` 是否在 `.gitignore` 中;会话文件 `app_session.json` 可能被误提交
|
||||
- **方案**:在 `.gitignore` 中显式添加 `*.env`(除 `.example`)、`app_session.json`、`runtime/`、`*.db`、`*.key`
|
||||
- **影响范围**:`.gitignore`
|
||||
|
||||
---
|
||||
|
||||
## 2. P1 — 核心体验缺陷
|
||||
|
||||
> 来源:代码审计报告 High/Medium 项 + 已知 bug + 功能缺口
|
||||
|
||||
### 2.1 手机端 `recentUpdatesOnly` 死开关修复
|
||||
|
||||
- **现状**:`gallery_tab.dart` 中 `recentUpdatesOnly` 状态变量只控制 Chip 高亮,从未作为 API 参数传入 `_load()`,切换无实际效果
|
||||
- **方案**:在 `_load()` 中将 `recentUpdatesOnly` 转换为 `order=updated_at` 或后端新增 `recent_only` 查询参数;或如果后端暂时不支持,先移除该 Chip
|
||||
- **影响范围**:`sakura_mobile/lib/src/pages/tabs/gallery_tab.dart`
|
||||
|
||||
### 2.2 Legado 书源搜索引擎完善
|
||||
|
||||
- **现状**:6 个默认种子源 `searchUrl` / `searchListRule` 全为 null,CSS 选择器解析不支持 class/ID/属性选择器,GBK 编码无容错
|
||||
- **方案**(分两步):
|
||||
- **2.2a 引擎层**:扩展 CSS 选择器支持 class(`.classname`)、ID(`#id`)、属性(`[attr=val]`)、后代(`A B`)选择器;GBK 解码增加容错参数
|
||||
- **2.2b 源数据层**:用真实可用的书源 JSON 替换默认种子源,或提供从 Legado 仓库一键导入的引导流程
|
||||
- **影响范围**:`legado_rule_parser.dart`、`legado_engine.dart`、`source_manager.dart`
|
||||
|
||||
### 2.3 手机端发现页滚动卡死修复
|
||||
|
||||
- **现状**:发现页 Feed 向上滚动时可能卡在底部
|
||||
- **方案**:排查 `feed_tab.dart` 中的 `ScrollController` 和分页加载逻辑,确保加载更多后正确更新 `hasMore` 状态,以及 `ListView` 的 `addPostFrameCallback` 不产生死循环
|
||||
- **影响范围**:`sakura_mobile/lib/src/pages/tabs/feed_tab.dart`
|
||||
|
||||
### 2.4 Web 管理后台页面完善
|
||||
|
||||
- **现状**:普通用户 Web 端详情页已从占位改为真实加载,但管理后台可能仍有死角
|
||||
- **方案**:
|
||||
- 完成管理员端用户管理、内容审核、统计面板页面
|
||||
- 普通用户端确保图片/漫画/小说三种内容类型均有完整的列表+详情页
|
||||
- **影响范围**:`sakura_admin_web/`
|
||||
|
||||
### 2.5 云同步上传失败队列与断点续传
|
||||
|
||||
- **现状**:`SAKURA_CLOUD_V1_STATUS.md` 明确标注"上传失败队列与断点恢复细节"尚未完成
|
||||
- **方案**:
|
||||
- 桌面端新增 `sync_failures` 表记录上传失败的文件,支持自动/手动重试
|
||||
- 上传分片支持断点续传(保存已上传分片索引)
|
||||
- 云端 `upload/complete` 接口增加分片校验
|
||||
- **影响范围**:`pixiv_app/core/cloud_sync.py`、`sakura_cloud/api/v1/endpoints/upload.py`
|
||||
|
||||
### 2.6 Alembic 数据库迁移完善
|
||||
|
||||
- **现状**:SAKURA_CLOUD_V1_STATUS 标注 Alembic 迁移尚未完成
|
||||
- **方案**:补全所有模型的 Alembic migration 脚本,确保 `alembic upgrade head` 一键建库;新增 CI 中检查迁移一致性
|
||||
- **影响范围**:`sakura_cloud/migrations/`
|
||||
|
||||
### 2.7 手机端 App 正式签名
|
||||
|
||||
- **现状**:APK 使用调试签名,不适合正式分发;README 和用户手册均强调此限制
|
||||
- **方案**:
|
||||
- 生成正式 keystore
|
||||
- 更新 `sakura_mobile/android/app/build.gradle.kts` 签名配置
|
||||
- 文档更新,区分 debug/release 构建流程
|
||||
- **影响范围**:`sakura_mobile/android/`、构建脚本
|
||||
|
||||
### 2.8 桌面端 Cookie 过期自动检测与提醒
|
||||
|
||||
- **现状**:Pixiv Cookie 过期后抓取静默失败,用户需手动排查
|
||||
- **方案**:
|
||||
- 在每次抓取前自动检测会话有效性(已有「检测会话」按钮逻辑,改为自动调用)
|
||||
- Cookie 将过期或已过期时弹窗提醒并引导重新导入
|
||||
- **影响范围**:`pixiv_app/core/auth.py`、`pixiv_app/gui/app.py`
|
||||
|
||||
### 2.9 下载失败自动重试与错误分类
|
||||
|
||||
- **现状**:`runtime/executor.py` 已有基本重试与错误分类,但未区分限流(429)、临时故障(5xx)、权限错误(403/404)
|
||||
- **方案**:
|
||||
- 429 → 自动退避重试(指数退避 + 抖动)
|
||||
- 5xx → 有限重试(最多 2 次)
|
||||
- 403/404 → 不重试,直接标记失败
|
||||
- 失败原因写入 `download_tasks` 表便于排查
|
||||
- **影响范围**:`pixiv_app/runtime/executor.py`、`pixiv_app/tasks/models.py`
|
||||
|
||||
### 2.10 图库搜索性能优化
|
||||
|
||||
- **现状**:图库 SQLite 查询在大数据量下(数万条作品)可能变慢,标签模糊搜索使用 `LIKE '%keyword%'` 无法走索引
|
||||
- **方案**:
|
||||
- 对 `artworks.title`、`artists.name`、`tags.name` 建立索引
|
||||
- 标签搜索改为 FTS5 全文索引
|
||||
- 云端的 `gallery` 查询为 `include_total` 参数已有优化,进一步对 `artworks.created_at`、`artworks.artist_id` 建立联合索引
|
||||
- **影响范围**:`pixiv_app/core/library.py`、`sakura_cloud/models/artwork.py`
|
||||
|
||||
---
|
||||
|
||||
## 3. P2 — 功能完善与增强
|
||||
|
||||
### 3.1 手机端推送通知(FCM)
|
||||
|
||||
- **现状**:Firebase 已集成但推送链路不完整;当前"每日提醒"仅是应用内摘要,非系统级推送
|
||||
- **方案**:
|
||||
- 完成 FCM token 注册与服务端对接
|
||||
- 云端提醒生成后通过 FCM 发送系统通知
|
||||
- 点击通知直接跳转到对应内容
|
||||
- 用户可配置"仅 Wi-Fi 下推送"、"免打扰时段"
|
||||
- **影响范围**:`sakura_mobile/lib/src/core/push_notifications.dart`、`sakura_cloud/services/push.py`
|
||||
|
||||
### 3.2 插件市场在线对接
|
||||
|
||||
- **现状**:`PluginMarketClient` 为占位类,仅支持本地文件系统发现插件
|
||||
- **方案**:
|
||||
- 云端提供 `/api/v1/plugins/` 端点(列表、下载、版本检查)
|
||||
- 桌面端「插件管理」面板增加"在线市场"标签页
|
||||
- 支持一键安装/更新/卸载
|
||||
- 插件版本兼容性检查
|
||||
- **影响范围**:`pixiv_app/core/plugin/market.py`、`pixiv_app/gui/plugin_panel.py`、`sakura_cloud/` 新增端点
|
||||
|
||||
### 3.3 视频下载与播放支持
|
||||
|
||||
- **现状**:X/Twitter 插件可抓取视频文件,但本地图库和缩略图墙未针对视频做优化;手机端无视频播放页
|
||||
- **方案**:
|
||||
- 缩略图墙增加视频缩略图(自动截取首帧或使用平台提供的 thumbnail)
|
||||
- 手机端新增视频播放页(基于 `video_player` 插件)
|
||||
- 图库列表支持视频类型筛选
|
||||
- **影响范围**:`pixiv_app/core/gallery_service.py`、`sakura_mobile/` 新增页面
|
||||
|
||||
### 3.4 阅读进度跨设备同步
|
||||
|
||||
- **现状**:手机端小说阅读器记录阅读进度,但仅本地存储(`online_reading_history.dart`)
|
||||
- **方案**:
|
||||
- 云端新增 `/api/v1/reading-progress/` 端点(GET/PUT)
|
||||
- 手机端在进度变更时自动上报云端
|
||||
- 桌面端缩略图墙也可展示阅读进度
|
||||
- 多设备间同账号自动同步阅读位置
|
||||
- **影响范围**:`sakura_mobile/lib/src/core/online_reading_history.dart`、`sakura_cloud/` 新增端点
|
||||
|
||||
### 3.5 内容合集 / 专辑功能
|
||||
|
||||
- **现状**:作品可以按作者、标签归类,但没有用户自定义合集的能力
|
||||
- **方案**:
|
||||
- 桌面端和手机端均支持创建"合集",将任意作品(图片/漫画/小说)归入合集
|
||||
- 合集支持排序、封面设置、备注
|
||||
- 云端同步合集数据
|
||||
- **影响范围**:`pixiv_app/core/library.py`(新增表)、`sakura_cloud/models/`(新增模型)、`sakura_mobile/` 新增页面
|
||||
|
||||
### 3.6 漫画连续阅读模式增强
|
||||
|
||||
- **现状**:手机端已有漫画分组与连续阅读,但体验可以更流畅
|
||||
- **方案**:
|
||||
- 左右滑动翻页(当前可能仅支持上下滚动)
|
||||
- 预加载下一页/上一页(当前页 ± 2 页)
|
||||
- 阅读进度条(当前页/总页数)
|
||||
- 双击缩放、长按保存
|
||||
- **影响范围**:`sakura_mobile/lib/src/pages/online_manga_reader_page.dart`
|
||||
|
||||
### 3.7 批量导出功能
|
||||
|
||||
- **现状**:用户可通过缩略图墙浏览,但无法批量导出原图
|
||||
- **方案**:
|
||||
- 缩略图墙增加"批量选择 → 导出原图"功能
|
||||
- 支持按作者/标签/收藏筛选后全选导出
|
||||
- 导出格式:原图 ZIP 包或保持目录结构
|
||||
- **影响范围**:`pixiv_app/core/gallery_service.py`、缩略图墙前端模板
|
||||
|
||||
### 3.8 X (Twitter) 作者时间线监控
|
||||
|
||||
- **现状**:X 插件支持抓取作者媒体和单个推文,但缺少"监控作者新推文"的主动检测
|
||||
- **方案**:
|
||||
- 桌面端新增"X 监控"面板,可添加监控的 X 用户名
|
||||
- 定时轮询作者时间线,检测新推文 → 自动下载 → 写入云端关注提醒
|
||||
- 与现有每日提醒系统打通
|
||||
- **影响范围**:`plugins/x/`、`pixiv_app/core/x_media_service.py`
|
||||
|
||||
### 3.9 下载内容去重增强
|
||||
|
||||
- **现状**:`download_tasks` 表有 `fingerprint` 唯一约束,`artwork_files.content_hash` 字段已预留但默认空字符串
|
||||
- **方案**:
|
||||
- 下载完成后计算文件 SHA256 并写入 `content_hash`
|
||||
- 新下载前先检查 `content_hash` 是否已存在,跳过重复文件
|
||||
- 缩略图墙可展示"重复文件"列表,支持清理
|
||||
- **影响范围**:`pixiv_app/core/downloader.py`、`pixiv_app/core/library.py`
|
||||
|
||||
### 3.10 代理池健康度评分
|
||||
|
||||
- **现状**:代理有 `fail_count` / `success_count` 但仅用于简单标记失败
|
||||
- **方案**:
|
||||
- 引入代理健康度评分(成功率、响应延迟、可用性百分比)
|
||||
- 代理按健康度排序,优先使用高分代理
|
||||
- 健康度过低的代理自动移出池
|
||||
- 代理池 UI 展示健康度统计
|
||||
- **影响范围**:`pixiv_app/core/proxy_pool.py`
|
||||
|
||||
---
|
||||
|
||||
## 4. P3 — 体验优化与锦上添花
|
||||
|
||||
### 4.1 桌面端深色模式
|
||||
|
||||
- **现状**:GUI 锁定 `ctk.set_appearance_mode("light")`
|
||||
- **方案**:增加"外观"设置项——浅色 / 深色 / 跟随系统;`customtkinter` 原生支持 `dark` 模式,改动极小
|
||||
- **影响范围**:`pixiv_app/gui/app.py`
|
||||
|
||||
### 4.2 缩略图墙 UI 现代化
|
||||
|
||||
- **现状**:缩略图墙是本地 WebUI(Jinja2 模板),功能齐全但 UI 风格较为基础
|
||||
- **方案**:
|
||||
- 引入轻量 CSS 框架(如 Pico.css 或 Water.css)美化
|
||||
- 支持网格/列表视图切换
|
||||
- 图片懒加载(当前可能一次性加载所有缩略图)
|
||||
- 虚拟滚动(大量作品时性能优化)
|
||||
- **影响范围**:`pixiv_app/core/gallery_service.py`、模板文件
|
||||
|
||||
### 4.3 手机端首页个性化配置
|
||||
|
||||
- **现状**:发现页展示"最近浏览的图片/漫画/小说"和"更新提醒"
|
||||
- **方案**:
|
||||
- 用户可选择发现页展示哪些模块(图片/漫画/小说/提醒)
|
||||
- 支持拖拽排序模块
|
||||
- "继续阅读"模块(直接从上次阅读位置进入)
|
||||
- **影响范围**:`sakura_mobile/lib/src/pages/tabs/feed_tab.dart`
|
||||
|
||||
### 4.4 下载速度与流量统计
|
||||
|
||||
- **现状**:桌面端有进度条但无历史统计
|
||||
- **方案**:
|
||||
- 记录每次下载的文件大小、耗时
|
||||
- 主界面增加"本次会话下载量"和"总计下载量"统计
|
||||
- 可查看每日/每周下载量趋势图(可选,使用 matplotlib)
|
||||
- **影响范围**:`pixiv_app/gui/app.py`、`pixiv_app/core/library.py`
|
||||
|
||||
### 4.5 国际化(i18n)支持
|
||||
|
||||
- **现状**:GUI 和手机端 UI 均为中文硬编码
|
||||
- **方案**:
|
||||
- 桌面端:提取所有 UI 字符串到 JSON 语言文件,支持中文/英文/日文
|
||||
- 手机端:使用 Flutter `intl` / `easy_localization` 实现多语言
|
||||
- 优先完成英文翻译以扩大用户群
|
||||
- **影响范围**:全局 UI
|
||||
|
||||
### 4.6 错误日志收集与诊断
|
||||
|
||||
- **现状**:日志分散在 `日志/` 目录和终端输出,排查问题需手动翻找
|
||||
- **方案**:
|
||||
- 桌面端增加"一键导出诊断包"按钮(日志 + 配置 + 数据库状态摘要,压缩为 ZIP)
|
||||
- 手机端增加"上传诊断日志"功能
|
||||
- 云端增加错误聚合面板(按错误类型/时间统计)
|
||||
- **影响范围**:`pixiv_app/`、`sakura_mobile/`、`sakura_cloud/`
|
||||
|
||||
### 4.7 桌面端系统托盘与后台运行
|
||||
|
||||
- **现状**:关闭 GUI 窗口即退出程序
|
||||
- **方案**:
|
||||
- 最小化到系统托盘(使用 `pystray` 或 `customtkinter` 配合系统托盘库)
|
||||
- 托盘菜单:显示/隐藏窗口、开始/停止抓取、退出
|
||||
- 下载完成后托盘气泡提示
|
||||
- **影响范围**:`pixiv_app/gui/app.py`
|
||||
|
||||
### 4.8 自动更新检查
|
||||
|
||||
- **现状**:用户需手动关注 releases 目录获取新版本
|
||||
- **方案**:
|
||||
- 桌面端启动时检查云端最新版本号
|
||||
- 有新版本时提示下载,可选"自动下载 → 下次启动替换"
|
||||
- 手机端同理(检查 APK 版本号)
|
||||
- **影响范围**:`pixiv_app/gui/app.py`、`sakura_mobile/lib/src/app.dart`、`sakura_cloud/` 新增版本检查端点
|
||||
|
||||
---
|
||||
|
||||
## 5. P4 — 长期架构演进
|
||||
|
||||
### 5.1 桌面端从 CustomTkinter 迁移到更现代化的 GUI 框架
|
||||
|
||||
- **理由**:CustomTkinter 功能有限,深色模式、动画、富文本等支持较弱;Windows 11 下风格略显过时
|
||||
- **候选方案**:
|
||||
- **Tauri (Rust + Web 前端)**:体积小、性能好、现代 UI,但需重写全部 Python 逻辑为 Rust 或通过 sidecar 通信
|
||||
- **Electron + Python 后端**:重用现有 Python 下载引擎,Electron 负责 UI,但体积大(~150MB+)
|
||||
- **PySide6 / PyQt6**:纯 Python 方案,迁移成本最低,支持 QSS 样式和丰富控件
|
||||
- **Flet (Flutter for Python)**:用 Python 写 Flutter UI,可与现有 Flutter 移动端共享设计语言
|
||||
- **建议**:优先评估 **Flet**(与 Flutter 移动端风格统一、纯 Python)或 **PySide6**(迁移成本最低)。不宜急于重写,等 P0/P1 完成后评估
|
||||
|
||||
### 5.2 iOS 客户端
|
||||
|
||||
- **理由**:当前仅支持 Android,iOS 用户无法使用
|
||||
- **方案**:
|
||||
- Flutter 跨平台特性使得 iOS 适配成本较低(大部分 Dart 代码可复用)
|
||||
- 需额外处理:iOS 文件系统权限、后台任务限制、APNs 推送替代 FCM
|
||||
- 需要 Apple Developer Program 账号($99/年)用于签名分发
|
||||
- **影响范围**:`sakura_mobile/ios/`(生成)、`sakura_mobile/lib/`(少量平台适配)
|
||||
|
||||
### 5.3 Web 端 PWA(渐进式 Web 应用)
|
||||
|
||||
- **理由**:桌面端 EXE 体积大(~68MB),部分用户只希望轻量浏览
|
||||
- **方案**:
|
||||
- 将缩略图墙升级为 PWA(Service Worker + 离线缓存 + 可安装到桌面)
|
||||
- 或单独开发 `sakura_web` 前端项目替代当前 `sakura_admin_web`
|
||||
- **影响范围**:`sakura_admin_web/` 或新建 `sakura_web/`
|
||||
|
||||
### 5.4 分布式爬虫完善
|
||||
|
||||
- **现状**:`distributed_crawler/` 目录有 Master-Worker 架构但尚不完整
|
||||
- **方案**:
|
||||
- 补全 Worker 节点注册、心跳、任务分配
|
||||
- Master 面板(Web UI)展示各 Worker 状态和任务进度
|
||||
- Docker Compose 一键部署 Master + N Workers
|
||||
- **影响范围**:`distributed_crawler/`
|
||||
|
||||
### 5.5 AI 辅助功能
|
||||
|
||||
- **理由**:PaddleOCR 已集成,可在其基础上扩展更多 AI 能力
|
||||
- **方案**:
|
||||
- **AI 标签自动生成**:对下载的图片使用轻量视觉模型(如 MobileNet/EfficientNet)自动打标签
|
||||
- **翻译增强**:手机端小说翻译当前仅支持"当前页翻译",可升级为整本翻译 + 双语对照
|
||||
- **图片去重**:基于感知哈希(pHash/dHash)的相似图片检测
|
||||
- **内容安全扫描**:对上传至云端的内容进行自动安全审核(NSFW 检测)
|
||||
- **影响范围**:`pixiv_app/core/ocr.py`、`sakura_cloud/services/ocr.py`
|
||||
|
||||
### 5.6 多用户/家庭共享
|
||||
|
||||
- **现状**:云同步为单用户设计,无分享功能
|
||||
- **方案**:
|
||||
- 云端新增"家庭组"/"共享空间"概念
|
||||
- 用户可邀请其他账号加入共享空间
|
||||
- 共享空间内的内容(图库/小说/漫画)成员均可访问
|
||||
- 权限控制:管理员/编辑/只读
|
||||
- **影响范围**:`sakura_cloud/` 新增模型与端点、手机端新增分享页面
|
||||
|
||||
---
|
||||
|
||||
## 6. 实施路线图
|
||||
|
||||
### 第一阶段:安全加固(第 1–2 周)
|
||||
|
||||
```
|
||||
P0-1 JWT 密钥强制生产配置 ████████░░░░░░░░ 1d
|
||||
P0-2 数据库/MinIO 凭据强制覆盖 ████████░░░░░░░░ 1d
|
||||
P0-3 HTTPS 支持 + 警告 ████████████░░░░ 1.5d
|
||||
P0-4 登录接口限流 ████████░░░░░░░░ 1d
|
||||
P0-5 笔趣阁 AES 随机化 ████░░░░░░░░░░░░ 0.5d
|
||||
P0-6 Chrome --no-sandbox 移除 ████░░░░░░░░░░░░ 0.5d
|
||||
P0-7 Firebase 初始化竞态修复 ████░░░░░░░░░░░░ 0.5d
|
||||
P0-8 ProxyPool 线程安全 ████████░░░░░░░░ 1d
|
||||
P0-9 .gitignore 完善 ████░░░░░░░░░░░░ 0.5d
|
||||
────────────────────────────────────────────────────────
|
||||
小计约 7.5 人天
|
||||
```
|
||||
|
||||
### 第二阶段:体验修复(第 3–6 周)
|
||||
|
||||
```
|
||||
P1-1 手机端死开关修复 ████░░░░░░░░░░░░ 0.5d
|
||||
P1-2 Legado 搜索引擎完善 ████████████████ 4d
|
||||
P1-3 发现页滚动卡死 ████████░░░░░░░░ 2d
|
||||
P1-4 Web 管理后台完善 ████████████░░░░ 3d
|
||||
P1-5 上传失败队列 + 断点续传 ████████████████ 4d
|
||||
P1-6 Alembic 迁移 ████████░░░░░░░░ 2d
|
||||
P1-7 正式签名 ████░░░░░░░░░░░░ 1d
|
||||
P1-8 Cookie 过期自动检测 ████████░░░░░░░░ 2d
|
||||
P1-9 下载失败重试优化 ████████░░░░░░░░ 2d
|
||||
P1-10 图库搜索性能 ████████████░░░░ 3d
|
||||
────────────────────────────────────────────────────────
|
||||
小计约 23.5 人天
|
||||
```
|
||||
|
||||
### 第三阶段:功能增强(第 7–14 周)
|
||||
|
||||
```
|
||||
P2-1 FCM 推送通知 ████████████████ 5d
|
||||
P2-2 插件市场在线对接 ████████████████ 5d
|
||||
P2-3 视频下载与播放 ████████████░░░░ 4d
|
||||
P2-4 阅读进度跨设备同步 ████████████░░░░ 4d
|
||||
P2-5 内容合集/专辑 ████████████████ 5d
|
||||
P2-6 漫画连续阅读增强 ████████░░░░░░░░ 3d
|
||||
P2-7 批量导出 ████████████░░░░ 4d
|
||||
P2-8 X 作者时间线监控 ████████████░░░░ 4d
|
||||
P2-9 下载去重增强 ████████░░░░░░░░ 3d
|
||||
P2-10 代理池健康度评分 ████████░░░░░░░░ 3d
|
||||
────────────────────────────────────────────────────────
|
||||
小计约 40 人天
|
||||
```
|
||||
|
||||
### 第四阶段及以后(按需排期)
|
||||
|
||||
```
|
||||
P3-1 深色模式 ████░░ 1d
|
||||
P3-2 缩略图墙 UI 现代化 ██████ 2d
|
||||
P3-3 首页个性化 ██████ 2d
|
||||
P3-4 下载统计 ████░░ 1.5d
|
||||
P3-5 国际化 ██████ 3d
|
||||
P3-6 错误诊断包 ██████ 2d
|
||||
P3-7 系统托盘 ██████ 2d
|
||||
P3-8 自动更新 ██████ 2d
|
||||
P4-X 长期架构演进 TBD
|
||||
────────────────────────────────────────────────────────
|
||||
小计约 15.5 人天
|
||||
```
|
||||
|
||||
### 关键里程碑
|
||||
|
||||
| 里程碑 | 时间 | 交付物 |
|
||||
|--------|------|--------|
|
||||
| **M1: 安全基线** | 第 2 周末 | 所有 P0 项修复完成,安全审计复测通过 |
|
||||
| **M2: 体验基线** | 第 6 周末 | 所有 P1 项修复完成,核心链路无阻断 |
|
||||
| **M3: 功能补全** | 第 14 周末 | P2 主要项完成,推送/插件市场/跨设备进度同步可用 |
|
||||
| **M4: 体验升级** | 第 20 周末 | P3 项完成,深色模式/国际化/自动更新上线 |
|
||||
|
||||
---
|
||||
|
||||
## 附录:已完成功能清单(便于对照)
|
||||
|
||||
| 模块 | 已具备能力 |
|
||||
|------|-----------|
|
||||
| **桌面端** | 多平台抓取、Cookie/密码登录、插件系统、本地图库 SQLite、缩略图墙 WebUI、云同步、代理池、任务队列 + 增量同步、OCR |
|
||||
| **云端** | FastAPI REST API、JWT 认证、图库/小说/文件 CRUD、关注管理、每日提醒摘要、分片上传、健康检查 |
|
||||
| **手机端** | 登录注册、发现/图片/漫画/书架/我的 五个标签页、小说阅读器(分页/收藏/翻译/进度)、漫画分组阅读、书源管理+Legado 引擎、在线搜索、FCM(部分) |
|
||||
| **插件** | Pixiv、X/Twitter、小红书、笔趣阁、长佩文学、AO3 |
|
||||
| **DevOps** | 一键构建 EXE/APK、Docker Compose 部署、Alembic 迁移 |
|
||||
|
||||
---
|
||||
|
||||
*本文档由代码审计报告、产品文档、CHANGELOG、SAKURA_CLOUD_V1_STATUS 及全仓库源码分析综合生成,供开发排期与需求评审参考。*
|
||||
@@ -0,0 +1,745 @@
|
||||
# 四大问题完整修复方案(终版)
|
||||
|
||||
> 更新日期:2026-06-16
|
||||
> 范围:Sakura Mobile (Flutter) + Sakura Cloud (FastAPI) + Sakura Admin Web (Vue.js) + Desktop GUI (Python)
|
||||
> 版本:v2.0 — 调整优先级为 P0/P1 两级、修正问题二的风险边界和工期、问题一拆为两层修复、问题四拆为开发/生产两套方案、问题三补搜索防抖
|
||||
|
||||
---
|
||||
|
||||
## 目录
|
||||
|
||||
1. [优先级总览](#零优先级总览)
|
||||
2. [P0-1:无法搜索本地不存在的图库(源不可用)](#一p0-1无法搜索本地不存在的图库源不可用)
|
||||
3. [P0-2:网页端显示 JSON 数据而非图片/小说/漫画](#二p0-2网页端显示-json-数据而非图片小说漫画)
|
||||
4. [P1-1:APP 发现页向上滚动卡在底部](#三p1-1app-发现页向上滚动卡在底部)
|
||||
5. [P1-2:导入源的位置排版有问题](#四p1-2导入源的位置排版有问题)
|
||||
6. [实施计划](#五实施计划)
|
||||
|
||||
---
|
||||
|
||||
## 零、优先级总览
|
||||
|
||||
| 等级 | 问题 | 严重程度 | 工期 | 风险 |
|
||||
|:----:|------|:--------:|:----:|:----:|
|
||||
| **P0-1** | 书源搜索不可用 | 核心功能阻断 | 3–5 天 | **高** |
|
||||
| **P0-2** | Web 端显示 JSON / 无法展示内容 | 入口不可用 | 1–2 天 | 低 |
|
||||
| **P1-1** | 发现页 Feed 滚动卡死 | 高频场景阻断 | 1–2 天 | 低 |
|
||||
| **P1-2** | 书源管理页排版和性能 | 管理体验差 | 0.5–1 天 | 低 |
|
||||
|
||||
**优先级判断依据:**
|
||||
- 问题二是"在线探索"的整个功能链路不通,影响范围最大。
|
||||
- 问题四是 Web 端入口整体不可用,新用户访问会直接流失。
|
||||
- 问题一如果高频复现(每次打开发现页都卡),也接近阻断级,但只影响单页面。
|
||||
- 问题三是管理后台的体验优化,不影响核心浏览链路。
|
||||
|
||||
---
|
||||
|
||||
## 一、P0-1:无法搜索本地不存在的图库(源不可用)
|
||||
|
||||
### 1.1 问题现象
|
||||
|
||||
"在线探索"(ExplorePage)搜索漫画/小说时,所有已启用的书源永远不返回结果。无论是否从仓库导入源,`legado_engine.search()` 都静默返回 `[]`,UI 显示"暂无搜索结果"。
|
||||
|
||||
### 1.2 根因分析
|
||||
|
||||
**审查文件(6 个):** `source_manager.dart`、`legado_engine.dart`、`legado_models.dart`、`legado_rule_parser.dart`、`unified_search.dart`、`explore_page.dart`
|
||||
|
||||
#### 根因 1(阻断级):默认种子源 URL = `example.invalid`,且 searchUrl/searchListRule 全空
|
||||
|
||||
**文件:** `source_manager.dart` 第 274–317 行
|
||||
|
||||
```dart
|
||||
LegadoSource(
|
||||
name: '笔趣阁',
|
||||
url: 'https://example.invalid/novel/biquge', // IANA 保留测试域名
|
||||
type: 0, enabled: true,
|
||||
// searchUrl、searchListRule —— 全部 null
|
||||
),
|
||||
```
|
||||
|
||||
`LegadoEngine.search()` 入口(第 36–39 行):
|
||||
|
||||
```dart
|
||||
if (searchUrl == null || listRule == null) {
|
||||
return const <UnifiedSearchResult>[]; // 静默短路
|
||||
}
|
||||
```
|
||||
|
||||
六个默认种子源全都没有搜索规则。不论用户是否"启用",搜索入口直接返回空。这些种子源是从设计之初就存在的占位符,但从未被替换为可用源。
|
||||
|
||||
#### 根因 2(阻断级):CSS 选择器仅支持 `<tag>...</tag>` 简单正则,不支持 class/ID/属性/后代
|
||||
|
||||
**文件:** `legado_rule_parser.dart` 第 86–96 行
|
||||
|
||||
```dart
|
||||
List<String> _cssFragments(String selector) {
|
||||
final tag = _selectorTag(selector); // 只取开头 a-zA-Z 部分作 tag
|
||||
if (tag == null) return const <String>[];
|
||||
final pattern = RegExp('<$tag\\b[^>]*>.*?</$tag>', ...);
|
||||
return pattern.allMatches(body)...;
|
||||
}
|
||||
```
|
||||
|
||||
Legado 源仓库中 95%+ 的规则使用 class 选择器(`.book-list > .item`),当前引擎完全无法解析。
|
||||
|
||||
#### 根因 3(阻断级):GBK 编码 HTML 抛 `FormatException`
|
||||
|
||||
**文件:** `legado_engine.dart` 第 216–220 行
|
||||
|
||||
```dart
|
||||
if (charset == 'gbk' || charset == 'gb2312' || ...) {
|
||||
return gbk.decode(bytes); // ← 无容错参数,遇非 GBK 字节直接抛异常
|
||||
}
|
||||
```
|
||||
|
||||
`utf8.decode` 使用了 `allowMalformed: true`,但 `gbk.decode` **没有容错**。大量声称 charset=gbk 的页面实际混入了非 GBK 字节(emoji、广告注入的特殊符号等),导致整个 `_fetchBody` 失败。
|
||||
|
||||
#### 根因 4:`_ruleType` 忽略传入的 `kind` 参数
|
||||
|
||||
**文件:** `legado_rule_parser.dart` 第 130–134 行
|
||||
|
||||
`selectList` 和 `select` 虽然接受 `kind` 参数,但传给 `_ruleType` 的逻辑有断裂——`_ruleType` 方法签名不接受 `kind`,只根据规则文本前缀判断类型。如果 `source.ruleKind == 'xpath'` 但规则文本不以 `xpath:` 开头,会错误地按 CSS 解析。
|
||||
|
||||
#### 根因 5:仓库 URL 在国内网络不可达 + 首次启动静默失败
|
||||
|
||||
`legado.aoaostar.com`、`gcore.jsdelivr.net`、`git.nyne.dev` 均可能被 DNS 污染或不可达。`initializeIfFirstRun()` 首次启动自动拉仓库,12 秒超时后静默失败,用户无法感知问题所在。
|
||||
|
||||
#### 根因 6:搜索结果 DOM 解析在 UI 线程
|
||||
|
||||
每条结果调一次 `html_parser.parse(fragment)`,搜索结果 20 条 × 6 源 = 120 次 DOM 解析。低端机上会造成 frame drop。
|
||||
|
||||
### 1.3 解决方案
|
||||
|
||||
**⚠️ 法律与风险声明:** 问题二的本质是半个解析引擎重构 + 第三方内容源集成。涉及版权法律风险、源站反爬策略、编码兼容性和解析性能。以下方案在技术层面给出修复路径,**实际实施前需要产品侧确认法律边界和使用条款**。
|
||||
|
||||
#### 方案 A:删除占位种子源,首次启动不做任何自动导入
|
||||
|
||||
**修改文件:** `source_manager.dart`
|
||||
|
||||
**策略——客户端不内置任何第三方书源仓库、书源 URL 或规则:**
|
||||
|
||||
```dart
|
||||
Future<void> initializeIfFirstRun() async {
|
||||
await load();
|
||||
final prefs = await SharedPreferences.getInstance();
|
||||
if (prefs.getBool(_initializedKey) == true) return;
|
||||
|
||||
// 不做任何自动导入。首次启动源列表为空。
|
||||
await prefs.setBool(_initializedKey, true);
|
||||
notifyListeners();
|
||||
}
|
||||
```
|
||||
|
||||
删除 `_seedDefaultSources()` 中的 6 个 `example.invalid` 占位源。首次启动后源列表为空,UI 显示引导入口:
|
||||
|
||||
> **当前没有可用书源。**
|
||||
> 书源规则是第三方网站的内容解析配置,请从你信任的仓库导入,或手动添加你有使用权限的书源。
|
||||
> [从内置仓库导入] [从 URL 导入]
|
||||
|
||||
**不内置任何仓库索引 JSON。** 仓库列表仅作为 UI 入口的快捷链接提供,不硬编码到客户端。
|
||||
|
||||
#### 方案 B:重写 CSS 选择器——引入 `html` 包 + Isolate 解析
|
||||
|
||||
**修改文件:** `legado_rule_parser.dart`、`pubspec.yaml`
|
||||
|
||||
```yaml
|
||||
dependencies:
|
||||
html: ^0.15.5
|
||||
```
|
||||
|
||||
```dart
|
||||
import 'package:html/parser.dart' as html_parser;
|
||||
|
||||
List<String> _cssFragments(String selector) {
|
||||
try {
|
||||
final doc = html_parser.parse(body);
|
||||
final elements = doc.querySelectorAll(selector);
|
||||
return elements.map((el) => el.outerHtml).where(_notEmpty).toList();
|
||||
} on Object {
|
||||
return _legacyCssFragments(selector); // 正则降级
|
||||
}
|
||||
}
|
||||
|
||||
String _cssText(String fragment, String selector) {
|
||||
final attrMatch = RegExp(r'@([a-zA-Z0-9_-]+)$').firstMatch(selector);
|
||||
final clean = selector.replaceFirst(RegExp(r'@([a-zA-Z0-9_-]+)$'), '').trim();
|
||||
try {
|
||||
final el = html_parser.parse(fragment).querySelector(clean);
|
||||
if (el == null) return '';
|
||||
if (attrMatch != null) return el.attributes[attrMatch.group(1)] ?? '';
|
||||
return el.text.trim();
|
||||
} on Object {
|
||||
return _legacyCssText(fragment, selector);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
对超过 20 条的搜索结果,在 `Isolate` 中批量解析:
|
||||
|
||||
```dart
|
||||
import 'dart:isolate';
|
||||
|
||||
Future<List<UnifiedSearchResult>> _parseBatch(
|
||||
List<_SearchRawResult> raw,
|
||||
) async {
|
||||
if (raw.length < 20) return _parseInline(raw);
|
||||
return Isolate.run(() => _parseInIsolate(raw));
|
||||
}
|
||||
```
|
||||
|
||||
#### 方案 C:修复 GBK/GB18030 解码容错
|
||||
|
||||
**修改文件:** `legado_engine.dart` 第 202–223 行
|
||||
|
||||
```dart
|
||||
String _decodeBody(List<int> bytes, {String? declaredCharset, ...}) {
|
||||
if (bytes.isEmpty) return '';
|
||||
final charset = (declaredCharset ?? _charsetFromContentType(contentType))
|
||||
?.trim().toLowerCase();
|
||||
|
||||
if (charset == 'gbk' || charset == 'gb2312' || charset == 'gb18030' || charset == 'gbk-0') {
|
||||
try {
|
||||
return gbk.decode(bytes);
|
||||
} on FormatException {
|
||||
return latin1.decode(bytes, allowInvalid: true); // 降级,保留可读部分
|
||||
}
|
||||
}
|
||||
|
||||
if (charset == 'latin1' || charset == 'iso-8859-1' || charset == 'windows-1252') {
|
||||
return latin1.decode(bytes, allowInvalid: true);
|
||||
}
|
||||
|
||||
try {
|
||||
return utf8.decode(bytes, allowMalformed: true);
|
||||
} on FormatException {
|
||||
return latin1.decode(bytes, allowInvalid: true);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### 方案 D:修复 `_ruleType`——优先使用传入的 `kind`
|
||||
|
||||
```dart
|
||||
String _resolveKind(String rule, String? explicitKind) {
|
||||
if (explicitKind != null && explicitKind.isNotEmpty) return explicitKind;
|
||||
final lower = rule.toLowerCase();
|
||||
if (lower.startsWith('json:') || lower.startsWith(r'$.')) return 'json';
|
||||
if (lower.startsWith('regex:')) return 'regex';
|
||||
return 'css';
|
||||
}
|
||||
```
|
||||
|
||||
调用方改为 `_resolveKind(normalized, kind)` 替代原来的 `kind ?? _ruleType(normalized)`。
|
||||
|
||||
#### 方案 E:多源异常隔离——单源失败不拖垮全局
|
||||
|
||||
**修改文件:** `legado_engine.dart` `searchAll()` 方法
|
||||
|
||||
当前实现已经是 `.catchError` 静默跳过,但错误信息被丢弃。改为:
|
||||
|
||||
```dart
|
||||
Future.wait(batch.map((source) =>
|
||||
search(source, keyword, page: page)
|
||||
.timeout(perSourceTimeout)
|
||||
.then((items) => items)
|
||||
.catchError((error) => <UnifiedSearchResult>[
|
||||
UnifiedSearchResult.error(
|
||||
sourceId: source.id,
|
||||
sourceName: source.name,
|
||||
errorMessage: _shortError(error),
|
||||
),
|
||||
]),
|
||||
));
|
||||
```
|
||||
|
||||
错误结果显示在 UI 中(如 `"笔趣阁 - 请求超时"`),用户可以看到哪些源失败了。
|
||||
|
||||
#### 方案 F:仓库镜像轮询 + 网络探测
|
||||
|
||||
为 5 个仓库 URL 各提供 2–3 个镜像域名。`importFromRepo()` 依次尝试,全部失败则提示用户手动输入 URL。
|
||||
|
||||
---
|
||||
|
||||
## 二、P0-2:网页端显示 JSON 数据而非图片/小说/漫画
|
||||
|
||||
### 2.1 问题现象
|
||||
|
||||
浏览器访问 Sakura Cloud 后端或 Admin Web 时,页面显示原始 JSON(`{"items": [...]}`),而非渲染好的图片网格、小说内容、漫画翻页器。
|
||||
|
||||
### 2.2 根因分析
|
||||
|
||||
**审查文件:** `sakura_cloud/main.py`、`sakura_cloud/api/middleware.py`、`sakura_admin_web/`、`pixiv_app/services/gallery_api.py`
|
||||
|
||||
#### 根因 1(阻断级):FastAPI 未注册 CORS 中间件
|
||||
|
||||
**文件:** `sakura_cloud/main.py`
|
||||
|
||||
```python
|
||||
app = FastAPI(...)
|
||||
# ← 缺少 app.add_middleware(CORSMiddleware, ...)
|
||||
```
|
||||
|
||||
Admin Web 开发时运行在 `localhost:5174`,后端在 `localhost:8000`。浏览器跨域请求被拦截,前端连 API 都调不通。CSP 中间件(`middleware.py`)处理的是内容安全策略,**不替代 CORS**。
|
||||
|
||||
#### 根因 2(阻断级):Cloud 后端未挂载 SPA 静态文件
|
||||
|
||||
**文件:** `sakura_cloud/main.py` 第 61–78 行
|
||||
|
||||
`/admin` 路由返回的是硬编码占位 HTML,不是 `sakura_admin_web/dist/index.html` 构建产物。任何 `/app`、`/login` 路径返回 404 或触发认证检查。
|
||||
|
||||
#### 根因 3:Admin Web 缺少内容浏览 Vue 组件
|
||||
|
||||
- `UserDashboard.vue`:仅导航卡片,无实际内容页
|
||||
- `UserDetailPage.vue`:占位组件——"详情数据由现有 API 按需加载"
|
||||
- 路由没有注册 `/gallery`、`/novels` 等内容浏览路由
|
||||
|
||||
#### 根因 4:CSP `img-src 'self'` 阻止外部图片
|
||||
|
||||
API 返回的 `thumbnail_url` 指向外部域名时,浏览器 CSP 引擎直接拦截。
|
||||
|
||||
### 2.3 解决方案
|
||||
|
||||
#### ⸻ 开发环境 ⸻
|
||||
|
||||
#### 方案 A:注册 CORS 中间件
|
||||
|
||||
**修改文件:** `sakura_cloud/main.py`
|
||||
|
||||
```python
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[
|
||||
"http://localhost:5174",
|
||||
"http://127.0.0.1:5174",
|
||||
"app://.",
|
||||
],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
```
|
||||
|
||||
#### 方案 B:修复 CSP 中间件
|
||||
|
||||
**修改文件:** `sakura_cloud/api/middleware.py`
|
||||
|
||||
```python
|
||||
"img-src 'self' data: blob: https: http:;"
|
||||
"connect-src 'self' http://localhost:* ws://localhost:*;"
|
||||
```
|
||||
|
||||
#### 方案 C:开发环境可选挂载 Vue dist
|
||||
|
||||
```python
|
||||
# sakura_cloud/main.py — 开发环境可选
|
||||
import os
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
frontend_dist = os.path.join(os.path.dirname(__file__), "..", "sakura_admin_web", "dist")
|
||||
if os.path.isdir(frontend_dist) and settings.debug:
|
||||
assets = os.path.join(frontend_dist, "assets")
|
||||
if os.path.isdir(assets):
|
||||
app.mount("/assets", StaticFiles(directory=assets), name="assets")
|
||||
|
||||
@app.get("/app/{full_path:path}")
|
||||
async def spa_fallback(full_path: str = ""):
|
||||
index = os.path.join(frontend_dist, "index.html")
|
||||
return FileResponse(index) if os.path.isfile(index) else HTMLResponse("Not built")
|
||||
```
|
||||
|
||||
**注意:不添加 API 路径自动重定向的中间件。** 这会导致 APP 的 Dio/浏览器 Extension 发起的 API 请求被误重定向。API 路径就让它返回 JSON——它是 REST API 的正确行为。前端 SPA 从 `/app` 入口加载后用 JS fetch 数据。
|
||||
|
||||
#### ⸻ 生产环境 ⸻
|
||||
|
||||
#### 方案 D:Nginx 托管 SPA + 反向代理 API
|
||||
|
||||
**推荐的生产结构:**
|
||||
|
||||
```
|
||||
Nginx
|
||||
├── / → /var/www/sakura_admin_web/dist/index.html
|
||||
├── /app → SPA fallback (try_files $uri /index.html)
|
||||
├── /login → SPA fallback
|
||||
├── /assets/ → 静态资源,expires 1y(Vite 产物带 hash)
|
||||
└── /api/ → proxy_pass http://127.0.0.1:8000
|
||||
```
|
||||
|
||||
**Nginx 配置示例:**
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 80;
|
||||
server_name admin.your-domain.com;
|
||||
root /var/www/sakura_admin_web/dist;
|
||||
index index.html;
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://127.0.0.1:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
location /assets/ {
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
FastAPI **不长期托管前端静态资源**,只负责 API。
|
||||
|
||||
#### 方案 E:新增内容浏览 Vue 组件
|
||||
|
||||
**新增文件:**
|
||||
|
||||
```
|
||||
sakura_admin_web/src/views/app/
|
||||
├── GalleryBrowsePage.vue # 画廊浏览:grid + 分页 + 搜索筛选
|
||||
├── NovelReadPage.vue # 小说阅读:章节列表 + 正文渲染
|
||||
└── MangaViewerPage.vue # 漫画翻页:PageView 横向翻页
|
||||
```
|
||||
|
||||
路由注册:
|
||||
|
||||
```typescript
|
||||
// router/index.ts
|
||||
{ path: '/app/gallery', component: () => import('@/views/app/GalleryBrowsePage.vue') },
|
||||
{ path: '/app/novels', component: () => import('@/views/app/NovelReadPage.vue') },
|
||||
{ path: '/app/manga/:id', component: () => import('@/views/app/MangaViewerPage.vue') },
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 三、P1-1:APP 发现页向上滚动卡在底部
|
||||
|
||||
### 3.1 问题现象
|
||||
|
||||
发现页从底部向上回滚时卡住,须等待数据加载完或强制下拉刷新才能恢复。
|
||||
|
||||
### 3.2 根因分析
|
||||
|
||||
**审查文件:** `feed_tab.dart`(829 行)
|
||||
|
||||
| 根因 | 位置 | 严重程度 |
|
||||
|------|------|:--------:|
|
||||
| `_ensureFutures()` 在 `build()` 中反复调用 | 第 64–66 行 | 高 |
|
||||
| `_refresh()` 用 `setState` 归零所有 future → 布局高度剧烈跳动 | 第 53–62 行 | **阻断** |
|
||||
| `_StatCard` 在列表末尾加载完成后高度变化 | 第 182–186 行 | 中 |
|
||||
| 内嵌横向 `ListView` 与纵向 `ListView` 手势竞争 | 第 443–507 行 | 中(部分设备) |
|
||||
| `ScrollController` 未显式管理生命周期 | 缺 | 低 |
|
||||
|
||||
### 3.3 解决方案——拆为两层
|
||||
|
||||
#### 第一层:必须做(P1 范围)
|
||||
|
||||
**1. `AsyncNotifier` 替代 `FutureBuilder` + 状态统一管理**
|
||||
|
||||
当前 `_FeedTabState` 用 5 个散落的 `Future` + `FutureBuilder` 管理异步状态,刷新时 `setState(null)` 归零所有 future 再在下一帧重建——这个模式是滚动问题的根源。
|
||||
|
||||
改为用一个统一的 `FeedState` 对象:
|
||||
|
||||
```dart
|
||||
class FeedState {
|
||||
final bool loading;
|
||||
final List<Artwork> todayArtworks;
|
||||
final List<Artwork> recentArtworks;
|
||||
final List<Novel> recentNovels;
|
||||
final List<Map<String, dynamic>> alerts;
|
||||
final Map<String, dynamic> stat;
|
||||
final String? error;
|
||||
}
|
||||
```
|
||||
|
||||
用 Riverpod `AsyncNotifier` 管理整个页面的数据加载。`build()` 只读取 `ref.watch(feedProvider)`,不调用 `_ensureFutures()`,不在 build 中创建 future。
|
||||
|
||||
**2. 显式 `ScrollController`**
|
||||
|
||||
```dart
|
||||
class _FeedTabState extends ConsumerState<FeedTab> {
|
||||
final ScrollController _scrollController = ScrollController();
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
_scrollController.dispose();
|
||||
super.dispose();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
外层 `ListView` 绑定此 controller。切换 tab 时通过 `_scrollController.hasClients` 检查位置有效性。
|
||||
|
||||
**3. `_StatCard` 固定高度**
|
||||
|
||||
```dart
|
||||
SizedBox(
|
||||
height: 106,
|
||||
child: _StatCard(stat: stat),
|
||||
),
|
||||
```
|
||||
|
||||
通过 `AnimatedSwitcher` 平滑过渡内容而非跳变布局。
|
||||
|
||||
**4. Refresh 状态统一管理**
|
||||
|
||||
`RefreshIndicator.onRefresh` 调用 provider 的 `ref.invalidate(feedProvider)`,由一个入口触发重载,不再散射到 5 个独立的 `setState(null)` 调用。
|
||||
|
||||
#### 第二层:如果仍有横向列表抢手势(专项修复)
|
||||
|
||||
仅当第一层修复后,在部分设备上仍出现"在对角线滑动时外层垂直滚动丢失"的问题时,才实施此层:
|
||||
|
||||
对横向 `_ArtworkStrip` 的 `ListView` 外套方向判定:
|
||||
|
||||
```dart
|
||||
SizedBox(
|
||||
height: 210,
|
||||
child: RawGestureDetector(
|
||||
gestures: {
|
||||
HorizontalDragGestureRecognizer:
|
||||
GestureRecognizerFactoryWithHandlers<HorizontalDragGestureRecognizer>(
|
||||
() => HorizontalDragGestureRecognizer(),
|
||||
(instance) {},
|
||||
),
|
||||
},
|
||||
child: ListView.separated(
|
||||
scrollDirection: Axis.horizontal,
|
||||
physics: const ClampingScrollPhysics(),
|
||||
...
|
||||
),
|
||||
),
|
||||
),
|
||||
```
|
||||
|
||||
`HorizontalDragGestureRecognizer` 在竞技场中只响应水平方向的拖拽,对角滑动时垂直分量不会被它吃掉,外层垂直 ListView 正常响应。
|
||||
|
||||
**不推荐 `PageStorageKey` 解决手势竞争。** `PageStorageKey` 的职责是页面切换后恢复滚动位置(`PageStorage` 机制),它不参与手势竞技场裁决。两项是不同层面的问题,不应混用。
|
||||
|
||||
---
|
||||
|
||||
## 四、P1-2:导入源的位置排版有问题
|
||||
|
||||
### 4.1 问题现象
|
||||
|
||||
书源管理页布局混乱:搜索栏在导入区底部(须滑过所有卡片才能搜索)、已导入 2200+ 条后 `for` 循环展开所有 Card 导致卡顿、5 个仓库全部展开占用过多空间。
|
||||
|
||||
### 4.2 根因分析
|
||||
|
||||
**审查文件:** `source_manager_page.dart`(465 行)
|
||||
|
||||
| 根因 | 位置 |
|
||||
|------|------|
|
||||
| 搜索栏放在仓库 + 导入卡片**之后** | 第 81–85 行 |
|
||||
| `for` 循环展开全部源,无懒加载 | 第 102–129 行 |
|
||||
| 5 个仓库全部展开,每个 ~120px | 第 376–413 行 |
|
||||
| 缺少分段标题和视觉层次 | 全页 |
|
||||
|
||||
### 4.3 解决方案
|
||||
|
||||
#### 方案 A:CustomScrollView + SliverAppBar 吸顶搜索栏
|
||||
|
||||
```dart
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final manager = ref.watch(sourceManagerProvider);
|
||||
final filteredSources = _filterSources(manager);
|
||||
|
||||
return Scaffold(
|
||||
body: CustomScrollView(
|
||||
slivers: [
|
||||
// 搜索栏吸顶
|
||||
SliverAppBar(
|
||||
pinned: true,
|
||||
title: const Text('书源管理'),
|
||||
bottom: PreferredSize(
|
||||
preferredSize: const Size.fromHeight(56),
|
||||
child: Padding(
|
||||
padding: const EdgeInsets.fromLTRB(16, 0, 16, 8),
|
||||
child: SearchBar(
|
||||
hintText: '搜索已导入的书源',
|
||||
leading: const Icon(Icons.search),
|
||||
onChanged: onSearchChanged, // 带防抖
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
|
||||
// 统计 + 维护 + 仓库 + 导入 —— 各 SliverToBoxAdapter
|
||||
SliverToBoxAdapter(child: _StatsCard(manager: manager)),
|
||||
SliverToBoxAdapter(child: _MaintenanceCard(...)),
|
||||
SliverToBoxAdapter(child: _CollapsibleRepoSection(...)),
|
||||
SliverToBoxAdapter(child: _CustomImportCard(...)),
|
||||
|
||||
// 分段标题
|
||||
SliverToBoxAdapter(
|
||||
child: Padding(
|
||||
padding: const EdgeInsets.fromLTRB(16, 8, 16, 4),
|
||||
child: Text('已导入 (${filteredSources.length})', ...),
|
||||
),
|
||||
),
|
||||
|
||||
// 源列表 —— 懒加载
|
||||
SliverList.builder(
|
||||
itemCount: filteredSources.length,
|
||||
itemBuilder: (context, index) {
|
||||
final source = filteredSources[index];
|
||||
return Card(
|
||||
child: SwitchListTile(
|
||||
value: source.enabled,
|
||||
onChanged: (value) =>
|
||||
ref.read(sourceManagerProvider).setEnabled(source.id, value),
|
||||
title: Text(source.name),
|
||||
subtitle: Text('${source.isManga ? '漫画' : '小说'} - ${source.url}'),
|
||||
secondary: Icon(source.isManga
|
||||
? Icons.menu_book_rounded
|
||||
: Icons.auto_stories_rounded),
|
||||
),
|
||||
);
|
||||
},
|
||||
),
|
||||
|
||||
if (filteredSources.isEmpty)
|
||||
SliverToBoxAdapter(
|
||||
child: const Padding(
|
||||
padding: EdgeInsets.all(28),
|
||||
child: Center(child: Text('还没有导入书源,请先导入一个书源仓库。')),
|
||||
),
|
||||
),
|
||||
|
||||
const SliverToBoxAdapter(child: SizedBox(height: 28)),
|
||||
],
|
||||
),
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
**不使用 `shrinkWrap: true` ——** `SliverList.builder` 本身就是懒加载,不需要这个参数。
|
||||
|
||||
#### 方案 B:搜索防抖
|
||||
|
||||
```dart
|
||||
Timer? _debounce;
|
||||
|
||||
void onSearchChanged(String value) {
|
||||
_debounce?.cancel();
|
||||
_debounce = Timer(const Duration(milliseconds: 180), () {
|
||||
setState(() {
|
||||
filter = value;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
_debounce?.cancel();
|
||||
customUrl.dispose();
|
||||
super.dispose();
|
||||
}
|
||||
```
|
||||
|
||||
180ms 防抖确保用户快速输入时不会每按一个键就触发一次全文过滤。
|
||||
|
||||
#### 方案 C:仓库列表改用 ExpansionTile
|
||||
|
||||
```dart
|
||||
class _CollapsibleRepoSection extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Card(
|
||||
child: ExpansionTile(
|
||||
title: const Text('内置书源仓库'),
|
||||
subtitle: Text('${SourceManager.builtinRepos.length} 个仓库可用'),
|
||||
initiallyExpanded: false,
|
||||
children: [
|
||||
for (final repo in SourceManager.builtinRepos)
|
||||
ListTile(
|
||||
leading: const Icon(Icons.cloud_download_outlined),
|
||||
title: Text(repo.name),
|
||||
subtitle: Text(repo.description),
|
||||
trailing: FilledButton.tonal(
|
||||
onPressed: ...,
|
||||
child: Text(repo.isSubscription ? '订阅' : '导入'),
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 五、实施计划
|
||||
|
||||
### 阶段 1:P0 可用性修复(3–6 天)
|
||||
|
||||
| 天数 | 任务 | 涉及文件 |
|
||||
|:----:|------|----------|
|
||||
| 1–3 | **问题二:书源搜索链路修复** | |
|
||||
| | └ 方案 A:删除占位种子源,空列表首次启动 | `source_manager.dart` |
|
||||
| | └ 方案 B:`html` 包重写 CSS 选择器 + Isolate | `legado_rule_parser.dart`、`pubspec.yaml` |
|
||||
| | └ 方案 C:GBK/GB18030 编码异常处理 | `legado_engine.dart` |
|
||||
| | └ 方案 D:`_ruleType` 优先使用 `kind` 参数 | `legado_rule_parser.dart` |
|
||||
| | └ 方案 E:多源异常隔离(错误透传 UI) | `legado_engine.dart` |
|
||||
| | └ 方案 F:仓库镜像轮询 + 网络探测 | `source_manager.dart` |
|
||||
| | └ **测试:** 真实书源搜索回归(≥10 个不同源站点) | |
|
||||
| 4–6 | **问题四:Web 端 JSON → 可视化内容** | |
|
||||
| | └ 方案 A:CORS 中间件(10min) | `sakura_cloud/main.py` |
|
||||
| | └ 方案 B:CSP 修复(10min) | `sakura_cloud/api/middleware.py` |
|
||||
| | └ 方案 C:开发环境可选挂载 Vue dist | `sakura_cloud/main.py` |
|
||||
| | └ 方案 D:Nginx 生产部署配置 | 新建 `sakura_admin_web/nginx.conf` |
|
||||
| | └ 方案 E:GalleryBrowsePage + NovelReadPage + MangaViewerPage | `sakura_admin_web/src/views/app/` |
|
||||
| | └ **测试:** 浏览器 `/app` → 图片网格 → 点击作品 → 大图 → 分页 |
|
||||
|
||||
### 阶段 2:P1 体验稳定性修复(2–4 天)
|
||||
|
||||
| 天数 | 任务 | 涉及文件 |
|
||||
|:----:|------|----------|
|
||||
| 1–2 | **问题一:Feed 滚动稳定性(第一层必做)** | |
|
||||
| | └ AsyncNotifier 替代 FutureBuilder + 状态统一管理 | `feed_tab.dart` |
|
||||
| | └ 显式 ScrollController + dispose | `feed_tab.dart` |
|
||||
| | └ StatCard 固定高度 106px | `feed_tab.dart` |
|
||||
| | └ Refresh 状态统一入口(Riverpod provider) | `feed_tab.dart` + `providers.dart` |
|
||||
| | └ 第二层(按需):RawGestureDetector 手势方向判定 | `feed_tab.dart` |
|
||||
| | └ **测试:** Android 真机快速上下滚动 × 50 次 | |
|
||||
| 3–4 | **问题三:书源管理页 Sliver 重构** | |
|
||||
| | └ CustomScrollView + SliverAppBar 吸顶搜索栏 | `source_manager_page.dart` |
|
||||
| | └ SliverList.builder 懒加载 | `source_manager_page.dart` |
|
||||
| | └ ExpansionTile 折叠仓库列表 | `source_manager_page.dart` |
|
||||
| | └ 搜索防抖 180ms + dispose 清理 | `source_manager_page.dart` |
|
||||
| | └ **测试:** 导入 2200+ 源后滚动 + 搜索 | |
|
||||
|
||||
### 阶段 3:回归测试(1–2 天)
|
||||
|
||||
| 测试项 | 方法 |
|
||||
|--------|------|
|
||||
| Android 真机滚动 | feed_tab 快速上下滑 × 50 次,观察有无卡顿 |
|
||||
| 2200+ 书源压力测试 | 从"精选书源仓库"导入全量 → 搜索 → 启用/禁用单个源 → 滚动 |
|
||||
| Web 端 `/app` | Chrome + Firefox 访问 → 图片网格加载 → 分页翻页 |
|
||||
| Web 端 `/api` | 直接访问 API 路径 → 确认返回 JSON(正常行为) |
|
||||
| Web 端图片加载 | 检查 CSP 日志 → 确认无 `img-src` 拦截 |
|
||||
| 无书源首次启动 | 清除 SharedPreferences → 冷启动 APP → 确认显示引导 UI 而非假源 |
|
||||
|
||||
### 风险矩阵
|
||||
|
||||
| 风险 | 等级 | 缓解措施 |
|
||||
|------|:----:|----------|
|
||||
| 问题二中 CSS 选择器 `html` 包性能不足 | 中 | 保留正则降级路径,低端机自动切降级 |
|
||||
| 问题二中真实书源测试覆盖不足 | 高 | 选择 ≥10 个已知稳定的 Legado 源作为回归集 |
|
||||
| 问题一中 AsyncNotifier 迁移引入新问题 | 低 | 先写测试用例覆盖原有 5 种数据加载路径 |
|
||||
| 问题四中 Nginx 配置与现有部署冲突 | 中 | 提供独立配置文件和部署检查清单 |
|
||||
| 书源法律边界 | 高 | 客户端不内置任何仓库 URL 或规则,由用户自行导入 |
|
||||
|
||||
---
|
||||
|
||||
> **版本记录:**
|
||||
> - v1.0 (2026-06-16):初始版本
|
||||
> - v1.1 (2026-06-16):技术评审修订——修正延迟数字、shrinkWrap 陷阱、重定向误伤、补充 CORS/ScrollController/GBK 异常/Isolate
|
||||
> - v2.0 (2026-06-16):终版——优先级改为 P0/P1、问题二风险升级为 3–5 天且不内置任何仓库、问题一拆为两层修复(第一层必做 + 第二层按需)、问题四拆为开发/生产两套方案、问题三补搜索防抖、三阶段实施计划
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
+778
@@ -0,0 +1,778 @@
|
||||
下面是整合后的**最终完整方案**,包括:漫画解析、章节完整性、图片加载、调试能力、搜索性能,以及你刚刚提到的顶部搜索框 UI 问题。
|
||||
Sakura 漫画模块完整改进方案
|
||||
=================
|
||||
|
||||
一、改进目标
|
||||
------
|
||||
|
||||
本轮改造解决 5 类问题:
|
||||
1. 漫画章节目录不全
|
||||
2. 漫画图片解析失败
|
||||
3. 图片 URL 有数量但无法显示
|
||||
4. 在线搜索速度慢
|
||||
5. 漫画 / 图片页顶部搜索框布局异常,在线搜索入口位置不合理
|
||||
|
||||
* * *
|
||||
|
||||
二、整体优先级
|
||||
=======
|
||||
|
||||
P0:顶部搜索框 UI 修复与在线搜索入口重构
|
||||
P1:图片 URL 合法性验证
|
||||
P2:PageImage + headers 拆分
|
||||
P3:图片 HTTP 探测与调试能力
|
||||
P4:TOC 分页抓全章节
|
||||
P5:章节排序支持番外
|
||||
P6:Script 图片变量提取
|
||||
P7:漫画调试页
|
||||
P8:搜索健康分与增量搜索
|
||||
|
||||
说明:
|
||||
P0 先做,因为它是明显 UI 问题,改动小、收益快。
|
||||
P1~P7 解决漫画“能不能看”。
|
||||
P8 最后做,解决“搜得慢”。
|
||||
|
||||
* * *
|
||||
|
||||
P0:顶部搜索框 UI 修复与在线搜索入口重构
|
||||
=======================
|
||||
|
||||
问题
|
||||
--
|
||||
|
||||
截图中红框区域出现:
|
||||
顶部搜索栏被状态栏 / 右上角按钮挤压
|
||||
漫画页和图片页都出现同类问题
|
||||
在线搜索入口放在右上角,不符合用户预期
|
||||
目标
|
||||
--
|
||||
|
||||
漫画页 / 图片页顶部布局稳定
|
||||
搜索框不被状态栏遮挡
|
||||
在线搜索入口集成到搜索框流程中
|
||||
不再在右上角放在线搜索按钮
|
||||
|
||||
推荐交互
|
||||
----
|
||||
|
||||
页面顶部只保留一个统一搜索框:
|
||||
[ 搜索图标 ] 搜索标题、作者或标签 [筛选] [刷新]
|
||||
|
||||
点击搜索框进入统一搜索页:
|
||||
SearchPage
|
||||
├─ 搜索输入框
|
||||
├─ 搜索范围切换:本地 / 在线 / 全部
|
||||
└─ 结果分区:
|
||||
├─ 本地漫画
|
||||
├─ 在线漫画源
|
||||
└─ 图片 / 插画
|
||||
代码结构
|
||||
----
|
||||
|
||||
新增统一组件:
|
||||
class SakuraSearchBar extends StatelessWidget {
|
||||
final String hintText;
|
||||
final VoidCallback onTap;
|
||||
final VoidCallback? onFilterTap;
|
||||
final VoidCallback? onRefreshTap;
|
||||
|
||||
const SakuraSearchBar({
|
||||
super.key,
|
||||
required this.hintText,
|
||||
required this.onTap,
|
||||
this.onFilterTap,
|
||||
this.onRefreshTap,
|
||||
});
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Material(
|
||||
elevation: 6,
|
||||
borderRadius: BorderRadius.circular(32),
|
||||
color: Theme.of(context).colorScheme.surface,
|
||||
child: InkWell(
|
||||
borderRadius: BorderRadius.circular(32),
|
||||
onTap: onTap,
|
||||
child: SizedBox(
|
||||
height: 56,
|
||||
child: Row(
|
||||
children: [
|
||||
const SizedBox(width: 18),
|
||||
const Icon(Icons.search),
|
||||
const SizedBox(width: 12),
|
||||
Expanded(
|
||||
child: Text(
|
||||
hintText,
|
||||
maxLines: 1,
|
||||
overflow: TextOverflow.ellipsis,
|
||||
style: Theme.of(context).textTheme.titleMedium,
|
||||
),
|
||||
),
|
||||
if (onFilterTap != null)
|
||||
IconButton(
|
||||
icon: const Icon(Icons.tune),
|
||||
onPressed: onFilterTap,
|
||||
),
|
||||
if (onRefreshTap != null)
|
||||
IconButton(
|
||||
icon: const Icon(Icons.refresh),
|
||||
onPressed: onRefreshTap,
|
||||
),
|
||||
const SizedBox(width: 8),
|
||||
],
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
页面顶部必须用:
|
||||
SafeArea(
|
||||
bottom: false,
|
||||
child: Padding(
|
||||
padding: const EdgeInsets.fromLTRB(16, 8, 16, 8),
|
||||
child: SakuraSearchBar(
|
||||
hintText: '搜索标题、作者或标签',
|
||||
onTap: () {
|
||||
Navigator.push(
|
||||
context,
|
||||
MaterialPageRoute(
|
||||
builder: (_) => UnifiedSearchPage(
|
||||
initialMode: SearchMode.manga,
|
||||
),
|
||||
),
|
||||
);
|
||||
},
|
||||
onFilterTap: _openFilter,
|
||||
onRefreshTap: _refresh,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
禁止再用:
|
||||
Stack + Positioned 顶部硬定位
|
||||
AppBar 右上角塞在线搜索入口
|
||||
状态栏高度手动写死
|
||||
|
||||
* * *
|
||||
|
||||
P1:图片 URL 合法性验证
|
||||
===============
|
||||
|
||||
目标
|
||||
--
|
||||
|
||||
不要相信 `1/17` 这种页数。它可能是:
|
||||
17 个真实图片
|
||||
17 个空字符串
|
||||
17 个 loading 图
|
||||
17 个 placeholder
|
||||
17 个错误 URL
|
||||
|
||||
新增模型:
|
||||
class ImageCandidate {
|
||||
final String raw;
|
||||
final String? url;
|
||||
final bool isValid;
|
||||
final String? reason;
|
||||
|
||||
const ImageCandidate({
|
||||
required this.raw,
|
||||
required this.url,
|
||||
required this.isValid,
|
||||
this.reason,
|
||||
});
|
||||
}
|
||||
|
||||
验证逻辑:
|
||||
ImageCandidate validateImageCandidate(String raw, String baseUrl) {
|
||||
final value = raw.trim();
|
||||
|
||||
if (value.isEmpty) {
|
||||
return ImageCandidate(
|
||||
raw: raw,
|
||||
url: null,
|
||||
isValid: false,
|
||||
reason: 'empty',
|
||||
);
|
||||
}
|
||||
|
||||
if (value.startsWith('data:')) {
|
||||
return ImageCandidate(
|
||||
raw: raw,
|
||||
url: null,
|
||||
isValid: false,
|
||||
reason: 'data-uri',
|
||||
);
|
||||
}
|
||||
|
||||
final lower = value.toLowerCase();
|
||||
if (lower.contains('loading') ||
|
||||
lower.contains('placeholder') ||
|
||||
lower.contains('logo') ||
|
||||
lower.contains('icon')) {
|
||||
return ImageCandidate(
|
||||
raw: raw,
|
||||
url: value,
|
||||
isValid: false,
|
||||
reason: 'placeholder',
|
||||
);
|
||||
}
|
||||
|
||||
final normalized = normalizeImageUrl(value, baseUrl);
|
||||
final uri = Uri.tryParse(normalized);
|
||||
|
||||
if (uri == null || !uri.hasScheme || uri.host.isEmpty) {
|
||||
return ImageCandidate(
|
||||
raw: raw,
|
||||
url: normalized,
|
||||
isValid: false,
|
||||
reason: 'invalid-url',
|
||||
);
|
||||
}
|
||||
|
||||
return ImageCandidate(
|
||||
raw: raw,
|
||||
url: normalized,
|
||||
isValid: true,
|
||||
);
|
||||
}
|
||||
|
||||
URL 归一化:
|
||||
String normalizeImageUrl(String raw, String baseUrl) {
|
||||
var u = raw.trim()
|
||||
.replaceAll(r'\/', '/')
|
||||
.replaceAll(r'\u002F', '/');
|
||||
|
||||
if (u.startsWith('//')) {
|
||||
u = 'https:$u';
|
||||
}
|
||||
|
||||
return Uri.parse(baseUrl).resolve(u).toString();
|
||||
}
|
||||
|
||||
* * *
|
||||
|
||||
P2:PageImage + headers 拆分
|
||||
=========================
|
||||
|
||||
问题
|
||||
--
|
||||
|
||||
Legado 漫画源经常返回:
|
||||
https://xxx.jpg,{"headers":{"Referer":"https://site.com"}}
|
||||
|
||||
如果直接丢给:
|
||||
Image.network(raw)
|
||||
|
||||
图片必炸。
|
||||
新模型
|
||||
---
|
||||
|
||||
class PageImage {
|
||||
final int index;
|
||||
final String raw;
|
||||
final String url;
|
||||
final Map<String, String> headers;
|
||||
|
||||
const PageImage({
|
||||
required this.index,
|
||||
required this.raw,
|
||||
required this.url,
|
||||
required this.headers,
|
||||
});
|
||||
}
|
||||
|
||||
拆分逻辑
|
||||
----
|
||||
|
||||
class ParsedImageSpec {
|
||||
final String url;
|
||||
final Map<String, String> headers;
|
||||
|
||||
ParsedImageSpec({
|
||||
required this.url,
|
||||
required this.headers,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
ParsedImageSpec splitLegadoImageSpec(String raw) {
|
||||
final idx = raw.indexOf(',{');
|
||||
if (idx < 0) {
|
||||
return ParsedImageSpec(url: raw.trim(), headers: {});
|
||||
}
|
||||
|
||||
final url = raw.substring(0, idx).trim();
|
||||
final jsonText = raw.substring(idx + 1).trim();
|
||||
|
||||
final headers = <String, String>{};
|
||||
|
||||
try {
|
||||
final decoded = jsonDecode(jsonText);
|
||||
final h = decoded['headers'];
|
||||
|
||||
if (h is Map) {
|
||||
h.forEach((k, v) {
|
||||
headers[k.toString()] = v.toString();
|
||||
});
|
||||
} else if (h is String) {
|
||||
final inner = jsonDecode(h);
|
||||
if (inner is Map) {
|
||||
inner.forEach((k, v) {
|
||||
headers[k.toString()] = v.toString();
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
return ParsedImageSpec(url: url, headers: headers);
|
||||
}
|
||||
|
||||
生成 PageImage:
|
||||
PageImage buildPageImage({
|
||||
required String raw,
|
||||
required String chapterUrl,
|
||||
required LegadoSource source,
|
||||
required int index,
|
||||
}) {
|
||||
final parsed = splitLegadoImageSpec(raw);
|
||||
|
||||
final headers = <String, String>{
|
||||
...source.headers,
|
||||
'User-Agent': source.userAgent ?? defaultUserAgent,
|
||||
'Referer': chapterUrl,
|
||||
...parsed.headers,
|
||||
};
|
||||
|
||||
return PageImage(
|
||||
index: index,
|
||||
raw: raw,
|
||||
url: normalizeImageUrl(parsed.url, chapterUrl),
|
||||
headers: headers,
|
||||
);
|
||||
}
|
||||
|
||||
阅读器改成:
|
||||
Image.network(
|
||||
page.url,
|
||||
headers: page.headers,
|
||||
fit: BoxFit.contain,
|
||||
errorBuilder: (_, error, stack) {
|
||||
return const Icon(Icons.broken_image);
|
||||
},
|
||||
)
|
||||
|
||||
* * *
|
||||
|
||||
P3:图片 HTTP 探测
|
||||
=============
|
||||
|
||||
目标
|
||||
--
|
||||
|
||||
判断到底是:
|
||||
URL 错
|
||||
headers 错
|
||||
反爬
|
||||
图片损坏
|
||||
Flutter 缓存问题
|
||||
|
||||
新增:
|
||||
class ImageProbeResult {
|
||||
final String url;
|
||||
final int? statusCode;
|
||||
final String? contentType;
|
||||
final int contentLength;
|
||||
final String first16BytesHex;
|
||||
final bool looksLikeImage;
|
||||
final String? error;
|
||||
|
||||
const ImageProbeResult({
|
||||
required this.url,
|
||||
this.statusCode,
|
||||
this.contentType,
|
||||
required this.contentLength,
|
||||
required this.first16BytesHex,
|
||||
required this.looksLikeImage,
|
||||
this.error,
|
||||
});
|
||||
}
|
||||
|
||||
探测:
|
||||
Future<ImageProbeResult> probeImage(Dio dio, PageImage image) async {
|
||||
try {
|
||||
final resp = await dio.get<List<int>>(
|
||||
image.url,
|
||||
options: Options(
|
||||
responseType: ResponseType.bytes,
|
||||
headers: image.headers,
|
||||
validateStatus: (_) => true,
|
||||
),
|
||||
);
|
||||
|
||||
final bytes = resp.data ?? const <int>[];
|
||||
final first16 = bytes
|
||||
.take(16)
|
||||
.map((b) => b.toRadixString(16).padLeft(2, '0'))
|
||||
.join();
|
||||
|
||||
return ImageProbeResult(
|
||||
url: image.url,
|
||||
statusCode: resp.statusCode,
|
||||
contentType: resp.headers.value('content-type'),
|
||||
contentLength: bytes.length,
|
||||
first16BytesHex: first16,
|
||||
looksLikeImage: isImageMagic(bytes),
|
||||
);
|
||||
} catch (e) {
|
||||
return ImageProbeResult(
|
||||
url: image.url,
|
||||
contentLength: 0,
|
||||
first16BytesHex: '',
|
||||
looksLikeImage: false,
|
||||
error: e.toString(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
魔数判断:
|
||||
bool isImageMagic(List<int> b) {
|
||||
if (b.length < 4) return false;
|
||||
|
||||
// JPG
|
||||
if (b[0] == 0xFF && b[1] == 0xD8 && b[2] == 0xFF) return true;
|
||||
|
||||
// PNG
|
||||
if (b[0] == 0x89 && b[1] == 0x50 && b[2] == 0x4E && b[3] == 0x47) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// WEBP
|
||||
if (b.length >= 12 &&
|
||||
b[0] == 0x52 &&
|
||||
b[1] == 0x49 &&
|
||||
b[2] == 0x46 &&
|
||||
b[3] == 0x46 &&
|
||||
b[8] == 0x57 &&
|
||||
b[9] == 0x45 &&
|
||||
b[10] == 0x42 &&
|
||||
b[11] == 0x50) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// GIF
|
||||
if (b[0] == 0x47 && b[1] == 0x49 && b[2] == 0x46 && b[3] == 0x38) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// BMP
|
||||
if (b[0] == 0x42 && b[1] == 0x4D) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
诊断表:
|
||||
403 / 444
|
||||
→ Referer / User-Agent / Cookie 问题
|
||||
|
||||
404
|
||||
→ 图片 URL 拼错或章节页已失效
|
||||
|
||||
200 + text/html
|
||||
→ 反爬页 / 登录页 / 错误页
|
||||
|
||||
200 + image/* + contentLength < 1024
|
||||
→ 可能是伪装错误图或破损图
|
||||
|
||||
200 + image/* + 魔数失败
|
||||
→ 响应不是有效图片
|
||||
|
||||
200 + image/* + 魔数成功但 Image.network 不显示
|
||||
→ headers 没传进 Image.network / 图片缓存污染 / Flutter 图片组件问题
|
||||
|
||||
* * *
|
||||
|
||||
P4:TOC 分页抓全章节
|
||||
=============
|
||||
|
||||
问题
|
||||
--
|
||||
|
||||
一人之下 600+ 话,但现在只显示几十章,说明目录分页没有抓全。
|
||||
支持字段
|
||||
----
|
||||
|
||||
tocNextPageRule
|
||||
nextTocUrl
|
||||
nextUrl
|
||||
nextPage
|
||||
tocNextPage
|
||||
|
||||
抓取流程
|
||||
----
|
||||
|
||||
firstTocUrl
|
||||
→ 请求当前页
|
||||
→ 解析 chapterList
|
||||
→ 解析 nextTocUrl
|
||||
→ resolve 成绝对 URL
|
||||
→ 合并章节并去重
|
||||
→ 继续下一页
|
||||
|
||||
终止条件
|
||||
----
|
||||
|
||||
nextUrl == null
|
||||
seenPageUrls 已访问
|
||||
连续 2 页新增章节为 0
|
||||
pageCount >= 100
|
||||
chapterCount >= 3000
|
||||
|
||||
建议参数:
|
||||
const maxTocPages = 100;
|
||||
const maxTocChapters = 3000;
|
||||
const emptyPageDelay = Duration(milliseconds: 500);
|
||||
|
||||
* * *
|
||||
|
||||
P5:章节排序支持番外
|
||||
===========
|
||||
|
||||
支持:
|
||||
第660话
|
||||
第660.5回
|
||||
Ch.660
|
||||
ch 660.5
|
||||
660话
|
||||
660.5 番外
|
||||
|
||||
章节号提取:
|
||||
double? extractChapterNo(String title) {
|
||||
var m = RegExp(r'第\s*(\d+(?:\.\d+)?)\s*[回话章节节]').firstMatch(title);
|
||||
if (m != null) return double.tryParse(m.group(1)!);
|
||||
|
||||
m = RegExp(r'[Cc]h\.?\s*(\d+(?:\.\d+)?)').firstMatch(title);
|
||||
if (m != null) return double.tryParse(m.group(1)!);
|
||||
|
||||
m = RegExp(r'^\s*(\d+(?:\.\d+)?)\s*(?:话|回|章|节)?').firstMatch(title);
|
||||
if (m != null) return double.tryParse(m.group(1)!);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
排序:
|
||||
chapters.sort((a, b) {
|
||||
final na = extractChapterNo(a.title);
|
||||
final nb = extractChapterNo(b.title);
|
||||
|
||||
if (na != null && nb != null && na != nb) {
|
||||
return na.compareTo(nb);
|
||||
}
|
||||
|
||||
return a.index.compareTo(b.index);
|
||||
});
|
||||
|
||||
* * *
|
||||
|
||||
P6:Script 图片变量提取
|
||||
================
|
||||
|
||||
当 `ruleContent.content` 失败或候选图片全无效时,从 rawHtml 的 script 中提取图片。
|
||||
|
||||
支持变量:
|
||||
newImgs
|
||||
imageList
|
||||
page_url
|
||||
chapterImages
|
||||
photosr
|
||||
img_data
|
||||
picTree
|
||||
|
||||
示例:
|
||||
List<String> extractImagesFromScripts(String html) {
|
||||
final out = <String>[];
|
||||
|
||||
final patterns = [
|
||||
RegExp(r'newImgs\s*=\s*(\[[\s\S]*?\])'),
|
||||
RegExp(r'imageList\s*=\s*(\[[\s\S]*?\])'),
|
||||
RegExp(r'"page_url"\s*:\s*(\[[\s\S]*?\])'),
|
||||
RegExp(r'chapterImages\s*=\s*(\[[\s\S]*?\])'),
|
||||
RegExp(r'photosr\s*=\s*(\[[\s\S]*?\])'),
|
||||
RegExp(r'img_data\s*=\s*(\[[\s\S]*?\])'),
|
||||
RegExp(r'picTree\s*=\s*(\[[\s\S]*?\])'),
|
||||
];
|
||||
|
||||
for (final p in patterns) {
|
||||
final m = p.firstMatch(html);
|
||||
if (m == null) continue;
|
||||
|
||||
final raw = m.group(1)!;
|
||||
final list = tryDecodeStringList(raw);
|
||||
out.addAll(list.map(sanitizeScriptUrl));
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
清洗:
|
||||
String sanitizeScriptUrl(String url) {
|
||||
return url
|
||||
.replaceAll(r'\/', '/')
|
||||
.replaceAll(r'\u002F', '/');
|
||||
}
|
||||
|
||||
* * *
|
||||
|
||||
P7:漫画调试页
|
||||
========
|
||||
|
||||
入口
|
||||
--
|
||||
|
||||
书籍详情页 → 长按章节 → 调试本章
|
||||
阅读器右上角菜单 → 调试当前章
|
||||
|
||||
展示内容
|
||||
----
|
||||
|
||||
{
|
||||
"sourceName": "包子漫画5",
|
||||
"bookTitle": "一人之下",
|
||||
"chapterTitle": "第652回",
|
||||
"chapterUrl": "...",
|
||||
"ruleContent": "...",
|
||||
"rawHtmlLength": 123456,
|
||||
"contentOutputLength": 2048,
|
||||
"rawCandidateCount": 17,
|
||||
"validCandidateCount": 0,
|
||||
"pageImageCount": 0,
|
||||
"firstCandidates": [
|
||||
{
|
||||
"raw": "...",
|
||||
"normalized": "...",
|
||||
"reason": "placeholder"
|
||||
}
|
||||
],
|
||||
"probe": [
|
||||
{
|
||||
"url": "...",
|
||||
"status": 200,
|
||||
"contentType": "image/jpeg",
|
||||
"contentLength": 312,
|
||||
"first16Bytes": "ffd8ff...",
|
||||
"looksLikeImage": false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
操作按钮
|
||||
----
|
||||
|
||||
复制调试信息
|
||||
重新检测前 5 张图
|
||||
切换 Referer:章节页 / 详情页 / 源首页
|
||||
清除图片缓存后重试
|
||||
打开原始章节 URL
|
||||
|
||||
* * *
|
||||
|
||||
P8:搜索健康分与增量搜索
|
||||
=============
|
||||
|
||||
问题
|
||||
--
|
||||
|
||||
搜索慢主要来自:
|
||||
坏源太多
|
||||
每个源 timeout 太长
|
||||
所有源一起等结果
|
||||
模型
|
||||
--
|
||||
|
||||
class SourceHealth {
|
||||
final String sourceId;
|
||||
|
||||
int successCount = 0;
|
||||
int failCount = 0;
|
||||
int timeoutCount = 0;
|
||||
int totalLatencyMs = 0;
|
||||
int consecutiveFails = 0;
|
||||
|
||||
DateTime? lastSuccessAt;
|
||||
DateTime? lastFailAt;
|
||||
}
|
||||
|
||||
策略
|
||||
--
|
||||
|
||||
健康源优先
|
||||
坏源降权
|
||||
连续失败 >= 5 临时跳过
|
||||
搜索超时 5~8 秒
|
||||
快速源先返回
|
||||
慢源后台补充
|
||||
|
||||
搜索阶段
|
||||
----
|
||||
|
||||
0~5s:快速源结果
|
||||
5~8s:普通源结果
|
||||
8s 后:慢源后台补
|
||||
|
||||
* * *
|
||||
|
||||
三、最终落地顺序
|
||||
========
|
||||
|
||||
第 1 阶段:UI 先止血
|
||||
-------------
|
||||
|
||||
P0:顶部搜索框 UI 修复
|
||||
P0:在线搜索入口放进统一搜索页
|
||||
|
||||
结果:
|
||||
漫画页 / 图片页顶部不再错位
|
||||
在线搜索不再占右上角
|
||||
搜索体验统一
|
||||
第 2 阶段:图片链路可观测
|
||||
--------------
|
||||
|
||||
P1:图片 URL 合法性验证
|
||||
P2:PageImage + headers 拆分
|
||||
P3:图片 HTTP 探测
|
||||
P7:漫画调试页
|
||||
|
||||
结果:
|
||||
可以准确判断图片失败原因
|
||||
不再靠猜
|
||||
第 3 阶段:章节抓全
|
||||
-----------
|
||||
|
||||
P4:TOC 分页
|
||||
P5:章节排序
|
||||
|
||||
结果:
|
||||
一人之下这类 600+ 话漫画可以抓全
|
||||
章节顺序稳定
|
||||
第 4 阶段:提升漫画源覆盖率
|
||||
---------------
|
||||
|
||||
P6:Script 图片变量提取
|
||||
|
||||
结果:
|
||||
更多漫画源能解析图片
|
||||
第 5 阶段:优化搜索速度
|
||||
-------------
|
||||
|
||||
P8:搜索健康分与增量搜索
|
||||
|
||||
结果:
|
||||
坏源不拖慢搜索
|
||||
结果更快出现
|
||||
|
||||
* * *
|
||||
|
||||
|
||||
=======
|
||||
@@ -0,0 +1,70 @@
|
||||
# 视频云同步与内容合集设计方案
|
||||
|
||||
## 目标
|
||||
|
||||
让 X、小红书、抖音采集到的视频和图片一样进入 Sakura Cloud,并能在手机端与 Web 端查看。云端对每个用户的视频总占用做 30GB 上限,Web 端从“后台列表”调整为适合阅读、看视频、浏览插画的内容合集。
|
||||
|
||||
## 云端设计
|
||||
|
||||
- 复用现有 `upload / sync / gallery / files` 链路,不新增数据库表,不改变 API 返回结构。
|
||||
- 视频仍作为 `Artwork` 保存,`content_type=video`,`mime_type=video/*`,文件实体复用 `FileBlob` 与 `UserFile`。
|
||||
- 上传接口允许 `resource_type=video`,视频文件进入 `uploads/videos/{user_id}/{year}/{month}`。
|
||||
- 新增配置:
|
||||
- `MAX_VIDEO_UPLOAD_SIZE_MB=2048`:单个视频默认最大 2GB。
|
||||
- `VIDEO_STORAGE_LIMIT_GB=30`:单用户视频总空间上限 30GB。
|
||||
- 配额计算以当前登录用户为准,从 `UserFile -> FileBlob` 统计该用户所有 `mime_type LIKE 'video/%'` 的文件大小。命中上限时返回 `413`,不会创建上传会话。
|
||||
- 已存在文件秒传时同样检查该用户的视频总配额,防止跨用户复用文件绕过限制。
|
||||
|
||||
## 桌面端采集同步设计
|
||||
|
||||
- `SyncManager` 根据 MIME 与 `content_type` 判断上传资源类型:
|
||||
- 小说:`novel`
|
||||
- 视频:`video`
|
||||
- 其他:`image`
|
||||
- X 手动预览下载和关注作者同步共用云端同步逻辑,视频文件下载成功后自动同步 metadata。
|
||||
- WebUI 通用插件下载路径也接入云同步,因此小红书、抖音插件只要返回 `files=[{type: "video"}]`,就能自动上传云端。
|
||||
- 小红书插件补充 DOM 视频解析,支持从 `video/source` 标签提取直接视频地址;搜索结果进入详情下载时也同时处理 `images + videos`。
|
||||
- 抖音插件已有视频文件抽取与下载逻辑,本方案复用其 `Resource.files` 输出,不重写下载器。
|
||||
|
||||
## 手机端设计
|
||||
|
||||
- `Artwork` 模型补充 `mimeType`、`fileSize`、`isVideo` 与 `primaryMediaPath`。
|
||||
- 图库页新增视频筛选,视频卡片显示播放叠层,不再伪装成普通插画。
|
||||
- 视频详情页展示视频封面、标题、作者、MIME、大小,并提供:
|
||||
- 播放视频
|
||||
- 下载到本地
|
||||
- 收藏
|
||||
- 为避免新增播放器依赖导致构建不稳定,当前采用“受保护下载到本地后调用系统播放器”的方式。
|
||||
- Android 通过 FileProvider 暴露 Sakura 下载目录中的临时读取 URI,不暴露任意路径。
|
||||
|
||||
## Web 端内容合集设计
|
||||
|
||||
- 普通用户首页改为内容合集:
|
||||
- 顶部概览:用户、空间、视频额度、最近内容。
|
||||
- 视频区:16:9 视频卡片,播放标记,显示作者与大小。
|
||||
- 插画区:缩略图网格。
|
||||
- 漫画区:阅读列表入口。
|
||||
- 小说区:阅读列表入口。
|
||||
- 在线搜索与提醒中心作为主操作入口。
|
||||
- 页面复用现有 API:
|
||||
- `/me/gallery?content_type=video`
|
||||
- `/me/gallery?content_type=illust`
|
||||
- `/me/gallery?content_type=manga`
|
||||
- `/novels/series`
|
||||
- CSS 使用现有粉白主题、8px 圆角和响应式网格,手机浏览时自动改为单列。
|
||||
|
||||
## 安全与边界
|
||||
|
||||
- 不上传第三方平台 Cookie、Token 或浏览器登录态。
|
||||
- 云端只保存视频文件与作品 metadata。
|
||||
- 视频配额按当前登录用户统计,避免水平越权。
|
||||
- 文件读取继续走现有 `/files` 和 gallery ownership 校验。
|
||||
- Android FileProvider 只给外部播放器一次性读取 URI,不暴露数据库或任意文件系统路径。
|
||||
|
||||
## 验收点
|
||||
|
||||
- X、小红书、抖音插件下载视频后,云同步开启时视频可进入 cloud gallery。
|
||||
- 超过 30GB 视频总配额时,上传初始化返回 413。
|
||||
- 手机端图库可以筛选视频,视频详情可以下载并调用系统播放器查看。
|
||||
- Web 普通用户首页能看到视频、插画、漫画、小说四类内容入口。
|
||||
- 现有图片、漫画、小说同步与阅读入口保持可用。
|
||||
+322
@@ -0,0 +1,322 @@
|
||||
# Sakura Downloader(樱云下载器)项目方案书
|
||||
|
||||
> 版本:v1.0 | 日期:2026-06-18 | 状态:已交付
|
||||
|
||||
---
|
||||
|
||||
## 一、项目概述
|
||||
|
||||
### 1.1 项目名称
|
||||
|
||||
**Sakura Downloader(樱云下载器)** — 个人媒体采集、本地图库管理与跨设备云同步系统。
|
||||
|
||||
### 1.2 项目定位
|
||||
|
||||
一个面向个人用户的**多媒体内容采集-整理-阅读一体化平台**,解决"从多个内容平台采集插画/漫画/小说 → 本地归档整理 → 在手机/桌面等多设备上随时浏览阅读"的完整链路。
|
||||
|
||||
### 1.3 核心价值
|
||||
|
||||
| 痛点 | 解决方案 |
|
||||
|------|----------|
|
||||
| 多平台内容分散(Pixiv、X、小红书、AO3、笔趣阁等) | 统一插件架构,一个入口采集所有平台 |
|
||||
| 下载后文件散乱、难以检索 | 本地图库 + SQLite 索引 + 缩略图墙 + 标签/作者搜索 |
|
||||
| 桌面下载了,手机上没法看 | Sakura Cloud 云端同步,手机端实时访问 |
|
||||
| 关注的作者更新了不知道 | 云端每日摘要提醒,手机端查看 |
|
||||
| 跨平台小说阅读体验差 | Flutter 移动端专为阅读优化:自动分页、进度记忆、翻译、字体调节 |
|
||||
|
||||
---
|
||||
|
||||
## 二、系统架构
|
||||
|
||||
### 2.1 整体拓扑
|
||||
|
||||
```text
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Sakura Cloud (FastAPI) │
|
||||
│ PostgreSQL + JWT Auth │
|
||||
│ api.kunkunxiaomao.top:8000 │
|
||||
└──────┬──────────────────────┬───────────────────────────┘
|
||||
│ REST API │ REST API
|
||||
▼ ▼
|
||||
┌──────────────┐ ┌──────────────────┐
|
||||
│ Windows 桌面端 │ │ Android 移动端 │
|
||||
│ Python + │ │ Flutter + │
|
||||
│ CustomTkinter │ │ Riverpod + Dio │
|
||||
│ (采集/整理中枢) │ │ (阅读/提醒终端) │
|
||||
└──────┬─────────┘ └──────────────────┘
|
||||
│ 本地 SQLite │
|
||||
▼ ▼
|
||||
┌──────────────┐ ┌──────────────┐
|
||||
│ 本地图库文件 │ │ 本地缓存/下载 │
|
||||
│ (图片/漫画/小说)│ │ (离线阅读) │
|
||||
└──────────────┘ └──────────────┘
|
||||
```
|
||||
|
||||
### 2.2 技术栈总览
|
||||
|
||||
| 层级 | 技术选型 |
|
||||
|------|----------|
|
||||
| **Windows 桌面端** | Python 3.8+ · CustomTkinter · SQLite · Playwright · Pillow |
|
||||
| **云端 API** | FastAPI · PostgreSQL · SQLAlchemy · Alembic · JWT · Docker |
|
||||
| **Android 移动端** | Flutter 3.x · Dart · Riverpod · Dio · shared_preferences |
|
||||
| **管理后台 Web** | Vite · TypeScript · Vue/React · Nginx |
|
||||
| **插件系统** | Python 动态加载 · manifest.json · 统一 PluginBase 接口 |
|
||||
| **打包分发** | PyInstaller (EXE) · Flutter build (APK) |
|
||||
|
||||
---
|
||||
|
||||
## 三、客户端种类与职责
|
||||
|
||||
### 3.1 Windows 桌面端(`pixiv_app/` + `main.py`)
|
||||
|
||||
**角色:采集与整理中枢**
|
||||
|
||||
| 功能模块 | 说明 |
|
||||
|----------|------|
|
||||
| **多平台采集** | 通过插件系统支持 Pixiv、X (Twitter)、小红书、AO3、笔趣阁等 |
|
||||
| **Cookie/密码登录** | 支持 Cookie 导入(推荐)和账号密码登录 |
|
||||
| **灵活抓取模式** | 作者全部作品、单作品/漫画、小说/系列、关键词搜索 |
|
||||
| **本地图库** | SQLite 索引 + 文件系统存储,支持缩略图生成 |
|
||||
| **缩略图墙** | 本地 Web 服务(127.0.0.1:8765),浏览器中浏览、搜索、筛选、收藏 |
|
||||
| **关注同步** | Pixiv 关注作者管理 + X 关注同步 |
|
||||
| **云同步** | 一键上传到 Sakura Cloud,自动同步新下载内容 |
|
||||
| **代理池** | 浏览器代理池 + HTTP 代理,规避反爬 |
|
||||
| **Playwright 集成** | 无头浏览器渲染 JS 页面(X 插件依赖) |
|
||||
| **指纹伪装** | 浏览器指纹随机化,降低封禁风险 |
|
||||
|
||||
**打包产物**:`sakuradownloader.exe` (~68 MB, PyInstaller 单文件)
|
||||
|
||||
### 3.2 Sakura Cloud 云端(`sakura_cloud/`)
|
||||
|
||||
**角色:数据与账号中心**
|
||||
|
||||
| 功能模块 | 说明 |
|
||||
|----------|------|
|
||||
| **用户系统** | 注册/登录/邮箱验证/JWT Token 鉴权 |
|
||||
| **媒体存储** | 图片/漫画/小说上传、存储、缩略图生成 |
|
||||
| **收藏系统** | 跨设备收藏同步 |
|
||||
| **关注与提醒** | 关注作者管理 + 每日更新摘要提醒 |
|
||||
| **阅读进度** | 小说阅读进度云端存储,跨设备同步 |
|
||||
| **管理后台** | SPA 管理控制台(可选) |
|
||||
| **自动运维** | 启动自动 migrate、定时清理过期上传、每 5 分钟检查提醒摘要 |
|
||||
|
||||
**部署方式**:Docker Compose(FastAPI + PostgreSQL)
|
||||
|
||||
### 3.3 Android 移动端(`sakura_mobile/`)
|
||||
|
||||
**角色:跨设备阅读与提醒终端**
|
||||
|
||||
| 功能模块 | 说明 |
|
||||
|----------|------|
|
||||
| **发现页** | 最近浏览 + 更新提醒摘要 |
|
||||
| **图片页** | 缩略图列表 · 详情 · 收藏 · 下载 · 编辑 · 删除 |
|
||||
| **漫画页** | 按漫画分组浏览,整组连续阅读,整组下载 |
|
||||
| **书架页** | 三列彩色方块风格,系列小说按书名归档,章节选择 |
|
||||
| **小说阅读器** | 自动分页 · 阅读进度记忆 · 收藏 · 当前页翻译 · 原文/译文切换 · 字体/行高/背景色调节 · 章节切换 |
|
||||
| **我的页** | 用户资料 · 云端空间 · 收藏管理 · 每日提醒开关/时间 · 提醒中心 |
|
||||
| **搜索** | 图片/漫画/小说全局搜索 |
|
||||
|
||||
**打包产物**:`app-release.apk` (~54 MB, 调试签名)
|
||||
|
||||
### 3.4 管理后台 Web(`sakura_admin_web/`)
|
||||
|
||||
**角色:云端管理控制台**
|
||||
|
||||
- 基于 Vite + TypeScript 的 SPA 单页应用
|
||||
- 通过 Sakura Cloud 的 `/admin` 路由挂载
|
||||
- 提供可视化的数据和用户管理界面
|
||||
- 可选部署(`admin_frontend_enabled` 配置项)
|
||||
|
||||
---
|
||||
|
||||
## 四、插件体系
|
||||
|
||||
### 4.1 插件架构
|
||||
|
||||
```text
|
||||
plugins/
|
||||
├── pixiv/ # Pixiv 插画/漫画/小说
|
||||
│ ├── manifest.json
|
||||
│ └── plugin.py
|
||||
├── x/ # X (Twitter) 媒体
|
||||
│ ├── manifest.json
|
||||
│ ├── plugin.py
|
||||
│ ├── playwright_client.py
|
||||
│ └── x_login.py
|
||||
├── xiaohongshu/ # 小红书
|
||||
├── ao3/ # Archive of Our Own
|
||||
└── biquge/ # 笔趣阁(小说)
|
||||
```
|
||||
|
||||
### 4.2 插件接口规范
|
||||
|
||||
每个插件只需实现 `PluginBase` 抽象类,包含:
|
||||
|
||||
- `platform_name` — 平台标识
|
||||
- `supports_modes` — 支持的抓取模式列表
|
||||
- `search()` — 搜索
|
||||
- `fetch_work()` — 获取作品详情
|
||||
- `fetch_author_works()` — 获取作者全部作品
|
||||
- `download()` — 下载媒体文件
|
||||
|
||||
通过 `manifest.json` 声明元数据,桌面端启动时自动发现并加载。
|
||||
|
||||
---
|
||||
|
||||
## 五、核心数据流
|
||||
|
||||
### 5.1 采集下载流程
|
||||
|
||||
```text
|
||||
用户输入(ID/链接/关键词)
|
||||
│
|
||||
▼
|
||||
任务解析器 (parser.py)
|
||||
│
|
||||
▼
|
||||
任务队列 (queue.py)
|
||||
│
|
||||
▼
|
||||
插件调度 → 对应平台 plugin.py
|
||||
│
|
||||
▼
|
||||
HTTP/Playwright 抓取 → 解析作品列表
|
||||
│
|
||||
▼
|
||||
下载器 (downloader.py) → 保存到本地图库
|
||||
│
|
||||
▼
|
||||
SQLite 索引写入 + 缩略图生成
|
||||
│
|
||||
▼
|
||||
(可选) 云同步上传到 Sakura Cloud
|
||||
```
|
||||
|
||||
### 5.2 云同步流程
|
||||
|
||||
```text
|
||||
桌面端下载完成
|
||||
│
|
||||
▼
|
||||
检查"启用云同步"开关
|
||||
│
|
||||
▼
|
||||
cloud_sync.py → POST /api/v1/media/upload
|
||||
│
|
||||
▼
|
||||
Sakura Cloud 存储 + 生成缩略图
|
||||
│
|
||||
▼
|
||||
移动端刷新 → GET /api/v1/media → 展示
|
||||
```
|
||||
|
||||
### 5.3 每日提醒流程
|
||||
|
||||
```text
|
||||
桌面端同步关注作者的新作品到云端
|
||||
│
|
||||
▼
|
||||
云端写入 follow + update_alert 记录
|
||||
│
|
||||
▼
|
||||
reminder_digest_loop (每5分钟检查)
|
||||
│
|
||||
▼
|
||||
到达用户设定的提醒时间 → 生成每日摘要
|
||||
│
|
||||
▼
|
||||
移动端"发现"页展示更新提醒
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 六、用法概览
|
||||
|
||||
### 6.1 桌面端典型工作流
|
||||
|
||||
1. **登录** → 导入 Pixiv Cookie(推荐)或账号密码登录
|
||||
2. **抓取** → 选择平台 → 选择模式 → 输入目标 → 开始抓取
|
||||
3. **浏览** → 打开缩略图墙 → 搜索/筛选/收藏
|
||||
4. **关注** → 在缩略图墙中关注作者 → 后续同步关注作者更新
|
||||
5. **云同步** → 登录 Sakura Cloud → 开启云同步 → 自动上传
|
||||
|
||||
### 6.2 移动端典型工作流
|
||||
|
||||
1. **安装** → 安装 APK → 允许未知来源
|
||||
2. **登录** → 使用与桌面端相同的 Sakura Cloud 账号
|
||||
3. **浏览** → 底部 5 个页签切换(发现/图片/漫画/书架/我的)
|
||||
4. **阅读小说** → 自动分页 · 调节字体 · 翻译 · 章节切换
|
||||
5. **提醒** → 我的 → 每日更新提醒 → 开启并设置时间
|
||||
|
||||
### 6.3 云端部署
|
||||
|
||||
```bash
|
||||
cd sakura_cloud
|
||||
docker-compose up -d
|
||||
# FastAPI :8000 + PostgreSQL :5432
|
||||
```
|
||||
|
||||
### 6.4 构建打包
|
||||
|
||||
```bash
|
||||
# 桌面 EXE
|
||||
scripts\build_desktop_exe.bat
|
||||
|
||||
# Android APK
|
||||
scripts\build_mobile_android.bat
|
||||
|
||||
# 复制到 releases/
|
||||
scripts\copy_release_artifacts.bat
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 七、项目交付物
|
||||
|
||||
| 交付物 | 路径 | 说明 |
|
||||
|--------|------|------|
|
||||
| Windows EXE | `releases/sakuradownloader.exe` | 双击即用,~68 MB |
|
||||
| Android APK | `releases/app-release.apk` | 调试签名,~54 MB |
|
||||
| 用户使用说明 | `使用说明-桌面版与安卓版.md` | 终端用户操作手册 |
|
||||
| 项目方案书 | `docs/项目方案书.md` | 本文档 |
|
||||
| 开发文档 | `项目文件/BUILD.md` | 构建与排错 |
|
||||
| 构建脚本 | `scripts/` | 一键打包脚本 |
|
||||
| 云端 API | `sakura_cloud/` | Docker 部署 |
|
||||
| 源码 | 全部 Python/Dart/TypeScript | 可二次开发 |
|
||||
|
||||
---
|
||||
|
||||
## 八、技术亮点
|
||||
|
||||
1. **插件化架构** — 新平台接入只需新增一个插件目录,不改动核心代码
|
||||
2. **多反爬策略** — Cookie 管理 · Playwright 无头浏览器 · 代理池 · 指纹伪装 · 限速退避
|
||||
3. **跨设备云同步** — 桌面采集 → 云端存储 → 手机阅读,全链路打通
|
||||
4. **沉浸式阅读体验** — 自动分页 · 进度记忆 · 即时翻译 · 个性化排版
|
||||
5. **智能提醒** — 关注作者更新云端汇总,非本地闹钟,静默不打扰
|
||||
6. **本地优先** — 所有数据本地 SQLite 存储,云端为可选项,离线也能用
|
||||
|
||||
---
|
||||
|
||||
## 九、后续演进方向
|
||||
|
||||
| 方向 | 说明 |
|
||||
|------|------|
|
||||
| iOS 客户端 | Flutter 已预留 iOS 平台支持,补充苹果开发者账号与签名 |
|
||||
| 正式应用签名 | APK 替换调试签名为正式 keystore,支持上架分发 |
|
||||
| 更多平台插件 | 持续扩充插件库(如 Fantia、Fanbox、DeviantArt 等) |
|
||||
| AI 能力增强 | OCR 文字识别(代码已预留)· AI 翻译优化 · 智能标签 |
|
||||
| 分布式爬取 | `distributed_crawler/` 目录已有 Master-Worker 架构原型 |
|
||||
| PWA 移动端 | 缩略图墙升级为 PWA,支持手机浏览器直接访问 |
|
||||
|
||||
---
|
||||
|
||||
## 十、许可与免责
|
||||
|
||||
- 本项目仅供个人学习、研究及合法使用。
|
||||
- 使用 X 插件需自行承担对应平台的使用条款风险。
|
||||
- APK 当前为调试签名,仅供个人测试,不宜正式分发。
|
||||
- 请遵守各内容平台的用户协议,合理使用爬取功能。
|
||||
|
||||
---
|
||||
|
||||
*文档生成于 2026-06-18 · Sakura Downloader Project*
|
||||
@@ -0,0 +1,10 @@
|
||||
from pixiv_app.gui.app import SakuraDownloaderGUI
|
||||
|
||||
|
||||
def main() -> None:
|
||||
app = SakuraDownloaderGUI()
|
||||
app.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Sakura downloader application package."""
|
||||
@@ -0,0 +1 @@
|
||||
"""Core business logic for Sakura downloader."""
|
||||
@@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import requests
|
||||
|
||||
from pixiv_app.core.downloader import USER_AGENT, build_headers
|
||||
|
||||
|
||||
LOGIN_PAGE_URL = "https://accounts.pixiv.net/login"
|
||||
LOGIN_AJAX_URL = "https://accounts.pixiv.net/ajax/login?lang=zh"
|
||||
VALIDATE_URL = "https://www.pixiv.net/setting_user.php"
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuthResult:
|
||||
success: bool
|
||||
message: str
|
||||
cookie: str = ""
|
||||
requires_verification: bool = False
|
||||
user_name: str = ""
|
||||
user_id: str = ""
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, path: str | Path = "pixiv_app_session.json") -> None:
|
||||
self.path = Path(path)
|
||||
|
||||
def load(self) -> dict[str, Any]:
|
||||
if not self.path.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(self.path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def save(self, payload: dict[str, Any]) -> None:
|
||||
self.path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
class PixivAuthClient:
|
||||
def __init__(self) -> None:
|
||||
self.timeout = 20
|
||||
|
||||
def validate_cookie(self, cookie: str) -> AuthResult:
|
||||
cookie = cookie.strip()
|
||||
if not cookie:
|
||||
return AuthResult(False, "Cookie 为空,请先填写或登录。")
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update(build_headers(cookie=cookie, referer="https://www.pixiv.net/"))
|
||||
try:
|
||||
response = session.get(VALIDATE_URL, allow_redirects=True, timeout=self.timeout)
|
||||
final_url = response.url
|
||||
if "return_to=" in final_url or "accounts.pixiv.net" in final_url:
|
||||
return AuthResult(False, "当前 Cookie 已失效或未登录,请更新会话。")
|
||||
user_id, user_name = self._extract_user_info(response.text)
|
||||
return AuthResult(True, "Cookie 有效,可直接开始抓取。", cookie=cookie, user_name=user_name, user_id=user_id)
|
||||
except Exception as exc:
|
||||
return AuthResult(False, f"Cookie 检测失败: {exc}")
|
||||
|
||||
def login_with_password(self, login_id: str, password: str) -> AuthResult:
|
||||
login_id = login_id.strip()
|
||||
password = password.strip()
|
||||
if not login_id or not password:
|
||||
return AuthResult(False, "账号和密码都不能为空。")
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update(
|
||||
{
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Origin": "https://accounts.pixiv.net",
|
||||
"Referer": LOGIN_PAGE_URL,
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
login_page = session.get(LOGIN_PAGE_URL, timeout=self.timeout)
|
||||
login_page.raise_for_status()
|
||||
tt = self._extract_tt(login_page.text)
|
||||
|
||||
payload = {
|
||||
"login_id": login_id,
|
||||
"password": password,
|
||||
"source": "accounts",
|
||||
"app_ios": "0",
|
||||
"ref": "",
|
||||
"return_to": "https://www.pixiv.net/",
|
||||
"g_recaptcha_response": "",
|
||||
"recaptcha_enterprise_score_token": "",
|
||||
}
|
||||
headers = dict(session.headers)
|
||||
if tt:
|
||||
headers["X-CSRF-Token"] = tt
|
||||
|
||||
response = session.post(
|
||||
LOGIN_AJAX_URL,
|
||||
data=urlencode(payload),
|
||||
headers={**headers, "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
|
||||
if isinstance(body, dict) and body.get("body", {}).get("success"):
|
||||
cookie = self._cookies_to_header(session.cookies)
|
||||
validation = self.validate_cookie(cookie)
|
||||
if validation.success:
|
||||
validation.message = "账号密码登录成功,会话已可用。"
|
||||
return validation
|
||||
return AuthResult(True, "登录接口返回成功,但会话校验未完全通过,请手动检测。", cookie=cookie)
|
||||
|
||||
body_payload = body.get("body", {}) if isinstance(body, dict) else {}
|
||||
errors = body_payload.get("errors", {}) if isinstance(body_payload, dict) else {}
|
||||
if errors.get("recaptcha") or body_payload.get("requireExtraVerification") or body_payload.get("requireTwoFactorAuthentication"):
|
||||
return AuthResult(
|
||||
False,
|
||||
"Pixiv 要求额外验证或二步验证,程序无法静默完成。请改用 Cookie 登录,或先在浏览器登录后导出 Cookie。",
|
||||
requires_verification=True,
|
||||
)
|
||||
message = self._flatten_error_message(errors) or "账号密码登录失败,请检查账号、密码或改用 Cookie。"
|
||||
return AuthResult(False, message)
|
||||
except Exception as exc:
|
||||
return AuthResult(False, f"账号密码登录失败: {exc}")
|
||||
|
||||
def _extract_tt(self, html_text: str) -> str:
|
||||
match = re.search(r'"pixivAccount\.tt":"([^"]+)"', html_text)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
def _extract_user_info(self, html_text: str) -> tuple[str, str]:
|
||||
user_id = ""
|
||||
user_name = ""
|
||||
patterns = [
|
||||
r'"userData":\{"id":"(\d+)","pixivId":"[^"]*","name":"([^"]+)"',
|
||||
r'"user_id":"(\d+)".*?"name":"([^"]+)"',
|
||||
]
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, html_text)
|
||||
if match:
|
||||
user_id = match.group(1)
|
||||
user_name = html.unescape(match.group(2))
|
||||
break
|
||||
return user_id, user_name
|
||||
|
||||
def _flatten_error_message(self, errors: dict[str, Any]) -> str:
|
||||
messages: list[str] = []
|
||||
for key, value in errors.items():
|
||||
if isinstance(value, str) and value.strip():
|
||||
messages.append(f"{key}: {value}")
|
||||
elif isinstance(value, list):
|
||||
text_items = [str(item).strip() for item in value if str(item).strip()]
|
||||
if text_items:
|
||||
messages.append(f"{key}: {'; '.join(text_items)}")
|
||||
return " | ".join(messages)
|
||||
|
||||
def _cookies_to_header(self, jar: requests.cookies.RequestsCookieJar) -> str:
|
||||
parts = [f"{cookie.name}={cookie.value}" for cookie in jar]
|
||||
return "; ".join(parts)
|
||||
@@ -0,0 +1,353 @@
|
||||
"""
|
||||
Playwright 浏览器代理池 - 支持动态换 IP
|
||||
功能:
|
||||
- 多代理轮换(轮询/随机/会话粘性)
|
||||
- 健康检查(自动剔除失效代理)
|
||||
- 延迟统计(加权选择最优代理)
|
||||
- 线程安全
|
||||
"""
|
||||
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, List, Dict, Any
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class RotationStrategy(Enum):
|
||||
"""轮换策略"""
|
||||
ROUND_ROBIN = "round_robin" # 轮询
|
||||
RANDOM = "random" # 随机
|
||||
WEIGHTED = "weighted" # 加权(按成功率+延迟)
|
||||
SESSION_STICKY = "session_sticky" # 会话粘性(同会话同IP)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProxyInfo:
|
||||
"""代理信息"""
|
||||
server: str # http://ip:port 或 socks5://ip:port
|
||||
username: str = ""
|
||||
password: str = ""
|
||||
|
||||
# 统计数据
|
||||
success_count: int = 0
|
||||
fail_count: int = 0
|
||||
total_latency_ms: float = 0.0
|
||||
avg_latency_ms: float = 0.0
|
||||
last_check_time: float = 0.0
|
||||
last_fail_time: float = 0.0
|
||||
|
||||
# 状态
|
||||
healthy: bool = True
|
||||
cooldown_until: float = 0.0 # 冷却结束时间
|
||||
|
||||
@property
|
||||
def proxy_url(self) -> str:
|
||||
"""获取完整的代理URL(含认证)"""
|
||||
if self.username and self.password:
|
||||
# http://user:pass@ip:port
|
||||
protocol = self.server.split("://")[0]
|
||||
rest = self.server.split("://")[1]
|
||||
return f"{protocol}://{self.username}:{self.password}@{rest}"
|
||||
return self.server
|
||||
|
||||
def to_requests_proxies(self) -> dict[str, str]:
|
||||
return {"http": self.proxy_url, "https": self.proxy_url}
|
||||
|
||||
def to_playwright_proxy(self) -> dict[str, str]:
|
||||
data = {"server": self.server}
|
||||
if self.username:
|
||||
data["username"] = self.username
|
||||
data["password"] = self.password
|
||||
return data
|
||||
|
||||
@property
|
||||
def success_rate(self) -> float:
|
||||
"""成功率"""
|
||||
total = self.success_count + self.fail_count
|
||||
if total == 0:
|
||||
return 1.0
|
||||
return self.success_count / total
|
||||
|
||||
@property
|
||||
def weight(self) -> float:
|
||||
"""
|
||||
计算代理权重(用于加权随机选择)
|
||||
考虑因素:成功率、延迟、冷却状态
|
||||
"""
|
||||
if not self.healthy:
|
||||
return 0.0
|
||||
|
||||
# 冷却中
|
||||
if self.cooldown_until > time.time():
|
||||
return 0.0
|
||||
|
||||
# 成功率权重
|
||||
success_weight = self.success_rate
|
||||
|
||||
# 延迟权重(延迟越低权重越高)
|
||||
if self.avg_latency_ms > 0:
|
||||
latency_weight = max(0.1, 1000.0 / max(self.avg_latency_ms, 1))
|
||||
else:
|
||||
latency_weight = 1.0
|
||||
|
||||
# 失败惩罚
|
||||
fail_penalty = 1.0 / (1.0 + self.fail_count * 0.5)
|
||||
|
||||
return success_weight * latency_weight * fail_penalty
|
||||
|
||||
|
||||
class ProxyPool:
|
||||
"""代理池 - 管理多个代理的获取和轮换"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
strategy: RotationStrategy = RotationStrategy.WEIGHTED,
|
||||
health_check_url: str = "https://httpbin.org/ip",
|
||||
health_check_timeout: int = 8,
|
||||
max_failures: int = 3,
|
||||
cooldown_seconds: int = 60,
|
||||
auto_health_check: bool = True,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
strategy: 轮换策略
|
||||
health_check_url: 健康检查URL
|
||||
health_check_timeout: 健康检查超时(秒)
|
||||
max_failures: 最大失败次数,超过后标记为不健康
|
||||
cooldown_seconds: 不健康代理的冷却时间(秒)
|
||||
auto_health_check: 是否自动健康检查
|
||||
"""
|
||||
self.strategy = strategy
|
||||
self.health_check_url = health_check_url
|
||||
self.health_check_timeout = health_check_timeout
|
||||
self.max_failures = max_failures
|
||||
self.cooldown_seconds = cooldown_seconds
|
||||
self.auto_health_check = auto_health_check
|
||||
|
||||
self._proxies: List[ProxyInfo] = []
|
||||
self._session_proxy_map: Dict[str, ProxyInfo] = {} # 会话粘性用
|
||||
self._round_robin_index = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add_proxy(self, server: str, username: str = "", password: str = "") -> None:
|
||||
"""添加单个代理"""
|
||||
with self._lock:
|
||||
# 去重
|
||||
existing = {p.server for p in self._proxies}
|
||||
if server not in existing:
|
||||
self._proxies.append(ProxyInfo(
|
||||
server=server,
|
||||
username=username,
|
||||
password=password,
|
||||
))
|
||||
|
||||
def add_proxy_url(self, value: str) -> None:
|
||||
parsed = parse_proxy_url(value)
|
||||
if parsed is not None:
|
||||
self.add_proxy(parsed.server, parsed.username, parsed.password)
|
||||
|
||||
def add_proxies(self, proxies: List[dict]) -> None:
|
||||
"""批量添加代理
|
||||
|
||||
Args:
|
||||
proxies: [{"server": "http://1.2.3.4:8080", "username": "", "password": ""}, ...]
|
||||
"""
|
||||
for p in proxies:
|
||||
self.add_proxy(
|
||||
server=p["server"],
|
||||
username=p.get("username", ""),
|
||||
password=p.get("password", ""),
|
||||
)
|
||||
|
||||
def add_proxy_text(self, text: str) -> int:
|
||||
count = 0
|
||||
for line in text.splitlines():
|
||||
parsed = parse_proxy_url(line)
|
||||
if parsed is None:
|
||||
continue
|
||||
self.add_proxy(parsed.server, parsed.username, parsed.password)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def remove_proxy(self, server: str) -> None:
|
||||
"""移除代理"""
|
||||
with self._lock:
|
||||
self._proxies = [p for p in self._proxies if p.server != server]
|
||||
|
||||
def get_proxy(self, session_id: str = None) -> Optional[ProxyInfo]:
|
||||
"""
|
||||
获取一个代理
|
||||
|
||||
Args:
|
||||
session_id: 会话ID,用于SESSION_STICKY策略
|
||||
|
||||
Returns:
|
||||
代理信息,如果没有可用代理返回None
|
||||
"""
|
||||
with self._lock:
|
||||
# 过滤出健康的代理
|
||||
now = time.time()
|
||||
healthy = [p for p in self._proxies if p.healthy and p.cooldown_until <= now]
|
||||
|
||||
if not healthy:
|
||||
# 尝试从冷却中恢复
|
||||
now = time.time()
|
||||
recovered = []
|
||||
for p in self._proxies:
|
||||
if not p.healthy and p.cooldown_until <= now:
|
||||
p.healthy = True
|
||||
p.fail_count = 0
|
||||
recovered.append(p)
|
||||
if recovered:
|
||||
healthy = recovered
|
||||
|
||||
if not healthy:
|
||||
return None
|
||||
|
||||
# 根据策略选择代理
|
||||
if self.strategy == RotationStrategy.RANDOM:
|
||||
return random.choice(healthy)
|
||||
|
||||
elif self.strategy == RotationStrategy.WEIGHTED:
|
||||
# 加权随机选择
|
||||
weights = [p.weight for p in healthy]
|
||||
total_weight = sum(weights)
|
||||
if total_weight <= 0:
|
||||
return random.choice(healthy)
|
||||
r = random.uniform(0, total_weight)
|
||||
cumsum = 0
|
||||
for i, w in enumerate(weights):
|
||||
cumsum += w
|
||||
if r <= cumsum:
|
||||
return healthy[i]
|
||||
return healthy[0]
|
||||
|
||||
elif self.strategy == RotationStrategy.SESSION_STICKY:
|
||||
if session_id and session_id in self._session_proxy_map:
|
||||
proxy = self._session_proxy_map[session_id]
|
||||
if proxy in healthy:
|
||||
return proxy
|
||||
# 新会话,分配一个代理
|
||||
proxy = self._get_round_robin(healthy)
|
||||
if session_id:
|
||||
self._session_proxy_map[session_id] = proxy
|
||||
return proxy
|
||||
|
||||
else: # ROUND_ROBIN
|
||||
return self._get_round_robin(healthy)
|
||||
|
||||
def _get_round_robin(self, proxies: List[ProxyInfo]) -> ProxyInfo:
|
||||
"""轮询获取代理"""
|
||||
proxy = proxies[self._round_robin_index % len(proxies)]
|
||||
self._round_robin_index += 1
|
||||
return proxy
|
||||
|
||||
def mark_success(self, proxy: ProxyInfo, latency_ms: float = 0) -> None:
|
||||
"""标记代理请求成功"""
|
||||
if proxy is None:
|
||||
return
|
||||
with self._lock:
|
||||
proxy.success_count += 1
|
||||
proxy.fail_count = 0
|
||||
proxy.healthy = True
|
||||
if latency_ms > 0:
|
||||
# 更新平均延迟(指数移动平均)
|
||||
if proxy.avg_latency_ms == 0:
|
||||
proxy.avg_latency_ms = latency_ms
|
||||
else:
|
||||
proxy.avg_latency_ms = proxy.avg_latency_ms * 0.7 + latency_ms * 0.3
|
||||
proxy.last_check_time = time.time()
|
||||
|
||||
def mark_failed(self, proxy: ProxyInfo) -> None:
|
||||
"""标记代理请求失败"""
|
||||
if proxy is None:
|
||||
return
|
||||
with self._lock:
|
||||
proxy.fail_count += 1
|
||||
if proxy.fail_count >= self.max_failures:
|
||||
proxy.healthy = False
|
||||
proxy.cooldown_until = time.time() + self.cooldown_seconds
|
||||
|
||||
def health_check(self, proxy: ProxyInfo = None) -> bool:
|
||||
"""
|
||||
健康检查单个或所有代理
|
||||
|
||||
Returns:
|
||||
如果检查通过返回True
|
||||
"""
|
||||
import requests
|
||||
|
||||
proxies_to_check = [proxy] if proxy else self._proxies
|
||||
|
||||
for p in proxies_to_check:
|
||||
if p is None:
|
||||
continue
|
||||
try:
|
||||
start = time.time()
|
||||
resp = requests.get(
|
||||
self.health_check_url,
|
||||
proxies={"http": p.proxy_url, "https": p.proxy_url},
|
||||
timeout=self.health_check_timeout,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
latency_ms = (time.time() - start) * 1000
|
||||
self.mark_success(p, latency_ms)
|
||||
else:
|
||||
self.mark_failed(p)
|
||||
except Exception:
|
||||
self.mark_failed(p)
|
||||
|
||||
return proxy.healthy if proxy else len(self.get_healthy_proxies()) > 0
|
||||
|
||||
def get_healthy_proxies(self) -> List[ProxyInfo]:
|
||||
"""获取所有健康代理"""
|
||||
with self._lock:
|
||||
return [p for p in self._proxies if p.healthy]
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""获取代理池统计信息"""
|
||||
with self._lock:
|
||||
healthy = [p for p in self._proxies if p.healthy]
|
||||
return {
|
||||
"total": len(self._proxies),
|
||||
"healthy": len(healthy),
|
||||
"unhealthy": len(self._proxies) - len(healthy),
|
||||
"proxies": [
|
||||
{
|
||||
"server": p.server,
|
||||
"healthy": p.healthy,
|
||||
"success_rate": p.success_rate,
|
||||
"avg_latency_ms": round(p.avg_latency_ms, 2),
|
||||
"fail_count": p.fail_count,
|
||||
}
|
||||
for p in self._proxies
|
||||
]
|
||||
}
|
||||
|
||||
def release_session(self, session_id: str) -> None:
|
||||
"""释放会话绑定的代理"""
|
||||
with self._lock:
|
||||
self._session_proxy_map.pop(session_id, None)
|
||||
|
||||
|
||||
def parse_proxy_url(value: str) -> Optional[ProxyInfo]:
|
||||
text = value.strip()
|
||||
if not text or text.startswith("#"):
|
||||
return None
|
||||
if "://" not in text:
|
||||
text = "http://" + text
|
||||
import re
|
||||
|
||||
match = re.match(r"^(https?|socks5)://(?:(?P<user>[^:@/]+):(?P<pwd>[^@/]+)@)?(?P<host>[\w.\-]+):(?P<port>\d{2,5})$", text, re.I)
|
||||
if not match:
|
||||
return None
|
||||
scheme = text.split("://", 1)[0].lower()
|
||||
host = match.group("host")
|
||||
port = match.group("port")
|
||||
user = match.group("user") or ""
|
||||
pwd = match.group("pwd") or ""
|
||||
return ProxyInfo(server=f"{scheme}://{host}:{port}", username=user, password=pwd)
|
||||
|
||||
|
||||
@@ -0,0 +1,602 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def file_sha256(path: str | Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open("rb") as fp:
|
||||
for chunk in iter(lambda: fp.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def persist_cloud_session_tokens(access_token: str, refresh_token: str = "") -> None:
|
||||
"""将刷新后的 Token 写回桌面会话文件,供 GUI 与 Gallery API 共用。"""
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.paths import app_session_file
|
||||
|
||||
store = SessionStore(app_session_file())
|
||||
payload = store.load()
|
||||
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
|
||||
cloud["access_token"] = access_token
|
||||
if refresh_token:
|
||||
cloud["refresh_token"] = refresh_token
|
||||
payload["cloud_sync"] = cloud
|
||||
store.save(payload)
|
||||
|
||||
|
||||
def _decode_jwt_exp(token: str) -> float:
|
||||
"""解码 JWT 的 exp 字段,返回 Unix 时间戳;解码失败返回 0。"""
|
||||
if not token or "." not in token:
|
||||
return 0.0
|
||||
try:
|
||||
payload = token.split(".")[1]
|
||||
payload += "=" * (4 - len(payload) % 4)
|
||||
decoded = json.loads(base64.urlsafe_b64decode(payload))
|
||||
return float(decoded.get("exp", 0))
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
class CloudSession:
|
||||
"""全局单例,统一管理云同步 Token 的读写、刷新与持久化。
|
||||
|
||||
所有需要访问 Sakura Cloud API 的模块(GUI、Gallery、FollowSync、XMedia)
|
||||
都通过此单例获取配置和创建客户端,确保 token 刷新互不冲突、变更全局可见。
|
||||
"""
|
||||
|
||||
_instance: CloudSession | None = None
|
||||
_instance_lock = threading.Lock()
|
||||
|
||||
def __new__(cls) -> CloudSession:
|
||||
if cls._instance is None:
|
||||
with cls._instance_lock:
|
||||
if cls._instance is None:
|
||||
instance = super().__new__(cls)
|
||||
instance._base_url = ""
|
||||
instance._access_token = ""
|
||||
instance._refresh_token = ""
|
||||
instance._enabled = False
|
||||
instance._refresh_lock = threading.Lock()
|
||||
instance._listeners: list[Callable[[str, str], None]] = []
|
||||
instance._last_refresh_time = 0.0
|
||||
cls._instance = instance
|
||||
return cls._instance
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 配置入口
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def configure(
|
||||
self,
|
||||
*,
|
||||
base_url: str = "",
|
||||
access_token: str = "",
|
||||
refresh_token: str = "",
|
||||
enabled: bool = False,
|
||||
) -> None:
|
||||
"""GUI 启动 / 登录 / 保存会话时调用,同步最新 token 到全局单例。"""
|
||||
self._base_url = (base_url or "").strip()
|
||||
self._access_token = (access_token or "").strip()
|
||||
self._refresh_token = (refresh_token or "").strip()
|
||||
self._enabled = bool(enabled)
|
||||
|
||||
def load_from_disk(self) -> dict[str, Any]:
|
||||
"""从磁盘会话文件读取云端配置并应用到单例,返回原始 payload。"""
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.paths import app_session_file
|
||||
|
||||
store = SessionStore(app_session_file())
|
||||
payload = store.load()
|
||||
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
|
||||
self.configure(
|
||||
base_url=str(cloud.get("base_url") or ""),
|
||||
access_token=str(cloud.get("access_token") or ""),
|
||||
refresh_token=str(cloud.get("refresh_token") or ""),
|
||||
enabled=bool(cloud.get("enabled", False)),
|
||||
)
|
||||
return payload
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self._enabled and bool(self._base_url) and bool(self._access_token)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Config 快照
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_config(self) -> CloudSyncConfig:
|
||||
"""返回当前 token 的 CloudSyncConfig 快照。
|
||||
|
||||
调用时机:每次创建 SakuraCloudClient 前。
|
||||
若 access_token 即将过期(< 5 分钟),主动触发一次刷新。
|
||||
"""
|
||||
if self._access_token and self._is_expired_or_near(threshold_seconds=300):
|
||||
self.try_refresh()
|
||||
return CloudSyncConfig(
|
||||
base_url=self._base_url,
|
||||
access_token=self._access_token,
|
||||
refresh_token=self._refresh_token,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 监听者管理
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_listener(self, callback: Callable[[str, str], None]) -> None:
|
||||
"""注册 token 刷新回调(如 GUI 的 _on_cloud_token_refreshed)。"""
|
||||
if callback not in self._listeners:
|
||||
self._listeners.append(callback)
|
||||
|
||||
def remove_listener(self, callback: Callable[[str, str], None]) -> None:
|
||||
try:
|
||||
self._listeners.remove(callback)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def notify_listeners(self, access_token: str, refresh_token: str) -> None:
|
||||
"""通知所有监听者 token 已更新(成功时为新的 token 对,失败时为空字符串)。"""
|
||||
for callback in self._listeners:
|
||||
try:
|
||||
callback(access_token, refresh_token)
|
||||
except Exception:
|
||||
logger.exception("CloudSession listener %s failed", getattr(callback, "__name__", callback))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Token 刷新(核心)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def try_refresh(self) -> bool:
|
||||
"""统一刷新入口。使用全局锁防止并发重复刷新。
|
||||
|
||||
Returns:
|
||||
True: 刷新成功,access_token 已更新
|
||||
False: 刷新失败,token 已被清空,监听者已收到空 token 通知
|
||||
"""
|
||||
refresh = self._refresh_token
|
||||
if not refresh:
|
||||
return False
|
||||
|
||||
with self._refresh_lock:
|
||||
# 二次检查:可能其他线程已刷新
|
||||
refresh = self._refresh_token
|
||||
if not refresh:
|
||||
return False
|
||||
|
||||
# 距上次刷新不足 5 秒,跳过(防抖)
|
||||
elapsed = time.time() - self._last_refresh_time
|
||||
if elapsed < 5.0 and self._access_token:
|
||||
return True
|
||||
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self._base_url}/api/v1/auth/refresh",
|
||||
json={"refresh_token": refresh},
|
||||
timeout=30.0,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.warning(
|
||||
"CloudSession refresh failed HTTP %s: %s",
|
||||
resp.status_code,
|
||||
resp.text[:200],
|
||||
)
|
||||
return False
|
||||
|
||||
body = resp.json()
|
||||
data = body.get("data", body) if isinstance(body, dict) else {}
|
||||
if not isinstance(data, dict):
|
||||
self._on_refresh_failed()
|
||||
return False
|
||||
|
||||
new_access = (data.get("access_token") or "").strip()
|
||||
new_refresh = (data.get("refresh_token") or "").strip()
|
||||
if not new_access:
|
||||
self._on_refresh_failed()
|
||||
return False
|
||||
|
||||
# 更新内存
|
||||
self._access_token = new_access
|
||||
if new_refresh:
|
||||
self._refresh_token = new_refresh
|
||||
self._last_refresh_time = time.time()
|
||||
|
||||
# 写盘
|
||||
self._persist()
|
||||
|
||||
# 通知监听者
|
||||
self.notify_listeners(self._access_token, self._refresh_token)
|
||||
logger.info("CloudSession token refreshed successfully")
|
||||
return True
|
||||
|
||||
except Exception:
|
||||
logger.exception("CloudSession refresh error")
|
||||
self._on_refresh_failed()
|
||||
return False
|
||||
|
||||
def _on_refresh_failed(self) -> None:
|
||||
"""刷新失败:清空 token + 写盘 + 通知监听者。"""
|
||||
self._access_token = ""
|
||||
self._refresh_token = ""
|
||||
self._last_refresh_time = time.time()
|
||||
self._persist()
|
||||
self.notify_listeners("", "")
|
||||
|
||||
def _persist(self) -> None:
|
||||
"""将当前 token 写入磁盘会话文件。异常记录日志但不抛出。"""
|
||||
try:
|
||||
persist_cloud_session_tokens(self._access_token, self._refresh_token)
|
||||
except Exception:
|
||||
logger.exception("CloudSession persist to disk failed")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# JWT 辅助
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _is_expired_or_near(self, threshold_seconds: int = 300) -> bool:
|
||||
exp = _decode_jwt_exp(self._access_token)
|
||||
if exp <= 0:
|
||||
return False
|
||||
return (exp - threshold_seconds) <= time.time()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CloudSyncConfig:
|
||||
base_url: str
|
||||
access_token: str
|
||||
refresh_token: str = ""
|
||||
chunk_size: int = 5 * 1024 * 1024
|
||||
timeout: float = 60.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncMetadata:
|
||||
resource_type: str = "artwork"
|
||||
source_platform: str = "pixiv"
|
||||
source_id: str | None = None
|
||||
content_type: str | None = None
|
||||
title: str = ""
|
||||
author: str | None = None
|
||||
author_id: str | None = None
|
||||
tags: list[str] = field(default_factory=list)
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
page_count: int = 1
|
||||
thumbnail_url: str | None = None
|
||||
description: str | None = None
|
||||
word_count: int = 0
|
||||
series_id: str | None = None
|
||||
series_title: str | None = None
|
||||
chapter_title: str | None = None
|
||||
chapter_order: int = 0
|
||||
|
||||
|
||||
class SyncFailureQueue:
|
||||
def __init__(self, path: str | Path | None = None) -> None:
|
||||
self.path = Path(path) if path else runtime_path("cloud_sync_failures.json")
|
||||
|
||||
def load(self) -> list[dict[str, Any]]:
|
||||
if not self.path.is_file():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(self.path.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, list) else []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def save(self, items: list[dict[str, Any]]) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.path.write_text(json.dumps(items, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
def add(self, file_path: str | Path, metadata: SyncMetadata, mime_type: str | None, error: str) -> None:
|
||||
now = time.time()
|
||||
file_path = str(Path(file_path))
|
||||
items = self.load()
|
||||
key = (file_path, metadata.source_platform, metadata.source_id)
|
||||
for item in items:
|
||||
item_key = (
|
||||
str(item.get("file_path") or ""),
|
||||
str(item.get("metadata", {}).get("source_platform") or ""),
|
||||
item.get("metadata", {}).get("source_id"),
|
||||
)
|
||||
if item_key == key:
|
||||
attempts = int(item.get("attempts") or 0)
|
||||
item.update(
|
||||
{
|
||||
"metadata": asdict(metadata),
|
||||
"mime_type": mime_type,
|
||||
"attempts": attempts,
|
||||
"last_error": error,
|
||||
"updated_at": now,
|
||||
"next_retry_at": now + min(3600, 60 * (2 ** min(attempts, 5))),
|
||||
}
|
||||
)
|
||||
self.save(items)
|
||||
return
|
||||
items.append(
|
||||
{
|
||||
"file_path": file_path,
|
||||
"metadata": asdict(metadata),
|
||||
"mime_type": mime_type,
|
||||
"attempts": 0,
|
||||
"last_error": error,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"next_retry_at": now + 60,
|
||||
}
|
||||
)
|
||||
self.save(items)
|
||||
|
||||
|
||||
class SakuraCloudClient:
|
||||
def __init__(
|
||||
self,
|
||||
config: CloudSyncConfig,
|
||||
*,
|
||||
on_token_refreshed: Callable[[str, str], None] | None = None,
|
||||
cloud_session: CloudSession | None = None,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.base_url = config.base_url.rstrip("/")
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({"Authorization": f"Bearer {config.access_token}"})
|
||||
self._cloud_session = cloud_session # 全局单例引用
|
||||
# 仅在没有 cloud_session 时使用独立锁和回调(兼容旧路径)
|
||||
if cloud_session is None:
|
||||
self._on_token_refreshed = on_token_refreshed or persist_cloud_session_tokens
|
||||
self._refresh_lock = threading.Lock()
|
||||
else:
|
||||
self._on_token_refreshed = None
|
||||
self._refresh_lock = None
|
||||
|
||||
def check_file(self, sha256: str, file_size: int) -> dict[str, Any]:
|
||||
return self._post("/api/v1/sync/check", {"sha256": sha256, "file_size": file_size})
|
||||
|
||||
def init_upload(
|
||||
self,
|
||||
*,
|
||||
sha256: str,
|
||||
file_name: str,
|
||||
file_size: int,
|
||||
mime_type: str | None,
|
||||
resource_type: str,
|
||||
) -> dict[str, Any]:
|
||||
return self._post(
|
||||
"/api/v1/upload/init",
|
||||
{
|
||||
"sha256": sha256,
|
||||
"file_name": file_name,
|
||||
"file_size": file_size,
|
||||
"mime_type": mime_type,
|
||||
"resource_type": resource_type,
|
||||
},
|
||||
)
|
||||
|
||||
def upload_chunk(self, upload_id: str, chunk: bytes) -> dict[str, Any]:
|
||||
response = self._request(
|
||||
"POST",
|
||||
"/api/v1/upload/chunk",
|
||||
data={"upload_id": upload_id},
|
||||
files={"chunk": ("chunk.bin", chunk)},
|
||||
)
|
||||
return self._unwrap(response)
|
||||
|
||||
def complete_upload(self, upload_id: str) -> dict[str, Any]:
|
||||
return self._post("/api/v1/upload/complete", {"upload_id": upload_id})
|
||||
|
||||
def sync_metadata(self, file_id: str, metadata: SyncMetadata) -> dict[str, Any]:
|
||||
payload = {
|
||||
"file_id": file_id,
|
||||
"resource_type": metadata.resource_type,
|
||||
"source_platform": metadata.source_platform,
|
||||
"source_id": metadata.source_id,
|
||||
"content_type": metadata.content_type,
|
||||
"title": metadata.title,
|
||||
"author": metadata.author,
|
||||
"author_id": metadata.author_id,
|
||||
"tags": metadata.tags,
|
||||
"metadata": metadata.metadata,
|
||||
"page_count": metadata.page_count,
|
||||
"thumbnail_url": metadata.thumbnail_url,
|
||||
"description": metadata.description,
|
||||
"word_count": metadata.word_count,
|
||||
"series_id": metadata.series_id,
|
||||
"series_title": metadata.series_title,
|
||||
"chapter_title": metadata.chapter_title,
|
||||
"chapter_order": metadata.chapter_order,
|
||||
}
|
||||
return self._post("/api/v1/sync/metadata", payload)
|
||||
|
||||
def upsert_follow(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return self._post("/api/v1/follows", payload)
|
||||
|
||||
def bulk_upsert_follows(self, items: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
return self._post("/api/v1/follows/bulk-upsert", {"items": items})
|
||||
|
||||
def create_alert(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return self._post("/api/v1/follows/alerts", payload)
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
json: dict[str, Any] | None = None,
|
||||
data: dict[str, Any] | None = None,
|
||||
files: dict[str, Any] | None = None,
|
||||
_retry_on_401: bool = True,
|
||||
) -> requests.Response:
|
||||
url = f"{self.base_url}{path}"
|
||||
response = self.session.request(
|
||||
method,
|
||||
url,
|
||||
json=json,
|
||||
data=data,
|
||||
files=files,
|
||||
timeout=self.config.timeout,
|
||||
)
|
||||
if response.status_code == 401 and _retry_on_401 and self._try_refresh_token():
|
||||
response = self.session.request(
|
||||
method,
|
||||
url,
|
||||
json=json,
|
||||
data=data,
|
||||
files=files,
|
||||
timeout=self.config.timeout,
|
||||
)
|
||||
return response
|
||||
|
||||
def _post(self, path: str, payload: dict[str, Any], *, _retry_on_401: bool = True) -> dict[str, Any]:
|
||||
response = self._request("POST", path, json=payload, _retry_on_401=_retry_on_401)
|
||||
return self._unwrap(response)
|
||||
|
||||
def _try_refresh_token(self) -> bool:
|
||||
# --- 新路径:委托给全局 CloudSession ---
|
||||
if self._cloud_session is not None:
|
||||
ok = self._cloud_session.try_refresh()
|
||||
if ok:
|
||||
# 刷新成功后同步更新本地 config 和 session headers
|
||||
new_config = self._cloud_session.get_config()
|
||||
self.config.access_token = new_config.access_token
|
||||
self.config.refresh_token = new_config.refresh_token
|
||||
self.session.headers.update({"Authorization": f"Bearer {new_config.access_token}"})
|
||||
return ok
|
||||
|
||||
# --- 旧路径(向后兼容,无 cloud_session 时使用) ---
|
||||
refresh = self.config.refresh_token.strip()
|
||||
if not refresh:
|
||||
return False
|
||||
with self._refresh_lock:
|
||||
refresh = self.config.refresh_token.strip()
|
||||
if not refresh:
|
||||
return False
|
||||
try:
|
||||
resp = requests.post(
|
||||
f"{self.base_url}/api/v1/auth/refresh",
|
||||
json={"refresh_token": refresh},
|
||||
timeout=self.config.timeout,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return False
|
||||
body = resp.json()
|
||||
data = body.get("data", body) if isinstance(body, dict) else {}
|
||||
if not isinstance(data, dict):
|
||||
return False
|
||||
new_access = (data.get("access_token") or "").strip()
|
||||
new_refresh = (data.get("refresh_token") or "").strip()
|
||||
if not new_access:
|
||||
return False
|
||||
self.config.access_token = new_access
|
||||
self.session.headers.update({"Authorization": f"Bearer {new_access}"})
|
||||
if new_refresh:
|
||||
self.config.refresh_token = new_refresh
|
||||
try:
|
||||
self._on_token_refreshed(self.config.access_token, self.config.refresh_token)
|
||||
except Exception:
|
||||
logger.exception("SakuraCloudClient._on_token_refreshed failed")
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("SakuraCloudClient._try_refresh_token error")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _unwrap(response: requests.Response) -> dict[str, Any]:
|
||||
response.raise_for_status()
|
||||
body = response.json()
|
||||
if isinstance(body, dict) and "data" in body:
|
||||
return body["data"]
|
||||
return body
|
||||
|
||||
|
||||
class SyncManager:
|
||||
def __init__(self, client: SakuraCloudClient, failure_queue: SyncFailureQueue | None = None) -> None:
|
||||
self.client = client
|
||||
self.failure_queue = failure_queue or SyncFailureQueue()
|
||||
|
||||
def sync_file(self, file_path: str | Path, metadata: SyncMetadata, *, mime_type: str | None = None) -> dict[str, Any]:
|
||||
try:
|
||||
return self._sync_file_once(file_path, metadata, mime_type=mime_type)
|
||||
except Exception as exc:
|
||||
self.failure_queue.add(file_path, metadata, mime_type, str(exc))
|
||||
raise
|
||||
|
||||
def _sync_file_once(self, file_path: str | Path, metadata: SyncMetadata, *, mime_type: str | None = None) -> dict[str, Any]:
|
||||
path = Path(file_path)
|
||||
sha256 = file_sha256(path)
|
||||
file_size = path.stat().st_size
|
||||
checked = self.client.check_file(sha256, file_size)
|
||||
file_id = checked.get("file_id") if checked.get("owned_by_user") else None
|
||||
|
||||
if not file_id:
|
||||
init = self.client.init_upload(
|
||||
sha256=sha256,
|
||||
file_name=path.name,
|
||||
file_size=file_size,
|
||||
mime_type=mime_type,
|
||||
resource_type=self._upload_resource_type(metadata, mime_type),
|
||||
)
|
||||
if init.get("exists") and init.get("file_id"):
|
||||
file_id = str(init["file_id"])
|
||||
else:
|
||||
upload_id = str(init["upload_id"])
|
||||
with path.open("rb") as fp:
|
||||
for chunk in iter(lambda: fp.read(self.client.config.chunk_size), b""):
|
||||
self.client.upload_chunk(upload_id, chunk)
|
||||
completed = self.client.complete_upload(upload_id)
|
||||
file_id = str(completed["file_id"])
|
||||
|
||||
return self.client.sync_metadata(str(file_id), metadata)
|
||||
|
||||
@staticmethod
|
||||
def _upload_resource_type(metadata: SyncMetadata, mime_type: str | None) -> str:
|
||||
if metadata.resource_type == "novel":
|
||||
return "novel"
|
||||
if (mime_type or "").lower().startswith("video/") or (metadata.content_type or "").lower() == "video":
|
||||
return "video"
|
||||
return "image"
|
||||
|
||||
def retry_failed(self, *, limit: int = 5) -> dict[str, int]:
|
||||
now = time.time()
|
||||
items = self.failure_queue.load()
|
||||
kept: list[dict[str, Any]] = []
|
||||
retried = 0
|
||||
succeeded = 0
|
||||
failed = 0
|
||||
for item in items:
|
||||
if retried >= limit or float(item.get("next_retry_at") or 0) > now:
|
||||
kept.append(item)
|
||||
continue
|
||||
path = Path(str(item.get("file_path") or ""))
|
||||
if not path.is_file():
|
||||
failed += 1
|
||||
continue
|
||||
try:
|
||||
metadata = SyncMetadata(**dict(item.get("metadata") or {}))
|
||||
self._sync_file_once(path, metadata, mime_type=item.get("mime_type"))
|
||||
succeeded += 1
|
||||
except Exception as exc:
|
||||
attempts = int(item.get("attempts") or 0) + 1
|
||||
item["attempts"] = attempts
|
||||
item["last_error"] = str(exc)
|
||||
item["updated_at"] = now
|
||||
item["next_retry_at"] = now + min(3600, 60 * (2 ** min(attempts, 5)))
|
||||
kept.append(item)
|
||||
failed += 1
|
||||
retried += 1
|
||||
self.failure_queue.save(kept)
|
||||
return {"retried": retried, "succeeded": succeeded, "failed": failed, "remaining": len(kept)}
|
||||
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
Cookie = dict[str, Any]
|
||||
|
||||
|
||||
def parse_cookie_text(text: str) -> list[Cookie]:
|
||||
"""Parse JSON cookie exports or Netscape cookies.txt content."""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
raise ValueError("Cookie 文件为空。")
|
||||
|
||||
try:
|
||||
payload = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
|
||||
if payload is not None:
|
||||
raw_items = payload.get("cookies", payload) if isinstance(payload, dict) else payload
|
||||
if not isinstance(raw_items, list):
|
||||
raise ValueError("Cookie JSON 必须是数组,或包含 cookies 数组。")
|
||||
cookies = [_normalize_json_cookie(item) for item in raw_items if isinstance(item, dict)]
|
||||
else:
|
||||
cookies = _parse_netscape_cookie_text(text)
|
||||
|
||||
cookies = [item for item in cookies if item.get("name") and item.get("value") is not None]
|
||||
if not cookies:
|
||||
raise ValueError("没有解析到有效 Cookie。")
|
||||
return cookies
|
||||
|
||||
|
||||
def filter_cookies_for_domains(cookies: list[Cookie], domains: tuple[str, ...]) -> list[Cookie]:
|
||||
result: list[Cookie] = []
|
||||
for cookie in cookies:
|
||||
domain = str(cookie.get("domain", "")).lower().lstrip(".")
|
||||
if any(domain == item or domain.endswith("." + item) for item in domains):
|
||||
result.append(cookie)
|
||||
return result
|
||||
|
||||
|
||||
def cookie_domains(cookies: list[Cookie]) -> list[str]:
|
||||
domains = sorted({str(item.get("domain", "")).lower().lstrip(".") for item in cookies if item.get("domain")})
|
||||
return [item for item in domains if item]
|
||||
|
||||
|
||||
def has_cookie_domain(cookies: list[Cookie], domains: tuple[str, ...]) -> bool:
|
||||
return bool(filter_cookies_for_domains(cookies, domains))
|
||||
|
||||
|
||||
def cookies_to_header(cookies: list[Cookie], domains: tuple[str, ...] | None = None) -> str:
|
||||
selected = filter_cookies_for_domains(cookies, domains) if domains else cookies
|
||||
pairs: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for cookie in selected:
|
||||
name = str(cookie.get("name", "")).strip()
|
||||
value = str(cookie.get("value", ""))
|
||||
if not name or name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
pairs.append(f"{name}={value}")
|
||||
if not pairs:
|
||||
raise ValueError("没有找到匹配站点的 Cookie。")
|
||||
return "; ".join(pairs)
|
||||
|
||||
|
||||
def cookies_to_playwright(cookies: list[Cookie], domains: tuple[str, ...] | None = None) -> list[Cookie]:
|
||||
selected = filter_cookies_for_domains(cookies, domains) if domains else cookies
|
||||
output_by_key: dict[tuple[str, str, str], Cookie] = {}
|
||||
for cookie in selected:
|
||||
name = str(cookie.get("name") or "").strip()
|
||||
domain = str(cookie.get("domain") or "").strip()
|
||||
if not name or not domain:
|
||||
continue
|
||||
item: Cookie = {
|
||||
"name": name,
|
||||
"value": str(cookie.get("value", "")),
|
||||
"domain": domain,
|
||||
"path": str(cookie.get("path") or "/"),
|
||||
"secure": bool(cookie.get("secure", False)),
|
||||
"httpOnly": bool(cookie.get("httpOnly", False)),
|
||||
}
|
||||
expires = cookie.get("expires")
|
||||
if expires is not None:
|
||||
try:
|
||||
expires_value = int(float(expires))
|
||||
if expires_value > 0:
|
||||
item["expires"] = expires_value
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
same_site = str(cookie.get("sameSite", "") or "").strip()
|
||||
if same_site in {"Strict", "Lax", "None"}:
|
||||
item["sameSite"] = same_site
|
||||
key = (str(item["domain"]).lower(), str(item["path"]), name)
|
||||
output_by_key[key] = item
|
||||
output = list(output_by_key.values())
|
||||
if not output:
|
||||
raise ValueError("没有找到匹配站点的 Cookie。")
|
||||
return output
|
||||
|
||||
|
||||
def save_playwright_cookies(cookies: list[Cookie], path: str | Path) -> Path:
|
||||
target = Path(path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json.dumps(cookies, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return target
|
||||
|
||||
|
||||
def cookie_summary(cookies: list[Cookie]) -> str:
|
||||
domains = sorted({str(item.get("domain", "")).lstrip(".") or "-" for item in cookies})
|
||||
shown = ", ".join(domains[:5])
|
||||
suffix = "..." if len(domains) > 5 else ""
|
||||
return f"{len(cookies)} 个 Cookie;域名:{shown}{suffix}"
|
||||
|
||||
|
||||
def _parse_netscape_cookie_text(text: str) -> list[Cookie]:
|
||||
cookies: list[Cookie] = []
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#HttpOnly_"):
|
||||
http_only = line.startswith("#HttpOnly_")
|
||||
line = line.removeprefix("#HttpOnly_")
|
||||
elif line.startswith("#"):
|
||||
continue
|
||||
else:
|
||||
http_only = False
|
||||
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 7:
|
||||
parts = line.split()
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
|
||||
domain, _include_subdomains, path, secure, expires, name = parts[:6]
|
||||
value = "\t".join(parts[6:])
|
||||
cookies.append(
|
||||
{
|
||||
"domain": domain,
|
||||
"path": path or "/",
|
||||
"secure": secure.upper() == "TRUE",
|
||||
"expires": int(expires) if str(expires).isdigit() else -1,
|
||||
"name": name,
|
||||
"value": value,
|
||||
"httpOnly": http_only,
|
||||
"sameSite": "Lax",
|
||||
}
|
||||
)
|
||||
return cookies
|
||||
|
||||
|
||||
def _normalize_json_cookie(item: Cookie) -> Cookie:
|
||||
expires = item.get("expires", item.get("expirationDate", item.get("expiry")))
|
||||
same_site = _normalize_same_site(item.get("sameSite"))
|
||||
return {
|
||||
"domain": str(item.get("domain") or item.get("host") or ""),
|
||||
"path": str(item.get("path") or "/"),
|
||||
"secure": bool(item.get("secure", False)),
|
||||
"expires": expires,
|
||||
"name": str(item.get("name") or ""),
|
||||
"value": str(item.get("value") or ""),
|
||||
"httpOnly": bool(item.get("httpOnly", item.get("httponly", False))),
|
||||
"sameSite": same_site,
|
||||
}
|
||||
|
||||
|
||||
def _normalize_same_site(value: Any) -> str:
|
||||
text = str(value or "").strip().lower()
|
||||
if text in {"strict", "samesitestrict"}:
|
||||
return "Strict"
|
||||
if text in {"none", "no_restriction", "samesitenone"}:
|
||||
return "None"
|
||||
return "Lax"
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
class DeviceType(Enum):
|
||||
WINDOWS_CHROME = "windows_chrome"
|
||||
WINDOWS_EDGE = "windows_edge"
|
||||
MAC_CHROME = "mac_chrome"
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrowserProfile:
|
||||
"""Stable browser context settings for one logical session.
|
||||
|
||||
This module intentionally avoids Canvas/WebGL spoofing or large fingerprint rotation.
|
||||
The goal is consistency inside a session, not impersonating many different users.
|
||||
"""
|
||||
|
||||
user_agent: str
|
||||
platform: str = "Win32"
|
||||
viewport: dict[str, int] = field(default_factory=lambda: {"width": 1920, "height": 1080})
|
||||
locale: str = "zh-CN"
|
||||
timezone_id: str = "Asia/Shanghai"
|
||||
color_scheme: str = "light"
|
||||
reduced_motion: str = "no-preference"
|
||||
device_scale_factor: float = 1.0
|
||||
is_mobile: bool = False
|
||||
has_touch: bool = False
|
||||
|
||||
def to_playwright_context_options(self) -> dict[str, Any]:
|
||||
return {
|
||||
"user_agent": self.user_agent,
|
||||
"viewport": dict(self.viewport),
|
||||
"locale": self.locale,
|
||||
"timezone_id": self.timezone_id,
|
||||
"color_scheme": self.color_scheme,
|
||||
"reduced_motion": self.reduced_motion,
|
||||
"device_scale_factor": self.device_scale_factor,
|
||||
"is_mobile": self.is_mobile,
|
||||
"has_touch": self.has_touch,
|
||||
}
|
||||
|
||||
def headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"User-Agent": self.user_agent,
|
||||
"Accept-Language": f"{self.locale},zh;q=0.9,en;q=0.8",
|
||||
}
|
||||
|
||||
|
||||
class BrowserProfileDatabase:
|
||||
@staticmethod
|
||||
def windows_chrome() -> BrowserProfile:
|
||||
return BrowserProfile(
|
||||
user_agent=real_browser_user_agent()
|
||||
or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
platform="Win32",
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def windows_edge() -> BrowserProfile:
|
||||
return BrowserProfile(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
platform="Win32",
|
||||
viewport={"width": 1536, "height": 864},
|
||||
device_scale_factor=1.25,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def mac_chrome() -> BrowserProfile:
|
||||
return BrowserProfile(
|
||||
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
platform="MacIntel",
|
||||
viewport={"width": 1512, "height": 982},
|
||||
device_scale_factor=2.0,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def all_desktop() -> list[BrowserProfile]:
|
||||
return [
|
||||
BrowserProfileDatabase.windows_chrome(),
|
||||
BrowserProfileDatabase.windows_edge(),
|
||||
BrowserProfileDatabase.mac_chrome(),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def by_device_type(device_type: DeviceType) -> BrowserProfile:
|
||||
mapping = {
|
||||
DeviceType.WINDOWS_CHROME: BrowserProfileDatabase.windows_chrome,
|
||||
DeviceType.WINDOWS_EDGE: BrowserProfileDatabase.windows_edge,
|
||||
DeviceType.MAC_CHROME: BrowserProfileDatabase.mac_chrome,
|
||||
}
|
||||
return mapping.get(device_type, BrowserProfileDatabase.windows_chrome)()
|
||||
|
||||
|
||||
class SessionFingerprintManager:
|
||||
"""Session-sticky browser profile manager."""
|
||||
|
||||
def __init__(self, profiles: list[BrowserProfile] | None = None, *, strategy: str = "sticky") -> None:
|
||||
self.profiles = profiles or BrowserProfileDatabase.all_desktop()
|
||||
self.strategy = strategy
|
||||
self._session_map: dict[str, BrowserProfile] = {}
|
||||
self._index = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def get_fingerprint(self, session_id: str = "default") -> BrowserProfile:
|
||||
with self._lock:
|
||||
if self.strategy == "sticky" and session_id in self._session_map:
|
||||
return copy.deepcopy(self._session_map[session_id])
|
||||
if self.strategy == "round_robin":
|
||||
profile = self.profiles[self._index % len(self.profiles)]
|
||||
self._index += 1
|
||||
elif self.strategy == "random":
|
||||
profile = random.choice(self.profiles)
|
||||
else:
|
||||
stable_index = abs(hash(session_id)) % len(self.profiles)
|
||||
profile = self.profiles[stable_index]
|
||||
self._session_map[session_id] = copy.deepcopy(profile)
|
||||
return copy.deepcopy(profile)
|
||||
|
||||
def get_stats(self) -> dict[str, Any]:
|
||||
return {
|
||||
"strategy": self.strategy,
|
||||
"total_profiles": len(self.profiles),
|
||||
"sticky_sessions": len(self._session_map),
|
||||
}
|
||||
|
||||
|
||||
Fingerprint = BrowserProfile
|
||||
FingerprintManager = SessionFingerprintManager
|
||||
FingerprintDatabase = BrowserProfileDatabase
|
||||
|
||||
|
||||
def real_browser_user_agent() -> str:
|
||||
"""Load a user-provided real browser UA, if configured."""
|
||||
env_value = os.environ.get("PIXIV_REAL_USER_AGENT", "").strip()
|
||||
if env_value:
|
||||
return env_value
|
||||
path_value = os.environ.get("PIXIV_REAL_USER_AGENT_FILE", "").strip()
|
||||
if path_value:
|
||||
path = Path(path_value)
|
||||
if path.is_file():
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
return ""
|
||||
|
||||
|
||||
def default_profile(session_id: str = "default") -> BrowserProfile:
|
||||
return SessionFingerprintManager().get_fingerprint(session_id)
|
||||
@@ -0,0 +1,597 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.downloader import download_user_works, fetch_user_profile, fetch_user_work_ids
|
||||
from pixiv_app.core.library_importer import LegacyDownloadImporter
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
|
||||
from pixiv_app.core.paths import app_session_file
|
||||
from pixiv_app.core.thumbnails import DEFAULT_THUMBNAIL_DIR, ThumbnailCache
|
||||
|
||||
|
||||
class LocalGalleryService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db_path: str | Path = DEFAULT_LIBRARY_DB,
|
||||
thumbnail_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
|
||||
) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
library = PixivLibrary(self.db_path)
|
||||
library.close()
|
||||
self.thumbnail_cache = ThumbnailCache(db_path=self.db_path, cache_dir=thumbnail_dir)
|
||||
|
||||
def list_artworks(
|
||||
self,
|
||||
*,
|
||||
query: str = "",
|
||||
search_scope: str = "all",
|
||||
tag: str = "",
|
||||
artist_id: int | None = None,
|
||||
artwork_type: str = "",
|
||||
include_restricted: bool = True,
|
||||
limit: int = 60,
|
||||
offset: int = 0,
|
||||
ensure_thumbnails: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
limit = max(1, min(int(limit), 200))
|
||||
offset = max(0, int(offset))
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
where: list[str] = []
|
||||
params: list[Any] = []
|
||||
joins = """
|
||||
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
|
||||
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
|
||||
LEFT JOIN thumbnail_cache tc ON tc.artwork_id = a.artwork_id AND tc.page_index = 0
|
||||
LEFT JOIN artwork_files af ON af.artwork_id = a.artwork_id AND af.page_index = 0
|
||||
LEFT JOIN artwork_ocr_texts ocr ON ocr.artwork_id = a.artwork_id
|
||||
"""
|
||||
if query:
|
||||
like = f"%{query}%"
|
||||
if search_scope == "ocr":
|
||||
where.append("(ocr.status = 'done' AND ocr.ocr_text LIKE ?)")
|
||||
params.append(like)
|
||||
elif search_scope == "title":
|
||||
where.append("(a.title LIKE ? OR a.caption LIKE ? OR ar.name LIKE ?)")
|
||||
params.extend([like, like, like])
|
||||
elif search_scope == "tags":
|
||||
joins += """
|
||||
LEFT JOIN artwork_tags at_search ON at_search.artwork_id = a.artwork_id
|
||||
LEFT JOIN tags t_search ON t_search.tag_id = at_search.tag_id
|
||||
"""
|
||||
where.append("t_search.name LIKE ?")
|
||||
params.append(like)
|
||||
else:
|
||||
joins += """
|
||||
LEFT JOIN artwork_tags at_search ON at_search.artwork_id = a.artwork_id
|
||||
LEFT JOIN tags t_search ON t_search.tag_id = at_search.tag_id
|
||||
"""
|
||||
where.append(
|
||||
"(a.title LIKE ? OR a.caption LIKE ? OR ar.name LIKE ? OR t_search.name LIKE ? "
|
||||
"OR (ocr.status = 'done' AND ocr.ocr_text LIKE ?))"
|
||||
)
|
||||
params.extend([like, like, like, like, like])
|
||||
if tag:
|
||||
joins += """
|
||||
JOIN artwork_tags at_filter ON at_filter.artwork_id = a.artwork_id
|
||||
JOIN tags t_filter ON t_filter.tag_id = at_filter.tag_id
|
||||
"""
|
||||
where.append("t_filter.name = ?")
|
||||
params.append(tag)
|
||||
if artist_id is not None:
|
||||
where.append("a.artist_id = ?")
|
||||
params.append(int(artist_id))
|
||||
if artwork_type:
|
||||
where.append("a.type = ?")
|
||||
params.append(artwork_type)
|
||||
else:
|
||||
where.append("a.type IN ('illust', 'manga')")
|
||||
if not include_restricted:
|
||||
where.append("a.x_restrict = 0")
|
||||
|
||||
where_sql = "WHERE " + " AND ".join(where) if where else ""
|
||||
total = library.connection.execute(
|
||||
f"SELECT COUNT(DISTINCT a.artwork_id) AS total FROM artworks a {joins} {where_sql}",
|
||||
params,
|
||||
).fetchone()["total"]
|
||||
rows = library.connection.execute(
|
||||
f"""
|
||||
SELECT
|
||||
a.artwork_id,
|
||||
a.type,
|
||||
a.title,
|
||||
a.artist_id,
|
||||
ar.name AS artist_name,
|
||||
ar.account AS artist_account,
|
||||
a.page_count,
|
||||
a.bookmark_count,
|
||||
a.like_count,
|
||||
a.view_count,
|
||||
a.x_restrict,
|
||||
a.ai_type,
|
||||
a.thumbnail_url,
|
||||
CASE WHEN fav.artwork_id IS NULL THEN 0 ELSE 1 END AS is_favorite,
|
||||
CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed,
|
||||
COALESCE(tc.thumbnail_path, '') AS thumbnail_path,
|
||||
COALESCE(af.file_path, '') AS primary_file_path,
|
||||
CASE
|
||||
WHEN ocr.status = 'done' AND ocr.ocr_text LIKE ? THEN 'ocr'
|
||||
ELSE ''
|
||||
END AS matched_by,
|
||||
a.updated_at
|
||||
FROM artworks a
|
||||
{joins}
|
||||
LEFT JOIN favorites fav ON fav.artwork_id = a.artwork_id
|
||||
{where_sql}
|
||||
GROUP BY a.artwork_id
|
||||
ORDER BY a.updated_at DESC, a.artwork_id DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
[f"%{query}%" if query else "", *params, limit, offset],
|
||||
).fetchall()
|
||||
|
||||
items = [dict(row) for row in rows]
|
||||
if ensure_thumbnails:
|
||||
self._ensure_list_thumbnails(items)
|
||||
items = self._refresh_thumbnail_paths(library, items)
|
||||
return {"total": int(total), "limit": limit, "offset": offset, "items": items}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def get_artwork(self, artwork_id: int, *, ensure_thumbnails: bool = True) -> dict[str, Any] | None:
|
||||
if ensure_thumbnails:
|
||||
self.thumbnail_cache.ensure_for_artwork(artwork_id)
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT a.*, ar.name AS artist_name, ar.account AS artist_account
|
||||
, CASE WHEN fav.artwork_id IS NULL THEN 0 ELSE 1 END AS is_favorite
|
||||
, CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed
|
||||
FROM artworks a
|
||||
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
|
||||
LEFT JOIN favorites fav ON fav.artwork_id = a.artwork_id
|
||||
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
|
||||
WHERE a.artwork_id = ?
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
files = library.connection.execute(
|
||||
"""
|
||||
SELECT f.*, tc.thumbnail_path, tc.width AS thumbnail_width, tc.height AS thumbnail_height
|
||||
FROM artwork_files f
|
||||
LEFT JOIN thumbnail_cache tc
|
||||
ON tc.artwork_id = f.artwork_id AND tc.page_index = f.page_index
|
||||
WHERE f.artwork_id = ?
|
||||
ORDER BY f.page_index ASC
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchall()
|
||||
tags = library.connection.execute(
|
||||
"""
|
||||
SELECT t.name, t.translated_name
|
||||
FROM tags t
|
||||
JOIN artwork_tags at ON at.tag_id = t.tag_id
|
||||
WHERE at.artwork_id = ?
|
||||
ORDER BY t.name ASC
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchall()
|
||||
history = library.connection.execute(
|
||||
"""
|
||||
SELECT status, downloaded_pages, skipped_pages, failed_pages, message, finished_at
|
||||
FROM download_history
|
||||
WHERE artwork_id = ?
|
||||
ORDER BY finished_at DESC
|
||||
LIMIT 20
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchall()
|
||||
return {
|
||||
"artwork": dict(row),
|
||||
"files": [dict(item) for item in files],
|
||||
"tags": [dict(item) for item in tags],
|
||||
"history": [dict(item) for item in history],
|
||||
}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def list_tags(self, *, query: str = "", limit: int = 100) -> list[dict[str, Any]]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
params: list[Any] = []
|
||||
where = ""
|
||||
if query:
|
||||
where = "WHERE t.name LIKE ? OR t.translated_name LIKE ?"
|
||||
params.extend([f"%{query}%", f"%{query}%"])
|
||||
rows = library.connection.execute(
|
||||
f"""
|
||||
SELECT t.name, t.translated_name, COUNT(at.artwork_id) AS artwork_count
|
||||
FROM tags t
|
||||
LEFT JOIN artwork_tags at ON at.tag_id = t.tag_id
|
||||
{where}
|
||||
GROUP BY t.tag_id
|
||||
ORDER BY artwork_count DESC, t.name ASC
|
||||
LIMIT ?
|
||||
""",
|
||||
[*params, max(1, min(int(limit), 500))],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def list_artists(self, *, query: str = "", limit: int = 100) -> list[dict[str, Any]]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
params: list[Any] = []
|
||||
where = ""
|
||||
if query:
|
||||
where = "WHERE ar.name LIKE ? OR ar.account LIKE ?"
|
||||
params.extend([f"%{query}%", f"%{query}%"])
|
||||
rows = library.connection.execute(
|
||||
f"""
|
||||
SELECT ar.artist_id, ar.name, ar.account, COUNT(a.artwork_id) AS artwork_count
|
||||
FROM artists ar
|
||||
LEFT JOIN artworks a ON a.artist_id = ar.artist_id
|
||||
{where}
|
||||
GROUP BY ar.artist_id
|
||||
ORDER BY
|
||||
CASE WHEN COALESCE(NULLIF(ar.name, ''), NULLIF(ar.account, '')) = '' THEN 1 ELSE 0 END,
|
||||
COALESCE(NULLIF(ar.name, ''), NULLIF(ar.account, ''), CAST(ar.artist_id AS TEXT)) ASC,
|
||||
artwork_count DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
[*params, max(1, min(int(limit), 500))],
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def get_artist(self, artist_id: int, *, limit: int = 60, offset: int = 0) -> dict[str, Any] | None:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT ar.*, COUNT(a.artwork_id) AS artwork_count,
|
||||
CASE WHEN fa.enabled = 1 THEN 1 ELSE 0 END AS is_followed,
|
||||
fa.last_synced_at, fa.last_error
|
||||
FROM artists ar
|
||||
LEFT JOIN artworks a ON a.artist_id = ar.artist_id
|
||||
LEFT JOIN followed_artists fa ON fa.artist_id = ar.artist_id
|
||||
WHERE ar.artist_id = ?
|
||||
GROUP BY ar.artist_id
|
||||
""",
|
||||
(artist_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {
|
||||
"artist": dict(row),
|
||||
"artworks": self.list_artworks(artist_id=artist_id, limit=limit, offset=offset),
|
||||
}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def list_favorites(self, *, limit: int = 60, offset: int = 0) -> dict[str, Any]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
total = library.connection.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS total
|
||||
FROM favorites fav
|
||||
JOIN artworks a ON a.artwork_id = fav.artwork_id
|
||||
WHERE a.type IN ('illust', 'manga')
|
||||
"""
|
||||
).fetchone()["total"]
|
||||
rows = library.connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
a.artwork_id, a.type, a.title, a.artist_id, ar.name AS artist_name,
|
||||
ar.account AS artist_account,
|
||||
a.page_count, a.bookmark_count, a.like_count, a.view_count,
|
||||
a.x_restrict, a.ai_type, a.thumbnail_url,
|
||||
1 AS is_favorite,
|
||||
CASE WHEN fa.artist_id IS NULL THEN 0 ELSE 1 END AS is_followed,
|
||||
COALESCE(tc.thumbnail_path, '') AS thumbnail_path,
|
||||
COALESCE(af.file_path, '') AS primary_file_path,
|
||||
fav.created_at AS favorite_at
|
||||
FROM favorites fav
|
||||
JOIN artworks a ON a.artwork_id = fav.artwork_id
|
||||
LEFT JOIN artists ar ON ar.artist_id = a.artist_id
|
||||
LEFT JOIN followed_artists fa ON fa.artist_id = a.artist_id AND fa.enabled = 1
|
||||
LEFT JOIN thumbnail_cache tc ON tc.artwork_id = a.artwork_id AND tc.page_index = 0
|
||||
LEFT JOIN artwork_files af ON af.artwork_id = a.artwork_id AND af.page_index = 0
|
||||
WHERE a.type IN ('illust', 'manga')
|
||||
ORDER BY fav.created_at DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
(max(1, min(int(limit), 200)), max(0, int(offset))),
|
||||
).fetchall()
|
||||
items = [dict(row) for row in rows]
|
||||
self._ensure_list_thumbnails(items)
|
||||
items = self._refresh_thumbnail_paths(library, items)
|
||||
return {"total": int(total), "limit": limit, "offset": offset, "items": items}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def set_favorite(self, artwork_id: int, favorite: bool, note: str = "") -> dict[str, Any]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
library.set_favorite(artwork_id, favorite, note)
|
||||
return {"artwork_id": artwork_id, "favorite": favorite}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def list_followed_artists(self) -> list[dict[str, Any]]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
rows = library.connection.execute(
|
||||
"""
|
||||
SELECT fa.artist_id, COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), '') AS name,
|
||||
COALESCE(NULLIF(ar.account, ''), '') AS account,
|
||||
fa.enabled, fa.last_synced_at, fa.last_error,
|
||||
COUNT(a.artwork_id) AS artwork_count
|
||||
FROM followed_artists fa
|
||||
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
|
||||
LEFT JOIN artworks a ON a.artist_id = fa.artist_id
|
||||
WHERE fa.enabled = 1
|
||||
GROUP BY fa.artist_id
|
||||
ORDER BY
|
||||
CASE WHEN COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, '')) = '' THEN 1 ELSE 0 END,
|
||||
COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, ''), CAST(fa.artist_id AS TEXT)) ASC,
|
||||
fa.artist_id ASC
|
||||
"""
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def set_followed_artist(self, artist_ref: int | str, followed: bool, name: str = "") -> dict[str, Any]:
|
||||
artist_id = self._resolve_artist_id(artist_ref)
|
||||
clean_name = self._clean_name(name)
|
||||
account = ""
|
||||
if followed and not clean_name:
|
||||
try:
|
||||
profile = fetch_user_profile(
|
||||
artist_id,
|
||||
cookie=str(SessionStore(app_session_file()).load().get("cookie", "")).strip(),
|
||||
)
|
||||
profile_info = profile.get("profile") if isinstance(profile.get("profile"), dict) else {}
|
||||
clean_name = self._clean_name(
|
||||
profile.get("name")
|
||||
or profile.get("userName")
|
||||
or profile.get("displayName")
|
||||
or profile_info.get("name")
|
||||
)
|
||||
account = self._clean_name(profile.get("account") or profile.get("userAccount"))
|
||||
except Exception:
|
||||
clean_name = ""
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
if clean_name or account:
|
||||
library.upsert_artist_profile(artist_id, name=clean_name, account=account)
|
||||
library.set_followed_artist(artist_id, followed, clean_name)
|
||||
return {
|
||||
"artist_id": artist_id,
|
||||
"followed": followed,
|
||||
"name": clean_name,
|
||||
"account": account,
|
||||
"cloud_synced": False,
|
||||
}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def _resolve_artist_id(self, artist_ref: int | str) -> int:
|
||||
text = str(artist_ref).strip()
|
||||
if not text:
|
||||
raise ValueError("Please enter a Pixiv artist ID, profile URL, or synced artist name.")
|
||||
|
||||
if text.lower().startswith("artist_") and text[7:].isdigit():
|
||||
return int(text[7:])
|
||||
|
||||
match = re.search(r"/users/(\d+)", text)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
if text.isdigit():
|
||||
return int(text)
|
||||
|
||||
lookup = text.lstrip("@")
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
rows = [
|
||||
dict(row)
|
||||
for row in library.connection.execute(
|
||||
"""
|
||||
SELECT artist_id, name, account
|
||||
FROM artists
|
||||
WHERE lower(name) = lower(?) OR lower(account) = lower(?)
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN lower(account) = lower(?) THEN 0
|
||||
WHEN lower(name) = lower(?) THEN 1
|
||||
ELSE 2
|
||||
END,
|
||||
artist_id ASC
|
||||
""",
|
||||
(lookup, lookup, lookup, lookup),
|
||||
).fetchall()
|
||||
]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
if len(rows) == 1:
|
||||
return int(rows[0]["artist_id"])
|
||||
if not rows:
|
||||
raise ValueError("Could not resolve Pixiv artist ID from that name. Enter an ID/profile URL or sync the artist first.")
|
||||
choices = ", ".join(str(int(row["artist_id"])) for row in rows[:5])
|
||||
raise ValueError(f"Artist name matched multiple Pixiv users; use an ID instead: {choices}")
|
||||
|
||||
@staticmethod
|
||||
def _clean_name(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
return "" if text.lower() in {"none", "null", "undefined"} else text
|
||||
|
||||
def scan_legacy_downloads(self, *, root: str | Path = "Sakura_Downloads") -> dict[str, Any]:
|
||||
summary = LegacyDownloadImporter(db_path=self.db_path).scan(root)
|
||||
self.thumbnail_cache.build_missing(limit=500)
|
||||
return summary.__dict__
|
||||
|
||||
def sync_followed_artists(
|
||||
self,
|
||||
*,
|
||||
cookie: str = "",
|
||||
max_new_per_artist: int = 20,
|
||||
download: bool = True,
|
||||
save_root: str | Path = "Sakura_Downloads",
|
||||
) -> dict[str, Any]:
|
||||
if not cookie:
|
||||
cookie = str(SessionStore(app_session_file()).load().get("cookie", "")).strip()
|
||||
if not cookie:
|
||||
raise ValueError("No Pixiv cookie available. Save a session in the GUI first or pass cookie explicitly.")
|
||||
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
followed = library.connection.execute(
|
||||
"""
|
||||
SELECT fa.artist_id,
|
||||
COALESCE(NULLIF(fa.name, ''), NULLIF(ar.name, ''), NULLIF(ar.account, ''), '') AS name,
|
||||
COALESCE(NULLIF(ar.account, ''), '') AS account
|
||||
FROM followed_artists fa
|
||||
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
|
||||
WHERE fa.enabled = 1
|
||||
ORDER BY fa.artist_id
|
||||
"""
|
||||
).fetchall()
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for row in followed:
|
||||
artist_id = int(row["artist_id"])
|
||||
artist_name = self._clean_name(row["name"])
|
||||
artist_account = self._clean_name(row["account"])
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
existing = {
|
||||
int(item["artwork_id"])
|
||||
for item in library.connection.execute(
|
||||
"SELECT artwork_id FROM artworks WHERE artist_id = ? AND type IN ('illust', 'manga')",
|
||||
(artist_id,),
|
||||
).fetchall()
|
||||
}
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
try:
|
||||
_, work_ids = fetch_user_work_ids(user_id=artist_id, cookie=cookie)
|
||||
new_ids = [work_id for work_id in work_ids if work_id not in existing][: max(0, int(max_new_per_artist))]
|
||||
downloaded = 0
|
||||
if download and new_ids:
|
||||
_, download_results = download_user_works(
|
||||
user_id=artist_id,
|
||||
cookie=cookie,
|
||||
max_workers=3,
|
||||
request_delay=0.2,
|
||||
save_root=save_root,
|
||||
work_ids=new_ids,
|
||||
)
|
||||
downloaded = len([item for item in download_results if item.ok])
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
library.mark_artist_synced(artist_id)
|
||||
finally:
|
||||
library.close()
|
||||
cloud_synced = False
|
||||
cloud_alerts = 0
|
||||
results.append(
|
||||
{
|
||||
"artist_id": artist_id,
|
||||
"remote_count": len(work_ids),
|
||||
"new_count": len(new_ids),
|
||||
"downloaded": downloaded,
|
||||
"cloud_synced": cloud_synced,
|
||||
"alerts": cloud_alerts,
|
||||
"error": "",
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
library.mark_artist_synced(artist_id, error=str(exc))
|
||||
finally:
|
||||
library.close()
|
||||
results.append({"artist_id": artist_id, "remote_count": 0, "new_count": 0, "downloaded": 0, "alerts": 0, "error": str(exc)})
|
||||
return {"artists": results}
|
||||
|
||||
def build_missing_thumbnails(self, *, limit: int = 200) -> list[dict[str, Any]]:
|
||||
return [item.__dict__ for item in self.thumbnail_cache.build_missing(limit=limit)]
|
||||
|
||||
def get_thumbnail_path(self, artwork_id: int, *, page_index: int = 0) -> Path | None:
|
||||
self.thumbnail_cache.ensure_for_artwork(artwork_id)
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT thumbnail_path
|
||||
FROM thumbnail_cache
|
||||
WHERE artwork_id = ? AND page_index = ?
|
||||
""",
|
||||
(artwork_id, page_index),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
path = Path(str(row["thumbnail_path"]))
|
||||
return path if path.exists() else None
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def get_source_file_path(self, artwork_id: int, *, page_index: int = 0) -> Path | None:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT file_path
|
||||
FROM artwork_files
|
||||
WHERE artwork_id = ? AND page_index = ?
|
||||
""",
|
||||
(artwork_id, page_index),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
path = Path(str(row["file_path"]))
|
||||
return path if path.exists() else None
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def _ensure_list_thumbnails(self, items: list[dict[str, Any]]) -> None:
|
||||
for item in items:
|
||||
if item.get("thumbnail_path"):
|
||||
continue
|
||||
if not item.get("primary_file_path"):
|
||||
continue
|
||||
self.thumbnail_cache.ensure_for_artwork(int(item["artwork_id"]))
|
||||
|
||||
def _refresh_thumbnail_paths(self, library: PixivLibrary, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
artwork_ids = [int(item["artwork_id"]) for item in items]
|
||||
if not artwork_ids:
|
||||
return items
|
||||
placeholders = ",".join("?" for _ in artwork_ids)
|
||||
rows = library.connection.execute(
|
||||
f"""
|
||||
SELECT artwork_id, thumbnail_path
|
||||
FROM thumbnail_cache
|
||||
WHERE page_index = 0 AND artwork_id IN ({placeholders})
|
||||
""",
|
||||
artwork_ids,
|
||||
).fetchall()
|
||||
thumbs = {int(row["artwork_id"]): str(row["thumbnail_path"]) for row in rows}
|
||||
for item in items:
|
||||
item["thumbnail_path"] = thumbs.get(int(item["artwork_id"]), item.get("thumbnail_path", ""))
|
||||
return items
|
||||
@@ -0,0 +1,703 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pixiv_app.core.paths import app_library_file
|
||||
|
||||
DEFAULT_LIBRARY_DB = app_library_file()
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadFileRecord:
|
||||
page_index: int
|
||||
file_path: str
|
||||
source_url: str
|
||||
status: str
|
||||
size_bytes: int = 0
|
||||
content_hash: str = ""
|
||||
|
||||
|
||||
class PixivLibrary:
|
||||
def __init__(self, db_path: str | Path = DEFAULT_LIBRARY_DB) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True) if self.db_path.parent != Path(".") else None
|
||||
# Allow sharing one connection across threads when guarded by a lock (e.g. TaskQueue).
|
||||
self.connection = sqlite3.connect(self.db_path, timeout=30, check_same_thread=False)
|
||||
self.connection.row_factory = sqlite3.Row
|
||||
self.connection.execute("PRAGMA journal_mode=WAL")
|
||||
self.connection.execute("PRAGMA foreign_keys=ON")
|
||||
self.setup()
|
||||
|
||||
def close(self) -> None:
|
||||
self.connection.close()
|
||||
|
||||
def setup(self) -> None:
|
||||
with self.connection:
|
||||
self.connection.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS artists (
|
||||
artist_id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
account TEXT,
|
||||
avatar_url TEXT,
|
||||
metadata_json TEXT NOT NULL DEFAULT '{}',
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artworks (
|
||||
artwork_id INTEGER PRIMARY KEY,
|
||||
artist_id INTEGER,
|
||||
type TEXT NOT NULL DEFAULT 'illust',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
caption TEXT NOT NULL DEFAULT '',
|
||||
page_count INTEGER NOT NULL DEFAULT 0,
|
||||
bookmark_count INTEGER NOT NULL DEFAULT 0,
|
||||
like_count INTEGER NOT NULL DEFAULT 0,
|
||||
view_count INTEGER NOT NULL DEFAULT 0,
|
||||
x_restrict INTEGER NOT NULL DEFAULT 0,
|
||||
ai_type INTEGER NOT NULL DEFAULT 0,
|
||||
thumbnail_url TEXT,
|
||||
metadata_json TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (artist_id) REFERENCES artists(artist_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
tag_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
translated_name TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artwork_tags (
|
||||
artwork_id INTEGER NOT NULL,
|
||||
tag_id INTEGER NOT NULL,
|
||||
PRIMARY KEY (artwork_id, tag_id),
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (tag_id) REFERENCES tags(tag_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artwork_files (
|
||||
artwork_id INTEGER NOT NULL,
|
||||
page_index INTEGER NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
source_url TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'unknown',
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (artwork_id, page_index),
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
artwork_id INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
downloaded_pages INTEGER NOT NULL DEFAULT 0,
|
||||
skipped_pages INTEGER NOT NULL DEFAULT 0,
|
||||
failed_pages INTEGER NOT NULL DEFAULT 0,
|
||||
message TEXT NOT NULL DEFAULT '',
|
||||
started_at TEXT,
|
||||
finished_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value_json TEXT NOT NULL DEFAULT '{}',
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS thumbnail_cache (
|
||||
artwork_id INTEGER NOT NULL,
|
||||
page_index INTEGER NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
thumbnail_path TEXT NOT NULL,
|
||||
width INTEGER NOT NULL DEFAULT 0,
|
||||
height INTEGER NOT NULL DEFAULT 0,
|
||||
source_mtime REAL NOT NULL DEFAULT 0,
|
||||
source_size INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (artwork_id, page_index),
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artwork_ocr_texts (
|
||||
artwork_id INTEGER PRIMARY KEY,
|
||||
language TEXT NOT NULL DEFAULT 'mixed',
|
||||
ocr_text TEXT NOT NULL DEFAULT '',
|
||||
confidence REAL,
|
||||
engine TEXT NOT NULL DEFAULT 'paddleocr',
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS favorites (
|
||||
artwork_id INTEGER PRIMARY KEY,
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (artwork_id) REFERENCES artworks(artwork_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS followed_artists (
|
||||
artist_id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
last_synced_at TEXT,
|
||||
last_error TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artworks_artist_id ON artworks(artist_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_artworks_type ON artworks(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_download_history_artwork_id ON download_history(artwork_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_artwork_files_path ON artwork_files(file_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_thumbnail_cache_path ON thumbnail_cache(thumbnail_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_artwork_ocr_texts_status ON artwork_ocr_texts(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_artwork_ocr_texts_text ON artwork_ocr_texts(ocr_text);
|
||||
CREATE INDEX IF NOT EXISTS idx_favorites_created_at ON favorites(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_followed_artists_enabled ON followed_artists(enabled);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_tasks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
fingerprint TEXT NOT NULL UNIQUE,
|
||||
target_type TEXT NOT NULL,
|
||||
target_id INTEGER,
|
||||
page_index INTEGER,
|
||||
task_kind TEXT NOT NULL,
|
||||
priority INTEGER NOT NULL DEFAULT 2,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
payload_json TEXT NOT NULL DEFAULT '{}',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
leased_until REAL,
|
||||
worker_id TEXT,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_download_tasks_status_pri_id
|
||||
ON download_tasks(status, priority DESC, id ASC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artist_sync_watermark (
|
||||
artist_id INTEGER PRIMARY KEY,
|
||||
last_synced_illust_id INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
"""
|
||||
)
|
||||
self._apply_schema_migrations()
|
||||
|
||||
def _apply_schema_migrations(self) -> None:
|
||||
"""Incremental ALTER for existing databases created before new columns/tables."""
|
||||
with self.connection:
|
||||
cols = {row[1] for row in self.connection.execute("PRAGMA table_info(artwork_files)")}
|
||||
if "content_hash" not in cols:
|
||||
self.connection.execute(
|
||||
"ALTER TABLE artwork_files ADD COLUMN content_hash TEXT NOT NULL DEFAULT ''"
|
||||
)
|
||||
# Drop legacy placeholder names like artist_12345 so UI can fall back cleanly.
|
||||
self.connection.execute(
|
||||
"UPDATE artists SET name = '' WHERE name = 'artist_' || artist_id"
|
||||
)
|
||||
self.connection.execute(
|
||||
"UPDATE followed_artists SET name = '' WHERE name = 'artist_' || artist_id"
|
||||
)
|
||||
self.connection.execute(
|
||||
"UPDATE artists SET name = '' WHERE lower(name) IN ('none', 'null', 'undefined')"
|
||||
)
|
||||
self.connection.execute(
|
||||
"UPDATE followed_artists SET name = '' WHERE lower(name) IN ('none', 'null', 'undefined')"
|
||||
)
|
||||
rows = self.connection.execute(
|
||||
"SELECT artwork_id, metadata_json FROM artworks WHERE artist_id IS NULL OR artist_id = 0"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
try:
|
||||
metadata = json.loads(str(row["metadata_json"] or "{}"))
|
||||
except Exception:
|
||||
continue
|
||||
artist_id = _as_int(metadata.get("userId") or metadata.get("author_id") or metadata.get("artistId"))
|
||||
if artist_id is None:
|
||||
continue
|
||||
self.ensure_artist_stub(artist_id, name=str(metadata.get("userName") or metadata.get("artistName") or ""))
|
||||
self.connection.execute(
|
||||
"UPDATE artworks SET artist_id = ?, updated_at = CURRENT_TIMESTAMP WHERE artwork_id = ?",
|
||||
(artist_id, int(row["artwork_id"])),
|
||||
)
|
||||
|
||||
def upsert_artwork(self, detail: dict[str, Any], *, artwork_type: str = "illust") -> None:
|
||||
source_id = _as_int(detail.get("illustId") or detail.get("id") or detail.get("novelId"))
|
||||
if source_id is None:
|
||||
return
|
||||
source_platform = str(detail.get("source_platform") or detail.get("source") or "")
|
||||
artwork_id = library_artwork_id(source_id, artwork_type, source_platform=source_platform)
|
||||
|
||||
artist_id = _as_int(detail.get("userId"))
|
||||
if artist_id is not None:
|
||||
self._upsert_artist(detail, artist_id)
|
||||
|
||||
tags = _extract_tags(detail)
|
||||
urls = detail.get("urls") if isinstance(detail.get("urls"), dict) else {}
|
||||
thumbnail_url = (
|
||||
detail.get("url")
|
||||
or detail.get("coverUrl")
|
||||
or urls.get("small")
|
||||
or urls.get("thumb")
|
||||
or urls.get("regular")
|
||||
or urls.get("original")
|
||||
)
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artworks (
|
||||
artwork_id, artist_id, type, title, caption, page_count,
|
||||
bookmark_count, like_count, view_count, x_restrict, ai_type,
|
||||
thumbnail_url, metadata_json, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id) DO UPDATE SET
|
||||
artist_id = excluded.artist_id,
|
||||
type = excluded.type,
|
||||
title = excluded.title,
|
||||
caption = excluded.caption,
|
||||
page_count = excluded.page_count,
|
||||
bookmark_count = excluded.bookmark_count,
|
||||
like_count = excluded.like_count,
|
||||
view_count = excluded.view_count,
|
||||
x_restrict = excluded.x_restrict,
|
||||
ai_type = excluded.ai_type,
|
||||
thumbnail_url = excluded.thumbnail_url,
|
||||
metadata_json = excluded.metadata_json,
|
||||
created_at = COALESCE(excluded.created_at, artworks.created_at),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
artwork_id,
|
||||
artist_id,
|
||||
artwork_type,
|
||||
str(detail.get("title") or ""),
|
||||
str(detail.get("description") or detail.get("caption") or ""),
|
||||
_as_int(detail.get("pageCount")) or 0,
|
||||
_as_int(detail.get("bookmarkCount")) or 0,
|
||||
_as_int(detail.get("likeCount")) or 0,
|
||||
_as_int(detail.get("viewCount")) or 0,
|
||||
_as_int(detail.get("xRestrict")) or 0,
|
||||
_as_int(detail.get("aiType")) or 0,
|
||||
str(thumbnail_url) if thumbnail_url else None,
|
||||
json.dumps({**detail, "sourceId": source_id}, ensure_ascii=False),
|
||||
str(detail.get("createDate") or detail.get("uploadDate") or "") or None,
|
||||
),
|
||||
)
|
||||
self.connection.execute("DELETE FROM artwork_tags WHERE artwork_id = ?", (artwork_id,))
|
||||
for name, translated_name in tags:
|
||||
tag_id = self._upsert_tag(name, translated_name)
|
||||
self.connection.execute(
|
||||
"INSERT OR IGNORE INTO artwork_tags (artwork_id, tag_id) VALUES (?, ?)",
|
||||
(artwork_id, tag_id),
|
||||
)
|
||||
|
||||
def ensure_artwork_stub(self, artwork_id: int, *, artwork_type: str = "illust", title: str = "", source_platform: str = "") -> int:
|
||||
db_artwork_id = library_artwork_id(artwork_id, artwork_type, source_platform=source_platform)
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO artworks (artwork_id, type, title, updated_at)
|
||||
VALUES (?, ?, ?, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
(db_artwork_id, artwork_type, title),
|
||||
)
|
||||
return db_artwork_id
|
||||
|
||||
def ensure_artist_stub(self, artist_id: int, *, name: str = "") -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO artists (artist_id, name, updated_at)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
""",
|
||||
(artist_id, name or ""),
|
||||
)
|
||||
|
||||
def upsert_artist_profile(self, artist_id: int, *, name: str = "", account: str = "", avatar_url: str = "") -> None:
|
||||
clean_name = _clean_optional_text(name)
|
||||
clean_account = _clean_optional_text(account)
|
||||
clean_avatar = _clean_optional_text(avatar_url)
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artists (artist_id, name, account, avatar_url, metadata_json, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
name = COALESCE(NULLIF(excluded.name, ''), artists.name),
|
||||
account = COALESCE(NULLIF(excluded.account, ''), artists.account),
|
||||
avatar_url = COALESCE(NULLIF(excluded.avatar_url, ''), artists.avatar_url),
|
||||
metadata_json = CASE
|
||||
WHEN excluded.metadata_json != '{}' THEN excluded.metadata_json
|
||||
ELSE artists.metadata_json
|
||||
END,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
int(artist_id),
|
||||
clean_name,
|
||||
clean_account,
|
||||
clean_avatar,
|
||||
json.dumps(
|
||||
{
|
||||
"userId": int(artist_id),
|
||||
"userName": clean_name,
|
||||
"userAccount": clean_account,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def set_artwork_artist(self, artwork_id: int, artist_id: int) -> None:
|
||||
with self.connection:
|
||||
self.ensure_artist_stub(artist_id)
|
||||
self.connection.execute(
|
||||
"""
|
||||
UPDATE artworks
|
||||
SET artist_id = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE artwork_id = ?
|
||||
""",
|
||||
(artist_id, artwork_id),
|
||||
)
|
||||
|
||||
def record_files(self, artwork_id: int, files: list[DownloadFileRecord]) -> None:
|
||||
with self.connection:
|
||||
for item in files:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artwork_files (
|
||||
artwork_id, page_index, file_path, source_url, status, size_bytes,
|
||||
content_hash, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id, page_index) DO UPDATE SET
|
||||
file_path = excluded.file_path,
|
||||
source_url = excluded.source_url,
|
||||
status = excluded.status,
|
||||
size_bytes = excluded.size_bytes,
|
||||
content_hash = CASE
|
||||
WHEN excluded.content_hash != '' THEN excluded.content_hash
|
||||
ELSE artwork_files.content_hash
|
||||
END,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
artwork_id,
|
||||
item.page_index,
|
||||
item.file_path,
|
||||
item.source_url,
|
||||
item.status,
|
||||
item.size_bytes,
|
||||
item.content_hash or "",
|
||||
),
|
||||
)
|
||||
|
||||
def record_download_history(
|
||||
self,
|
||||
*,
|
||||
artwork_id: int,
|
||||
status: str,
|
||||
downloaded_pages: int,
|
||||
skipped_pages: int,
|
||||
failed_pages: int,
|
||||
message: str,
|
||||
started_at: str | None = None,
|
||||
) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO download_history (
|
||||
artwork_id, status, downloaded_pages, skipped_pages, failed_pages, message, started_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(artwork_id, status, downloaded_pages, skipped_pages, failed_pages, message, started_at),
|
||||
)
|
||||
|
||||
def update_sync_state(self, key: str, value: dict[str, Any]) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO sync_state (key, value_json, updated_at)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
value_json = excluded.value_json,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(key, json.dumps(value, ensure_ascii=False)),
|
||||
)
|
||||
|
||||
def upsert_thumbnail(
|
||||
self,
|
||||
*,
|
||||
artwork_id: int,
|
||||
page_index: int,
|
||||
source_path: str,
|
||||
thumbnail_path: str,
|
||||
width: int,
|
||||
height: int,
|
||||
source_mtime: float,
|
||||
source_size: int,
|
||||
) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO thumbnail_cache (
|
||||
artwork_id, page_index, source_path, thumbnail_path, width, height,
|
||||
source_mtime, source_size, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id, page_index) DO UPDATE SET
|
||||
source_path = excluded.source_path,
|
||||
thumbnail_path = excluded.thumbnail_path,
|
||||
width = excluded.width,
|
||||
height = excluded.height,
|
||||
source_mtime = excluded.source_mtime,
|
||||
source_size = excluded.source_size,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(artwork_id, page_index, source_path, thumbnail_path, width, height, source_mtime, source_size),
|
||||
)
|
||||
|
||||
def upsert_ocr_text(
|
||||
self,
|
||||
artwork_id: int,
|
||||
*,
|
||||
text: str,
|
||||
language: str = "mixed",
|
||||
confidence: float | None = None,
|
||||
engine: str = "paddleocr",
|
||||
status: str = "done",
|
||||
attempts: int = 1,
|
||||
) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artwork_ocr_texts (
|
||||
artwork_id, language, ocr_text, confidence, engine, status, attempts, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id) DO UPDATE SET
|
||||
language = excluded.language,
|
||||
ocr_text = excluded.ocr_text,
|
||||
confidence = excluded.confidence,
|
||||
engine = excluded.engine,
|
||||
status = excluded.status,
|
||||
attempts = excluded.attempts,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(int(artwork_id), language, text, confidence, engine, status, attempts),
|
||||
)
|
||||
|
||||
def get_ocr_text(self, artwork_id: int) -> sqlite3.Row | None:
|
||||
return self.connection.execute(
|
||||
"SELECT * FROM artwork_ocr_texts WHERE artwork_id = ?",
|
||||
(int(artwork_id),),
|
||||
).fetchone()
|
||||
|
||||
def set_favorite(self, artwork_id: int, favorite: bool, note: str = "") -> None:
|
||||
with self.connection:
|
||||
if favorite:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO favorites (artwork_id, note, created_at)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artwork_id) DO UPDATE SET note = excluded.note
|
||||
""",
|
||||
(artwork_id, note),
|
||||
)
|
||||
else:
|
||||
self.connection.execute("DELETE FROM favorites WHERE artwork_id = ?", (artwork_id,))
|
||||
|
||||
def set_followed_artist(self, artist_id: int, followed: bool, name: str = "") -> None:
|
||||
with self.connection:
|
||||
self.ensure_artist_stub(artist_id, name=name)
|
||||
if followed:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO followed_artists (artist_id, name, enabled, created_at)
|
||||
VALUES (?, ?, 1, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
name = COALESCE(NULLIF(excluded.name, ''), followed_artists.name),
|
||||
enabled = 1
|
||||
""",
|
||||
(artist_id, name),
|
||||
)
|
||||
else:
|
||||
self.connection.execute(
|
||||
"UPDATE followed_artists SET enabled = 0 WHERE artist_id = ?",
|
||||
(artist_id,),
|
||||
)
|
||||
|
||||
def upsert_origin_followed_artist(
|
||||
self,
|
||||
artist_id: int,
|
||||
*,
|
||||
name: str = "",
|
||||
account: str = "",
|
||||
avatar_url: str = "",
|
||||
) -> str:
|
||||
clean_name = _clean_optional_text(name)
|
||||
before = self.connection.execute(
|
||||
"""
|
||||
SELECT fa.enabled, fa.name AS follow_name, ar.name AS artist_name,
|
||||
ar.account, ar.avatar_url
|
||||
FROM followed_artists fa
|
||||
LEFT JOIN artists ar ON ar.artist_id = fa.artist_id
|
||||
WHERE fa.artist_id = ?
|
||||
""",
|
||||
(int(artist_id),),
|
||||
).fetchone()
|
||||
status = "created"
|
||||
if before is not None:
|
||||
changed = (
|
||||
not bool(before["enabled"])
|
||||
or (clean_name and clean_name != str(before["follow_name"] or before["artist_name"] or ""))
|
||||
or (account and account != str(before["account"] or ""))
|
||||
or (avatar_url and avatar_url != str(before["avatar_url"] or ""))
|
||||
)
|
||||
status = "updated" if changed else "skipped"
|
||||
with self.connection:
|
||||
self.upsert_artist_profile(
|
||||
int(artist_id),
|
||||
name=clean_name,
|
||||
account=account,
|
||||
avatar_url=avatar_url,
|
||||
)
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO followed_artists (artist_id, name, enabled, last_synced_at, last_error, created_at)
|
||||
VALUES (?, ?, 1, CURRENT_TIMESTAMP, '', CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
name = COALESCE(NULLIF(excluded.name, ''), followed_artists.name),
|
||||
enabled = 1,
|
||||
last_synced_at = CURRENT_TIMESTAMP,
|
||||
last_error = ''
|
||||
""",
|
||||
(int(artist_id), clean_name),
|
||||
)
|
||||
return status
|
||||
|
||||
def mark_artist_synced(self, artist_id: int, *, error: str = "") -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO followed_artists (artist_id, enabled, last_synced_at, last_error)
|
||||
VALUES (?, 1, CURRENT_TIMESTAMP, ?)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
last_synced_at = CURRENT_TIMESTAMP,
|
||||
last_error = excluded.last_error
|
||||
""",
|
||||
(artist_id, error),
|
||||
)
|
||||
|
||||
def _upsert_artist(self, detail: dict[str, Any], artist_id: int) -> None:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO artists (artist_id, name, account, avatar_url, metadata_json, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
account = COALESCE(excluded.account, artists.account),
|
||||
avatar_url = COALESCE(excluded.avatar_url, artists.avatar_url),
|
||||
metadata_json = excluded.metadata_json,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
artist_id,
|
||||
str(detail.get("userName") or detail.get("artistName") or ""),
|
||||
str(detail.get("userAccount") or "") or None,
|
||||
str(detail.get("profileImageUrl") or detail.get("userImage") or "") or None,
|
||||
json.dumps(
|
||||
{
|
||||
"userId": artist_id,
|
||||
"userName": detail.get("userName"),
|
||||
"userAccount": detail.get("userAccount"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def _upsert_tag(self, name: str, translated_name: str | None) -> int:
|
||||
self.connection.execute(
|
||||
"""
|
||||
INSERT INTO tags (name, translated_name)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
translated_name = COALESCE(excluded.translated_name, tags.translated_name)
|
||||
""",
|
||||
(name, translated_name),
|
||||
)
|
||||
row = self.connection.execute("SELECT tag_id FROM tags WHERE name = ?", (name,)).fetchone()
|
||||
return int(row["tag_id"])
|
||||
|
||||
|
||||
def _as_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _clean_optional_text(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
return "" if text.lower() in {"none", "null", "undefined"} else text
|
||||
|
||||
|
||||
def library_artwork_id(source_id: int, artwork_type: str = "illust", source_platform: str = "") -> int:
|
||||
"""Map a source-native ID into the unified library artwork_id namespace.
|
||||
|
||||
Pixiv novels use negative source_id; AO3 works use a negative offset
|
||||
range to avoid collisions with Pixiv novel IDs.
|
||||
"""
|
||||
if artwork_type == "novel":
|
||||
if source_platform == "ao3":
|
||||
# AO3: use negative offset range -(200M + work_id) to stay clear
|
||||
# of Pixiv novel IDs (currently -1 .. -199_999_999).
|
||||
AO3_OFFSET = 200_000_000
|
||||
return -(AO3_OFFSET + abs(int(source_id)))
|
||||
return -abs(int(source_id))
|
||||
return int(source_id)
|
||||
|
||||
|
||||
def _extract_tags(detail: dict[str, Any]) -> list[tuple[str, str | None]]:
|
||||
raw_tags = detail.get("tags")
|
||||
if isinstance(raw_tags, dict):
|
||||
raw_tags = raw_tags.get("tags")
|
||||
if not isinstance(raw_tags, list):
|
||||
return []
|
||||
|
||||
result: list[tuple[str, str | None]] = []
|
||||
for item in raw_tags:
|
||||
if isinstance(item, str):
|
||||
result.append((item, None))
|
||||
continue
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
name = item.get("tag") or item.get("name")
|
||||
if not name:
|
||||
continue
|
||||
translation = item.get("translation")
|
||||
translated_name = None
|
||||
if isinstance(translation, dict):
|
||||
translated_name = translation.get("en") or translation.get("zh") or translation.get("romaji")
|
||||
elif isinstance(translation, str):
|
||||
translated_name = translation
|
||||
result.append((str(name), str(translated_name) if translated_name else None))
|
||||
return result
|
||||
@@ -0,0 +1,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, DownloadFileRecord, PixivLibrary
|
||||
|
||||
|
||||
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
|
||||
ILLUST_FILE_PATTERN = re.compile(r"(?P<artwork_id>\d+)_p(?P<page_index>\d+)", re.IGNORECASE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportSummary:
|
||||
scanned_files: int = 0
|
||||
imported_files: int = 0
|
||||
skipped_files: int = 0
|
||||
imported_artworks: int = 0
|
||||
|
||||
|
||||
class LegacyDownloadImporter:
|
||||
def __init__(self, *, db_path: str | Path = DEFAULT_LIBRARY_DB) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
|
||||
def scan(self, root: str | Path = "Sakura_Downloads") -> ImportSummary:
|
||||
root_path = Path(root)
|
||||
summary = ImportSummary()
|
||||
if not root_path.exists():
|
||||
return summary
|
||||
|
||||
library = PixivLibrary(self.db_path)
|
||||
imported_artworks: set[int] = set()
|
||||
try:
|
||||
for file_path in root_path.rglob("*"):
|
||||
if not file_path.is_file() or file_path.suffix.lower() not in IMAGE_SUFFIXES:
|
||||
continue
|
||||
summary.scanned_files += 1
|
||||
match = ILLUST_FILE_PATTERN.search(file_path.stem)
|
||||
if match is None:
|
||||
summary.skipped_files += 1
|
||||
continue
|
||||
artwork_id = int(match.group("artwork_id"))
|
||||
page_index = int(match.group("page_index"))
|
||||
library.ensure_artwork_stub(artwork_id, artwork_type="illust", title=str(artwork_id))
|
||||
artist_id = self._infer_artist_id(root_path, file_path)
|
||||
if artist_id is not None:
|
||||
library.set_artwork_artist(artwork_id, artist_id)
|
||||
library.record_files(
|
||||
artwork_id,
|
||||
[
|
||||
DownloadFileRecord(
|
||||
page_index=page_index,
|
||||
file_path=str(file_path.resolve()),
|
||||
source_url="",
|
||||
status="imported",
|
||||
size_bytes=file_path.stat().st_size,
|
||||
)
|
||||
],
|
||||
)
|
||||
imported_artworks.add(artwork_id)
|
||||
summary.imported_files += 1
|
||||
summary.imported_artworks = len(imported_artworks)
|
||||
return summary
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def _infer_artist_id(self, root_path: Path, file_path: Path) -> int | None:
|
||||
try:
|
||||
relative = file_path.relative_to(root_path)
|
||||
except ValueError:
|
||||
return None
|
||||
if not relative.parts:
|
||||
return None
|
||||
first = relative.parts[0]
|
||||
return int(first) if first.isdigit() else None
|
||||
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
|
||||
|
||||
|
||||
OcrExtractor = Callable[[Path], tuple[str, float | None, str]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class LocalOcrResult:
|
||||
artwork_id: int
|
||||
status: str
|
||||
text_length: int = 0
|
||||
confidence: float | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class LocalOcrIndexer:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db_path: str | Path = DEFAULT_LIBRARY_DB,
|
||||
enabled: bool | None = None,
|
||||
max_dimension: int | None = None,
|
||||
language: str | None = None,
|
||||
max_concurrency: int | None = None,
|
||||
) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.enabled = enabled if enabled is not None else _env_bool("SAKURA_LOCAL_OCR_ENABLED", False)
|
||||
self.max_dimension = max_dimension or int(os.getenv("SAKURA_LOCAL_OCR_MAX_DIMENSION", "1800"))
|
||||
self.language = language or os.getenv("SAKURA_LOCAL_OCR_LANGUAGE", "ch")
|
||||
self._semaphore = threading.BoundedSemaphore(
|
||||
max_concurrency or int(os.getenv("SAKURA_LOCAL_OCR_MAX_CONCURRENCY", "1"))
|
||||
)
|
||||
|
||||
def index_artwork(self, artwork_id: int, *, extractor: OcrExtractor | None = None) -> LocalOcrResult:
|
||||
if not self.enabled and extractor is None:
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="disabled")
|
||||
if not self._semaphore.acquire(blocking=False):
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="busy")
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
existing = library.get_ocr_text(artwork_id)
|
||||
if existing is not None and str(existing["status"]) == "done" and str(existing["ocr_text"]).strip():
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(str(existing["ocr_text"])))
|
||||
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT file_path
|
||||
FROM artwork_files
|
||||
WHERE artwork_id = ?
|
||||
ORDER BY page_index ASC
|
||||
LIMIT 1
|
||||
""",
|
||||
(int(artwork_id),),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="file_not_found")
|
||||
path = Path(str(row["file_path"]))
|
||||
if path.suffix.lower() not in {".jpg", ".jpeg", ".png", ".webp"}:
|
||||
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="unsupported_file_type")
|
||||
|
||||
text, confidence, language = (extractor or self._paddle_ocr_extract)(path)
|
||||
library.upsert_ocr_text(
|
||||
artwork_id,
|
||||
text=text.strip(),
|
||||
confidence=confidence,
|
||||
language=language or self.language,
|
||||
status="done",
|
||||
attempts=1,
|
||||
)
|
||||
logging.info("Local OCR done artwork=%s text_len=%s confidence=%s", artwork_id, len(text.strip()), confidence)
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(text.strip()), confidence=confidence)
|
||||
except Exception as exc:
|
||||
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
|
||||
logging.warning("Local OCR failed artwork=%s error=%s", artwork_id, exc.__class__.__name__)
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="failed", error=exc.__class__.__name__)
|
||||
finally:
|
||||
library.close()
|
||||
self._semaphore.release()
|
||||
|
||||
def _paddle_ocr_extract(self, path: Path) -> tuple[str, float | None, str]:
|
||||
from PIL import Image
|
||||
from paddleocr import PaddleOCR
|
||||
|
||||
work_path = path
|
||||
with Image.open(path) as image:
|
||||
original_size = image.size
|
||||
image.thumbnail((self.max_dimension, self.max_dimension))
|
||||
if image.size != original_size:
|
||||
tmp = path.with_suffix(".local-ocr-resized.jpg")
|
||||
image.convert("RGB").save(tmp, quality=90)
|
||||
work_path = tmp
|
||||
try:
|
||||
ocr = PaddleOCR(use_angle_cls=True, lang=self.language, use_gpu=False)
|
||||
result = ocr.ocr(str(work_path), cls=True)
|
||||
texts: list[str] = []
|
||||
scores: list[float] = []
|
||||
for page in result or []:
|
||||
for line in page or []:
|
||||
if len(line) >= 2 and isinstance(line[1], (list, tuple)):
|
||||
text = str(line[1][0]).strip()
|
||||
if text:
|
||||
texts.append(text)
|
||||
try:
|
||||
scores.append(float(line[1][1]))
|
||||
except (TypeError, ValueError, IndexError):
|
||||
pass
|
||||
confidence = sum(scores) / len(scores) if scores else None
|
||||
return "\n".join(texts), confidence, self.language
|
||||
finally:
|
||||
if work_path != path:
|
||||
work_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
value = os.getenv(name)
|
||||
if value is None:
|
||||
return default
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Bridge URL-level plugin parsing with the rest of the app (tasks/GUI logging).
|
||||
|
||||
Full TaskQueue integration can map Resource → DownloadTaskSpec per site.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
|
||||
from pixiv_app.core.plugin.manager import PluginManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrchestrationResult:
|
||||
plugin_name: str
|
||||
resources: list[Resource]
|
||||
|
||||
|
||||
class PluginTaskOrchestrator:
|
||||
"""Resolve plugins and run parse/download without replacing the legacy Pixiv GUI flow."""
|
||||
|
||||
def __init__(self, plugin_manager: PluginManager) -> None:
|
||||
self.plugin_manager = plugin_manager
|
||||
|
||||
def get_plugin_for_url(self, url: str) -> BasePlugin | None:
|
||||
return self.plugin_manager.get_plugin_for_url(url)
|
||||
|
||||
def parse_url(self, url: str) -> OrchestrationResult:
|
||||
plugin = self.get_plugin_for_url(url)
|
||||
if plugin is None:
|
||||
raise PluginParseError(f"No plugin registered for: {url}")
|
||||
resources = plugin.parse(url)
|
||||
return OrchestrationResult(plugin_name=plugin.name, resources=resources)
|
||||
|
||||
def download_resource(
|
||||
self,
|
||||
plugin_name: str,
|
||||
resource: Resource,
|
||||
save_path: Path,
|
||||
) -> list[Path]:
|
||||
plugin = self.plugin_manager.plugins.get(plugin_name)
|
||||
if plugin is None:
|
||||
raise PluginParseError(f"Plugin not loaded: {plugin_name}")
|
||||
return plugin.download(resource, save_path)
|
||||
@@ -0,0 +1,423 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.downloader import create_session, request_json
|
||||
from pixiv_app.core.gallery_service import LocalGalleryService
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
|
||||
from pixiv_app.core.paths import app_session_file
|
||||
from pixiv_app.core.x_media_service import XMediaService, _normalize_username
|
||||
|
||||
|
||||
ProgressCallback = Callable[[dict[str, Any]], None]
|
||||
|
||||
|
||||
@dataclass
|
||||
class OriginFollow:
|
||||
platform: str
|
||||
external_id: str
|
||||
name: str = ""
|
||||
handle: str = ""
|
||||
avatar_url: str = ""
|
||||
profile_url: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class OriginFollowSyncResult:
|
||||
platform: str
|
||||
created: int = 0
|
||||
updated: int = 0
|
||||
skipped: int = 0
|
||||
failed: int = 0
|
||||
cloud_created: int = 0
|
||||
cloud_updated: int = 0
|
||||
cloud_skipped: int = 0
|
||||
pages: int = 0
|
||||
errors: list[dict[str, str]] = field(default_factory=list)
|
||||
progress: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"platform": self.platform,
|
||||
"created": self.created,
|
||||
"updated": self.updated,
|
||||
"skipped": self.skipped,
|
||||
"failed": self.failed,
|
||||
"cloud_created": self.cloud_created,
|
||||
"cloud_updated": self.cloud_updated,
|
||||
"cloud_skipped": self.cloud_skipped,
|
||||
"pages": self.pages,
|
||||
"errors": self.errors,
|
||||
"progress": self.progress,
|
||||
}
|
||||
|
||||
|
||||
class OriginFollowSyncService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db_path: str = DEFAULT_LIBRARY_DB,
|
||||
gallery_service: LocalGalleryService | None = None,
|
||||
x_media_service: XMediaService | None = None,
|
||||
) -> None:
|
||||
self.db_path = db_path
|
||||
self.gallery_service = gallery_service or LocalGalleryService(db_path=db_path)
|
||||
self.x_media_service = x_media_service or XMediaService()
|
||||
|
||||
def sync(self, platform: str, *, progress: ProgressCallback | None = None) -> dict[str, Any]:
|
||||
normalized = platform.strip().lower()
|
||||
if normalized == "pixiv":
|
||||
return self.sync_pixiv(progress=progress).to_dict()
|
||||
if normalized == "x":
|
||||
return self.sync_x(progress=progress).to_dict()
|
||||
raise ValueError("请选择 Pixiv 或 X。")
|
||||
|
||||
def sync_pixiv(self, *, progress: ProgressCallback | None = None) -> OriginFollowSyncResult:
|
||||
cookie = str(SessionStore(app_session_file()).load().get("cookie", "")).strip()
|
||||
if not cookie:
|
||||
raise ValueError("Pixiv Cookie 不存在,请先导入或保存 Pixiv 登录态。")
|
||||
|
||||
result = OriginFollowSyncResult(platform="pixiv")
|
||||
session = create_session(cookie)
|
||||
try:
|
||||
user_id = self._fetch_pixiv_current_user_id(session, cookie)
|
||||
follows = self._fetch_pixiv_following(session, cookie, user_id, result, progress=progress)
|
||||
finally:
|
||||
session.close()
|
||||
self._upsert_pixiv_local(follows, result)
|
||||
return result
|
||||
|
||||
def sync_x(self, *, progress: ProgressCallback | None = None) -> OriginFollowSyncResult:
|
||||
plugin = self.x_media_service._get_plugin()
|
||||
if not plugin.validate():
|
||||
raise ValueError("X Playwright 不可用,请先安装 Playwright 并完成 X 登录。")
|
||||
|
||||
result = OriginFollowSyncResult(platform="x")
|
||||
with plugin._playwright_lock:
|
||||
follows = asyncio.run(self._fetch_x_following(plugin, result, progress=progress))
|
||||
self._upsert_x_local(follows, result)
|
||||
return result
|
||||
|
||||
def _emit(
|
||||
self,
|
||||
result: OriginFollowSyncResult,
|
||||
progress: ProgressCallback | None,
|
||||
payload: dict[str, Any],
|
||||
) -> None:
|
||||
result.progress.append(payload)
|
||||
if progress is not None:
|
||||
progress(payload)
|
||||
|
||||
def _fetch_pixiv_current_user_id(self, session: Any, cookie: str) -> str:
|
||||
for url in (
|
||||
"https://www.pixiv.net/ajax/user/self?lang=zh",
|
||||
"https://www.pixiv.net/ajax/user/self",
|
||||
):
|
||||
try:
|
||||
payload = request_json(session, url, cookie=cookie, context="Pixiv 当前账号")
|
||||
body = payload.get("body") if isinstance(payload, dict) else {}
|
||||
user_id = _first_text(body, "userId", "id")
|
||||
if user_id:
|
||||
return user_id
|
||||
except Exception:
|
||||
continue
|
||||
try:
|
||||
response = session.get("https://www.pixiv.net/", timeout=30)
|
||||
response.raise_for_status()
|
||||
match = re.search(r'"userData"\s*:\s*\{[^}]*"id"\s*:\s*"?(?P<id>\d+)"?', response.text)
|
||||
if match:
|
||||
return match.group("id")
|
||||
except Exception:
|
||||
pass
|
||||
raise ValueError("Pixiv Cookie 可能已失效:无法识别当前登录账号。")
|
||||
|
||||
def _fetch_pixiv_following(
|
||||
self,
|
||||
session: Any,
|
||||
cookie: str,
|
||||
user_id: str,
|
||||
result: OriginFollowSyncResult,
|
||||
*,
|
||||
progress: ProgressCallback | None = None,
|
||||
) -> list[OriginFollow]:
|
||||
follows: list[OriginFollow] = []
|
||||
seen: set[str] = set()
|
||||
limit = 100
|
||||
offset = 0
|
||||
page = 1
|
||||
while True:
|
||||
self._emit(result, progress, {"platform": "pixiv", "page": page, "message": f"正在同步第 {page} 页"})
|
||||
url = (
|
||||
f"https://www.pixiv.net/ajax/user/{user_id}/following"
|
||||
f"?offset={offset}&limit={limit}&rest=show&tag=&acceptingRequests=0&lang=zh"
|
||||
)
|
||||
try:
|
||||
payload = request_json(
|
||||
session,
|
||||
url,
|
||||
cookie=cookie,
|
||||
referer=f"https://www.pixiv.net/users/{user_id}/following",
|
||||
context=f"Pixiv 关注列表第 {page} 页",
|
||||
)
|
||||
except Exception as exc:
|
||||
if page == 1:
|
||||
raise ValueError("Pixiv Cookie 可能已失效:无法读取关注列表。") from exc
|
||||
result.failed += 1
|
||||
result.errors.append({"id": f"page:{page}", "reason": str(exc)})
|
||||
break
|
||||
users = _extract_pixiv_users(payload)
|
||||
result.pages = page
|
||||
if not users:
|
||||
break
|
||||
for item in users:
|
||||
follow = _pixiv_follow_from_payload(item)
|
||||
if follow is None:
|
||||
result.failed += 1
|
||||
result.errors.append({"id": "unknown", "reason": "Pixiv 作者记录缺少 userId"})
|
||||
continue
|
||||
if follow.external_id in seen:
|
||||
result.skipped += 1
|
||||
continue
|
||||
seen.add(follow.external_id)
|
||||
follows.append(follow)
|
||||
if len(users) < limit:
|
||||
break
|
||||
offset += limit
|
||||
page += 1
|
||||
return follows
|
||||
|
||||
async def _fetch_x_following(
|
||||
self,
|
||||
plugin: Any,
|
||||
result: OriginFollowSyncResult,
|
||||
*,
|
||||
progress: ProgressCallback | None = None,
|
||||
) -> list[OriginFollow]:
|
||||
from plugins.x import selectors
|
||||
from plugins.x.playwright_client import XPlaywrightClient
|
||||
|
||||
client = XPlaywrightClient(plugin.config)
|
||||
follows: list[OriginFollow] = []
|
||||
seen: set[str] = set()
|
||||
try:
|
||||
await client.start()
|
||||
page = await client.new_page_goto("https://x.com/home")
|
||||
try:
|
||||
await client.raise_if_guest_wall(page)
|
||||
profile_href = await page.locator(selectors.PROFILE_LINK).first.get_attribute("href", timeout=8000)
|
||||
username = _x_username_from_href(profile_href or "")
|
||||
if not username:
|
||||
raise ValueError("X 未登录:请先在插件管理里完成 X 登录。")
|
||||
await page.goto(f"https://x.com/{username}{selectors.FOLLOWING_URL_SUFFIX}", wait_until="domcontentloaded")
|
||||
await client.raise_if_guest_wall(page)
|
||||
stale_rounds = 0
|
||||
page_no = 1
|
||||
while stale_rounds < 4 and page_no <= 80:
|
||||
self._emit(result, progress, {"platform": "x", "page": page_no, "message": f"正在同步第 {page_no} 页"})
|
||||
added = await _collect_x_user_cells(page, seen, follows)
|
||||
result.pages = page_no
|
||||
stale_rounds = stale_rounds + 1 if added == 0 else 0
|
||||
await client.human_scroll_page(page)
|
||||
page_no += 1
|
||||
if not follows:
|
||||
raise ValueError("X 未读取到 following 列表,请确认账号已登录且 following 页面可访问。")
|
||||
finally:
|
||||
await page.close()
|
||||
finally:
|
||||
await client.close()
|
||||
return follows
|
||||
|
||||
def _upsert_pixiv_local(self, follows: list[OriginFollow], result: OriginFollowSyncResult) -> None:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
for follow in follows:
|
||||
try:
|
||||
status = library.upsert_origin_followed_artist(
|
||||
int(follow.external_id),
|
||||
name=follow.name,
|
||||
account=follow.handle,
|
||||
avatar_url=follow.avatar_url,
|
||||
)
|
||||
_apply_local_status(result, status)
|
||||
except Exception as exc:
|
||||
result.failed += 1
|
||||
result.errors.append({"id": follow.external_id, "reason": str(exc)})
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def _upsert_x_local(self, follows: list[OriginFollow], result: OriginFollowSyncResult) -> None:
|
||||
rows = self.x_media_service._load_follow_rows()
|
||||
now = _now_text()
|
||||
by_handle = {str(item.get("username", "")).lower(): item for item in rows}
|
||||
for follow in follows:
|
||||
handle = _normalize_username(follow.handle or follow.external_id)
|
||||
if not handle:
|
||||
result.failed += 1
|
||||
result.errors.append({"id": follow.external_id, "reason": "X 用户名为空"})
|
||||
continue
|
||||
row = by_handle.get(handle.lower())
|
||||
status = "created"
|
||||
if row is None:
|
||||
row = {
|
||||
"username": handle,
|
||||
"creator_name": follow.name or handle,
|
||||
"enabled": True,
|
||||
"profile_url": follow.profile_url or f"https://x.com/{handle}",
|
||||
"avatar_url": follow.avatar_url,
|
||||
"latest_tweet_id": "",
|
||||
"latest_published_at": "",
|
||||
"last_checked_at": now,
|
||||
"last_error": "",
|
||||
"seen_tweet_ids": [],
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
rows.append(row)
|
||||
by_handle[handle.lower()] = row
|
||||
else:
|
||||
changed = (
|
||||
not bool(row.get("enabled", True))
|
||||
or (follow.name and follow.name != str(row.get("creator_name") or ""))
|
||||
or (follow.avatar_url and follow.avatar_url != str(row.get("avatar_url") or ""))
|
||||
)
|
||||
status = "updated" if changed else "skipped"
|
||||
row["username"] = handle
|
||||
row["creator_name"] = follow.name or str(row.get("creator_name") or handle)
|
||||
row["enabled"] = True
|
||||
row["profile_url"] = follow.profile_url or f"https://x.com/{handle}"
|
||||
row["avatar_url"] = follow.avatar_url or str(row.get("avatar_url") or "")
|
||||
row["last_checked_at"] = now
|
||||
row["last_error"] = ""
|
||||
row["updated_at"] = now
|
||||
_apply_local_status(result, status)
|
||||
self.x_media_service._save_follow_rows(rows)
|
||||
|
||||
def _apply_local_status(result: OriginFollowSyncResult, status: str) -> None:
|
||||
if status == "created":
|
||||
result.created += 1
|
||||
elif status == "updated":
|
||||
result.updated += 1
|
||||
else:
|
||||
result.skipped += 1
|
||||
|
||||
|
||||
def _extract_pixiv_users(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
body = payload.get("body") if isinstance(payload, dict) else payload
|
||||
if isinstance(body, list):
|
||||
return [item for item in body if isinstance(item, dict)]
|
||||
if not isinstance(body, dict):
|
||||
return []
|
||||
candidates = (
|
||||
body.get("users"),
|
||||
body.get("followUsers"),
|
||||
body.get("userPreviews"),
|
||||
body.get("thumbnails", {}).get("user") if isinstance(body.get("thumbnails"), dict) else None,
|
||||
)
|
||||
for candidate in candidates:
|
||||
if isinstance(candidate, list):
|
||||
return [item for item in candidate if isinstance(item, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def _pixiv_follow_from_payload(item: dict[str, Any]) -> OriginFollow | None:
|
||||
user_id = _first_text(item, "userId", "user_id", "id")
|
||||
if not user_id:
|
||||
user = item.get("user") if isinstance(item.get("user"), dict) else {}
|
||||
user_id = _first_text(user, "userId", "user_id", "id")
|
||||
item = {**item, **user}
|
||||
if not user_id:
|
||||
return None
|
||||
name = _first_text(item, "name", "userName", "displayName")
|
||||
handle = _first_text(item, "userAccount", "account", "screenName")
|
||||
avatar = _first_text(item, "imageBig", "image", "profileImageUrl", "profile_image_url")
|
||||
return OriginFollow(
|
||||
platform="pixiv",
|
||||
external_id=str(user_id),
|
||||
name=name,
|
||||
handle=handle,
|
||||
avatar_url=avatar,
|
||||
profile_url=f"https://www.pixiv.net/users/{user_id}",
|
||||
)
|
||||
|
||||
|
||||
async def _collect_x_user_cells(page: Any, seen: set[str], follows: list[OriginFollow]) -> int:
|
||||
from plugins.x import selectors
|
||||
|
||||
added = 0
|
||||
cells = await page.locator(selectors.USER_CELL).all()
|
||||
for cell in cells:
|
||||
try:
|
||||
data = await cell.evaluate(
|
||||
"""(node) => {
|
||||
const text = node.innerText || "";
|
||||
const links = Array.from(node.querySelectorAll('a[href^="/"]')).map(a => a.getAttribute("href") || "");
|
||||
const img = node.querySelector('img[src*="profile_images"]');
|
||||
return { text, links, avatar: img ? img.getAttribute("src") : "" };
|
||||
}"""
|
||||
)
|
||||
handle = _x_handle_from_cell(data)
|
||||
if not handle or handle.lower() in seen:
|
||||
continue
|
||||
seen.add(handle.lower())
|
||||
follows.append(
|
||||
OriginFollow(
|
||||
platform="x",
|
||||
external_id=handle,
|
||||
name=_x_display_name_from_cell(data, handle),
|
||||
handle=handle,
|
||||
avatar_url=str(data.get("avatar") or ""),
|
||||
profile_url=f"https://x.com/{handle}",
|
||||
)
|
||||
)
|
||||
added += 1
|
||||
except Exception:
|
||||
continue
|
||||
return added
|
||||
|
||||
|
||||
def _x_handle_from_cell(data: dict[str, Any]) -> str:
|
||||
for href in data.get("links") or []:
|
||||
username = _x_username_from_href(str(href))
|
||||
if username:
|
||||
return username
|
||||
text = str(data.get("text") or "")
|
||||
match = re.search(r"@([A-Za-z0-9_]{1,15})", text)
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def _x_display_name_from_cell(data: dict[str, Any], handle: str) -> str:
|
||||
text = str(data.get("text") or "")
|
||||
for line in [item.strip() for item in text.splitlines() if item.strip()]:
|
||||
if not line.startswith("@") and line.lower() not in {"follow", "following"}:
|
||||
return line
|
||||
return handle
|
||||
|
||||
|
||||
def _x_username_from_href(href: str) -> str:
|
||||
match = re.match(r"^/?([A-Za-z0-9_]{1,15})(?:/)?$", href.strip())
|
||||
if not match:
|
||||
return ""
|
||||
username = match.group(1)
|
||||
if username.lower() in {"home", "explore", "notifications", "messages", "settings", "i"}:
|
||||
return ""
|
||||
return username
|
||||
|
||||
|
||||
def _first_text(payload: Any, *keys: str) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
return ""
|
||||
for key in keys:
|
||||
value = payload.get(key)
|
||||
if value is not None and str(value).strip():
|
||||
return str(value).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _now_text() -> str:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
APP_NAME = "SakuraDownloader"
|
||||
LEGACY_APP_NAME = "PixivDownloader"
|
||||
|
||||
|
||||
def is_frozen() -> bool:
|
||||
return bool(getattr(sys, "frozen", False))
|
||||
|
||||
|
||||
def resource_root() -> Path:
|
||||
if is_frozen():
|
||||
return Path(getattr(sys, "_MEIPASS")).resolve()
|
||||
return Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def executable_dir() -> Path:
|
||||
if is_frozen():
|
||||
return Path(sys.executable).resolve().parent
|
||||
return resource_root()
|
||||
|
||||
|
||||
def data_root() -> Path:
|
||||
override = os.environ.get("SAKURA_APP_DATA_DIR") or os.environ.get("PIXIV_APP_DATA_DIR")
|
||||
if override:
|
||||
root = Path(override)
|
||||
elif is_frozen():
|
||||
local_appdata = os.environ.get("LOCALAPPDATA")
|
||||
root = Path(local_appdata) / APP_NAME if local_appdata else executable_dir() / "data"
|
||||
legacy_root = Path(local_appdata) / LEGACY_APP_NAME if local_appdata else None
|
||||
if legacy_root and legacy_root.exists() and not root.exists():
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
for name in ("pixiv_app_session.json", "runtime", "profiles", "plugins"):
|
||||
src = legacy_root / name
|
||||
dst = root / name
|
||||
if src.exists() and not dst.exists():
|
||||
if src.is_dir():
|
||||
import shutil
|
||||
|
||||
shutil.copytree(src, dst)
|
||||
else:
|
||||
dst.write_bytes(src.read_bytes())
|
||||
else:
|
||||
root = resource_root()
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
def runtime_path(*parts: str) -> Path:
|
||||
path = data_root() / "runtime"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path.joinpath(*parts)
|
||||
|
||||
|
||||
def profiles_root() -> Path:
|
||||
path = data_root() / "profiles"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def plugins_root() -> Path:
|
||||
return resource_root() / "plugins"
|
||||
|
||||
|
||||
def external_plugins_root() -> Path:
|
||||
"""Plugins shipped beside the executable, outside the PyInstaller bundle."""
|
||||
return executable_dir() / "plugins"
|
||||
|
||||
|
||||
def user_plugins_root() -> Path:
|
||||
path = data_root() / "plugins"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def plugin_roots() -> list[Path]:
|
||||
roots = [plugins_root(), external_plugins_root(), user_plugins_root()]
|
||||
result: list[Path] = []
|
||||
for root in roots:
|
||||
if root not in result:
|
||||
result.append(root)
|
||||
return result
|
||||
|
||||
|
||||
def webui_root() -> Path:
|
||||
return resource_root() / "pixiv_app" / "webui"
|
||||
|
||||
|
||||
def app_session_file() -> Path:
|
||||
return data_root() / "pixiv_app_session.json"
|
||||
|
||||
|
||||
def app_library_file() -> Path:
|
||||
"""Persistent local library DB, writable even when the EXE directory is read-only."""
|
||||
target = data_root() / "pixiv_app_library.db"
|
||||
if is_frozen() and not target.exists():
|
||||
legacy = executable_dir() / "pixiv_app_library.db"
|
||||
if legacy.is_file():
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(legacy, target)
|
||||
return target
|
||||
|
||||
|
||||
def downloads_root() -> Path:
|
||||
path = executable_dir() / "Sakura_Downloads"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def playwright_browsers_root() -> Path | None:
|
||||
local_appdata = os.environ.get("LOCALAPPDATA", "").strip()
|
||||
if not local_appdata:
|
||||
return None
|
||||
return Path(local_appdata) / "ms-playwright"
|
||||
|
||||
|
||||
def configure_playwright_runtime_env() -> Path | None:
|
||||
root = playwright_browsers_root()
|
||||
if root is None:
|
||||
return None
|
||||
if root.exists() and "PLAYWRIGHT_BROWSERS_PATH" not in os.environ:
|
||||
os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(root)
|
||||
return root
|
||||
|
||||
|
||||
def preferred_chromium_executable() -> Path | None:
|
||||
root = configure_playwright_runtime_env()
|
||||
if root is None or not root.exists():
|
||||
return None
|
||||
|
||||
candidates: list[tuple[int, float, Path]] = []
|
||||
for browser_dir in root.glob("chromium-*"):
|
||||
if not browser_dir.is_dir():
|
||||
continue
|
||||
revision = _parse_revision(browser_dir.name)
|
||||
for relative in (
|
||||
Path("chrome-win64") / "chrome.exe",
|
||||
Path("chrome-win") / "chrome.exe",
|
||||
):
|
||||
executable = browser_dir / relative
|
||||
if executable.is_file():
|
||||
try:
|
||||
mtime = executable.stat().st_mtime
|
||||
except OSError:
|
||||
mtime = 0.0
|
||||
candidates.append((revision, mtime, executable))
|
||||
break
|
||||
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
candidates.sort(key=lambda item: (item[0], item[1]), reverse=True)
|
||||
return candidates[0][2]
|
||||
|
||||
|
||||
def explain_missing_browser(exc: Exception, *, feature_name: str) -> str:
|
||||
text = str(exc)
|
||||
lowered = text.lower()
|
||||
if "executable doesn't exist" not in lowered and "browsertype.launch_persistent_context" not in lowered:
|
||||
return text
|
||||
|
||||
return (
|
||||
f"{feature_name} 需要 Playwright Chromium 浏览器,但当前系统里没有可用的浏览器文件。\n"
|
||||
"请先运行同目录下的 install_playwright_chromium.bat,安装完成后重新打开程序再试。"
|
||||
)
|
||||
|
||||
|
||||
def _parse_revision(name: str) -> int:
|
||||
try:
|
||||
return int(name.rsplit("-", 1)[-1])
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Plugin system: base types, manager, generator, optional market client."""
|
||||
|
||||
from pixiv_app.core.plugin.base import (
|
||||
BasePlugin,
|
||||
PluginAuthError,
|
||||
PluginDownloadError,
|
||||
PluginError,
|
||||
PluginManifest,
|
||||
PluginParseError,
|
||||
Resource,
|
||||
)
|
||||
from pixiv_app.core.plugin.manager import PluginEvent, PluginManager
|
||||
|
||||
__all__ = [
|
||||
"BasePlugin",
|
||||
"PluginAuthError",
|
||||
"PluginDownloadError",
|
||||
"PluginError",
|
||||
"PluginEvent",
|
||||
"PluginManager",
|
||||
"PluginManifest",
|
||||
"PluginParseError",
|
||||
"Resource",
|
||||
]
|
||||
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
Multi-site plugin core abstractions.
|
||||
|
||||
解析与下载采用同步接口,便于与现有 requests / 线程模型集成;
|
||||
若站点 SDK 仅提供 asyncio,可在插件内部使用 asyncio.run 包装。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class Resource:
|
||||
"""Unified resource model returned by plugins."""
|
||||
|
||||
id: str
|
||||
url: str
|
||||
title: str
|
||||
author: str
|
||||
author_id: str
|
||||
files: list[dict[str, Any]]
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
thumbnail: str | None = None
|
||||
created_at: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PluginManifest:
|
||||
"""Optional manifest shipped as manifest.json next to plugin.py."""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
domain: str
|
||||
author: str = ""
|
||||
description: str = ""
|
||||
requires_auth: bool = False
|
||||
min_platform_version: str = "1.0.0"
|
||||
tags: list[str] = field(default_factory=list)
|
||||
dependencies: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class PluginError(Exception):
|
||||
"""Base class for plugin failures."""
|
||||
|
||||
|
||||
class PluginAuthError(PluginError):
|
||||
"""Authentication or authorization failed."""
|
||||
|
||||
|
||||
class PluginParseError(PluginError):
|
||||
"""URL or API response could not be parsed."""
|
||||
|
||||
|
||||
class PluginDownloadError(PluginError):
|
||||
"""Download or filesystem error."""
|
||||
|
||||
|
||||
class BasePlugin(ABC):
|
||||
"""Interface implemented by every site plugin."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def domain(self) -> str:
|
||||
"""Primary domain, e.g. pixiv.net."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""Human-readable plugin name."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def version(self) -> str:
|
||||
"""Semantic version string."""
|
||||
|
||||
@abstractmethod
|
||||
def can_handle(self, url: str) -> bool:
|
||||
"""Return True if this plugin should process the URL or raw token."""
|
||||
|
||||
@abstractmethod
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
"""Resolve URL into one or more downloadable resources."""
|
||||
|
||||
@abstractmethod
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
"""Download files for ``resource`` under ``save_path``."""
|
||||
|
||||
@abstractmethod
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
"""HTTP headers for requests (Cookie, Bearer token, etc.)."""
|
||||
|
||||
def validate(self) -> bool:
|
||||
"""Optional self-check (credentials present, deps installed)."""
|
||||
return True
|
||||
|
||||
def get_config_schema(self) -> dict[str, Any]:
|
||||
"""Optional JSON-schema-like dict for dynamic GUI forms."""
|
||||
return {}
|
||||
|
||||
def load_config(self) -> dict[str, Any]:
|
||||
"""Override to load persisted plugin settings."""
|
||||
return {}
|
||||
|
||||
def save_config(self, data: dict[str, Any]) -> None:
|
||||
"""Override to persist plugin settings."""
|
||||
|
||||
__all__ = [
|
||||
"BasePlugin",
|
||||
"PluginAuthError",
|
||||
"PluginDownloadError",
|
||||
"PluginError",
|
||||
"PluginManifest",
|
||||
"PluginParseError",
|
||||
"Resource",
|
||||
]
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||
|
||||
|
||||
def _class_name_from_plugin_name(name: str) -> str:
|
||||
parts = re.split(r"[\s_]+", name.strip())
|
||||
stem = "".join(p[:1].upper() + p[1:].lower() if p else "" for p in parts if p)
|
||||
return f"{stem}Plugin" if stem else "GeneratedPlugin"
|
||||
|
||||
|
||||
class PluginGenerator:
|
||||
"""Generate a skeleton plugin directory from Jinja2 templates."""
|
||||
|
||||
def __init__(self, template_dir: str | Path | None = None) -> None:
|
||||
base = Path(__file__).resolve().parent / "templates"
|
||||
self.template_dir = Path(template_dir) if template_dir else base
|
||||
self.env = Environment(
|
||||
loader=FileSystemLoader(str(self.template_dir)),
|
||||
autoescape=select_autoescape(enabled_extensions=()),
|
||||
)
|
||||
|
||||
def generate(
|
||||
self,
|
||||
name: str,
|
||||
domain: str,
|
||||
author: str,
|
||||
description: str,
|
||||
*,
|
||||
output_root: str | Path = "plugins",
|
||||
) -> Path:
|
||||
slug = name.strip().lower().replace(" ", "_")
|
||||
plugin_dir = Path(output_root) / slug
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
context: dict[str, Any] = {
|
||||
"name": name.strip(),
|
||||
"domain": domain.strip(),
|
||||
"author": author.strip(),
|
||||
"description": description.strip(),
|
||||
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"class_name": _class_name_from_plugin_name(name),
|
||||
}
|
||||
|
||||
py_template = self.env.get_template("base_plugin.py.jinja")
|
||||
(plugin_dir / "plugin.py").write_text(py_template.render(**context), encoding="utf-8")
|
||||
|
||||
manifest_template = self.env.get_template("manifest.json.jinja")
|
||||
(plugin_dir / "manifest.json").write_text(manifest_template.render(**context), encoding="utf-8")
|
||||
|
||||
init_file = plugin_dir / "__init__.py"
|
||||
if not init_file.exists():
|
||||
init_file.write_text('"""Generated plugin."""\n', encoding="utf-8")
|
||||
|
||||
return plugin_dir.resolve()
|
||||
|
||||
def generate_from_gui(self, form_data: dict[str, str], *, output_root: str | Path = "plugins") -> Path:
|
||||
return self.generate(
|
||||
name=form_data["name"],
|
||||
domain=form_data["domain"],
|
||||
author=form_data["author"],
|
||||
description=form_data["description"],
|
||||
output_root=output_root,
|
||||
)
|
||||
@@ -0,0 +1,194 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginManifest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PluginEvent(str, Enum):
|
||||
BEFORE_LOAD = "before_load"
|
||||
AFTER_LOAD = "after_load"
|
||||
BEFORE_UNLOAD = "before_unload"
|
||||
AFTER_UNLOAD = "after_unload"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
class PluginManager:
|
||||
"""Discover and load plugins from filesystem directories."""
|
||||
|
||||
def __init__(self, plugin_dirs: list[str | Path]) -> None:
|
||||
self.plugin_dirs = [Path(d).resolve() for d in plugin_dirs]
|
||||
self.plugins: dict[str, BasePlugin] = {}
|
||||
self.manifests: dict[str, PluginManifest] = {}
|
||||
self.loaded_modules: dict[str, Any] = {}
|
||||
self._paths: dict[str, Path] = {}
|
||||
self.load_errors: dict[Path, str] = {}
|
||||
self.event_handlers: dict[PluginEvent, list[Callable[..., None]]] = defaultdict(list)
|
||||
|
||||
def on_event(self, event: PluginEvent, callback: Callable[..., None]) -> None:
|
||||
self.event_handlers[event].append(callback)
|
||||
|
||||
def _emit(self, event: PluginEvent, *args: Any, **kwargs: Any) -> None:
|
||||
for cb in self.event_handlers.get(event, []):
|
||||
try:
|
||||
cb(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
logger.exception("Plugin event handler failed: %s", exc)
|
||||
|
||||
def discover(self) -> list[Path]:
|
||||
"""Return paths to plugin.py files."""
|
||||
found: list[Path] = []
|
||||
for root in self.plugin_dirs:
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for subdir in sorted(root.iterdir()):
|
||||
if not subdir.is_dir():
|
||||
continue
|
||||
candidate = subdir / "plugin.py"
|
||||
if candidate.is_file():
|
||||
found.append(candidate)
|
||||
return found
|
||||
|
||||
def _load_manifest(self, plugin_dir: Path) -> PluginManifest | None:
|
||||
manifest_path = plugin_dir / "manifest.json"
|
||||
if not manifest_path.is_file():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
return PluginManifest(
|
||||
name=str(data.get("name", plugin_dir.name)),
|
||||
version=str(data.get("version", "0.0.0")),
|
||||
domain=str(data.get("domain", "")),
|
||||
author=str(data.get("author", "")),
|
||||
description=str(data.get("description", "")),
|
||||
requires_auth=bool(data.get("requires_auth", False)),
|
||||
min_platform_version=str(data.get("min_platform_version", "1.0.0")),
|
||||
tags=list(data.get("tags") or []),
|
||||
dependencies=list(data.get("dependencies") or []),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Invalid manifest %s: %s", manifest_path, exc)
|
||||
return None
|
||||
|
||||
def load_plugin(self, plugin_path: Path) -> BasePlugin | None:
|
||||
"""Load a single plugin from ``.../plugin.py``."""
|
||||
plugin_path = plugin_path.resolve()
|
||||
plugin_dir = plugin_path.parent
|
||||
module_name = f"sakura_dynamic_plugin_{plugin_dir.name}"
|
||||
module_stem = module_name.rsplit(".", 1)[0]
|
||||
for loaded_name in list(sys.modules):
|
||||
if loaded_name == module_name or loaded_name.startswith(f"{module_name}."):
|
||||
del sys.modules[loaded_name]
|
||||
self.load_errors.pop(plugin_path, None)
|
||||
|
||||
self._emit(PluginEvent.BEFORE_LOAD, plugin_path)
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name,
|
||||
plugin_path,
|
||||
submodule_search_locations=[str(plugin_dir)],
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
self._emit(PluginEvent.ERROR, plugin_path, "invalid spec")
|
||||
self.load_errors[plugin_path] = "invalid spec"
|
||||
return None
|
||||
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
module.__package__ = module_stem
|
||||
module.__file__ = str(plugin_path)
|
||||
sys.modules[module_name] = module
|
||||
try:
|
||||
spec.loader.exec_module(module)
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to load %s", plugin_path)
|
||||
self._emit(PluginEvent.ERROR, plugin_path, exc)
|
||||
self.load_errors[plugin_path] = f"{type(exc).__name__}: {exc}"
|
||||
return None
|
||||
|
||||
plugin_class = getattr(module, "plugin_class", None)
|
||||
if plugin_class is None:
|
||||
logger.error("Module %s has no plugin_class export", plugin_path)
|
||||
self._emit(PluginEvent.ERROR, plugin_path, "missing plugin_class")
|
||||
self.load_errors[plugin_path] = "missing plugin_class"
|
||||
return None
|
||||
|
||||
try:
|
||||
plugin = plugin_class()
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to instantiate plugin from %s", plugin_path)
|
||||
self._emit(PluginEvent.ERROR, plugin_path, exc)
|
||||
self.load_errors[plugin_path] = f"{type(exc).__name__}: {exc}"
|
||||
return None
|
||||
|
||||
if not isinstance(plugin, BasePlugin):
|
||||
logger.error("plugin_class is not BasePlugin: %s", plugin_path)
|
||||
self.load_errors[plugin_path] = "plugin_class is not BasePlugin"
|
||||
return None
|
||||
|
||||
key = plugin.name
|
||||
manifest = self._load_manifest(plugin_dir)
|
||||
if manifest:
|
||||
self.manifests[key] = manifest
|
||||
|
||||
self.plugins[key] = plugin
|
||||
self.loaded_modules[key] = module
|
||||
self._paths[key] = plugin_path
|
||||
self.load_errors.pop(plugin_path, None)
|
||||
self._emit(PluginEvent.AFTER_LOAD, plugin)
|
||||
return plugin
|
||||
|
||||
def load_all(self) -> dict[str, BasePlugin]:
|
||||
for path in self.discover():
|
||||
try:
|
||||
self.load_plugin(path)
|
||||
except Exception as exc:
|
||||
logger.exception("Failed during load_all %s: %s", path, exc)
|
||||
return self.plugins
|
||||
|
||||
def reload_all_from_disk(self) -> None:
|
||||
"""Unload every plugin entry then scan directories again."""
|
||||
for name in list(self.plugins.keys()):
|
||||
self.unload_plugin(name)
|
||||
self.load_errors.clear()
|
||||
self.load_all()
|
||||
|
||||
def reload_plugin(self, name: str) -> bool:
|
||||
path = self._paths.get(name)
|
||||
if path is None:
|
||||
return False
|
||||
self.unload_plugin(name)
|
||||
return self.load_plugin(path) is not None
|
||||
|
||||
def get_plugin_path(self, name: str) -> Path | None:
|
||||
return self._paths.get(name)
|
||||
|
||||
def unload_plugin(self, name: str) -> None:
|
||||
self._emit(PluginEvent.BEFORE_UNLOAD, name)
|
||||
self.plugins.pop(name, None)
|
||||
mod = self.loaded_modules.pop(name, None)
|
||||
self.manifests.pop(name, None)
|
||||
self._paths.pop(name, None)
|
||||
if mod is not None and getattr(mod, "__name__", None) in sys.modules:
|
||||
del sys.modules[mod.__name__]
|
||||
self._emit(PluginEvent.AFTER_UNLOAD, name)
|
||||
|
||||
def get_plugin_for_url(self, url: str) -> BasePlugin | None:
|
||||
for plugin in self.plugins.values():
|
||||
if plugin.can_handle(url):
|
||||
return plugin
|
||||
return None
|
||||
|
||||
def check_dependencies(self, manifest: PluginManifest) -> bool:
|
||||
for dep in manifest.dependencies:
|
||||
if dep not in self.plugins:
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,29 @@
|
||||
"""
|
||||
Optional plugin market client — stub for future HTTP catalog integration.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
class PluginMarketClient:
|
||||
def __init__(self, api_url: str = "") -> None:
|
||||
self.api_url = api_url.rstrip("/")
|
||||
|
||||
def list_plugins(self) -> list[dict[str, Any]]:
|
||||
if not self.api_url:
|
||||
return []
|
||||
response = requests.get(f"{self.api_url}/plugins", timeout=15)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return list(data.get("plugins", []))
|
||||
|
||||
def install_plugin(self, _plugin_name: str) -> bool:
|
||||
"""Download zip and extract into plugins/ — not implemented."""
|
||||
return False
|
||||
|
||||
def check_updates(self) -> list[dict[str, Any]]:
|
||||
return []
|
||||
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
{{ name }} plugin — auto-generated at {{ generated_at }}.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pixiv_app.core.plugin.base import BasePlugin, Resource
|
||||
|
||||
|
||||
class {{ class_name }}(BasePlugin):
|
||||
domain = "{{ domain }}"
|
||||
name = "{{ name }}"
|
||||
version = "1.0.0"
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
return "{{ domain }}" in url.lower()
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
raise NotImplementedError("Implement URL parsing and build Resource list.")
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
raise NotImplementedError("Download files for resource into save_path.")
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
return {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
|
||||
|
||||
|
||||
plugin_class = {{ class_name }}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "{{ name }}",
|
||||
"version": "1.0.0",
|
||||
"domain": "{{ domain }}",
|
||||
"author": "{{ author }}",
|
||||
"description": "{{ description }}",
|
||||
"requires_auth": false,
|
||||
"tags": ["general"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProxyInfo:
|
||||
ip: str
|
||||
port: int
|
||||
protocol: str = "http"
|
||||
source: str = "manual"
|
||||
location: str = ""
|
||||
org: str = ""
|
||||
last_check: float = 0.0
|
||||
success_count: int = 0
|
||||
fail_count: int = 0
|
||||
|
||||
@property
|
||||
def proxy_url(self) -> str:
|
||||
return f"{self.protocol}://{self.ip}:{self.port}"
|
||||
|
||||
@property
|
||||
def is_alive(self) -> bool:
|
||||
return self.success_count > 0 or self.fail_count < 3
|
||||
|
||||
|
||||
def parse_proxy_text(text: str) -> list[ProxyInfo]:
|
||||
proxies: list[ProxyInfo] = []
|
||||
seen: set[tuple[str, int, str]] = set()
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "://" not in line:
|
||||
line = f"http://{line}"
|
||||
match = re.match(r"^(https?)://([\w\.\-]+):(\d{2,5})$", line, re.IGNORECASE)
|
||||
if not match:
|
||||
continue
|
||||
protocol, ip, port_text = match.groups()
|
||||
key = (ip, int(port_text), protocol.lower())
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
proxies.append(ProxyInfo(ip=ip, port=int(port_text), protocol=protocol.lower(), source="manual"))
|
||||
return proxies
|
||||
|
||||
|
||||
class QuakeClient:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str = "",
|
||||
cookie: str = "",
|
||||
mode: str = "api_v3",
|
||||
request_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
self.api_key = api_key.strip()
|
||||
self.cookie = cookie.strip()
|
||||
self.mode = mode
|
||||
self.request_kwargs = dict(request_kwargs or {})
|
||||
self.base_url = "https://quake.360.net/api/v3"
|
||||
self.web_assoc_url = "https://quake.360.net/api/search/field/association/quake_service"
|
||||
|
||||
def _api_headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"X-QuakeToken": self.api_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def _web_headers(self) -> dict[str, str]:
|
||||
headers = {
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Content-Type": "application/json",
|
||||
"Origin": "https://quake.360.net",
|
||||
"Referer": "https://quake.360.net/quake/",
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
if self.cookie:
|
||||
headers["Cookie"] = self.cookie
|
||||
return headers
|
||||
|
||||
def search_ips_api(self, query: str, size: int = 100, start: int = 0) -> list[dict[str, Any]]:
|
||||
if not self.api_key:
|
||||
return []
|
||||
url = f"{self.base_url}/search/quake_service"
|
||||
payload = {"query": query, "start": start, "size": min(size, 500)}
|
||||
kwargs = dict(self.request_kwargs)
|
||||
kwargs.setdefault("timeout", 30)
|
||||
try:
|
||||
response = requests.post(url, headers=self._api_headers(), json=payload, **kwargs)
|
||||
if response.status_code != 200:
|
||||
return []
|
||||
body = response.json()
|
||||
return body.get("data", []) if isinstance(body, dict) else []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def search_ips_web_assoc(self, search_content: str) -> list[dict[str, Any]]:
|
||||
if not self.cookie:
|
||||
return []
|
||||
payload = {
|
||||
"search_content": search_content,
|
||||
"device": {
|
||||
"device_type": "PC",
|
||||
"os": "Windows",
|
||||
"os_version": "10.0",
|
||||
"language": "zh_CN",
|
||||
"network": "4g",
|
||||
"browser_info": "Chrome",
|
||||
"fingerprint": "pixiv-proxy-pool",
|
||||
"user_agent": self._web_headers()["User-Agent"],
|
||||
"date": time.strftime("%Y/%m/%d %H:%M:%S"),
|
||||
"UUID": "pixiv-proxy-association",
|
||||
},
|
||||
}
|
||||
kwargs = dict(self.request_kwargs)
|
||||
kwargs.setdefault("timeout", 30)
|
||||
try:
|
||||
response = requests.post(
|
||||
self.web_assoc_url,
|
||||
headers=self._web_headers(),
|
||||
json=payload,
|
||||
**kwargs,
|
||||
)
|
||||
if response.status_code != 200:
|
||||
return []
|
||||
text = response.text
|
||||
ips = sorted(set(re.findall(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", text)))
|
||||
return [{"ip": ip, "port": 80, "organization": "", "location": {}} for ip in ips]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def get_foreign_proxies(
|
||||
self,
|
||||
size: int = 120,
|
||||
countries: Optional[list[str]] = None,
|
||||
) -> list[ProxyInfo]:
|
||||
proxy_ports = [80, 443, 8080, 3128, 8888, 1080, 8118]
|
||||
port_query = " OR ".join([f"port:{port}" for port in proxy_ports])
|
||||
country_codes = countries or ["US", "JP", "KR", "SG", "HK", "TW", "DE", "FR", "GB", "CA", "AU"]
|
||||
country_query = " OR ".join([f'country:"{code}"' for code in country_codes])
|
||||
query = f"({port_query}) AND ({country_query}) AND response:ok"
|
||||
|
||||
if self.mode == "web_assoc":
|
||||
data = self.search_ips_web_assoc(query)
|
||||
else:
|
||||
data = self.search_ips_api(query=query, size=size)
|
||||
|
||||
proxies: list[ProxyInfo] = []
|
||||
seen: set[tuple[str, int, str]] = set()
|
||||
for item in data:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
ip = item.get("ip")
|
||||
port = item.get("port")
|
||||
if not ip or not isinstance(port, int):
|
||||
continue
|
||||
protocol = "https" if port in (443, 8443) else "http"
|
||||
key = (ip, port, protocol)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
location_obj = item.get("location", {}) if isinstance(item.get("location"), dict) else {}
|
||||
location = "/".join(
|
||||
[str(location_obj.get("country", "")).strip(), str(location_obj.get("province", "")).strip()]
|
||||
).strip("/")
|
||||
proxies.append(
|
||||
ProxyInfo(
|
||||
ip=ip,
|
||||
port=port,
|
||||
protocol=protocol,
|
||||
source=f"quake:{self.mode}",
|
||||
location=location,
|
||||
org=str(item.get("organization", "") or item.get("org", "")),
|
||||
)
|
||||
)
|
||||
return proxies
|
||||
|
||||
|
||||
class ProxyPool:
|
||||
def __init__(self, test_url: str = "https://httpbin.org/ip", test_timeout: int = 6):
|
||||
self.proxies: list[ProxyInfo] = []
|
||||
self.working_proxies: list[ProxyInfo] = []
|
||||
self.current_index = 0
|
||||
self.test_url = test_url
|
||||
self.test_timeout = test_timeout
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({"User-Agent": "SakuraDownloaderProxyPool/1.0"})
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add_proxies(self, proxies: list[ProxyInfo]) -> None:
|
||||
with self._lock:
|
||||
existing = {(p.ip, p.port, p.protocol) for p in self.proxies}
|
||||
for proxy in proxies:
|
||||
key = (proxy.ip, proxy.port, proxy.protocol)
|
||||
if key not in existing:
|
||||
existing.add(key)
|
||||
self.proxies.append(proxy)
|
||||
|
||||
def verify_proxy(self, proxy: ProxyInfo) -> bool:
|
||||
target_proxies = {"http": proxy.proxy_url, "https": proxy.proxy_url}
|
||||
try:
|
||||
response = self.session.get(
|
||||
self.test_url,
|
||||
proxies=target_proxies,
|
||||
timeout=self.test_timeout,
|
||||
verify=False,
|
||||
)
|
||||
if response.status_code == 200:
|
||||
with self._lock:
|
||||
proxy.last_check = time.time()
|
||||
proxy.success_count += 1
|
||||
proxy.fail_count = 0
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
with self._lock:
|
||||
proxy.fail_count += 1
|
||||
return False
|
||||
|
||||
def verify_all(self, max_workers: int = 16, max_proxies: int = 80) -> list[ProxyInfo]:
|
||||
with self._lock:
|
||||
to_verify = list(self.proxies[:max_proxies])
|
||||
self.working_proxies = []
|
||||
if not to_verify:
|
||||
return self.working_proxies
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
future_map = {executor.submit(self.verify_proxy, proxy): proxy for proxy in to_verify}
|
||||
for future in as_completed(future_map):
|
||||
proxy = future_map[future]
|
||||
try:
|
||||
if future.result():
|
||||
with self._lock:
|
||||
self.working_proxies.append(proxy)
|
||||
except Exception:
|
||||
continue
|
||||
with self._lock:
|
||||
return list(self.working_proxies)
|
||||
|
||||
def collect_until_target(
|
||||
self,
|
||||
quake_client: QuakeClient,
|
||||
target_working: int = 5,
|
||||
stable_rounds: int = 2,
|
||||
fetch_batch_size: int = 60,
|
||||
verify_batch_size: int = 30,
|
||||
max_rounds: int = 6,
|
||||
max_workers: int = 16,
|
||||
countries: Optional[list[str]] = None,
|
||||
) -> list[ProxyInfo]:
|
||||
stable_hits = 0
|
||||
for _ in range(max_rounds):
|
||||
if stable_hits >= stable_rounds:
|
||||
break
|
||||
new_items = quake_client.get_foreign_proxies(size=fetch_batch_size, countries=countries)
|
||||
if new_items:
|
||||
self.add_proxies(new_items)
|
||||
self.verify_all(max_workers=max_workers, max_proxies=verify_batch_size)
|
||||
with self._lock:
|
||||
working_count = len(self.working_proxies)
|
||||
if working_count >= target_working:
|
||||
stable_hits += 1
|
||||
else:
|
||||
stable_hits = 0
|
||||
with self._lock:
|
||||
return list(self.working_proxies)
|
||||
|
||||
def get_proxy(self, rotate: bool = True) -> Optional[ProxyInfo]:
|
||||
with self._lock:
|
||||
if not self.working_proxies:
|
||||
return None
|
||||
if not rotate:
|
||||
return random.choice(self.working_proxies)
|
||||
proxy = self.working_proxies[self.current_index % len(self.working_proxies)]
|
||||
self.current_index += 1
|
||||
return proxy
|
||||
|
||||
def mark_failed(self, proxy: ProxyInfo) -> None:
|
||||
with self._lock:
|
||||
proxy.fail_count += 1
|
||||
if proxy.fail_count >= 3 and proxy in self.working_proxies:
|
||||
self.working_proxies.remove(proxy)
|
||||
@@ -0,0 +1,145 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
|
||||
|
||||
DEFAULT_THUMBNAIL_DIR = runtime_path("thumbnails")
|
||||
SUPPORTED_IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".gif"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ThumbnailResult:
|
||||
artwork_id: int
|
||||
page_index: int
|
||||
source_path: str
|
||||
thumbnail_path: str
|
||||
width: int
|
||||
height: int
|
||||
generated: bool
|
||||
|
||||
|
||||
class ThumbnailCache:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db_path: str | Path = DEFAULT_LIBRARY_DB,
|
||||
cache_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
|
||||
size: tuple[int, int] = (360, 360),
|
||||
quality: int = 86,
|
||||
) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.cache_dir = Path(cache_dir)
|
||||
self.size = size
|
||||
self.quality = quality
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def ensure_for_artwork(self, artwork_id: int) -> list[ThumbnailResult]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
rows = library.connection.execute(
|
||||
"""
|
||||
SELECT artwork_id, page_index, file_path
|
||||
FROM artwork_files
|
||||
WHERE artwork_id = ?
|
||||
ORDER BY page_index ASC
|
||||
""",
|
||||
(artwork_id,),
|
||||
).fetchall()
|
||||
return [result for row in rows if (result := self.ensure_for_file(library, row)) is not None]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def build_missing(self, *, limit: int = 200) -> list[ThumbnailResult]:
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
rows = library.connection.execute(
|
||||
"""
|
||||
SELECT f.artwork_id, f.page_index, f.file_path
|
||||
FROM artwork_files f
|
||||
LEFT JOIN thumbnail_cache t
|
||||
ON t.artwork_id = f.artwork_id AND t.page_index = f.page_index
|
||||
WHERE t.thumbnail_path IS NULL
|
||||
OR t.source_path != f.file_path
|
||||
ORDER BY f.updated_at DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [result for row in rows if (result := self.ensure_for_file(library, row)) is not None]
|
||||
finally:
|
||||
library.close()
|
||||
|
||||
def ensure_for_file(self, library: PixivLibrary, row) -> ThumbnailResult | None:
|
||||
source_path = Path(str(row["file_path"]))
|
||||
if source_path.suffix.lower() not in SUPPORTED_IMAGE_SUFFIXES or not source_path.exists():
|
||||
return None
|
||||
stat = source_path.stat()
|
||||
thumbnail_path = self._thumbnail_path(int(row["artwork_id"]), int(row["page_index"]), source_path)
|
||||
|
||||
cached = library.connection.execute(
|
||||
"""
|
||||
SELECT thumbnail_path, width, height, source_mtime, source_size
|
||||
FROM thumbnail_cache
|
||||
WHERE artwork_id = ? AND page_index = ?
|
||||
""",
|
||||
(int(row["artwork_id"]), int(row["page_index"])),
|
||||
).fetchone()
|
||||
if (
|
||||
cached
|
||||
and Path(str(cached["thumbnail_path"])).exists()
|
||||
and float(cached["source_mtime"]) == stat.st_mtime
|
||||
and int(cached["source_size"]) == stat.st_size
|
||||
):
|
||||
return ThumbnailResult(
|
||||
artwork_id=int(row["artwork_id"]),
|
||||
page_index=int(row["page_index"]),
|
||||
source_path=str(source_path),
|
||||
thumbnail_path=str(cached["thumbnail_path"]),
|
||||
width=int(cached["width"]),
|
||||
height=int(cached["height"]),
|
||||
generated=False,
|
||||
)
|
||||
|
||||
width, height = self._generate(source_path, thumbnail_path)
|
||||
library.upsert_thumbnail(
|
||||
artwork_id=int(row["artwork_id"]),
|
||||
page_index=int(row["page_index"]),
|
||||
source_path=str(source_path),
|
||||
thumbnail_path=str(thumbnail_path),
|
||||
width=width,
|
||||
height=height,
|
||||
source_mtime=stat.st_mtime,
|
||||
source_size=stat.st_size,
|
||||
)
|
||||
return ThumbnailResult(
|
||||
artwork_id=int(row["artwork_id"]),
|
||||
page_index=int(row["page_index"]),
|
||||
source_path=str(source_path),
|
||||
thumbnail_path=str(thumbnail_path),
|
||||
width=width,
|
||||
height=height,
|
||||
generated=True,
|
||||
)
|
||||
|
||||
def _generate(self, source_path: Path, thumbnail_path: Path) -> tuple[int, int]:
|
||||
try:
|
||||
from PIL import Image, ImageOps
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("Pillow is required to generate thumbnails. Install it with: pip install Pillow") from exc
|
||||
|
||||
thumbnail_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with Image.open(source_path) as image:
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image.thumbnail(self.size, Image.Resampling.LANCZOS)
|
||||
if image.mode not in {"RGB", "L"}:
|
||||
image = image.convert("RGB")
|
||||
image.save(thumbnail_path, format="JPEG", quality=self.quality, optimize=True)
|
||||
return image.size
|
||||
|
||||
def _thumbnail_path(self, artwork_id: int, page_index: int, source_path: Path) -> Path:
|
||||
bucket = str(abs(artwork_id) % 1000).zfill(3)
|
||||
return self.cache_dir / bucket / f"{artwork_id}_{page_index}_{source_path.stem}.jpg"
|
||||
@@ -0,0 +1,405 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.cookie_import import (
|
||||
cookie_summary,
|
||||
cookies_to_playwright,
|
||||
parse_cookie_text,
|
||||
save_playwright_cookies,
|
||||
)
|
||||
from pixiv_app.core.paths import (
|
||||
app_session_file,
|
||||
downloads_root,
|
||||
plugin_roots,
|
||||
plugins_root as app_plugins_root,
|
||||
runtime_path,
|
||||
)
|
||||
from pixiv_app.core.plugin.base import Resource
|
||||
from pixiv_app.core.plugin.manager import PluginManager
|
||||
|
||||
|
||||
def project_root() -> Path:
|
||||
return app_plugins_root().parent
|
||||
|
||||
|
||||
class XMediaService:
|
||||
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
|
||||
VIDEO_EXTS = {".mp4", ".mov", ".webm", ".m4v"}
|
||||
|
||||
def __init__(self, *, plugins_root: str | Path | None = None, save_root: str | Path | None = None) -> None:
|
||||
self.plugins_root = Path(plugins_root) if plugins_root else app_plugins_root()
|
||||
self.save_root = Path(save_root) if save_root else downloads_root() / "x"
|
||||
self.cookie_path = runtime_path("x_cookies.json")
|
||||
self.follow_store_path = runtime_path("x_followed_authors.json")
|
||||
self._manager: PluginManager | None = None
|
||||
self._preview_cache: dict[str, list[Resource]] = {}
|
||||
|
||||
def import_cookie_text(self, text: str) -> dict[str, Any]:
|
||||
cookies = parse_cookie_text(text)
|
||||
x_cookies = cookies_to_playwright(cookies, ("x.com", "twitter.com"))
|
||||
path = save_playwright_cookies(x_cookies, self.cookie_path)
|
||||
plugin = self._get_plugin()
|
||||
if hasattr(plugin, "reset_playwright_session"):
|
||||
plugin.reset_playwright_session()
|
||||
return {"count": len(x_cookies), "path": str(path), "summary": cookie_summary(x_cookies)}
|
||||
|
||||
def preview_author_media(self, username: str, *, media_type: str = "all", max_items: int = 120) -> dict[str, Any]:
|
||||
handle = _normalize_username(username)
|
||||
media_type = _normalize_media_type(media_type)
|
||||
max_items = max(1, min(int(max_items), 500))
|
||||
url = f"https://x.com/{handle}/media"
|
||||
|
||||
plugin = self._get_plugin()
|
||||
if not plugin.validate():
|
||||
raise ValueError("X plugin is unavailable. Install playwright/httpx and run playwright install chromium first.")
|
||||
|
||||
resources = plugin.parse(url)
|
||||
resources = _filter_resources(resources, media_type)[:max_items]
|
||||
preview_id = uuid.uuid4().hex
|
||||
self._preview_cache[preview_id] = resources
|
||||
return {
|
||||
"preview_id": preview_id,
|
||||
"username": handle,
|
||||
"media_type": media_type,
|
||||
"count": len(resources),
|
||||
"items": [_resource_to_json(item, i) for i, item in enumerate(resources)],
|
||||
"created_at": int(time.time()),
|
||||
}
|
||||
|
||||
def download_preview(self, preview_id: str, *, selected_ids: list[str] | None = None) -> dict[str, Any]:
|
||||
resources = self._preview_cache.get(preview_id)
|
||||
if resources is None:
|
||||
raise ValueError("Preview expired. Re-run the X media preview.")
|
||||
selected = set(selected_ids or [])
|
||||
if selected:
|
||||
resources = [item for item in resources if item.id in selected]
|
||||
if not resources:
|
||||
raise ValueError("No X media items selected.")
|
||||
|
||||
plugin = self._get_plugin()
|
||||
self.save_root.mkdir(parents=True, exist_ok=True)
|
||||
results: list[dict[str, Any]] = []
|
||||
downloaded = 0
|
||||
failed = 0
|
||||
for resource in resources:
|
||||
try:
|
||||
paths = plugin.download(resource, self.save_root)
|
||||
downloaded += len(paths)
|
||||
results.append(
|
||||
{
|
||||
"id": resource.id,
|
||||
"ok": bool(paths),
|
||||
"paths": [str(path) for path in paths],
|
||||
"title": resource.title,
|
||||
"synced_files": 0,
|
||||
}
|
||||
)
|
||||
if not paths:
|
||||
failed += 1
|
||||
except Exception as exc: # Keep batch downloads moving.
|
||||
failed += 1
|
||||
results.append({"id": resource.id, "ok": False, "error": str(exc), "title": resource.title})
|
||||
|
||||
return {
|
||||
"requested": len(resources),
|
||||
"downloaded_files": downloaded,
|
||||
"synced_files": 0,
|
||||
"failed_items": failed,
|
||||
"items": results,
|
||||
}
|
||||
|
||||
def list_local_media(self, *, limit: int = 160, offset: int = 0) -> dict[str, Any]:
|
||||
self.save_root.mkdir(parents=True, exist_ok=True)
|
||||
limit = max(1, min(int(limit), 500))
|
||||
offset = max(0, int(offset))
|
||||
files = [
|
||||
path
|
||||
for path in self.save_root.rglob("*")
|
||||
if path.is_file() and path.suffix.lower() in self.IMAGE_EXTS | self.VIDEO_EXTS
|
||||
]
|
||||
files.sort(key=lambda path: path.stat().st_mtime, reverse=True)
|
||||
page = files[offset : offset + limit]
|
||||
return {
|
||||
"total": len(files),
|
||||
"items": [self._local_media_to_json(path) for path in page],
|
||||
}
|
||||
|
||||
def delete_local_media(self, media_ids: list[str]) -> dict[str, Any]:
|
||||
deleted: list[str] = []
|
||||
failed: list[dict[str, str]] = []
|
||||
for media_id in media_ids:
|
||||
try:
|
||||
path = self.resolve_local_media(media_id)
|
||||
path.unlink()
|
||||
deleted.append(media_id)
|
||||
except Exception as exc:
|
||||
failed.append({"id": media_id, "error": str(exc)})
|
||||
return {"deleted": deleted, "failed": failed}
|
||||
|
||||
def resolve_local_media(self, media_id: str) -> Path:
|
||||
rel_text = _decode_media_id(media_id)
|
||||
target = (self.save_root / rel_text).resolve()
|
||||
root = self.save_root.resolve()
|
||||
if root != target and root not in target.parents:
|
||||
raise ValueError("Illegal media path.")
|
||||
if not target.is_file():
|
||||
raise ValueError("Local media file does not exist.")
|
||||
return target
|
||||
|
||||
def list_followed_authors(self) -> list[dict[str, Any]]:
|
||||
rows = [item for item in self._load_follow_rows() if bool(item.get("enabled", True))]
|
||||
rows.sort(key=lambda item: ((str(item.get("creator_name") or "") or str(item.get("username") or "")).lower()))
|
||||
return rows
|
||||
|
||||
def set_followed_author(self, username: str, *, followed: bool = True, creator_name: str = "") -> dict[str, Any]:
|
||||
handle = _normalize_username(username)
|
||||
rows = self._load_follow_rows()
|
||||
row = next((item for item in rows if str(item.get("username", "")).lower() == handle.lower()), None)
|
||||
now = _now_text()
|
||||
if row is None:
|
||||
row = {
|
||||
"username": handle,
|
||||
"creator_name": creator_name.strip() or handle,
|
||||
"enabled": True,
|
||||
"profile_url": f"https://x.com/{handle}",
|
||||
"latest_tweet_id": "",
|
||||
"latest_published_at": "",
|
||||
"last_checked_at": "",
|
||||
"last_error": "",
|
||||
"seen_tweet_ids": [],
|
||||
"created_at": now,
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
row["username"] = handle
|
||||
row["creator_name"] = creator_name.strip() or str(row.get("creator_name") or handle)
|
||||
row["enabled"] = bool(followed)
|
||||
row["profile_url"] = f"https://x.com/{handle}"
|
||||
row["updated_at"] = now
|
||||
if not followed:
|
||||
row["last_error"] = ""
|
||||
|
||||
self._save_follow_rows(rows)
|
||||
return dict(row)
|
||||
|
||||
def sync_followed_authors(self, *, download: bool = True, max_new_per_author: int = 20) -> dict[str, Any]:
|
||||
followed = self.list_followed_authors()
|
||||
if not followed:
|
||||
return {"authors": []}
|
||||
|
||||
plugin = self._get_plugin()
|
||||
if not plugin.validate():
|
||||
raise ValueError("X plugin is unavailable. Import a valid X cookie and ensure Playwright is installed.")
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
rows = self._load_follow_rows()
|
||||
|
||||
for row in rows:
|
||||
if not bool(row.get("enabled", True)):
|
||||
continue
|
||||
|
||||
handle = str(row.get("username") or "").strip()
|
||||
if not handle:
|
||||
continue
|
||||
|
||||
try:
|
||||
parse_for_sync = getattr(plugin, "parse_for_sync", plugin.parse)
|
||||
resources = parse_for_sync(f"https://x.com/{handle}")
|
||||
grouped = _group_resources_by_tweet(resources)
|
||||
seen_ids = [str(item) for item in row.get("seen_tweet_ids", []) if str(item).strip()]
|
||||
seen_set = set(seen_ids)
|
||||
new_groups = [item for item in grouped if item["tweet_id"] and item["tweet_id"] not in seen_set]
|
||||
new_groups = new_groups[: max(0, int(max_new_per_author))]
|
||||
|
||||
downloaded_files = 0
|
||||
synced_files = 0
|
||||
created_alerts = 0
|
||||
new_ids: list[str] = []
|
||||
|
||||
for group in new_groups:
|
||||
representative = group["resource"]
|
||||
new_ids.append(group["tweet_id"])
|
||||
|
||||
if download:
|
||||
for resource in group["resources"]:
|
||||
paths = plugin.download(resource, self.save_root)
|
||||
downloaded_files += len(paths)
|
||||
|
||||
latest_group = grouped[0] if grouped else None
|
||||
if latest_group is not None:
|
||||
representative = latest_group["resource"]
|
||||
row["creator_name"] = representative.author or str(row.get("creator_name") or handle)
|
||||
row["latest_tweet_id"] = latest_group["tweet_id"]
|
||||
row["latest_published_at"] = representative.created_at or str(row.get("latest_published_at") or "")
|
||||
|
||||
row["last_checked_at"] = _now_text()
|
||||
row["last_error"] = ""
|
||||
row["updated_at"] = _now_text()
|
||||
row["seen_tweet_ids"] = _merge_seen_ids(new_ids, seen_ids)
|
||||
|
||||
results.append(
|
||||
{
|
||||
"username": handle,
|
||||
"creator_name": row.get("creator_name") or handle,
|
||||
"new_posts": len(new_groups),
|
||||
"downloaded_files": downloaded_files,
|
||||
"synced_files": synced_files,
|
||||
"alerts": created_alerts,
|
||||
"error": "",
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
row["last_checked_at"] = _now_text()
|
||||
row["last_error"] = str(exc)
|
||||
row["updated_at"] = _now_text()
|
||||
results.append(
|
||||
{
|
||||
"username": handle,
|
||||
"creator_name": row.get("creator_name") or handle,
|
||||
"new_posts": 0,
|
||||
"downloaded_files": 0,
|
||||
"synced_files": 0,
|
||||
"alerts": 0,
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
self._save_follow_rows(rows)
|
||||
return {"authors": results}
|
||||
|
||||
def _get_plugin(self):
|
||||
if self._manager is None:
|
||||
self._manager = PluginManager(plugin_roots() if self.plugins_root == app_plugins_root() else [self.plugins_root])
|
||||
self._manager.load_all()
|
||||
plugin = self._manager.plugins.get("X")
|
||||
if plugin is None:
|
||||
raise ValueError("X plugin is missing. Confirm plugins/x exists.")
|
||||
return plugin
|
||||
|
||||
def _load_follow_rows(self) -> list[dict[str, Any]]:
|
||||
if not self.follow_store_path.is_file():
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(self.follow_store_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return []
|
||||
if isinstance(payload, list):
|
||||
return [dict(item) for item in payload if isinstance(item, dict)]
|
||||
return []
|
||||
|
||||
def _save_follow_rows(self, rows: list[dict[str, Any]]) -> None:
|
||||
self.follow_store_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.follow_store_path.write_text(
|
||||
json.dumps(rows, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def _session_payload(self) -> dict[str, Any]:
|
||||
return SessionStore(app_session_file()).load()
|
||||
|
||||
def _local_media_to_json(self, path: Path) -> dict[str, Any]:
|
||||
rel = path.relative_to(self.save_root)
|
||||
stat = path.stat()
|
||||
kind = "video" if path.suffix.lower() in self.VIDEO_EXTS else "image"
|
||||
media_id = _encode_media_id(str(rel).replace("\\", "/"))
|
||||
return {
|
||||
"id": media_id,
|
||||
"name": path.name,
|
||||
"relative_path": str(rel).replace("\\", "/"),
|
||||
"kind": kind,
|
||||
"size": stat.st_size,
|
||||
"modified_at": int(stat.st_mtime),
|
||||
"url": f"/media/x-local/{media_id}",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_username(username: str) -> str:
|
||||
value = username.strip()
|
||||
if value.startswith("@"):
|
||||
value = value[1:]
|
||||
if "/" in value:
|
||||
parts = [item for item in value.replace("https://", "").replace("http://", "").split("/") if item]
|
||||
value = parts[1] if parts and parts[0].lower() in {"x.com", "twitter.com", "www.x.com", "www.twitter.com"} else parts[0]
|
||||
if not value:
|
||||
raise ValueError("Please enter an X username.")
|
||||
if len(value) > 15 or not value.replace("_", "").isalnum():
|
||||
raise ValueError("Invalid X username format.")
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_media_type(media_type: str) -> str:
|
||||
value = (media_type or "all").strip().lower()
|
||||
if value not in {"all", "image", "video"}:
|
||||
raise ValueError("media_type must be all, image, or video.")
|
||||
return value
|
||||
|
||||
|
||||
def _filter_resources(resources: list[Resource], media_type: str) -> list[Resource]:
|
||||
if media_type == "all":
|
||||
return resources
|
||||
return [item for item in resources if any(file.get("type", "image") == media_type for file in item.files)]
|
||||
|
||||
|
||||
def _resource_to_json(resource: Resource, index: int) -> dict[str, Any]:
|
||||
payload = asdict(resource)
|
||||
payload["index"] = index
|
||||
payload["kind"] = "video" if any(file.get("type") == "video" for file in resource.files) else "image"
|
||||
payload["file_count"] = len(resource.files)
|
||||
return payload
|
||||
|
||||
|
||||
def _group_resources_by_tweet(resources: list[Resource]) -> list[dict[str, Any]]:
|
||||
grouped: dict[str, dict[str, Any]] = {}
|
||||
ordered: list[str] = []
|
||||
for resource in resources:
|
||||
tweet_id = str(resource.metadata.get("tweet_id") or resource.id or "")
|
||||
key = tweet_id or resource.id
|
||||
if key not in grouped:
|
||||
grouped[key] = {
|
||||
"tweet_id": tweet_id,
|
||||
"resource": resource,
|
||||
"resources": [resource],
|
||||
}
|
||||
ordered.append(key)
|
||||
else:
|
||||
grouped[key]["resources"].append(resource)
|
||||
return [grouped[key] for key in ordered]
|
||||
|
||||
|
||||
def _merge_seen_ids(new_ids: list[str], existing_ids: list[str], *, limit: int = 200) -> list[str]:
|
||||
merged: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in [*new_ids, *existing_ids]:
|
||||
value = str(item).strip()
|
||||
if not value or value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
merged.append(value)
|
||||
if len(merged) >= limit:
|
||||
break
|
||||
return merged
|
||||
|
||||
|
||||
def _encode_media_id(value: str) -> str:
|
||||
from base64 import urlsafe_b64encode
|
||||
|
||||
return urlsafe_b64encode(value.encode("utf-8")).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def _decode_media_id(value: str) -> str:
|
||||
from base64 import urlsafe_b64decode
|
||||
|
||||
padding = "=" * (-len(value) % 4)
|
||||
return urlsafe_b64decode((value + padding).encode("ascii")).decode("utf-8")
|
||||
|
||||
|
||||
def _now_text() -> str:
|
||||
return datetime.now().isoformat(timespec="seconds")
|
||||
@@ -0,0 +1 @@
|
||||
"""GUI layer for Sakura downloader."""
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,343 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from tkinter import filedialog, messagebox
|
||||
|
||||
import customtkinter as ctk
|
||||
|
||||
from pixiv_app.core.paths import plugin_roots, user_plugins_root
|
||||
from pixiv_app.core.plugin.base import BasePlugin
|
||||
from pixiv_app.core.plugin.generator import PluginGenerator
|
||||
from pixiv_app.core.plugin.manager import PluginManager
|
||||
|
||||
|
||||
def _slugify_plugin_name(value: str) -> str:
|
||||
text = re.sub(r"[^0-9A-Za-z_\-\u4e00-\u9fff]+", "_", value.strip())
|
||||
text = text.strip("._-")
|
||||
return text or "imported_plugin"
|
||||
|
||||
|
||||
class PluginPanel(ctk.CTkFrame):
|
||||
"""Generic plugin list, import, reload, and template tools."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
plugin_manager: PluginManager,
|
||||
*,
|
||||
user_root: Path | None = None,
|
||||
text_color: str = "#1e3a5f",
|
||||
muted_color: str = "#6f8dac",
|
||||
border_color: str = "#d6e8fb",
|
||||
danger_color: str = "#eb7d7d",
|
||||
) -> None:
|
||||
super().__init__(parent, fg_color="transparent")
|
||||
self.manager = plugin_manager
|
||||
self.user_root = user_root or user_plugins_root()
|
||||
self._text_color = text_color
|
||||
self._muted_color = muted_color
|
||||
self._border_color = border_color
|
||||
self._danger_color = danger_color
|
||||
|
||||
title_frame = ctk.CTkFrame(self, fg_color="transparent")
|
||||
title_frame.pack(fill="x", padx=12, pady=(10, 6))
|
||||
ctk.CTkLabel(
|
||||
title_frame,
|
||||
text="插件管理",
|
||||
font=ctk.CTkFont(size=18, weight="bold"),
|
||||
text_color=text_color,
|
||||
).pack(side="left")
|
||||
|
||||
btn_row = ctk.CTkFrame(title_frame, fg_color="transparent")
|
||||
btn_row.pack(side="right")
|
||||
ctk.CTkButton(btn_row, text="导入 .py", width=88, command=self.import_python_plugin).pack(side="left", padx=4)
|
||||
ctk.CTkButton(btn_row, text="生成模板", width=88, command=self.open_create_dialog).pack(side="left", padx=4)
|
||||
ctk.CTkButton(btn_row, text="重新加载", width=88, command=self.reload_all).pack(side="left", padx=4)
|
||||
|
||||
hint = (
|
||||
"导入一个 Python 插件文件即可扩展站点;插件需导出 plugin_class,"
|
||||
"并继承 pixiv_app.core.plugin.base.BasePlugin。"
|
||||
)
|
||||
ctk.CTkLabel(
|
||||
self,
|
||||
text=hint,
|
||||
text_color=muted_color,
|
||||
font=ctk.CTkFont(size=12),
|
||||
wraplength=760,
|
||||
justify="left",
|
||||
).pack(anchor="w", padx=14, pady=(0, 8))
|
||||
|
||||
self.summary_var = ctk.StringVar(value="正在读取插件状态...")
|
||||
ctk.CTkLabel(
|
||||
self,
|
||||
textvariable=self.summary_var,
|
||||
text_color=text_color,
|
||||
font=ctk.CTkFont(size=12, weight="bold"),
|
||||
anchor="w",
|
||||
).pack(fill="x", padx=14, pady=(0, 8))
|
||||
|
||||
path_row = ctk.CTkFrame(self, fg_color="transparent")
|
||||
path_row.pack(fill="x", padx=12, pady=(0, 8))
|
||||
ctk.CTkLabel(path_row, text="用户插件目录:", text_color=text_color, font=ctk.CTkFont(size=12, weight="bold")).pack(
|
||||
side="left"
|
||||
)
|
||||
ctk.CTkLabel(path_row, text=str(self.user_root), text_color=muted_color, font=ctk.CTkFont(size=11)).pack(
|
||||
side="left", fill="x", expand=True
|
||||
)
|
||||
|
||||
self.container = ctk.CTkScrollableFrame(self, fg_color="transparent")
|
||||
self.container.pack(fill="both", expand=True, padx=8, pady=(0, 8))
|
||||
|
||||
self.error_container = ctk.CTkFrame(self, fg_color="transparent")
|
||||
self.error_container.pack(fill="x", padx=8, pady=(0, 8))
|
||||
|
||||
self.log_text = ctk.CTkTextbox(self, height=120, font=ctk.CTkFont(family="Consolas", size=11))
|
||||
self.log_text.pack(fill="x", padx=12, pady=(0, 12))
|
||||
|
||||
self.refresh()
|
||||
|
||||
def log(self, message: str) -> None:
|
||||
stamp = datetime.now().strftime("%H:%M:%S")
|
||||
self.log_text.insert("end", f"[{stamp}] {message}\n")
|
||||
self.log_text.see("end")
|
||||
|
||||
def refresh(self) -> None:
|
||||
for child in self.container.winfo_children():
|
||||
child.destroy()
|
||||
for child in self.error_container.winfo_children():
|
||||
child.destroy()
|
||||
if not self.manager.plugins:
|
||||
ctk.CTkLabel(
|
||||
self.container,
|
||||
text="没有发现插件。请点击「导入 .py」或「生成模板」。",
|
||||
text_color=self._muted_color,
|
||||
justify="left",
|
||||
).pack(anchor="w", pady=8)
|
||||
for name, plugin in sorted(self.manager.plugins.items(), key=lambda x: x[0].lower()):
|
||||
self._add_plugin_card(name, plugin)
|
||||
ready_count = sum(1 for plugin in self.manager.plugins.values() if plugin.validate())
|
||||
self.summary_var.set(
|
||||
f"已加载 {len(self.manager.plugins)} 个插件 · 可用 {ready_count} 个 · "
|
||||
f"加载失败 {len(self.manager.load_errors)} 个"
|
||||
)
|
||||
self._add_error_cards()
|
||||
|
||||
def _add_error_cards(self) -> None:
|
||||
if not self.manager.load_errors:
|
||||
return
|
||||
title = ctk.CTkLabel(
|
||||
self.error_container,
|
||||
text="加载失败的插件",
|
||||
font=ctk.CTkFont(size=14, weight="bold"),
|
||||
text_color=self._danger_color,
|
||||
)
|
||||
title.pack(anchor="w", padx=4, pady=(8, 4))
|
||||
for path, error in sorted(self.manager.load_errors.items(), key=lambda x: str(x[0]).lower()):
|
||||
card = ctk.CTkFrame(self.error_container, corner_radius=10, border_width=1, border_color=self._danger_color)
|
||||
card.pack(fill="x", pady=4)
|
||||
ctk.CTkLabel(
|
||||
card,
|
||||
text=str(path),
|
||||
font=ctk.CTkFont(size=11),
|
||||
text_color=self._text_color,
|
||||
wraplength=760,
|
||||
justify="left",
|
||||
).pack(anchor="w", padx=10, pady=(8, 2))
|
||||
ctk.CTkLabel(
|
||||
card,
|
||||
text=error,
|
||||
font=ctk.CTkFont(size=11),
|
||||
text_color=self._danger_color,
|
||||
wraplength=760,
|
||||
justify="left",
|
||||
).pack(anchor="w", padx=10, pady=(0, 8))
|
||||
|
||||
def _add_plugin_card(self, name: str, plugin: BasePlugin) -> None:
|
||||
card = ctk.CTkFrame(self.container, corner_radius=12, border_width=1, border_color=self._border_color)
|
||||
card.pack(fill="x", pady=6)
|
||||
|
||||
top = ctk.CTkFrame(card, fg_color="transparent")
|
||||
top.pack(fill="x", padx=12, pady=(10, 4))
|
||||
ctk.CTkLabel(top, text=plugin.name, font=ctk.CTkFont(size=15, weight="bold"), text_color=self._text_color).pack(
|
||||
side="left"
|
||||
)
|
||||
ctk.CTkLabel(top, text=f"v{plugin.version}", font=ctk.CTkFont(size=12), text_color=self._muted_color).pack(
|
||||
side="left", padx=10
|
||||
)
|
||||
is_ready = plugin.validate()
|
||||
st = "可用" if is_ready else "需配置"
|
||||
ctk.CTkLabel(
|
||||
top,
|
||||
text=st,
|
||||
font=ctk.CTkFont(size=12, weight="bold"),
|
||||
text_color="#3f9b72" if is_ready else self._danger_color,
|
||||
).pack(side="right")
|
||||
|
||||
path = self.manager.get_plugin_path(name)
|
||||
source = "用户插件" if path and self.user_root in path.parents else "内置插件"
|
||||
ctk.CTkLabel(
|
||||
card,
|
||||
text=f"域名 {plugin.domain} 来源 {source}",
|
||||
font=ctk.CTkFont(size=12),
|
||||
text_color=self._muted_color,
|
||||
).pack(anchor="w", padx=12, pady=(0, 4))
|
||||
if path:
|
||||
ctk.CTkLabel(
|
||||
card,
|
||||
text=str(path),
|
||||
font=ctk.CTkFont(size=11),
|
||||
text_color=self._muted_color,
|
||||
wraplength=760,
|
||||
justify="left",
|
||||
).pack(anchor="w", padx=12, pady=(0, 8))
|
||||
|
||||
actions = ctk.CTkFrame(card, fg_color="transparent")
|
||||
actions.pack(fill="x", padx=12, pady=(0, 10))
|
||||
ctk.CTkButton(actions, text="重载", width=72, command=lambda n=name: self.reload_one(n)).pack(side="left", padx=(0, 6))
|
||||
if path and self.user_root in path.parents:
|
||||
ctk.CTkButton(
|
||||
actions,
|
||||
text="删除文件",
|
||||
width=86,
|
||||
fg_color=self._danger_color,
|
||||
command=lambda n=name: self.delete_user_plugin(n),
|
||||
).pack(side="left")
|
||||
else:
|
||||
ctk.CTkButton(actions, text="从内存卸载", width=92, command=lambda n=name: self.unload_one(n)).pack(side="left")
|
||||
|
||||
def import_python_plugin(self) -> None:
|
||||
path = filedialog.askopenfilename(
|
||||
title="导入 Python 插件",
|
||||
filetypes=[("Python plugin", "*.py"), ("All files", "*.*")],
|
||||
)
|
||||
if not path:
|
||||
return
|
||||
src = Path(path)
|
||||
if not src.is_file():
|
||||
messagebox.showerror("导入失败", "文件不存在。")
|
||||
return
|
||||
|
||||
plugin_dir = self.user_root / _slugify_plugin_name(src.stem)
|
||||
index = 2
|
||||
while plugin_dir.exists() and (plugin_dir / "plugin.py").resolve() != src.resolve():
|
||||
plugin_dir = self.user_root / f"{_slugify_plugin_name(src.stem)}_{index}"
|
||||
index += 1
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
dst = plugin_dir / "plugin.py"
|
||||
|
||||
try:
|
||||
shutil.copy2(src, dst)
|
||||
plugin = self.manager.load_plugin(dst)
|
||||
except Exception as exc:
|
||||
messagebox.showerror("导入失败", str(exc))
|
||||
return
|
||||
if plugin is None:
|
||||
messagebox.showerror("导入失败", "插件未能加载。请确认文件中导出了 plugin_class。")
|
||||
return
|
||||
|
||||
self.refresh()
|
||||
self.log(f"已导入插件: {plugin.name} -> {dst}")
|
||||
messagebox.showinfo("导入完成", f"已导入插件:{plugin.name}\n\n{dst}")
|
||||
|
||||
def reload_all(self) -> None:
|
||||
self.manager.reload_all_from_disk()
|
||||
self.refresh()
|
||||
self.log("已重新加载全部插件。")
|
||||
|
||||
def reload_one(self, name: str) -> None:
|
||||
if self.manager.reload_plugin(name):
|
||||
self.refresh()
|
||||
self.log(f"已重载插件: {name}")
|
||||
else:
|
||||
self.log(f"重载失败: {name}")
|
||||
|
||||
def unload_one(self, name: str) -> None:
|
||||
self.manager.unload_plugin(name)
|
||||
self.refresh()
|
||||
self.log(f"已从内存卸载: {name}")
|
||||
|
||||
def delete_user_plugin(self, name: str) -> None:
|
||||
path = self.manager.get_plugin_path(name)
|
||||
if path is None or self.user_root not in path.parents:
|
||||
messagebox.showinfo("提示", "只能删除用户导入的插件文件。")
|
||||
return
|
||||
if not messagebox.askyesno("确认删除", f"将删除插件目录:\n{path.parent}\n\n是否继续?"):
|
||||
return
|
||||
self.manager.unload_plugin(name)
|
||||
shutil.rmtree(path.parent, ignore_errors=True)
|
||||
self.manager.reload_all_from_disk()
|
||||
self.refresh()
|
||||
self.log(f"已删除用户插件: {name}")
|
||||
|
||||
def open_create_dialog(self) -> None:
|
||||
CreatePluginDialog(self, self.user_root, on_created=lambda: (self.reload_all(), self.refresh()))
|
||||
|
||||
|
||||
class CreatePluginDialog(ctk.CTkToplevel):
|
||||
def __init__(self, parent, plugins_root: Path, *, on_created: Callable[[], None] | None = None) -> None:
|
||||
super().__init__(parent)
|
||||
self.title("生成插件模板")
|
||||
self.geometry("520x420")
|
||||
self.plugins_root = plugins_root
|
||||
self._on_created = on_created
|
||||
self.generator = PluginGenerator()
|
||||
|
||||
pad = {"padx": 16, "pady": 8}
|
||||
ctk.CTkLabel(self, text="名称").pack(anchor="w", **pad)
|
||||
self.name_entry = ctk.CTkEntry(self, placeholder_text="例如 MySite")
|
||||
self.name_entry.pack(fill="x", **pad)
|
||||
|
||||
ctk.CTkLabel(self, text="域名").pack(anchor="w", **pad)
|
||||
self.domain_entry = ctk.CTkEntry(self, placeholder_text="例如 example.com")
|
||||
self.domain_entry.pack(fill="x", **pad)
|
||||
|
||||
ctk.CTkLabel(self, text="作者").pack(anchor="w", **pad)
|
||||
self.author_entry = ctk.CTkEntry(self)
|
||||
self.author_entry.pack(fill="x", **pad)
|
||||
|
||||
ctk.CTkLabel(self, text="描述").pack(anchor="w", **pad)
|
||||
self.desc_text = ctk.CTkTextbox(self, height=100)
|
||||
self.desc_text.pack(fill="x", **pad)
|
||||
|
||||
ctk.CTkButton(self, text="生成到用户插件目录", command=self._generate).pack(pady=16)
|
||||
|
||||
def _generate(self) -> None:
|
||||
data = {
|
||||
"name": self.name_entry.get().strip(),
|
||||
"domain": self.domain_entry.get().strip(),
|
||||
"author": self.author_entry.get().strip(),
|
||||
"description": self.desc_text.get("1.0", "end").strip(),
|
||||
}
|
||||
if not data["name"] or not data["domain"]:
|
||||
messagebox.showerror("校验失败", "请填写名称与域名。")
|
||||
return
|
||||
try:
|
||||
path = self.generator.generate_from_gui(data, output_root=self.plugins_root)
|
||||
except Exception as exc:
|
||||
messagebox.showerror("失败", str(exc))
|
||||
return
|
||||
messagebox.showinfo("完成", f"已生成: {path}")
|
||||
self.destroy()
|
||||
if self._on_created:
|
||||
self._on_created()
|
||||
|
||||
|
||||
class PluginPanelWindow(ctk.CTkToplevel):
|
||||
"""Standalone window hosting PluginPanel."""
|
||||
|
||||
def __init__(self, parent, *, cookie: str = "") -> None:
|
||||
super().__init__(parent)
|
||||
self.title("插件管理")
|
||||
self.geometry("760x620")
|
||||
user_root = user_plugins_root()
|
||||
self.manager = PluginManager(plugin_roots())
|
||||
self.manager.load_all()
|
||||
pixiv = self.manager.plugins.get("Pixiv")
|
||||
if pixiv is not None and hasattr(pixiv, "set_auth"):
|
||||
pixiv.set_auth(cookie)
|
||||
panel = PluginPanel(self, self.manager, user_root=user_root)
|
||||
panel.pack(fill="both", expand=True)
|
||||
@@ -0,0 +1,160 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
import customtkinter as ctk
|
||||
|
||||
|
||||
class ProxyDialog(ctk.CTkToplevel):
|
||||
def __init__(
|
||||
self,
|
||||
master,
|
||||
initial_config: dict,
|
||||
on_save: Callable[[dict], None],
|
||||
on_action: Optional[Callable[[str, dict], str | dict[str, Any]]] = None,
|
||||
):
|
||||
super().__init__(master)
|
||||
self.title("代理池设置")
|
||||
self.geometry("760x700")
|
||||
self.minsize(700, 640)
|
||||
self.on_save = on_save
|
||||
self.on_action = on_action
|
||||
self.initial_config = initial_config
|
||||
|
||||
self.use_proxy_var = ctk.BooleanVar(value=bool(initial_config.get("enabled", False)))
|
||||
self.use_quake_var = ctk.BooleanVar(value=bool(initial_config.get("use_quake", False)))
|
||||
self.quake_mode_var = ctk.StringVar(value=str(initial_config.get("quake_mode", "api_v3")))
|
||||
self.status_var = ctk.StringVar(value="准备就绪")
|
||||
|
||||
self._build_ui()
|
||||
self.transient(master)
|
||||
self.grab_set()
|
||||
|
||||
def _build_ui(self) -> None:
|
||||
container = ctk.CTkFrame(self, fg_color="transparent")
|
||||
container.pack(fill="both", expand=True, padx=16, pady=16)
|
||||
|
||||
ctk.CTkLabel(container, text="代理池设置", font=ctk.CTkFont(size=24, weight="bold")).pack(anchor="w", pady=(0, 6))
|
||||
ctk.CTkLabel(
|
||||
container,
|
||||
text="预校验通过的代理会自动回填到下面的代理列表里,保存后下次可直接使用。",
|
||||
text_color="#6f8dac",
|
||||
font=ctk.CTkFont(size=12),
|
||||
).pack(anchor="w", pady=(0, 12))
|
||||
|
||||
toggle_row = ctk.CTkFrame(container, fg_color="#f7fbff", corner_radius=12)
|
||||
toggle_row.pack(fill="x", pady=(0, 10), padx=2)
|
||||
ctk.CTkSwitch(toggle_row, text="启用代理池", variable=self.use_proxy_var).pack(anchor="w", padx=12, pady=(8, 6))
|
||||
ctk.CTkSwitch(toggle_row, text="启用 Quake 自动爬取", variable=self.use_quake_var).pack(anchor="w", padx=12, pady=(0, 10))
|
||||
|
||||
mode_row = ctk.CTkFrame(container, fg_color="transparent")
|
||||
mode_row.pack(fill="x", pady=(0, 8))
|
||||
ctk.CTkLabel(mode_row, text="Quake 模式:", width=120).pack(side="left")
|
||||
ctk.CTkOptionMenu(mode_row, values=["api_v3", "web_assoc"], variable=self.quake_mode_var, width=220).pack(side="left")
|
||||
|
||||
key_row = ctk.CTkFrame(container, fg_color="transparent")
|
||||
key_row.pack(fill="x", pady=(0, 8))
|
||||
ctk.CTkLabel(key_row, text="Quake API Key:", width=120).pack(side="left")
|
||||
self.api_key_entry = ctk.CTkEntry(key_row, placeholder_text="输入 Quake API Key")
|
||||
self.api_key_entry.pack(side="left", fill="x", expand=True)
|
||||
self.api_key_entry.insert(0, str(self.initial_config.get("quake_api_key", "")))
|
||||
|
||||
cookie_row = ctk.CTkFrame(container, fg_color="transparent")
|
||||
cookie_row.pack(fill="x", pady=(0, 8))
|
||||
ctk.CTkLabel(cookie_row, text="Quake Cookie:", width=120).pack(side="left")
|
||||
self.cookie_entry = ctk.CTkEntry(cookie_row, placeholder_text="输入 Quake Cookie(web_assoc 模式需要)")
|
||||
self.cookie_entry.pack(side="left", fill="x", expand=True)
|
||||
self.cookie_entry.insert(0, str(self.initial_config.get("quake_cookie", "")))
|
||||
|
||||
country_row = ctk.CTkFrame(container, fg_color="transparent")
|
||||
country_row.pack(fill="x", pady=(0, 8))
|
||||
ctk.CTkLabel(country_row, text="海外国家代码:", width=120).pack(side="left")
|
||||
self.country_entry = ctk.CTkEntry(country_row, placeholder_text="US,JP,KR,SG,HK,TW,DE,FR,GB,CA,AU")
|
||||
self.country_entry.pack(side="left", fill="x", expand=True)
|
||||
self.country_entry.insert(0, str(self.initial_config.get("countries_text", "US,JP,KR,SG,HK,TW,DE,FR,GB,CA,AU")))
|
||||
|
||||
tool_row = ctk.CTkFrame(container, fg_color="transparent")
|
||||
tool_row.pack(fill="x", pady=(8, 6))
|
||||
self.manual_input = ctk.CTkEntry(tool_row, placeholder_text="快速添加单个代理,如 1.2.3.4:8080")
|
||||
self.manual_input.pack(side="left", fill="x", expand=True)
|
||||
ctk.CTkButton(tool_row, text="手动添加", width=110, command=self._add_manual_proxy).pack(side="left", padx=(8, 0))
|
||||
self.fetch_button = ctk.CTkButton(tool_row, text="开始爬取并预校验", width=170, command=self._trigger_fetch)
|
||||
self.fetch_button.pack(side="left", padx=(8, 0))
|
||||
|
||||
ctk.CTkLabel(
|
||||
container,
|
||||
text="代理列表(每行一个,支持 ip:port / http://ip:port / https://ip:port)",
|
||||
anchor="w",
|
||||
).pack(fill="x", pady=(8, 0))
|
||||
self.proxy_text = ctk.CTkTextbox(container, height=360)
|
||||
self.proxy_text.pack(fill="both", expand=True, pady=(6, 12))
|
||||
self.proxy_text.insert("1.0", str(self.initial_config.get("manual_text", "")))
|
||||
|
||||
status_row = ctk.CTkFrame(container, fg_color="#f7fbff", corner_radius=12)
|
||||
status_row.pack(fill="x", pady=(0, 10))
|
||||
ctk.CTkLabel(status_row, textvariable=self.status_var, text_color="#365b85", font=ctk.CTkFont(size=12)).pack(
|
||||
anchor="w", padx=12, pady=8
|
||||
)
|
||||
|
||||
button_row = ctk.CTkFrame(container, fg_color="transparent")
|
||||
button_row.pack(fill="x")
|
||||
ctk.CTkButton(button_row, text="取消", command=self.destroy, width=120).pack(side="right", padx=(8, 0))
|
||||
ctk.CTkButton(button_row, text="保存", command=self._save, width=120).pack(side="right")
|
||||
|
||||
def _collect_config(self) -> dict:
|
||||
return {
|
||||
"enabled": bool(self.use_proxy_var.get()),
|
||||
"use_quake": bool(self.use_quake_var.get()),
|
||||
"quake_mode": self.quake_mode_var.get().strip() or "api_v3",
|
||||
"quake_api_key": self.api_key_entry.get().strip(),
|
||||
"quake_cookie": self.cookie_entry.get().strip(),
|
||||
"manual_text": self.proxy_text.get("1.0", "end").strip(),
|
||||
"countries_text": self.country_entry.get().strip(),
|
||||
}
|
||||
|
||||
def _add_manual_proxy(self) -> None:
|
||||
value = self.manual_input.get().strip()
|
||||
if not value:
|
||||
self.status_var.set("请输入代理地址后再添加。")
|
||||
return
|
||||
old_text = self.proxy_text.get("1.0", "end").strip()
|
||||
new_text = f"{old_text}\n{value}".strip()
|
||||
self.proxy_text.delete("1.0", "end")
|
||||
self.proxy_text.insert("1.0", new_text)
|
||||
self.manual_input.delete(0, "end")
|
||||
self.status_var.set("已加入手动代理列表。")
|
||||
|
||||
def _trigger_fetch(self) -> None:
|
||||
if not self.on_action:
|
||||
self.status_var.set("当前版本未启用预校验动作。")
|
||||
return
|
||||
self.fetch_button.configure(state="disabled", text="处理中...")
|
||||
self.status_var.set("正在爬取并校验代理,请稍候...")
|
||||
|
||||
config = self._collect_config()
|
||||
|
||||
def worker() -> None:
|
||||
try:
|
||||
result = self.on_action("fetch_preview", config)
|
||||
except Exception as exc:
|
||||
result = {"status": f"预校验失败: {exc}"}
|
||||
self.after(0, lambda: self._finish_fetch(result))
|
||||
|
||||
threading.Thread(target=worker, daemon=True).start()
|
||||
|
||||
def _finish_fetch(self, result: str | dict[str, Any]) -> None:
|
||||
self.fetch_button.configure(state="normal", text="开始爬取并预校验")
|
||||
if isinstance(result, dict):
|
||||
manual_text = result.get("manual_text")
|
||||
if isinstance(manual_text, str):
|
||||
self.proxy_text.delete("1.0", "end")
|
||||
self.proxy_text.insert("1.0", manual_text)
|
||||
status = str(result.get("status", "预校验完成"))
|
||||
self.status_var.set(status)
|
||||
return
|
||||
self.status_var.set(str(result))
|
||||
|
||||
def _save(self) -> None:
|
||||
self.on_save(self._collect_config())
|
||||
self.destroy()
|
||||
@@ -0,0 +1 @@
|
||||
"""Local service entry points for Sakura downloader."""
|
||||
@@ -0,0 +1,500 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import mimetypes
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import asdict
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from pixiv_app.core.auth import SessionStore
|
||||
from pixiv_app.core.cookie_import import cookie_summary, cookies_to_header, cookies_to_playwright, parse_cookie_text
|
||||
from pixiv_app.core.gallery_service import LocalGalleryService
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB
|
||||
from pixiv_app.core.origin_follow_sync import OriginFollowSyncService
|
||||
from pixiv_app.core.paths import app_session_file, downloads_root, plugin_roots, webui_root
|
||||
from pixiv_app.core.plugin.manager import PluginManager
|
||||
from pixiv_app.core.thumbnails import DEFAULT_THUMBNAIL_DIR
|
||||
from pixiv_app.core.x_media_service import XMediaService
|
||||
|
||||
|
||||
class GalleryApiServer:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
host: str = "127.0.0.1",
|
||||
port: int = 8765,
|
||||
db_path: str | Path = DEFAULT_LIBRARY_DB,
|
||||
thumbnail_dir: str | Path = DEFAULT_THUMBNAIL_DIR,
|
||||
web_root: str | Path | None = None,
|
||||
) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.service = LocalGalleryService(db_path=db_path, thumbnail_dir=thumbnail_dir)
|
||||
self.x_media_service = XMediaService()
|
||||
self.origin_follow_sync_service = OriginFollowSyncService(
|
||||
db_path=str(db_path),
|
||||
gallery_service=self.service,
|
||||
x_media_service=self.x_media_service,
|
||||
)
|
||||
self.plugin_manager = PluginManager(plugin_roots())
|
||||
self.plugin_manager.load_all()
|
||||
self.plugin_preview_cache: dict[str, tuple[str, list]] = {}
|
||||
self.web_root = Path(web_root) if web_root else webui_root()
|
||||
|
||||
def serve_forever(self) -> None:
|
||||
service = self.service
|
||||
x_media_service = self.x_media_service
|
||||
origin_follow_sync_service = self.origin_follow_sync_service
|
||||
plugin_manager = self.plugin_manager
|
||||
plugin_preview_cache = self.plugin_preview_cache
|
||||
web_root = self.web_root
|
||||
|
||||
Handler = type(
|
||||
"GalleryApiHandlerForServer",
|
||||
(GalleryApiHandler,),
|
||||
{
|
||||
"gallery_service": service,
|
||||
"x_media_service": x_media_service,
|
||||
"origin_follow_sync_service": origin_follow_sync_service,
|
||||
"plugin_manager": plugin_manager,
|
||||
"plugin_preview_cache": plugin_preview_cache,
|
||||
"web_root": web_root,
|
||||
},
|
||||
)
|
||||
|
||||
server = ThreadingHTTPServer((self.host, self.port), Handler)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
server.server_close()
|
||||
|
||||
|
||||
class GalleryApiHandler(BaseHTTPRequestHandler):
|
||||
gallery_service: LocalGalleryService
|
||||
x_media_service: XMediaService
|
||||
origin_follow_sync_service: OriginFollowSyncService
|
||||
plugin_manager: PluginManager
|
||||
plugin_preview_cache: dict[str, tuple[str, list]]
|
||||
web_root: Path
|
||||
|
||||
def do_GET(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
query = parse_qs(parsed.query)
|
||||
try:
|
||||
if parsed.path in {"/", "/index.html"}:
|
||||
self._send_static_file(self.web_root / "index.html")
|
||||
elif parsed.path.startswith("/assets/"):
|
||||
self._send_static_file(self.web_root / parsed.path.removeprefix("/assets/"))
|
||||
elif parsed.path == "/api/health":
|
||||
self._send_json({"status": "ok"})
|
||||
elif parsed.path == "/api/session/status":
|
||||
self._send_json(self._handle_session_status())
|
||||
elif parsed.path == "/api/plugins/pages":
|
||||
self._send_json(self._handle_plugin_pages())
|
||||
elif parsed.path == "/api/artworks":
|
||||
self._send_json(self._handle_artworks(query))
|
||||
elif parsed.path == "/api/favorites":
|
||||
self._send_json(self.gallery_service.list_favorites(limit=_int(query, "limit", 60), offset=_int(query, "offset", 0)))
|
||||
elif parsed.path.startswith("/api/artworks/"):
|
||||
self._send_json(self._handle_artwork_detail(parsed.path))
|
||||
elif parsed.path == "/api/tags":
|
||||
self._send_json(self.gallery_service.list_tags(query=_first(query, "q"), limit=_int(query, "limit", 100)))
|
||||
elif parsed.path == "/api/artists":
|
||||
self._send_json(self.gallery_service.list_artists(query=_first(query, "q"), limit=_int(query, "limit", 100)))
|
||||
elif parsed.path.startswith("/api/artists/"):
|
||||
self._send_json(self._handle_artist_detail(parsed.path, query))
|
||||
elif parsed.path == "/api/followed-artists":
|
||||
self._send_json(self.gallery_service.list_followed_artists())
|
||||
elif parsed.path == "/api/thumbnails/build":
|
||||
self._send_json({"items": self.gallery_service.build_missing_thumbnails(limit=_int(query, "limit", 200))})
|
||||
elif parsed.path == "/api/import/legacy":
|
||||
self._send_json(self.gallery_service.scan_legacy_downloads(root=_first(query, "root", "Sakura_Downloads")))
|
||||
elif parsed.path == "/api/sync/followed":
|
||||
self._send_json(
|
||||
self.gallery_service.sync_followed_artists(
|
||||
max_new_per_artist=_int(query, "max_new", 20),
|
||||
download=_bool(query, "download", True),
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/x/local-media":
|
||||
self._send_json(
|
||||
self.x_media_service.list_local_media(
|
||||
limit=_int(query, "limit", 160),
|
||||
offset=_int(query, "offset", 0),
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/x/follows":
|
||||
self._send_json(self.x_media_service.list_followed_authors())
|
||||
elif parsed.path.startswith("/media/thumbnail/"):
|
||||
self._send_file(self._resolve_media_path(parsed.path, thumbnail=True))
|
||||
elif parsed.path.startswith("/media/source/"):
|
||||
self._send_file(self._resolve_media_path(parsed.path, thumbnail=False))
|
||||
elif parsed.path.startswith("/media/x-local/"):
|
||||
self._send_file(self.x_media_service.resolve_local_media(parsed.path.rsplit("/", 1)[-1]))
|
||||
else:
|
||||
self._send_json({"error": "not found"}, status=404)
|
||||
except ValueError as exc:
|
||||
self._send_json({"error": str(exc)}, status=400)
|
||||
except Exception as exc:
|
||||
self._send_json({"error": str(exc)}, status=500)
|
||||
|
||||
def do_POST(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
try:
|
||||
payload = self._read_json_body()
|
||||
if parsed.path == "/api/favorites":
|
||||
self._send_json(
|
||||
self.gallery_service.set_favorite(
|
||||
int(payload["artwork_id"]),
|
||||
bool(payload.get("favorite", True)),
|
||||
str(payload.get("note", "")),
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/followed-artists":
|
||||
self._send_json(
|
||||
self.gallery_service.set_followed_artist(
|
||||
payload.get("artist_id", ""),
|
||||
bool(payload.get("followed", True)),
|
||||
str(payload.get("name", "")),
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/import/legacy":
|
||||
self._send_json(self.gallery_service.scan_legacy_downloads(root=str(payload.get("root", "Sakura_Downloads"))))
|
||||
elif parsed.path == "/api/sync/followed":
|
||||
self._send_json(
|
||||
self.gallery_service.sync_followed_artists(
|
||||
cookie=str(payload.get("cookie", "")),
|
||||
max_new_per_artist=int(payload.get("max_new", 20)),
|
||||
download=bool(payload.get("download", True)),
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/cookies/import":
|
||||
self._send_json(self._handle_cookie_import(payload))
|
||||
elif parsed.path == "/api/origin-follows/sync":
|
||||
self._send_json(
|
||||
self.origin_follow_sync_service.sync(
|
||||
str(payload.get("platform", "")),
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/x/media/preview":
|
||||
self._send_json(
|
||||
self.x_media_service.preview_author_media(
|
||||
str(payload.get("username", "")),
|
||||
media_type=str(payload.get("media_type", "all")),
|
||||
max_items=int(payload.get("max_items", 120)),
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/x/media/download":
|
||||
selected_ids = payload.get("selected_ids")
|
||||
if selected_ids is not None and not isinstance(selected_ids, list):
|
||||
raise ValueError("selected_ids 必须是数组。")
|
||||
self._send_json(
|
||||
self.x_media_service.download_preview(
|
||||
str(payload.get("preview_id", "")),
|
||||
selected_ids=[str(item) for item in selected_ids] if selected_ids is not None else None,
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/x/local-media/delete":
|
||||
media_ids = payload.get("ids", [])
|
||||
if not isinstance(media_ids, list):
|
||||
raise ValueError("ids 必须是数组。")
|
||||
self._send_json(self.x_media_service.delete_local_media([str(item) for item in media_ids]))
|
||||
elif parsed.path == "/api/x/follows":
|
||||
self._send_json(
|
||||
self.x_media_service.set_followed_author(
|
||||
str(payload.get("username", "")),
|
||||
followed=bool(payload.get("followed", True)),
|
||||
creator_name=str(payload.get("name", "")),
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/x/follows/sync":
|
||||
self._send_json(
|
||||
self.x_media_service.sync_followed_authors(
|
||||
download=bool(payload.get("download", True)),
|
||||
max_new_per_author=int(payload.get("max_new", 20)),
|
||||
)
|
||||
)
|
||||
elif parsed.path == "/api/xiaohongshu/preview":
|
||||
self._send_json(self._handle_plugin_preview("小红书", payload))
|
||||
elif parsed.path == "/api/xiaohongshu/download":
|
||||
self._send_json(self._handle_plugin_download(payload))
|
||||
else:
|
||||
self._send_json({"error": "not found"}, status=404)
|
||||
except ValueError as exc:
|
||||
self._send_json({"error": str(exc)}, status=400)
|
||||
except Exception as exc:
|
||||
self._send_json({"error": str(exc)}, status=500)
|
||||
|
||||
def do_OPTIONS(self) -> None:
|
||||
self.send_response(204)
|
||||
self._send_cors_headers()
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, format: str, *args) -> None:
|
||||
return None
|
||||
|
||||
def _handle_artworks(self, query: dict[str, list[str]]) -> dict:
|
||||
artist_id = _first(query, "artist_id")
|
||||
return self.gallery_service.list_artworks(
|
||||
query=_first(query, "q"),
|
||||
search_scope=_first(query, "search_scope") or "all",
|
||||
tag=_first(query, "tag"),
|
||||
artist_id=int(artist_id) if artist_id else None,
|
||||
artwork_type=_first(query, "type"),
|
||||
include_restricted=_bool(query, "include_restricted", True),
|
||||
limit=_int(query, "limit", 60),
|
||||
offset=_int(query, "offset", 0),
|
||||
ensure_thumbnails=_bool(query, "ensure_thumbnails", True),
|
||||
)
|
||||
|
||||
def _handle_plugin_pages(self) -> dict:
|
||||
self.plugin_manager.reload_all_from_disk()
|
||||
items = []
|
||||
if "X" in self.plugin_manager.plugins:
|
||||
items.append(
|
||||
{
|
||||
"id": "x-media",
|
||||
"view": "xMedia",
|
||||
"title": "X 作者媒体",
|
||||
"description": "按作者名称解析图片或视频并选择下载",
|
||||
}
|
||||
)
|
||||
if "小红书" in self.plugin_manager.plugins:
|
||||
plugin = self.plugin_manager.plugins["小红书"]
|
||||
items.append(
|
||||
{
|
||||
"id": "xiaohongshu",
|
||||
"view": "xiaohongshu",
|
||||
"title": "小红书",
|
||||
"description": "搜索关键词或解析笔记链接,预览后下载",
|
||||
"available": bool(plugin.validate()),
|
||||
}
|
||||
)
|
||||
return {"items": items}
|
||||
|
||||
def _handle_plugin_preview(self, plugin_name: str, payload: dict) -> dict:
|
||||
plugin = self.plugin_manager.plugins.get(plugin_name)
|
||||
if plugin is None:
|
||||
raise ValueError(f"未安装插件: {plugin_name}")
|
||||
if not plugin.validate():
|
||||
raise ValueError(f"{plugin_name} 插件依赖不可用,请先安装 Playwright。")
|
||||
target = str(payload.get("target", "")).strip()
|
||||
if not target:
|
||||
raise ValueError("请输入解析目标。")
|
||||
limit = max(1, min(int(payload.get("limit", 20)), 50))
|
||||
resources = plugin.parse(target)[:limit]
|
||||
preview_id = uuid.uuid4().hex
|
||||
self.plugin_preview_cache[preview_id] = (plugin_name, resources)
|
||||
return {
|
||||
"preview_id": preview_id,
|
||||
"count": len(resources),
|
||||
"items": [_resource_to_json(item, i) for i, item in enumerate(resources)],
|
||||
}
|
||||
|
||||
def _handle_plugin_download(self, payload: dict) -> dict:
|
||||
preview_id = str(payload.get("preview_id", ""))
|
||||
cached = self.plugin_preview_cache.get(preview_id)
|
||||
if cached is None:
|
||||
raise ValueError("预览结果已过期,请重新解析。")
|
||||
plugin_name, resources = cached
|
||||
plugin = self.plugin_manager.plugins.get(plugin_name)
|
||||
if plugin is None:
|
||||
raise ValueError(f"插件已卸载: {plugin_name}")
|
||||
selected_ids = payload.get("selected_ids")
|
||||
selected = set(str(item) for item in selected_ids) if isinstance(selected_ids, list) else set()
|
||||
if selected:
|
||||
resources = [item for item in resources if item.id in selected]
|
||||
save_root = downloads_root()
|
||||
results = []
|
||||
downloaded = 0
|
||||
failed = 0
|
||||
for resource in resources:
|
||||
try:
|
||||
paths = plugin.download(resource, save_root)
|
||||
downloaded += len(paths)
|
||||
results.append({
|
||||
"id": resource.id,
|
||||
"ok": bool(paths),
|
||||
"paths": [str(path) for path in paths],
|
||||
"synced_files": 0,
|
||||
})
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
results.append({"id": resource.id, "ok": False, "error": str(exc)})
|
||||
return {
|
||||
"requested": len(resources),
|
||||
"downloaded_files": downloaded,
|
||||
"synced_files": 0,
|
||||
"failed_items": failed,
|
||||
"items": results,
|
||||
}
|
||||
|
||||
def _handle_artwork_detail(self, path: str) -> dict:
|
||||
artwork_id = int(path.rsplit("/", 1)[-1])
|
||||
item = self.gallery_service.get_artwork(artwork_id)
|
||||
if item is None:
|
||||
raise ValueError(f"Artwork not found: {artwork_id}")
|
||||
return item
|
||||
|
||||
def _handle_artist_detail(self, path: str, query: dict[str, list[str]]) -> dict:
|
||||
artist_id = int(path.rsplit("/", 1)[-1])
|
||||
item = self.gallery_service.get_artist(
|
||||
artist_id,
|
||||
limit=_int(query, "limit", 60),
|
||||
offset=_int(query, "offset", 0),
|
||||
)
|
||||
if item is None:
|
||||
raise ValueError(f"Artist not found: {artist_id}")
|
||||
return item
|
||||
|
||||
def _resolve_media_path(self, path: str, *, thumbnail: bool) -> Path:
|
||||
parts = path.strip("/").split("/")
|
||||
if len(parts) != 4:
|
||||
raise ValueError("Expected /media/{thumbnail|source}/{artwork_id}/{page_index}")
|
||||
artwork_id = int(parts[2])
|
||||
page_index = int(parts[3])
|
||||
media_path = (
|
||||
self.gallery_service.get_thumbnail_path(artwork_id, page_index=page_index)
|
||||
if thumbnail
|
||||
else self.gallery_service.get_source_file_path(artwork_id, page_index=page_index)
|
||||
)
|
||||
if media_path is None:
|
||||
raise ValueError("Media file not found")
|
||||
return media_path
|
||||
|
||||
def _handle_cookie_import(self, payload: dict) -> dict:
|
||||
platform = str(payload.get("platform", "")).strip().lower()
|
||||
text = str(payload.get("text", ""))
|
||||
if platform in {"pixiv", "px"}:
|
||||
cookies = parse_cookie_text(text)
|
||||
pixiv_cookies = cookies_to_playwright(cookies, ("pixiv.net",))
|
||||
header = cookies_to_header(pixiv_cookies, ("pixiv.net",))
|
||||
store = SessionStore(app_session_file())
|
||||
session_payload = store.load()
|
||||
session_payload.update(
|
||||
{
|
||||
"login_mode": "cookie",
|
||||
"cookie": header,
|
||||
"cookie_json": pixiv_cookies,
|
||||
"login_id": "",
|
||||
"saved_at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
)
|
||||
store.save(session_payload)
|
||||
return {"platform": "pixiv", "count": len(pixiv_cookies), "summary": cookie_summary(pixiv_cookies)}
|
||||
if platform in {"x", "twitter"}:
|
||||
result = self.x_media_service.import_cookie_text(text)
|
||||
result["platform"] = "x"
|
||||
return result
|
||||
raise ValueError("platform 必须是 pixiv 或 x。")
|
||||
|
||||
def _handle_session_status(self) -> dict:
|
||||
payload = SessionStore(app_session_file()).load()
|
||||
pixiv_cookie = str(payload.get("cookie") or "").strip()
|
||||
cookie_json = payload.get("cookie_json")
|
||||
cloud = payload.get("cloud_sync") if isinstance(payload.get("cloud_sync"), dict) else {}
|
||||
return {
|
||||
"pixiv": {
|
||||
"has_cookie": bool(pixiv_cookie or cookie_json),
|
||||
"saved_at": str(payload.get("saved_at") or ""),
|
||||
"login_mode": str(payload.get("login_mode") or ""),
|
||||
},
|
||||
"cloud": {
|
||||
"enabled": bool(cloud.get("enabled")),
|
||||
"has_token": bool(str(cloud.get("access_token") or "").strip()),
|
||||
"base_url": str(cloud.get("base_url") or "").strip(),
|
||||
},
|
||||
}
|
||||
|
||||
def _send_json(self, payload: dict | list, *, status: int = 200) -> None:
|
||||
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self._send_cors_headers()
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_file(self, path: Path) -> None:
|
||||
content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
||||
body = path.read_bytes()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_static_file(self, path: Path) -> None:
|
||||
if not path.exists() or not path.is_file():
|
||||
self._send_json({"error": "not found"}, status=404)
|
||||
return
|
||||
self._send_file(path)
|
||||
|
||||
def _send_cors_headers(self) -> None:
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
self.send_header("Access-Control-Allow-Headers", "Content-Type")
|
||||
|
||||
def _read_json_body(self) -> dict:
|
||||
length = int(self.headers.get("Content-Length", "0") or "0")
|
||||
if length == 0:
|
||||
return {}
|
||||
body = self.rfile.read(length)
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("JSON body must be an object")
|
||||
return payload
|
||||
|
||||
|
||||
def _first(query: dict[str, list[str]], key: str, default: str = "") -> str:
|
||||
values = query.get(key)
|
||||
if not values:
|
||||
return default
|
||||
return values[0]
|
||||
|
||||
|
||||
def _int(query: dict[str, list[str]], key: str, default: int) -> int:
|
||||
value = _first(query, key)
|
||||
if not value:
|
||||
return default
|
||||
return int(value)
|
||||
|
||||
|
||||
def _bool(query: dict[str, list[str]], key: str, default: bool) -> bool:
|
||||
value = _first(query, key)
|
||||
if not value:
|
||||
return default
|
||||
return value.lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _resource_to_json(resource, index: int) -> dict:
|
||||
payload = asdict(resource)
|
||||
payload["index"] = index
|
||||
payload["kind"] = "video" if any(file.get("type") == "video" for file in resource.files) else "image"
|
||||
payload["file_count"] = len(resource.files)
|
||||
return payload
|
||||
|
||||
|
||||
def _plugin_platform_slug(plugin_name: str) -> str:
|
||||
lowered = plugin_name.lower()
|
||||
if "douyin" in lowered or "抖音" in plugin_name:
|
||||
return "douyin"
|
||||
if "xiaohongshu" in lowered or "小红书" in plugin_name:
|
||||
return "xiaohongshu"
|
||||
if lowered == "x":
|
||||
return "x"
|
||||
return lowered.replace(" ", "_")[:20] or "plugin"
|
||||
|
||||
|
||||
def _resource_file_type(resource, index: int, path: Path) -> str:
|
||||
if index < len(resource.files):
|
||||
file_type = str(resource.files[index].get("type") or "").lower()
|
||||
if file_type == "video":
|
||||
return "video"
|
||||
mime_type = mimetypes.guess_type(str(path))[0] or ""
|
||||
return "video" if mime_type.startswith("video/") else "illust"
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from pixiv_app.services.gallery_api import GalleryApiServer
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Run local Sakura gallery API")
|
||||
parser.add_argument("--host", default="127.0.0.1")
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
parser.add_argument("--db", default="pixiv_app_library.db")
|
||||
parser.add_argument("--thumbnail-dir", default="runtime/thumbnails")
|
||||
parser.add_argument("--web-root", default="")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logging.info("Local gallery API listening on http://%s:%s", args.host, args.port)
|
||||
GalleryApiServer(
|
||||
host=args.host,
|
||||
port=args.port,
|
||||
db_path=args.db,
|
||||
thumbnail_dir=args.thumbnail_dir,
|
||||
web_root=args.web_root or None,
|
||||
).serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,356 @@
|
||||
"""
|
||||
X (Twitter) authentication helper — Playwright **persistent context** only.
|
||||
|
||||
- First-time login is always manual (including email verification); no credential stuffing.
|
||||
- Session persistence is Chromium's user_data_dir (cookies + localStorage live inside it).
|
||||
- Optional JSON cookie export into ``profiles/x/<profile>/cookies/`` for backup only.
|
||||
|
||||
Directory layout (created automatically)::
|
||||
|
||||
profiles/
|
||||
x/
|
||||
<profile_key>/
|
||||
chromium_profile/ # user_data_dir → persistent session (real storage)
|
||||
cookies/ # optional Playwright cookie exports (JSON)
|
||||
localStorage/ # reserved / notes (Chromium keeps LS inside chromium_profile)
|
||||
|
||||
Usage::
|
||||
|
||||
auth = XAuthManager(profile_key="default")
|
||||
await auth.start_session()
|
||||
page = await auth.get_page()
|
||||
await page.goto("https://x.com/i/flow/login")
|
||||
# ... user completes login manually ...
|
||||
if await auth.is_logged_in():
|
||||
...
|
||||
await auth.close()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from playwright.async_api import BrowserContext, Page, Playwright, async_playwright
|
||||
|
||||
from pixiv_app.core.paths import profiles_root
|
||||
from pixiv_app.core.playwright_runtime import configure_playwright_runtime_env, preferred_chromium_executable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _default_profiles_root() -> Path:
|
||||
return profiles_root()
|
||||
|
||||
|
||||
class XAuthManager:
|
||||
"""
|
||||
Manage a long-lived Playwright Chromium **persistent** profile for x.com.
|
||||
|
||||
Not thread-safe: use one instance per logical browser session, or guard externally.
|
||||
"""
|
||||
|
||||
HOME_URL = "https://x.com/home"
|
||||
LOGIN_FLOW_HINT = "https://x.com/i/flow/login"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
profile_key: str = "default",
|
||||
profiles_root: Path | None = None,
|
||||
headless: bool = True,
|
||||
auto_reconnect: bool = True,
|
||||
max_health_retries: int = 2,
|
||||
) -> None:
|
||||
"""
|
||||
:param profile_key: Subdirectory under ``profiles/x/`` for multi-account switching.
|
||||
:param profiles_root: Defaults to ``<project>/profiles``.
|
||||
:param headless: Headless for automated checks; use ``False`` for first manual login.
|
||||
:param auto_reconnect: If ``start_session`` finds a dead context, restart Playwright.
|
||||
:param max_health_retries: Retries when validating session after transient failures.
|
||||
"""
|
||||
self._profile_key = profile_key
|
||||
root = profiles_root or _default_profiles_root()
|
||||
self._profile_home = root / "x" / profile_key
|
||||
self._user_data_dir = self._profile_home / "chromium_profile"
|
||||
self._cookies_dir = self._profile_home / "cookies"
|
||||
self._localstorage_dir = self._profile_home / "localStorage"
|
||||
|
||||
self._headless = headless
|
||||
self._auto_reconnect = auto_reconnect
|
||||
self._max_health_retries = max(1, max_health_retries)
|
||||
|
||||
self._pw: Playwright | None = None
|
||||
self._context: BrowserContext | None = None
|
||||
self._primary_page: Page | None = None
|
||||
|
||||
self._ensure_storage_layout()
|
||||
|
||||
def _ensure_storage_layout(self) -> None:
|
||||
"""Create profile dirs. Real cookie/localStorage persistence is under ``chromium_profile``."""
|
||||
self._user_data_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._cookies_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._localstorage_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@property
|
||||
def user_data_dir(self) -> Path:
|
||||
"""Playwright ``user_data_dir`` passed to ``launch_persistent_context``."""
|
||||
return self._user_data_dir
|
||||
|
||||
def _context_alive(self) -> bool:
|
||||
if self._context is None:
|
||||
return False
|
||||
try:
|
||||
_ = self._context.pages
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def start_session(self, *, force_new: bool = False) -> BrowserContext:
|
||||
"""
|
||||
Launch or reuse persistent Chromium. Session is restored from disk automatically.
|
||||
|
||||
:param force_new: Close existing context and open a fresh one (same profile dir).
|
||||
"""
|
||||
if self._context_alive() and not force_new:
|
||||
return self._context # type: ignore[return-value]
|
||||
|
||||
if self._context is not None:
|
||||
await self.close()
|
||||
|
||||
self._pw = await async_playwright().start()
|
||||
try:
|
||||
self._context = await self._launch_persistent(self._pw)
|
||||
except Exception:
|
||||
await self._pw.stop()
|
||||
self._pw = None
|
||||
raise
|
||||
|
||||
self._primary_page = None
|
||||
logger.info("XAuthManager: persistent context ready (%s)", self._user_data_dir)
|
||||
return self._context
|
||||
|
||||
async def _launch_persistent(self, pw: Playwright) -> BrowserContext:
|
||||
"""Single place for launch_persistent_context configuration."""
|
||||
self._user_data_dir.mkdir(parents=True, exist_ok=True)
|
||||
configure_playwright_runtime_env()
|
||||
|
||||
base_kw: dict[str, Any] = {
|
||||
"user_data_dir": str(self._user_data_dir),
|
||||
"headless": self._headless,
|
||||
"viewport": {"width": 1280, "height": 800},
|
||||
"locale": "zh-CN",
|
||||
"ignore_default_args": ["--enable-automation"],
|
||||
"args": ["--disable-blink-features=AutomationControlled"],
|
||||
"user_agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
|
||||
import os
|
||||
|
||||
env_ch = os.environ.get("X_PLAYWRIGHT_CHANNEL", "").strip()
|
||||
local_executable = preferred_chromium_executable()
|
||||
attempts: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for extra in (
|
||||
{"channel": env_ch} if env_ch else {},
|
||||
{"executable_path": str(local_executable)} if local_executable else {},
|
||||
{"channel": "chrome"},
|
||||
{},
|
||||
):
|
||||
key = repr(extra)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
attempts.append(extra)
|
||||
|
||||
last_err: Exception | None = None
|
||||
for extra in attempts:
|
||||
kw = dict(base_kw)
|
||||
kw.update(extra)
|
||||
try:
|
||||
ctx = await pw.chromium.launch_persistent_context(**kw)
|
||||
await ctx.add_init_script(
|
||||
"""
|
||||
try {
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
} catch (e) {}
|
||||
"""
|
||||
)
|
||||
return ctx
|
||||
except Exception as exc:
|
||||
last_err = exc
|
||||
logger.warning("launch_persistent_context failed (%s): %s", extra, exc)
|
||||
continue
|
||||
|
||||
assert last_err is not None
|
||||
raise last_err
|
||||
|
||||
async def get_context(self) -> BrowserContext:
|
||||
if not self._context_alive():
|
||||
if self._auto_reconnect:
|
||||
return await self.start_session()
|
||||
raise RuntimeError("No browser session; call start_session() first.")
|
||||
return self._context # type: ignore[return-value]
|
||||
|
||||
async def get_page(self, *, new: bool = False) -> Page:
|
||||
"""
|
||||
Return a page for navigation. Default reuses one tab; set ``new=True`` for a fresh tab.
|
||||
"""
|
||||
ctx = await self.get_context()
|
||||
if new:
|
||||
return await ctx.new_page()
|
||||
|
||||
if self._primary_page is not None:
|
||||
try:
|
||||
_ = self._primary_page.url
|
||||
return self._primary_page
|
||||
except Exception:
|
||||
self._primary_page = None
|
||||
|
||||
if ctx.pages:
|
||||
self._primary_page = ctx.pages[0]
|
||||
return self._primary_page
|
||||
|
||||
self._primary_page = await ctx.new_page()
|
||||
return self._primary_page
|
||||
|
||||
async def is_logged_in(self, *, page: Page | None = None, navigate: bool = True) -> bool:
|
||||
"""
|
||||
Heuristic login detection (DOM + URL). Not guaranteed if X changes UI.
|
||||
|
||||
- ``False`` if still on login / signup flow URLs or login UI is visible.
|
||||
- ``True`` if home timeline / primary column or account UI is present.
|
||||
"""
|
||||
p = page if page is not None else await self.get_page()
|
||||
|
||||
if navigate:
|
||||
await p.goto(self.HOME_URL, wait_until="domcontentloaded", timeout=45_000)
|
||||
await asyncio.sleep(0.6)
|
||||
|
||||
url = (p.url or "").lower()
|
||||
if any(
|
||||
x in url
|
||||
for x in (
|
||||
"/i/flow/login",
|
||||
"/i/flow/signup",
|
||||
"/login",
|
||||
)
|
||||
):
|
||||
return False
|
||||
|
||||
# Logged-in indicators (best-effort; X changes selectors periodically)
|
||||
logged_in_selectors = (
|
||||
'[data-testid="SideNav_AccountSwitcher_Button"]',
|
||||
'[data-testid="AppTabBar_Home_Link"]',
|
||||
'[data-testid="primaryColumn"]',
|
||||
'[data-testid="tweet"]',
|
||||
)
|
||||
for sel in logged_in_selectors:
|
||||
try:
|
||||
el = await p.wait_for_selector(sel, timeout=2500, state="visible")
|
||||
if el:
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Login / gate modal text (guest wall)
|
||||
try:
|
||||
guest = p.get_by_text(
|
||||
"Sign in to X",
|
||||
exact=False,
|
||||
)
|
||||
if await guest.count() and await guest.first.is_visible(timeout=800):
|
||||
return False
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
async def session_healthy(self) -> bool:
|
||||
"""True if browser context responds and ``is_logged_in`` succeeds within retries."""
|
||||
for attempt in range(self._max_health_retries):
|
||||
try:
|
||||
if not self._context_alive():
|
||||
return False
|
||||
return await self.is_logged_in(navigate=True)
|
||||
except Exception as exc:
|
||||
logger.debug("session_healthy attempt %s: %s", attempt + 1, exc)
|
||||
await asyncio.sleep(1.0)
|
||||
return False
|
||||
|
||||
async def export_playwright_cookies(self, filename: str = "playwright_cookies.json") -> Path:
|
||||
"""Optional backup: write context cookies to ``profiles/x/<key>/cookies/``."""
|
||||
ctx = await self.get_context()
|
||||
cookies = await ctx.cookies()
|
||||
path = self._cookies_dir / filename
|
||||
path.write_text(json.dumps(cookies, indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close Playwright; persistent data remains on disk for next run."""
|
||||
if self._primary_page is not None:
|
||||
try:
|
||||
await self._primary_page.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._primary_page = None
|
||||
|
||||
if self._context is not None:
|
||||
try:
|
||||
await self._context.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._context = None
|
||||
|
||||
if self._pw is not None:
|
||||
try:
|
||||
await self._pw.stop()
|
||||
except Exception:
|
||||
pass
|
||||
self._pw = None
|
||||
|
||||
logger.info("XAuthManager: closed (%s)", self._profile_key)
|
||||
|
||||
# --- sync helpers for Tkinter / scripts without asyncio ---
|
||||
|
||||
def start_session_sync(self, *, force_new: bool = False) -> BrowserContext:
|
||||
return asyncio.run(self.start_session(force_new=force_new))
|
||||
|
||||
def get_page_sync(self, *, new: bool = False) -> Page:
|
||||
return asyncio.run(self.get_page(new=new))
|
||||
|
||||
def is_logged_in_sync(self, *, navigate: bool = True) -> bool:
|
||||
return asyncio.run(self.is_logged_in(navigate=navigate))
|
||||
|
||||
def close_sync(self) -> None:
|
||||
asyncio.run(self.close())
|
||||
|
||||
|
||||
async def _demo() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
auth = XAuthManager(profile_key="default", headless=False)
|
||||
try:
|
||||
await auth.start_session()
|
||||
page = await auth.get_page()
|
||||
await page.goto(XAuthManager.LOGIN_FLOW_HINT, wait_until="domcontentloaded")
|
||||
print(
|
||||
"浏览器已打开登录流程。请在窗口中手动完成登录(含邮箱验证),完成后回到终端按 Enter…",
|
||||
)
|
||||
await asyncio.to_thread(input)
|
||||
ok = await auth.is_logged_in()
|
||||
print("is_logged_in:", ok)
|
||||
if ok:
|
||||
path = await auth.export_playwright_cookies()
|
||||
print("Cookies 备份:", path)
|
||||
finally:
|
||||
await auth.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(_demo())
|
||||
@@ -0,0 +1,156 @@
|
||||
"""
|
||||
Helpers for X (Twitter) web login flow in Playwright.
|
||||
|
||||
X often opens a combined sheet titled 「注册 X」 even from /i/flow/login; users must switch to
|
||||
the actual sign-in path or they loop after entering email. These helpers only automate UI clicks
|
||||
(no credential stuffing, no bypass).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from playwright.async_api import Page
|
||||
|
||||
LOGIN_URL_X = "https://x.com/i/flow/login"
|
||||
LOGIN_URL_TWITTER = "https://twitter.com/i/flow/login"
|
||||
|
||||
# When X shows this, login inside Playwright often cannot proceed (rate limit / bot heuristics).
|
||||
X_FLOW_ERROR_RECOVERY_HINT_CN = (
|
||||
"这是 X 端返回的通用错误,常见于自动化浏览器或风控,并不代表本程序损坏。\n\n"
|
||||
"可行做法(任选):\n"
|
||||
"1)点击弹窗「重试」后等待几分钟再试,或更换网络(如手机热点);\n"
|
||||
"2)在插件管理中「清除本 Profile 浏览器数据」后重新「手动登录」;\n"
|
||||
"3)用本机普通 Chrome 登录 x.com,导出 Cookie 为 JSON,放到 runtime/x_cookies.json,再点「同步到抓取」。\n\n"
|
||||
"我们无法绕过 X/Google 的风控策略。"
|
||||
)
|
||||
|
||||
|
||||
async def detect_x_generic_flow_error(page: Page) -> bool:
|
||||
"""True if the grey modal 「出错了 / Something went wrong」 is likely visible."""
|
||||
try:
|
||||
dlg = page.locator('[role="dialog"]')
|
||||
if await dlg.count() > 0:
|
||||
text = (await dlg.first.inner_text())[:1200]
|
||||
low = text.lower()
|
||||
if ("出错了" in text or "something went wrong" in low) and (
|
||||
"重试" in text or "reload" in low or "retry" in low or "重新加载" in text
|
||||
):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
async def prefer_login_over_signup(page: Page, *, max_attempts: int = 5) -> None:
|
||||
"""
|
||||
If the modal looks like sign-up (注册 X / Sign up for X), try to switch to login.
|
||||
|
||||
Safe to call multiple times; no-op when login sheet is already shown.
|
||||
"""
|
||||
for _ in range(max_attempts):
|
||||
await asyncio.sleep(0.35)
|
||||
|
||||
# Already looks like login-oriented sheet (best-effort)
|
||||
try:
|
||||
login_heading = page.get_by_role("heading", name=re.compile(r"登录.*X|Sign in to X|登录或注册", re.I))
|
||||
if await login_heading.count() > 0 and await login_heading.first.is_visible(timeout=400):
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
signup_visible = False
|
||||
try:
|
||||
su = page.get_by_text(re.compile(r"注册 X|Sign up for X"), exact=False)
|
||||
if await su.count() > 0:
|
||||
signup_visible = await su.first.is_visible(timeout=500)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not signup_visible:
|
||||
try:
|
||||
dlg = page.locator('[role="dialog"]')
|
||||
if await dlg.count() > 0:
|
||||
t = await dlg.first.inner_text()
|
||||
if "注册 X" in t or "Sign up for X" in t:
|
||||
signup_visible = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not signup_visible:
|
||||
return
|
||||
|
||||
clicked = await _try_click_login_switch(page)
|
||||
if not clicked:
|
||||
break
|
||||
|
||||
|
||||
async def _try_click_login_switch(page: Page) -> bool:
|
||||
"""Return True if a qualifying control was clicked."""
|
||||
|
||||
# Known Twitter/X test ids (may change)
|
||||
for tid in ("signupSheetSwitchToLogin", "switch_to_login"):
|
||||
try:
|
||||
loc = page.locator(f'[data-testid="{tid}"]')
|
||||
if await loc.count() > 0 and await loc.first.is_visible(timeout=600):
|
||||
await loc.first.click(timeout=3000)
|
||||
await asyncio.sleep(0.8)
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Footer / inverse of 「还没有账号?注册」 → often 「登录」 link nearby
|
||||
try:
|
||||
inv = page.get_by_text(re.compile(r"已有账号|已有帐户|Already have an account", re.I))
|
||||
if await inv.count() > 0:
|
||||
await inv.first.click(timeout=3500)
|
||||
await asyncio.sleep(0.85)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
plain = page.get_by_role("link", name=re.compile(r"^(登录|Sign in)$", re.I))
|
||||
if await plain.count() > 0:
|
||||
await plain.first.click(timeout=3500)
|
||||
await asyncio.sleep(0.85)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
href_login = page.locator('a[href*="login"]').filter(has_not_text=re.compile(r"Google|Apple", re.I))
|
||||
if await href_login.count() > 0 and await href_login.first.is_visible(timeout=600):
|
||||
await href_login.first.click(timeout=3500)
|
||||
await asyncio.sleep(0.85)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
link = page.locator('a[href="/login"]').first
|
||||
if await link.is_visible(timeout=600):
|
||||
await link.click(timeout=3500)
|
||||
await asyncio.sleep(0.85)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
async def goto_x_login_entry(page: Page, *, timeout: float = 90_000) -> None:
|
||||
"""Navigate to official login flow and try to leave sign-up shell."""
|
||||
await page.goto(LOGIN_URL_X, wait_until="domcontentloaded", timeout=timeout)
|
||||
await prefer_login_over_signup(page)
|
||||
|
||||
# Some locales keep showing 「注册 X」 on x.com; twitter.com entry occasionally differs.
|
||||
try:
|
||||
su = page.get_by_text(re.compile(r"注册 X|Sign up for X"), exact=False)
|
||||
if await su.count() > 0 and await su.first.is_visible(timeout=700):
|
||||
await page.goto(LOGIN_URL_TWITTER, wait_until="domcontentloaded", timeout=timeout)
|
||||
await prefer_login_over_signup(page)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Task models, SQLite-backed queue, batch parser, and watermark helpers."""
|
||||
|
||||
from pixiv_app.tasks.models import DownloadTaskSpec, ParsePreview, ParsedLine, TaskPriority
|
||||
from pixiv_app.tasks.parser import parse_batch_for_mode, split_input_blob
|
||||
from pixiv_app.tasks.queue import TaskQueue
|
||||
from pixiv_app.tasks.watermark import advance_watermark, filter_newer_than_watermark, get_last_synced_illust_id
|
||||
|
||||
__all__ = [
|
||||
"DownloadTaskSpec",
|
||||
"ParsePreview",
|
||||
"ParsedLine",
|
||||
"TaskPriority",
|
||||
"TaskQueue",
|
||||
"advance_watermark",
|
||||
"filter_newer_than_watermark",
|
||||
"get_last_synced_illust_id",
|
||||
"parse_batch_for_mode",
|
||||
"split_input_blob",
|
||||
]
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import IntEnum
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
class TaskPriority(IntEnum):
|
||||
LOW = 1
|
||||
MEDIUM = 2
|
||||
HIGH = 3
|
||||
|
||||
|
||||
TaskStatus = Literal["pending", "running", "done", "failed"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DownloadTaskSpec:
|
||||
"""Serializable unit stored in SQLite via TaskQueue (payload omits secrets)."""
|
||||
|
||||
target_type: Literal["illust", "novel"]
|
||||
target_id: int
|
||||
task_kind: str
|
||||
priority: TaskPriority
|
||||
page_index: int | None = None
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def fingerprint(self) -> str:
|
||||
page_part = self.page_index if self.page_index is not None else -1
|
||||
return f"{self.task_kind}:{self.target_type}:{self.target_id}:{page_part}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedLine:
|
||||
"""One logical input token after splitting (before expansion to illust ids)."""
|
||||
|
||||
category: Literal["user", "illust", "novel", "tag", "keyword"]
|
||||
id_value: int
|
||||
tag: str = ""
|
||||
keyword_text: str = ""
|
||||
raw_token: str = ""
|
||||
pixiv_kind: str = "user"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsePreview:
|
||||
"""Statistics for GUI preview."""
|
||||
|
||||
total_lines: int
|
||||
by_category: dict[str, int]
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Literal
|
||||
|
||||
from pixiv_app.core.downloader import parse_pixiv_target
|
||||
from pixiv_app.tasks.models import ParsePreview, ParsedLine
|
||||
|
||||
|
||||
def split_input_blob(blob: str) -> list[str]:
|
||||
parts = re.split(r"[\s,,;;\n\r]+", blob.strip())
|
||||
return [p for p in parts if p]
|
||||
|
||||
|
||||
def parse_batch_for_mode(
|
||||
raw: str,
|
||||
*,
|
||||
ui_mode: Literal["user", "illust", "manga", "novel", "keyword"],
|
||||
) -> tuple[list[ParsedLine], ParsePreview]:
|
||||
"""
|
||||
Parse multi-line / comma / space separated input into structured intents.
|
||||
|
||||
ui_mode matches GUI combo selection so validation stays aligned with existing rules.
|
||||
"""
|
||||
tokens = split_input_blob(raw)
|
||||
lines: list[ParsedLine] = []
|
||||
stats: dict[str, int] = {}
|
||||
|
||||
def bump(key: str) -> None:
|
||||
stats[key] = stats.get(key, 0) + 1
|
||||
|
||||
if ui_mode == "keyword":
|
||||
kw = " ".join(tokens) if tokens else raw.strip()
|
||||
if kw.startswith("#"):
|
||||
kw = kw[1:].strip()
|
||||
if kw:
|
||||
lines.append(ParsedLine(category="keyword", id_value=0, keyword_text=kw, raw_token=raw.strip()))
|
||||
bump("keyword")
|
||||
return lines, ParsePreview(total_lines=len(lines), by_category=stats)
|
||||
|
||||
for token in tokens:
|
||||
t = token.strip()
|
||||
if not t:
|
||||
continue
|
||||
if t.startswith("#"):
|
||||
tag = t[1:].strip()
|
||||
if tag:
|
||||
lines.append(ParsedLine(category="tag", id_value=0, tag=tag, raw_token=t))
|
||||
bump("tag")
|
||||
continue
|
||||
|
||||
try:
|
||||
kind, num_id = parse_pixiv_target(t)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if ui_mode == "novel":
|
||||
if kind == "novel":
|
||||
lines.append(ParsedLine(category="novel", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("novel")
|
||||
elif kind == "novel_series":
|
||||
lines.append(ParsedLine(category="novel", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("novel")
|
||||
elif kind == "unknown":
|
||||
lines.append(ParsedLine(category="novel", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("novel")
|
||||
continue
|
||||
|
||||
if ui_mode in {"illust", "manga"}:
|
||||
if kind == "illust":
|
||||
lines.append(ParsedLine(category="illust", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("illust")
|
||||
elif kind == "unknown":
|
||||
lines.append(ParsedLine(category="illust", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("illust")
|
||||
continue
|
||||
|
||||
# ui_mode == "user"
|
||||
if kind == "user":
|
||||
lines.append(ParsedLine(category="user", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("user")
|
||||
elif kind == "illust":
|
||||
lines.append(ParsedLine(category="user", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("user")
|
||||
elif kind == "unknown":
|
||||
lines.append(ParsedLine(category="user", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("user")
|
||||
|
||||
return lines, ParsePreview(total_lines=len(lines), by_category=stats)
|
||||
@@ -0,0 +1,238 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pixiv_app.tasks.models import DownloadTaskSpec
|
||||
|
||||
|
||||
class TaskQueue:
|
||||
"""SQLite-backed queue with lease/reclaim; thread-safe for concurrent workers."""
|
||||
|
||||
def __init__(self, conn: sqlite3.Connection) -> None:
|
||||
self._conn = conn
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def enqueue(self, tasks: list[DownloadTaskSpec]) -> tuple[int, int]:
|
||||
inserted = 0
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
for t in tasks:
|
||||
fp = t.fingerprint()
|
||||
cur = self._conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO download_tasks (
|
||||
fingerprint, target_type, target_id, page_index, task_kind,
|
||||
priority, status, payload_json
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)
|
||||
""",
|
||||
(
|
||||
fp,
|
||||
t.target_type,
|
||||
t.target_id,
|
||||
t.page_index,
|
||||
t.task_kind,
|
||||
int(t.priority),
|
||||
json.dumps(t.payload, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
inserted += cur.rowcount
|
||||
skipped = len(tasks) - inserted
|
||||
return inserted, skipped
|
||||
|
||||
def requeue_failed_by_fingerprints(self, fingerprints: list[str]) -> int:
|
||||
"""Reset `failed` rows to `pending` so a retry can pick them up."""
|
||||
if not fingerprints:
|
||||
return 0
|
||||
placeholders = ",".join("?" * len(fingerprints))
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
cur = self._conn.execute(
|
||||
f"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'pending',
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = '',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE fingerprint IN ({placeholders}) AND status = 'failed'
|
||||
""",
|
||||
fingerprints,
|
||||
)
|
||||
return int(cur.rowcount or 0)
|
||||
|
||||
def requeue_done_by_fingerprints(self, fingerprints: list[str]) -> int:
|
||||
"""Reset completed rows to `pending` so the user can rerun the same task batch."""
|
||||
if not fingerprints:
|
||||
return 0
|
||||
placeholders = ",".join("?" * len(fingerprints))
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
cur = self._conn.execute(
|
||||
f"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'pending',
|
||||
attempts = 0,
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = '',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE fingerprint IN ({placeholders}) AND status = 'done'
|
||||
""",
|
||||
fingerprints,
|
||||
)
|
||||
return int(cur.rowcount or 0)
|
||||
|
||||
def count_incomplete_for_fingerprints(self, fingerprints: list[str]) -> int:
|
||||
"""Rows for these fingerprints that are not successfully finished (`done`)."""
|
||||
if not fingerprints:
|
||||
return 0
|
||||
placeholders = ",".join("?" * len(fingerprints))
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) FROM download_tasks
|
||||
WHERE fingerprint IN ({placeholders}) AND status != 'done'
|
||||
""",
|
||||
fingerprints,
|
||||
).fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
def lease(
|
||||
self,
|
||||
worker_id: str,
|
||||
*,
|
||||
limit: int = 4,
|
||||
lease_seconds: float = 180.0,
|
||||
) -> list[int]:
|
||||
"""Grab up to `limit` tasks; reclaims expired leases."""
|
||||
now = time.time()
|
||||
deadline = now + lease_seconds
|
||||
ids: list[int] = []
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
for _ in range(limit):
|
||||
cur = self._conn.execute(
|
||||
"""
|
||||
WITH picked AS (
|
||||
SELECT id FROM download_tasks
|
||||
WHERE status = 'pending'
|
||||
OR (status = 'running' AND leased_until IS NOT NULL AND leased_until < ?)
|
||||
ORDER BY priority DESC, id ASC
|
||||
LIMIT 1
|
||||
)
|
||||
UPDATE download_tasks
|
||||
SET status = 'running',
|
||||
worker_id = ?,
|
||||
leased_until = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id IN (SELECT id FROM picked)
|
||||
RETURNING id
|
||||
""",
|
||||
(now, worker_id, deadline),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
break
|
||||
ids.append(int(row[0]))
|
||||
return ids
|
||||
|
||||
def load_task(self, task_id: int) -> sqlite3.Row | None:
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT * FROM download_tasks WHERE id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
return row
|
||||
|
||||
def mark_done(self, task_id: int) -> None:
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'done',
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = '',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(task_id,),
|
||||
)
|
||||
|
||||
def mark_failed_final(self, task_id: int, message: str) -> None:
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'failed',
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(message[:2000], task_id),
|
||||
)
|
||||
|
||||
def requeue_retry(self, task_id: int, message: str, max_attempts: int = 4) -> bool:
|
||||
"""Return True if scheduled for another attempt."""
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
row = self._conn.execute(
|
||||
"SELECT attempts FROM download_tasks WHERE id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
attempts = int(row[0]) + 1
|
||||
if attempts >= max_attempts:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'failed',
|
||||
attempts = ?,
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(attempts, message[:2000], task_id),
|
||||
)
|
||||
return False
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'pending',
|
||||
attempts = ?,
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(attempts, message[:2000], task_id),
|
||||
)
|
||||
return True
|
||||
|
||||
def count_by_status(self, status: str) -> int:
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT COUNT(*) FROM download_tasks WHERE status = ?",
|
||||
(status,),
|
||||
).fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
def snapshot_counts(self) -> dict[str, int]:
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"""
|
||||
SELECT status, COUNT(*) FROM download_tasks GROUP BY status
|
||||
"""
|
||||
).fetchall()
|
||||
return {str(r[0]): int(r[1]) for r in rows}
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
|
||||
|
||||
def get_last_synced_illust_id(conn: sqlite3.Connection, artist_id: int) -> int:
|
||||
row = conn.execute(
|
||||
"SELECT last_synced_illust_id FROM artist_sync_watermark WHERE artist_id = ?",
|
||||
(artist_id,),
|
||||
).fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
|
||||
def advance_watermark(conn: sqlite3.Connection, artist_id: int, illust_id: int) -> None:
|
||||
"""Raise watermark to at least illust_id for incremental author sync."""
|
||||
with conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO artist_sync_watermark (artist_id, last_synced_illust_id, updated_at)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
last_synced_illust_id = MAX(
|
||||
artist_sync_watermark.last_synced_illust_id,
|
||||
excluded.last_synced_illust_id
|
||||
),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(artist_id, illust_id),
|
||||
)
|
||||
|
||||
|
||||
def filter_newer_than_watermark(work_ids: list[int], watermark: int) -> list[int]:
|
||||
"""Keep illust ids strictly greater than last synced watermark (Pixiv ids grow over time)."""
|
||||
return [wid for wid in work_ids if wid > watermark]
|
||||
@@ -0,0 +1,947 @@
|
||||
const state = {
|
||||
view: "home",
|
||||
query: "",
|
||||
tag: "",
|
||||
type: "",
|
||||
artistId: "",
|
||||
offset: 0,
|
||||
limit: 48,
|
||||
total: 0,
|
||||
xPreviewId: "",
|
||||
xPreviewUsername: "",
|
||||
xPreviewAuthor: "",
|
||||
xhsPreviewId: "",
|
||||
};
|
||||
|
||||
const els = {
|
||||
search: document.querySelector("#searchInput"),
|
||||
type: document.querySelector("#typeFilter"),
|
||||
status: document.querySelector("#statusLine"),
|
||||
grid: document.querySelector("#artworkGrid"),
|
||||
tags: document.querySelector("#tagList"),
|
||||
artists: document.querySelector("#artistList"),
|
||||
followed: document.querySelector("#followedList"),
|
||||
xFollowed: document.querySelector("#xFollowedList"),
|
||||
title: document.querySelector("#viewTitle"),
|
||||
subtitle: document.querySelector("#viewSubtitle"),
|
||||
pageInfo: document.querySelector("#pageInfo"),
|
||||
dialog: document.querySelector("#detailDialog"),
|
||||
detail: document.querySelector("#detailContent"),
|
||||
pixivCookieFile: document.querySelector("#pixivCookieFile"),
|
||||
pixivSessionStatus: document.querySelector("#pixivSessionStatus"),
|
||||
refreshSessionButton: document.querySelector("#refreshSessionButton"),
|
||||
xCookieFile: document.querySelector("#xCookieFile"),
|
||||
xUsername: document.querySelector("#xUsernameInput"),
|
||||
xMediaType: document.querySelector("#xMediaType"),
|
||||
xMaxItems: document.querySelector("#xMaxItems"),
|
||||
xMediaGrid: document.querySelector("#xMediaGrid"),
|
||||
xMediaSummary: document.querySelector("#xMediaSummary"),
|
||||
xSelectAll: document.querySelector("#xSelectAll"),
|
||||
xFollowUsername: document.querySelector("#xFollowUsername"),
|
||||
xFollowName: document.querySelector("#xFollowName"),
|
||||
xLocalGrid: document.querySelector("#xLocalGrid"),
|
||||
xLocalSummary: document.querySelector("#xLocalSummary"),
|
||||
xLocalSelectAll: document.querySelector("#xLocalSelectAll"),
|
||||
xhsNavButton: document.querySelector("#xiaohongshuNavButton"),
|
||||
xhsTarget: document.querySelector("#xhsTargetInput"),
|
||||
xhsLimit: document.querySelector("#xhsLimitInput"),
|
||||
xhsGrid: document.querySelector("#xhsGrid"),
|
||||
xhsSummary: document.querySelector("#xhsSummary"),
|
||||
xhsSelectAll: document.querySelector("#xhsSelectAll"),
|
||||
originFollowSyncDialog: document.querySelector("#originFollowSyncDialog"),
|
||||
originFollowSyncResult: document.querySelector("#originFollowSyncResult"),
|
||||
};
|
||||
|
||||
function setStatus(text) {
|
||||
els.status.textContent = text;
|
||||
}
|
||||
|
||||
function isPlaceholderPixivName(name, artistId) {
|
||||
const value = String(name || "").trim();
|
||||
if (!value) {
|
||||
return true;
|
||||
}
|
||||
const idText = String(artistId || "").trim();
|
||||
return Boolean(idText) && value.toLowerCase() === `artist_${idText}`.toLowerCase();
|
||||
}
|
||||
|
||||
function pixivArtistStoredName(item) {
|
||||
const artistId = item?.artist_id ?? item?.artistId ?? item?.creator_id ?? "";
|
||||
const name = String(item?.name || item?.artist_name || item?.creator_name || "").trim();
|
||||
return isPlaceholderPixivName(name, artistId) ? "" : name;
|
||||
}
|
||||
|
||||
function pixivArtistDisplayName(item) {
|
||||
const artistId = item?.artist_id ?? item?.artistId ?? item?.creator_id ?? "";
|
||||
const name = pixivArtistStoredName(item);
|
||||
if (name) {
|
||||
return name;
|
||||
}
|
||||
const account = String(item?.account || item?.artist_account || item?.creator_handle || "").trim().replace(/^@/, "");
|
||||
if (account) {
|
||||
return `@${account}`;
|
||||
}
|
||||
return artistId ? `Pixiv ID ${artistId}` : "Pixiv 作者";
|
||||
}
|
||||
|
||||
window.addEventListener("unhandledrejection", (event) => {
|
||||
setStatus(event.reason?.message || "操作失败,请查看后端日志。");
|
||||
});
|
||||
|
||||
async function api(path, options = {}) {
|
||||
const response = await fetch(path, {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
...options,
|
||||
});
|
||||
const payload = await response.json();
|
||||
if (!response.ok || payload.error) {
|
||||
throw new Error(payload.error || `HTTP ${response.status}`);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
async function loadSessionStatus() {
|
||||
if (!els.pixivSessionStatus) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const payload = await api("/api/session/status");
|
||||
const pixiv = payload.pixiv || {};
|
||||
const cookieText = pixiv.has_cookie
|
||||
? `已共享桌面端 Pixiv 会话${pixiv.saved_at ? `(${pixiv.saved_at})` : ""}`
|
||||
: "未检测到桌面端 Pixiv 会话,请先在 EXE 登录或导入 Cookie。";
|
||||
els.pixivSessionStatus.textContent = cookieText;
|
||||
return payload;
|
||||
} catch (error) {
|
||||
els.pixivSessionStatus.textContent = `会话状态读取失败:${error.message || error}`;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function artworkUrl(item, pageIndex = 0, kind = "thumbnail") {
|
||||
return `/media/${kind}/${item.artwork_id}/${pageIndex}`;
|
||||
}
|
||||
|
||||
function renderArtworkGrid(payload) {
|
||||
state.total = payload.total || 0;
|
||||
els.grid.innerHTML = "";
|
||||
if (!payload.items.length) {
|
||||
els.grid.innerHTML = `<div class="empty">还没有作品。可以先下载作品,或点击“扫描旧目录”导入已有文件。</div>`;
|
||||
} else {
|
||||
els.grid.append(...payload.items.map(renderArtworkCard));
|
||||
}
|
||||
const currentPage = Math.floor(state.offset / state.limit) + 1;
|
||||
const totalPages = Math.max(1, Math.ceil(state.total / state.limit));
|
||||
els.pageInfo.textContent = `${currentPage} / ${totalPages}`;
|
||||
}
|
||||
|
||||
function renderArtworkCard(item) {
|
||||
const card = document.createElement("article");
|
||||
card.className = "artwork-card";
|
||||
const thumb = item.thumbnail_path
|
||||
? `<img loading="lazy" src="${artworkUrl(item)}" alt="${escapeHtml(item.title || String(item.artwork_id))}">`
|
||||
: `<div class="thumb-placeholder">${item.type === "novel" ? "小说" : "暂无缩略图"}</div>`;
|
||||
card.innerHTML = `
|
||||
<button class="thumb-button" data-open="${item.artwork_id}">${thumb}</button>
|
||||
<div class="artwork-meta">
|
||||
<div class="artwork-title">${escapeHtml(item.title || String(item.artwork_id))}</div>
|
||||
<div class="artwork-sub">${escapeHtml(item.artist_name || "未知作者")} · ${escapeHtml(item.type || "")}</div>
|
||||
<div class="card-actions">
|
||||
<button class="small-button ${item.is_favorite ? "active" : ""}" data-fav="${item.artwork_id}" data-on="${item.is_favorite ? 1 : 0}">
|
||||
${item.is_favorite ? "已收藏" : "收藏"}
|
||||
</button>
|
||||
${item.artist_id ? `<button class="small-button" data-artist="${item.artist_id}">作者</button>` : ""}
|
||||
${item.artist_id ? `<button class="small-button ${Number(item.is_followed || 0) ? "active" : ""}" data-follow="${item.artist_id}" data-name="${escapeHtml(pixivArtistStoredName(item))}" data-followed="${Number(item.is_followed || 0)}">${Number(item.is_followed || 0) ? "取消关注" : "关注作者"}</button>` : ""}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
const artworkSub = card.querySelector(".artwork-sub");
|
||||
if (artworkSub) {
|
||||
artworkSub.textContent = `${pixivArtistDisplayName(item)} · ${item.type || ""}`;
|
||||
}
|
||||
const artworkFollow = card.querySelector("[data-follow]");
|
||||
if (artworkFollow) {
|
||||
artworkFollow.dataset.name = pixivArtistStoredName(item);
|
||||
}
|
||||
return card;
|
||||
}
|
||||
|
||||
async function loadArtworks() {
|
||||
const params = new URLSearchParams({
|
||||
limit: state.limit,
|
||||
offset: state.offset,
|
||||
ensure_thumbnails: "true",
|
||||
});
|
||||
if (state.query) params.set("q", state.query);
|
||||
if (state.tag) params.set("tag", state.tag);
|
||||
if (state.type) params.set("type", state.type);
|
||||
if (state.artistId) params.set("artist_id", state.artistId);
|
||||
setStatus("正在加载图库...");
|
||||
const payload = await api(`/api/artworks?${params}`);
|
||||
renderArtworkGrid(payload);
|
||||
setStatus(`已加载 ${payload.items.length} / ${payload.total} 个作品`);
|
||||
}
|
||||
|
||||
async function loadFavorites() {
|
||||
setStatus("正在加载收藏...");
|
||||
const params = new URLSearchParams({ limit: state.limit, offset: state.offset });
|
||||
const payload = await api(`/api/favorites?${params}`);
|
||||
renderArtworkGrid(payload);
|
||||
setStatus(`收藏 ${payload.total} 个作品`);
|
||||
}
|
||||
|
||||
async function loadTags() {
|
||||
const tags = await api("/api/tags?limit=80");
|
||||
els.tags.innerHTML = "";
|
||||
if (!tags.length) {
|
||||
els.tags.innerHTML = `<span class="tag-chip">暂无标签</span>`;
|
||||
return;
|
||||
}
|
||||
for (const tag of tags) {
|
||||
const button = document.createElement("button");
|
||||
button.className = `tag-chip ${state.tag === tag.name ? "active" : ""}`;
|
||||
button.textContent = `${tag.name} ${tag.artwork_count}`;
|
||||
button.dataset.tag = tag.name;
|
||||
els.tags.append(button);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadArtists() {
|
||||
setStatus("正在加载作者...");
|
||||
const artists = await api("/api/artists?limit=200");
|
||||
els.artists.innerHTML = "";
|
||||
if (!artists.length) {
|
||||
els.artists.innerHTML = `<div class="empty">暂无作者数据。</div>`;
|
||||
} else {
|
||||
els.artists.append(...artists.map(renderArtistRow));
|
||||
}
|
||||
setStatus(`已加载 ${artists.length} 位作者`);
|
||||
}
|
||||
|
||||
async function loadFollowed() {
|
||||
const followed = await api("/api/followed-artists");
|
||||
els.followed.innerHTML = "";
|
||||
if (!followed.length) {
|
||||
els.followed.innerHTML = `<div class="empty">还没有关注作者。</div>`;
|
||||
} else {
|
||||
els.followed.append(...followed.map(renderArtistRow));
|
||||
}
|
||||
}
|
||||
|
||||
async function loadXFollowed() {
|
||||
const followed = await api("/api/x/follows");
|
||||
els.xFollowed.innerHTML = "";
|
||||
if (!followed.length) {
|
||||
els.xFollowed.innerHTML = `<div class="empty">还没有 X 关注作者。</div>`;
|
||||
} else {
|
||||
els.xFollowed.append(...followed.map(renderXFollowRow));
|
||||
}
|
||||
}
|
||||
|
||||
function renderArtistRow(artist) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "artist-row";
|
||||
const isFollowed = Number(artist.enabled ?? artist.is_followed ?? 0);
|
||||
row.innerHTML = `
|
||||
<div>
|
||||
<strong>${escapeHtml(artist.name || `artist_${artist.artist_id}`)}</strong>
|
||||
<span>ID ${artist.artist_id} · ${artist.artwork_count || 0} 个作品${artist.last_synced_at ? ` · 上次同步 ${artist.last_synced_at}` : ""}</span>
|
||||
</div>
|
||||
<div class="card-actions">
|
||||
<button class="small-button" data-artist="${artist.artist_id}">详情</button>
|
||||
<button class="small-button ${isFollowed ? "active" : ""}" data-follow="${artist.artist_id}" data-name="${escapeHtml(artist.name || "")}" data-followed="${isFollowed ? 1 : 0}">${isFollowed ? "取消关注" : "关注"}</button>
|
||||
</div>
|
||||
`;
|
||||
const artistTitle = row.querySelector("strong");
|
||||
if (artistTitle) {
|
||||
artistTitle.textContent = pixivArtistDisplayName(artist);
|
||||
}
|
||||
const artistFollow = row.querySelector("[data-follow]");
|
||||
if (artistFollow) {
|
||||
artistFollow.dataset.name = pixivArtistStoredName(artist);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
function renderXFollowRow(artist) {
|
||||
const row = document.createElement("div");
|
||||
row.className = "artist-row";
|
||||
const handle = artist.username || artist.creator_handle || "";
|
||||
const isFollowed = Number(artist.enabled ?? 1);
|
||||
const latest = artist.latest_published_at ? ` · 最新 ${artist.latest_published_at}` : "";
|
||||
const checked = artist.last_checked_at ? ` · 检查 ${artist.last_checked_at}` : "";
|
||||
const error = artist.last_error ? ` · ${escapeHtml(artist.last_error)}` : "";
|
||||
row.innerHTML = `
|
||||
<div>
|
||||
<strong>${escapeHtml(artist.creator_name || handle || "x_author")}</strong>
|
||||
<span>@${escapeHtml(handle)}${latest}${checked}${error}</span>
|
||||
</div>
|
||||
<div class="card-actions">
|
||||
<button class="small-button" data-open-x="${escapeHtml(handle)}">去预览</button>
|
||||
<button class="small-button ${isFollowed ? "active" : ""}" data-follow-x="${escapeHtml(handle)}" data-name="${escapeHtml(artist.creator_name || "")}" data-followed-x="${isFollowed ? 1 : 0}">${isFollowed ? "取消关注" : "关注"}</button>
|
||||
</div>
|
||||
`;
|
||||
return row;
|
||||
}
|
||||
|
||||
async function openArtwork(artworkId) {
|
||||
const payload = await api(`/api/artworks/${artworkId}`);
|
||||
const artwork = payload.artwork;
|
||||
const files = payload.files || [];
|
||||
const tags = payload.tags || [];
|
||||
const firstFile = files[0];
|
||||
const image = firstFile
|
||||
? `<img src="/media/source/${artwork.artwork_id}/${firstFile.page_index}" alt="${escapeHtml(artwork.title)}">`
|
||||
: `<div class="thumb-placeholder">没有本地文件</div>`;
|
||||
els.detail.innerHTML = `
|
||||
<div class="detail-layout">
|
||||
<div class="detail-media">${image}</div>
|
||||
<div class="detail-info">
|
||||
<h2>${escapeHtml(artwork.title || String(artwork.artwork_id))}</h2>
|
||||
<p>${escapeHtml(artwork.artist_name || "未知作者")} · ${escapeHtml(artwork.type)}</p>
|
||||
<div class="card-actions">
|
||||
<button class="small-button ${artwork.is_favorite ? "active" : ""}" data-fav="${artwork.artwork_id}" data-on="${artwork.is_favorite ? 1 : 0}">
|
||||
${artwork.is_favorite ? "已收藏" : "收藏"}
|
||||
</button>
|
||||
${artwork.artist_id ? `<button class="small-button ${Number(artwork.is_followed || 0) ? "active" : ""}" data-follow="${artwork.artist_id}" data-name="${escapeHtml(artwork.artist_name || "")}" data-followed="${Number(artwork.is_followed || 0)}">${Number(artwork.is_followed || 0) ? "取消关注" : "关注作者"}</button>` : ""}
|
||||
</div>
|
||||
<div class="detail-tags">
|
||||
${tags.map((tag) => `<button class="tag-chip" data-tag="${escapeHtml(tag.name)}">${escapeHtml(tag.name)}</button>`).join("")}
|
||||
</div>
|
||||
<p>${escapeHtml(stripHtml(artwork.caption || ""))}</p>
|
||||
<p>页数 ${files.length || artwork.page_count || 0} · 收藏 ${artwork.bookmark_count || 0} · 浏览 ${artwork.view_count || 0}</p>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
const detailArtist = els.detail.querySelector(".detail-info p");
|
||||
if (detailArtist) {
|
||||
detailArtist.textContent = `${pixivArtistDisplayName(artwork)} · ${artwork.type}`;
|
||||
}
|
||||
const detailFollow = els.detail.querySelector("[data-follow]");
|
||||
if (detailFollow) {
|
||||
detailFollow.dataset.name = pixivArtistStoredName(artwork);
|
||||
}
|
||||
els.dialog.showModal();
|
||||
}
|
||||
|
||||
async function showArtist(artistId) {
|
||||
setActiveView("gallery");
|
||||
state.artistId = String(artistId);
|
||||
state.tag = "";
|
||||
state.offset = 0;
|
||||
const detail = await api(`/api/artists/${artistId}?limit=1`);
|
||||
els.title.textContent = detail.artist.name || `作者 ${artistId}`;
|
||||
els.subtitle.textContent = `作者 ID ${artistId} · ${detail.artist.artwork_count || 0} 个本地作品`;
|
||||
await loadArtworks();
|
||||
els.title.textContent = pixivArtistDisplayName(detail.artist);
|
||||
}
|
||||
|
||||
async function toggleFavorite(artworkId, currentOn) {
|
||||
await api("/api/favorites", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ artwork_id: Number(artworkId), favorite: !Number(currentOn) }),
|
||||
});
|
||||
await reloadCurrent();
|
||||
}
|
||||
|
||||
async function followArtist(artistId, name = "") {
|
||||
await setPixivFollow(artistId, name, true);
|
||||
}
|
||||
|
||||
async function setPixivFollow(artistId, name = "", followed = true) {
|
||||
await api("/api/followed-artists", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ artist_id: String(artistId).trim(), name, followed }),
|
||||
});
|
||||
setStatus(followed ? `已关注作者 ${artistId}` : `已取消关注作者 ${artistId}`);
|
||||
await Promise.all([loadFollowed(), reloadCurrent()]);
|
||||
}
|
||||
|
||||
async function followXAuthor(username, name = "") {
|
||||
await setXFollow(username, name, true);
|
||||
}
|
||||
|
||||
async function setXFollow(username, name = "", followed = true) {
|
||||
const handle = username.trim().replace(/^@/, "");
|
||||
if (!handle) {
|
||||
return;
|
||||
}
|
||||
await api("/api/x/follows", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ username: handle, name, followed }),
|
||||
});
|
||||
setStatus(followed ? `已关注 X 作者 @${handle}` : `已取消关注 X 作者 @${handle}`);
|
||||
await loadXFollowed();
|
||||
}
|
||||
|
||||
async function scanLegacy() {
|
||||
setStatus("正在扫描旧下载目录...");
|
||||
const result = await api("/api/import/legacy", { method: "POST", body: JSON.stringify({ root: "Sakura_Downloads" }) });
|
||||
setStatus(`扫描完成:导入 ${result.imported_files} 个文件,${result.imported_artworks} 个作品`);
|
||||
await Promise.all([loadTags(), reloadCurrent()]);
|
||||
}
|
||||
|
||||
async function syncFollowed() {
|
||||
setStatus("正在同步关注作者...");
|
||||
try {
|
||||
const result = await api("/api/sync/followed", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ download: true, max_new: 20 }),
|
||||
});
|
||||
const downloaded = result.artists.reduce((sum, item) => sum + (item.downloaded || 0), 0);
|
||||
setStatus(`同步完成:下载 ${downloaded} 个新作品`);
|
||||
await Promise.all([loadFollowed(), loadTags(), reloadCurrent()]);
|
||||
} catch (error) {
|
||||
setStatus(error.message || "同步关注作者失败");
|
||||
}
|
||||
}
|
||||
|
||||
async function syncXFollowed() {
|
||||
setStatus("正在同步 X 关注...");
|
||||
const result = await api("/api/x/follows/sync", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ download: true, max_new: 20 }),
|
||||
});
|
||||
const total = (result.authors || []).reduce((sum, item) => sum + (item.new_posts || 0), 0);
|
||||
setStatus(`X 同步完成:发现 ${total} 个新动态`);
|
||||
await loadXFollowed();
|
||||
}
|
||||
|
||||
function openOriginFollowSyncDialog() {
|
||||
els.originFollowSyncResult.textContent = "请选择要同步的平台。";
|
||||
els.originFollowSyncDialog.showModal();
|
||||
}
|
||||
|
||||
async function syncOriginFollows(platform) {
|
||||
const label = platform === "pixiv" ? "Pixiv" : "X";
|
||||
const button = platform === "pixiv" ? document.querySelector("#syncOriginPixivButton") : document.querySelector("#syncOriginXButton");
|
||||
if (button?.disabled) {
|
||||
return;
|
||||
}
|
||||
if (button) {
|
||||
button.disabled = true;
|
||||
}
|
||||
setStatus(`正在同步 ${label} 原站关注...`);
|
||||
els.originFollowSyncResult.textContent = `正在同步 ${label} 原站关注,请稍候...`;
|
||||
try {
|
||||
if (platform === "pixiv") {
|
||||
const session = await loadSessionStatus();
|
||||
if (session && !session.pixiv?.has_cookie) {
|
||||
throw new Error("未检测到桌面端 Pixiv 会话,请先在 EXE 登录或导入 Cookie。");
|
||||
}
|
||||
}
|
||||
const result = await api("/api/origin-follows/sync", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ platform }),
|
||||
});
|
||||
const lines = [
|
||||
`${label} 原站关注同步完成`,
|
||||
`同步页数:${result.pages || 0}`,
|
||||
`新增:${result.created || 0} 位`,
|
||||
`更新:${result.updated || 0} 位`,
|
||||
`跳过:${result.skipped || 0} 位`,
|
||||
`失败:${result.failed || 0} 位`,
|
||||
];
|
||||
if (Array.isArray(result.progress) && result.progress.length) {
|
||||
lines.push("", "进度:");
|
||||
for (const item of result.progress.slice(-20)) {
|
||||
lines.push(item.message || `正在同步第 ${item.page || "?"} 页`);
|
||||
}
|
||||
}
|
||||
if (Array.isArray(result.errors) && result.errors.length) {
|
||||
lines.push("", "失败原因:");
|
||||
for (const item of result.errors.slice(0, 10)) {
|
||||
lines.push(`${item.id || "-"}:${item.reason || ""}`);
|
||||
}
|
||||
}
|
||||
els.originFollowSyncResult.textContent = lines.join("\n");
|
||||
setStatus(`${label} 原站关注同步完成:新增 ${result.created || 0},更新 ${result.updated || 0},跳过 ${result.skipped || 0},失败 ${result.failed || 0}`);
|
||||
if (platform === "pixiv") {
|
||||
await loadFollowed();
|
||||
} else {
|
||||
await loadXFollowed();
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error.message || `${label} 原站关注同步失败`;
|
||||
els.originFollowSyncResult.textContent = message;
|
||||
setStatus(message);
|
||||
} finally {
|
||||
if (button) {
|
||||
button.disabled = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function followCurrentX() {
|
||||
const username = els.xUsername.value.trim();
|
||||
if (!username) {
|
||||
setStatus("请先输入 X 作者名。");
|
||||
return;
|
||||
}
|
||||
const handle = username.replace(/^@/, "");
|
||||
const creatorName = state.xPreviewUsername === handle ? state.xPreviewAuthor : "";
|
||||
await followXAuthor(username, creatorName || handle);
|
||||
}
|
||||
|
||||
async function buildThumbnails() {
|
||||
setStatus("正在生成缩略图...");
|
||||
const result = await api("/api/thumbnails/build?limit=500");
|
||||
setStatus(`缩略图完成:处理 ${result.items.length} 个文件`);
|
||||
await reloadCurrent();
|
||||
}
|
||||
|
||||
async function loadPluginPages() {
|
||||
const payload = await api("/api/plugins/pages");
|
||||
const pages = payload.items || [];
|
||||
const hasXhs = pages.some((item) => item.view === "xiaohongshu");
|
||||
els.xhsNavButton.classList.toggle("hidden", !hasXhs);
|
||||
}
|
||||
|
||||
async function importCookieFile(platform, fileInput) {
|
||||
const file = fileInput.files?.[0];
|
||||
if (!file) {
|
||||
setStatus("请先选择 cookies.txt 或 json 文件。");
|
||||
return;
|
||||
}
|
||||
setStatus(`正在导入 ${platform} Cookie...`);
|
||||
const text = await file.text();
|
||||
const result = await api("/api/cookies/import", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ platform, text }),
|
||||
});
|
||||
setStatus(`Cookie 导入完成:${result.summary || `${result.count || 0} 个 Cookie`}`);
|
||||
}
|
||||
|
||||
async function previewXMedia(event) {
|
||||
event.preventDefault();
|
||||
const username = els.xUsername.value.trim();
|
||||
if (!username) {
|
||||
setStatus("请输入 X 作者名称。");
|
||||
return;
|
||||
}
|
||||
els.xMediaGrid.innerHTML = `<div class="empty">正在解析 X 媒体页,首次启动 Playwright 可能需要一些时间。</div>`;
|
||||
els.xMediaSummary.textContent = "解析中...";
|
||||
const submitButton = event.submitter || document.querySelector("#xMediaForm button[type='submit']");
|
||||
if (submitButton) {
|
||||
submitButton.disabled = true;
|
||||
}
|
||||
setStatus(`正在解析 X 作者 @${username.replace(/^@/, "")}...`);
|
||||
try {
|
||||
const payload = await api("/api/x/media/preview", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
username,
|
||||
media_type: els.xMediaType.value,
|
||||
max_items: Number(els.xMaxItems.value || 120),
|
||||
}),
|
||||
});
|
||||
state.xPreviewId = payload.preview_id;
|
||||
renderXMedia(payload);
|
||||
setStatus(`X 解析完成:${payload.count} 个媒体项`);
|
||||
} catch (error) {
|
||||
state.xPreviewId = "";
|
||||
els.xMediaSummary.textContent = "解析失败";
|
||||
els.xMediaGrid.innerHTML = `<div class="empty">${escapeHtml(error.message || "X 解析失败")}</div>`;
|
||||
setStatus(error.message || "X 解析失败");
|
||||
} finally {
|
||||
if (submitButton) {
|
||||
submitButton.disabled = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function renderXMedia(payload) {
|
||||
els.xMediaGrid.innerHTML = "";
|
||||
els.xMediaSummary.textContent = `${payload.username} · ${payload.count} 个媒体项`;
|
||||
if (!payload.items.length) {
|
||||
els.xMediaGrid.innerHTML = `<div class="empty">没有解析到媒体。可能需要先导入 X Cookie,或该作者媒体页为空。</div>`;
|
||||
return;
|
||||
}
|
||||
els.xMediaGrid.append(...payload.items.map(renderXMediaCard));
|
||||
els.xSelectAll.checked = true;
|
||||
}
|
||||
|
||||
function renderXMediaCard(item) {
|
||||
const card = document.createElement("article");
|
||||
card.className = "x-media-card";
|
||||
const preview = item.thumbnail || item.files?.[0]?.url || "";
|
||||
const media = item.kind === "video"
|
||||
? `<div class="video-tile">VIDEO</div>`
|
||||
: `<img loading="lazy" referrerpolicy="no-referrer" src="${escapeHtml(preview)}" alt="${escapeHtml(item.title || item.id)}">`;
|
||||
card.innerHTML = `
|
||||
<label class="x-check">
|
||||
<input type="checkbox" data-x-id="${escapeHtml(item.id)}" checked />
|
||||
<span>${item.kind === "video" ? "视频" : "图片"}</span>
|
||||
</label>
|
||||
<div class="x-thumb">${media}</div>
|
||||
<div class="artwork-meta">
|
||||
<div class="artwork-title">${escapeHtml(item.title || item.id)}</div>
|
||||
<div class="artwork-sub">@${escapeHtml(item.author_id || "")} · ${item.file_count || 1} 个文件</div>
|
||||
</div>
|
||||
`;
|
||||
return card;
|
||||
}
|
||||
|
||||
async function downloadXSelected() {
|
||||
if (!state.xPreviewId) {
|
||||
setStatus("请先解析 X 作者媒体。");
|
||||
return;
|
||||
}
|
||||
const selected = [...document.querySelectorAll("[data-x-id]:checked")].map((item) => item.dataset.xId);
|
||||
if (!selected.length) {
|
||||
setStatus("没有选中的媒体。");
|
||||
return;
|
||||
}
|
||||
setStatus(`正在下载 ${selected.length} 个 X 媒体项...`);
|
||||
const result = await api("/api/x/media/download", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ preview_id: state.xPreviewId, selected_ids: selected }),
|
||||
});
|
||||
setStatus(`X 下载完成:保存 ${result.downloaded_files} 个文件,失败 ${result.failed_items} 项`);
|
||||
await loadXLocalMedia();
|
||||
}
|
||||
|
||||
async function loadXLocalMedia() {
|
||||
const payload = await api("/api/x/local-media?limit=160");
|
||||
els.xLocalGrid.innerHTML = "";
|
||||
els.xLocalSummary.textContent = `本地 ${payload.total || 0} 个文件`;
|
||||
if (!payload.items.length) {
|
||||
els.xLocalGrid.innerHTML = `<div class="empty">还没有本地 X 图片或视频。</div>`;
|
||||
return;
|
||||
}
|
||||
els.xLocalGrid.append(...payload.items.map(renderXLocalCard));
|
||||
}
|
||||
|
||||
function renderXLocalCard(item) {
|
||||
const card = document.createElement("article");
|
||||
card.className = "x-media-card";
|
||||
const media = item.kind === "video"
|
||||
? `<video src="${escapeHtml(item.url)}" muted preload="metadata" controls></video>`
|
||||
: `<img loading="lazy" src="${escapeHtml(item.url)}" alt="${escapeHtml(item.name)}">`;
|
||||
card.innerHTML = `
|
||||
<label class="x-check">
|
||||
<input type="checkbox" data-x-local-id="${escapeHtml(item.id)}" />
|
||||
<span>${item.kind === "video" ? "视频" : "图片"}</span>
|
||||
</label>
|
||||
<div class="x-thumb">${media}</div>
|
||||
<div class="artwork-meta">
|
||||
<div class="artwork-title">${escapeHtml(item.name)}</div>
|
||||
<div class="artwork-sub">${escapeHtml(item.relative_path)} · ${formatBytes(item.size)}</div>
|
||||
</div>
|
||||
`;
|
||||
return card;
|
||||
}
|
||||
|
||||
async function deleteXLocalSelected() {
|
||||
const selected = [...document.querySelectorAll("[data-x-local-id]:checked")].map((item) => item.dataset.xLocalId);
|
||||
if (!selected.length) {
|
||||
setStatus("没有选中的本地 X 文件。");
|
||||
return;
|
||||
}
|
||||
setStatus(`正在删除 ${selected.length} 个本地 X 文件...`);
|
||||
const result = await api("/api/x/local-media/delete", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ ids: selected }),
|
||||
});
|
||||
setStatus(`删除完成:${result.deleted.length} 个,失败 ${result.failed.length} 个`);
|
||||
els.xLocalSelectAll.checked = false;
|
||||
await loadXLocalMedia();
|
||||
}
|
||||
|
||||
async function previewXhs(event) {
|
||||
event.preventDefault();
|
||||
const target = els.xhsTarget.value.trim();
|
||||
if (!target) {
|
||||
setStatus("请输入小红书关键词或笔记链接。");
|
||||
return;
|
||||
}
|
||||
els.xhsGrid.innerHTML = `<div class="empty">正在解析小红书内容...</div>`;
|
||||
els.xhsSummary.textContent = "解析中...";
|
||||
const payload = await api("/api/xiaohongshu/preview", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ target, limit: Number(els.xhsLimit.value || 20) }),
|
||||
});
|
||||
state.xhsPreviewId = payload.preview_id;
|
||||
renderXhs(payload);
|
||||
setStatus(`小红书解析完成:${payload.count} 个项目`);
|
||||
}
|
||||
|
||||
function renderXhs(payload) {
|
||||
els.xhsGrid.innerHTML = "";
|
||||
els.xhsSummary.textContent = `${payload.count} 个项目`;
|
||||
if (!payload.items.length) {
|
||||
els.xhsGrid.innerHTML = `<div class="empty">没有解析到内容,可能需要先登录或换一个关键词。</div>`;
|
||||
return;
|
||||
}
|
||||
els.xhsGrid.append(...payload.items.map(renderXhsCard));
|
||||
els.xhsSelectAll.checked = true;
|
||||
}
|
||||
|
||||
function renderXhsCard(item) {
|
||||
const card = document.createElement("article");
|
||||
card.className = "x-media-card";
|
||||
const preview = item.thumbnail || item.metadata?.cover_url || "";
|
||||
const media = preview
|
||||
? `<img loading="lazy" referrerpolicy="no-referrer" src="${escapeHtml(preview)}" alt="${escapeHtml(item.title || item.id)}">`
|
||||
: `<div class="thumb-placeholder">暂无预览</div>`;
|
||||
card.innerHTML = `
|
||||
<label class="x-check">
|
||||
<input type="checkbox" data-xhs-id="${escapeHtml(item.id)}" checked />
|
||||
<span>笔记</span>
|
||||
</label>
|
||||
<div class="x-thumb">${media}</div>
|
||||
<div class="artwork-meta">
|
||||
<div class="artwork-title">${escapeHtml(item.title || item.id)}</div>
|
||||
<div class="artwork-sub">${escapeHtml(item.author || "")} · ${item.file_count || 0} 个文件</div>
|
||||
</div>
|
||||
`;
|
||||
return card;
|
||||
}
|
||||
|
||||
async function downloadXhsSelected() {
|
||||
if (!state.xhsPreviewId) {
|
||||
setStatus("请先解析小红书内容。");
|
||||
return;
|
||||
}
|
||||
const selected = [...document.querySelectorAll("[data-xhs-id]:checked")].map((item) => item.dataset.xhsId);
|
||||
if (!selected.length) {
|
||||
setStatus("没有选中的小红书项目。");
|
||||
return;
|
||||
}
|
||||
setStatus(`正在下载 ${selected.length} 个小红书项目...`);
|
||||
const result = await api("/api/xiaohongshu/download", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ preview_id: state.xhsPreviewId, selected_ids: selected }),
|
||||
});
|
||||
setStatus(`小红书下载完成:保存 ${result.downloaded_files} 个文件,失败 ${result.failed_items} 项`);
|
||||
}
|
||||
|
||||
function setActiveView(view) {
|
||||
state.view = view;
|
||||
document.querySelectorAll(".nav-button").forEach((button) => {
|
||||
button.classList.toggle("active", button.dataset.view === view);
|
||||
});
|
||||
document.querySelectorAll(".view").forEach((section) => section.classList.remove("active"));
|
||||
if (view === "home") {
|
||||
document.querySelector("#homeView").classList.add("active");
|
||||
} else if (view === "artists") {
|
||||
document.querySelector("#artistsView").classList.add("active");
|
||||
} else if (view === "sync") {
|
||||
document.querySelector("#syncView").classList.add("active");
|
||||
} else if (view === "xMedia") {
|
||||
document.querySelector("#xMediaView").classList.add("active");
|
||||
} else if (view === "xiaohongshu") {
|
||||
document.querySelector("#xiaohongshuView").classList.add("active");
|
||||
} else {
|
||||
document.querySelector("#galleryView").classList.add("active");
|
||||
}
|
||||
}
|
||||
|
||||
async function reloadCurrent() {
|
||||
if (state.view === "home") {
|
||||
setStatus("首页已就绪。选择一个入口开始整理或同步内容。");
|
||||
} else if (state.view === "favorites") {
|
||||
els.title.textContent = "本地收藏";
|
||||
els.subtitle.textContent = "你标记收藏的作品";
|
||||
await loadFavorites();
|
||||
} else if (state.view === "artists") {
|
||||
await loadArtists();
|
||||
} else if (state.view === "sync") {
|
||||
await Promise.all([loadFollowed(), loadXFollowed()]);
|
||||
} else if (state.view === "xMedia") {
|
||||
await loadXLocalMedia();
|
||||
setStatus("X 作者媒体页已就绪。");
|
||||
} else if (state.view === "xiaohongshu") {
|
||||
setStatus("小红书插件页已就绪。");
|
||||
} else {
|
||||
if (!state.artistId) {
|
||||
els.title.textContent = state.tag ? `标签:${state.tag}` : "全部作品";
|
||||
els.subtitle.textContent = state.query ? `搜索:${state.query}` : "本地已入库作品";
|
||||
}
|
||||
await loadArtworks();
|
||||
}
|
||||
}
|
||||
|
||||
function clearFilters() {
|
||||
state.query = "";
|
||||
state.tag = "";
|
||||
state.type = "";
|
||||
state.artistId = "";
|
||||
state.offset = 0;
|
||||
els.search.value = "";
|
||||
els.type.value = "";
|
||||
}
|
||||
|
||||
function escapeHtml(value) {
|
||||
return String(value ?? "").replace(/[&<>"']/g, (char) => ({
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
'"': """,
|
||||
"'": "'",
|
||||
})[char]);
|
||||
}
|
||||
|
||||
function stripHtml(value) {
|
||||
const tmp = document.createElement("div");
|
||||
tmp.innerHTML = value;
|
||||
return tmp.textContent || tmp.innerText || "";
|
||||
}
|
||||
|
||||
function formatBytes(value) {
|
||||
const size = Number(value || 0);
|
||||
if (size < 1024) return `${size} B`;
|
||||
if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KB`;
|
||||
return `${(size / 1024 / 1024).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
document.querySelectorAll(".nav-button").forEach((button) => {
|
||||
button.addEventListener("click", async () => {
|
||||
setActiveView(button.dataset.view);
|
||||
state.offset = 0;
|
||||
if (state.view !== "gallery") {
|
||||
state.artistId = "";
|
||||
}
|
||||
await reloadCurrent();
|
||||
});
|
||||
});
|
||||
|
||||
els.search.addEventListener("input", debounce(async () => {
|
||||
state.query = els.search.value.trim();
|
||||
state.offset = 0;
|
||||
setActiveView("gallery");
|
||||
await reloadCurrent();
|
||||
}, 280));
|
||||
|
||||
els.type.addEventListener("change", async () => {
|
||||
state.type = els.type.value;
|
||||
state.offset = 0;
|
||||
setActiveView("gallery");
|
||||
await reloadCurrent();
|
||||
});
|
||||
|
||||
document.addEventListener("click", async (event) => {
|
||||
const target = event.target.closest("button");
|
||||
if (!target) return;
|
||||
if (target.dataset.viewAction) {
|
||||
setActiveView(target.dataset.viewAction);
|
||||
state.offset = 0;
|
||||
await reloadCurrent();
|
||||
return;
|
||||
}
|
||||
if (target.dataset.open) await openArtwork(target.dataset.open);
|
||||
if (target.dataset.fav) await toggleFavorite(target.dataset.fav, target.dataset.on);
|
||||
if (target.dataset.artist) await showArtist(target.dataset.artist);
|
||||
if (target.dataset.tag) {
|
||||
state.tag = target.dataset.tag;
|
||||
state.artistId = "";
|
||||
state.offset = 0;
|
||||
setActiveView("gallery");
|
||||
await loadTags();
|
||||
await reloadCurrent();
|
||||
}
|
||||
if (target.dataset.follow) {
|
||||
const nextFollowed = target.dataset.followed === "1" ? false : true;
|
||||
await setPixivFollow(target.dataset.follow, target.dataset.name || "", nextFollowed);
|
||||
}
|
||||
if (target.dataset.followX) {
|
||||
const nextFollowed = target.dataset.followedX === "1" ? false : true;
|
||||
await setXFollow(target.dataset.followX, target.dataset.name || "", nextFollowed);
|
||||
}
|
||||
if (target.dataset.openX) {
|
||||
els.xUsername.value = `@${target.dataset.openX}`;
|
||||
setActiveView("xMedia");
|
||||
setStatus(`已填入 @${target.dataset.openX},可以直接解析最新媒体。`);
|
||||
}
|
||||
});
|
||||
|
||||
document.querySelector("#clearFiltersButton").addEventListener("click", async () => {
|
||||
clearFilters();
|
||||
await loadTags();
|
||||
await reloadCurrent();
|
||||
});
|
||||
|
||||
document.querySelector("#prevPageButton").addEventListener("click", async () => {
|
||||
state.offset = Math.max(0, state.offset - state.limit);
|
||||
await reloadCurrent();
|
||||
});
|
||||
|
||||
document.querySelector("#nextPageButton").addEventListener("click", async () => {
|
||||
if (state.offset + state.limit < state.total) {
|
||||
state.offset += state.limit;
|
||||
await reloadCurrent();
|
||||
}
|
||||
});
|
||||
|
||||
document.querySelector("#scanButton").addEventListener("click", scanLegacy);
|
||||
document.querySelector("#homeScanButton")?.addEventListener("click", scanLegacy);
|
||||
document.querySelector("#originFollowSyncButton").addEventListener("click", openOriginFollowSyncDialog);
|
||||
document.querySelector("#closeOriginFollowSyncDialogButton").addEventListener("click", () => els.originFollowSyncDialog.close());
|
||||
document.querySelector("#syncOriginPixivButton").addEventListener("click", () => syncOriginFollows("pixiv"));
|
||||
document.querySelector("#syncOriginXButton").addEventListener("click", () => syncOriginFollows("x"));
|
||||
document.querySelector("#syncButton").addEventListener("click", syncFollowed);
|
||||
document.querySelector("#syncXButton").addEventListener("click", syncXFollowed);
|
||||
document.querySelector("#buildThumbsButton").addEventListener("click", buildThumbnails);
|
||||
document.querySelector("#homeThumbButton")?.addEventListener("click", buildThumbnails);
|
||||
document.querySelector("#closeDialogButton").addEventListener("click", () => els.dialog.close());
|
||||
document.querySelector("#importPixivCookieButton")?.addEventListener("click", () => importCookieFile("pixiv", els.pixivCookieFile));
|
||||
els.refreshSessionButton?.addEventListener("click", loadSessionStatus);
|
||||
document.querySelector("#importXCookieButton").addEventListener("click", () => importCookieFile("x", els.xCookieFile));
|
||||
document.querySelector("#xMediaForm").addEventListener("submit", previewXMedia);
|
||||
document.querySelector("#followCurrentXButton").addEventListener("click", followCurrentX);
|
||||
document.querySelector("#downloadXSelectedButton").addEventListener("click", downloadXSelected);
|
||||
document.querySelector("#refreshXLocalButton").addEventListener("click", loadXLocalMedia);
|
||||
document.querySelector("#deleteXLocalButton").addEventListener("click", deleteXLocalSelected);
|
||||
document.querySelector("#xhsForm").addEventListener("submit", previewXhs);
|
||||
document.querySelector("#downloadXhsSelectedButton").addEventListener("click", downloadXhsSelected);
|
||||
els.xSelectAll.addEventListener("change", () => {
|
||||
document.querySelectorAll("[data-x-id]").forEach((item) => {
|
||||
item.checked = els.xSelectAll.checked;
|
||||
});
|
||||
});
|
||||
els.xLocalSelectAll.addEventListener("change", () => {
|
||||
document.querySelectorAll("[data-x-local-id]").forEach((item) => {
|
||||
item.checked = els.xLocalSelectAll.checked;
|
||||
});
|
||||
});
|
||||
els.xhsSelectAll.addEventListener("change", () => {
|
||||
document.querySelectorAll("[data-xhs-id]").forEach((item) => {
|
||||
item.checked = els.xhsSelectAll.checked;
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelector("#followForm").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const id = document.querySelector("#followArtistId").value.trim();
|
||||
const name = document.querySelector("#followArtistName").value.trim();
|
||||
if (!id) return;
|
||||
await followArtist(id, name);
|
||||
event.target.reset();
|
||||
});
|
||||
|
||||
document.querySelector("#xFollowForm").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const username = els.xFollowUsername.value.trim();
|
||||
const name = els.xFollowName.value.trim();
|
||||
if (!username) return;
|
||||
await followXAuthor(username, name);
|
||||
event.target.reset();
|
||||
});
|
||||
|
||||
function debounce(fn, delay) {
|
||||
let timer = 0;
|
||||
return (...args) => {
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(() => fn(...args), delay);
|
||||
};
|
||||
}
|
||||
|
||||
async function boot() {
|
||||
try {
|
||||
await Promise.all([loadPluginPages(), loadSessionStatus(), loadTags(), loadArtworks(), loadFollowed(), loadXFollowed()]);
|
||||
} catch (error) {
|
||||
setStatus(error.message);
|
||||
}
|
||||
}
|
||||
|
||||
boot();
|
||||
@@ -0,0 +1,284 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Sakura 本地媒体库</title>
|
||||
<link rel="stylesheet" href="/assets/styles.css" />
|
||||
</head>
|
||||
<body>
|
||||
<aside class="sidebar">
|
||||
<div class="brand">
|
||||
<div class="brand-mark">SK</div>
|
||||
<div>
|
||||
<h1>Sakura</h1>
|
||||
<p>本地媒体库</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<nav class="nav">
|
||||
<button class="nav-button active" data-view="home">首页</button>
|
||||
<button class="nav-button" data-view="gallery">图库</button>
|
||||
<button class="nav-button" data-view="favorites">收藏</button>
|
||||
<button class="nav-button" data-view="artists">作者</button>
|
||||
<button class="nav-button" data-view="sync">关注同步</button>
|
||||
</nav>
|
||||
|
||||
<section class="panel plugin-panel">
|
||||
<h2>插件页面</h2>
|
||||
<div id="pluginNav" class="nav compact-nav">
|
||||
<button class="nav-button" data-view="xMedia">X 作者媒体</button>
|
||||
<button id="xiaohongshuNavButton" class="nav-button hidden" data-view="xiaohongshu">小红书</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel">
|
||||
<h2>标签</h2>
|
||||
<div id="tagList" class="tag-list"></div>
|
||||
</section>
|
||||
</aside>
|
||||
|
||||
<main class="main">
|
||||
<header class="toolbar">
|
||||
<div class="search-wrap">
|
||||
<input id="searchInput" type="search" placeholder="搜索标题、作者、简介或标签" />
|
||||
<select id="typeFilter" aria-label="类型筛选">
|
||||
<option value="">全部类型</option>
|
||||
<option value="illust">插画</option>
|
||||
<option value="manga">漫画</option>
|
||||
<option value="novel">小说</option>
|
||||
<option value="video">视频</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="toolbar-actions">
|
||||
<button id="buildThumbsButton" class="ghost-button">生成缩略图</button>
|
||||
<button id="scanButton" class="primary-button">扫描旧目录</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section id="statusLine" class="status-line">准备就绪</section>
|
||||
|
||||
<section id="homeView" class="view active">
|
||||
<div class="welcome-card">
|
||||
<div>
|
||||
<p class="eyebrow">本地优先</p>
|
||||
<h2>今天想看什么?</h2>
|
||||
<p>插画、漫画、小说和视频都保存在本机。你可以直接浏览、搜索和收藏。</p>
|
||||
</div>
|
||||
<button class="primary-button" data-view-action="gallery">打开图库</button>
|
||||
</div>
|
||||
|
||||
<div class="quick-grid">
|
||||
<button class="quick-card" data-view-action="gallery">
|
||||
<strong>查看本地图库</strong>
|
||||
<span>按标题、作者、类型和标签筛选已经入库的作品。</span>
|
||||
</button>
|
||||
<button class="quick-card" data-view-action="sync">
|
||||
<strong>同步关注作者</strong>
|
||||
<span>读取 Pixiv / X 关注列表,把新作品下载到本地媒体库。</span>
|
||||
</button>
|
||||
<button class="quick-card" id="homeScanButton" type="button">
|
||||
<strong>导入旧下载目录</strong>
|
||||
<span>把 Sakura_Downloads 里的旧文件重新扫描进图库。</span>
|
||||
</button>
|
||||
<button class="quick-card" id="homeThumbButton" type="button">
|
||||
<strong>修复缩略图</strong>
|
||||
<span>图片不显示时,重新生成缺失的预览图。</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="help-steps">
|
||||
<article>
|
||||
<b>1</b>
|
||||
<div>
|
||||
<strong>在 EXE 主界面导入 Cookie</strong>
|
||||
<span>Pixiv、X、小红书等登录态会保存到本地运行目录。</span>
|
||||
</div>
|
||||
</article>
|
||||
<article>
|
||||
<b>2</b>
|
||||
<div>
|
||||
<strong>选择平台和目标后开始抓取</strong>
|
||||
<span>作者、单作品、关键词、插件链接都可以从主界面发起。</span>
|
||||
</div>
|
||||
</article>
|
||||
<article>
|
||||
<b>3</b>
|
||||
<div>
|
||||
<strong>回到图库整理内容</strong>
|
||||
<span>收藏、关注作者、生成缩略图和查看详情都在这里完成。</span>
|
||||
</div>
|
||||
</article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="galleryView" class="view">
|
||||
<div class="section-head">
|
||||
<div>
|
||||
<h2 id="viewTitle">全部作品</h2>
|
||||
<p id="viewSubtitle">本地已入库作品</p>
|
||||
</div>
|
||||
<button id="clearFiltersButton" class="ghost-button">清除筛选</button>
|
||||
</div>
|
||||
<div id="artworkGrid" class="artwork-grid"></div>
|
||||
<div class="pager">
|
||||
<button id="prevPageButton" class="ghost-button">上一页</button>
|
||||
<span id="pageInfo">0 / 0</span>
|
||||
<button id="nextPageButton" class="ghost-button">下一页</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="artistsView" class="view">
|
||||
<div class="section-head">
|
||||
<div>
|
||||
<h2>作者</h2>
|
||||
<p>按本地作品数量排序</p>
|
||||
</div>
|
||||
</div>
|
||||
<div id="artistList" class="artist-list"></div>
|
||||
</section>
|
||||
|
||||
<section id="syncView" class="view">
|
||||
<div class="import-strip">
|
||||
<div>
|
||||
<strong>同步原站关注</strong>
|
||||
<span>读取当前 Pixiv / X 登录账号的关注列表,保存到本地。</span>
|
||||
</div>
|
||||
<button id="originFollowSyncButton" class="primary-button" type="button">同步原站关注</button>
|
||||
</div>
|
||||
<div class="section-head">
|
||||
<div>
|
||||
<h2>关注同步</h2>
|
||||
<p>使用已保存 Cookie 同步关注作者的新作品</p>
|
||||
</div>
|
||||
<button id="syncButton" class="primary-button">同步关注作者</button>
|
||||
</div>
|
||||
<div class="import-strip session-strip">
|
||||
<div>
|
||||
<strong>Pixiv 登录态</strong>
|
||||
<span id="pixivSessionStatus">正在读取桌面端已保存的 Pixiv 会话...</span>
|
||||
</div>
|
||||
<button id="refreshSessionButton" class="ghost-button" type="button">刷新会话状态</button>
|
||||
</div>
|
||||
<form id="followForm" class="follow-form">
|
||||
<input id="followArtistId" inputmode="text" placeholder="作者 ID、作者名或主页链接" />
|
||||
<input id="followArtistName" placeholder="备注名称,可选" />
|
||||
<button class="ghost-button" type="submit">添加关注</button>
|
||||
</form>
|
||||
<div id="followedList" class="artist-list"></div>
|
||||
<div class="section-head">
|
||||
<div>
|
||||
<h2>X 关注</h2>
|
||||
<p>按时间线检查 X 作者的最新媒体推文,并保存到本地媒体库。</p>
|
||||
</div>
|
||||
<button id="syncXButton" class="primary-button" type="button">同步 X 关注</button>
|
||||
</div>
|
||||
<form id="xFollowForm" class="follow-form">
|
||||
<input id="xFollowUsername" placeholder="X 用户名,例如 @nasa" />
|
||||
<input id="xFollowName" placeholder="备注名称,可选" />
|
||||
<button class="ghost-button" type="submit">添加 X 关注</button>
|
||||
</form>
|
||||
<div id="xFollowedList" class="artist-list"></div>
|
||||
</section>
|
||||
|
||||
<section id="xMediaView" class="view">
|
||||
<div class="section-head">
|
||||
<div>
|
||||
<h2>X 作者媒体</h2>
|
||||
<p>输入作者名称,解析媒体页中的图片或视频,勾选后下载到本地。</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="import-strip">
|
||||
<div>
|
||||
<strong>X Cookie</strong>
|
||||
<span>导入 Get cookies.txt 导出的 txt/json,会写入 runtime/x_cookies.json。</span>
|
||||
</div>
|
||||
<label class="file-button">
|
||||
选择 txt
|
||||
<input id="xCookieFile" type="file" accept=".txt,.json" />
|
||||
</label>
|
||||
<button id="importXCookieButton" class="ghost-button" type="button">导入 X Cookie</button>
|
||||
</div>
|
||||
|
||||
<form id="xMediaForm" class="x-media-form">
|
||||
<input id="xUsernameInput" placeholder="作者名称,例如 nasa 或 @nasa" autocomplete="off" />
|
||||
<select id="xMediaType">
|
||||
<option value="all">图片和视频</option>
|
||||
<option value="image">只看图片</option>
|
||||
<option value="video">只看视频</option>
|
||||
</select>
|
||||
<input id="xMaxItems" type="number" min="1" max="500" value="120" />
|
||||
<button class="primary-button" type="submit">解析媒体</button>
|
||||
</form>
|
||||
|
||||
<div class="x-actions">
|
||||
<label><input id="xSelectAll" type="checkbox" checked /> 全选</label>
|
||||
<button id="followCurrentXButton" class="ghost-button" type="button">关注当前作者</button>
|
||||
<button id="downloadXSelectedButton" class="primary-button" type="button">下载选中</button>
|
||||
<span id="xMediaSummary">尚未解析</span>
|
||||
</div>
|
||||
|
||||
<div id="xMediaGrid" class="x-media-grid"></div>
|
||||
|
||||
<div class="section-head x-local-head">
|
||||
<div>
|
||||
<h2>本地 X 媒体</h2>
|
||||
<p>查看已下载的 X 图片和视频,可删除本地文件。</p>
|
||||
</div>
|
||||
<div class="toolbar-actions">
|
||||
<button id="refreshXLocalButton" class="ghost-button" type="button">刷新</button>
|
||||
<button id="deleteXLocalButton" class="ghost-button danger-button" type="button">删除选中</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="x-actions">
|
||||
<label><input id="xLocalSelectAll" type="checkbox" /> 全选本地文件</label>
|
||||
<span id="xLocalSummary">等待加载</span>
|
||||
</div>
|
||||
<div id="xLocalGrid" class="x-media-grid"></div>
|
||||
</section>
|
||||
|
||||
<section id="xiaohongshuView" class="view">
|
||||
<div class="section-head">
|
||||
<div>
|
||||
<h2>小红书</h2>
|
||||
<p>输入关键词或笔记链接,先预览再勾选下载。</p>
|
||||
</div>
|
||||
</div>
|
||||
<form id="xhsForm" class="x-media-form">
|
||||
<input id="xhsTargetInput" placeholder="关键词,或 xiaohongshu.com/explore/... 链接" autocomplete="off" />
|
||||
<input id="xhsLimitInput" type="number" min="1" max="50" value="20" />
|
||||
<button class="primary-button" type="submit">解析预览</button>
|
||||
</form>
|
||||
<div class="x-actions">
|
||||
<label><input id="xhsSelectAll" type="checkbox" checked /> 全选</label>
|
||||
<button id="downloadXhsSelectedButton" class="primary-button" type="button">下载选中</button>
|
||||
<span id="xhsSummary">等待解析</span>
|
||||
</div>
|
||||
<div id="xhsGrid" class="x-media-grid"></div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<dialog id="originFollowSyncDialog" class="detail-dialog">
|
||||
<div class="dialog-head">
|
||||
<div>
|
||||
<strong>同步原站关注</strong>
|
||||
<span>选择平台后,会读取当前登录账号的关注列表并保存到本地。</span>
|
||||
</div>
|
||||
<button id="closeOriginFollowSyncDialogButton" class="ghost-button" type="button">关闭</button>
|
||||
</div>
|
||||
<div class="dialog-actions">
|
||||
<button id="syncOriginPixivButton" class="primary-button" type="button">Pixiv</button>
|
||||
<button id="syncOriginXButton" class="primary-button" type="button">X</button>
|
||||
</div>
|
||||
<pre id="originFollowSyncResult" class="sync-result"></pre>
|
||||
</dialog>
|
||||
|
||||
<dialog id="detailDialog" class="detail-dialog">
|
||||
<button id="closeDialogButton" class="close-button" aria-label="关闭">×</button>
|
||||
<div id="detailContent"></div>
|
||||
</dialog>
|
||||
|
||||
<script src="/assets/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,756 @@
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #f6f7f9;
|
||||
--surface: #ffffff;
|
||||
--surface-soft: #faf5f7;
|
||||
--line: #e5dce1;
|
||||
--text: #242229;
|
||||
--muted: #706b73;
|
||||
--primary: #cc5f91;
|
||||
--primary-strong: #af477a;
|
||||
--accent: #277d78;
|
||||
--ok: #3f9b72;
|
||||
--shadow: 0 10px 28px rgba(48, 39, 44, 0.09);
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
display: grid;
|
||||
grid-template-columns: 248px minmax(0, 1fr);
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font-family: "Segoe UI", "Microsoft YaHei", Arial, sans-serif;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
button,
|
||||
input,
|
||||
select {
|
||||
font: inherit;
|
||||
}
|
||||
|
||||
button {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.sidebar {
|
||||
min-height: 100vh;
|
||||
border-right: 1px solid var(--line);
|
||||
background: #ffffff;
|
||||
padding: 22px 18px;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.brand {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
.brand-mark {
|
||||
width: 46px;
|
||||
height: 46px;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
background: var(--primary);
|
||||
color: white;
|
||||
border-radius: 8px;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.brand h1,
|
||||
.brand p,
|
||||
.panel h2,
|
||||
.section-head h2,
|
||||
.section-head p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.brand h1 {
|
||||
font-size: 19px;
|
||||
font-weight: 750;
|
||||
}
|
||||
|
||||
.brand p,
|
||||
.section-head p {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
margin-top: 3px;
|
||||
}
|
||||
|
||||
.nav {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
margin-bottom: 22px;
|
||||
}
|
||||
|
||||
.nav-button,
|
||||
.ghost-button,
|
||||
.primary-button {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 10px;
|
||||
min-height: 40px;
|
||||
padding: 0 14px;
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.nav-button {
|
||||
text-align: left;
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
.nav-button.active,
|
||||
.primary-button {
|
||||
background: var(--primary);
|
||||
border-color: var(--primary);
|
||||
color: white;
|
||||
box-shadow: 0 8px 18px rgba(204, 95, 145, 0.2);
|
||||
}
|
||||
|
||||
.primary-button:hover {
|
||||
background: var(--primary-strong);
|
||||
}
|
||||
|
||||
.ghost-button:hover,
|
||||
.nav-button:hover {
|
||||
border-color: var(--primary);
|
||||
color: var(--primary);
|
||||
}
|
||||
|
||||
.danger-button:hover {
|
||||
border-color: var(--accent);
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.panel {
|
||||
border-top: 1px solid var(--line);
|
||||
padding-top: 18px;
|
||||
}
|
||||
|
||||
.plugin-panel {
|
||||
margin-bottom: 18px;
|
||||
}
|
||||
|
||||
.panel h2 {
|
||||
font-size: 14px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.compact-nav {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.tag-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.tag-chip {
|
||||
border: 1px solid var(--line);
|
||||
background: #fff7fb;
|
||||
border-radius: 999px;
|
||||
padding: 6px 10px;
|
||||
color: var(--muted);
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.tag-chip.active {
|
||||
color: white;
|
||||
background: var(--accent);
|
||||
border-color: var(--accent);
|
||||
}
|
||||
|
||||
.main {
|
||||
min-width: 0;
|
||||
padding: 24px 30px 40px;
|
||||
}
|
||||
|
||||
.welcome-card {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: #ffffff;
|
||||
box-shadow: var(--shadow);
|
||||
padding: 28px 30px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 20px;
|
||||
margin-bottom: 18px;
|
||||
}
|
||||
|
||||
.welcome-card h2 {
|
||||
margin: 0 0 8px;
|
||||
font-size: 28px;
|
||||
line-height: 1.22;
|
||||
}
|
||||
|
||||
.welcome-card p {
|
||||
margin: 0;
|
||||
color: var(--muted);
|
||||
max-width: 660px;
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
.eyebrow {
|
||||
color: var(--primary) !important;
|
||||
font-size: 13px;
|
||||
font-weight: 750;
|
||||
margin-bottom: 6px !important;
|
||||
}
|
||||
|
||||
.quick-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 14px;
|
||||
margin-bottom: 18px;
|
||||
}
|
||||
|
||||
.quick-card {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: var(--surface);
|
||||
min-height: 132px;
|
||||
padding: 18px;
|
||||
text-align: left;
|
||||
box-shadow: 0 8px 22px rgba(130, 66, 98, 0.08);
|
||||
}
|
||||
|
||||
.quick-card:hover {
|
||||
border-color: var(--primary);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.quick-card strong,
|
||||
.quick-card span {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.quick-card strong {
|
||||
font-size: 17px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.quick-card span {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
line-height: 1.55;
|
||||
}
|
||||
|
||||
.help-steps {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.help-steps article {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: rgba(255, 255, 255, 0.86);
|
||||
padding: 16px;
|
||||
display: flex;
|
||||
gap: 14px;
|
||||
align-items: flex-start;
|
||||
}
|
||||
|
||||
.help-steps b {
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
border-radius: 999px;
|
||||
background: #ffe3ef;
|
||||
color: var(--primary);
|
||||
display: grid;
|
||||
place-items: center;
|
||||
}
|
||||
|
||||
.help-steps strong,
|
||||
.help-steps span {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.help-steps span {
|
||||
color: var(--muted);
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.search-wrap {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(220px, 420px) 150px;
|
||||
gap: 10px;
|
||||
width: min(100%, 610px);
|
||||
}
|
||||
|
||||
.search-wrap input,
|
||||
.search-wrap select,
|
||||
.follow-form input,
|
||||
.x-media-form input,
|
||||
.x-media-form select {
|
||||
height: 42px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
padding: 0 12px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.toolbar-actions {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.status-line {
|
||||
min-height: 32px;
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
background: rgba(255, 255, 255, 0.72);
|
||||
border: 1px solid rgba(240, 213, 226, 0.8);
|
||||
border-radius: 8px;
|
||||
padding: 8px 12px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.view {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.view.active {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.hidden {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
.section-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.section-head h2 {
|
||||
font-size: 24px;
|
||||
}
|
||||
|
||||
.artwork-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(168px, 1fr));
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.artwork-card {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 12px;
|
||||
background: var(--surface);
|
||||
overflow: hidden;
|
||||
box-shadow: 0 8px 22px rgba(130, 66, 98, 0.08);
|
||||
}
|
||||
|
||||
.thumb-button {
|
||||
border: 0;
|
||||
padding: 0;
|
||||
width: 100%;
|
||||
aspect-ratio: 1 / 1;
|
||||
background: #dde5ec;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.thumb-button img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.thumb-placeholder {
|
||||
height: 100%;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: var(--muted);
|
||||
padding: 16px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.artwork-meta {
|
||||
padding: 10px;
|
||||
display: grid;
|
||||
gap: 7px;
|
||||
}
|
||||
|
||||
.artwork-title {
|
||||
font-weight: 650;
|
||||
line-height: 1.35;
|
||||
min-height: 38px;
|
||||
}
|
||||
|
||||
.artwork-sub {
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.card-actions {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.small-button {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 7px;
|
||||
background: var(--surface-soft);
|
||||
min-height: 30px;
|
||||
padding: 0 9px;
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.small-button.active {
|
||||
background: var(--accent);
|
||||
border-color: var(--accent);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.pager {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 12px;
|
||||
margin: 22px 0 0;
|
||||
}
|
||||
|
||||
.artist-list {
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.artist-row {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 12px;
|
||||
background: var(--surface);
|
||||
min-height: 66px;
|
||||
padding: 12px 14px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.artist-row strong {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.artist-row span {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.follow-form {
|
||||
display: grid;
|
||||
grid-template-columns: 160px minmax(180px, 320px) 110px;
|
||||
gap: 10px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.import-strip {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 12px;
|
||||
background: var(--surface);
|
||||
padding: 12px;
|
||||
display: grid;
|
||||
grid-template-columns: minmax(220px, 1fr) auto auto;
|
||||
gap: 10px;
|
||||
align-items: center;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.import-strip strong,
|
||||
.import-strip span {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.import-strip span,
|
||||
.x-actions {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.file-button {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
min-height: 38px;
|
||||
padding: 0 14px;
|
||||
display: inline-grid;
|
||||
place-items: center;
|
||||
background: var(--surface-soft);
|
||||
color: var(--text);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.file-button input {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.x-media-form {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(220px, 1fr) 150px 120px;
|
||||
gap: 10px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
#xMediaForm {
|
||||
grid-template-columns: minmax(220px, 1fr) 150px 110px 120px;
|
||||
}
|
||||
|
||||
.x-actions {
|
||||
min-height: 38px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 14px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.x-actions label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.x-media-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(180px, 1fr));
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.x-media-card {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: var(--surface);
|
||||
overflow: hidden;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.x-local-head {
|
||||
border-top: 1px solid var(--line);
|
||||
padding-top: 22px;
|
||||
margin-top: 28px;
|
||||
}
|
||||
|
||||
.x-check {
|
||||
position: absolute;
|
||||
z-index: 2;
|
||||
left: 8px;
|
||||
top: 8px;
|
||||
min-height: 30px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
border-radius: 8px;
|
||||
background: rgba(255, 255, 255, 0.92);
|
||||
padding: 4px 8px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.x-thumb {
|
||||
width: 100%;
|
||||
aspect-ratio: 1 / 1;
|
||||
background: #101820;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
}
|
||||
|
||||
.x-thumb img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.x-thumb video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: contain;
|
||||
background: #101820;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.video-tile {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: white;
|
||||
background: #223140;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.detail-dialog {
|
||||
width: min(980px, calc(100vw - 36px));
|
||||
max-height: calc(100vh - 36px);
|
||||
border: 0;
|
||||
border-radius: 14px;
|
||||
padding: 0;
|
||||
box-shadow: var(--shadow);
|
||||
}
|
||||
|
||||
.detail-dialog::backdrop {
|
||||
background: rgba(9, 18, 28, 0.5);
|
||||
}
|
||||
|
||||
.dialog-head {
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
padding: 20px 20px 0;
|
||||
}
|
||||
|
||||
.dialog-head strong,
|
||||
.dialog-head span {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.dialog-head span {
|
||||
color: var(--muted);
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.dialog-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
padding: 18px 20px 0;
|
||||
}
|
||||
|
||||
.sync-result {
|
||||
background: #fff7fa;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
color: var(--text);
|
||||
margin: 18px 20px 20px;
|
||||
max-height: 260px;
|
||||
overflow: auto;
|
||||
padding: 12px;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
.close-button {
|
||||
position: absolute;
|
||||
right: 12px;
|
||||
top: 12px;
|
||||
width: 34px;
|
||||
height: 34px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 50%;
|
||||
background: var(--surface);
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
.detail-layout {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(260px, 52%) minmax(260px, 1fr);
|
||||
min-height: 520px;
|
||||
}
|
||||
|
||||
.detail-media {
|
||||
background: #101820;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.detail-media img {
|
||||
max-width: 100%;
|
||||
max-height: 72vh;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
.detail-info {
|
||||
padding: 28px;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
.detail-info h2 {
|
||||
margin: 0 0 8px;
|
||||
font-size: 26px;
|
||||
}
|
||||
|
||||
.detail-info p {
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.detail-tags {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin: 16px 0;
|
||||
}
|
||||
|
||||
.empty {
|
||||
border: 1px dashed var(--line);
|
||||
border-radius: 14px;
|
||||
padding: 34px;
|
||||
background: var(--surface);
|
||||
color: var(--muted);
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
@media (max-width: 860px) {
|
||||
body {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.sidebar {
|
||||
position: static;
|
||||
min-height: auto;
|
||||
}
|
||||
|
||||
.quick-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.welcome-card {
|
||||
align-items: stretch;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.toolbar,
|
||||
.section-head,
|
||||
.artist-row {
|
||||
align-items: stretch;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.search-wrap,
|
||||
.follow-form,
|
||||
.import-strip,
|
||||
.x-media-form {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.toolbar-actions {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.detail-layout {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
"""Bundled AO3 (Archive of Our Own) plugin package."""
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "AO3",
|
||||
"version": "1.0.0",
|
||||
"domain": "archiveofourown.org",
|
||||
"author": "SakuraHub",
|
||||
"description": "AO3 fanfiction archive novel plugin. Supports work URL parsing, metadata extraction, EPUB/HTML download, cookie-based auth, and update checking.",
|
||||
"requires_auth": false,
|
||||
"min_platform_version": "1.0.0",
|
||||
"tags": ["novel", "fanfiction", "ao3"],
|
||||
"dependencies": []
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "笔趣阁",
|
||||
"version": "3.0.0",
|
||||
"domain": "www.bqg551.cc",
|
||||
"author": "Sakura Downloader",
|
||||
"description": "基于 bqg551 阅读站公开 API 的小说搜索、目录解析、章节抓取与 TXT 导出。",
|
||||
"requires_auth": false,
|
||||
"tags": ["novel", "bqg551", "biquge", "bookshelf"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
Loaded 100 of 412 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user