commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,19 @@
|
||||
"""Task models, SQLite-backed queue, batch parser, and watermark helpers."""
|
||||
|
||||
from pixiv_app.tasks.models import DownloadTaskSpec, ParsePreview, ParsedLine, TaskPriority
|
||||
from pixiv_app.tasks.parser import parse_batch_for_mode, split_input_blob
|
||||
from pixiv_app.tasks.queue import TaskQueue
|
||||
from pixiv_app.tasks.watermark import advance_watermark, filter_newer_than_watermark, get_last_synced_illust_id
|
||||
|
||||
__all__ = [
|
||||
"DownloadTaskSpec",
|
||||
"ParsePreview",
|
||||
"ParsedLine",
|
||||
"TaskPriority",
|
||||
"TaskQueue",
|
||||
"advance_watermark",
|
||||
"filter_newer_than_watermark",
|
||||
"get_last_synced_illust_id",
|
||||
"parse_batch_for_mode",
|
||||
"split_input_blob",
|
||||
]
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import IntEnum
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
class TaskPriority(IntEnum):
|
||||
LOW = 1
|
||||
MEDIUM = 2
|
||||
HIGH = 3
|
||||
|
||||
|
||||
TaskStatus = Literal["pending", "running", "done", "failed"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DownloadTaskSpec:
|
||||
"""Serializable unit stored in SQLite via TaskQueue (payload omits secrets)."""
|
||||
|
||||
target_type: Literal["illust", "novel"]
|
||||
target_id: int
|
||||
task_kind: str
|
||||
priority: TaskPriority
|
||||
page_index: int | None = None
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def fingerprint(self) -> str:
|
||||
page_part = self.page_index if self.page_index is not None else -1
|
||||
return f"{self.task_kind}:{self.target_type}:{self.target_id}:{page_part}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedLine:
|
||||
"""One logical input token after splitting (before expansion to illust ids)."""
|
||||
|
||||
category: Literal["user", "illust", "novel", "tag", "keyword"]
|
||||
id_value: int
|
||||
tag: str = ""
|
||||
keyword_text: str = ""
|
||||
raw_token: str = ""
|
||||
pixiv_kind: str = "user"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsePreview:
|
||||
"""Statistics for GUI preview."""
|
||||
|
||||
total_lines: int
|
||||
by_category: dict[str, int]
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Literal
|
||||
|
||||
from pixiv_app.core.downloader import parse_pixiv_target
|
||||
from pixiv_app.tasks.models import ParsePreview, ParsedLine
|
||||
|
||||
|
||||
def split_input_blob(blob: str) -> list[str]:
|
||||
parts = re.split(r"[\s,,;;\n\r]+", blob.strip())
|
||||
return [p for p in parts if p]
|
||||
|
||||
|
||||
def parse_batch_for_mode(
|
||||
raw: str,
|
||||
*,
|
||||
ui_mode: Literal["user", "illust", "manga", "novel", "keyword"],
|
||||
) -> tuple[list[ParsedLine], ParsePreview]:
|
||||
"""
|
||||
Parse multi-line / comma / space separated input into structured intents.
|
||||
|
||||
ui_mode matches GUI combo selection so validation stays aligned with existing rules.
|
||||
"""
|
||||
tokens = split_input_blob(raw)
|
||||
lines: list[ParsedLine] = []
|
||||
stats: dict[str, int] = {}
|
||||
|
||||
def bump(key: str) -> None:
|
||||
stats[key] = stats.get(key, 0) + 1
|
||||
|
||||
if ui_mode == "keyword":
|
||||
kw = " ".join(tokens) if tokens else raw.strip()
|
||||
if kw.startswith("#"):
|
||||
kw = kw[1:].strip()
|
||||
if kw:
|
||||
lines.append(ParsedLine(category="keyword", id_value=0, keyword_text=kw, raw_token=raw.strip()))
|
||||
bump("keyword")
|
||||
return lines, ParsePreview(total_lines=len(lines), by_category=stats)
|
||||
|
||||
for token in tokens:
|
||||
t = token.strip()
|
||||
if not t:
|
||||
continue
|
||||
if t.startswith("#"):
|
||||
tag = t[1:].strip()
|
||||
if tag:
|
||||
lines.append(ParsedLine(category="tag", id_value=0, tag=tag, raw_token=t))
|
||||
bump("tag")
|
||||
continue
|
||||
|
||||
try:
|
||||
kind, num_id = parse_pixiv_target(t)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if ui_mode == "novel":
|
||||
if kind == "novel":
|
||||
lines.append(ParsedLine(category="novel", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("novel")
|
||||
elif kind == "novel_series":
|
||||
lines.append(ParsedLine(category="novel", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("novel")
|
||||
elif kind == "unknown":
|
||||
lines.append(ParsedLine(category="novel", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("novel")
|
||||
continue
|
||||
|
||||
if ui_mode in {"illust", "manga"}:
|
||||
if kind == "illust":
|
||||
lines.append(ParsedLine(category="illust", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("illust")
|
||||
elif kind == "unknown":
|
||||
lines.append(ParsedLine(category="illust", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("illust")
|
||||
continue
|
||||
|
||||
# ui_mode == "user"
|
||||
if kind == "user":
|
||||
lines.append(ParsedLine(category="user", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("user")
|
||||
elif kind == "illust":
|
||||
lines.append(ParsedLine(category="user", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("user")
|
||||
elif kind == "unknown":
|
||||
lines.append(ParsedLine(category="user", id_value=num_id, raw_token=t, pixiv_kind=kind))
|
||||
bump("user")
|
||||
|
||||
return lines, ParsePreview(total_lines=len(lines), by_category=stats)
|
||||
@@ -0,0 +1,238 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pixiv_app.tasks.models import DownloadTaskSpec
|
||||
|
||||
|
||||
class TaskQueue:
|
||||
"""SQLite-backed queue with lease/reclaim; thread-safe for concurrent workers."""
|
||||
|
||||
def __init__(self, conn: sqlite3.Connection) -> None:
|
||||
self._conn = conn
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def enqueue(self, tasks: list[DownloadTaskSpec]) -> tuple[int, int]:
|
||||
inserted = 0
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
for t in tasks:
|
||||
fp = t.fingerprint()
|
||||
cur = self._conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO download_tasks (
|
||||
fingerprint, target_type, target_id, page_index, task_kind,
|
||||
priority, status, payload_json
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)
|
||||
""",
|
||||
(
|
||||
fp,
|
||||
t.target_type,
|
||||
t.target_id,
|
||||
t.page_index,
|
||||
t.task_kind,
|
||||
int(t.priority),
|
||||
json.dumps(t.payload, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
inserted += cur.rowcount
|
||||
skipped = len(tasks) - inserted
|
||||
return inserted, skipped
|
||||
|
||||
def requeue_failed_by_fingerprints(self, fingerprints: list[str]) -> int:
|
||||
"""Reset `failed` rows to `pending` so a retry can pick them up."""
|
||||
if not fingerprints:
|
||||
return 0
|
||||
placeholders = ",".join("?" * len(fingerprints))
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
cur = self._conn.execute(
|
||||
f"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'pending',
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = '',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE fingerprint IN ({placeholders}) AND status = 'failed'
|
||||
""",
|
||||
fingerprints,
|
||||
)
|
||||
return int(cur.rowcount or 0)
|
||||
|
||||
def requeue_done_by_fingerprints(self, fingerprints: list[str]) -> int:
|
||||
"""Reset completed rows to `pending` so the user can rerun the same task batch."""
|
||||
if not fingerprints:
|
||||
return 0
|
||||
placeholders = ",".join("?" * len(fingerprints))
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
cur = self._conn.execute(
|
||||
f"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'pending',
|
||||
attempts = 0,
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = '',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE fingerprint IN ({placeholders}) AND status = 'done'
|
||||
""",
|
||||
fingerprints,
|
||||
)
|
||||
return int(cur.rowcount or 0)
|
||||
|
||||
def count_incomplete_for_fingerprints(self, fingerprints: list[str]) -> int:
|
||||
"""Rows for these fingerprints that are not successfully finished (`done`)."""
|
||||
if not fingerprints:
|
||||
return 0
|
||||
placeholders = ",".join("?" * len(fingerprints))
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) FROM download_tasks
|
||||
WHERE fingerprint IN ({placeholders}) AND status != 'done'
|
||||
""",
|
||||
fingerprints,
|
||||
).fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
def lease(
|
||||
self,
|
||||
worker_id: str,
|
||||
*,
|
||||
limit: int = 4,
|
||||
lease_seconds: float = 180.0,
|
||||
) -> list[int]:
|
||||
"""Grab up to `limit` tasks; reclaims expired leases."""
|
||||
now = time.time()
|
||||
deadline = now + lease_seconds
|
||||
ids: list[int] = []
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
for _ in range(limit):
|
||||
cur = self._conn.execute(
|
||||
"""
|
||||
WITH picked AS (
|
||||
SELECT id FROM download_tasks
|
||||
WHERE status = 'pending'
|
||||
OR (status = 'running' AND leased_until IS NOT NULL AND leased_until < ?)
|
||||
ORDER BY priority DESC, id ASC
|
||||
LIMIT 1
|
||||
)
|
||||
UPDATE download_tasks
|
||||
SET status = 'running',
|
||||
worker_id = ?,
|
||||
leased_until = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id IN (SELECT id FROM picked)
|
||||
RETURNING id
|
||||
""",
|
||||
(now, worker_id, deadline),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
break
|
||||
ids.append(int(row[0]))
|
||||
return ids
|
||||
|
||||
def load_task(self, task_id: int) -> sqlite3.Row | None:
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT * FROM download_tasks WHERE id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
return row
|
||||
|
||||
def mark_done(self, task_id: int) -> None:
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'done',
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = '',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(task_id,),
|
||||
)
|
||||
|
||||
def mark_failed_final(self, task_id: int, message: str) -> None:
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'failed',
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(message[:2000], task_id),
|
||||
)
|
||||
|
||||
def requeue_retry(self, task_id: int, message: str, max_attempts: int = 4) -> bool:
|
||||
"""Return True if scheduled for another attempt."""
|
||||
with self._lock:
|
||||
with self._conn:
|
||||
row = self._conn.execute(
|
||||
"SELECT attempts FROM download_tasks WHERE id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
attempts = int(row[0]) + 1
|
||||
if attempts >= max_attempts:
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'failed',
|
||||
attempts = ?,
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(attempts, message[:2000], task_id),
|
||||
)
|
||||
return False
|
||||
self._conn.execute(
|
||||
"""
|
||||
UPDATE download_tasks
|
||||
SET status = 'pending',
|
||||
attempts = ?,
|
||||
leased_until = NULL,
|
||||
worker_id = NULL,
|
||||
last_error = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(attempts, message[:2000], task_id),
|
||||
)
|
||||
return True
|
||||
|
||||
def count_by_status(self, status: str) -> int:
|
||||
with self._lock:
|
||||
row = self._conn.execute(
|
||||
"SELECT COUNT(*) FROM download_tasks WHERE status = ?",
|
||||
(status,),
|
||||
).fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
def snapshot_counts(self) -> dict[str, int]:
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"""
|
||||
SELECT status, COUNT(*) FROM download_tasks GROUP BY status
|
||||
"""
|
||||
).fetchall()
|
||||
return {str(r[0]): int(r[1]) for r in rows}
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
|
||||
|
||||
def get_last_synced_illust_id(conn: sqlite3.Connection, artist_id: int) -> int:
|
||||
row = conn.execute(
|
||||
"SELECT last_synced_illust_id FROM artist_sync_watermark WHERE artist_id = ?",
|
||||
(artist_id,),
|
||||
).fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
|
||||
def advance_watermark(conn: sqlite3.Connection, artist_id: int, illust_id: int) -> None:
|
||||
"""Raise watermark to at least illust_id for incremental author sync."""
|
||||
with conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO artist_sync_watermark (artist_id, last_synced_illust_id, updated_at)
|
||||
VALUES (?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(artist_id) DO UPDATE SET
|
||||
last_synced_illust_id = MAX(
|
||||
artist_sync_watermark.last_synced_illust_id,
|
||||
excluded.last_synced_illust_id
|
||||
),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(artist_id, illust_id),
|
||||
)
|
||||
|
||||
|
||||
def filter_newer_than_watermark(work_ids: list[int], watermark: int) -> list[int]:
|
||||
"""Keep illust ids strictly greater than last synced watermark (Pixiv ids grow over time)."""
|
||||
return [wid for wid in work_ids if wid > watermark]
|
||||
Reference in new issue
Block a user