first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+238
View File
@@ -0,0 +1,238 @@
from __future__ import annotations
import json
import sqlite3
import threading
import time
from pixiv_app.tasks.models import DownloadTaskSpec
class TaskQueue:
"""SQLite-backed queue with lease/reclaim; thread-safe for concurrent workers."""
def __init__(self, conn: sqlite3.Connection) -> None:
self._conn = conn
self._lock = threading.Lock()
def enqueue(self, tasks: list[DownloadTaskSpec]) -> tuple[int, int]:
inserted = 0
with self._lock:
with self._conn:
for t in tasks:
fp = t.fingerprint()
cur = self._conn.execute(
"""
INSERT OR IGNORE INTO download_tasks (
fingerprint, target_type, target_id, page_index, task_kind,
priority, status, payload_json
)
VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)
""",
(
fp,
t.target_type,
t.target_id,
t.page_index,
t.task_kind,
int(t.priority),
json.dumps(t.payload, ensure_ascii=False),
),
)
inserted += cur.rowcount
skipped = len(tasks) - inserted
return inserted, skipped
def requeue_failed_by_fingerprints(self, fingerprints: list[str]) -> int:
"""Reset `failed` rows to `pending` so a retry can pick them up."""
if not fingerprints:
return 0
placeholders = ",".join("?" * len(fingerprints))
with self._lock:
with self._conn:
cur = self._conn.execute(
f"""
UPDATE download_tasks
SET status = 'pending',
leased_until = NULL,
worker_id = NULL,
last_error = '',
updated_at = CURRENT_TIMESTAMP
WHERE fingerprint IN ({placeholders}) AND status = 'failed'
""",
fingerprints,
)
return int(cur.rowcount or 0)
def requeue_done_by_fingerprints(self, fingerprints: list[str]) -> int:
"""Reset completed rows to `pending` so the user can rerun the same task batch."""
if not fingerprints:
return 0
placeholders = ",".join("?" * len(fingerprints))
with self._lock:
with self._conn:
cur = self._conn.execute(
f"""
UPDATE download_tasks
SET status = 'pending',
attempts = 0,
leased_until = NULL,
worker_id = NULL,
last_error = '',
updated_at = CURRENT_TIMESTAMP
WHERE fingerprint IN ({placeholders}) AND status = 'done'
""",
fingerprints,
)
return int(cur.rowcount or 0)
def count_incomplete_for_fingerprints(self, fingerprints: list[str]) -> int:
"""Rows for these fingerprints that are not successfully finished (`done`)."""
if not fingerprints:
return 0
placeholders = ",".join("?" * len(fingerprints))
with self._lock:
row = self._conn.execute(
f"""
SELECT COUNT(*) FROM download_tasks
WHERE fingerprint IN ({placeholders}) AND status != 'done'
""",
fingerprints,
).fetchone()
return int(row[0]) if row else 0
def lease(
self,
worker_id: str,
*,
limit: int = 4,
lease_seconds: float = 180.0,
) -> list[int]:
"""Grab up to `limit` tasks; reclaims expired leases."""
now = time.time()
deadline = now + lease_seconds
ids: list[int] = []
with self._lock:
with self._conn:
for _ in range(limit):
cur = self._conn.execute(
"""
WITH picked AS (
SELECT id FROM download_tasks
WHERE status = 'pending'
OR (status = 'running' AND leased_until IS NOT NULL AND leased_until < ?)
ORDER BY priority DESC, id ASC
LIMIT 1
)
UPDATE download_tasks
SET status = 'running',
worker_id = ?,
leased_until = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id IN (SELECT id FROM picked)
RETURNING id
""",
(now, worker_id, deadline),
)
row = cur.fetchone()
if row is None:
break
ids.append(int(row[0]))
return ids
def load_task(self, task_id: int) -> sqlite3.Row | None:
with self._lock:
row = self._conn.execute(
"SELECT * FROM download_tasks WHERE id = ?",
(task_id,),
).fetchone()
return row
def mark_done(self, task_id: int) -> None:
with self._lock:
with self._conn:
self._conn.execute(
"""
UPDATE download_tasks
SET status = 'done',
leased_until = NULL,
worker_id = NULL,
last_error = '',
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(task_id,),
)
def mark_failed_final(self, task_id: int, message: str) -> None:
with self._lock:
with self._conn:
self._conn.execute(
"""
UPDATE download_tasks
SET status = 'failed',
leased_until = NULL,
worker_id = NULL,
last_error = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(message[:2000], task_id),
)
def requeue_retry(self, task_id: int, message: str, max_attempts: int = 4) -> bool:
"""Return True if scheduled for another attempt."""
with self._lock:
with self._conn:
row = self._conn.execute(
"SELECT attempts FROM download_tasks WHERE id = ?",
(task_id,),
).fetchone()
if row is None:
return False
attempts = int(row[0]) + 1
if attempts >= max_attempts:
self._conn.execute(
"""
UPDATE download_tasks
SET status = 'failed',
attempts = ?,
leased_until = NULL,
worker_id = NULL,
last_error = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(attempts, message[:2000], task_id),
)
return False
self._conn.execute(
"""
UPDATE download_tasks
SET status = 'pending',
attempts = ?,
leased_until = NULL,
worker_id = NULL,
last_error = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""",
(attempts, message[:2000], task_id),
)
return True
def count_by_status(self, status: str) -> int:
with self._lock:
row = self._conn.execute(
"SELECT COUNT(*) FROM download_tasks WHERE status = ?",
(status,),
).fetchone()
return int(row[0]) if row else 0
def snapshot_counts(self) -> dict[str, int]:
with self._lock:
rows = self._conn.execute(
"""
SELECT status, COUNT(*) FROM download_tasks GROUP BY status
"""
).fetchall()
return {str(r[0]): int(r[1]) for r in rows}