239 lines
8.7 KiB
Python
239 lines
8.7 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from pixiv_app.tasks.models import DownloadTaskSpec
|
|
|
|
|
|
class TaskQueue:
|
|
"""SQLite-backed queue with lease/reclaim; thread-safe for concurrent workers."""
|
|
|
|
def __init__(self, conn: sqlite3.Connection) -> None:
|
|
self._conn = conn
|
|
self._lock = threading.Lock()
|
|
|
|
def enqueue(self, tasks: list[DownloadTaskSpec]) -> tuple[int, int]:
|
|
inserted = 0
|
|
with self._lock:
|
|
with self._conn:
|
|
for t in tasks:
|
|
fp = t.fingerprint()
|
|
cur = self._conn.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO download_tasks (
|
|
fingerprint, target_type, target_id, page_index, task_kind,
|
|
priority, status, payload_json
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)
|
|
""",
|
|
(
|
|
fp,
|
|
t.target_type,
|
|
t.target_id,
|
|
t.page_index,
|
|
t.task_kind,
|
|
int(t.priority),
|
|
json.dumps(t.payload, ensure_ascii=False),
|
|
),
|
|
)
|
|
inserted += cur.rowcount
|
|
skipped = len(tasks) - inserted
|
|
return inserted, skipped
|
|
|
|
def requeue_failed_by_fingerprints(self, fingerprints: list[str]) -> int:
|
|
"""Reset `failed` rows to `pending` so a retry can pick them up."""
|
|
if not fingerprints:
|
|
return 0
|
|
placeholders = ",".join("?" * len(fingerprints))
|
|
with self._lock:
|
|
with self._conn:
|
|
cur = self._conn.execute(
|
|
f"""
|
|
UPDATE download_tasks
|
|
SET status = 'pending',
|
|
leased_until = NULL,
|
|
worker_id = NULL,
|
|
last_error = '',
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE fingerprint IN ({placeholders}) AND status = 'failed'
|
|
""",
|
|
fingerprints,
|
|
)
|
|
return int(cur.rowcount or 0)
|
|
|
|
def requeue_done_by_fingerprints(self, fingerprints: list[str]) -> int:
|
|
"""Reset completed rows to `pending` so the user can rerun the same task batch."""
|
|
if not fingerprints:
|
|
return 0
|
|
placeholders = ",".join("?" * len(fingerprints))
|
|
with self._lock:
|
|
with self._conn:
|
|
cur = self._conn.execute(
|
|
f"""
|
|
UPDATE download_tasks
|
|
SET status = 'pending',
|
|
attempts = 0,
|
|
leased_until = NULL,
|
|
worker_id = NULL,
|
|
last_error = '',
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE fingerprint IN ({placeholders}) AND status = 'done'
|
|
""",
|
|
fingerprints,
|
|
)
|
|
return int(cur.rowcount or 0)
|
|
|
|
def count_incomplete_for_fingerprints(self, fingerprints: list[str]) -> int:
|
|
"""Rows for these fingerprints that are not successfully finished (`done`)."""
|
|
if not fingerprints:
|
|
return 0
|
|
placeholders = ",".join("?" * len(fingerprints))
|
|
with self._lock:
|
|
row = self._conn.execute(
|
|
f"""
|
|
SELECT COUNT(*) FROM download_tasks
|
|
WHERE fingerprint IN ({placeholders}) AND status != 'done'
|
|
""",
|
|
fingerprints,
|
|
).fetchone()
|
|
return int(row[0]) if row else 0
|
|
|
|
def lease(
|
|
self,
|
|
worker_id: str,
|
|
*,
|
|
limit: int = 4,
|
|
lease_seconds: float = 180.0,
|
|
) -> list[int]:
|
|
"""Grab up to `limit` tasks; reclaims expired leases."""
|
|
now = time.time()
|
|
deadline = now + lease_seconds
|
|
ids: list[int] = []
|
|
with self._lock:
|
|
with self._conn:
|
|
for _ in range(limit):
|
|
cur = self._conn.execute(
|
|
"""
|
|
WITH picked AS (
|
|
SELECT id FROM download_tasks
|
|
WHERE status = 'pending'
|
|
OR (status = 'running' AND leased_until IS NOT NULL AND leased_until < ?)
|
|
ORDER BY priority DESC, id ASC
|
|
LIMIT 1
|
|
)
|
|
UPDATE download_tasks
|
|
SET status = 'running',
|
|
worker_id = ?,
|
|
leased_until = ?,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id IN (SELECT id FROM picked)
|
|
RETURNING id
|
|
""",
|
|
(now, worker_id, deadline),
|
|
)
|
|
row = cur.fetchone()
|
|
if row is None:
|
|
break
|
|
ids.append(int(row[0]))
|
|
return ids
|
|
|
|
def load_task(self, task_id: int) -> sqlite3.Row | None:
|
|
with self._lock:
|
|
row = self._conn.execute(
|
|
"SELECT * FROM download_tasks WHERE id = ?",
|
|
(task_id,),
|
|
).fetchone()
|
|
return row
|
|
|
|
def mark_done(self, task_id: int) -> None:
|
|
with self._lock:
|
|
with self._conn:
|
|
self._conn.execute(
|
|
"""
|
|
UPDATE download_tasks
|
|
SET status = 'done',
|
|
leased_until = NULL,
|
|
worker_id = NULL,
|
|
last_error = '',
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(task_id,),
|
|
)
|
|
|
|
def mark_failed_final(self, task_id: int, message: str) -> None:
|
|
with self._lock:
|
|
with self._conn:
|
|
self._conn.execute(
|
|
"""
|
|
UPDATE download_tasks
|
|
SET status = 'failed',
|
|
leased_until = NULL,
|
|
worker_id = NULL,
|
|
last_error = ?,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(message[:2000], task_id),
|
|
)
|
|
|
|
def requeue_retry(self, task_id: int, message: str, max_attempts: int = 4) -> bool:
|
|
"""Return True if scheduled for another attempt."""
|
|
with self._lock:
|
|
with self._conn:
|
|
row = self._conn.execute(
|
|
"SELECT attempts FROM download_tasks WHERE id = ?",
|
|
(task_id,),
|
|
).fetchone()
|
|
if row is None:
|
|
return False
|
|
attempts = int(row[0]) + 1
|
|
if attempts >= max_attempts:
|
|
self._conn.execute(
|
|
"""
|
|
UPDATE download_tasks
|
|
SET status = 'failed',
|
|
attempts = ?,
|
|
leased_until = NULL,
|
|
worker_id = NULL,
|
|
last_error = ?,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(attempts, message[:2000], task_id),
|
|
)
|
|
return False
|
|
self._conn.execute(
|
|
"""
|
|
UPDATE download_tasks
|
|
SET status = 'pending',
|
|
attempts = ?,
|
|
leased_until = NULL,
|
|
worker_id = NULL,
|
|
last_error = ?,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(attempts, message[:2000], task_id),
|
|
)
|
|
return True
|
|
|
|
def count_by_status(self, status: str) -> int:
|
|
with self._lock:
|
|
row = self._conn.execute(
|
|
"SELECT COUNT(*) FROM download_tasks WHERE status = ?",
|
|
(status,),
|
|
).fetchone()
|
|
return int(row[0]) if row else 0
|
|
|
|
def snapshot_counts(self) -> dict[str, int]:
|
|
with self._lock:
|
|
rows = self._conn.execute(
|
|
"""
|
|
SELECT status, COUNT(*) FROM download_tasks GROUP BY status
|
|
"""
|
|
).fetchall()
|
|
return {str(r[0]): int(r[1]) for r in rows}
|