first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+42
View File
@@ -0,0 +1,42 @@
from __future__ import annotations
import json
from redis.asyncio import Redis
from .models import CrawlResult, CrawlTask, DeadLetterEntry, TaskPriority
class RedisPriorityQueue:
def __init__(self, redis: Redis, namespace: str, result_queue_name: str, dead_letter_queue_name: str | None = None) -> None:
self.redis = redis
self.namespace = namespace
self.result_queue_name = result_queue_name
self.dead_letter_queue_name = dead_letter_queue_name or f"{namespace}:dead_letter"
def _task_key(self, worker_id: str, priority: TaskPriority) -> str:
return f"{self.namespace}:tasks:{worker_id}:{priority.value}"
async def enqueue_task(self, worker_id: str, task: CrawlTask) -> None:
await self.redis.rpush(self._task_key(worker_id, task.priority), json.dumps(task.to_dict()))
async def dequeue_task(self, worker_id: str, timeout: int = 5) -> CrawlTask | None:
keys = [self._task_key(worker_id, priority) for priority in (TaskPriority.HIGH, TaskPriority.MEDIUM, TaskPriority.LOW)]
result = await self.redis.blpop(keys, timeout=timeout)
if not result:
return None
_, payload = result
return CrawlTask.from_dict(json.loads(payload))
async def publish_result(self, result: CrawlResult) -> None:
await self.redis.rpush(self.result_queue_name, json.dumps(result.to_dict()))
async def consume_result(self, timeout: int = 5) -> CrawlResult | None:
result = await self.redis.blpop(self.result_queue_name, timeout=timeout)
if not result:
return None
_, payload = result
return CrawlResult.from_dict(json.loads(payload))
async def publish_dead_letter(self, entry: DeadLetterEntry) -> None:
await self.redis.rpush(self.dead_letter_queue_name, json.dumps(entry.to_dict(), ensure_ascii=False))