commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from .models import CrawlResult, CrawlTask, DeadLetterEntry, TaskPriority
|
||||
|
||||
|
||||
class RedisPriorityQueue:
|
||||
def __init__(self, redis: Redis, namespace: str, result_queue_name: str, dead_letter_queue_name: str | None = None) -> None:
|
||||
self.redis = redis
|
||||
self.namespace = namespace
|
||||
self.result_queue_name = result_queue_name
|
||||
self.dead_letter_queue_name = dead_letter_queue_name or f"{namespace}:dead_letter"
|
||||
|
||||
def _task_key(self, worker_id: str, priority: TaskPriority) -> str:
|
||||
return f"{self.namespace}:tasks:{worker_id}:{priority.value}"
|
||||
|
||||
async def enqueue_task(self, worker_id: str, task: CrawlTask) -> None:
|
||||
await self.redis.rpush(self._task_key(worker_id, task.priority), json.dumps(task.to_dict()))
|
||||
|
||||
async def dequeue_task(self, worker_id: str, timeout: int = 5) -> CrawlTask | None:
|
||||
keys = [self._task_key(worker_id, priority) for priority in (TaskPriority.HIGH, TaskPriority.MEDIUM, TaskPriority.LOW)]
|
||||
result = await self.redis.blpop(keys, timeout=timeout)
|
||||
if not result:
|
||||
return None
|
||||
_, payload = result
|
||||
return CrawlTask.from_dict(json.loads(payload))
|
||||
|
||||
async def publish_result(self, result: CrawlResult) -> None:
|
||||
await self.redis.rpush(self.result_queue_name, json.dumps(result.to_dict()))
|
||||
|
||||
async def consume_result(self, timeout: int = 5) -> CrawlResult | None:
|
||||
result = await self.redis.blpop(self.result_queue_name, timeout=timeout)
|
||||
if not result:
|
||||
return None
|
||||
_, payload = result
|
||||
return CrawlResult.from_dict(json.loads(payload))
|
||||
|
||||
async def publish_dead_letter(self, entry: DeadLetterEntry) -> None:
|
||||
await self.redis.rpush(self.dead_letter_queue_name, json.dumps(entry.to_dict(), ensure_ascii=False))
|
||||
Reference in new issue
Block a user