190 lines
5.5 KiB
Python
190 lines
5.5 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import asdict, dataclass, field
|
|
from datetime import UTC, datetime
|
|
from enum import Enum
|
|
from hashlib import sha1
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
from uuid import uuid4
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(UTC).isoformat()
|
|
|
|
|
|
def make_url_hash(url: str) -> str:
|
|
return sha1(url.encode("utf-8")).hexdigest()
|
|
|
|
|
|
class TaskPriority(str, Enum):
|
|
HIGH = "high"
|
|
MEDIUM = "medium"
|
|
LOW = "low"
|
|
|
|
|
|
class TaskStatus(str, Enum):
|
|
PENDING = "pending"
|
|
RUNNING = "running"
|
|
SUCCESS = "success"
|
|
FAILED = "failed"
|
|
RETRY = "retry"
|
|
|
|
|
|
@dataclass
|
|
class CrawlTask:
|
|
task_id: str
|
|
url: str
|
|
spider: str
|
|
method: str = "GET"
|
|
headers: dict[str, str] = field(default_factory=dict)
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
body: str | None = None
|
|
priority: TaskPriority = TaskPriority.MEDIUM
|
|
status: TaskStatus = TaskStatus.PENDING
|
|
retries: int = 0
|
|
max_retries: int = 4
|
|
assigned_worker: str | None = None
|
|
created_at: str = field(default_factory=utc_now)
|
|
updated_at: str = field(default_factory=utc_now)
|
|
schedule_name: str | None = None
|
|
|
|
@property
|
|
def domain(self) -> str:
|
|
return urlparse(self.url).netloc
|
|
|
|
@property
|
|
def fingerprint(self) -> str:
|
|
payload = f"{self.method}:{self.url}:{self.spider}:{self.body or ''}"
|
|
return sha1(payload.encode("utf-8")).hexdigest()
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
payload = asdict(self)
|
|
payload["priority"] = self.priority.value
|
|
payload["status"] = self.status.value
|
|
return payload
|
|
|
|
@classmethod
|
|
def from_dict(cls, payload: dict[str, Any]) -> "CrawlTask":
|
|
return cls(
|
|
task_id=str(payload["task_id"]),
|
|
url=str(payload["url"]),
|
|
spider=str(payload["spider"]),
|
|
method=str(payload.get("method", "GET")),
|
|
headers=dict(payload.get("headers", {})),
|
|
metadata=dict(payload.get("metadata", {})),
|
|
body=payload.get("body"),
|
|
priority=TaskPriority(payload.get("priority", TaskPriority.MEDIUM.value)),
|
|
status=TaskStatus(payload.get("status", TaskStatus.PENDING.value)),
|
|
retries=int(payload.get("retries", 0)),
|
|
max_retries=int(payload.get("max_retries", 4)),
|
|
assigned_worker=payload.get("assigned_worker"),
|
|
created_at=str(payload.get("created_at", utc_now())),
|
|
updated_at=str(payload.get("updated_at", utc_now())),
|
|
schedule_name=payload.get("schedule_name"),
|
|
)
|
|
|
|
@classmethod
|
|
def create(
|
|
cls,
|
|
*,
|
|
url: str,
|
|
spider: str,
|
|
method: str = "GET",
|
|
headers: dict[str, str] | None = None,
|
|
metadata: dict[str, Any] | None = None,
|
|
body: str | None = None,
|
|
priority: str = "medium",
|
|
max_retries: int = 4,
|
|
schedule_name: str | None = None,
|
|
) -> "CrawlTask":
|
|
return cls(
|
|
task_id=uuid4().hex,
|
|
url=url,
|
|
spider=spider,
|
|
method=method.upper(),
|
|
headers=headers or {},
|
|
metadata=metadata or {},
|
|
body=body,
|
|
priority=TaskPriority(priority),
|
|
max_retries=max_retries,
|
|
schedule_name=schedule_name,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class CrawlResult:
|
|
task_id: str
|
|
worker_id: str
|
|
status: TaskStatus
|
|
url: str
|
|
spider: str
|
|
http_status: int | None = None
|
|
data: dict[str, Any] = field(default_factory=dict)
|
|
error: str | None = None
|
|
finished_at: str = field(default_factory=utc_now)
|
|
|
|
@property
|
|
def url_hash(self) -> str:
|
|
return make_url_hash(self.url)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
payload = asdict(self)
|
|
payload["status"] = self.status.value
|
|
payload["url_hash"] = self.url_hash
|
|
return payload
|
|
|
|
@classmethod
|
|
def from_dict(cls, payload: dict[str, Any]) -> "CrawlResult":
|
|
return cls(
|
|
task_id=str(payload["task_id"]),
|
|
worker_id=str(payload["worker_id"]),
|
|
status=TaskStatus(payload["status"]),
|
|
url=str(payload["url"]),
|
|
spider=str(payload["spider"]),
|
|
http_status=payload.get("http_status"),
|
|
data=dict(payload.get("data", {})),
|
|
error=payload.get("error"),
|
|
finished_at=str(payload.get("finished_at", utc_now())),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class DeadLetterEntry:
|
|
task: dict[str, Any]
|
|
result: dict[str, Any] | None = None
|
|
reason: str = ""
|
|
failed_at: str = field(default_factory=utc_now)
|
|
last_error: str | None = None
|
|
response_status: int | None = None
|
|
retry_count: int = 0
|
|
worker_id: str | None = None
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return asdict(self)
|
|
|
|
@classmethod
|
|
def from_task_result(cls, task: CrawlTask, result: CrawlResult, reason: str | None = None) -> "DeadLetterEntry":
|
|
return cls(
|
|
task=task.to_dict(),
|
|
result=result.to_dict(),
|
|
reason=reason or result.error or "Task failed",
|
|
last_error=result.error,
|
|
response_status=result.http_status,
|
|
retry_count=task.retries,
|
|
worker_id=result.worker_id,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class WorkerHeartbeat:
|
|
worker_id: str
|
|
concurrency: int
|
|
active_tasks: int
|
|
last_seen: str = field(default_factory=utc_now)
|
|
host: str | None = None
|
|
tags: list[str] = field(default_factory=list)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return asdict(self)
|