Files
SakuraHub/distributed_crawler/crawler/master.py
T

346 lines
15 KiB
Python
Raw Normal View History

2026-11-16 20:45:22 +08:00
from __future__ import annotations
import asyncio
import logging
from time import time
from typing import Any
from html import escape
from aiohttp import web
from redis.asyncio import Redis
from .bloom import RedisBloomFilter
from .config import AppSettings
from .hash_ring import ConsistentHashRing
from .models import CrawlResult, CrawlTask, DeadLetterEntry, TaskPriority, TaskStatus, WorkerHeartbeat
from .queue import RedisPriorityQueue
from .results import ResultRepository, build_result_repository
from .scheduler import ScheduleService
from .state import CheckpointManager
LOGGER = logging.getLogger(__name__)
class MasterNode:
def __init__(self, settings: AppSettings) -> None:
self.settings = settings
self.redis = Redis.from_url(settings.redis_dsn, decode_responses=True)
self.queue = RedisPriorityQueue(
self.redis,
settings.master.queue_name,
settings.master.result_queue_name,
settings.master.dead_letter_queue_name,
)
self.checkpoints = CheckpointManager(self.redis, settings.resolve_path(settings.checkpoint_dir), settings.master.queue_name)
self.bloom = RedisBloomFilter.from_capacity(
self.redis,
key=f"{settings.master.queue_name}:bloom",
capacity=settings.master.bloom_capacity,
false_positive_rate=settings.master.bloom_false_positive_rate,
hashes=settings.master.bloom_hashes,
)
self.hash_ring = ConsistentHashRing()
self.result_repo: ResultRepository = build_result_repository(
settings.storage,
settings.resolve_path(settings.storage.jsonl_path),
)
self.scheduler = ScheduleService(settings, self.submit_task)
self.workers: dict[str, WorkerHeartbeat] = {}
self.app = web.Application()
self.app.add_routes(
[
web.post("/tasks", self.handle_submit_task),
web.post("/heartbeat", self.handle_heartbeat),
web.get("/workers", self.handle_workers),
web.get("/stats", self.handle_stats),
web.get("/health", self.handle_health),
web.get("/dashboard", self.handle_dashboard),
]
)
self._background_tasks: list[asyncio.Task] = []
async def start(self) -> None:
await self.result_repo.setup()
await self._recover_local_checkpoints()
self.scheduler.start()
self._background_tasks = [
asyncio.create_task(self.consume_results_loop(), name="consume-results"),
asyncio.create_task(self.reap_stale_workers_loop(), name="reap-workers"),
]
async def close(self) -> None:
for task in self._background_tasks:
task.cancel()
await asyncio.gather(*self._background_tasks, return_exceptions=True)
await self.scheduler.shutdown()
await self.result_repo.close()
await self.redis.aclose()
async def submit_task(self, task: CrawlTask) -> bool:
duplicate = await self.bloom.contains_or_add(task.fingerprint)
if duplicate:
LOGGER.info("Skip duplicate task %s", task.url)
return False
worker_id = self._assign_worker(task)
task.assigned_worker = worker_id
task.status = TaskStatus.PENDING
await self.checkpoints.save(task)
await self.queue.enqueue_task(worker_id, task)
LOGGER.info("Submitted task %s to %s", task.task_id, worker_id)
return True
async def handle_submit_task(self, request: web.Request) -> web.Response:
payload = await request.json()
task = CrawlTask.create(
url=payload["url"],
spider=payload.get("spider", self.settings.master.default_spider),
method=payload.get("method", "GET"),
headers=payload.get("headers"),
metadata=payload.get("metadata"),
body=payload.get("body"),
priority=payload.get("priority", TaskPriority.MEDIUM.value),
max_retries=int(payload.get("max_retries", self.settings.worker.retries)),
)
accepted = await self.submit_task(task)
return web.json_response({"accepted": accepted, "task_id": task.task_id, "worker_id": task.assigned_worker})
async def handle_heartbeat(self, request: web.Request) -> web.Response:
payload = await request.json()
heartbeat = WorkerHeartbeat(
worker_id=str(payload["worker_id"]),
concurrency=int(payload["concurrency"]),
active_tasks=int(payload["active_tasks"]),
host=payload.get("host"),
tags=list(payload.get("tags", [])),
)
self.workers[heartbeat.worker_id] = heartbeat
self.hash_ring.rebuild(list(self.workers.keys()))
return web.json_response({"ok": True, "known_workers": len(self.workers)})
async def handle_workers(self, _: web.Request) -> web.Response:
return web.json_response({"workers": [item.to_dict() for item in self.workers.values()]})
async def handle_stats(self, _: web.Request) -> web.Response:
return web.json_response(
{
"workers": len(self.workers),
"queues": {
worker_id: {
priority.value: await self.redis.llen(self.queue._task_key(worker_id, priority))
for priority in TaskPriority
}
for worker_id in self.workers or {"default": None}
},
"dead_letters": await self.redis.llen(self.queue.dead_letter_queue_name),
}
)
async def handle_health(self, _: web.Request) -> web.Response:
pong = await self.redis.ping()
return web.json_response({"status": "ok", "redis": bool(pong), "workers": len(self.workers)})
async def handle_dashboard(self, _: web.Request) -> web.Response:
queue_stats = await self._collect_queue_stats()
session_count = await self._count_sessions()
dead_letter_count = await self.redis.llen(self.queue.dead_letter_queue_name)
html = self._render_dashboard_html(
queue_stats=queue_stats,
session_count=session_count,
dead_letter_count=dead_letter_count,
)
return web.Response(text=html, content_type="text/html")
async def consume_results_loop(self) -> None:
while True:
try:
result = await self.queue.consume_result(timeout=5)
if result is None:
continue
await self.process_result(result)
except asyncio.CancelledError:
raise
except Exception:
LOGGER.exception("Result consumer failed")
await asyncio.sleep(1)
async def process_result(self, result: CrawlResult) -> None:
task = await self.checkpoints.load(result.task_id)
if task is None:
LOGGER.warning("Result for unknown task %s", result.task_id)
return
task.status = result.status
await self.checkpoints.mark_status(task, result.status)
await self.result_repo.store(result)
if result.status == TaskStatus.FAILED:
await self.queue.publish_dead_letter(DeadLetterEntry.from_task_result(task, result))
LOGGER.warning(
"Dead-lettered task task_id=%s worker_id=%s request_id=%s reason=%s",
task.task_id,
result.worker_id,
result.data.get("request_id", task.metadata.get("request_id", task.task_id)),
result.error,
)
async def reap_stale_workers_loop(self) -> None:
while True:
try:
await asyncio.sleep(self.settings.master.worker_ttl_seconds)
now = time()
stale_workers = [
worker_id
for worker_id, heartbeat in self.workers.items()
if now - self._heartbeat_epoch(heartbeat.last_seen) > self.settings.master.worker_ttl_seconds
]
for worker_id in stale_workers:
LOGGER.warning("Worker %s is stale, recovering tasks", worker_id)
recovered = await self.checkpoints.recover_stale(worker_id, self.settings.master.stale_task_seconds)
for task in recovered:
task.retries += 1
if task.retries > task.max_retries:
task.status = TaskStatus.FAILED
await self.checkpoints.mark_status(task, TaskStatus.FAILED)
result = CrawlResult(
task_id=task.task_id,
worker_id=worker_id,
status=TaskStatus.FAILED,
url=task.url,
spider=task.spider,
error="Task exceeded retry budget during worker recovery",
)
await self.result_repo.store(result)
await self.queue.publish_dead_letter(DeadLetterEntry.from_task_result(task, result))
continue
task.assigned_worker = self._assign_worker(task, exclude=worker_id)
await self.checkpoints.save(task)
await self.queue.enqueue_task(task.assigned_worker, task)
self.workers.pop(worker_id, None)
self.hash_ring.rebuild(list(self.workers.keys()))
except asyncio.CancelledError:
raise
except Exception:
LOGGER.exception("Stale worker reaper failed")
async def _recover_local_checkpoints(self) -> None:
tasks = await self.checkpoints.recover_local()
for task in tasks:
task.assigned_worker = self._assign_worker(task)
task.status = TaskStatus.RETRY
await self.checkpoints.save(task)
await self.queue.enqueue_task(task.assigned_worker, task)
async def _collect_queue_stats(self) -> dict[str, dict[str, int]]:
worker_keys = list(self.workers.keys()) or ["default"]
result: dict[str, dict[str, int]] = {}
for worker_id in worker_keys:
result[worker_id] = {
priority.value: await self.redis.llen(self.queue._task_key(worker_id, priority))
for priority in TaskPriority
}
return result
async def _count_sessions(self) -> int:
pattern = f"{self.settings.session.redis_prefix}:*"
count = 0
async for _ in self.redis.scan_iter(match=pattern):
count += 1
return count
def _render_dashboard_html(self, queue_stats: dict[str, dict[str, int]], session_count: int, dead_letter_count: int) -> str:
worker_rows = []
for worker_id, hb in self.workers.items():
load = hb.active_tasks / max(hb.concurrency, 1)
worker_rows.append(
f"<tr><td>{escape(worker_id)}</td><td>{hb.active_tasks}</td><td>{hb.concurrency}</td><td>{load:.2f}</td><td>{escape(hb.last_seen)}</td></tr>"
)
if not worker_rows:
worker_rows.append("<tr><td colspan='5'>暂无在线 Worker</td></tr>")
queue_blocks = []
for worker_id, levels in queue_stats.items():
queue_blocks.append(
"<div class='card'><h3>Queue - "
+ escape(worker_id)
+ "</h3><p>high: "
+ str(levels.get("high", 0))
+ " | medium: "
+ str(levels.get("medium", 0))
+ " | low: "
+ str(levels.get("low", 0))
+ "</p></div>"
)
queue_html = "\n".join(queue_blocks)
enabled_auth_domains = ", ".join(sorted(self.settings.auth.domains.keys())) if self.settings.auth.domains else "-"
enabled_proxy = "enabled" if self.settings.proxy.enabled else "disabled"
enabled_session = "enabled" if self.settings.session.enabled else "disabled"
enabled_auth = "enabled" if self.settings.auth.enabled else "disabled"
return f"""<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<title>Distributed Crawler Dashboard</title>
<style>
body {{ font-family: Segoe UI, Arial, sans-serif; background:#f6f8fb; margin:0; padding:24px; color:#1f2d3d; }}
.grid {{ display:grid; grid-template-columns: repeat(auto-fill, minmax(280px,1fr)); gap:12px; margin-bottom:16px; }}
.card {{ background:#fff; border:1px solid #dce6f4; border-radius:10px; padding:12px 14px; }}
h1 {{ margin:0 0 14px 0; font-size:24px; }}
h3 {{ margin:0 0 8px 0; font-size:16px; }}
table {{ width:100%; border-collapse: collapse; background:#fff; border:1px solid #dce6f4; }}
th, td {{ text-align:left; padding:10px; border-bottom:1px solid #eef2f8; font-size:13px; }}
th {{ background:#f1f5fc; }}
.muted {{ color:#607086; font-size:12px; }}
code {{ background:#eef3fb; padding:2px 6px; border-radius:4px; }}
</style>
</head>
<body>
<h1>Distributed Crawler Dashboard</h1>
<div class="grid">
<div class="card"><h3>Workers</h3><p>{len(self.workers)}</p></div>
<div class="card"><h3>Session Keys</h3><p>{session_count}</p></div>
<div class="card"><h3>Dead Letters</h3><p>{dead_letter_count}</p></div>
<div class="card"><h3>Proxy</h3><p>{enabled_proxy}</p></div>
<div class="card"><h3>Auto Login</h3><p>{enabled_auth}</p></div>
<div class="card"><h3>Session Sharing</h3><p>{enabled_session}</p></div>
<div class="card"><h3>Auth Domains</h3><p>{escape(enabled_auth_domains)}</p></div>
</div>
<div class="grid">
{queue_html}
</div>
<div class="card">
<h3>Worker Runtime</h3>
<table>
<thead><tr><th>worker_id</th><th>active_tasks</th><th>concurrency</th><th>load</th><th>last_seen</th></tr></thead>
<tbody>
{''.join(worker_rows)}
</tbody>
</table>
</div>
<p class="muted">API endpoints: <code>/tasks</code> <code>/workers</code> <code>/stats</code> <code>/health</code></p>
</body>
</html>"""
def _assign_worker(self, task: CrawlTask, exclude: str | None = None) -> str:
candidates = [worker_id for worker_id in self.workers if worker_id != exclude]
if not candidates:
return "default"
self.hash_ring.rebuild(candidates)
preferred = self.hash_ring.get_node(task.url) or candidates[0]
preferred_heartbeat = self.workers.get(preferred)
if preferred_heartbeat is None:
return preferred
preferred_load = preferred_heartbeat.active_tasks / max(preferred_heartbeat.concurrency, 1)
if preferred_load < 0.9:
return preferred
least_loaded = min(
candidates,
key=lambda worker_id: self.workers[worker_id].active_tasks / max(self.workers[worker_id].concurrency, 1),
)
return least_loaded
def _heartbeat_epoch(self, timestamp: str) -> float:
from datetime import datetime
return datetime.fromisoformat(timestamp).timestamp()