Files
SakuraHub/distributed_crawler/crawler/http.py
T
2026-11-16 20:45:22 +08:00

181 lines
7.5 KiB
Python

from __future__ import annotations
import json
import random
from pathlib import Path
from time import monotonic
from typing import Any
import aiohttp
from .auth import AuthProvider
from .circuit_breaker import DomainCircuitBreaker
from .config import AppSettings
from .models import CrawlTask
from .proxy import ProxyPool
from .rate_limit import DomainRateLimiter
from .sessions import DualSessionStore
from .signatures import SignatureProvider
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/126.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4) AppleWebKit/605.1.15 Version/17.4 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Edg/126.0.0.0 Safari/537.36",
]
class AsyncHttpClient:
def __init__(
self,
settings: AppSettings,
rate_limiter: DomainRateLimiter,
proxy_pool: ProxyPool,
session_store: DualSessionStore,
auth_provider: AuthProvider,
signature_provider: SignatureProvider,
) -> None:
self.settings = settings
self.rate_limiter = rate_limiter
self.proxy_pool = proxy_pool
self.session_store = session_store
self.auth_provider = auth_provider
self.signature_provider = signature_provider
self.session: aiohttp.ClientSession | None = None
self.circuit_breaker = DomainCircuitBreaker(settings.circuit_breaker)
async def start(self) -> None:
timeout = aiohttp.ClientTimeout(
total=self.settings.worker.request_timeout_seconds,
connect=self.settings.worker.connect_timeout_seconds,
)
connector = aiohttp.TCPConnector(limit=self.settings.worker.concurrency, ssl=False)
self.session = aiohttp.ClientSession(timeout=timeout, connector=connector)
async def close(self) -> None:
if self.session is not None:
await self.session.close()
async def fetch(self, task: CrawlTask) -> dict[str, Any]:
if self.session is None:
raise RuntimeError("HTTP client is not started")
await self.rate_limiter.acquire(task.domain)
self.circuit_breaker.before_request(task.domain)
proxy = await self.proxy_pool.acquire()
session_payload = await self.session_store.load(task.domain)
if self.settings.session.enabled and session_payload.get("expired"):
session_payload = await self.auth_provider.refresh_session(task.domain, session_payload)
await self.session_store.save(task.domain, session_payload)
headers = dict(task.headers)
headers.setdefault("User-Agent", random.choice(USER_AGENTS))
headers.setdefault("Accept", "*/*")
if task.metadata.get("referer"):
headers.setdefault("Referer", str(task.metadata["referer"]))
if task.metadata.get("origin"):
headers.setdefault("Origin", str(task.metadata["origin"]))
headers.setdefault("X-Task-Id", task.task_id)
if task.metadata.get("request_id"):
headers.setdefault("X-Request-Id", str(task.metadata["request_id"]))
if task.metadata.get("worker_id"):
headers.setdefault("X-Worker-Id", str(task.metadata["worker_id"]))
cookie_header = session_payload.get("cookie")
if cookie_header:
headers.setdefault("Cookie", str(cookie_header))
signature_headers = await self.signature_provider.sign(
{
"url": task.url,
"method": task.method,
"body": task.body,
"metadata": task.metadata,
}
)
headers.update(signature_headers)
started_at = monotonic()
try:
async with self.session.request(
task.method,
task.url,
headers=headers,
data=task.body.encode("utf-8") if isinstance(task.body, str) else None,
proxy=proxy.url if proxy else None,
) as response:
content_type = response.headers.get("Content-Type", "")
body_bytes = await response.read()
if "application/json" in content_type:
body: Any = json.loads(body_bytes.decode("utf-8", errors="ignore"))
else:
body = body_bytes.decode("utf-8", errors="ignore")
response_data = {
"status": response.status,
"headers": dict(response.headers),
"body": body,
"body_bytes": body_bytes,
"proxy": proxy.url if proxy else None,
}
if self.settings.session.enabled:
if self._is_session_expired(task.domain, response.status, body):
session_payload["expired"] = True
await self.session_store.save(task.domain, session_payload)
else:
if "Set-Cookie" in response.headers:
cookie_header = self._extract_cookie_header(response)
if cookie_header:
session_payload["cookie"] = cookie_header
session_payload["expired"] = False
await self.session_store.save(task.domain, session_payload)
if response.status >= 500:
self.circuit_breaker.record_failure(task.domain)
else:
self.circuit_breaker.record_success(task.domain)
await self.proxy_pool.mark_success(proxy, (monotonic() - started_at) * 1000)
return response_data
except Exception:
self.circuit_breaker.record_failure(task.domain)
await self.proxy_pool.mark_failed(proxy)
raise
async def save_binary(self, task: CrawlTask, file_name: str, body_bytes: bytes) -> str:
save_dir = self.settings.resolve_path(self.settings.worker.download_dir)
save_dir.mkdir(parents=True, exist_ok=True)
target = save_dir / file_name
target.write_bytes(body_bytes)
return str(target)
def _is_session_expired(self, domain: str, status_code: int, body: Any) -> bool:
if not self.settings.auth.enabled:
return False
domain_cfg = self.settings.auth.domains.get(domain)
if domain_cfg is None:
return False
if status_code in set(domain_cfg.expire_status_codes):
return True
if isinstance(body, str):
lower_body = body.lower()
return any(keyword in lower_body for keyword in domain_cfg.expire_body_keywords)
if isinstance(body, dict):
packed = json.dumps(body, ensure_ascii=False).lower()
return any(keyword in packed for keyword in domain_cfg.expire_body_keywords)
return False
def _extract_cookie_header(self, response: aiohttp.ClientResponse) -> str:
values = response.headers.getall("Set-Cookie", [])
cookie_parts: list[str] = []
for value in values:
head = value.split(";", 1)[0].strip()
if "=" in head:
cookie_parts.append(head)
# keep order and remove duplicates
seen: set[str] = set()
deduped: list[str] = []
for item in cookie_parts:
key = item.split("=", 1)[0].strip()
if key in seen:
continue
seen.add(key)
deduped.append(item)
return "; ".join(deduped)