206 lines
7.1 KiB
Python
206 lines
7.1 KiB
Python
from __future__ import annotations
|
|||
|
|
|
||
|
|
from dataclasses import dataclass, field
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
import yaml
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class MasterSettings:
|
||
|
|
host: str = "127.0.0.1"
|
||
|
|
port: int = 8088
|
||
|
|
worker_ttl_seconds: int = 30
|
||
|
|
stale_task_seconds: int = 120
|
||
|
|
bloom_capacity: int = 1_000_000
|
||
|
|
bloom_false_positive_rate: float = 0.01
|
||
|
|
bloom_hashes: int | None = None
|
||
|
|
queue_name: str = "crawler"
|
||
|
|
result_queue_name: str = "crawler:results"
|
||
|
|
dead_letter_queue_name: str = "crawler:dead_letter"
|
||
|
|
default_spider: str = "generic_fetch"
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class WorkerSettings:
|
||
|
|
worker_id: str = "worker-1"
|
||
|
|
concurrency: int = 100
|
||
|
|
heartbeat_interval_seconds: int = 5
|
||
|
|
request_timeout_seconds: int = 20
|
||
|
|
connect_timeout_seconds: int = 10
|
||
|
|
retries: int = 4
|
||
|
|
base_backoff_seconds: float = 1.0
|
||
|
|
max_backoff_seconds: float = 16.0
|
||
|
|
download_dir: str = "./runtime/downloads"
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class StorageSettings:
|
||
|
|
backend: str = "jsonl"
|
||
|
|
jsonl_path: str = "./runtime/results/results.jsonl"
|
||
|
|
mongodb_dsn: str = "mongodb://127.0.0.1:27017"
|
||
|
|
mongodb_database: str = "crawler"
|
||
|
|
mongodb_collection: str = "results"
|
||
|
|
postgres_dsn: str = "postgresql://postgres:postgres@127.0.0.1:5432/crawler"
|
||
|
|
postgres_table: str = "crawl_results"
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class ProxySettings:
|
||
|
|
enabled: bool = False
|
||
|
|
verify_url: str = "https://httpbin.org/ip"
|
||
|
|
verify_timeout_seconds: int = 8
|
||
|
|
refresh_interval_seconds: int = 300
|
||
|
|
failure_threshold: int = 3
|
||
|
|
cooldown_seconds: int = 60
|
||
|
|
static_proxies: list[str] = field(default_factory=list)
|
||
|
|
provider_api_url: str = ""
|
||
|
|
provider_auth_header: str = ""
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class CircuitBreakerSettings:
|
||
|
|
enabled: bool = True
|
||
|
|
failure_threshold: int = 8
|
||
|
|
recovery_timeout_seconds: int = 30
|
||
|
|
half_open_success_threshold: int = 2
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class SessionSettings:
|
||
|
|
enabled: bool = True
|
||
|
|
redis_prefix: str = "crawler:sessions"
|
||
|
|
local_backup_path: str = "./runtime/sessions/sessions.json"
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class AuthDomainSettings:
|
||
|
|
login_url: str = ""
|
||
|
|
username: str = ""
|
||
|
|
password: str = ""
|
||
|
|
username_field: str = "username"
|
||
|
|
password_field: str = "password"
|
||
|
|
submit_type: str = "form"
|
||
|
|
extra_payload: dict[str, Any] = field(default_factory=dict)
|
||
|
|
success_status_codes: list[int] = field(default_factory=lambda: [200])
|
||
|
|
expire_status_codes: list[int] = field(default_factory=lambda: [401, 403])
|
||
|
|
expire_body_keywords: list[str] = field(default_factory=lambda: ["login required", "please login", "unauthorized"])
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class AuthSettings:
|
||
|
|
enabled: bool = False
|
||
|
|
domains: dict[str, AuthDomainSettings] = field(default_factory=dict)
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class DomainRateLimitSettings:
|
||
|
|
qps: float = 3.0
|
||
|
|
burst: int = 6
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class RateLimitSettings:
|
||
|
|
default_qps: float = 3.0
|
||
|
|
default_burst: int = 6
|
||
|
|
per_domain: dict[str, DomainRateLimitSettings] = field(default_factory=dict)
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class ScheduleTaskSettings:
|
||
|
|
name: str
|
||
|
|
trigger: str
|
||
|
|
task: dict[str, Any]
|
||
|
|
priority: str = "medium"
|
||
|
|
seconds: int | None = None
|
||
|
|
minutes: int | None = None
|
||
|
|
cron: str | None = None
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class AppSettings:
|
||
|
|
app_name: str = "distributed-crawler"
|
||
|
|
redis_dsn: str = "redis://127.0.0.1:6379/0"
|
||
|
|
checkpoint_dir: str = "./runtime/checkpoints"
|
||
|
|
master: MasterSettings = field(default_factory=MasterSettings)
|
||
|
|
worker: WorkerSettings = field(default_factory=WorkerSettings)
|
||
|
|
storage: StorageSettings = field(default_factory=StorageSettings)
|
||
|
|
proxy: ProxySettings = field(default_factory=ProxySettings)
|
||
|
|
circuit_breaker: CircuitBreakerSettings = field(default_factory=CircuitBreakerSettings)
|
||
|
|
session: SessionSettings = field(default_factory=SessionSettings)
|
||
|
|
auth: AuthSettings = field(default_factory=AuthSettings)
|
||
|
|
rate_limit: RateLimitSettings = field(default_factory=RateLimitSettings)
|
||
|
|
schedules: list[ScheduleTaskSettings] = field(default_factory=list)
|
||
|
|
source_path: Path | None = None
|
||
|
|
|
||
|
|
@property
|
||
|
|
def base_dir(self) -> Path:
|
||
|
|
if self.source_path is None:
|
||
|
|
return Path.cwd()
|
||
|
|
return self.source_path.parent
|
||
|
|
|
||
|
|
def resolve_path(self, value: str) -> Path:
|
||
|
|
candidate = Path(value)
|
||
|
|
if candidate.is_absolute():
|
||
|
|
return candidate
|
||
|
|
return (self.base_dir / candidate).resolve()
|
||
|
|
|
||
|
|
|
||
|
|
def _load_domain_limits(raw_limits: dict[str, Any]) -> dict[str, DomainRateLimitSettings]:
|
||
|
|
result: dict[str, DomainRateLimitSettings] = {}
|
||
|
|
for domain, item in raw_limits.items():
|
||
|
|
result[domain] = DomainRateLimitSettings(
|
||
|
|
qps=float(item.get("qps", 3.0)),
|
||
|
|
burst=int(item.get("burst", 6)),
|
||
|
|
)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _load_auth_domains(raw_domains: dict[str, Any]) -> dict[str, AuthDomainSettings]:
|
||
|
|
result: dict[str, AuthDomainSettings] = {}
|
||
|
|
for domain, item in raw_domains.items():
|
||
|
|
if not isinstance(item, dict):
|
||
|
|
continue
|
||
|
|
result[domain] = AuthDomainSettings(
|
||
|
|
login_url=str(item.get("login_url", "")),
|
||
|
|
username=str(item.get("username", "")),
|
||
|
|
password=str(item.get("password", "")),
|
||
|
|
username_field=str(item.get("username_field", "username")),
|
||
|
|
password_field=str(item.get("password_field", "password")),
|
||
|
|
submit_type=str(item.get("submit_type", "form")),
|
||
|
|
extra_payload=dict(item.get("extra_payload", {})),
|
||
|
|
success_status_codes=[int(v) for v in item.get("success_status_codes", [200])],
|
||
|
|
expire_status_codes=[int(v) for v in item.get("expire_status_codes", [401, 403])],
|
||
|
|
expire_body_keywords=[str(v).lower() for v in item.get("expire_body_keywords", ["login required", "please login", "unauthorized"])],
|
||
|
|
)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def load_settings(path: str | Path) -> AppSettings:
|
||
|
|
config_path = Path(path).resolve()
|
||
|
|
data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
|
||
|
|
|
||
|
|
settings = AppSettings(
|
||
|
|
app_name=str(data.get("app_name", "distributed-crawler")),
|
||
|
|
redis_dsn=str(data.get("redis_dsn", "redis://127.0.0.1:6379/0")),
|
||
|
|
checkpoint_dir=str(data.get("checkpoint_dir", "./runtime/checkpoints")),
|
||
|
|
master=MasterSettings(**(data.get("master") or {})),
|
||
|
|
worker=WorkerSettings(**(data.get("worker") or {})),
|
||
|
|
storage=StorageSettings(**(data.get("storage") or {})),
|
||
|
|
proxy=ProxySettings(**(data.get("proxy") or {})),
|
||
|
|
circuit_breaker=CircuitBreakerSettings(**(data.get("circuit_breaker") or {})),
|
||
|
|
session=SessionSettings(**(data.get("session") or {})),
|
||
|
|
auth=AuthSettings(
|
||
|
|
enabled=bool((data.get("auth") or {}).get("enabled", False)),
|
||
|
|
domains=_load_auth_domains((data.get("auth") or {}).get("domains", {})),
|
||
|
|
),
|
||
|
|
rate_limit=RateLimitSettings(
|
||
|
|
default_qps=float((data.get("rate_limit") or {}).get("default_qps", 3.0)),
|
||
|
|
default_burst=int((data.get("rate_limit") or {}).get("default_burst", 6)),
|
||
|
|
per_domain=_load_domain_limits((data.get("rate_limit") or {}).get("per_domain", {})),
|
||
|
|
),
|
||
|
|
schedules=[ScheduleTaskSettings(**item) for item in data.get("schedules", [])],
|
||
|
|
source_path=config_path,
|
||
|
|
)
|
||
|
|
return settings
|