first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+206
View File
@@ -0,0 +1,206 @@
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import yaml
@dataclass
class MasterSettings:
host: str = "127.0.0.1"
port: int = 8088
worker_ttl_seconds: int = 30
stale_task_seconds: int = 120
bloom_capacity: int = 1_000_000
bloom_false_positive_rate: float = 0.01
bloom_hashes: int | None = None
queue_name: str = "crawler"
result_queue_name: str = "crawler:results"
dead_letter_queue_name: str = "crawler:dead_letter"
default_spider: str = "generic_fetch"
@dataclass
class WorkerSettings:
worker_id: str = "worker-1"
concurrency: int = 100
heartbeat_interval_seconds: int = 5
request_timeout_seconds: int = 20
connect_timeout_seconds: int = 10
retries: int = 4
base_backoff_seconds: float = 1.0
max_backoff_seconds: float = 16.0
download_dir: str = "./runtime/downloads"
@dataclass
class StorageSettings:
backend: str = "jsonl"
jsonl_path: str = "./runtime/results/results.jsonl"
mongodb_dsn: str = "mongodb://127.0.0.1:27017"
mongodb_database: str = "crawler"
mongodb_collection: str = "results"
postgres_dsn: str = "postgresql://postgres:postgres@127.0.0.1:5432/crawler"
postgres_table: str = "crawl_results"
@dataclass
class ProxySettings:
enabled: bool = False
verify_url: str = "https://httpbin.org/ip"
verify_timeout_seconds: int = 8
refresh_interval_seconds: int = 300
failure_threshold: int = 3
cooldown_seconds: int = 60
static_proxies: list[str] = field(default_factory=list)
provider_api_url: str = ""
provider_auth_header: str = ""
@dataclass
class CircuitBreakerSettings:
enabled: bool = True
failure_threshold: int = 8
recovery_timeout_seconds: int = 30
half_open_success_threshold: int = 2
@dataclass
class SessionSettings:
enabled: bool = True
redis_prefix: str = "crawler:sessions"
local_backup_path: str = "./runtime/sessions/sessions.json"
@dataclass
class AuthDomainSettings:
login_url: str = ""
username: str = ""
password: str = ""
username_field: str = "username"
password_field: str = "password"
submit_type: str = "form"
extra_payload: dict[str, Any] = field(default_factory=dict)
success_status_codes: list[int] = field(default_factory=lambda: [200])
expire_status_codes: list[int] = field(default_factory=lambda: [401, 403])
expire_body_keywords: list[str] = field(default_factory=lambda: ["login required", "please login", "unauthorized"])
@dataclass
class AuthSettings:
enabled: bool = False
domains: dict[str, AuthDomainSettings] = field(default_factory=dict)
@dataclass
class DomainRateLimitSettings:
qps: float = 3.0
burst: int = 6
@dataclass
class RateLimitSettings:
default_qps: float = 3.0
default_burst: int = 6
per_domain: dict[str, DomainRateLimitSettings] = field(default_factory=dict)
@dataclass
class ScheduleTaskSettings:
name: str
trigger: str
task: dict[str, Any]
priority: str = "medium"
seconds: int | None = None
minutes: int | None = None
cron: str | None = None
@dataclass
class AppSettings:
app_name: str = "distributed-crawler"
redis_dsn: str = "redis://127.0.0.1:6379/0"
checkpoint_dir: str = "./runtime/checkpoints"
master: MasterSettings = field(default_factory=MasterSettings)
worker: WorkerSettings = field(default_factory=WorkerSettings)
storage: StorageSettings = field(default_factory=StorageSettings)
proxy: ProxySettings = field(default_factory=ProxySettings)
circuit_breaker: CircuitBreakerSettings = field(default_factory=CircuitBreakerSettings)
session: SessionSettings = field(default_factory=SessionSettings)
auth: AuthSettings = field(default_factory=AuthSettings)
rate_limit: RateLimitSettings = field(default_factory=RateLimitSettings)
schedules: list[ScheduleTaskSettings] = field(default_factory=list)
source_path: Path | None = None
@property
def base_dir(self) -> Path:
if self.source_path is None:
return Path.cwd()
return self.source_path.parent
def resolve_path(self, value: str) -> Path:
candidate = Path(value)
if candidate.is_absolute():
return candidate
return (self.base_dir / candidate).resolve()
def _load_domain_limits(raw_limits: dict[str, Any]) -> dict[str, DomainRateLimitSettings]:
result: dict[str, DomainRateLimitSettings] = {}
for domain, item in raw_limits.items():
result[domain] = DomainRateLimitSettings(
qps=float(item.get("qps", 3.0)),
burst=int(item.get("burst", 6)),
)
return result
def _load_auth_domains(raw_domains: dict[str, Any]) -> dict[str, AuthDomainSettings]:
result: dict[str, AuthDomainSettings] = {}
for domain, item in raw_domains.items():
if not isinstance(item, dict):
continue
result[domain] = AuthDomainSettings(
login_url=str(item.get("login_url", "")),
username=str(item.get("username", "")),
password=str(item.get("password", "")),
username_field=str(item.get("username_field", "username")),
password_field=str(item.get("password_field", "password")),
submit_type=str(item.get("submit_type", "form")),
extra_payload=dict(item.get("extra_payload", {})),
success_status_codes=[int(v) for v in item.get("success_status_codes", [200])],
expire_status_codes=[int(v) for v in item.get("expire_status_codes", [401, 403])],
expire_body_keywords=[str(v).lower() for v in item.get("expire_body_keywords", ["login required", "please login", "unauthorized"])],
)
return result
def load_settings(path: str | Path) -> AppSettings:
config_path = Path(path).resolve()
data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
settings = AppSettings(
app_name=str(data.get("app_name", "distributed-crawler")),
redis_dsn=str(data.get("redis_dsn", "redis://127.0.0.1:6379/0")),
checkpoint_dir=str(data.get("checkpoint_dir", "./runtime/checkpoints")),
master=MasterSettings(**(data.get("master") or {})),
worker=WorkerSettings(**(data.get("worker") or {})),
storage=StorageSettings(**(data.get("storage") or {})),
proxy=ProxySettings(**(data.get("proxy") or {})),
circuit_breaker=CircuitBreakerSettings(**(data.get("circuit_breaker") or {})),
session=SessionSettings(**(data.get("session") or {})),
auth=AuthSettings(
enabled=bool((data.get("auth") or {}).get("enabled", False)),
domains=_load_auth_domains((data.get("auth") or {}).get("domains", {})),
),
rate_limit=RateLimitSettings(
default_qps=float((data.get("rate_limit") or {}).get("default_qps", 3.0)),
default_burst=int((data.get("rate_limit") or {}).get("default_burst", 6)),
per_domain=_load_domain_limits((data.get("rate_limit") or {}).get("per_domain", {})),
),
schedules=[ScheduleTaskSettings(**item) for item in data.get("schedules", [])],
source_path=config_path,
)
return settings