Files
2026-11-16 20:45:22 +08:00

94 lines
2.1 KiB
YAML

app_name: distributed-crawler
redis_dsn: redis://127.0.0.1:6379/0
checkpoint_dir: ./runtime/checkpoints
master:
host: 127.0.0.1
port: 8088
worker_ttl_seconds: 30
stale_task_seconds: 120
bloom_capacity: 1000000
bloom_false_positive_rate: 0.01
bloom_hashes:
queue_name: crawler
result_queue_name: crawler:results
dead_letter_queue_name: crawler:dead_letter
default_spider: generic_fetch
worker:
worker_id: worker-1
concurrency: 100
heartbeat_interval_seconds: 5
request_timeout_seconds: 20
connect_timeout_seconds: 10
retries: 4
base_backoff_seconds: 1
max_backoff_seconds: 16
download_dir: ./runtime/downloads
storage:
backend: jsonl
jsonl_path: ./runtime/results/results.jsonl
mongodb_dsn: mongodb://127.0.0.1:27017
mongodb_database: crawler
mongodb_collection: results
postgres_dsn: postgresql://postgres:postgres@127.0.0.1:5432/crawler
postgres_table: crawl_results
proxy:
enabled: false
verify_url: https://httpbin.org/ip
verify_timeout_seconds: 8
refresh_interval_seconds: 300
failure_threshold: 3
cooldown_seconds: 60
static_proxies: []
provider_api_url: ""
provider_auth_header: ""
circuit_breaker:
enabled: true
failure_threshold: 8
recovery_timeout_seconds: 30
half_open_success_threshold: 2
session:
enabled: true
redis_prefix: crawler:sessions
local_backup_path: ./runtime/sessions/sessions.json
auth:
enabled: false
domains:
example.com:
login_url: https://example.com/api/login
username: your_username
password: your_password
username_field: username
password_field: password
submit_type: json # json or form
extra_payload: {}
success_status_codes: [200]
expire_status_codes: [401, 403]
expire_body_keywords: ["login required", "please login", "unauthorized"]
rate_limit:
default_qps: 3.0
default_burst: 6
per_domain:
httpbin.org:
qps: 2.0
burst: 4
schedules:
- name: sample-httpbin
trigger: interval
seconds: 300
priority: medium
task:
spider: generic_fetch
url: https://httpbin.org/get
method: GET
metadata:
source: schedule