first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+34
View File
@@ -0,0 +1,34 @@
from __future__ import annotations
import argparse
import asyncio
import logging
from crawler.config import load_settings
from crawler.worker import WorkerNode
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--config", required=True)
parser.add_argument("--worker-id", required=False)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
settings = load_settings(args.config)
if args.worker_id:
settings.worker.worker_id = args.worker_id
node = WorkerNode(settings)
await node.start()
logging.getLogger(__name__).info("Worker started: %s", settings.worker.worker_id)
try:
while True:
await asyncio.sleep(3600)
finally:
await node.close()
if __name__ == "__main__":
asyncio.run(main())