Files
SakuraHub/distributed_crawler/crawler/scheduler.py
T
2026-11-16 20:45:22 +08:00

44 lines
1.6 KiB
Python

from __future__ import annotations
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.interval import IntervalTrigger
from .config import AppSettings
from .models import CrawlTask
class ScheduleService:
def __init__(self, settings: AppSettings, submit_callback) -> None:
self.settings = settings
self.submit_callback = submit_callback
self.scheduler = AsyncIOScheduler()
def start(self) -> None:
for item in self.settings.schedules:
if item.trigger == "cron" and item.cron:
trigger = CronTrigger.from_crontab(item.cron)
else:
seconds = item.seconds or 0
minutes = item.minutes or 0
trigger = IntervalTrigger(seconds=seconds, minutes=minutes)
self.scheduler.add_job(self._submit_job, trigger=trigger, args=[item], id=item.name, replace_existing=True)
self.scheduler.start()
async def shutdown(self) -> None:
self.scheduler.shutdown(wait=False)
async def _submit_job(self, item) -> None:
payload = dict(item.task)
task = CrawlTask.create(
url=payload["url"],
spider=payload.get("spider", self.settings.master.default_spider),
method=payload.get("method", "GET"),
headers=payload.get("headers"),
metadata=payload.get("metadata"),
body=payload.get("body"),
priority=item.priority,
schedule_name=item.name,
)
await self.submit_callback(task)