44 lines
1.6 KiB
Python
44 lines
1.6 KiB
Python
from __future__ import annotations
|
|
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
from apscheduler.triggers.interval import IntervalTrigger
|
|
|
|
from .config import AppSettings
|
|
from .models import CrawlTask
|
|
|
|
|
|
class ScheduleService:
|
|
def __init__(self, settings: AppSettings, submit_callback) -> None:
|
|
self.settings = settings
|
|
self.submit_callback = submit_callback
|
|
self.scheduler = AsyncIOScheduler()
|
|
|
|
def start(self) -> None:
|
|
for item in self.settings.schedules:
|
|
if item.trigger == "cron" and item.cron:
|
|
trigger = CronTrigger.from_crontab(item.cron)
|
|
else:
|
|
seconds = item.seconds or 0
|
|
minutes = item.minutes or 0
|
|
trigger = IntervalTrigger(seconds=seconds, minutes=minutes)
|
|
self.scheduler.add_job(self._submit_job, trigger=trigger, args=[item], id=item.name, replace_existing=True)
|
|
self.scheduler.start()
|
|
|
|
async def shutdown(self) -> None:
|
|
self.scheduler.shutdown(wait=False)
|
|
|
|
async def _submit_job(self, item) -> None:
|
|
payload = dict(item.task)
|
|
task = CrawlTask.create(
|
|
url=payload["url"],
|
|
spider=payload.get("spider", self.settings.master.default_spider),
|
|
method=payload.get("method", "GET"),
|
|
headers=payload.get("headers"),
|
|
metadata=payload.get("metadata"),
|
|
body=payload.get("body"),
|
|
priority=item.priority,
|
|
schedule_name=item.name,
|
|
)
|
|
await self.submit_callback(task)
|