commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
from .config import AppSettings
|
||||
from .models import CrawlTask
|
||||
|
||||
|
||||
class ScheduleService:
|
||||
def __init__(self, settings: AppSettings, submit_callback) -> None:
|
||||
self.settings = settings
|
||||
self.submit_callback = submit_callback
|
||||
self.scheduler = AsyncIOScheduler()
|
||||
|
||||
def start(self) -> None:
|
||||
for item in self.settings.schedules:
|
||||
if item.trigger == "cron" and item.cron:
|
||||
trigger = CronTrigger.from_crontab(item.cron)
|
||||
else:
|
||||
seconds = item.seconds or 0
|
||||
minutes = item.minutes or 0
|
||||
trigger = IntervalTrigger(seconds=seconds, minutes=minutes)
|
||||
self.scheduler.add_job(self._submit_job, trigger=trigger, args=[item], id=item.name, replace_existing=True)
|
||||
self.scheduler.start()
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
self.scheduler.shutdown(wait=False)
|
||||
|
||||
async def _submit_job(self, item) -> None:
|
||||
payload = dict(item.task)
|
||||
task = CrawlTask.create(
|
||||
url=payload["url"],
|
||||
spider=payload.get("spider", self.settings.master.default_spider),
|
||||
method=payload.get("method", "GET"),
|
||||
headers=payload.get("headers"),
|
||||
metadata=payload.get("metadata"),
|
||||
body=payload.get("body"),
|
||||
priority=item.priority,
|
||||
schedule_name=item.name,
|
||||
)
|
||||
await self.submit_callback(task)
|
||||
Reference in new issue
Block a user