48 lines
1.4 KiB
Python
48 lines
1.4 KiB
Python
import logging
|
|
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
|
|
from app.api.deps import get_crawl_service
|
|
from app.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
scheduler = AsyncIOScheduler()
|
|
|
|
|
|
async def scheduled_crawl():
|
|
logger.info("开始定时爬取任务")
|
|
service = get_crawl_service()
|
|
names = service.get_spider_names()
|
|
for name in names:
|
|
try:
|
|
results = await service.run_spider(name)
|
|
for r in results:
|
|
if not r.success:
|
|
logger.error(f"Spider {name} 失败: {r.error_message}")
|
|
else:
|
|
logger.info(f"Spider {name} 完成: {r.total_count} 条")
|
|
except Exception as e:
|
|
logger.error(f"Spider {name} 异常: {e}")
|
|
logger.info("定时爬取任务完成")
|
|
|
|
|
|
def start_scheduler():
|
|
if not settings.scheduler_enabled:
|
|
return
|
|
trigger = CronTrigger.from_crontab(settings.scheduler_cron, timezone="Asia/Shanghai")
|
|
scheduler.add_job(
|
|
scheduled_crawl,
|
|
trigger=trigger,
|
|
id="scheduled_crawl",
|
|
name="定时爬取",
|
|
)
|
|
scheduler.start()
|
|
logger.info(f"APScheduler 已启动 (cron: {settings.scheduler_cron})")
|
|
|
|
|
|
def shutdown_scheduler():
|
|
if scheduler.running:
|
|
scheduler.shutdown(wait=False)
|
|
logger.info("APScheduler 已停止")
|