v6ole
7455d7e426
130 issues auto-fixed (import ordering, UP045/UP006 type annotations), 33 issues manually fixed (E712/E501/E402/E722 + N818 rename + per-file wechat ignore for N8xx naming conventions). All 33 tests pass.
49 lines
1.3 KiB
Python
49 lines
1.3 KiB
Python
import logging
|
|
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
|
|
from app.api.deps import get_crawl_service
|
|
from app.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
scheduler = AsyncIOScheduler()
|
|
|
|
|
|
async def scheduled_crawl():
|
|
logger.info("开始定时爬取任务")
|
|
service = get_crawl_service()
|
|
names = service.get_spider_names()
|
|
for name in names:
|
|
try:
|
|
results = await service.run_spider(name)
|
|
for r in results:
|
|
if not r.success:
|
|
logger.error(f"Spider {name} 失败: {r.error_message}")
|
|
else:
|
|
logger.info(f"Spider {name} 完成: {r.total_count} 条")
|
|
except Exception as e:
|
|
logger.error(f"Spider {name} 异常: {e}")
|
|
logger.info("定时爬取任务完成")
|
|
|
|
|
|
def start_scheduler():
|
|
if not settings.scheduler_enabled:
|
|
return
|
|
scheduler.add_job(
|
|
scheduled_crawl,
|
|
"cron",
|
|
hour="8,14,18",
|
|
minute="0",
|
|
id="scheduled_crawl",
|
|
name="定时爬取",
|
|
timezone="Asia/Shanghai",
|
|
)
|
|
scheduler.start()
|
|
logger.info("APScheduler 已启动 (8:00, 14:00, 18:00)")
|
|
|
|
|
|
def shutdown_scheduler():
|
|
if scheduler.running:
|
|
scheduler.shutdown(wait=False)
|
|
logger.info("APScheduler 已停止")
|