v6ole
cbeb3c3504
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline（爬取→存储→筛选→去重→推送） - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告：爬虫每次从 API 按日期倒序抓取，未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
69 lines
2.1 KiB
Python
69 lines
2.1 KiB
Python
import logging
|
|
from datetime import datetime, time
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
|
|
from app.api.deps import get_crawl_service
|
|
from app.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
scheduler = AsyncIOScheduler()
|
|
|
|
NIGHT_START = time(22, 0)
|
|
NIGHT_END = time(6, 0)
|
|
TZ = ZoneInfo("Asia/Shanghai")
|
|
|
|
|
|
def _is_night_time() -> bool:
|
|
"""22:00 ~ 次日 06:00 夜间时段"""
|
|
current = datetime.now(TZ).time()
|
|
return current >= NIGHT_START or current < NIGHT_END
|
|
|
|
|
|
async def scheduled_crawl():
|
|
if _is_night_time():
|
|
logger.info("夜间时段 (22:00-06:00)，跳过爬取")
|
|
return
|
|
|
|
logger.info("开始定时爬取任务")
|
|
service = get_crawl_service()
|
|
for name in service.get_spider_names():
|
|
try:
|
|
results = await service.run_spider(name)
|
|
for r in results:
|
|
if not r.success:
|
|
logger.error(f"Spider {name} 失败: {r.error_message}")
|
|
elif r.pipeline_result:
|
|
logger.info(
|
|
f"Spider {name}: 抓取{r.total_count}条, "
|
|
f"新增{r.pipeline_result.stored}条, "
|
|
f"通知{r.pipeline_result.notified}条"
|
|
)
|
|
else:
|
|
logger.info(f"Spider {name}: 抓取{r.total_count}条 (无新增)")
|
|
except Exception as e:
|
|
logger.error(f"Spider {name} 异常: {e}")
|
|
logger.info("定时爬取任务完成")
|
|
|
|
|
|
def start_scheduler():
|
|
if not settings.scheduler_enabled:
|
|
return
|
|
trigger = CronTrigger.from_crontab(settings.scheduler_cron, timezone=TZ)
|
|
scheduler.add_job(
|
|
scheduled_crawl,
|
|
trigger=trigger,
|
|
id="scheduled_crawl",
|
|
name="定时爬取",
|
|
)
|
|
scheduler.start()
|
|
logger.info(f"APScheduler 已启动 (cron: {settings.scheduler_cron})")
|
|
|
|
|
|
def shutdown_scheduler():
|
|
if scheduler.running:
|
|
scheduler.shutdown(wait=False)
|
|
logger.info("APScheduler 已停止")
|