fix: 连接 Pipeline 到爬取流程 + 夜间跳过 + 断网恢复

- CrawlService.run_spider 现在自动调用 PostCrawlPipeline(爬取→存储→筛选→去重→推送)
- 定时任务添加 22:00-06:00 夜间跳过逻辑
- 断网恢复后不会漏公告:爬虫每次从 API 按日期倒序抓取,未推送的公告下次自动补推
- Pipeline 的 _exclude_sent + mark_sent 防止重复推送
This commit is contained in:
2026-05-09 16:10:30 +08:00
parent d851dafea9
commit cbeb3c3504
6 changed files with 106 additions and 25 deletions
+25 -4
View File
@@ -1,4 +1,6 @@
import logging
from datetime import datetime, time
from zoneinfo import ZoneInfo
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
@@ -9,19 +11,38 @@ from app.config import settings
logger = logging.getLogger(__name__)
scheduler = AsyncIOScheduler()
NIGHT_START = time(22, 0)
NIGHT_END = time(6, 0)
TZ = ZoneInfo("Asia/Shanghai")
def _is_night_time() -> bool:
"""22:00 ~ 次日 06:00 夜间时段"""
current = datetime.now(TZ).time()
return current >= NIGHT_START or current < NIGHT_END
async def scheduled_crawl():
if _is_night_time():
logger.info("夜间时段 (22:00-06:00),跳过爬取")
return
logger.info("开始定时爬取任务")
service = get_crawl_service()
names = service.get_spider_names()
for name in names:
for name in service.get_spider_names():
try:
results = await service.run_spider(name)
for r in results:
if not r.success:
logger.error(f"Spider {name} 失败: {r.error_message}")
elif r.pipeline_result:
logger.info(
f"Spider {name}: 抓取{r.total_count}条, "
f"新增{r.pipeline_result.stored}条, "
f"通知{r.pipeline_result.notified}"
)
else:
logger.info(f"Spider {name} 完成: {r.total_count} ")
logger.info(f"Spider {name}: 抓取{r.total_count} (无新增)")
except Exception as e:
logger.error(f"Spider {name} 异常: {e}")
logger.info("定时爬取任务完成")
@@ -30,7 +51,7 @@ async def scheduled_crawl():
def start_scheduler():
if not settings.scheduler_enabled:
return
trigger = CronTrigger.from_crontab(settings.scheduler_cron, timezone="Asia/Shanghai")
trigger = CronTrigger.from_crontab(settings.scheduler_cron, timezone=TZ)
scheduler.add_job(
scheduled_crawl,
trigger=trigger,