fix: 连接 Pipeline 到爬取流程 + 夜间跳过 + 断网恢复
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline(爬取→存储→筛选→去重→推送) - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告:爬虫每次从 API 按日期倒序抓取,未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
This commit is contained in:
@@ -12,13 +12,21 @@ async def trigger_crawl(request: CrawlTriggerRequest):
|
||||
names = service.get_spider_names()
|
||||
|
||||
all_results = []
|
||||
total_stored = 0
|
||||
total_notified = 0
|
||||
for name in names:
|
||||
results = await service.run_spider(name)
|
||||
all_results.extend(results)
|
||||
for r in results:
|
||||
if r.pipeline_result:
|
||||
total_stored += r.pipeline_result.stored
|
||||
total_notified += r.pipeline_result.notified
|
||||
|
||||
return {
|
||||
"spiders_run": names,
|
||||
"total_announcements": sum(r.total_count for r in all_results),
|
||||
"total_stored": total_stored,
|
||||
"total_notified": total_notified,
|
||||
"errors": [r.error_message for r in all_results if not r.success],
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user