fix: 连接 Pipeline 到爬取流程 + 夜间跳过 + 断网恢复

- CrawlService.run_spider 现在自动调用 PostCrawlPipeline(爬取→存储→筛选→去重→推送)
- 定时任务添加 22:00-06:00 夜间跳过逻辑
- 断网恢复后不会漏公告:爬虫每次从 API 按日期倒序抓取,未推送的公告下次自动补推
- Pipeline 的 _exclude_sent + mark_sent 防止重复推送
This commit is contained in:
2026-05-09 16:10:30 +08:00
parent d851dafea9
commit cbeb3c3504
6 changed files with 106 additions and 25 deletions
+7 -2
View File
@@ -3,10 +3,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.crawler.dahuagov_spider import DahuagovSpider
from app.crawler.gxgp_spider import GXGPSpider
from app.services.crawl_service import CrawlService
from app.services.notification_service import NotificationService
async def get_db() -> AsyncSession:
from app.main import async_session # 延迟导入避免循环引用
from app.main import async_session
async with async_session() as session:
yield session
@@ -17,7 +18,11 @@ _crawl_service: CrawlService | None = None
def get_crawl_service() -> CrawlService:
global _crawl_service
if _crawl_service is None:
_crawl_service = CrawlService()
from app.main import async_session
_crawl_service = CrawlService(
db_session_factory=async_session,
notification_service=NotificationService(),
)
_crawl_service.register(GXGPSpider())
_crawl_service.register(DahuagovSpider())
return _crawl_service