fix: 连接 Pipeline 到爬取流程 + 夜间跳过 + 断网恢复
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline(爬取→存储→筛选→去重→推送) - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告:爬虫每次从 API 按日期倒序抓取,未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
This commit is contained in:
+7
-2
@@ -3,10 +3,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.crawler.dahuagov_spider import DahuagovSpider
|
||||
from app.crawler.gxgp_spider import GXGPSpider
|
||||
from app.services.crawl_service import CrawlService
|
||||
from app.services.notification_service import NotificationService
|
||||
|
||||
|
||||
async def get_db() -> AsyncSession:
|
||||
from app.main import async_session # 延迟导入避免循环引用
|
||||
from app.main import async_session
|
||||
async with async_session() as session:
|
||||
yield session
|
||||
|
||||
@@ -17,7 +18,11 @@ _crawl_service: CrawlService | None = None
|
||||
def get_crawl_service() -> CrawlService:
|
||||
global _crawl_service
|
||||
if _crawl_service is None:
|
||||
_crawl_service = CrawlService()
|
||||
from app.main import async_session
|
||||
_crawl_service = CrawlService(
|
||||
db_session_factory=async_session,
|
||||
notification_service=NotificationService(),
|
||||
)
|
||||
_crawl_service.register(GXGPSpider())
|
||||
_crawl_service.register(DahuagovSpider())
|
||||
return _crawl_service
|
||||
|
||||
Reference in New Issue
Block a user