fix: 连接 Pipeline 到爬取流程 + 夜间跳过 + 断网恢复

- CrawlService.run_spider 现在自动调用 PostCrawlPipeline(爬取→存储→筛选→去重→推送)
- 定时任务添加 22:00-06:00 夜间跳过逻辑
- 断网恢复后不会漏公告:爬虫每次从 API 按日期倒序抓取,未推送的公告下次自动补推
- Pipeline 的 _exclude_sent + mark_sent 防止重复推送
This commit is contained in:
2026-05-09 16:10:30 +08:00
parent d851dafea9
commit cbeb3c3504
6 changed files with 106 additions and 25 deletions
+12 -4
View File
@@ -1,3 +1,4 @@
from unittest.mock import AsyncMock
import pytest
@@ -49,9 +50,16 @@ class MockDahuagovSpider(BaseSpider):
)
def _make_service():
return CrawlService(
db_session_factory=AsyncMock(),
notification_service=AsyncMock(),
)
@pytest.mark.asyncio
async def test_crawl_service_registers_spiders():
service = CrawlService()
service = _make_service()
service.register(MockGXGPSpider())
service.register(MockDahuagovSpider())
assert len(service.spiders) == 2
@@ -59,7 +67,7 @@ async def test_crawl_service_registers_spiders():
@pytest.mark.asyncio
async def test_crawl_service_run_all():
service = CrawlService()
service = _make_service()
service.register(MockGXGPSpider())
service.register(MockDahuagovSpider())
results = await service.run_all()
@@ -71,7 +79,7 @@ async def test_crawl_service_run_all():
@pytest.mark.asyncio
async def test_crawl_service_run_specific():
service = CrawlService()
service = _make_service()
service.register(MockGXGPSpider())
service.register(MockDahuagovSpider())
results = await service.run_spider("mock_dahuagov")
@@ -81,7 +89,7 @@ async def test_crawl_service_run_specific():
@pytest.mark.asyncio
async def test_crawl_service_run_unknown():
service = CrawlService()
service = _make_service()
results = await service.run_spider("nonexistent")
assert len(results) == 1
assert results[0].success is False