fix: 连接 Pipeline 到爬取流程 + 夜间跳过 + 断网恢复
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline(爬取→存储→筛选→去重→推送) - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告:爬虫每次从 API 按日期倒序抓取,未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -49,9 +50,16 @@ class MockDahuagovSpider(BaseSpider):
|
||||
)
|
||||
|
||||
|
||||
def _make_service():
|
||||
return CrawlService(
|
||||
db_session_factory=AsyncMock(),
|
||||
notification_service=AsyncMock(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_service_registers_spiders():
|
||||
service = CrawlService()
|
||||
service = _make_service()
|
||||
service.register(MockGXGPSpider())
|
||||
service.register(MockDahuagovSpider())
|
||||
assert len(service.spiders) == 2
|
||||
@@ -59,7 +67,7 @@ async def test_crawl_service_registers_spiders():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_service_run_all():
|
||||
service = CrawlService()
|
||||
service = _make_service()
|
||||
service.register(MockGXGPSpider())
|
||||
service.register(MockDahuagovSpider())
|
||||
results = await service.run_all()
|
||||
@@ -71,7 +79,7 @@ async def test_crawl_service_run_all():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_service_run_specific():
|
||||
service = CrawlService()
|
||||
service = _make_service()
|
||||
service.register(MockGXGPSpider())
|
||||
service.register(MockDahuagovSpider())
|
||||
results = await service.run_spider("mock_dahuagov")
|
||||
@@ -81,7 +89,7 @@ async def test_crawl_service_run_specific():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_service_run_unknown():
|
||||
service = CrawlService()
|
||||
service = _make_service()
|
||||
results = await service.run_spider("nonexistent")
|
||||
assert len(results) == 1
|
||||
assert results[0].success is False
|
||||
|
||||
Reference in New Issue
Block a user