v6ole
cbeb3c3504
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline（爬取→存储→筛选→去重→推送） - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告：爬虫每次从 API 按日期倒序抓取，未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
96 lines
2.5 KiB
Python
96 lines
2.5 KiB
Python
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
|
|
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
|
from app.services.crawl_service import CrawlService
|
|
|
|
|
|
class MockGXGPSpider(BaseSpider):
|
|
name = "mock_gxgp"
|
|
source_code = "mock_gxgp"
|
|
source_name = "Mock GXGP"
|
|
|
|
async def crawl(self, **kwargs):
|
|
return CrawlResult(
|
|
source_code=self.source_code,
|
|
source_name=self.source_name,
|
|
total_count=10,
|
|
new_count=5,
|
|
announcements=[],
|
|
)
|
|
|
|
def get_pipeline_config(self):
|
|
return PipelineConfig(
|
|
filter_enabled=True,
|
|
keywords=["大化"],
|
|
notify_mode="filtered",
|
|
)
|
|
|
|
|
|
class MockDahuagovSpider(BaseSpider):
|
|
name = "mock_dahuagov"
|
|
source_code = "mock_dahuagov"
|
|
source_name = "Mock Dahuagov"
|
|
|
|
async def crawl(self, **kwargs):
|
|
return CrawlResult(
|
|
source_code=self.source_code,
|
|
source_name=self.source_name,
|
|
total_count=3,
|
|
new_count=3,
|
|
announcements=[],
|
|
)
|
|
|
|
def get_pipeline_config(self):
|
|
return PipelineConfig(
|
|
filter_enabled=False,
|
|
notify_mode="all",
|
|
mark_sent=True,
|
|
)
|
|
|
|
|
|
def _make_service():
|
|
return CrawlService(
|
|
db_session_factory=AsyncMock(),
|
|
notification_service=AsyncMock(),
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_crawl_service_registers_spiders():
|
|
service = _make_service()
|
|
service.register(MockGXGPSpider())
|
|
service.register(MockDahuagovSpider())
|
|
assert len(service.spiders) == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_crawl_service_run_all():
|
|
service = _make_service()
|
|
service.register(MockGXGPSpider())
|
|
service.register(MockDahuagovSpider())
|
|
results = await service.run_all()
|
|
assert len(results) == 2
|
|
assert results[0].total_count == 10
|
|
assert results[1].total_count == 3
|
|
assert all(r.success for r in results)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_crawl_service_run_specific():
|
|
service = _make_service()
|
|
service.register(MockGXGPSpider())
|
|
service.register(MockDahuagovSpider())
|
|
results = await service.run_spider("mock_dahuagov")
|
|
assert len(results) == 1
|
|
assert results[0].source_code == "mock_dahuagov"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_crawl_service_run_unknown():
|
|
service = _make_service()
|
|
results = await service.run_spider("nonexistent")
|
|
assert len(results) == 1
|
|
assert results[0].success is False
|