v6ole
7455d7e426
130 issues auto-fixed (import ordering, UP045/UP006 type annotations), 33 issues manually fixed (E712/E501/E402/E722 + N818 rename + per-file wechat ignore for N8xx naming conventions). All 33 tests pass.
96 lines
3.5 KiB
Python
96 lines
3.5 KiB
Python
from datetime import datetime
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from app.crawler.base import CrawlResult, PipelineConfig
|
|
from app.services.pipeline import PostCrawlPipeline
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_filtered_mode():
|
|
config = PipelineConfig(
|
|
filter_enabled=True,
|
|
keywords=["大化"],
|
|
dedup_enabled=True,
|
|
notify_mode="filtered",
|
|
mark_sent=False,
|
|
)
|
|
result = CrawlResult(
|
|
source_code="test", source_name="test",
|
|
total_count=3, new_count=3,
|
|
announcements=[
|
|
{
|
|
"title": "大化县公告", "publish_date": datetime(2026, 5, 9),
|
|
"purchase_name": "大化县", "content_url": "https://1.com",
|
|
"source_code": "test", "source_name": "测试",
|
|
"announcement_type": "purchase", "crawl_mode": "auto",
|
|
"is_new": True, "is_today": True,
|
|
"content_hash": "abc123",
|
|
},
|
|
{
|
|
"title": "南宁市公告", "publish_date": datetime(2026, 5, 9),
|
|
"purchase_name": "南宁市", "content_url": "https://2.com",
|
|
"source_code": "test", "source_name": "测试",
|
|
"announcement_type": "purchase", "crawl_mode": "auto",
|
|
"is_new": True, "is_today": True,
|
|
"content_hash": "def456",
|
|
},
|
|
],
|
|
)
|
|
|
|
mock_db = AsyncMock()
|
|
mock_notify = AsyncMock()
|
|
|
|
pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
|
|
|
|
with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
|
|
with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=1)):
|
|
pipe_result = await pipeline.process(
|
|
result.announcements, config
|
|
)
|
|
assert pipe_result.stored == 2
|
|
assert pipe_result.filtered == 1
|
|
assert pipe_result.notified == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_all_mode():
|
|
config = PipelineConfig(
|
|
filter_enabled=False,
|
|
keywords=[],
|
|
dedup_enabled=True,
|
|
notify_mode="all",
|
|
mark_sent=True,
|
|
)
|
|
result = CrawlResult(
|
|
source_code="dahuagov", source_name="大化县政府网",
|
|
total_count=2, new_count=2,
|
|
announcements=[
|
|
{
|
|
"title": f"公告{i}", "publish_date": datetime(2026, 5, 9),
|
|
"purchase_name": "大化县", "content_url": f"https://x.com/{i}",
|
|
"source_code": "dahuagov", "source_name": "大化县政府网采购公告",
|
|
"announcement_type": "purchase", "crawl_mode": "auto",
|
|
"is_new": True, "is_today": True,
|
|
"content_hash": f"hash{i}",
|
|
}
|
|
for i in range(2)
|
|
],
|
|
)
|
|
|
|
mock_db = AsyncMock()
|
|
mock_notify = AsyncMock()
|
|
pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
|
|
|
|
with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
|
|
with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=2)):
|
|
with patch.object(pipeline, "_mark_sent", AsyncMock(return_value=2)):
|
|
pipe_result = await pipeline.process(
|
|
result.announcements, config
|
|
)
|
|
assert pipe_result.stored == 2
|
|
assert pipe_result.filtered == 0
|
|
assert pipe_result.notified == 2
|
|
pipeline._mark_sent.assert_awaited_once()
|