v6ole
d851dafea9
- Pipeline 新增 _exclude_sent() 方法，通知前查询数据库跳过 is_sent=True 的公告 - GXGP Spider mark_sent 改为 True，配合去重逻辑防止每次爬取重复通知
102 lines
3.8 KiB
Python
102 lines
3.8 KiB
Python
from datetime import datetime
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from app.crawler.base import CrawlResult, PipelineConfig
|
|
from app.services.pipeline import PostCrawlPipeline
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_filtered_mode():
|
|
config = PipelineConfig(
|
|
filter_enabled=True,
|
|
keywords=["大化"],
|
|
dedup_enabled=True,
|
|
notify_mode="filtered",
|
|
mark_sent=False,
|
|
)
|
|
result = CrawlResult(
|
|
source_code="test", source_name="test",
|
|
total_count=3, new_count=3,
|
|
announcements=[
|
|
{
|
|
"title": "大化县公告", "publish_date": datetime(2026, 5, 9),
|
|
"purchase_name": "大化县", "content_url": "https://1.com",
|
|
"source_code": "test", "source_name": "测试",
|
|
"announcement_type": "purchase", "crawl_mode": "auto",
|
|
"is_new": True, "is_today": True,
|
|
"content_hash": "abc123",
|
|
},
|
|
{
|
|
"title": "南宁市公告", "publish_date": datetime(2026, 5, 9),
|
|
"purchase_name": "南宁市", "content_url": "https://2.com",
|
|
"source_code": "test", "source_name": "测试",
|
|
"announcement_type": "purchase", "crawl_mode": "auto",
|
|
"is_new": True, "is_today": True,
|
|
"content_hash": "def456",
|
|
},
|
|
],
|
|
)
|
|
|
|
mock_db = AsyncMock()
|
|
mock_notify = AsyncMock()
|
|
|
|
pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
|
|
|
|
with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
|
|
with patch.object(pipeline, "_exclude_sent", AsyncMock(
|
|
side_effect=lambda anns: [a for a in anns if "大化" in a["title"]]
|
|
)):
|
|
with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=1)):
|
|
pipe_result = await pipeline.process(
|
|
result.announcements, config
|
|
)
|
|
assert pipe_result.stored == 2
|
|
assert pipe_result.filtered == 1
|
|
assert pipe_result.notified == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_all_mode():
|
|
config = PipelineConfig(
|
|
filter_enabled=False,
|
|
keywords=[],
|
|
dedup_enabled=True,
|
|
notify_mode="all",
|
|
mark_sent=True,
|
|
)
|
|
result = CrawlResult(
|
|
source_code="dahuagov", source_name="大化县政府网",
|
|
total_count=2, new_count=2,
|
|
announcements=[
|
|
{
|
|
"title": f"公告{i}", "publish_date": datetime(2026, 5, 9),
|
|
"purchase_name": "大化县", "content_url": f"https://x.com/{i}",
|
|
"source_code": "dahuagov", "source_name": "大化县政府网采购公告",
|
|
"announcement_type": "purchase", "crawl_mode": "auto",
|
|
"is_new": True, "is_today": True,
|
|
"content_hash": f"hash{i}",
|
|
}
|
|
for i in range(2)
|
|
],
|
|
)
|
|
|
|
mock_db = AsyncMock()
|
|
mock_notify = AsyncMock()
|
|
pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
|
|
|
|
with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
|
|
with patch.object(pipeline, "_exclude_sent", AsyncMock(
|
|
side_effect=lambda anns: anns
|
|
)):
|
|
with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=2)):
|
|
with patch.object(pipeline, "_mark_sent", AsyncMock(return_value=2)):
|
|
pipe_result = await pipeline.process(
|
|
result.announcements, config
|
|
)
|
|
assert pipe_result.stored == 2
|
|
assert pipe_result.filtered == 0
|
|
assert pipe_result.notified == 2
|
|
pipeline._mark_sent.assert_awaited_once()
|