feat: 添加 PostCrawlPipeline 统一管道 + 筛选服务 + 测试

tests/test_services/test_filter_service.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+from app.services.filter_service import filter_by_keywords, filter_by_date
+
+
+def test_filter_by_keywords_match():
+    announcements = [
+        {"title": "大化县采购公告", "purchase_name": "大化县财政局",
+         "source_code": "test", "source_name": "test"},
+        {"title": "南宁市采购公告", "purchase_name": "南宁市财政局",
+         "source_code": "test", "source_name": "test"},
+    ]
+    result = filter_by_keywords(announcements, ["大化"])
+    assert len(result) == 1
+    assert result[0]["title"] == "大化县采购公告"
+
+
+def test_filter_by_keywords_no_keywords():
+    announcements = [
+        {"title": "大化县采购公告", "purchase_name": "x",
+         "source_code": "test", "source_name": "test"},
+    ]
+    result = filter_by_keywords(announcements, [])
+    assert len(result) == 1
+
+
+def test_filter_by_date_range():
+    today = datetime(2026, 5, 9)
+    announcements = [
+        {"title": "t1", "publish_date": datetime(2026, 5, 9),
+         "source_code": "test", "source_name": "test"},
+        {"title": "t2", "publish_date": datetime(2026, 5, 1),
+         "source_code": "test", "source_name": "test"},
+        {"title": "t3", "publish_date": datetime(2026, 4, 30),
+         "source_code": "test", "source_name": "test"},
+    ]
+    result = filter_by_date(announcements, start_date=today, end_date=today)
+    assert len(result) == 1
+    assert result[0]["title"] == "t1"

tests/test_services/test_pipeline.py

@@ -0,0 +1,93 @@
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+from datetime import datetime
+from app.services.pipeline import PostCrawlPipeline
+from app.crawler.base import CrawlResult, PipelineConfig
+
+
+@pytest.mark.asyncio
+async def test_pipeline_filtered_mode():
+    config = PipelineConfig(
+        filter_enabled=True,
+        keywords=["大化"],
+        dedup_enabled=True,
+        notify_mode="filtered",
+        mark_sent=False,
+    )
+    result = CrawlResult(
+        source_code="test", source_name="test",
+        total_count=3, new_count=3,
+        announcements=[
+            {
+                "title": "大化县公告", "publish_date": datetime(2026, 5, 9),
+                "purchase_name": "大化县", "content_url": "https://1.com",
+                "source_code": "test", "source_name": "测试",
+                "announcement_type": "purchase", "crawl_mode": "auto",
+                "is_new": True, "is_today": True,
+                "content_hash": "abc123",
+            },
+            {
+                "title": "南宁市公告", "publish_date": datetime(2026, 5, 9),
+                "purchase_name": "南宁市", "content_url": "https://2.com",
+                "source_code": "test", "source_name": "测试",
+                "announcement_type": "purchase", "crawl_mode": "auto",
+                "is_new": True, "is_today": True,
+                "content_hash": "def456",
+            },
+        ],
+    )
+
+    mock_db = AsyncMock()
+    mock_notify = AsyncMock()
+
+    pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
+
+    with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
+        with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=1)):
+            pipe_result = await pipeline.process(
+                result.announcements, config
+            )
+            assert pipe_result.stored == 2
+            assert pipe_result.filtered == 1
+            assert pipe_result.notified == 1
+
+
+@pytest.mark.asyncio
+async def test_pipeline_all_mode():
+    config = PipelineConfig(
+        filter_enabled=False,
+        keywords=[],
+        dedup_enabled=True,
+        notify_mode="all",
+        mark_sent=True,
+    )
+    result = CrawlResult(
+        source_code="dahuagov", source_name="大化县政府网",
+        total_count=2, new_count=2,
+        announcements=[
+            {
+                "title": f"公告{i}", "publish_date": datetime(2026, 5, 9),
+                "purchase_name": "大化县", "content_url": f"https://x.com/{i}",
+                "source_code": "dahuagov", "source_name": "大化县政府网采购公告",
+                "announcement_type": "purchase", "crawl_mode": "auto",
+                "is_new": True, "is_today": True,
+                "content_hash": f"hash{i}",
+            }
+            for i in range(2)
+        ],
+    )
+
+    mock_db = AsyncMock()
+    mock_notify = AsyncMock()
+    pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
+
+    with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
+        with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=2)):
+            with patch.object(pipeline, "_mark_sent", AsyncMock(return_value=2)):
+                pipe_result = await pipeline.process(
+                    result.announcements, config
+                )
+                assert pipe_result.stored == 2
+                assert pipe_result.filtered == 0
+                assert pipe_result.notified == 2
+                pipeline._mark_sent.assert_awaited_once()