feat: 添加 CrawlService 爬取编排器 + 测试

app/services/crawl_service.py

@@ -0,0 +1,36 @@
+from typing import Dict, List
+from app.crawler.base import BaseSpider, CrawlResult
+
+
+class CrawlService:
+    def __init__(self):
+        self.spiders: Dict[str, BaseSpider] = {}
+
+    def register(self, spider: BaseSpider):
+        self.spiders[spider.name] = spider
+
+    async def run_all(self) -> List[CrawlResult]:
+        results = []
+        for name, spider in self.spiders.items():
+            result = await spider.crawl()
+            results.append(result)
+        return results
+
+    async def run_spider(self, name: str, **kwargs) -> List[CrawlResult]:
+        spider = self.spiders.get(name)
+        if spider is None:
+            return [CrawlResult(
+                source_code=name, source_name=name,
+                error_message=f"Spider not found: {name}"
+            )]
+        result = await spider.crawl(**kwargs)
+        return [result]
+
+    def get_spider_names(self) -> List[str]:
+        return list(self.spiders.keys())
+
+    def get_pipeline_config(self, name: str):
+        spider = self.spiders.get(name)
+        if spider:
+            return spider.get_pipeline_config()
+        return None

tests/test_services/test_crawl_service.py

@@ -0,0 +1,86 @@
+import pytest
+from datetime import datetime
+from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
+from app.services.crawl_service import CrawlService
+
+
+class MockGXGPSpider(BaseSpider):
+    name = "mock_gxgp"
+    source_code = "mock_gxgp"
+    source_name = "Mock GXGP"
+
+    async def crawl(self, **kwargs):
+        return CrawlResult(
+            source_code=self.source_code,
+            source_name=self.source_name,
+            total_count=10,
+            new_count=5,
+            announcements=[],
+        )
+
+    def get_pipeline_config(self):
+        return PipelineConfig(
+            filter_enabled=True,
+            keywords=["大化"],
+            notify_mode="filtered",
+        )
+
+
+class MockDahuagovSpider(BaseSpider):
+    name = "mock_dahuagov"
+    source_code = "mock_dahuagov"
+    source_name = "Mock Dahuagov"
+
+    async def crawl(self, **kwargs):
+        return CrawlResult(
+            source_code=self.source_code,
+            source_name=self.source_name,
+            total_count=3,
+            new_count=3,
+            announcements=[],
+        )
+
+    def get_pipeline_config(self):
+        return PipelineConfig(
+            filter_enabled=False,
+            notify_mode="all",
+            mark_sent=True,
+        )
+
+
+@pytest.mark.asyncio
+async def test_crawl_service_registers_spiders():
+    service = CrawlService()
+    service.register(MockGXGPSpider())
+    service.register(MockDahuagovSpider())
+    assert len(service.spiders) == 2
+
+
+@pytest.mark.asyncio
+async def test_crawl_service_run_all():
+    service = CrawlService()
+    service.register(MockGXGPSpider())
+    service.register(MockDahuagovSpider())
+    results = await service.run_all()
+    assert len(results) == 2
+    assert results[0].total_count == 10
+    assert results[1].total_count == 3
+    assert all(r.success for r in results)
+
+
+@pytest.mark.asyncio
+async def test_crawl_service_run_specific():
+    service = CrawlService()
+    service.register(MockGXGPSpider())
+    service.register(MockDahuagovSpider())
+    results = await service.run_spider("mock_dahuagov")
+    assert len(results) == 1
+    assert results[0].source_code == "mock_dahuagov"
+
+
+@pytest.mark.asyncio
+async def test_crawl_service_run_unknown():
+    service = CrawlService()
+    results = await service.run_spider("nonexistent")
+    assert len(results) == 1
+    assert results[0].success is False