From 022d34805232c3ef51b1ce0b63376bea7d340056 Mon Sep 17 00:00:00 2001
From: v6ole <voole@vip.qq.com>
Date: Sat, 9 May 2026 13:50:58 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=20CrawlService=20?=
 =?UTF-8?q?=E7=88=AC=E5=8F=96=E7=BC=96=E6=8E=92=E5=99=A8=20+=20=E6=B5=8B?=
 =?UTF-8?q?=E8=AF=95?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/services/__init__.py                  |  0
 app/services/crawl_service.py             | 36 ++++++++++
 tests/test_services/__init__.py           |  0
 tests/test_services/test_crawl_service.py | 86 +++++++++++++++++++++++
 4 files changed, 122 insertions(+)
 create mode 100644 app/services/__init__.py
 create mode 100644 app/services/crawl_service.py
 create mode 100644 tests/test_services/__init__.py
 create mode 100644 tests/test_services/test_crawl_service.py

diff --git a/app/services/__init__.py b/app/services/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/app/services/crawl_service.py b/app/services/crawl_service.py
new file mode 100644
index 0000000..5f9c396
--- /dev/null
+++ b/app/services/crawl_service.py
@@ -0,0 +1,36 @@
+from typing import Dict, List
+from app.crawler.base import BaseSpider, CrawlResult
+
+
+class CrawlService:
+    def __init__(self):
+        self.spiders: Dict[str, BaseSpider] = {}
+
+    def register(self, spider: BaseSpider):
+        self.spiders[spider.name] = spider
+
+    async def run_all(self) -> List[CrawlResult]:
+        results = []
+        for name, spider in self.spiders.items():
+            result = await spider.crawl()
+            results.append(result)
+        return results
+
+    async def run_spider(self, name: str, **kwargs) -> List[CrawlResult]:
+        spider = self.spiders.get(name)
+        if spider is None:
+            return [CrawlResult(
+                source_code=name, source_name=name,
+                error_message=f"Spider not found: {name}"
+            )]
+        result = await spider.crawl(**kwargs)
+        return [result]
+
+    def get_spider_names(self) -> List[str]:
+        return list(self.spiders.keys())
+
+    def get_pipeline_config(self, name: str):
+        spider = self.spiders.get(name)
+        if spider:
+            return spider.get_pipeline_config()
+        return None
diff --git a/tests/test_services/__init__.py b/tests/test_services/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_services/test_crawl_service.py b/tests/test_services/test_crawl_service.py
new file mode 100644
index 0000000..e593950
--- /dev/null
+++ b/tests/test_services/test_crawl_service.py
@@ -0,0 +1,86 @@
+import pytest
+from datetime import datetime
+from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
+from app.services.crawl_service import CrawlService
+
+
+class MockGXGPSpider(BaseSpider):
+    name = "mock_gxgp"
+    source_code = "mock_gxgp"
+    source_name = "Mock GXGP"
+
+    async def crawl(self, **kwargs):
+        return CrawlResult(
+            source_code=self.source_code,
+            source_name=self.source_name,
+            total_count=10,
+            new_count=5,
+            announcements=[],
+        )
+
+    def get_pipeline_config(self):
+        return PipelineConfig(
+            filter_enabled=True,
+            keywords=["大化"],
+            notify_mode="filtered",
+        )
+
+
+class MockDahuagovSpider(BaseSpider):
+    name = "mock_dahuagov"
+    source_code = "mock_dahuagov"
+    source_name = "Mock Dahuagov"
+
+    async def crawl(self, **kwargs):
+        return CrawlResult(
+            source_code=self.source_code,
+            source_name=self.source_name,
+            total_count=3,
+            new_count=3,
+            announcements=[],
+        )
+
+    def get_pipeline_config(self):
+        return PipelineConfig(
+            filter_enabled=False,
+            notify_mode="all",
+            mark_sent=True,
+        )
+
+
+@pytest.mark.asyncio
+async def test_crawl_service_registers_spiders():
+    service = CrawlService()
+    service.register(MockGXGPSpider())
+    service.register(MockDahuagovSpider())
+    assert len(service.spiders) == 2
+
+
+@pytest.mark.asyncio
+async def test_crawl_service_run_all():
+    service = CrawlService()
+    service.register(MockGXGPSpider())
+    service.register(MockDahuagovSpider())
+    results = await service.run_all()
+    assert len(results) == 2
+    assert results[0].total_count == 10
+    assert results[1].total_count == 3
+    assert all(r.success for r in results)
+
+
+@pytest.mark.asyncio
+async def test_crawl_service_run_specific():
+    service = CrawlService()
+    service.register(MockGXGPSpider())
+    service.register(MockDahuagovSpider())
+    results = await service.run_spider("mock_dahuagov")
+    assert len(results) == 1
+    assert results[0].source_code == "mock_dahuagov"
+
+
+@pytest.mark.asyncio
+async def test_crawl_service_run_unknown():
+    service = CrawlService()
+    results = await service.run_spider("nonexistent")
+    assert len(results) == 1
+    assert results[0].success is False