feat: 添加 Dahuagov Spider（大化县政府网爬虫）+ 测试

2026-05-09 13:47:48 +08:00
parent b1bbf64287
commit 5e0582d499
2 changed files with 134 additions and 0 deletions
@@ -0,0 +1,72 @@
+import asyncio
+import random
+from datetime import datetime
+import httpx
+from app.config import settings
+from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
+from app.crawler.parsers import parse_dahuagov_html
+
+
+class DahuagovSpider(BaseSpider):
+    name = "dahuagov"
+    source_code = "dahuagov"
+    source_name = "大化县政府网采购公告"
+
+    BASE_URL = "http://www.gxdh.gov.cn"
+    ANNOUNCEMENT_PATH = "/xxgk/zdlyxxgk/ggzypzly/zfcgly/cggg/"
+
+    def get_pipeline_config(self) -> PipelineConfig:
+        return PipelineConfig(
+            filter_enabled=False,
+            keywords=[],
+            dedup_enabled=True,
+            notify_mode="all",
+            mark_sent=True,
+        )
+
+    async def crawl(self) -> CrawlResult:
+        start_time = datetime.now()
+        url = self.BASE_URL + self.ANNOUNCEMENT_PATH
+
+        async with httpx.AsyncClient(timeout=settings.crawler_timeout) as client:
+            await self._delay()
+            try:
+                headers = {
+                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+                    "Accept": "text/html,application/xhtml+xml",
+                    "Accept-Language": "zh-CN,zh;q=0.9",
+                    "Referer": self.BASE_URL,
+                }
+                response = await client.get(url, headers=headers)
+                if response.status_code != 200:
+                    return CrawlResult(
+                        source_code=self.source_code,
+                        source_name=self.source_name,
+                        error_message=f"HTTP {response.status_code}",
+                        crawled_at=start_time,
+                    )
+                html = response.text
+            except Exception as e:
+                return CrawlResult(
+                    source_code=self.source_code,
+                    source_name=self.source_name,
+                    error_message=str(e),
+                    crawled_at=start_time,
+                )
+
+        announcements = parse_dahuagov_html(html, start_time)
+        duration = (datetime.now() - start_time).total_seconds()
+
+        return CrawlResult(
+            source_code=self.source_code,
+            source_name=self.source_name,
+            total_count=len(announcements),
+            new_count=len(announcements),
+            announcements=announcements,
+            crawled_at=start_time,
+            duration=duration,
+        )
+
+    async def _delay(self):
+        delay = random.uniform(1.0, 3.0)
+        await asyncio.sleep(delay)
@@ -0,0 +1,62 @@
+import pytest
+from unittest.mock import AsyncMock, patch, MagicMock
+from app.crawler.dahuagov_spider import DahuagovSpider
+from app.crawler.base import PipelineConfig
+
+
+@pytest.mark.asyncio
+async def test_dahuagov_spider_attributes():
+    spider = DahuagovSpider()
+    assert spider.name == "dahuagov"
+    assert spider.source_code == "dahuagov"
+    assert spider.source_name == "大化县政府网采购公告"
+
+
+@pytest.mark.asyncio
+async def test_dahuagov_spider_pipeline_config():
+    spider = DahuagovSpider()
+    config = spider.get_pipeline_config()
+    assert isinstance(config, PipelineConfig)
+    assert config.filter_enabled is False
+    assert config.notify_mode == "all"
+    assert config.mark_sent is True
+
+
+@pytest.mark.asyncio
+async def test_dahuagov_spider_crawl():
+    spider = DahuagovSpider()
+
+    html = """
+    <html><body>
+      <ul class="more-list">
+        <li><span>2026-05-08</span>
+            <a href="./detail/1.html" title="公告1">公告1</a></li>
+        <li><span>2026-05-07</span>
+            <a href="./detail/2.html" title="公告2">公告2</a></li>
+      </ul>
+    </body></html>
+    """
+
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.text = html
+
+    with patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
+        result = await spider.crawl()
+        assert result.total_count == 2
+        assert len(result.announcements) == 2
+        assert result.success is True
+
+
+@pytest.mark.asyncio
+async def test_dahuagov_spider_crawl_empty():
+    spider = DahuagovSpider()
+
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.text = "<html><body></body></html>"
+
+    with patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
+        result = await spider.crawl()
+        assert result.total_count == 0
+        assert result.success is True