diff --git a/app/crawler/dahuagov_spider.py b/app/crawler/dahuagov_spider.py new file mode 100644 index 0000000..f95ae3d --- /dev/null +++ b/app/crawler/dahuagov_spider.py @@ -0,0 +1,72 @@ +import asyncio +import random +from datetime import datetime +import httpx +from app.config import settings +from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig +from app.crawler.parsers import parse_dahuagov_html + + +class DahuagovSpider(BaseSpider): + name = "dahuagov" + source_code = "dahuagov" + source_name = "大化县政府网采购公告" + + BASE_URL = "http://www.gxdh.gov.cn" + ANNOUNCEMENT_PATH = "/xxgk/zdlyxxgk/ggzypzly/zfcgly/cggg/" + + def get_pipeline_config(self) -> PipelineConfig: + return PipelineConfig( + filter_enabled=False, + keywords=[], + dedup_enabled=True, + notify_mode="all", + mark_sent=True, + ) + + async def crawl(self) -> CrawlResult: + start_time = datetime.now() + url = self.BASE_URL + self.ANNOUNCEMENT_PATH + + async with httpx.AsyncClient(timeout=settings.crawler_timeout) as client: + await self._delay() + try: + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Accept": "text/html,application/xhtml+xml", + "Accept-Language": "zh-CN,zh;q=0.9", + "Referer": self.BASE_URL, + } + response = await client.get(url, headers=headers) + if response.status_code != 200: + return CrawlResult( + source_code=self.source_code, + source_name=self.source_name, + error_message=f"HTTP {response.status_code}", + crawled_at=start_time, + ) + html = response.text + except Exception as e: + return CrawlResult( + source_code=self.source_code, + source_name=self.source_name, + error_message=str(e), + crawled_at=start_time, + ) + + announcements = parse_dahuagov_html(html, start_time) + duration = (datetime.now() - start_time).total_seconds() + + return CrawlResult( + source_code=self.source_code, + source_name=self.source_name, + total_count=len(announcements), + new_count=len(announcements), + announcements=announcements, + crawled_at=start_time, + duration=duration, + ) + + async def _delay(self): + delay = random.uniform(1.0, 3.0) + await asyncio.sleep(delay) diff --git a/tests/test_crawler/test_dahuagov_spider.py b/tests/test_crawler/test_dahuagov_spider.py new file mode 100644 index 0000000..7f20a37 --- /dev/null +++ b/tests/test_crawler/test_dahuagov_spider.py @@ -0,0 +1,62 @@ +import pytest +from unittest.mock import AsyncMock, patch, MagicMock +from app.crawler.dahuagov_spider import DahuagovSpider +from app.crawler.base import PipelineConfig + + +@pytest.mark.asyncio +async def test_dahuagov_spider_attributes(): + spider = DahuagovSpider() + assert spider.name == "dahuagov" + assert spider.source_code == "dahuagov" + assert spider.source_name == "大化县政府网采购公告" + + +@pytest.mark.asyncio +async def test_dahuagov_spider_pipeline_config(): + spider = DahuagovSpider() + config = spider.get_pipeline_config() + assert isinstance(config, PipelineConfig) + assert config.filter_enabled is False + assert config.notify_mode == "all" + assert config.mark_sent is True + + +@pytest.mark.asyncio +async def test_dahuagov_spider_crawl(): + spider = DahuagovSpider() + + html = """ + + + + """ + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = html + + with patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)): + result = await spider.crawl() + assert result.total_count == 2 + assert len(result.announcements) == 2 + assert result.success is True + + +@pytest.mark.asyncio +async def test_dahuagov_spider_crawl_empty(): + spider = DahuagovSpider() + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "" + + with patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)): + result = await spider.crawl() + assert result.total_count == 0 + assert result.success is True