feat: 添加 Dahuagov Spider（大化县政府网爬虫）+ 测试
This commit is contained in:
@@ -0,0 +1,72 @@
|
|||||||
|
import asyncio
|
||||||
|
import random
|
||||||
|
from datetime import datetime
|
||||||
|
import httpx
|
||||||
|
from app.config import settings
|
||||||
|
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||||
|
from app.crawler.parsers import parse_dahuagov_html
|
||||||
|
|
||||||
|
|
||||||
|
class DahuagovSpider(BaseSpider):
|
||||||
|
name = "dahuagov"
|
||||||
|
source_code = "dahuagov"
|
||||||
|
source_name = "大化县政府网采购公告"
|
||||||
|
|
||||||
|
BASE_URL = "http://www.gxdh.gov.cn"
|
||||||
|
ANNOUNCEMENT_PATH = "/xxgk/zdlyxxgk/ggzypzly/zfcgly/cggg/"
|
||||||
|
|
||||||
|
def get_pipeline_config(self) -> PipelineConfig:
|
||||||
|
return PipelineConfig(
|
||||||
|
filter_enabled=False,
|
||||||
|
keywords=[],
|
||||||
|
dedup_enabled=True,
|
||||||
|
notify_mode="all",
|
||||||
|
mark_sent=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def crawl(self) -> CrawlResult:
|
||||||
|
start_time = datetime.now()
|
||||||
|
url = self.BASE_URL + self.ANNOUNCEMENT_PATH
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(timeout=settings.crawler_timeout) as client:
|
||||||
|
await self._delay()
|
||||||
|
try:
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||||
|
"Referer": self.BASE_URL,
|
||||||
|
}
|
||||||
|
response = await client.get(url, headers=headers)
|
||||||
|
if response.status_code != 200:
|
||||||
|
return CrawlResult(
|
||||||
|
source_code=self.source_code,
|
||||||
|
source_name=self.source_name,
|
||||||
|
error_message=f"HTTP {response.status_code}",
|
||||||
|
crawled_at=start_time,
|
||||||
|
)
|
||||||
|
html = response.text
|
||||||
|
except Exception as e:
|
||||||
|
return CrawlResult(
|
||||||
|
source_code=self.source_code,
|
||||||
|
source_name=self.source_name,
|
||||||
|
error_message=str(e),
|
||||||
|
crawled_at=start_time,
|
||||||
|
)
|
||||||
|
|
||||||
|
announcements = parse_dahuagov_html(html, start_time)
|
||||||
|
duration = (datetime.now() - start_time).total_seconds()
|
||||||
|
|
||||||
|
return CrawlResult(
|
||||||
|
source_code=self.source_code,
|
||||||
|
source_name=self.source_name,
|
||||||
|
total_count=len(announcements),
|
||||||
|
new_count=len(announcements),
|
||||||
|
announcements=announcements,
|
||||||
|
crawled_at=start_time,
|
||||||
|
duration=duration,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _delay(self):
|
||||||
|
delay = random.uniform(1.0, 3.0)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import pytest
|
||||||
|
from unittest.mock import AsyncMock, patch, MagicMock
|
||||||
|
from app.crawler.dahuagov_spider import DahuagovSpider
|
||||||
|
from app.crawler.base import PipelineConfig
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dahuagov_spider_attributes():
|
||||||
|
spider = DahuagovSpider()
|
||||||
|
assert spider.name == "dahuagov"
|
||||||
|
assert spider.source_code == "dahuagov"
|
||||||
|
assert spider.source_name == "大化县政府网采购公告"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dahuagov_spider_pipeline_config():
|
||||||
|
spider = DahuagovSpider()
|
||||||
|
config = spider.get_pipeline_config()
|
||||||
|
assert isinstance(config, PipelineConfig)
|
||||||
|
assert config.filter_enabled is False
|
||||||
|
assert config.notify_mode == "all"
|
||||||
|
assert config.mark_sent is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dahuagov_spider_crawl():
|
||||||
|
spider = DahuagovSpider()
|
||||||
|
|
||||||
|
html = """
|
||||||
|
<html><body>
|
||||||
|
<ul class="more-list">
|
||||||
|
<li><span>2026-05-08</span>
|
||||||
|
<a href="./detail/1.html" title="公告1">公告1</a></li>
|
||||||
|
<li><span>2026-05-07</span>
|
||||||
|
<a href="./detail/2.html" title="公告2">公告2</a></li>
|
||||||
|
</ul>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.status_code = 200
|
||||||
|
mock_response.text = html
|
||||||
|
|
||||||
|
with patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
|
||||||
|
result = await spider.crawl()
|
||||||
|
assert result.total_count == 2
|
||||||
|
assert len(result.announcements) == 2
|
||||||
|
assert result.success is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dahuagov_spider_crawl_empty():
|
||||||
|
spider = DahuagovSpider()
|
||||||
|
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.status_code = 200
|
||||||
|
mock_response.text = "<html><body></body></html>"
|
||||||
|
|
||||||
|
with patch("httpx.AsyncClient.get", AsyncMock(return_value=mock_response)):
|
||||||
|
result = await spider.crawl()
|
||||||
|
assert result.total_count == 0
|
||||||
|
assert result.success is True
|
||||||
Reference in New Issue
Block a user