v6ole
b2d0503bc2
列表页只有日期(00:00)，改为并发抓取详情页解析 <meta name="PubDate"> 获取精确时分秒，失败时保留列表页日期作为 fallback。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
102 lines
3.6 KiB
Python
102 lines
3.6 KiB
Python
import asyncio
|
|
import random
|
|
from datetime import datetime
|
|
|
|
import httpx
|
|
|
|
from app.config import settings
|
|
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
|
from app.crawler.parsers import parse_dahuagov_html, parse_dahuagov_detail_pubdate
|
|
|
|
|
|
class DahuagovSpider(BaseSpider):
|
|
name = "dahuagov"
|
|
source_code = "dahuagov"
|
|
source_name = "大化县政府网采购公告"
|
|
|
|
BASE_URL = "http://www.gxdh.gov.cn"
|
|
ANNOUNCEMENT_PATH = "/xxgk/zdlyxxgk/ggzypzly/zfcgly/cggg/"
|
|
|
|
def get_pipeline_config(self) -> PipelineConfig:
|
|
return PipelineConfig(
|
|
filter_enabled=False,
|
|
keywords=[],
|
|
dedup_enabled=True,
|
|
notify_mode="all",
|
|
mark_sent=True,
|
|
)
|
|
|
|
async def crawl(self) -> CrawlResult:
|
|
start_time = datetime.now()
|
|
url = self.BASE_URL + self.ANNOUNCEMENT_PATH
|
|
|
|
async with httpx.AsyncClient(timeout=settings.crawler_timeout) as client:
|
|
await self._delay()
|
|
try:
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
"Accept": "text/html,application/xhtml+xml",
|
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
|
"Referer": self.BASE_URL,
|
|
}
|
|
response = await client.get(url, headers=headers)
|
|
if response.status_code != 200:
|
|
return CrawlResult(
|
|
source_code=self.source_code,
|
|
source_name=self.source_name,
|
|
error_message=f"HTTP {response.status_code}",
|
|
crawled_at=start_time,
|
|
)
|
|
html = response.text
|
|
except Exception as e:
|
|
return CrawlResult(
|
|
source_code=self.source_code,
|
|
source_name=self.source_name,
|
|
error_message=str(e),
|
|
crawled_at=start_time,
|
|
)
|
|
|
|
announcements = parse_dahuagov_html(html, start_time)
|
|
await self._fetch_detail_dates(client, announcements, headers)
|
|
|
|
duration = (datetime.now() - start_time).total_seconds()
|
|
|
|
return CrawlResult(
|
|
source_code=self.source_code,
|
|
source_name=self.source_name,
|
|
total_count=len(announcements),
|
|
new_count=len(announcements),
|
|
announcements=announcements,
|
|
crawled_at=start_time,
|
|
duration=duration,
|
|
)
|
|
|
|
async def _fetch_detail_dates(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
announcements: list[dict],
|
|
headers: dict,
|
|
) -> None:
|
|
"""并发抓取详情页，用 PubDate meta 更新精确发布时间"""
|
|
sem = asyncio.Semaphore(5)
|
|
|
|
async def fetch_one(ann: dict) -> None:
|
|
async with sem:
|
|
await asyncio.sleep(random.uniform(0.3, 0.8))
|
|
try:
|
|
resp = await client.get(ann["content_url"], headers=headers)
|
|
if resp.status_code == 200:
|
|
pub = parse_dahuagov_detail_pubdate(resp.text)
|
|
if pub:
|
|
ann["publish_date"] = pub
|
|
ann["is_today"] = pub.date() == datetime.now().date()
|
|
except Exception:
|
|
pass # 保留列表页日期作为 fallback
|
|
|
|
await asyncio.gather(*[fetch_one(ann) for ann in announcements])
|
|
|
|
async def _delay(self):
|
|
delay = random.uniform(1.0, 3.0)
|
|
await asyncio.sleep(delay)
|
|
|