From b2d0503bc231a361c123ae6d7cbadc454086ab7f Mon Sep 17 00:00:00 2001
From: v6ole <voole@vip.qq.com>
Date: Tue, 12 May 2026 17:16:33 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20=E5=A4=A7=E5=8C=96=E5=8E=BF=E6=94=BF?=
 =?UTF-8?q?=E5=BA=9C=E7=BD=91=E5=85=AC=E5=91=8A=E6=8A=93=E5=8F=96=E8=AF=A6?=
 =?UTF-8?q?=E6=83=85=E9=A1=B5=E8=8E=B7=E5=8F=96=E7=B2=BE=E7=A1=AE=E5=8F=91?=
 =?UTF-8?q?=E5=B8=83=E6=97=B6=E9=97=B4?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

列表页只有日期(00:00)，改为并发抓取详情页解析 <meta name="PubDate">
获取精确时分秒，失败时保留列表页日期作为 fallback。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 app/crawler/dahuagov_spider.py | 31 +++++++++++++++++++++++++++++--
 app/crawler/parsers.py         | 15 +++++++++++++++
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/app/crawler/dahuagov_spider.py b/app/crawler/dahuagov_spider.py
index 0f20964..bfb33aa 100644
--- a/app/crawler/dahuagov_spider.py
+++ b/app/crawler/dahuagov_spider.py
@@ -6,7 +6,7 @@ import httpx
 
 from app.config import settings
 from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
-from app.crawler.parsers import parse_dahuagov_html
+from app.crawler.parsers import parse_dahuagov_html, parse_dahuagov_detail_pubdate
 
 
 class DahuagovSpider(BaseSpider):
@@ -56,7 +56,9 @@ class DahuagovSpider(BaseSpider):
                     crawled_at=start_time,
                 )
 
-        announcements = parse_dahuagov_html(html, start_time)
+            announcements = parse_dahuagov_html(html, start_time)
+            await self._fetch_detail_dates(client, announcements, headers)
+
         duration = (datetime.now() - start_time).total_seconds()
 
         return CrawlResult(
@@ -69,6 +71,31 @@ class DahuagovSpider(BaseSpider):
             duration=duration,
         )
 
+    async def _fetch_detail_dates(
+        self,
+        client: httpx.AsyncClient,
+        announcements: list[dict],
+        headers: dict,
+    ) -> None:
+        """并发抓取详情页，用 PubDate meta 更新精确发布时间"""
+        sem = asyncio.Semaphore(5)
+
+        async def fetch_one(ann: dict) -> None:
+            async with sem:
+                await asyncio.sleep(random.uniform(0.3, 0.8))
+                try:
+                    resp = await client.get(ann["content_url"], headers=headers)
+                    if resp.status_code == 200:
+                        pub = parse_dahuagov_detail_pubdate(resp.text)
+                        if pub:
+                            ann["publish_date"] = pub
+                            ann["is_today"] = pub.date() == datetime.now().date()
+                except Exception:
+                    pass  # 保留列表页日期作为 fallback
+
+        await asyncio.gather(*[fetch_one(ann) for ann in announcements])
+
     async def _delay(self):
         delay = random.uniform(1.0, 3.0)
         await asyncio.sleep(delay)
+
diff --git a/app/crawler/parsers.py b/app/crawler/parsers.py
index 2d92fd0..b9c32d9 100644
--- a/app/crawler/parsers.py
+++ b/app/crawler/parsers.py
@@ -134,6 +134,21 @@ def parse_dahuagov_html(html: str, crawled_at: datetime) -> list[dict[str, Any]]
     return results
 
 
+def parse_dahuagov_detail_pubdate(html: str) -> datetime | None:
+    """从详情页 <meta name="PubDate"> 解析精确发布时间"""
+    soup = BeautifulSoup(html, "html.parser")
+    meta = soup.find("meta", attrs={"name": "PubDate"})
+    if not meta:
+        return None
+    content = meta.get("content", "").strip()
+    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d"):
+        try:
+            return datetime.strptime(content, fmt)
+        except ValueError:
+            continue
+    return None
+
+
 def _generate_hash(ann: dict[str, Any]) -> str:
     content = (
         f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"