fix: 大化县政府网公告抓取详情页获取精确发布时间
列表页只有日期(00:00),改为并发抓取详情页解析 <meta name="PubDate"> 获取精确时分秒,失败时保留列表页日期作为 fallback。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -134,6 +134,21 @@ def parse_dahuagov_html(html: str, crawled_at: datetime) -> list[dict[str, Any]]
|
||||
return results
|
||||
|
||||
|
||||
def parse_dahuagov_detail_pubdate(html: str) -> datetime | None:
|
||||
"""从详情页 <meta name="PubDate"> 解析精确发布时间"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
meta = soup.find("meta", attrs={"name": "PubDate"})
|
||||
if not meta:
|
||||
return None
|
||||
content = meta.get("content", "").strip()
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(content, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _generate_hash(ann: dict[str, Any]) -> str:
|
||||
content = (
|
||||
f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"
|
||||
|
||||
Reference in New Issue
Block a user