feat: 接入 DeepSeek AI 分析公告是否为中国电信可承接项目
- 新增 ai_enabled/ai_api_key/ai_base_url/ai_model 等配置项,通过 .env 管理 - 新增 app/services/ai_analyzer.py — DeepSeek API 调用 + 详情页正文提取 - 新增 extract_page_content() 从详情页抓取正文供 AI 分析 - Announcement 模型新增 ai_relevant / ai_analysis 字段 - 流水线集成 AI 分析步骤(关键词匹配后、通知前) - AI 标记为可承接的项目额外发送 markdown 着重通知 - 创建 alembic 迁移版本 6e8f4c2d1b0a
This commit is contained in:
@@ -149,6 +149,57 @@ def parse_dahuagov_detail_pubdate(html: str) -> datetime | None:
|
||||
return None
|
||||
|
||||
|
||||
async def extract_page_content(url: str, timeout: int = 30) -> str | None:
|
||||
"""抓取详情页并用 BeautifulSoup 提取正文纯文本"""
|
||||
import httpx
|
||||
|
||||
try:
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=headers)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
|
||||
# 尝试多种常见正文容器选择器
|
||||
selectors = [
|
||||
"div.article-content", "div.content", "div.TRS_Editor",
|
||||
"div.Custom_UnionStyle", "div.pages_content", "div#content",
|
||||
"div.main-content", "article", ".article", ".detail-content",
|
||||
".text-content", ".news-content", ".detail-article",
|
||||
]
|
||||
for selector in selectors:
|
||||
container = soup.select_one(selector)
|
||||
if container:
|
||||
# 移除脚本和样式
|
||||
for tag in container.find_all(["script", "style"]):
|
||||
tag.decompose()
|
||||
text = container.get_text(separator="\n", strip=True)
|
||||
if len(text) > 50: # 至少50字才算有效正文
|
||||
return text
|
||||
|
||||
# 兜底:取 body 内所有文本
|
||||
body = soup.find("body")
|
||||
if body:
|
||||
for tag in body.find_all(["script", "style", "nav", "footer", "header"]):
|
||||
tag.decompose()
|
||||
text = body.get_text(separator="\n", strip=True)
|
||||
# 移除过长的空白行
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
text = "\n".join(lines[:200]) # 最多取前200行
|
||||
if len(text) > 50:
|
||||
return text
|
||||
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _generate_hash(ann: dict[str, Any]) -> str:
|
||||
content = (
|
||||
f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"
|
||||
|
||||
Reference in New Issue
Block a user