feat: 接入 DeepSeek AI 分析公告是否为中国电信可承接项目
- 新增 ai_enabled/ai_api_key/ai_base_url/ai_model 等配置项，通过 .env 管理 - 新增 app/services/ai_analyzer.py — DeepSeek API 调用 + 详情页正文提取 - 新增 extract_page_content() 从详情页抓取正文供 AI 分析 - Announcement 模型新增 ai_relevant / ai_analysis 字段 - 流水线集成 AI 分析步骤（关键词匹配后、通知前） - AI 标记为可承接的项目额外发送 markdown 着重通知 - 创建 alembic 迁移版本 6e8f4c2d1b0a
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.crawler.parsers import extract_page_content
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AiResult:
|
||||
"""DeepSeek 分析结果"""
|
||||
is_relevant: bool = False
|
||||
reason: str = ""
|
||||
business_type: str = ""
|
||||
error: str | None = None
|
||||
content_snippet: str | None = None # 提取到的正文前 200 字，供入库参考
|
||||
|
||||
|
||||
class AiAnalyzer:
|
||||
"""AI 分析器 — 调用 DeepSeek 判断公告是否为中国电信可承接项目"""
|
||||
|
||||
def __init__(self):
|
||||
self.api_key = settings.ai_api_key
|
||||
self.base_url = settings.ai_base_url.rstrip("/")
|
||||
self.model = settings.ai_model
|
||||
self.timeout = settings.ai_timeout
|
||||
self.prompt_template = settings.ai_prompt_template
|
||||
|
||||
async def analyze(self, announcement: dict[str, Any]) -> AiResult:
|
||||
"""分析单条公告"""
|
||||
if not self.api_key:
|
||||
return AiResult(error="AI_API_KEY 未配置")
|
||||
|
||||
# 1. 获取公告正文
|
||||
content_url = announcement.get("content_url", "")
|
||||
content = None
|
||||
content_snippet = None
|
||||
if content_url:
|
||||
content = await extract_page_content(content_url, self.timeout)
|
||||
|
||||
if content:
|
||||
content_snippet = content[:200]
|
||||
else:
|
||||
logger.warning("无法获取公告正文: %s", content_url)
|
||||
|
||||
# 2. 构建 prompt
|
||||
prompt = self.prompt_template.format(
|
||||
title=announcement.get("title", ""),
|
||||
purchase_name=announcement.get("purchase_name", ""),
|
||||
announcement_type=announcement.get("announcement_type", ""),
|
||||
content=content or "（无法获取正文，请仅根据标题和采购人信息判断）",
|
||||
)
|
||||
|
||||
# 3. 调用 DeepSeek API
|
||||
try:
|
||||
result = await self._call_deepseek(prompt)
|
||||
if result.error:
|
||||
return AiResult(error=result.error, content_snippet=content_snippet)
|
||||
return AiResult(
|
||||
is_relevant=result.is_relevant,
|
||||
reason=result.reason,
|
||||
business_type=result.business_type,
|
||||
content_snippet=content_snippet,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("AI 分析异常")
|
||||
return AiResult(error=str(e), content_snippet=content_snippet)
|
||||
|
||||
async def analyze_batch(
|
||||
self, announcements: list[dict[str, Any]], max_concurrent: int = 3
|
||||
) -> list[AiResult]:
|
||||
"""批量分析，控制并发数"""
|
||||
import asyncio
|
||||
|
||||
sem = asyncio.Semaphore(max_concurrent)
|
||||
|
||||
async def _limited(ann: dict[str, Any]) -> AiResult:
|
||||
async with sem:
|
||||
return await self.analyze(ann)
|
||||
|
||||
tasks = [_limited(ann) for ann in announcements]
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
async def _call_deepseek(self, prompt: str) -> AiResult:
|
||||
"""调用 DeepSeek Chat API"""
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "你是一个专业的政府采购项目分析师。请根据公告信息判断是否为中国电信可以承接的项目，并用 JSON 格式回答。",
|
||||
},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0.3, # 低温度，提高判断一致性
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.post(url, headers=headers, json=payload)
|
||||
if response.status_code != 200:
|
||||
return AiResult(
|
||||
error=f"API 请求失败 (HTTP {response.status_code}): {response.text[:200]}"
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
choices = data.get("choices", [])
|
||||
if not choices:
|
||||
return AiResult(error="API 返回空 choices")
|
||||
|
||||
content = choices[0].get("message", {}).get("content", "")
|
||||
return self._parse_response(content)
|
||||
|
||||
@staticmethod
|
||||
def _parse_response(content: str) -> AiResult:
|
||||
"""从 LLM 回复中提取 JSON 结果"""
|
||||
# 清理可能的 markdown 代码块标记
|
||||
content = content.strip()
|
||||
if content.startswith("```"):
|
||||
# 移除 ```json 或 ``` 包裹
|
||||
lines = content.split("\n")
|
||||
if lines[0].strip().startswith("```"):
|
||||
lines = lines[1:]
|
||||
if lines and lines[-1].strip() == "```":
|
||||
lines = lines[:-1]
|
||||
content = "\n".join(lines).strip()
|
||||
|
||||
try:
|
||||
result = json.loads(content)
|
||||
return AiResult(
|
||||
is_relevant=bool(result.get("is_relevant", False)),
|
||||
reason=str(result.get("reason", "")),
|
||||
business_type=str(result.get("business_type", "")),
|
||||
)
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
logger.warning("JSON 解析失败: %s\n原始内容: %s", e, content[:200])
|
||||
return AiResult(error=f"JSON 解析失败: {e}")
|
||||
@@ -17,6 +17,7 @@ class NotificationService:
|
||||
sent = 0
|
||||
for ann in announcements:
|
||||
try:
|
||||
# 发送普通 textcard
|
||||
title = ann.get("title", "")
|
||||
if len(title) > 128:
|
||||
title = title[:125] + "..."
|
||||
@@ -36,7 +37,42 @@ class NotificationService:
|
||||
|
||||
if await self.client.send_textcard(title, description, url):
|
||||
sent += 1
|
||||
|
||||
# AI 标记为可承接的，额外发送着重通知
|
||||
ai_result = ann.get("ai_result")
|
||||
if ai_result and ai_result.get("is_relevant"):
|
||||
await self._send_ai_emphasis(ann, ai_result)
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return sent
|
||||
|
||||
async def _send_ai_emphasis(
|
||||
self, ann: dict[str, Any], ai_result: dict[str, Any]
|
||||
) -> bool:
|
||||
"""发送 AI 分析的着重通知（markdown 格式）"""
|
||||
title = ann.get("title", "")
|
||||
purchase_name = ann.get("purchase_name", "")
|
||||
pub_date = ann.get("publish_date")
|
||||
time_str = pub_date.strftime("%Y-%m-%d %H:%M") if pub_date else "时间未知"
|
||||
url = ann.get("content_url", "")
|
||||
|
||||
reason = ai_result.get("reason", "")
|
||||
business_type = ai_result.get("business_type", "")
|
||||
|
||||
# 企业微信 markdown 格式
|
||||
md = (
|
||||
f"{settings.ai_analysis_title}\n"
|
||||
f"---\n"
|
||||
f"**标题：** [{title}]({url})\n"
|
||||
f"> 采购人：{purchase_name}\n"
|
||||
f"> 发布时间：{time_str}\n\n"
|
||||
f"**🤖 AI 分析：**\n"
|
||||
f"> {reason}\n\n"
|
||||
f"**🏷 业务分类：** {business_type}\n"
|
||||
f"---\n"
|
||||
f"[📄 查看公告原文]({url})"
|
||||
)
|
||||
|
||||
return await self.client.send_markdown(md)
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from app.config import settings
|
||||
from app.crawler.base import PipelineConfig, PipelineResult
|
||||
from app.services.filter_service import dedup_by_hash
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PostCrawlPipeline:
|
||||
def __init__(self, db_session, notification_service):
|
||||
@@ -41,6 +45,10 @@ class PostCrawlPipeline:
|
||||
if to_notify:
|
||||
to_notify = await self._exclude_sent(to_notify)
|
||||
|
||||
# 5.5 AI 分析（可选）
|
||||
if settings.ai_enabled and to_notify:
|
||||
await self._ai_analyze(to_notify)
|
||||
|
||||
# 6. Notify
|
||||
if config.notify_mode == "all":
|
||||
result.notified = await self._send_notifications(to_notify)
|
||||
@@ -129,3 +137,52 @@ class PostCrawlPipeline:
|
||||
def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool:
|
||||
text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}"
|
||||
return any(kw in text for kw in keywords)
|
||||
|
||||
async def _ai_analyze(self, announcements: list[dict[str, Any]]) -> None:
|
||||
"""对公告列表执行 AI 分析，将结果附加到每条公告的 ai_result 字段，并更新数据库"""
|
||||
from app.services.ai_analyzer import AiAnalyzer
|
||||
|
||||
analyzer = AiAnalyzer()
|
||||
logger.info("AI 分析开始：共 %d 条公告", len(announcements))
|
||||
|
||||
results = await analyzer.analyze_batch(announcements)
|
||||
|
||||
ai_updates = []
|
||||
for ann, ai_result in zip(announcements, results):
|
||||
ann["ai_result"] = {
|
||||
"is_relevant": ai_result.is_relevant,
|
||||
"reason": ai_result.reason,
|
||||
"business_type": ai_result.business_type,
|
||||
}
|
||||
if ai_result.error:
|
||||
logger.warning("AI 分析失败 [%s]: %s", ann.get("title", "")[:30], ai_result.error)
|
||||
else:
|
||||
ai_updates.append({
|
||||
"content_hash": ann["content_hash"],
|
||||
"ai_relevant": ai_result.is_relevant,
|
||||
"ai_analysis": ai_result.reason,
|
||||
})
|
||||
if ai_result.is_relevant:
|
||||
logger.info("AI 标记可承接项目: %s (%s)", ann.get("title", "")[:40], ai_result.business_type)
|
||||
|
||||
# 批量更新数据库中的 AI 分析结果
|
||||
if ai_updates:
|
||||
await self._update_ai_results(ai_updates)
|
||||
|
||||
async def _update_ai_results(self, updates: list[dict[str, Any]]) -> None:
|
||||
"""批量更新公告的 AI 分析结果到数据库"""
|
||||
from sqlalchemy import update
|
||||
|
||||
from app.models.announcement import Announcement
|
||||
|
||||
for u in updates:
|
||||
stmt = (
|
||||
update(Announcement)
|
||||
.where(Announcement.content_hash == u["content_hash"])
|
||||
.values(
|
||||
ai_relevant=u["ai_relevant"],
|
||||
ai_analysis=u["ai_analysis"],
|
||||
)
|
||||
)
|
||||
await self.db.execute(stmt)
|
||||
await self.db.commit()
|
||||
|
||||
Reference in New Issue
Block a user