Compare commits
20 Commits
c3e06c997f
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 18f415b363 | |||
| 103de16a8f | |||
| a51161b5f3 | |||
| 68215aa804 | |||
| 754214692e | |||
| 136941d84e | |||
| 4f16134955 | |||
| c01a3d427b | |||
| dc3ca1f76c | |||
| 6d1d512fca | |||
| b2d0503bc2 | |||
| 77c6722a92 | |||
| 048fca283d | |||
| 0121017272 | |||
| 24d844bf44 | |||
| 9f90e16661 | |||
| d84c838e76 | |||
| e1c9a985ca | |||
| 2996920277 | |||
| 1f18d2ec87 |
+21
-4
@@ -20,11 +20,28 @@ WECHAT_TOKEN=
|
||||
WECHAT_ENCODING_AES_KEY=
|
||||
WECHAT_PORT=18001
|
||||
WECHAT_HOST=0.0.0.0
|
||||
# 企微 API 代理(可选),用于绕过 IP 白名单限制
|
||||
# 留空则直连 https://qyapi.weixin.qq.com
|
||||
WECHAT_API_BASE_URL=https://qyapi.weixin.qq.com
|
||||
|
||||
# 定时任务
|
||||
SCHEDULER_ENABLED=true
|
||||
SCHEDULER_CRON=0 8,14,18 * * *
|
||||
SCHEDULER_CRON=0 8-21 * * *
|
||||
|
||||
# Markdown
|
||||
MARKDOWN_ENABLED=true
|
||||
MARKDOWN_OUTPUT_FILE=onu.md
|
||||
# LogHive 日志系统
|
||||
LOGHIVE_ENDPOINT=http://10.10.10.14:8000
|
||||
LOGHIVE_PROJECT=gx-gp-notify
|
||||
LOGHIVE_API_KEY=
|
||||
|
||||
# AI 分析 (DeepSeek)
|
||||
AI_ENABLED=false
|
||||
# AI 管理白名单(企微用户ID,逗号分隔),留空表示所有人可操作
|
||||
AI_WHITELIST=
|
||||
AI_API_KEY=sk-your-deepseek-api-key
|
||||
AI_BASE_URL=https://api.deepseek.com/v1
|
||||
AI_MODEL=deepseek-chat
|
||||
AI_TIMEOUT=30
|
||||
AI_ANALYSIS_TITLE=🔔 中国电信可承接项目
|
||||
# 自定义分析提示词(可选),不设置则用代码内置默认值
|
||||
# 使用 \n 表示换行,支持 {title} {purchase_name} {announcement_type} {content} 四个占位符
|
||||
# AI_PROMPT_TEMPLATE=你是一个政府采购项目分析师...\n\n--- 公告信息 ---\n标题:{title}\n...
|
||||
|
||||
@@ -12,3 +12,7 @@ dist/
|
||||
config.yaml
|
||||
gx_gp_monitor/config/config.yaml
|
||||
onu.md
|
||||
vendor/build/
|
||||
vendor/*.egg-info/
|
||||
# Added by code-review-graph
|
||||
.code-review-graph/
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# GX-GP-Notify
|
||||
|
||||
广西政府采购公告监控系统,自动爬取广西政府采购网和大化县政府网的公告,通过企业微信推送关键词匹配的公告。
|
||||
|
||||
## 功能
|
||||
|
||||
- **定时爬取**:按 cron 规则自动爬取,工作日 8:00-21:00 每小时执行
|
||||
- **关键词过滤**:广西政采网公告按关键词过滤(默认:`大化`),大化县政府网全量爬取
|
||||
- **企微推送**:新公告实时推送到企业微信应用
|
||||
- **去重**:基于内容 hash 去重,避免重复入库和推送
|
||||
- **节假日感知**:非工作日自动跳过爬取
|
||||
|
||||
## 数据来源
|
||||
|
||||
| 来源 | 说明 | 过滤 |
|
||||
|------|------|------|
|
||||
| 广西政府采购网 | 采购公告、结果公告、合同公告等 13 个分类 | 关键词过滤 |
|
||||
| 大化县政府网 | 大化县门户网站采购公告 | 全量爬取 |
|
||||
|
||||
## 企微菜单功能
|
||||
|
||||
| 菜单 | 说明 |
|
||||
|------|------|
|
||||
| 最新公告 | 触发一次爬取,推送两个来源合并后最新 6 条(时间升序) |
|
||||
| 手动爬取 | 立即触发全量爬取并推送新公告 |
|
||||
| 今日统计 | 查看今日新增和累计公告数 |
|
||||
| 系统状态 | 查看待推送数量、定时任务状态 |
|
||||
| 监控配置 | 查看关键词、爬取页数、定时规则 |
|
||||
| 工作日状态 | 查看今天是否为工作日 |
|
||||
| 同步节假日 | 同步当年节假日数据 |
|
||||
| 暂停/恢复任务 | 控制定时爬取任务 |
|
||||
|
||||
## 快速部署
|
||||
|
||||
### 1. 配置环境变量
|
||||
|
||||
复制 `.env.example` 为 `.env` 并填写:
|
||||
|
||||
```env
|
||||
# 数据库
|
||||
DATABASE_URL=postgresql+asyncpg://user:password@host:5432/gx-gp-notify
|
||||
|
||||
# 企业微信
|
||||
WECHAT_CORP_ID=your_corp_id
|
||||
WECHAT_AGENT_ID=your_agent_id
|
||||
WECHAT_SECRET=your_secret
|
||||
WECHAT_TOKEN=your_token
|
||||
WECHAT_ENCODING_AES_KEY=your_aes_key
|
||||
|
||||
# 爬虫
|
||||
CRAWLER_KEYWORDS=["大化"]
|
||||
CRAWLER_MAX_PAGES=10
|
||||
SCHEDULER_CRON=0 8-21 * * 1-5
|
||||
```
|
||||
|
||||
### 2. 启动服务
|
||||
|
||||
```bash
|
||||
docker compose -f docker/docker-compose.yml up -d
|
||||
```
|
||||
|
||||
### 3. 数据库迁移
|
||||
|
||||
```bash
|
||||
docker compose -f docker/docker-compose.yml exec app alembic upgrade head
|
||||
```
|
||||
|
||||
### 4. 配置企微回调
|
||||
|
||||
在企业微信后台将回调 URL 设置为:
|
||||
|
||||
```
|
||||
http://your-server:18001/wechat/callback
|
||||
```
|
||||
|
||||
## 开发
|
||||
|
||||
```bash
|
||||
# 安装依赖
|
||||
pip install -e ".[dev]"
|
||||
|
||||
# 运行测试
|
||||
pytest
|
||||
|
||||
# 代码检查
|
||||
ruff check .
|
||||
```
|
||||
|
||||
## 技术栈
|
||||
|
||||
- **框架**:FastAPI + uvicorn
|
||||
- **数据库**:PostgreSQL + SQLAlchemy (async) + Alembic
|
||||
- **爬虫**:httpx + BeautifulSoup4
|
||||
- **定时任务**:APScheduler
|
||||
- **推送**:企业微信应用消息 API
|
||||
- **部署**:Docker
|
||||
|
||||
## 项目结构
|
||||
|
||||
```
|
||||
app/
|
||||
├── api/ # HTTP 接口(公告查询、手动触发爬取等)
|
||||
├── crawler/ # 爬虫(广西政采网、大化县政府网)
|
||||
├── models/ # 数据库模型
|
||||
├── scheduler/ # 定时任务
|
||||
├── services/ # 业务逻辑(pipeline、过滤、通知)
|
||||
└── wechat/ # 企业微信(消息处理、推送客户端)
|
||||
```
|
||||
@@ -2,6 +2,7 @@ import asyncio
|
||||
from alembic import context
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from app.models.announcement import Base
|
||||
from app.models.holiday import ChineseHoliday # noqa: F401
|
||||
from app.config import settings
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""add_chinese_holidays_table
|
||||
|
||||
Revision ID: 1f59799a5083
|
||||
Revises: eed20ee8cc26
|
||||
Create Date: 2026-05-09 18:14:31.144256
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '1f59799a5083'
|
||||
down_revision: Union[str, Sequence[str], None] = 'eed20ee8cc26'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
op.create_table(
|
||||
'chinese_holidays',
|
||||
sa.Column('date', sa.Date(), nullable=False),
|
||||
sa.Column('is_workday', sa.Boolean(), nullable=False, server_default='true'),
|
||||
sa.Column('year', sa.Integer(), nullable=False),
|
||||
sa.Column('description', sa.String(length=100), nullable=False, server_default=''),
|
||||
sa.PrimaryKeyConstraint('date'),
|
||||
)
|
||||
op.create_index('idx_chinese_holidays_year', 'chinese_holidays', ['year'])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
op.drop_index('idx_chinese_holidays_year', table_name='chinese_holidays')
|
||||
op.drop_table('chinese_holidays')
|
||||
@@ -0,0 +1,30 @@
|
||||
"""add_ai_analysis_columns
|
||||
|
||||
Revision ID: 6e8f4c2d1b0a
|
||||
Revises: 1f59799a5083
|
||||
Create Date: 2026-05-21 10:00:00.000000
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '6e8f4c2d1b0a'
|
||||
down_revision: Union[str, Sequence[str], None] = '1f59799a5083'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""添加 AI 分析相关字段"""
|
||||
op.add_column('announcements', sa.Column('ai_relevant', sa.Boolean(), nullable=True))
|
||||
op.add_column('announcements', sa.Column('ai_analysis', sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""回滚"""
|
||||
op.drop_column('announcements', 'ai_analysis')
|
||||
op.drop_column('announcements', 'ai_relevant')
|
||||
+21
-7
@@ -9,7 +9,10 @@ router = APIRouter()
|
||||
@router.post("/crawl/trigger")
|
||||
async def trigger_crawl(request: CrawlTriggerRequest):
|
||||
service = get_crawl_service()
|
||||
names = service.get_spider_names()
|
||||
if request.spider_name:
|
||||
names = [request.spider_name]
|
||||
else:
|
||||
names = service.get_spider_names()
|
||||
|
||||
all_results = []
|
||||
total_stored = 0
|
||||
@@ -46,9 +49,20 @@ async def crawl_sources():
|
||||
|
||||
from app.config import settings
|
||||
sources = json.loads(settings.announcement_sources)
|
||||
return {
|
||||
"sources": [
|
||||
{"code": code, "name": info["name"], "type": info["type"]}
|
||||
for code, info in sources.items()
|
||||
]
|
||||
}
|
||||
result = [
|
||||
{"code": code, "name": info["name"], "type": info["type"]}
|
||||
for code, info in sources.items()
|
||||
]
|
||||
# 加入独立爬虫来源(非 GXGP 子来源的独立 Spider)
|
||||
service = get_crawl_service()
|
||||
for name in service.get_spider_names():
|
||||
if name == "gxgp":
|
||||
continue # gxgp 的子来源已在上面列出
|
||||
spider = service.spiders.get(name)
|
||||
if spider:
|
||||
result.append({
|
||||
"code": spider.source_code,
|
||||
"name": spider.source_name,
|
||||
"type": "independent",
|
||||
})
|
||||
return {"sources": result}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.deps import get_db
|
||||
from app.services.holiday_service import now_in_china, sync_holidays
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/holidays/sync")
|
||||
async def sync_holidays_endpoint(
|
||||
year: int | None = Query(None, description="同步年份,默认当前年份"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
if year is None:
|
||||
year = now_in_china().year
|
||||
count = await sync_holidays(db, year)
|
||||
return {"status": "ok", "year": year, "synced": count}
|
||||
|
||||
|
||||
@router.get("/holidays/today")
|
||||
async def get_today_status(db: AsyncSession = Depends(get_db)):
|
||||
from app.services.holiday_service import is_workday
|
||||
today = now_in_china()
|
||||
workday = await is_workday(db, today)
|
||||
return {
|
||||
"date": today.isoformat(),
|
||||
"is_workday": workday,
|
||||
"message": "工作日,正常爬取" if workday else "非工作日,跳过爬取",
|
||||
}
|
||||
+2
-1
@@ -1,10 +1,11 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api import announcements, crawl, wechat
|
||||
from app.api import announcements, crawl, holidays, wechat
|
||||
from app.api import scheduler as scheduler_module
|
||||
|
||||
api_router = APIRouter(prefix="/api/v1")
|
||||
api_router.include_router(announcements.router, tags=["announcements"])
|
||||
api_router.include_router(crawl.router, tags=["crawl"])
|
||||
api_router.include_router(holidays.router, tags=["holidays"])
|
||||
api_router.include_router(wechat.router, tags=["wechat"])
|
||||
api_router.include_router(scheduler_module.router)
|
||||
|
||||
+2
-2
@@ -52,7 +52,7 @@ async def wechat_callback(request: Request):
|
||||
event = xml_tree.find("Event")
|
||||
event_key = xml_tree.find("EventKey")
|
||||
from_user = xml_tree.find("FromUserName")
|
||||
handler.handle_event(
|
||||
await handler.handle_event(
|
||||
event.text if event is not None else "",
|
||||
event_key.text if event_key is not None else None,
|
||||
from_user.text if from_user is not None else "",
|
||||
@@ -60,7 +60,7 @@ async def wechat_callback(request: Request):
|
||||
elif msg_type == "text":
|
||||
content = xml_tree.find("Content")
|
||||
from_user = xml_tree.find("FromUserName")
|
||||
handler.handle_text(
|
||||
await handler.handle_text(
|
||||
content.text if content is not None else "",
|
||||
from_user.text if from_user is not None else "",
|
||||
)
|
||||
|
||||
+43
-4
@@ -1,5 +1,6 @@
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from pydantic import field_validator
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
@@ -28,14 +29,52 @@ class Settings(BaseSettings):
|
||||
wechat_encoding_aes_key: str = ""
|
||||
wechat_port: int = 18001
|
||||
wechat_host: str = "0.0.0.0"
|
||||
wechat_api_base_url: str = "https://qyapi.weixin.qq.com"
|
||||
|
||||
# 定时任务
|
||||
scheduler_enabled: bool = True
|
||||
scheduler_cron: str = "0 8,14,18 * * *"
|
||||
scheduler_cron: str = "0 8-21 * * *"
|
||||
|
||||
# Markdown
|
||||
markdown_enabled: bool = True
|
||||
markdown_output_file: str = "onu.md"
|
||||
# LogHive 日志系统
|
||||
loghive_endpoint: str = "http://10.10.10.14:8000"
|
||||
loghive_project: str = "gx-gp-notify"
|
||||
loghive_api_key: str = ""
|
||||
|
||||
# AI 分析 (DeepSeek)
|
||||
ai_enabled: bool = False
|
||||
ai_whitelist: str = ""
|
||||
ai_api_key: str = ""
|
||||
ai_base_url: str = "https://api.deepseek.com/v1"
|
||||
ai_model: str = "deepseek-chat"
|
||||
ai_timeout: int = 30
|
||||
ai_analysis_title: str = "🔔 中国电信可承接项目"
|
||||
ai_prompt_template: str = (
|
||||
"你是一个政府采购项目分析师,专门帮助中国电信识别可以承接的项目。\n\n"
|
||||
"可承接范围包括但不限于:\n"
|
||||
"- 通信工程、光缆建设、基站建设\n"
|
||||
"- 信息化系统建设、系统集成\n"
|
||||
"- 云计算、大数据、政务云\n"
|
||||
"- 物联网、智慧城市、智慧园区\n"
|
||||
"- 安防监控、视频会议、应急通信\n"
|
||||
"- 网络运维、网络优化、IDC 服务\n"
|
||||
"- 5G 应用、专线服务\n\n"
|
||||
"--- 公告信息 ---\n"
|
||||
"标题:{title}\n"
|
||||
"采购人:{purchase_name}\n"
|
||||
"公告类型:{announcement_type}\n\n"
|
||||
"--- 公告正文 ---\n"
|
||||
"{content}\n\n"
|
||||
"请用 JSON 格式回答:\n"
|
||||
'{{"is_relevant": true/false, "reason": "简要判断理由", "business_type": "业务分类"}}'
|
||||
)
|
||||
|
||||
@field_validator("ai_prompt_template", mode="before")
|
||||
@classmethod
|
||||
def convert_newlines(cls, v: str) -> str:
|
||||
"""将 .env 中字面 \\n 转换为真实换行"""
|
||||
if isinstance(v, str) and "\\n" in v:
|
||||
return v.replace("\\n", "\n")
|
||||
return v
|
||||
|
||||
# 公告来源(JSON 字符串,从环境变量读取)
|
||||
announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}' # noqa: E501
|
||||
|
||||
@@ -9,7 +9,6 @@ class PipelineResult:
|
||||
stored: int = 0
|
||||
filtered: int = 0
|
||||
notified: int = 0
|
||||
markdown_generated: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -6,7 +6,7 @@ import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||
from app.crawler.parsers import parse_dahuagov_html
|
||||
from app.crawler.parsers import parse_dahuagov_html, parse_dahuagov_detail_pubdate
|
||||
|
||||
|
||||
class DahuagovSpider(BaseSpider):
|
||||
@@ -56,7 +56,9 @@ class DahuagovSpider(BaseSpider):
|
||||
crawled_at=start_time,
|
||||
)
|
||||
|
||||
announcements = parse_dahuagov_html(html, start_time)
|
||||
announcements = parse_dahuagov_html(html, start_time)
|
||||
await self._fetch_detail_dates(client, announcements, headers)
|
||||
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
|
||||
return CrawlResult(
|
||||
@@ -69,6 +71,31 @@ class DahuagovSpider(BaseSpider):
|
||||
duration=duration,
|
||||
)
|
||||
|
||||
async def _fetch_detail_dates(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
announcements: list[dict],
|
||||
headers: dict,
|
||||
) -> None:
|
||||
"""并发抓取详情页,用 PubDate meta 更新精确发布时间"""
|
||||
sem = asyncio.Semaphore(5)
|
||||
|
||||
async def fetch_one(ann: dict) -> None:
|
||||
async with sem:
|
||||
await asyncio.sleep(random.uniform(0.3, 0.8))
|
||||
try:
|
||||
resp = await client.get(ann["content_url"], headers=headers)
|
||||
if resp.status_code == 200:
|
||||
pub = parse_dahuagov_detail_pubdate(resp.text)
|
||||
if pub:
|
||||
ann["publish_date"] = pub
|
||||
ann["is_today"] = pub.date() == datetime.now().date()
|
||||
except Exception:
|
||||
pass # 保留列表页日期作为 fallback
|
||||
|
||||
await asyncio.gather(*[fetch_one(ann) for ann in announcements])
|
||||
|
||||
async def _delay(self):
|
||||
delay = random.uniform(1.0, 3.0)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
|
||||
+186
-1
@@ -2,6 +2,9 @@ import hashlib
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
_TZ = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
@@ -30,7 +33,7 @@ def parse_gxgp_api_response(
|
||||
if not timestamp:
|
||||
continue
|
||||
try:
|
||||
publish_date = datetime.fromtimestamp(int(timestamp) / 1000)
|
||||
publish_date = datetime.fromtimestamp(int(timestamp) / 1000, tz=_TZ).replace(tzinfo=None)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
@@ -131,6 +134,188 @@ def parse_dahuagov_html(html: str, crawled_at: datetime) -> list[dict[str, Any]]
|
||||
return results
|
||||
|
||||
|
||||
def parse_dahuagov_detail_pubdate(html: str) -> datetime | None:
|
||||
"""从详情页 <meta name="PubDate"> 解析精确发布时间"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
meta = soup.find("meta", attrs={"name": "PubDate"})
|
||||
if not meta:
|
||||
return None
|
||||
content = meta.get("content", "").strip()
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(content, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
async def extract_page_content(url: str, timeout: int = 30) -> str | None:
|
||||
"""抓取详情页并用 BeautifulSoup 提取正文纯文本"""
|
||||
import urllib.parse
|
||||
|
||||
import httpx
|
||||
|
||||
# ---- 站点专用处理 ----
|
||||
|
||||
# ① 政采云 SPA (HTTP API, 无需渲染)
|
||||
if "zfcg.gxzf.gov.cn" in url and "articleId=" in url:
|
||||
return await _extract_zcy_content(url, timeout)
|
||||
|
||||
# ② 大化县政府网 (静态HTML)
|
||||
if "www.gxdh.gov.cn" in url or "gxdh.gov.cn" in url:
|
||||
return await _extract_dahuagov_content(url, timeout)
|
||||
|
||||
# ---- 通用 HTML 提取 ----
|
||||
try:
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=headers)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
|
||||
# 尝试多种常见正文容器选择器
|
||||
selectors = [
|
||||
"div.article-content", "div.content", "div.TRS_Editor",
|
||||
"div.Custom_UnionStyle", "div.pages_content", "div#content",
|
||||
"div.main-content", "article", ".article", ".detail-content",
|
||||
".text-content", ".news-content", ".detail-article",
|
||||
"div.article-con", ".trs_editor_view", ".TRS_UEDITOR",
|
||||
".trs_paper_default", ".article-content",
|
||||
]
|
||||
for selector in selectors:
|
||||
container = soup.select_one(selector)
|
||||
if container:
|
||||
for tag in container.find_all(["script", "style"]):
|
||||
tag.decompose()
|
||||
text = container.get_text(separator="\n", strip=True)
|
||||
if len(text) > 50:
|
||||
return text
|
||||
|
||||
# 兜底:取 body 内所有文本
|
||||
body = soup.find("body")
|
||||
if body:
|
||||
for tag in body.find_all(["script", "style", "nav", "footer", "header"]):
|
||||
tag.decompose()
|
||||
text = body.get_text(separator="\n", strip=True)
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
text = "\n".join(lines[:200])
|
||||
if len(text) > 50:
|
||||
return text
|
||||
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def _extract_zcy_content(url: str, timeout: int = 30) -> str | None:
|
||||
"""从政采云 SPA 隐藏 API 提取公告正文"""
|
||||
import urllib.parse
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
params = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
||||
article_id = params.get("articleId", [None])[0]
|
||||
parent_id = params.get("parentId", [None])[0]
|
||||
if not article_id:
|
||||
return None
|
||||
|
||||
api_url = "https://zfcg.gxzf.gov.cn/portal/detail"
|
||||
if parent_id:
|
||||
api_url += f"?articleId={article_id}&parentId={parent_id}"
|
||||
else:
|
||||
api_url += f"?articleId={article_id}"
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Referer": url,
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
resp = await client.get(api_url, headers=headers)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
|
||||
data = resp.json()
|
||||
if not data.get("success"):
|
||||
return None
|
||||
|
||||
content_html = data.get("result", {}).get("data", {}).get("content", "")
|
||||
if not content_html:
|
||||
return None
|
||||
|
||||
soup = BeautifulSoup(content_html, "html.parser")
|
||||
for tag in soup.find_all(["script", "style"]):
|
||||
tag.decompose()
|
||||
text = soup.get_text(separator="\n", strip=True)
|
||||
|
||||
# 清理过短行和多余空白
|
||||
lines = [l.strip() for l in text.split("\n") if len(l.strip()) > 5]
|
||||
text = "\n".join(lines[:300])
|
||||
return text if len(text) > 50 else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def _extract_dahuagov_content(url: str, timeout: int = 30) -> str | None:
|
||||
"""从大化县政府网详情页提取正文"""
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=headers)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
|
||||
# 大化县政府网正文容器
|
||||
selectors = [
|
||||
"div.article-con", ".trs_editor_view", ".TRS_UEDITOR",
|
||||
".trs_paper_default", "div.content", "div.TRS_Editor",
|
||||
"div.article-content", "div.Custom_UnionStyle",
|
||||
"div#content", "div.main-content", "article", ".detail-content",
|
||||
]
|
||||
for selector in selectors:
|
||||
container = soup.select_one(selector)
|
||||
if container:
|
||||
for tag in container.find_all(["script", "style"]):
|
||||
tag.decompose()
|
||||
text = container.get_text(separator="\n", strip=True)
|
||||
if len(text) > 50:
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
return "\n".join(lines[:300])
|
||||
|
||||
# 兜底
|
||||
body = soup.find("body")
|
||||
if body:
|
||||
for tag in body.find_all(["script", "style", "nav", "footer", "header"]):
|
||||
tag.decompose()
|
||||
text = body.get_text(separator="\n", strip=True)
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
text = "\n".join(lines[:200])
|
||||
if len(text) > 50:
|
||||
return text
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _generate_hash(ann: dict[str, Any]) -> str:
|
||||
content = (
|
||||
f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"
|
||||
|
||||
+11
@@ -22,6 +22,17 @@ async def lifespan(app: FastAPI):
|
||||
level=getattr(logging, settings.log_level),
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
)
|
||||
# LogHive 远程日志(仅当配置了 API Key 时启用)
|
||||
if settings.loghive_api_key:
|
||||
from loghive_client import LogHiveHandler
|
||||
handler = LogHiveHandler(
|
||||
project=settings.loghive_project,
|
||||
api_key=settings.loghive_api_key,
|
||||
endpoint=settings.loghive_endpoint,
|
||||
level=logging.INFO,
|
||||
)
|
||||
logging.getLogger().addHandler(handler)
|
||||
logging.info("LogHive 日志系统已连接")
|
||||
from app.scheduler.jobs import shutdown_scheduler, start_scheduler
|
||||
start_scheduler()
|
||||
yield
|
||||
|
||||
@@ -25,6 +25,8 @@ class Announcement(Base):
|
||||
is_new: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
is_sent: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
keyword_matched: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
ai_relevant: Mapped[bool | None] = mapped_column(Boolean, nullable=True, default=None)
|
||||
ai_analysis: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now()) # noqa: E501
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy import Boolean, Date, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.announcement import Base
|
||||
|
||||
|
||||
class ChineseHoliday(Base):
|
||||
__tablename__ = "chinese_holidays"
|
||||
|
||||
date: Mapped[date] = mapped_column(Date, primary_key=True)
|
||||
is_workday: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
year: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
|
||||
description: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
||||
@@ -29,6 +29,7 @@ class AnnouncementListResponse(BaseModel):
|
||||
|
||||
|
||||
class CrawlTriggerRequest(BaseModel):
|
||||
spider_name: str | None = None
|
||||
keywords: list[str] | None = None
|
||||
sources: list[str] | None = None
|
||||
manual: bool = False
|
||||
|
||||
+21
-8
@@ -1,30 +1,43 @@
|
||||
import logging
|
||||
from datetime import datetime, time
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from app.api.deps import get_crawl_service
|
||||
from app.api.deps import get_db, get_crawl_service
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
scheduler = AsyncIOScheduler()
|
||||
|
||||
NIGHT_START = time(22, 0)
|
||||
NIGHT_END = time(6, 0)
|
||||
NIGHT_START = 22 # 22:00
|
||||
NIGHT_END = 8 # 08:00
|
||||
TZ = ZoneInfo("Asia/Shanghai")
|
||||
|
||||
|
||||
def _is_night_time() -> bool:
|
||||
"""22:00 ~ 次日 06:00 夜间时段"""
|
||||
current = datetime.now(TZ).time()
|
||||
"""22:00 ~ 次日 08:00 夜间时段"""
|
||||
current = datetime.now(TZ).hour
|
||||
return current >= NIGHT_START or current < NIGHT_END
|
||||
|
||||
|
||||
async def scheduled_crawl():
|
||||
async def _should_skip() -> bool:
|
||||
"""检查是否应该跳过爬取"""
|
||||
if _is_night_time():
|
||||
logger.info("夜间时段 (22:00-06:00),跳过爬取")
|
||||
logger.info("夜间时段 (22:00-08:00),跳过爬取")
|
||||
return True
|
||||
|
||||
from app.services.holiday_service import is_workday
|
||||
async for db in get_db():
|
||||
if not await is_workday(db):
|
||||
logger.info("非工作日,跳过爬取")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def scheduled_crawl():
|
||||
if await _should_skip():
|
||||
return
|
||||
|
||||
logger.info("开始定时爬取任务")
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.crawler.parsers import extract_page_content
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AiResult:
|
||||
"""DeepSeek 分析结果"""
|
||||
is_relevant: bool = False
|
||||
reason: str = ""
|
||||
business_type: str = ""
|
||||
error: str | None = None
|
||||
content_snippet: str | None = None # 提取到的正文前 200 字,供入库参考
|
||||
|
||||
|
||||
class AiAnalyzer:
|
||||
"""AI 分析器 — 调用 DeepSeek 判断公告是否为中国电信可承接项目"""
|
||||
|
||||
def __init__(self):
|
||||
self.api_key = settings.ai_api_key
|
||||
self.base_url = settings.ai_base_url.rstrip("/")
|
||||
self.model = settings.ai_model
|
||||
self.timeout = settings.ai_timeout
|
||||
self.prompt_template = settings.ai_prompt_template
|
||||
|
||||
async def analyze(self, announcement: dict[str, Any]) -> AiResult:
|
||||
"""分析单条公告"""
|
||||
if not self.api_key:
|
||||
return AiResult(error="AI_API_KEY 未配置")
|
||||
|
||||
# 1. 获取公告正文
|
||||
content_url = announcement.get("content_url", "")
|
||||
content = None
|
||||
content_snippet = None
|
||||
if content_url:
|
||||
content = await extract_page_content(content_url, self.timeout)
|
||||
|
||||
if content:
|
||||
content_snippet = content[:200]
|
||||
else:
|
||||
logger.warning("无法获取公告正文: %s", content_url)
|
||||
|
||||
# 2. 构建 prompt
|
||||
prompt = self.prompt_template.format(
|
||||
title=announcement.get("title", ""),
|
||||
purchase_name=announcement.get("purchase_name", ""),
|
||||
announcement_type=announcement.get("announcement_type", ""),
|
||||
content=content or "(无法获取正文,请仅根据标题和采购人信息判断)",
|
||||
)
|
||||
|
||||
# 3. 调用 DeepSeek API
|
||||
try:
|
||||
result = await self._call_deepseek(prompt)
|
||||
if result.error:
|
||||
return AiResult(error=result.error, content_snippet=content_snippet)
|
||||
return AiResult(
|
||||
is_relevant=result.is_relevant,
|
||||
reason=result.reason,
|
||||
business_type=result.business_type,
|
||||
content_snippet=content_snippet,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("AI 分析异常")
|
||||
return AiResult(error=str(e), content_snippet=content_snippet)
|
||||
|
||||
async def analyze_batch(
|
||||
self, announcements: list[dict[str, Any]], max_concurrent: int = 3
|
||||
) -> list[AiResult]:
|
||||
"""批量分析,控制并发数"""
|
||||
import asyncio
|
||||
|
||||
sem = asyncio.Semaphore(max_concurrent)
|
||||
|
||||
async def _limited(ann: dict[str, Any]) -> AiResult:
|
||||
async with sem:
|
||||
return await self.analyze(ann)
|
||||
|
||||
tasks = [_limited(ann) for ann in announcements]
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
async def _call_deepseek(self, prompt: str) -> AiResult:
|
||||
"""调用 DeepSeek Chat API"""
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "你是一个专业的政府采购项目分析师。请根据公告信息判断是否为中国电信可以承接的项目,并用 JSON 格式回答。",
|
||||
},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0.3, # 低温度,提高判断一致性
|
||||
"max_tokens": 512,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.post(url, headers=headers, json=payload)
|
||||
if response.status_code != 200:
|
||||
return AiResult(
|
||||
error=f"API 请求失败 (HTTP {response.status_code}): {response.text[:200]}"
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
choices = data.get("choices", [])
|
||||
if not choices:
|
||||
return AiResult(error="API 返回空 choices")
|
||||
|
||||
content = choices[0].get("message", {}).get("content", "")
|
||||
return self._parse_response(content)
|
||||
|
||||
@staticmethod
|
||||
def _parse_response(content: str) -> AiResult:
|
||||
"""从 LLM 回复中提取 JSON 结果"""
|
||||
# 清理可能的 markdown 代码块标记
|
||||
content = content.strip()
|
||||
if content.startswith("```"):
|
||||
# 移除 ```json 或 ``` 包裹
|
||||
lines = content.split("\n")
|
||||
if lines[0].strip().startswith("```"):
|
||||
lines = lines[1:]
|
||||
if lines and lines[-1].strip() == "```":
|
||||
lines = lines[:-1]
|
||||
content = "\n".join(lines).strip()
|
||||
|
||||
try:
|
||||
result = json.loads(content)
|
||||
return AiResult(
|
||||
is_relevant=bool(result.get("is_relevant", False)),
|
||||
reason=str(result.get("reason", "")),
|
||||
business_type=str(result.get("business_type", "")),
|
||||
)
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
logger.warning("JSON 解析失败: %s\n原始内容: %s", e, content[:200])
|
||||
return AiResult(error=f"JSON 解析失败: {e}")
|
||||
@@ -0,0 +1,51 @@
|
||||
"""AI 运行时状态 — 支持企微菜单动态开关"""
|
||||
|
||||
import logging
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 运行时覆盖值,None 表示使用 settings.ai_enabled
|
||||
_runtime_override: bool | None = None
|
||||
|
||||
|
||||
def is_ai_enabled() -> bool:
|
||||
"""获取 AI 分析当前是否启用(考虑运行时覆盖)"""
|
||||
if _runtime_override is not None:
|
||||
return _runtime_override
|
||||
return settings.ai_enabled
|
||||
|
||||
|
||||
def set_ai_enabled(enabled: bool) -> bool:
|
||||
"""设置 AI 分析运行时开关,返回是否真的发生了变化"""
|
||||
global _runtime_override
|
||||
current = is_ai_enabled()
|
||||
if enabled == current:
|
||||
return False
|
||||
_runtime_override = enabled
|
||||
status = "启用" if enabled else "禁用"
|
||||
logger.info("AI 分析已通过企微菜单%s", status)
|
||||
return True
|
||||
|
||||
|
||||
def get_ai_status_text() -> str:
|
||||
"""获取 AI 状态文本"""
|
||||
return "已启用" if is_ai_enabled() else "已禁用"
|
||||
|
||||
|
||||
def get_whitelist() -> list[str]:
|
||||
"""获取 AI 白名单用户列表"""
|
||||
raw = settings.ai_whitelist
|
||||
if not raw:
|
||||
return []
|
||||
return [u.strip() for u in raw.split(",") if u.strip()]
|
||||
|
||||
|
||||
def is_whitelisted(user_id: str) -> bool:
|
||||
"""检查用户是否在 AI 白名单中(不区分大小写)"""
|
||||
whitelist = get_whitelist()
|
||||
if not whitelist:
|
||||
# 白名单为空则所有人都可以操作
|
||||
return True
|
||||
return user_id.lower() in [u.lower() for u in whitelist]
|
||||
@@ -0,0 +1,101 @@
|
||||
import logging
|
||||
from datetime import date
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.holiday import ChineseHoliday
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HOLIDAY_API = "http://timor.tech/api/holiday/year"
|
||||
|
||||
|
||||
def now_in_china() -> date:
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
return datetime.now(ZoneInfo("Asia/Shanghai")).date()
|
||||
|
||||
|
||||
async def sync_holidays(db: AsyncSession, year: int) -> int:
|
||||
"""同步中国节假日数据,返回更新的记录数"""
|
||||
url = f"{HOLIDAY_API}/{year}"
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
response = await client.get(url)
|
||||
data = response.json()
|
||||
|
||||
if data.get("code") != 0:
|
||||
logger.error(f"节假日 API 返回错误: {data}")
|
||||
return 0
|
||||
|
||||
holidays = data.get("holiday", {})
|
||||
if not holidays:
|
||||
return 0
|
||||
|
||||
# 也标记周末(周六日但非调休工作日)
|
||||
from datetime import timedelta
|
||||
current = date(year, 1, 1)
|
||||
end = date(year, 12, 31)
|
||||
|
||||
records: dict[date, dict] = {}
|
||||
while current <= end:
|
||||
dow = current.weekday() # 0=Mon, 6=Sun
|
||||
# 默认:周一~五为工作日,周六日为非工作日
|
||||
default_workday = dow < 5
|
||||
records[current] = {
|
||||
"date": current,
|
||||
"is_workday": default_workday,
|
||||
"year": year,
|
||||
"description": "",
|
||||
}
|
||||
current += timedelta(days=1)
|
||||
|
||||
# 覆盖节假日数据
|
||||
# holiday=true → 休息日(无论 wage 值)
|
||||
# holiday=false → 调休补班日(周末也要上班)
|
||||
for date_str, info_str in holidays.items():
|
||||
d = date.fromisoformat(f"{year}-{date_str}")
|
||||
if d.year != year:
|
||||
continue
|
||||
info = info_str if isinstance(info_str, dict) else {}
|
||||
holiday = info.get("holiday", False)
|
||||
name = info.get("name", "")
|
||||
|
||||
records[d]["description"] = name
|
||||
records[d]["is_workday"] = not holiday # holiday=false → 调休工作日
|
||||
|
||||
# Upsert
|
||||
values = list(records.values())
|
||||
stmt = pg_insert(ChineseHoliday).values(values)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["date"],
|
||||
set_={"is_workday": stmt.excluded.is_workday,
|
||||
"description": stmt.excluded.description},
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
await db.commit()
|
||||
logger.info(f"已同步 {year} 年节假日,{len(values)} 天")
|
||||
return result.rowcount
|
||||
|
||||
|
||||
async def is_workday(db: AsyncSession, day: date | None = None) -> bool:
|
||||
"""判断某天是否为工作日"""
|
||||
if day is None:
|
||||
day = now_in_china()
|
||||
from sqlalchemy import select
|
||||
result = await db.execute(
|
||||
select(ChineseHoliday.is_workday).where(ChineseHoliday.date == day)
|
||||
)
|
||||
row = result.fetchone()
|
||||
if row is None:
|
||||
# 无数据时,按周判断
|
||||
return day.weekday() < 5
|
||||
return row[0]
|
||||
|
||||
|
||||
async def was_yesterday_workday(db: AsyncSession) -> bool:
|
||||
"""昨天是工作日吗"""
|
||||
from datetime import timedelta
|
||||
yesterday = now_in_china() - timedelta(days=1)
|
||||
return await is_workday(db, yesterday)
|
||||
@@ -17,6 +17,7 @@ class NotificationService:
|
||||
sent = 0
|
||||
for ann in announcements:
|
||||
try:
|
||||
# 发送普通 textcard
|
||||
title = ann.get("title", "")
|
||||
if len(title) > 128:
|
||||
title = title[:125] + "..."
|
||||
@@ -36,7 +37,42 @@ class NotificationService:
|
||||
|
||||
if await self.client.send_textcard(title, description, url):
|
||||
sent += 1
|
||||
|
||||
# AI 标记为可承接的,额外发送着重通知
|
||||
ai_result = ann.get("ai_result")
|
||||
if ai_result and ai_result.get("is_relevant"):
|
||||
await self._send_ai_emphasis(ann, ai_result)
|
||||
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return sent
|
||||
|
||||
async def _send_ai_emphasis(
|
||||
self, ann: dict[str, Any], ai_result: dict[str, Any]
|
||||
) -> bool:
|
||||
"""发送 AI 分析的着重通知(markdown 格式)"""
|
||||
title = ann.get("title", "")
|
||||
purchase_name = ann.get("purchase_name", "")
|
||||
pub_date = ann.get("publish_date")
|
||||
time_str = pub_date.strftime("%Y-%m-%d %H:%M") if pub_date else "时间未知"
|
||||
url = ann.get("content_url", "")
|
||||
|
||||
reason = ai_result.get("reason", "")
|
||||
business_type = ai_result.get("business_type", "")
|
||||
|
||||
# 企业微信 markdown 格式
|
||||
md = (
|
||||
f"{settings.ai_analysis_title}\n"
|
||||
f"---\n"
|
||||
f"**标题:** [{title}]({url})\n"
|
||||
f"> 采购人:{purchase_name}\n"
|
||||
f"> 发布时间:{time_str}\n\n"
|
||||
f"**🤖 AI 分析:**\n"
|
||||
f"> {reason}\n\n"
|
||||
f"**🏷 业务分类:** {business_type}\n"
|
||||
f"---\n"
|
||||
f"[📄 查看公告原文]({url})"
|
||||
)
|
||||
|
||||
return await self.client.send_markdown(md)
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from app.config import settings
|
||||
from app.crawler.base import PipelineConfig, PipelineResult
|
||||
from app.services.ai_state import is_ai_enabled
|
||||
from app.services.filter_service import dedup_by_hash
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PostCrawlPipeline:
|
||||
def __init__(self, db_session, notification_service):
|
||||
@@ -20,24 +25,32 @@ class PostCrawlPipeline:
|
||||
if config.dedup_enabled:
|
||||
announcements = dedup_by_hash(announcements)
|
||||
|
||||
# 2. Store to database
|
||||
# 2. 先标记 keyword_matched,再存库
|
||||
if config.filter_enabled and config.keywords:
|
||||
for a in announcements:
|
||||
a["keyword_matched"] = self._match_keywords(a, config.keywords)
|
||||
|
||||
# 3. Store to database
|
||||
stored = await self._save_to_db(announcements)
|
||||
result.stored = stored
|
||||
|
||||
to_notify = announcements
|
||||
|
||||
# 3. Filter
|
||||
# 4. Filter
|
||||
if config.filter_enabled and config.keywords:
|
||||
before = len(to_notify)
|
||||
to_notify = [a for a in to_notify
|
||||
if self._match_keywords(a, config.keywords)]
|
||||
to_notify = [a for a in to_notify if a.get("keyword_matched")]
|
||||
result.filtered = before - len(to_notify)
|
||||
|
||||
# 4. Skip already-notified
|
||||
# 5. Skip already-notified
|
||||
if to_notify:
|
||||
to_notify = await self._exclude_sent(to_notify)
|
||||
|
||||
# 5. Notify
|
||||
# 5.5 AI 分析(可选,支持运行时开关)
|
||||
if is_ai_enabled() and to_notify:
|
||||
await self._ai_analyze(to_notify)
|
||||
|
||||
# 6. Notify
|
||||
if config.notify_mode == "all":
|
||||
result.notified = await self._send_notifications(to_notify)
|
||||
elif config.notify_mode == "filtered":
|
||||
@@ -46,7 +59,7 @@ class PostCrawlPipeline:
|
||||
elif not config.filter_enabled:
|
||||
result.notified = await self._send_notifications(to_notify)
|
||||
|
||||
# 6. Mark sent
|
||||
# 7. Mark sent
|
||||
if config.mark_sent and result.notified > 0:
|
||||
await self._mark_sent(to_notify)
|
||||
|
||||
@@ -125,3 +138,52 @@ class PostCrawlPipeline:
|
||||
def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool:
|
||||
text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}"
|
||||
return any(kw in text for kw in keywords)
|
||||
|
||||
async def _ai_analyze(self, announcements: list[dict[str, Any]]) -> None:
|
||||
"""对公告列表执行 AI 分析,将结果附加到每条公告的 ai_result 字段,并更新数据库"""
|
||||
from app.services.ai_analyzer import AiAnalyzer
|
||||
|
||||
analyzer = AiAnalyzer()
|
||||
logger.info("AI 分析开始:共 %d 条公告", len(announcements))
|
||||
|
||||
results = await analyzer.analyze_batch(announcements)
|
||||
|
||||
ai_updates = []
|
||||
for ann, ai_result in zip(announcements, results):
|
||||
ann["ai_result"] = {
|
||||
"is_relevant": ai_result.is_relevant,
|
||||
"reason": ai_result.reason,
|
||||
"business_type": ai_result.business_type,
|
||||
}
|
||||
if ai_result.error:
|
||||
logger.warning("AI 分析失败 [%s]: %s", ann.get("title", "")[:30], ai_result.error)
|
||||
else:
|
||||
ai_updates.append({
|
||||
"content_hash": ann["content_hash"],
|
||||
"ai_relevant": ai_result.is_relevant,
|
||||
"ai_analysis": ai_result.reason,
|
||||
})
|
||||
if ai_result.is_relevant:
|
||||
logger.info("AI 标记可承接项目: %s (%s)", ann.get("title", "")[:40], ai_result.business_type)
|
||||
|
||||
# 批量更新数据库中的 AI 分析结果
|
||||
if ai_updates:
|
||||
await self._update_ai_results(ai_updates)
|
||||
|
||||
async def _update_ai_results(self, updates: list[dict[str, Any]]) -> None:
|
||||
"""批量更新公告的 AI 分析结果到数据库"""
|
||||
from sqlalchemy import update
|
||||
|
||||
from app.models.announcement import Announcement
|
||||
|
||||
for u in updates:
|
||||
stmt = (
|
||||
update(Announcement)
|
||||
.where(Announcement.content_hash == u["content_hash"])
|
||||
.values(
|
||||
ai_relevant=u["ai_relevant"],
|
||||
ai_analysis=u["ai_analysis"],
|
||||
)
|
||||
)
|
||||
await self.db.execute(stmt)
|
||||
await self.db.commit()
|
||||
|
||||
+15
-3
@@ -1,9 +1,12 @@
|
||||
import logging
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WeChatClient:
|
||||
def __init__(self):
|
||||
@@ -15,7 +18,7 @@ class WeChatClient:
|
||||
if self._access_token and now < self._token_expires_at:
|
||||
return self._access_token
|
||||
|
||||
url = "https://qyapi.weixin.qq.com/cgi-bin/gettoken"
|
||||
url = f"{settings.wechat_api_base_url}/cgi-bin/gettoken"
|
||||
params = {
|
||||
"corpid": settings.wechat_corp_id,
|
||||
"corpsecret": settings.wechat_secret,
|
||||
@@ -27,6 +30,7 @@ class WeChatClient:
|
||||
self._access_token = data["access_token"]
|
||||
self._token_expires_at = now + data.get("expires_in", 7200) - 300
|
||||
return self._access_token
|
||||
logger.error(f"获取 access_token 失败: {data}")
|
||||
return None
|
||||
|
||||
async def send_text(self, content: str, to_user: str = "@all") -> bool:
|
||||
@@ -59,9 +63,10 @@ class WeChatClient:
|
||||
) -> bool:
|
||||
token = await self._get_access_token()
|
||||
if not token:
|
||||
logger.error("无法获取 access_token,跳过消息发送")
|
||||
return False
|
||||
|
||||
url = "https://qyapi.weixin.qq.com/cgi-bin/message/send"
|
||||
url = f"{settings.wechat_api_base_url}/cgi-bin/message/send"
|
||||
params = {"access_token": token}
|
||||
body = {
|
||||
"touser": to_user,
|
||||
@@ -73,4 +78,11 @@ class WeChatClient:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
response = await client.post(url, params=params, json=body)
|
||||
data = response.json()
|
||||
return data.get("errcode") == 0
|
||||
errcode = data.get("errcode")
|
||||
if errcode == 0:
|
||||
return True
|
||||
logger.error(
|
||||
f"企业微信消息发送失败: errcode={errcode} errmsg={data.get('errmsg')} "
|
||||
f"msgtype={msgtype} touser={to_user}"
|
||||
)
|
||||
return False
|
||||
|
||||
+332
-2
@@ -1,7 +1,21 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from app.config import settings
|
||||
from app.wechat.crypto import WXBizMsgCrypt
|
||||
from app.wechat.client import WeChatClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 防重入:记录最近一次触发爬取的时间戳,60秒内不重复执行
|
||||
_last_crawl_time: float = 0.0
|
||||
_crawl_lock = asyncio.Lock()
|
||||
|
||||
# 防重入:最新公告爬取,60秒内不重复执行
|
||||
_last_latest_time: float = 0.0
|
||||
_latest_lock = asyncio.Lock()
|
||||
|
||||
|
||||
class WeChatMessageHandler:
|
||||
@@ -11,6 +25,7 @@ class WeChatMessageHandler:
|
||||
sEncodingAESKey=settings.wechat_encoding_aes_key,
|
||||
sReceiveId=settings.wechat_corp_id,
|
||||
)
|
||||
self.client = WeChatClient()
|
||||
|
||||
def verify_url(
|
||||
self, msg_signature: str, timestamp: str, nonce: str, echostr: str
|
||||
@@ -48,10 +63,325 @@ class WeChatMessageHandler:
|
||||
return encrypted
|
||||
return None
|
||||
|
||||
def handle_event(
|
||||
async def handle_event(
|
||||
self, event: str, event_key: str | None, from_user: str
|
||||
) -> str | None:
|
||||
if event != "click" or not event_key:
|
||||
return None
|
||||
|
||||
if event_key == "today_stats":
|
||||
return await self._handle_today_stats(from_user)
|
||||
elif event_key == "trigger_crawl":
|
||||
return await self._handle_trigger_crawl(from_user)
|
||||
elif event_key == "sync_holidays":
|
||||
return await self._handle_sync_holidays(from_user)
|
||||
elif event_key == "monitor_config":
|
||||
return await self._handle_monitor_config(from_user)
|
||||
elif event_key == "system_status":
|
||||
return await self._handle_system_status(from_user)
|
||||
elif event_key == "workday_status":
|
||||
return await self._handle_workday_status(from_user)
|
||||
elif event_key == "latest_announcements":
|
||||
return await self._handle_latest_announcements(from_user)
|
||||
elif event_key == "pause_scheduler":
|
||||
return await self._handle_pause_scheduler(from_user)
|
||||
elif event_key == "resume_scheduler":
|
||||
return await self._handle_resume_scheduler(from_user)
|
||||
elif event_key == "toggle_ai":
|
||||
return await self._handle_toggle_ai(from_user)
|
||||
return None
|
||||
|
||||
def handle_text(self, content: str, from_user: str) -> str | None:
|
||||
async def handle_text(self, content: str, from_user: str) -> str | None:
|
||||
"""文本消息:返回帮助提示"""
|
||||
help_text = (
|
||||
"请使用菜单操作:\n"
|
||||
"---\n"
|
||||
"📋 最新公告 - 获取最新公告\n"
|
||||
"📊 查询 → 监控配置/系统状态/今日工作日\n"
|
||||
"⚙️ 系统管理 → 立即爬取/同步节假日/AI 分析"
|
||||
)
|
||||
ok = await self.client.send_text(help_text, from_user)
|
||||
if not ok:
|
||||
logger.warning(f"发送帮助消息失败, touser={from_user}")
|
||||
return None
|
||||
|
||||
async def _handle_today_stats(self, from_user: str) -> str | None:
|
||||
from app.api.deps import get_db
|
||||
|
||||
try:
|
||||
async for db in get_db():
|
||||
from sqlalchemy import func, select
|
||||
from app.models.announcement import Announcement
|
||||
|
||||
total_result = await db.execute(
|
||||
select(func.count()).select_from(Announcement)
|
||||
)
|
||||
total = total_result.scalar() or 0
|
||||
|
||||
today_result = await db.execute(
|
||||
select(func.count()).where(
|
||||
func.date(Announcement.publish_date) == func.current_date()
|
||||
).select_from(Announcement)
|
||||
)
|
||||
today = today_result.scalar() or 0
|
||||
|
||||
text = f"今日新增: {today} 条\n累计公告: {total} 条"
|
||||
await self.client.send_text(text, from_user)
|
||||
except Exception as e:
|
||||
logger.error(f"查询统计失败: {e}")
|
||||
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||
|
||||
async def _handle_trigger_crawl(self, from_user: str) -> str | None:
|
||||
global _last_crawl_time
|
||||
from app.api.deps import get_crawl_service
|
||||
|
||||
# 防重入:企业微信会对同一事件重试多次,60秒内只执行一次
|
||||
async with _crawl_lock:
|
||||
now = time.monotonic()
|
||||
if now - _last_crawl_time < 60:
|
||||
remaining = int(60 - (now - _last_crawl_time))
|
||||
logger.info(f"爬取请求被忽略(防重入),距上次 {now - _last_crawl_time:.1f}s")
|
||||
await self.client.send_text(
|
||||
f"爬取任务进行中,请 {remaining} 秒后再试", from_user
|
||||
)
|
||||
return None
|
||||
_last_crawl_time = now
|
||||
|
||||
await self.client.send_text("开始爬取,请稍候...", from_user)
|
||||
|
||||
try:
|
||||
service = get_crawl_service()
|
||||
results = await service.run_all()
|
||||
total = sum(r.total_count for r in results)
|
||||
stored = sum(
|
||||
r.pipeline_result.stored for r in results
|
||||
if r.pipeline_result
|
||||
)
|
||||
notified = sum(
|
||||
r.pipeline_result.notified for r in results
|
||||
if r.pipeline_result
|
||||
)
|
||||
errors = [r.error_message for r in results if not r.success]
|
||||
msg = f"爬取完成\n抓取: {total} 条\n新增: {stored} 条\n推送: {notified} 条"
|
||||
if errors:
|
||||
msg += f"\n异常: {errors[0][:50]}"
|
||||
await self.client.send_text(msg, from_user)
|
||||
except Exception as e:
|
||||
logger.error(f"手动爬取失败: {e}")
|
||||
await self.client.send_text(f"爬取失败: {e}", from_user)
|
||||
|
||||
async def _handle_sync_holidays(self, from_user: str) -> str | None:
|
||||
from app.services.holiday_service import now_in_china, sync_holidays
|
||||
from app.api.deps import get_db
|
||||
|
||||
try:
|
||||
async for db in get_db():
|
||||
year = now_in_china().year
|
||||
count = await sync_holidays(db, year)
|
||||
await self.client.send_text(
|
||||
f"已同步 {year} 年节假日\n共 {count} 条记录", from_user,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"同步节假日失败: {e}")
|
||||
await self.client.send_text(f"同步失败: {e}", from_user)
|
||||
|
||||
async def _handle_monitor_config(self, from_user: str) -> str | None:
|
||||
import json
|
||||
from app.services.ai_state import get_ai_status_text
|
||||
try:
|
||||
keywords = settings.crawler_keywords
|
||||
sources = json.loads(settings.announcement_sources)
|
||||
source_names = "、".join(v["name"] for v in sources.values())
|
||||
text = (
|
||||
f"📋 监控配置\n"
|
||||
f"---\n"
|
||||
f"监控关键词: {', '.join(keywords)}\n"
|
||||
f"爬取页数: {settings.crawler_max_pages} 页\n"
|
||||
f"定时规则: {settings.scheduler_cron}\n"
|
||||
f"公告来源: {source_names}\n"
|
||||
f"---\n"
|
||||
f"🤖 AI 分析: {get_ai_status_text()}\n"
|
||||
f"AI 模型: {settings.ai_model}\n"
|
||||
f"重点标记: {settings.ai_analysis_title}"
|
||||
)
|
||||
await self.client.send_text(text, from_user)
|
||||
except Exception as e:
|
||||
logger.error(f"查询监控配置失败: {e}")
|
||||
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||
|
||||
async def _handle_system_status(self, from_user: str) -> str | None:
|
||||
from app.api.deps import get_db
|
||||
from sqlalchemy import func, select
|
||||
from app.models.announcement import Announcement
|
||||
from app.services.ai_state import get_ai_status_text
|
||||
|
||||
try:
|
||||
async for db in get_db():
|
||||
total_result = await db.execute(
|
||||
select(func.count()).select_from(Announcement)
|
||||
)
|
||||
total = total_result.scalar() or 0
|
||||
|
||||
today_result = await db.execute(
|
||||
select(func.count()).where(
|
||||
func.date(Announcement.publish_date) == func.current_date()
|
||||
).select_from(Announcement)
|
||||
)
|
||||
today = today_result.scalar() or 0
|
||||
|
||||
unsent_result = await db.execute(
|
||||
select(func.count()).where(
|
||||
Announcement.is_sent == False, # noqa: E712
|
||||
Announcement.keyword_matched == True, # noqa: E712
|
||||
).select_from(Announcement)
|
||||
)
|
||||
unsent = unsent_result.scalar() or 0
|
||||
|
||||
# AI 标记统计
|
||||
ai_relevant_result = await db.execute(
|
||||
select(func.count()).where(
|
||||
Announcement.ai_relevant == True, # noqa: E712
|
||||
).select_from(Announcement)
|
||||
)
|
||||
ai_relevant = ai_relevant_result.scalar() or 0
|
||||
|
||||
scheduler_status = "已启用" if settings.scheduler_enabled else "已禁用"
|
||||
text = (
|
||||
f"📊 系统状态\n"
|
||||
f"---\n"
|
||||
f"累计公告: {total} 条\n"
|
||||
f"今日新增: {today} 条\n"
|
||||
f"待推送: {unsent} 条\n"
|
||||
f"---\n"
|
||||
f"定时任务: {scheduler_status}\n"
|
||||
f"定时规则: {settings.scheduler_cron}\n"
|
||||
f"---\n"
|
||||
f"🤖 AI 分析: {get_ai_status_text()}\n"
|
||||
f"AI 标记项目: {ai_relevant} 条"
|
||||
)
|
||||
await self.client.send_text(text, from_user)
|
||||
except Exception as e:
|
||||
logger.error(f"查询系统状态失败: {e}")
|
||||
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||
|
||||
async def _handle_workday_status(self, from_user: str) -> str | None:
|
||||
from app.services.holiday_service import now_in_china, is_workday
|
||||
from app.api.deps import get_db
|
||||
|
||||
try:
|
||||
async for db in get_db():
|
||||
today = now_in_china()
|
||||
workday = await is_workday(db, today)
|
||||
status = "工作日,正常爬取" if workday else "非工作日,跳过爬取"
|
||||
text = f"今天 {today.strftime('%Y-%m-%d %A')}\n{status}"
|
||||
await self.client.send_text(text, from_user)
|
||||
except Exception as e:
|
||||
logger.error(f"查询工作日状态失败: {e}")
|
||||
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||
|
||||
async def _handle_latest_announcements(self, from_user: str) -> str | None:
|
||||
global _last_latest_time
|
||||
from app.api.deps import get_db, get_crawl_service
|
||||
from sqlalchemy import select, asc, or_
|
||||
from app.models.announcement import Announcement
|
||||
|
||||
# 防重入:60秒内整个函数只执行一次(含推送)
|
||||
async with _latest_lock:
|
||||
now = time.monotonic()
|
||||
if (now - _last_latest_time) < 60:
|
||||
logger.info("最新公告请求被忽略(防重入)")
|
||||
return None
|
||||
_last_latest_time = now
|
||||
|
||||
try:
|
||||
await self.client.send_text("正在获取最新公告,请稍候...", from_user)
|
||||
service = get_crawl_service()
|
||||
await service.run_all()
|
||||
|
||||
# 查:广西政采网关键词匹配 + 大化县政府网全部,取最新6条后按时间升序推送
|
||||
async for db in get_db():
|
||||
from sqlalchemy import select, desc, or_
|
||||
result = await db.execute(
|
||||
select(Announcement)
|
||||
.where(or_(
|
||||
Announcement.keyword_matched == True, # noqa: E712
|
||||
Announcement.source_code == "dahuagov",
|
||||
))
|
||||
.order_by(desc(Announcement.publish_date))
|
||||
.limit(6)
|
||||
)
|
||||
items = result.scalars().all()
|
||||
|
||||
if not items:
|
||||
await self.client.send_text("暂无公告", from_user)
|
||||
return None
|
||||
|
||||
# 按时间升序推送,企微里向上滑即为时间正序
|
||||
for ann in sorted(items, key=lambda a: a.publish_date):
|
||||
title = ann.title if len(ann.title) <= 128 else ann.title[:125] + "..."
|
||||
purchase_name = ann.purchase_name or ""
|
||||
if len(purchase_name) > 25:
|
||||
purchase_name = purchase_name[:22] + "..."
|
||||
time_str = ann.publish_date.strftime("%Y-%m-%d %H:%M") if ann.publish_date else "时间未知"
|
||||
source_name = ann.source_name or ann.source_code or ""
|
||||
description = f"{source_name} | {purchase_name} | {time_str}"
|
||||
await self.client.send_textcard(title, description, ann.content_url or "", from_user)
|
||||
await asyncio.sleep(1)
|
||||
except Exception as e:
|
||||
logger.error(f"查询最新公告失败: {e}")
|
||||
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||
|
||||
async def _handle_pause_scheduler(self, from_user: str) -> str | None:
|
||||
try:
|
||||
from app.scheduler.jobs import scheduler
|
||||
if scheduler.running:
|
||||
scheduler.pause()
|
||||
await self.client.send_text("定时任务已暂停", from_user)
|
||||
else:
|
||||
await self.client.send_text("定时任务未在运行", from_user)
|
||||
except Exception as e:
|
||||
logger.error(f"暂停定时任务失败: {e}")
|
||||
await self.client.send_text(f"操作失败: {e}", from_user)
|
||||
|
||||
async def _handle_toggle_ai(self, from_user: str) -> str | None:
|
||||
from app.services.ai_state import (
|
||||
get_ai_status_text,
|
||||
get_whitelist,
|
||||
is_ai_enabled,
|
||||
is_whitelisted,
|
||||
set_ai_enabled,
|
||||
)
|
||||
|
||||
# 白名单校验
|
||||
whitelist = get_whitelist()
|
||||
if whitelist and not is_whitelisted(from_user):
|
||||
await self.client.send_text(
|
||||
f"⚠️ 你没有权限操作 AI 分析开关\n"
|
||||
f"当前仅以下用户可操作:\n{', '.join(whitelist)}",
|
||||
from_user,
|
||||
)
|
||||
return None
|
||||
|
||||
current = is_ai_enabled()
|
||||
changed = set_ai_enabled(not current)
|
||||
if not changed:
|
||||
await self.client.send_text(
|
||||
f"AI 分析当前已是「{get_ai_status_text()}」状态,无需切换",
|
||||
from_user,
|
||||
)
|
||||
return None
|
||||
|
||||
await self.client.send_text(
|
||||
f"✅ AI 分析已切换为「{get_ai_status_text()}」\n"
|
||||
f"下次爬取触发时生效",
|
||||
from_user,
|
||||
)
|
||||
|
||||
async def _handle_resume_scheduler(self, from_user: str) -> str | None:
|
||||
try:
|
||||
from app.scheduler.jobs import scheduler
|
||||
scheduler.resume()
|
||||
await self.client.send_text("定时任务已恢复", from_user)
|
||||
except Exception as e:
|
||||
logger.error(f"恢复定时任务失败: {e}")
|
||||
await self.client.send_text(f"操作失败: {e}", from_user)
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MENU = {
|
||||
"button": [
|
||||
{
|
||||
"name": "最新公告",
|
||||
"type": "click",
|
||||
"key": "latest_announcements",
|
||||
},
|
||||
{
|
||||
"name": "查询",
|
||||
"sub_button": [
|
||||
{
|
||||
"name": "监控配置",
|
||||
"type": "click",
|
||||
"key": "monitor_config",
|
||||
},
|
||||
{
|
||||
"name": "系统状态",
|
||||
"type": "click",
|
||||
"key": "system_status",
|
||||
},
|
||||
{
|
||||
"name": "今日工作日",
|
||||
"type": "click",
|
||||
"key": "workday_status",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "系统管理",
|
||||
"sub_button": [
|
||||
{
|
||||
"name": "立即爬取",
|
||||
"type": "click",
|
||||
"key": "trigger_crawl",
|
||||
},
|
||||
{
|
||||
"name": "同步节假日",
|
||||
"type": "click",
|
||||
"key": "sync_holidays",
|
||||
},
|
||||
{
|
||||
"name": "AI 分析",
|
||||
"type": "click",
|
||||
"key": "toggle_ai",
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
class MenuManager:
|
||||
def __init__(self, client=None):
|
||||
self.client = client
|
||||
|
||||
async def _get_token(self) -> str | None:
|
||||
from app.wechat.client import WeChatClient
|
||||
c = self.client or WeChatClient()
|
||||
return await c._get_access_token()
|
||||
|
||||
async def create(self) -> bool:
|
||||
token = await self._get_token()
|
||||
if not token:
|
||||
return False
|
||||
|
||||
url = f"{settings.wechat_api_base_url}/cgi-bin/menu/create"
|
||||
params = {"access_token": token, "agentid": int(settings.wechat_agent_id)}
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
response = await client.post(url, params=params, json=MENU)
|
||||
data = response.json()
|
||||
if data.get("errcode") == 0:
|
||||
logger.info("企微菜单创建成功")
|
||||
return True
|
||||
logger.error(f"企微菜单创建失败: {data}")
|
||||
return False
|
||||
|
||||
async def delete(self) -> bool:
|
||||
token = await self._get_token()
|
||||
if not token:
|
||||
return False
|
||||
|
||||
url = f"{settings.wechat_api_base_url}/cgi-bin/menu/delete"
|
||||
params = {"access_token": token, "agentid": int(settings.wechat_agent_id)}
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
response = await client.get(url, params=params)
|
||||
data = response.json()
|
||||
return data.get("errcode") == 0
|
||||
|
||||
async def get(self) -> dict | None:
|
||||
token = await self._get_token()
|
||||
if not token:
|
||||
return None
|
||||
|
||||
url = f"{settings.wechat_api_base_url}/cgi-bin/menu/get"
|
||||
params = {"access_token": token, "agentid": int(settings.wechat_agent_id)}
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
response = await client.get(url, params=params)
|
||||
data = response.json()
|
||||
if data.get("errcode") == 0:
|
||||
return data
|
||||
return None
|
||||
+6
-1
@@ -1,5 +1,9 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
LABEL maintainer="GX-gp-notify Dev Team" \
|
||||
version="2.0.0" \
|
||||
description="广西政府采购网公告监控系统"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
@@ -7,7 +11,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY pyproject.toml .
|
||||
RUN pip install --no-cache-dir -e ".[dev]"
|
||||
COPY vendor/ vendor/
|
||||
RUN pip install --no-cache-dir -e ".[dev]" && pip install --no-cache-dir ./vendor/
|
||||
|
||||
COPY . .
|
||||
|
||||
|
||||
@@ -3,6 +3,11 @@ services:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile
|
||||
tags:
|
||||
- gx-gp-notify:2.0.0
|
||||
- gx-gp-notify:latest
|
||||
image: gx-gp-notify:2.0.0
|
||||
container_name: gx-gp-notify
|
||||
ports:
|
||||
- "18001:8000"
|
||||
env_file:
|
||||
|
||||
@@ -53,7 +53,6 @@ def test_pipeline_result_defaults():
|
||||
assert result.stored == 0
|
||||
assert result.filtered == 0
|
||||
assert result.notified == 0
|
||||
assert result.markdown_generated is False
|
||||
|
||||
|
||||
def test_generate_content_hash():
|
||||
|
||||
Vendored
+84
@@ -0,0 +1,84 @@
|
||||
# LogHive Client SDK
|
||||
|
||||
Python client SDK for sending logs to [LogHive](https://github.com/your-org/loghive).
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install loghive-client
|
||||
```
|
||||
|
||||
Or install from source:
|
||||
|
||||
```bash
|
||||
cd client
|
||||
pip install .
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Sync mode (recommended for scripts, Django, Flask)
|
||||
|
||||
```python
|
||||
from loghive_client import LogHiveLogger
|
||||
|
||||
logger = LogHiveLogger(
|
||||
project="my-awesome-app",
|
||||
api_key="your-api-key",
|
||||
endpoint="http://localhost:8000",
|
||||
)
|
||||
|
||||
logger.info("Server started", extra={"port": 8080})
|
||||
logger.error("Database timeout", exc_info=True)
|
||||
```
|
||||
|
||||
### Async mode (for FastAPI, aiohttp, asyncio)
|
||||
|
||||
```python
|
||||
from loghive_client import AsyncLogHiveLogger
|
||||
import asyncio
|
||||
|
||||
async def main():
|
||||
async with AsyncLogHiveLogger(
|
||||
project="my-api",
|
||||
api_key="your-api-key",
|
||||
endpoint="http://localhost:8000",
|
||||
) as logger:
|
||||
await logger.info("API started")
|
||||
# ...
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
### Standard logging integration (zero code change)
|
||||
|
||||
Add the handler to your existing logger:
|
||||
|
||||
```python
|
||||
import logging
|
||||
from loghive_client import LogHiveHandler
|
||||
|
||||
handler = LogHiveHandler("my-project", "api-key", "http://localhost:8000")
|
||||
logging.getLogger().addHandler(handler)
|
||||
|
||||
# All existing logger calls now forward to LogHive
|
||||
logging.info("This goes to LogHive too!")
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
| Param | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `project` | (required) | Your project name in LogHive |
|
||||
| `api_key` | (required) | Your project's API key |
|
||||
| `endpoint` | `http://localhost:8000` | LogHive server URL |
|
||||
| `batch_size` | 50 | Max entries per HTTP request |
|
||||
| `flush_interval` | 2.0 | Seconds between flushes |
|
||||
| `max_retries` | 3 | Retries on failure |
|
||||
| `timeout` | 5.0 | HTTP request timeout |
|
||||
|
||||
## Trace ID (request correlation)
|
||||
|
||||
```python
|
||||
logger.set_trace_id("req-abc-123")
|
||||
```
|
||||
Vendored
+20
@@ -0,0 +1,20 @@
|
||||
"""LogHive Client — Send logs from your Python projects to LogHive.
|
||||
|
||||
Usage:
|
||||
from loghive_client import LogHiveLogger
|
||||
|
||||
logger = LogHiveLogger(
|
||||
project="my-project",
|
||||
api_key="your-api-key",
|
||||
endpoint="http://localhost:8000",
|
||||
)
|
||||
|
||||
logger.info("User logged in", extra={"user_id": 42})
|
||||
logger.error("Database connection failed", exc_info=True)
|
||||
"""
|
||||
|
||||
from loghive_client.client import LogHiveLogger
|
||||
from loghive_client.async_client import AsyncLogHiveLogger
|
||||
from loghive_client.handler import LogHiveHandler
|
||||
|
||||
__all__ = ["LogHiveLogger", "AsyncLogHiveLogger", "LogHiveHandler"]
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
"""Async LogHive client — for use in asyncio-based projects (e.g., FastAPI, aiohttp)."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import traceback
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AsyncLogHiveLogger:
|
||||
"""Async logger for asyncio applications.
|
||||
|
||||
Uses an async background task to batch and send log entries.
|
||||
Ideal for FastAPI / Starlette / aiohttp projects.
|
||||
|
||||
Usage:
|
||||
logger = AsyncLogHiveLogger("my-project", "api-key", "http://localhost:8000")
|
||||
await logger.start()
|
||||
|
||||
await logger.info("Request processed", extra={"path": "/api/users"})
|
||||
|
||||
await logger.stop()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
project: str,
|
||||
api_key: str,
|
||||
endpoint: str = "http://localhost:8000",
|
||||
batch_size: int = 50,
|
||||
flush_interval: float = 2.0,
|
||||
max_retries: int = 3,
|
||||
timeout: float = 5.0,
|
||||
):
|
||||
self.project = project
|
||||
self.api_key = api_key
|
||||
self.endpoint = endpoint.rstrip("/")
|
||||
self.batch_size = batch_size
|
||||
self.flush_interval = flush_interval
|
||||
self.max_retries = max_retries
|
||||
self.timeout = timeout
|
||||
|
||||
self._queue: asyncio.Queue = asyncio.Queue()
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self._client: Optional[httpx.AsyncClient] = None
|
||||
self._stop_event = asyncio.Event()
|
||||
|
||||
async def start(self):
|
||||
"""Start the background flush task."""
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._client = httpx.AsyncClient(timeout=self.timeout)
|
||||
self._stop_event.clear()
|
||||
self._task = asyncio.create_task(self._flush_loop())
|
||||
logger.debug("AsyncLogHiveLogger started for project '%s'", self.project)
|
||||
|
||||
async def stop(self, flush: bool = True):
|
||||
"""Stop the background task."""
|
||||
self._stop_event.set()
|
||||
if flush:
|
||||
await self._flush_now()
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
logger.debug("AsyncLogHiveLogger stopped for project '%s'", self.project)
|
||||
|
||||
async def __aenter__(self):
|
||||
await self.start()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
await self.stop()
|
||||
|
||||
async def _flush_loop(self):
|
||||
"""Background loop that periodically flushes the queue."""
|
||||
while not self._stop_event.is_set():
|
||||
await self._flush_now()
|
||||
await asyncio.sleep(self.flush_interval)
|
||||
|
||||
async def _flush_now(self):
|
||||
"""Flush all currently queued entries."""
|
||||
entries = []
|
||||
while len(entries) < self.batch_size:
|
||||
try:
|
||||
entry = self._queue.get_nowait()
|
||||
entries.append(entry)
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
|
||||
if not entries:
|
||||
return
|
||||
|
||||
await self._send_batch(entries)
|
||||
|
||||
async def _send_batch(self, entries: List[Dict[str, Any]]):
|
||||
"""Send a batch with retries."""
|
||||
url = urljoin(self.endpoint, "/api/logs/ingest")
|
||||
payload = {"project": self.project, "entries": entries}
|
||||
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
resp = await self._client.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
)
|
||||
if resp.status_code == 201:
|
||||
return
|
||||
elif resp.status_code == 401:
|
||||
logger.error("LogHive: Invalid API key — dropping batch")
|
||||
return
|
||||
else:
|
||||
logger.warning(
|
||||
"LogHive: HTTP %d (attempt %d/%d)",
|
||||
resp.status_code,
|
||||
attempt + 1,
|
||||
self.max_retries,
|
||||
)
|
||||
except httpx.RequestError as e:
|
||||
logger.warning(
|
||||
"LogHive: Connection error (attempt %d/%d): %s",
|
||||
attempt + 1,
|
||||
self.max_retries,
|
||||
e,
|
||||
)
|
||||
|
||||
if attempt < self.max_retries - 1:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
|
||||
logger.error("LogHive: Failed to send %d entries after %d retries", len(entries), self.max_retries)
|
||||
|
||||
def _enqueue(self, level: str, message: str, **kwargs):
|
||||
"""Enqueue a log entry."""
|
||||
entry = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"level": level,
|
||||
"message": message,
|
||||
"logger": kwargs.pop("logger", None) or __name__,
|
||||
"extra": kwargs,
|
||||
}
|
||||
|
||||
exc_info = kwargs.pop("exc_info", None)
|
||||
if exc_info:
|
||||
if isinstance(exc_info, BaseException):
|
||||
entry["exception"] = "".join(
|
||||
traceback.format_exception(type(exc_info), exc_info, exc_info.__traceback__)
|
||||
)
|
||||
elif exc_info is True:
|
||||
entry["exception"] = traceback.format_exc()
|
||||
|
||||
self._queue.put_nowait(entry)
|
||||
|
||||
# ── Public API ─────────────────────────────────────────────
|
||||
|
||||
async def debug(self, message: str, **kwargs):
|
||||
self._enqueue("debug", message, **kwargs)
|
||||
|
||||
async def info(self, message: str, **kwargs):
|
||||
self._enqueue("info", message, **kwargs)
|
||||
|
||||
async def warning(self, message: str, **kwargs):
|
||||
self._enqueue("warning", message, **kwargs)
|
||||
|
||||
async def error(self, message: str, **kwargs):
|
||||
self._enqueue("error", message, **kwargs)
|
||||
|
||||
async def critical(self, message: str, **kwargs):
|
||||
self._enqueue("critical", message, **kwargs)
|
||||
Vendored
+183
@@ -0,0 +1,183 @@
|
||||
"""Synchronous LogHive client — uses threading for non-blocking sends."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from datetime import datetime, timezone
|
||||
from queue import Queue, Empty
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LogHiveLogger:
|
||||
"""Synchronous logger that sends logs to LogHive in the background.
|
||||
|
||||
Uses a background thread with a queue to avoid blocking the main
|
||||
application on network I/O.
|
||||
|
||||
Usage:
|
||||
logger = LogHiveLogger("my-project", "api-key-here", "http://localhost:8000")
|
||||
logger.info("Hello, world!")
|
||||
logger.error("Something broke", exc_info=True)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
project: str,
|
||||
api_key: str,
|
||||
endpoint: str = "http://localhost:8000",
|
||||
batch_size: int = 50,
|
||||
flush_interval: float = 2.0,
|
||||
max_retries: int = 3,
|
||||
timeout: float = 5.0,
|
||||
auto_start: bool = True,
|
||||
):
|
||||
self.project = project
|
||||
self.api_key = api_key
|
||||
self.endpoint = endpoint.rstrip("/")
|
||||
self.batch_size = batch_size
|
||||
self.flush_interval = flush_interval
|
||||
self.max_retries = max_retries
|
||||
self.timeout = timeout
|
||||
|
||||
self._queue: Queue = Queue()
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
|
||||
if auto_start:
|
||||
self.start()
|
||||
|
||||
def start(self):
|
||||
"""Start the background flush thread."""
|
||||
if self._thread and self._thread.is_alive():
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(target=self._flush_loop, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self, flush: bool = True):
|
||||
"""Stop the background thread, optionally flushing remaining logs."""
|
||||
self._stop_event.set()
|
||||
if flush and self._thread:
|
||||
self._flush_now()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=5)
|
||||
|
||||
def _flush_loop(self):
|
||||
"""Background loop that periodically flushes the queue."""
|
||||
while not self._stop_event.is_set():
|
||||
self._flush_now()
|
||||
self._stop_event.wait(self.flush_interval)
|
||||
|
||||
def _flush_now(self):
|
||||
"""Flush all currently queued log entries."""
|
||||
entries = []
|
||||
while len(entries) < self.batch_size:
|
||||
try:
|
||||
entry = self._queue.get_nowait()
|
||||
entries.append(entry)
|
||||
except Empty:
|
||||
break
|
||||
|
||||
if not entries:
|
||||
return
|
||||
|
||||
self._send_batch(entries)
|
||||
|
||||
def _send_batch(self, entries: List[Dict[str, Any]]):
|
||||
"""Send a batch of entries to the LogHive API, with retries."""
|
||||
url = urljoin(self.endpoint, "/api/logs/ingest")
|
||||
payload = {"project": self.project, "entries": entries}
|
||||
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
resp = httpx.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
if resp.status_code == 201:
|
||||
return
|
||||
elif resp.status_code == 401:
|
||||
logger.error("LogHive: Invalid API key — dropping batch")
|
||||
return
|
||||
else:
|
||||
logger.warning(
|
||||
"LogHive: HTTP %d (attempt %d/%d)",
|
||||
resp.status_code,
|
||||
attempt + 1,
|
||||
self.max_retries,
|
||||
)
|
||||
except httpx.RequestError as e:
|
||||
logger.warning(
|
||||
"LogHive: Connection error (attempt %d/%d): %s",
|
||||
attempt + 1,
|
||||
self.max_retries,
|
||||
e,
|
||||
)
|
||||
|
||||
if attempt < self.max_retries - 1:
|
||||
time.sleep(2 ** attempt)
|
||||
|
||||
logger.error("LogHive: Failed to send %d entries after %d retries", len(entries), self.max_retries)
|
||||
|
||||
def _enqueue(self, level: str, message: str, **kwargs):
|
||||
"""Enqueue a log entry for async sending."""
|
||||
entry = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"level": level,
|
||||
"message": message,
|
||||
"logger": kwargs.pop("logger", None) or __name__,
|
||||
"module": kwargs.pop("module", None),
|
||||
"function": kwargs.pop("function", None),
|
||||
"line_no": kwargs.pop("line_no", None),
|
||||
"trace_id": kwargs.pop("trace_id", None),
|
||||
"extra": kwargs,
|
||||
}
|
||||
|
||||
# Handle exception info
|
||||
exc_info = kwargs.pop("exc_info", None)
|
||||
if exc_info:
|
||||
if isinstance(exc_info, BaseException):
|
||||
entry["exception"] = "".join(
|
||||
traceback.format_exception(type(exc_info), exc_info, exc_info.__traceback__)
|
||||
)
|
||||
elif exc_info is True:
|
||||
entry["exception"] = traceback.format_exc()
|
||||
|
||||
self._queue.put_nowait(entry)
|
||||
|
||||
# ── Public API (matching standard logging levels) ──────────
|
||||
|
||||
def debug(self, message: str, **kwargs):
|
||||
self._enqueue("debug", message, **kwargs)
|
||||
|
||||
def info(self, message: str, **kwargs):
|
||||
self._enqueue("info", message, **kwargs)
|
||||
|
||||
def warning(self, message: str, **kwargs):
|
||||
self._enqueue("warning", message, **kwargs)
|
||||
|
||||
def error(self, message: str, **kwargs):
|
||||
self._enqueue("error", message, **kwargs)
|
||||
|
||||
def critical(self, message: str, **kwargs):
|
||||
self._enqueue("critical", message, **kwargs)
|
||||
|
||||
def log(self, level: str, message: str, **kwargs):
|
||||
"""Log a message with an explicit level string."""
|
||||
self._enqueue(level, message, **kwargs)
|
||||
|
||||
def set_trace_id(self, trace_id: str):
|
||||
"""Set a trace_id for request correlation (used in web frameworks)."""
|
||||
self._current_trace_id = trace_id
|
||||
|
||||
def __del__(self):
|
||||
self.stop(flush=True)
|
||||
Vendored
+72
@@ -0,0 +1,72 @@
|
||||
"""Python logging.Handler integration — use LogHive with the stdlib logging module.
|
||||
|
||||
This allows you to replace or augment your existing logging setup with
|
||||
zero code changes (just add a handler to your logger).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from loghive_client.client import LogHiveLogger
|
||||
|
||||
|
||||
class LogHiveHandler(logging.Handler):
|
||||
"""A logging.Handler that sends records to LogHive.
|
||||
|
||||
Use it with Python's standard logging module:
|
||||
import logging
|
||||
from loghive_client import LogHiveHandler
|
||||
|
||||
handler = LogHiveHandler("my-project", "api-key", "http://localhost:8000")
|
||||
logging.getLogger().addHandler(handler)
|
||||
|
||||
All existing logger calls (logger.info, logger.error, etc.) will
|
||||
automatically forward to LogHive.
|
||||
"""
|
||||
|
||||
LEVEL_MAP = {
|
||||
logging.DEBUG: "debug",
|
||||
logging.INFO: "info",
|
||||
logging.WARNING: "warning",
|
||||
logging.ERROR: "error",
|
||||
logging.CRITICAL: "critical",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
project: str,
|
||||
api_key: str,
|
||||
endpoint: str = "http://localhost:8000",
|
||||
level: int = logging.INFO,
|
||||
):
|
||||
super().__init__(level=level)
|
||||
self._client = LogHiveLogger(
|
||||
project=project,
|
||||
api_key=api_key,
|
||||
endpoint=endpoint,
|
||||
)
|
||||
|
||||
def emit(self, record: logging.LogRecord):
|
||||
"""Send a log record to LogHive."""
|
||||
try:
|
||||
level = self.LEVEL_MAP.get(record.levelno, "info")
|
||||
extra = {
|
||||
"logger": record.name,
|
||||
"module": record.module,
|
||||
"function": record.funcName,
|
||||
"line_no": record.lineno,
|
||||
}
|
||||
if record.exc_info and record.exc_info[0]:
|
||||
import traceback
|
||||
extra["exception"] = "".join(
|
||||
traceback.format_exception(*record.exc_info)
|
||||
)
|
||||
|
||||
self._client._enqueue(level, record.getMessage(), **extra)
|
||||
except Exception:
|
||||
self.handleError(record)
|
||||
|
||||
def close(self):
|
||||
"""Flush and close."""
|
||||
self._client.stop(flush=True)
|
||||
super().close()
|
||||
Vendored
+23
@@ -0,0 +1,23 @@
|
||||
"""Setup script for loghive-client."""
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
setup(
|
||||
name="loghive-client",
|
||||
version="0.1.0",
|
||||
description="LogHive client SDK — push logs from your Python projects to LogHive",
|
||||
author="LogHive",
|
||||
packages=find_packages(),
|
||||
install_requires=[
|
||||
"httpx>=0.27.0",
|
||||
],
|
||||
python_requires=">=3.10",
|
||||
classifiers=[
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: Developers",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
],
|
||||
)
|
||||
Reference in New Issue
Block a user