Compare commits
26 Commits
d114d5dd6a
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 18f415b363 | |||
| 103de16a8f | |||
| a51161b5f3 | |||
| 68215aa804 | |||
| 754214692e | |||
| 136941d84e | |||
| 4f16134955 | |||
| c01a3d427b | |||
| dc3ca1f76c | |||
| 6d1d512fca | |||
| b2d0503bc2 | |||
| 77c6722a92 | |||
| 048fca283d | |||
| 0121017272 | |||
| 24d844bf44 | |||
| 9f90e16661 | |||
| d84c838e76 | |||
| e1c9a985ca | |||
| 2996920277 | |||
| 1f18d2ec87 | |||
| c3e06c997f | |||
| 248866d73f | |||
| cbeb3c3504 | |||
| d851dafea9 | |||
| 6fbdf097b3 | |||
| 241465ea7d |
+21
-4
@@ -20,11 +20,28 @@ WECHAT_TOKEN=
|
|||||||
WECHAT_ENCODING_AES_KEY=
|
WECHAT_ENCODING_AES_KEY=
|
||||||
WECHAT_PORT=18001
|
WECHAT_PORT=18001
|
||||||
WECHAT_HOST=0.0.0.0
|
WECHAT_HOST=0.0.0.0
|
||||||
|
# 企微 API 代理(可选),用于绕过 IP 白名单限制
|
||||||
|
# 留空则直连 https://qyapi.weixin.qq.com
|
||||||
|
WECHAT_API_BASE_URL=https://qyapi.weixin.qq.com
|
||||||
|
|
||||||
# 定时任务
|
# 定时任务
|
||||||
SCHEDULER_ENABLED=true
|
SCHEDULER_ENABLED=true
|
||||||
SCHEDULER_CRON=0 8,14,18 * * *
|
SCHEDULER_CRON=0 8-21 * * *
|
||||||
|
|
||||||
# Markdown
|
# LogHive 日志系统
|
||||||
MARKDOWN_ENABLED=true
|
LOGHIVE_ENDPOINT=http://10.10.10.14:8000
|
||||||
MARKDOWN_OUTPUT_FILE=onu.md
|
LOGHIVE_PROJECT=gx-gp-notify
|
||||||
|
LOGHIVE_API_KEY=
|
||||||
|
|
||||||
|
# AI 分析 (DeepSeek)
|
||||||
|
AI_ENABLED=false
|
||||||
|
# AI 管理白名单(企微用户ID,逗号分隔),留空表示所有人可操作
|
||||||
|
AI_WHITELIST=
|
||||||
|
AI_API_KEY=sk-your-deepseek-api-key
|
||||||
|
AI_BASE_URL=https://api.deepseek.com/v1
|
||||||
|
AI_MODEL=deepseek-chat
|
||||||
|
AI_TIMEOUT=30
|
||||||
|
AI_ANALYSIS_TITLE=🔔 中国电信可承接项目
|
||||||
|
# 自定义分析提示词(可选),不设置则用代码内置默认值
|
||||||
|
# 使用 \n 表示换行,支持 {title} {purchase_name} {announcement_type} {content} 四个占位符
|
||||||
|
# AI_PROMPT_TEMPLATE=你是一个政府采购项目分析师...\n\n--- 公告信息 ---\n标题:{title}\n...
|
||||||
|
|||||||
@@ -12,3 +12,7 @@ dist/
|
|||||||
config.yaml
|
config.yaml
|
||||||
gx_gp_monitor/config/config.yaml
|
gx_gp_monitor/config/config.yaml
|
||||||
onu.md
|
onu.md
|
||||||
|
vendor/build/
|
||||||
|
vendor/*.egg-info/
|
||||||
|
# Added by code-review-graph
|
||||||
|
.code-review-graph/
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
# GX-GP-Notify
|
||||||
|
|
||||||
|
广西政府采购公告监控系统,自动爬取广西政府采购网和大化县政府网的公告,通过企业微信推送关键词匹配的公告。
|
||||||
|
|
||||||
|
## 功能
|
||||||
|
|
||||||
|
- **定时爬取**:按 cron 规则自动爬取,工作日 8:00-21:00 每小时执行
|
||||||
|
- **关键词过滤**:广西政采网公告按关键词过滤(默认:`大化`),大化县政府网全量爬取
|
||||||
|
- **企微推送**:新公告实时推送到企业微信应用
|
||||||
|
- **去重**:基于内容 hash 去重,避免重复入库和推送
|
||||||
|
- **节假日感知**:非工作日自动跳过爬取
|
||||||
|
|
||||||
|
## 数据来源
|
||||||
|
|
||||||
|
| 来源 | 说明 | 过滤 |
|
||||||
|
|------|------|------|
|
||||||
|
| 广西政府采购网 | 采购公告、结果公告、合同公告等 13 个分类 | 关键词过滤 |
|
||||||
|
| 大化县政府网 | 大化县门户网站采购公告 | 全量爬取 |
|
||||||
|
|
||||||
|
## 企微菜单功能
|
||||||
|
|
||||||
|
| 菜单 | 说明 |
|
||||||
|
|------|------|
|
||||||
|
| 最新公告 | 触发一次爬取,推送两个来源合并后最新 6 条(时间升序) |
|
||||||
|
| 手动爬取 | 立即触发全量爬取并推送新公告 |
|
||||||
|
| 今日统计 | 查看今日新增和累计公告数 |
|
||||||
|
| 系统状态 | 查看待推送数量、定时任务状态 |
|
||||||
|
| 监控配置 | 查看关键词、爬取页数、定时规则 |
|
||||||
|
| 工作日状态 | 查看今天是否为工作日 |
|
||||||
|
| 同步节假日 | 同步当年节假日数据 |
|
||||||
|
| 暂停/恢复任务 | 控制定时爬取任务 |
|
||||||
|
|
||||||
|
## 快速部署
|
||||||
|
|
||||||
|
### 1. 配置环境变量
|
||||||
|
|
||||||
|
复制 `.env.example` 为 `.env` 并填写:
|
||||||
|
|
||||||
|
```env
|
||||||
|
# 数据库
|
||||||
|
DATABASE_URL=postgresql+asyncpg://user:password@host:5432/gx-gp-notify
|
||||||
|
|
||||||
|
# 企业微信
|
||||||
|
WECHAT_CORP_ID=your_corp_id
|
||||||
|
WECHAT_AGENT_ID=your_agent_id
|
||||||
|
WECHAT_SECRET=your_secret
|
||||||
|
WECHAT_TOKEN=your_token
|
||||||
|
WECHAT_ENCODING_AES_KEY=your_aes_key
|
||||||
|
|
||||||
|
# 爬虫
|
||||||
|
CRAWLER_KEYWORDS=["大化"]
|
||||||
|
CRAWLER_MAX_PAGES=10
|
||||||
|
SCHEDULER_CRON=0 8-21 * * 1-5
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. 启动服务
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker/docker-compose.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. 数据库迁移
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker/docker-compose.yml exec app alembic upgrade head
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. 配置企微回调
|
||||||
|
|
||||||
|
在企业微信后台将回调 URL 设置为:
|
||||||
|
|
||||||
|
```
|
||||||
|
http://your-server:18001/wechat/callback
|
||||||
|
```
|
||||||
|
|
||||||
|
## 开发
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 安装依赖
|
||||||
|
pip install -e ".[dev]"
|
||||||
|
|
||||||
|
# 运行测试
|
||||||
|
pytest
|
||||||
|
|
||||||
|
# 代码检查
|
||||||
|
ruff check .
|
||||||
|
```
|
||||||
|
|
||||||
|
## 技术栈
|
||||||
|
|
||||||
|
- **框架**:FastAPI + uvicorn
|
||||||
|
- **数据库**:PostgreSQL + SQLAlchemy (async) + Alembic
|
||||||
|
- **爬虫**:httpx + BeautifulSoup4
|
||||||
|
- **定时任务**:APScheduler
|
||||||
|
- **推送**:企业微信应用消息 API
|
||||||
|
- **部署**:Docker
|
||||||
|
|
||||||
|
## 项目结构
|
||||||
|
|
||||||
|
```
|
||||||
|
app/
|
||||||
|
├── api/ # HTTP 接口(公告查询、手动触发爬取等)
|
||||||
|
├── crawler/ # 爬虫(广西政采网、大化县政府网)
|
||||||
|
├── models/ # 数据库模型
|
||||||
|
├── scheduler/ # 定时任务
|
||||||
|
├── services/ # 业务逻辑(pipeline、过滤、通知)
|
||||||
|
└── wechat/ # 企业微信(消息处理、推送客户端)
|
||||||
|
```
|
||||||
@@ -1 +0,0 @@
|
|||||||
Generic single-database configuration.
|
|
||||||
@@ -2,6 +2,7 @@ import asyncio
|
|||||||
from alembic import context
|
from alembic import context
|
||||||
from sqlalchemy.ext.asyncio import create_async_engine
|
from sqlalchemy.ext.asyncio import create_async_engine
|
||||||
from app.models.announcement import Base
|
from app.models.announcement import Base
|
||||||
|
from app.models.holiday import ChineseHoliday # noqa: F401
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
target_metadata = Base.metadata
|
target_metadata = Base.metadata
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
"""add_chinese_holidays_table
|
||||||
|
|
||||||
|
Revision ID: 1f59799a5083
|
||||||
|
Revises: eed20ee8cc26
|
||||||
|
Create Date: 2026-05-09 18:14:31.144256
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '1f59799a5083'
|
||||||
|
down_revision: Union[str, Sequence[str], None] = 'eed20ee8cc26'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Upgrade schema."""
|
||||||
|
op.create_table(
|
||||||
|
'chinese_holidays',
|
||||||
|
sa.Column('date', sa.Date(), nullable=False),
|
||||||
|
sa.Column('is_workday', sa.Boolean(), nullable=False, server_default='true'),
|
||||||
|
sa.Column('year', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('description', sa.String(length=100), nullable=False, server_default=''),
|
||||||
|
sa.PrimaryKeyConstraint('date'),
|
||||||
|
)
|
||||||
|
op.create_index('idx_chinese_holidays_year', 'chinese_holidays', ['year'])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Downgrade schema."""
|
||||||
|
op.drop_index('idx_chinese_holidays_year', table_name='chinese_holidays')
|
||||||
|
op.drop_table('chinese_holidays')
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
"""add_ai_analysis_columns
|
||||||
|
|
||||||
|
Revision ID: 6e8f4c2d1b0a
|
||||||
|
Revises: 1f59799a5083
|
||||||
|
Create Date: 2026-05-21 10:00:00.000000
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '6e8f4c2d1b0a'
|
||||||
|
down_revision: Union[str, Sequence[str], None] = '1f59799a5083'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""添加 AI 分析相关字段"""
|
||||||
|
op.add_column('announcements', sa.Column('ai_relevant', sa.Boolean(), nullable=True))
|
||||||
|
op.add_column('announcements', sa.Column('ai_analysis', sa.Text(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""回滚"""
|
||||||
|
op.drop_column('announcements', 'ai_analysis')
|
||||||
|
op.drop_column('announcements', 'ai_relevant')
|
||||||
+25
-3
@@ -9,16 +9,27 @@ router = APIRouter()
|
|||||||
@router.post("/crawl/trigger")
|
@router.post("/crawl/trigger")
|
||||||
async def trigger_crawl(request: CrawlTriggerRequest):
|
async def trigger_crawl(request: CrawlTriggerRequest):
|
||||||
service = get_crawl_service()
|
service = get_crawl_service()
|
||||||
|
if request.spider_name:
|
||||||
|
names = [request.spider_name]
|
||||||
|
else:
|
||||||
names = service.get_spider_names()
|
names = service.get_spider_names()
|
||||||
|
|
||||||
all_results = []
|
all_results = []
|
||||||
|
total_stored = 0
|
||||||
|
total_notified = 0
|
||||||
for name in names:
|
for name in names:
|
||||||
results = await service.run_spider(name)
|
results = await service.run_spider(name)
|
||||||
all_results.extend(results)
|
all_results.extend(results)
|
||||||
|
for r in results:
|
||||||
|
if r.pipeline_result:
|
||||||
|
total_stored += r.pipeline_result.stored
|
||||||
|
total_notified += r.pipeline_result.notified
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"spiders_run": names,
|
"spiders_run": names,
|
||||||
"total_announcements": sum(r.total_count for r in all_results),
|
"total_announcements": sum(r.total_count for r in all_results),
|
||||||
|
"total_stored": total_stored,
|
||||||
|
"total_notified": total_notified,
|
||||||
"errors": [r.error_message for r in all_results if not r.success],
|
"errors": [r.error_message for r in all_results if not r.success],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -38,9 +49,20 @@ async def crawl_sources():
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
sources = json.loads(settings.announcement_sources)
|
sources = json.loads(settings.announcement_sources)
|
||||||
return {
|
result = [
|
||||||
"sources": [
|
|
||||||
{"code": code, "name": info["name"], "type": info["type"]}
|
{"code": code, "name": info["name"], "type": info["type"]}
|
||||||
for code, info in sources.items()
|
for code, info in sources.items()
|
||||||
]
|
]
|
||||||
}
|
# 加入独立爬虫来源(非 GXGP 子来源的独立 Spider)
|
||||||
|
service = get_crawl_service()
|
||||||
|
for name in service.get_spider_names():
|
||||||
|
if name == "gxgp":
|
||||||
|
continue # gxgp 的子来源已在上面列出
|
||||||
|
spider = service.spiders.get(name)
|
||||||
|
if spider:
|
||||||
|
result.append({
|
||||||
|
"code": spider.source_code,
|
||||||
|
"name": spider.source_name,
|
||||||
|
"type": "independent",
|
||||||
|
})
|
||||||
|
return {"sources": result}
|
||||||
|
|||||||
+7
-2
@@ -3,10 +3,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
from app.crawler.dahuagov_spider import DahuagovSpider
|
from app.crawler.dahuagov_spider import DahuagovSpider
|
||||||
from app.crawler.gxgp_spider import GXGPSpider
|
from app.crawler.gxgp_spider import GXGPSpider
|
||||||
from app.services.crawl_service import CrawlService
|
from app.services.crawl_service import CrawlService
|
||||||
|
from app.services.notification_service import NotificationService
|
||||||
|
|
||||||
|
|
||||||
async def get_db() -> AsyncSession:
|
async def get_db() -> AsyncSession:
|
||||||
from app.main import async_session # 延迟导入避免循环引用
|
from app.main import async_session
|
||||||
async with async_session() as session:
|
async with async_session() as session:
|
||||||
yield session
|
yield session
|
||||||
|
|
||||||
@@ -17,7 +18,11 @@ _crawl_service: CrawlService | None = None
|
|||||||
def get_crawl_service() -> CrawlService:
|
def get_crawl_service() -> CrawlService:
|
||||||
global _crawl_service
|
global _crawl_service
|
||||||
if _crawl_service is None:
|
if _crawl_service is None:
|
||||||
_crawl_service = CrawlService()
|
from app.main import async_session
|
||||||
|
_crawl_service = CrawlService(
|
||||||
|
db_session_factory=async_session,
|
||||||
|
notification_service=NotificationService(),
|
||||||
|
)
|
||||||
_crawl_service.register(GXGPSpider())
|
_crawl_service.register(GXGPSpider())
|
||||||
_crawl_service.register(DahuagovSpider())
|
_crawl_service.register(DahuagovSpider())
|
||||||
return _crawl_service
|
return _crawl_service
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
from fastapi import APIRouter, Depends, Query
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.api.deps import get_db
|
||||||
|
from app.services.holiday_service import now_in_china, sync_holidays
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/holidays/sync")
|
||||||
|
async def sync_holidays_endpoint(
|
||||||
|
year: int | None = Query(None, description="同步年份,默认当前年份"),
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
):
|
||||||
|
if year is None:
|
||||||
|
year = now_in_china().year
|
||||||
|
count = await sync_holidays(db, year)
|
||||||
|
return {"status": "ok", "year": year, "synced": count}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/holidays/today")
|
||||||
|
async def get_today_status(db: AsyncSession = Depends(get_db)):
|
||||||
|
from app.services.holiday_service import is_workday
|
||||||
|
today = now_in_china()
|
||||||
|
workday = await is_workday(db, today)
|
||||||
|
return {
|
||||||
|
"date": today.isoformat(),
|
||||||
|
"is_workday": workday,
|
||||||
|
"message": "工作日,正常爬取" if workday else "非工作日,跳过爬取",
|
||||||
|
}
|
||||||
+2
-1
@@ -1,10 +1,11 @@
|
|||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
|
|
||||||
from app.api import announcements, crawl, wechat
|
from app.api import announcements, crawl, holidays, wechat
|
||||||
from app.api import scheduler as scheduler_module
|
from app.api import scheduler as scheduler_module
|
||||||
|
|
||||||
api_router = APIRouter(prefix="/api/v1")
|
api_router = APIRouter(prefix="/api/v1")
|
||||||
api_router.include_router(announcements.router, tags=["announcements"])
|
api_router.include_router(announcements.router, tags=["announcements"])
|
||||||
api_router.include_router(crawl.router, tags=["crawl"])
|
api_router.include_router(crawl.router, tags=["crawl"])
|
||||||
|
api_router.include_router(holidays.router, tags=["holidays"])
|
||||||
api_router.include_router(wechat.router, tags=["wechat"])
|
api_router.include_router(wechat.router, tags=["wechat"])
|
||||||
api_router.include_router(scheduler_module.router)
|
api_router.include_router(scheduler_module.router)
|
||||||
|
|||||||
+2
-2
@@ -52,7 +52,7 @@ async def wechat_callback(request: Request):
|
|||||||
event = xml_tree.find("Event")
|
event = xml_tree.find("Event")
|
||||||
event_key = xml_tree.find("EventKey")
|
event_key = xml_tree.find("EventKey")
|
||||||
from_user = xml_tree.find("FromUserName")
|
from_user = xml_tree.find("FromUserName")
|
||||||
handler.handle_event(
|
await handler.handle_event(
|
||||||
event.text if event is not None else "",
|
event.text if event is not None else "",
|
||||||
event_key.text if event_key is not None else None,
|
event_key.text if event_key is not None else None,
|
||||||
from_user.text if from_user is not None else "",
|
from_user.text if from_user is not None else "",
|
||||||
@@ -60,7 +60,7 @@ async def wechat_callback(request: Request):
|
|||||||
elif msg_type == "text":
|
elif msg_type == "text":
|
||||||
content = xml_tree.find("Content")
|
content = xml_tree.find("Content")
|
||||||
from_user = xml_tree.find("FromUserName")
|
from_user = xml_tree.find("FromUserName")
|
||||||
handler.handle_text(
|
await handler.handle_text(
|
||||||
content.text if content is not None else "",
|
content.text if content is not None else "",
|
||||||
from_user.text if from_user is not None else "",
|
from_user.text if from_user is not None else "",
|
||||||
)
|
)
|
||||||
|
|||||||
+43
-4
@@ -1,5 +1,6 @@
|
|||||||
|
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
from pydantic import field_validator
|
||||||
|
|
||||||
|
|
||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
@@ -28,14 +29,52 @@ class Settings(BaseSettings):
|
|||||||
wechat_encoding_aes_key: str = ""
|
wechat_encoding_aes_key: str = ""
|
||||||
wechat_port: int = 18001
|
wechat_port: int = 18001
|
||||||
wechat_host: str = "0.0.0.0"
|
wechat_host: str = "0.0.0.0"
|
||||||
|
wechat_api_base_url: str = "https://qyapi.weixin.qq.com"
|
||||||
|
|
||||||
# 定时任务
|
# 定时任务
|
||||||
scheduler_enabled: bool = True
|
scheduler_enabled: bool = True
|
||||||
scheduler_cron: str = "0 8,14,18 * * *"
|
scheduler_cron: str = "0 8-21 * * *"
|
||||||
|
|
||||||
# Markdown
|
# LogHive 日志系统
|
||||||
markdown_enabled: bool = True
|
loghive_endpoint: str = "http://10.10.10.14:8000"
|
||||||
markdown_output_file: str = "onu.md"
|
loghive_project: str = "gx-gp-notify"
|
||||||
|
loghive_api_key: str = ""
|
||||||
|
|
||||||
|
# AI 分析 (DeepSeek)
|
||||||
|
ai_enabled: bool = False
|
||||||
|
ai_whitelist: str = ""
|
||||||
|
ai_api_key: str = ""
|
||||||
|
ai_base_url: str = "https://api.deepseek.com/v1"
|
||||||
|
ai_model: str = "deepseek-chat"
|
||||||
|
ai_timeout: int = 30
|
||||||
|
ai_analysis_title: str = "🔔 中国电信可承接项目"
|
||||||
|
ai_prompt_template: str = (
|
||||||
|
"你是一个政府采购项目分析师,专门帮助中国电信识别可以承接的项目。\n\n"
|
||||||
|
"可承接范围包括但不限于:\n"
|
||||||
|
"- 通信工程、光缆建设、基站建设\n"
|
||||||
|
"- 信息化系统建设、系统集成\n"
|
||||||
|
"- 云计算、大数据、政务云\n"
|
||||||
|
"- 物联网、智慧城市、智慧园区\n"
|
||||||
|
"- 安防监控、视频会议、应急通信\n"
|
||||||
|
"- 网络运维、网络优化、IDC 服务\n"
|
||||||
|
"- 5G 应用、专线服务\n\n"
|
||||||
|
"--- 公告信息 ---\n"
|
||||||
|
"标题:{title}\n"
|
||||||
|
"采购人:{purchase_name}\n"
|
||||||
|
"公告类型:{announcement_type}\n\n"
|
||||||
|
"--- 公告正文 ---\n"
|
||||||
|
"{content}\n\n"
|
||||||
|
"请用 JSON 格式回答:\n"
|
||||||
|
'{{"is_relevant": true/false, "reason": "简要判断理由", "business_type": "业务分类"}}'
|
||||||
|
)
|
||||||
|
|
||||||
|
@field_validator("ai_prompt_template", mode="before")
|
||||||
|
@classmethod
|
||||||
|
def convert_newlines(cls, v: str) -> str:
|
||||||
|
"""将 .env 中字面 \\n 转换为真实换行"""
|
||||||
|
if isinstance(v, str) and "\\n" in v:
|
||||||
|
return v.replace("\\n", "\n")
|
||||||
|
return v
|
||||||
|
|
||||||
# 公告来源(JSON 字符串,从环境变量读取)
|
# 公告来源(JSON 字符串,从环境变量读取)
|
||||||
announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}' # noqa: E501
|
announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}' # noqa: E501
|
||||||
|
|||||||
+8
-8
@@ -4,6 +4,13 @@ from dataclasses import dataclass, field
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PipelineResult:
|
||||||
|
stored: int = 0
|
||||||
|
filtered: int = 0
|
||||||
|
notified: int = 0
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class CrawlResult:
|
class CrawlResult:
|
||||||
source_code: str
|
source_code: str
|
||||||
@@ -14,6 +21,7 @@ class CrawlResult:
|
|||||||
error_message: str | None = None
|
error_message: str | None = None
|
||||||
crawled_at: datetime = field(default_factory=datetime.now)
|
crawled_at: datetime = field(default_factory=datetime.now)
|
||||||
duration: float = 0.0
|
duration: float = 0.0
|
||||||
|
pipeline_result: PipelineResult | None = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def success(self) -> bool:
|
def success(self) -> bool:
|
||||||
@@ -29,14 +37,6 @@ class PipelineConfig:
|
|||||||
mark_sent: bool = False
|
mark_sent: bool = False
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class PipelineResult:
|
|
||||||
stored: int = 0
|
|
||||||
filtered: int = 0
|
|
||||||
notified: int = 0
|
|
||||||
markdown_generated: bool = False
|
|
||||||
|
|
||||||
|
|
||||||
class BaseSpider(ABC):
|
class BaseSpider(ABC):
|
||||||
name: str
|
name: str
|
||||||
source_code: str
|
source_code: str
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import httpx
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||||
from app.crawler.parsers import parse_dahuagov_html
|
from app.crawler.parsers import parse_dahuagov_html, parse_dahuagov_detail_pubdate
|
||||||
|
|
||||||
|
|
||||||
class DahuagovSpider(BaseSpider):
|
class DahuagovSpider(BaseSpider):
|
||||||
@@ -57,6 +57,8 @@ class DahuagovSpider(BaseSpider):
|
|||||||
)
|
)
|
||||||
|
|
||||||
announcements = parse_dahuagov_html(html, start_time)
|
announcements = parse_dahuagov_html(html, start_time)
|
||||||
|
await self._fetch_detail_dates(client, announcements, headers)
|
||||||
|
|
||||||
duration = (datetime.now() - start_time).total_seconds()
|
duration = (datetime.now() - start_time).total_seconds()
|
||||||
|
|
||||||
return CrawlResult(
|
return CrawlResult(
|
||||||
@@ -69,6 +71,31 @@ class DahuagovSpider(BaseSpider):
|
|||||||
duration=duration,
|
duration=duration,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def _fetch_detail_dates(
|
||||||
|
self,
|
||||||
|
client: httpx.AsyncClient,
|
||||||
|
announcements: list[dict],
|
||||||
|
headers: dict,
|
||||||
|
) -> None:
|
||||||
|
"""并发抓取详情页,用 PubDate meta 更新精确发布时间"""
|
||||||
|
sem = asyncio.Semaphore(5)
|
||||||
|
|
||||||
|
async def fetch_one(ann: dict) -> None:
|
||||||
|
async with sem:
|
||||||
|
await asyncio.sleep(random.uniform(0.3, 0.8))
|
||||||
|
try:
|
||||||
|
resp = await client.get(ann["content_url"], headers=headers)
|
||||||
|
if resp.status_code == 200:
|
||||||
|
pub = parse_dahuagov_detail_pubdate(resp.text)
|
||||||
|
if pub:
|
||||||
|
ann["publish_date"] = pub
|
||||||
|
ann["is_today"] = pub.date() == datetime.now().date()
|
||||||
|
except Exception:
|
||||||
|
pass # 保留列表页日期作为 fallback
|
||||||
|
|
||||||
|
await asyncio.gather(*[fetch_one(ann) for ann in announcements])
|
||||||
|
|
||||||
async def _delay(self):
|
async def _delay(self):
|
||||||
delay = random.uniform(1.0, 3.0)
|
delay = random.uniform(1.0, 3.0)
|
||||||
await asyncio.sleep(delay)
|
await asyncio.sleep(delay)
|
||||||
|
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ class GXGPSpider(BaseSpider):
|
|||||||
keywords=list(settings.crawler_keywords),
|
keywords=list(settings.crawler_keywords),
|
||||||
dedup_enabled=True,
|
dedup_enabled=True,
|
||||||
notify_mode="filtered",
|
notify_mode="filtered",
|
||||||
mark_sent=False,
|
mark_sent=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def crawl(self, sources: list[str] | None = None,
|
async def crawl(self, sources: list[str] | None = None,
|
||||||
|
|||||||
+186
-1
@@ -2,6 +2,9 @@ import hashlib
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
|
_TZ = ZoneInfo("Asia/Shanghai")
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
@@ -30,7 +33,7 @@ def parse_gxgp_api_response(
|
|||||||
if not timestamp:
|
if not timestamp:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
publish_date = datetime.fromtimestamp(int(timestamp) / 1000)
|
publish_date = datetime.fromtimestamp(int(timestamp) / 1000, tz=_TZ).replace(tzinfo=None)
|
||||||
except (ValueError, TypeError):
|
except (ValueError, TypeError):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -131,6 +134,188 @@ def parse_dahuagov_html(html: str, crawled_at: datetime) -> list[dict[str, Any]]
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def parse_dahuagov_detail_pubdate(html: str) -> datetime | None:
|
||||||
|
"""从详情页 <meta name="PubDate"> 解析精确发布时间"""
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
meta = soup.find("meta", attrs={"name": "PubDate"})
|
||||||
|
if not meta:
|
||||||
|
return None
|
||||||
|
content = meta.get("content", "").strip()
|
||||||
|
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d"):
|
||||||
|
try:
|
||||||
|
return datetime.strptime(content, fmt)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def extract_page_content(url: str, timeout: int = 30) -> str | None:
|
||||||
|
"""抓取详情页并用 BeautifulSoup 提取正文纯文本"""
|
||||||
|
import urllib.parse
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
# ---- 站点专用处理 ----
|
||||||
|
|
||||||
|
# ① 政采云 SPA (HTTP API, 无需渲染)
|
||||||
|
if "zfcg.gxzf.gov.cn" in url and "articleId=" in url:
|
||||||
|
return await _extract_zcy_content(url, timeout)
|
||||||
|
|
||||||
|
# ② 大化县政府网 (静态HTML)
|
||||||
|
if "www.gxdh.gov.cn" in url or "gxdh.gov.cn" in url:
|
||||||
|
return await _extract_dahuagov_content(url, timeout)
|
||||||
|
|
||||||
|
# ---- 通用 HTML 提取 ----
|
||||||
|
try:
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||||
|
}
|
||||||
|
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, headers=headers)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return None
|
||||||
|
|
||||||
|
soup = BeautifulSoup(resp.text, "html.parser")
|
||||||
|
|
||||||
|
# 尝试多种常见正文容器选择器
|
||||||
|
selectors = [
|
||||||
|
"div.article-content", "div.content", "div.TRS_Editor",
|
||||||
|
"div.Custom_UnionStyle", "div.pages_content", "div#content",
|
||||||
|
"div.main-content", "article", ".article", ".detail-content",
|
||||||
|
".text-content", ".news-content", ".detail-article",
|
||||||
|
"div.article-con", ".trs_editor_view", ".TRS_UEDITOR",
|
||||||
|
".trs_paper_default", ".article-content",
|
||||||
|
]
|
||||||
|
for selector in selectors:
|
||||||
|
container = soup.select_one(selector)
|
||||||
|
if container:
|
||||||
|
for tag in container.find_all(["script", "style"]):
|
||||||
|
tag.decompose()
|
||||||
|
text = container.get_text(separator="\n", strip=True)
|
||||||
|
if len(text) > 50:
|
||||||
|
return text
|
||||||
|
|
||||||
|
# 兜底:取 body 内所有文本
|
||||||
|
body = soup.find("body")
|
||||||
|
if body:
|
||||||
|
for tag in body.find_all(["script", "style", "nav", "footer", "header"]):
|
||||||
|
tag.decompose()
|
||||||
|
text = body.get_text(separator="\n", strip=True)
|
||||||
|
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||||
|
text = "\n".join(lines[:200])
|
||||||
|
if len(text) > 50:
|
||||||
|
return text
|
||||||
|
|
||||||
|
return None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def _extract_zcy_content(url: str, timeout: int = 30) -> str | None:
|
||||||
|
"""从政采云 SPA 隐藏 API 提取公告正文"""
|
||||||
|
import urllib.parse
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
params = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
||||||
|
article_id = params.get("articleId", [None])[0]
|
||||||
|
parent_id = params.get("parentId", [None])[0]
|
||||||
|
if not article_id:
|
||||||
|
return None
|
||||||
|
|
||||||
|
api_url = "https://zfcg.gxzf.gov.cn/portal/detail"
|
||||||
|
if parent_id:
|
||||||
|
api_url += f"?articleId={article_id}&parentId={parent_id}"
|
||||||
|
else:
|
||||||
|
api_url += f"?articleId={article_id}"
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||||
|
"Accept": "application/json, text/plain, */*",
|
||||||
|
"Referer": url,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||||
|
resp = await client.get(api_url, headers=headers)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return None
|
||||||
|
|
||||||
|
data = resp.json()
|
||||||
|
if not data.get("success"):
|
||||||
|
return None
|
||||||
|
|
||||||
|
content_html = data.get("result", {}).get("data", {}).get("content", "")
|
||||||
|
if not content_html:
|
||||||
|
return None
|
||||||
|
|
||||||
|
soup = BeautifulSoup(content_html, "html.parser")
|
||||||
|
for tag in soup.find_all(["script", "style"]):
|
||||||
|
tag.decompose()
|
||||||
|
text = soup.get_text(separator="\n", strip=True)
|
||||||
|
|
||||||
|
# 清理过短行和多余空白
|
||||||
|
lines = [l.strip() for l in text.split("\n") if len(l.strip()) > 5]
|
||||||
|
text = "\n".join(lines[:300])
|
||||||
|
return text if len(text) > 50 else None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def _extract_dahuagov_content(url: str, timeout: int = 30) -> str | None:
|
||||||
|
"""从大化县政府网详情页提取正文"""
|
||||||
|
import httpx
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||||
|
"Accept": "text/html,application/xhtml+xml",
|
||||||
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||||
|
resp = await client.get(url, headers=headers)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return None
|
||||||
|
|
||||||
|
soup = BeautifulSoup(resp.text, "html.parser")
|
||||||
|
|
||||||
|
# 大化县政府网正文容器
|
||||||
|
selectors = [
|
||||||
|
"div.article-con", ".trs_editor_view", ".TRS_UEDITOR",
|
||||||
|
".trs_paper_default", "div.content", "div.TRS_Editor",
|
||||||
|
"div.article-content", "div.Custom_UnionStyle",
|
||||||
|
"div#content", "div.main-content", "article", ".detail-content",
|
||||||
|
]
|
||||||
|
for selector in selectors:
|
||||||
|
container = soup.select_one(selector)
|
||||||
|
if container:
|
||||||
|
for tag in container.find_all(["script", "style"]):
|
||||||
|
tag.decompose()
|
||||||
|
text = container.get_text(separator="\n", strip=True)
|
||||||
|
if len(text) > 50:
|
||||||
|
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||||
|
return "\n".join(lines[:300])
|
||||||
|
|
||||||
|
# 兜底
|
||||||
|
body = soup.find("body")
|
||||||
|
if body:
|
||||||
|
for tag in body.find_all(["script", "style", "nav", "footer", "header"]):
|
||||||
|
tag.decompose()
|
||||||
|
text = body.get_text(separator="\n", strip=True)
|
||||||
|
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||||
|
text = "\n".join(lines[:200])
|
||||||
|
if len(text) > 50:
|
||||||
|
return text
|
||||||
|
return None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _generate_hash(ann: dict[str, Any]) -> str:
|
def _generate_hash(ann: dict[str, Any]) -> str:
|
||||||
content = (
|
content = (
|
||||||
f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"
|
f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"
|
||||||
|
|||||||
+11
@@ -22,6 +22,17 @@ async def lifespan(app: FastAPI):
|
|||||||
level=getattr(logging, settings.log_level),
|
level=getattr(logging, settings.log_level),
|
||||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||||
)
|
)
|
||||||
|
# LogHive 远程日志(仅当配置了 API Key 时启用)
|
||||||
|
if settings.loghive_api_key:
|
||||||
|
from loghive_client import LogHiveHandler
|
||||||
|
handler = LogHiveHandler(
|
||||||
|
project=settings.loghive_project,
|
||||||
|
api_key=settings.loghive_api_key,
|
||||||
|
endpoint=settings.loghive_endpoint,
|
||||||
|
level=logging.INFO,
|
||||||
|
)
|
||||||
|
logging.getLogger().addHandler(handler)
|
||||||
|
logging.info("LogHive 日志系统已连接")
|
||||||
from app.scheduler.jobs import shutdown_scheduler, start_scheduler
|
from app.scheduler.jobs import shutdown_scheduler, start_scheduler
|
||||||
start_scheduler()
|
start_scheduler()
|
||||||
yield
|
yield
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ class Announcement(Base):
|
|||||||
is_new: Mapped[bool] = mapped_column(Boolean, default=True)
|
is_new: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||||
is_sent: Mapped[bool] = mapped_column(Boolean, default=False)
|
is_sent: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||||
keyword_matched: Mapped[bool] = mapped_column(Boolean, default=False)
|
keyword_matched: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||||
|
ai_relevant: Mapped[bool | None] = mapped_column(Boolean, nullable=True, default=None)
|
||||||
|
ai_analysis: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now()) # noqa: E501
|
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now()) # noqa: E501
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
from datetime import date
|
||||||
|
|
||||||
|
from sqlalchemy import Boolean, Date, Integer, String
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from app.models.announcement import Base
|
||||||
|
|
||||||
|
|
||||||
|
class ChineseHoliday(Base):
|
||||||
|
__tablename__ = "chinese_holidays"
|
||||||
|
|
||||||
|
date: Mapped[date] = mapped_column(Date, primary_key=True)
|
||||||
|
is_workday: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||||
|
year: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
|
||||||
|
description: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
||||||
@@ -29,6 +29,7 @@ class AnnouncementListResponse(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class CrawlTriggerRequest(BaseModel):
|
class CrawlTriggerRequest(BaseModel):
|
||||||
|
spider_name: str | None = None
|
||||||
keywords: list[str] | None = None
|
keywords: list[str] | None = None
|
||||||
sources: list[str] | None = None
|
sources: list[str] | None = None
|
||||||
manual: bool = False
|
manual: bool = False
|
||||||
|
|||||||
+42
-9
@@ -1,26 +1,61 @@
|
|||||||
import logging
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||||
|
from apscheduler.triggers.cron import CronTrigger
|
||||||
|
|
||||||
from app.api.deps import get_crawl_service
|
from app.api.deps import get_db, get_crawl_service
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
scheduler = AsyncIOScheduler()
|
scheduler = AsyncIOScheduler()
|
||||||
|
|
||||||
|
NIGHT_START = 22 # 22:00
|
||||||
|
NIGHT_END = 8 # 08:00
|
||||||
|
TZ = ZoneInfo("Asia/Shanghai")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_night_time() -> bool:
|
||||||
|
"""22:00 ~ 次日 08:00 夜间时段"""
|
||||||
|
current = datetime.now(TZ).hour
|
||||||
|
return current >= NIGHT_START or current < NIGHT_END
|
||||||
|
|
||||||
|
|
||||||
|
async def _should_skip() -> bool:
|
||||||
|
"""检查是否应该跳过爬取"""
|
||||||
|
if _is_night_time():
|
||||||
|
logger.info("夜间时段 (22:00-08:00),跳过爬取")
|
||||||
|
return True
|
||||||
|
|
||||||
|
from app.services.holiday_service import is_workday
|
||||||
|
async for db in get_db():
|
||||||
|
if not await is_workday(db):
|
||||||
|
logger.info("非工作日,跳过爬取")
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
async def scheduled_crawl():
|
async def scheduled_crawl():
|
||||||
|
if await _should_skip():
|
||||||
|
return
|
||||||
|
|
||||||
logger.info("开始定时爬取任务")
|
logger.info("开始定时爬取任务")
|
||||||
service = get_crawl_service()
|
service = get_crawl_service()
|
||||||
names = service.get_spider_names()
|
for name in service.get_spider_names():
|
||||||
for name in names:
|
|
||||||
try:
|
try:
|
||||||
results = await service.run_spider(name)
|
results = await service.run_spider(name)
|
||||||
for r in results:
|
for r in results:
|
||||||
if not r.success:
|
if not r.success:
|
||||||
logger.error(f"Spider {name} 失败: {r.error_message}")
|
logger.error(f"Spider {name} 失败: {r.error_message}")
|
||||||
|
elif r.pipeline_result:
|
||||||
|
logger.info(
|
||||||
|
f"Spider {name}: 抓取{r.total_count}条, "
|
||||||
|
f"新增{r.pipeline_result.stored}条, "
|
||||||
|
f"通知{r.pipeline_result.notified}条"
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
logger.info(f"Spider {name} 完成: {r.total_count} 条")
|
logger.info(f"Spider {name}: 抓取{r.total_count}条 (无新增)")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Spider {name} 异常: {e}")
|
logger.error(f"Spider {name} 异常: {e}")
|
||||||
logger.info("定时爬取任务完成")
|
logger.info("定时爬取任务完成")
|
||||||
@@ -29,17 +64,15 @@ async def scheduled_crawl():
|
|||||||
def start_scheduler():
|
def start_scheduler():
|
||||||
if not settings.scheduler_enabled:
|
if not settings.scheduler_enabled:
|
||||||
return
|
return
|
||||||
|
trigger = CronTrigger.from_crontab(settings.scheduler_cron, timezone=TZ)
|
||||||
scheduler.add_job(
|
scheduler.add_job(
|
||||||
scheduled_crawl,
|
scheduled_crawl,
|
||||||
"cron",
|
trigger=trigger,
|
||||||
hour="8,14,18",
|
|
||||||
minute="0",
|
|
||||||
id="scheduled_crawl",
|
id="scheduled_crawl",
|
||||||
name="定时爬取",
|
name="定时爬取",
|
||||||
timezone="Asia/Shanghai",
|
|
||||||
)
|
)
|
||||||
scheduler.start()
|
scheduler.start()
|
||||||
logger.info("APScheduler 已启动 (8:00, 14:00, 18:00)")
|
logger.info(f"APScheduler 已启动 (cron: {settings.scheduler_cron})")
|
||||||
|
|
||||||
|
|
||||||
def shutdown_scheduler():
|
def shutdown_scheduler():
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
from app.crawler.parsers import extract_page_content
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AiResult:
|
||||||
|
"""DeepSeek 分析结果"""
|
||||||
|
is_relevant: bool = False
|
||||||
|
reason: str = ""
|
||||||
|
business_type: str = ""
|
||||||
|
error: str | None = None
|
||||||
|
content_snippet: str | None = None # 提取到的正文前 200 字,供入库参考
|
||||||
|
|
||||||
|
|
||||||
|
class AiAnalyzer:
|
||||||
|
"""AI 分析器 — 调用 DeepSeek 判断公告是否为中国电信可承接项目"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.api_key = settings.ai_api_key
|
||||||
|
self.base_url = settings.ai_base_url.rstrip("/")
|
||||||
|
self.model = settings.ai_model
|
||||||
|
self.timeout = settings.ai_timeout
|
||||||
|
self.prompt_template = settings.ai_prompt_template
|
||||||
|
|
||||||
|
async def analyze(self, announcement: dict[str, Any]) -> AiResult:
|
||||||
|
"""分析单条公告"""
|
||||||
|
if not self.api_key:
|
||||||
|
return AiResult(error="AI_API_KEY 未配置")
|
||||||
|
|
||||||
|
# 1. 获取公告正文
|
||||||
|
content_url = announcement.get("content_url", "")
|
||||||
|
content = None
|
||||||
|
content_snippet = None
|
||||||
|
if content_url:
|
||||||
|
content = await extract_page_content(content_url, self.timeout)
|
||||||
|
|
||||||
|
if content:
|
||||||
|
content_snippet = content[:200]
|
||||||
|
else:
|
||||||
|
logger.warning("无法获取公告正文: %s", content_url)
|
||||||
|
|
||||||
|
# 2. 构建 prompt
|
||||||
|
prompt = self.prompt_template.format(
|
||||||
|
title=announcement.get("title", ""),
|
||||||
|
purchase_name=announcement.get("purchase_name", ""),
|
||||||
|
announcement_type=announcement.get("announcement_type", ""),
|
||||||
|
content=content or "(无法获取正文,请仅根据标题和采购人信息判断)",
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3. 调用 DeepSeek API
|
||||||
|
try:
|
||||||
|
result = await self._call_deepseek(prompt)
|
||||||
|
if result.error:
|
||||||
|
return AiResult(error=result.error, content_snippet=content_snippet)
|
||||||
|
return AiResult(
|
||||||
|
is_relevant=result.is_relevant,
|
||||||
|
reason=result.reason,
|
||||||
|
business_type=result.business_type,
|
||||||
|
content_snippet=content_snippet,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("AI 分析异常")
|
||||||
|
return AiResult(error=str(e), content_snippet=content_snippet)
|
||||||
|
|
||||||
|
async def analyze_batch(
|
||||||
|
self, announcements: list[dict[str, Any]], max_concurrent: int = 3
|
||||||
|
) -> list[AiResult]:
|
||||||
|
"""批量分析,控制并发数"""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
sem = asyncio.Semaphore(max_concurrent)
|
||||||
|
|
||||||
|
async def _limited(ann: dict[str, Any]) -> AiResult:
|
||||||
|
async with sem:
|
||||||
|
return await self.analyze(ann)
|
||||||
|
|
||||||
|
tasks = [_limited(ann) for ann in announcements]
|
||||||
|
return await asyncio.gather(*tasks)
|
||||||
|
|
||||||
|
async def _call_deepseek(self, prompt: str) -> AiResult:
|
||||||
|
"""调用 DeepSeek Chat API"""
|
||||||
|
url = f"{self.base_url}/chat/completions"
|
||||||
|
headers = {
|
||||||
|
"Authorization": f"Bearer {self.api_key}",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
}
|
||||||
|
payload = {
|
||||||
|
"model": self.model,
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "你是一个专业的政府采购项目分析师。请根据公告信息判断是否为中国电信可以承接的项目,并用 JSON 格式回答。",
|
||||||
|
},
|
||||||
|
{"role": "user", "content": prompt},
|
||||||
|
],
|
||||||
|
"temperature": 0.3, # 低温度,提高判断一致性
|
||||||
|
"max_tokens": 512,
|
||||||
|
}
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||||
|
response = await client.post(url, headers=headers, json=payload)
|
||||||
|
if response.status_code != 200:
|
||||||
|
return AiResult(
|
||||||
|
error=f"API 请求失败 (HTTP {response.status_code}): {response.text[:200]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
choices = data.get("choices", [])
|
||||||
|
if not choices:
|
||||||
|
return AiResult(error="API 返回空 choices")
|
||||||
|
|
||||||
|
content = choices[0].get("message", {}).get("content", "")
|
||||||
|
return self._parse_response(content)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_response(content: str) -> AiResult:
|
||||||
|
"""从 LLM 回复中提取 JSON 结果"""
|
||||||
|
# 清理可能的 markdown 代码块标记
|
||||||
|
content = content.strip()
|
||||||
|
if content.startswith("```"):
|
||||||
|
# 移除 ```json 或 ``` 包裹
|
||||||
|
lines = content.split("\n")
|
||||||
|
if lines[0].strip().startswith("```"):
|
||||||
|
lines = lines[1:]
|
||||||
|
if lines and lines[-1].strip() == "```":
|
||||||
|
lines = lines[:-1]
|
||||||
|
content = "\n".join(lines).strip()
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = json.loads(content)
|
||||||
|
return AiResult(
|
||||||
|
is_relevant=bool(result.get("is_relevant", False)),
|
||||||
|
reason=str(result.get("reason", "")),
|
||||||
|
business_type=str(result.get("business_type", "")),
|
||||||
|
)
|
||||||
|
except (json.JSONDecodeError, ValueError) as e:
|
||||||
|
logger.warning("JSON 解析失败: %s\n原始内容: %s", e, content[:200])
|
||||||
|
return AiResult(error=f"JSON 解析失败: {e}")
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
"""AI 运行时状态 — 支持企微菜单动态开关"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 运行时覆盖值,None 表示使用 settings.ai_enabled
|
||||||
|
_runtime_override: bool | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def is_ai_enabled() -> bool:
|
||||||
|
"""获取 AI 分析当前是否启用(考虑运行时覆盖)"""
|
||||||
|
if _runtime_override is not None:
|
||||||
|
return _runtime_override
|
||||||
|
return settings.ai_enabled
|
||||||
|
|
||||||
|
|
||||||
|
def set_ai_enabled(enabled: bool) -> bool:
|
||||||
|
"""设置 AI 分析运行时开关,返回是否真的发生了变化"""
|
||||||
|
global _runtime_override
|
||||||
|
current = is_ai_enabled()
|
||||||
|
if enabled == current:
|
||||||
|
return False
|
||||||
|
_runtime_override = enabled
|
||||||
|
status = "启用" if enabled else "禁用"
|
||||||
|
logger.info("AI 分析已通过企微菜单%s", status)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def get_ai_status_text() -> str:
|
||||||
|
"""获取 AI 状态文本"""
|
||||||
|
return "已启用" if is_ai_enabled() else "已禁用"
|
||||||
|
|
||||||
|
|
||||||
|
def get_whitelist() -> list[str]:
|
||||||
|
"""获取 AI 白名单用户列表"""
|
||||||
|
raw = settings.ai_whitelist
|
||||||
|
if not raw:
|
||||||
|
return []
|
||||||
|
return [u.strip() for u in raw.split(",") if u.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def is_whitelisted(user_id: str) -> bool:
|
||||||
|
"""检查用户是否在 AI 白名单中(不区分大小写)"""
|
||||||
|
whitelist = get_whitelist()
|
||||||
|
if not whitelist:
|
||||||
|
# 白名单为空则所有人都可以操作
|
||||||
|
return True
|
||||||
|
return user_id.lower() in [u.lower() for u in whitelist]
|
||||||
@@ -1,9 +1,19 @@
|
|||||||
|
import logging
|
||||||
|
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||||
|
|
||||||
from app.crawler.base import BaseSpider, CrawlResult
|
from app.crawler.base import BaseSpider, CrawlResult
|
||||||
|
from app.services.notification_service import NotificationService
|
||||||
|
from app.services.pipeline import PostCrawlPipeline
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class CrawlService:
|
class CrawlService:
|
||||||
def __init__(self):
|
def __init__(self, db_session_factory: async_sessionmaker,
|
||||||
|
notification_service: NotificationService):
|
||||||
|
self.db_session_factory = db_session_factory
|
||||||
|
self.notification_service = notification_service
|
||||||
self.spiders: dict[str, BaseSpider] = {}
|
self.spiders: dict[str, BaseSpider] = {}
|
||||||
|
|
||||||
def register(self, spider: BaseSpider):
|
def register(self, spider: BaseSpider):
|
||||||
@@ -11,19 +21,47 @@ class CrawlService:
|
|||||||
|
|
||||||
async def run_all(self) -> list[CrawlResult]:
|
async def run_all(self) -> list[CrawlResult]:
|
||||||
results = []
|
results = []
|
||||||
for name, spider in self.spiders.items():
|
for name in self.spiders:
|
||||||
result = await spider.crawl()
|
spider_results = await self.run_spider(name)
|
||||||
results.append(result)
|
results.extend(spider_results)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
async def run_spider(self, name: str, **kwargs) -> list[CrawlResult]:
|
async def run_spider(self, name: str) -> list[CrawlResult]:
|
||||||
spider = self.spiders.get(name)
|
spider = self.spiders.get(name)
|
||||||
if spider is None:
|
if spider is None:
|
||||||
return [CrawlResult(
|
return [CrawlResult(
|
||||||
source_code=name, source_name=name,
|
source_code=name, source_name=name,
|
||||||
error_message=f"Spider not found: {name}"
|
error_message=f"Spider not found: {name}",
|
||||||
)]
|
)]
|
||||||
result = await spider.crawl(**kwargs)
|
|
||||||
|
# 1. Crawl
|
||||||
|
result = await spider.crawl()
|
||||||
|
|
||||||
|
# 2. Pipeline: store → filter → notify
|
||||||
|
if result.success and result.announcements:
|
||||||
|
try:
|
||||||
|
config = spider.get_pipeline_config()
|
||||||
|
async with self.db_session_factory() as db:
|
||||||
|
pipeline = PostCrawlPipeline(
|
||||||
|
db_session=db,
|
||||||
|
notification_service=self.notification_service,
|
||||||
|
)
|
||||||
|
pipe_result = await pipeline.process(
|
||||||
|
result.announcements, config,
|
||||||
|
)
|
||||||
|
result.pipeline_result = pipe_result
|
||||||
|
result.new_count = pipe_result.stored
|
||||||
|
logger.info(
|
||||||
|
f"Spider {name}: total={result.total_count}, "
|
||||||
|
f"stored={pipe_result.stored}, "
|
||||||
|
f"filtered={pipe_result.filtered}, "
|
||||||
|
f"notified={pipe_result.notified}"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Spider {name} pipeline 异常: {e}")
|
||||||
|
if not result.error_message:
|
||||||
|
result.error_message = str(e)
|
||||||
|
|
||||||
return [result]
|
return [result]
|
||||||
|
|
||||||
def get_spider_names(self) -> list[str]:
|
def get_spider_names(self) -> list[str]:
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import logging
|
||||||
|
from datetime import date
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.holiday import ChineseHoliday
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
HOLIDAY_API = "http://timor.tech/api/holiday/year"
|
||||||
|
|
||||||
|
|
||||||
|
def now_in_china() -> date:
|
||||||
|
from datetime import datetime
|
||||||
|
from zoneinfo import ZoneInfo
|
||||||
|
return datetime.now(ZoneInfo("Asia/Shanghai")).date()
|
||||||
|
|
||||||
|
|
||||||
|
async def sync_holidays(db: AsyncSession, year: int) -> int:
|
||||||
|
"""同步中国节假日数据,返回更新的记录数"""
|
||||||
|
url = f"{HOLIDAY_API}/{year}"
|
||||||
|
async with httpx.AsyncClient(timeout=15) as client:
|
||||||
|
response = await client.get(url)
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
if data.get("code") != 0:
|
||||||
|
logger.error(f"节假日 API 返回错误: {data}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
holidays = data.get("holiday", {})
|
||||||
|
if not holidays:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# 也标记周末(周六日但非调休工作日)
|
||||||
|
from datetime import timedelta
|
||||||
|
current = date(year, 1, 1)
|
||||||
|
end = date(year, 12, 31)
|
||||||
|
|
||||||
|
records: dict[date, dict] = {}
|
||||||
|
while current <= end:
|
||||||
|
dow = current.weekday() # 0=Mon, 6=Sun
|
||||||
|
# 默认:周一~五为工作日,周六日为非工作日
|
||||||
|
default_workday = dow < 5
|
||||||
|
records[current] = {
|
||||||
|
"date": current,
|
||||||
|
"is_workday": default_workday,
|
||||||
|
"year": year,
|
||||||
|
"description": "",
|
||||||
|
}
|
||||||
|
current += timedelta(days=1)
|
||||||
|
|
||||||
|
# 覆盖节假日数据
|
||||||
|
# holiday=true → 休息日(无论 wage 值)
|
||||||
|
# holiday=false → 调休补班日(周末也要上班)
|
||||||
|
for date_str, info_str in holidays.items():
|
||||||
|
d = date.fromisoformat(f"{year}-{date_str}")
|
||||||
|
if d.year != year:
|
||||||
|
continue
|
||||||
|
info = info_str if isinstance(info_str, dict) else {}
|
||||||
|
holiday = info.get("holiday", False)
|
||||||
|
name = info.get("name", "")
|
||||||
|
|
||||||
|
records[d]["description"] = name
|
||||||
|
records[d]["is_workday"] = not holiday # holiday=false → 调休工作日
|
||||||
|
|
||||||
|
# Upsert
|
||||||
|
values = list(records.values())
|
||||||
|
stmt = pg_insert(ChineseHoliday).values(values)
|
||||||
|
stmt = stmt.on_conflict_do_update(
|
||||||
|
index_elements=["date"],
|
||||||
|
set_={"is_workday": stmt.excluded.is_workday,
|
||||||
|
"description": stmt.excluded.description},
|
||||||
|
)
|
||||||
|
result = await db.execute(stmt)
|
||||||
|
await db.commit()
|
||||||
|
logger.info(f"已同步 {year} 年节假日,{len(values)} 天")
|
||||||
|
return result.rowcount
|
||||||
|
|
||||||
|
|
||||||
|
async def is_workday(db: AsyncSession, day: date | None = None) -> bool:
|
||||||
|
"""判断某天是否为工作日"""
|
||||||
|
if day is None:
|
||||||
|
day = now_in_china()
|
||||||
|
from sqlalchemy import select
|
||||||
|
result = await db.execute(
|
||||||
|
select(ChineseHoliday.is_workday).where(ChineseHoliday.date == day)
|
||||||
|
)
|
||||||
|
row = result.fetchone()
|
||||||
|
if row is None:
|
||||||
|
# 无数据时,按周判断
|
||||||
|
return day.weekday() < 5
|
||||||
|
return row[0]
|
||||||
|
|
||||||
|
|
||||||
|
async def was_yesterday_workday(db: AsyncSession) -> bool:
|
||||||
|
"""昨天是工作日吗"""
|
||||||
|
from datetime import timedelta
|
||||||
|
yesterday = now_in_china() - timedelta(days=1)
|
||||||
|
return await is_workday(db, yesterday)
|
||||||
@@ -17,6 +17,7 @@ class NotificationService:
|
|||||||
sent = 0
|
sent = 0
|
||||||
for ann in announcements:
|
for ann in announcements:
|
||||||
try:
|
try:
|
||||||
|
# 发送普通 textcard
|
||||||
title = ann.get("title", "")
|
title = ann.get("title", "")
|
||||||
if len(title) > 128:
|
if len(title) > 128:
|
||||||
title = title[:125] + "..."
|
title = title[:125] + "..."
|
||||||
@@ -30,17 +31,48 @@ class NotificationService:
|
|||||||
|
|
||||||
source_name = ann.get("source_name", "")
|
source_name = ann.get("source_name", "")
|
||||||
|
|
||||||
description = (
|
description = f"{source_name} | {purchase_name} | {time_str}"
|
||||||
f'<div style="font-size: 14px; margin-top: 8px;">'
|
|
||||||
f'{source_name} | {purchase_name} | {time_str}'
|
|
||||||
f'</div>'
|
|
||||||
)
|
|
||||||
|
|
||||||
url = ann.get("content_url", "")
|
url = ann.get("content_url", "")
|
||||||
|
|
||||||
if await self.client.send_textcard(title, description, url):
|
if await self.client.send_textcard(title, description, url):
|
||||||
sent += 1
|
sent += 1
|
||||||
|
|
||||||
|
# AI 标记为可承接的,额外发送着重通知
|
||||||
|
ai_result = ann.get("ai_result")
|
||||||
|
if ai_result and ai_result.get("is_relevant"):
|
||||||
|
await self._send_ai_emphasis(ann, ai_result)
|
||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
return sent
|
return sent
|
||||||
|
|
||||||
|
async def _send_ai_emphasis(
|
||||||
|
self, ann: dict[str, Any], ai_result: dict[str, Any]
|
||||||
|
) -> bool:
|
||||||
|
"""发送 AI 分析的着重通知(markdown 格式)"""
|
||||||
|
title = ann.get("title", "")
|
||||||
|
purchase_name = ann.get("purchase_name", "")
|
||||||
|
pub_date = ann.get("publish_date")
|
||||||
|
time_str = pub_date.strftime("%Y-%m-%d %H:%M") if pub_date else "时间未知"
|
||||||
|
url = ann.get("content_url", "")
|
||||||
|
|
||||||
|
reason = ai_result.get("reason", "")
|
||||||
|
business_type = ai_result.get("business_type", "")
|
||||||
|
|
||||||
|
# 企业微信 markdown 格式
|
||||||
|
md = (
|
||||||
|
f"{settings.ai_analysis_title}\n"
|
||||||
|
f"---\n"
|
||||||
|
f"**标题:** [{title}]({url})\n"
|
||||||
|
f"> 采购人:{purchase_name}\n"
|
||||||
|
f"> 发布时间:{time_str}\n\n"
|
||||||
|
f"**🤖 AI 分析:**\n"
|
||||||
|
f"> {reason}\n\n"
|
||||||
|
f"**🏷 业务分类:** {business_type}\n"
|
||||||
|
f"---\n"
|
||||||
|
f"[📄 查看公告原文]({url})"
|
||||||
|
)
|
||||||
|
|
||||||
|
return await self.client.send_markdown(md)
|
||||||
|
|||||||
@@ -1,8 +1,13 @@
|
|||||||
|
import logging
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
from app.crawler.base import PipelineConfig, PipelineResult
|
from app.crawler.base import PipelineConfig, PipelineResult
|
||||||
|
from app.services.ai_state import is_ai_enabled
|
||||||
from app.services.filter_service import dedup_by_hash
|
from app.services.filter_service import dedup_by_hash
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class PostCrawlPipeline:
|
class PostCrawlPipeline:
|
||||||
def __init__(self, db_session, notification_service):
|
def __init__(self, db_session, notification_service):
|
||||||
@@ -20,20 +25,32 @@ class PostCrawlPipeline:
|
|||||||
if config.dedup_enabled:
|
if config.dedup_enabled:
|
||||||
announcements = dedup_by_hash(announcements)
|
announcements = dedup_by_hash(announcements)
|
||||||
|
|
||||||
# 2. Store to database
|
# 2. 先标记 keyword_matched,再存库
|
||||||
|
if config.filter_enabled and config.keywords:
|
||||||
|
for a in announcements:
|
||||||
|
a["keyword_matched"] = self._match_keywords(a, config.keywords)
|
||||||
|
|
||||||
|
# 3. Store to database
|
||||||
stored = await self._save_to_db(announcements)
|
stored = await self._save_to_db(announcements)
|
||||||
result.stored = stored
|
result.stored = stored
|
||||||
|
|
||||||
to_notify = announcements
|
to_notify = announcements
|
||||||
|
|
||||||
# 3. Filter
|
# 4. Filter
|
||||||
if config.filter_enabled and config.keywords:
|
if config.filter_enabled and config.keywords:
|
||||||
before = len(to_notify)
|
before = len(to_notify)
|
||||||
to_notify = [a for a in to_notify
|
to_notify = [a for a in to_notify if a.get("keyword_matched")]
|
||||||
if self._match_keywords(a, config.keywords)]
|
|
||||||
result.filtered = before - len(to_notify)
|
result.filtered = before - len(to_notify)
|
||||||
|
|
||||||
# 4. Notify
|
# 5. Skip already-notified
|
||||||
|
if to_notify:
|
||||||
|
to_notify = await self._exclude_sent(to_notify)
|
||||||
|
|
||||||
|
# 5.5 AI 分析(可选,支持运行时开关)
|
||||||
|
if is_ai_enabled() and to_notify:
|
||||||
|
await self._ai_analyze(to_notify)
|
||||||
|
|
||||||
|
# 6. Notify
|
||||||
if config.notify_mode == "all":
|
if config.notify_mode == "all":
|
||||||
result.notified = await self._send_notifications(to_notify)
|
result.notified = await self._send_notifications(to_notify)
|
||||||
elif config.notify_mode == "filtered":
|
elif config.notify_mode == "filtered":
|
||||||
@@ -42,7 +59,7 @@ class PostCrawlPipeline:
|
|||||||
elif not config.filter_enabled:
|
elif not config.filter_enabled:
|
||||||
result.notified = await self._send_notifications(to_notify)
|
result.notified = await self._send_notifications(to_notify)
|
||||||
|
|
||||||
# 5. Mark sent
|
# 7. Mark sent
|
||||||
if config.mark_sent and result.notified > 0:
|
if config.mark_sent and result.notified > 0:
|
||||||
await self._mark_sent(to_notify)
|
await self._mark_sent(to_notify)
|
||||||
|
|
||||||
@@ -99,7 +116,74 @@ class PostCrawlPipeline:
|
|||||||
await self.db.commit()
|
await self.db.commit()
|
||||||
return result.rowcount
|
return result.rowcount
|
||||||
|
|
||||||
|
async def _exclude_sent(self, announcements: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.models.announcement import Announcement
|
||||||
|
|
||||||
|
hashes = [a["content_hash"] for a in announcements if a.get("content_hash")]
|
||||||
|
if not hashes:
|
||||||
|
return announcements
|
||||||
|
|
||||||
|
stmt = select(Announcement.content_hash).where(
|
||||||
|
Announcement.content_hash.in_(hashes),
|
||||||
|
Announcement.is_sent == True, # noqa: E712
|
||||||
|
)
|
||||||
|
result = await self.db.execute(stmt)
|
||||||
|
sent_hashes = {row[0] for row in result.fetchall()}
|
||||||
|
|
||||||
|
return [a for a in announcements if a.get("content_hash") not in sent_hashes]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool:
|
def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool:
|
||||||
text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}"
|
text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}"
|
||||||
return any(kw in text for kw in keywords)
|
return any(kw in text for kw in keywords)
|
||||||
|
|
||||||
|
async def _ai_analyze(self, announcements: list[dict[str, Any]]) -> None:
|
||||||
|
"""对公告列表执行 AI 分析,将结果附加到每条公告的 ai_result 字段,并更新数据库"""
|
||||||
|
from app.services.ai_analyzer import AiAnalyzer
|
||||||
|
|
||||||
|
analyzer = AiAnalyzer()
|
||||||
|
logger.info("AI 分析开始:共 %d 条公告", len(announcements))
|
||||||
|
|
||||||
|
results = await analyzer.analyze_batch(announcements)
|
||||||
|
|
||||||
|
ai_updates = []
|
||||||
|
for ann, ai_result in zip(announcements, results):
|
||||||
|
ann["ai_result"] = {
|
||||||
|
"is_relevant": ai_result.is_relevant,
|
||||||
|
"reason": ai_result.reason,
|
||||||
|
"business_type": ai_result.business_type,
|
||||||
|
}
|
||||||
|
if ai_result.error:
|
||||||
|
logger.warning("AI 分析失败 [%s]: %s", ann.get("title", "")[:30], ai_result.error)
|
||||||
|
else:
|
||||||
|
ai_updates.append({
|
||||||
|
"content_hash": ann["content_hash"],
|
||||||
|
"ai_relevant": ai_result.is_relevant,
|
||||||
|
"ai_analysis": ai_result.reason,
|
||||||
|
})
|
||||||
|
if ai_result.is_relevant:
|
||||||
|
logger.info("AI 标记可承接项目: %s (%s)", ann.get("title", "")[:40], ai_result.business_type)
|
||||||
|
|
||||||
|
# 批量更新数据库中的 AI 分析结果
|
||||||
|
if ai_updates:
|
||||||
|
await self._update_ai_results(ai_updates)
|
||||||
|
|
||||||
|
async def _update_ai_results(self, updates: list[dict[str, Any]]) -> None:
|
||||||
|
"""批量更新公告的 AI 分析结果到数据库"""
|
||||||
|
from sqlalchemy import update
|
||||||
|
|
||||||
|
from app.models.announcement import Announcement
|
||||||
|
|
||||||
|
for u in updates:
|
||||||
|
stmt = (
|
||||||
|
update(Announcement)
|
||||||
|
.where(Announcement.content_hash == u["content_hash"])
|
||||||
|
.values(
|
||||||
|
ai_relevant=u["ai_relevant"],
|
||||||
|
ai_analysis=u["ai_analysis"],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await self.db.execute(stmt)
|
||||||
|
await self.db.commit()
|
||||||
|
|||||||
+15
-3
@@ -1,9 +1,12 @@
|
|||||||
|
import logging
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class WeChatClient:
|
class WeChatClient:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
@@ -15,7 +18,7 @@ class WeChatClient:
|
|||||||
if self._access_token and now < self._token_expires_at:
|
if self._access_token and now < self._token_expires_at:
|
||||||
return self._access_token
|
return self._access_token
|
||||||
|
|
||||||
url = "https://qyapi.weixin.qq.com/cgi-bin/gettoken"
|
url = f"{settings.wechat_api_base_url}/cgi-bin/gettoken"
|
||||||
params = {
|
params = {
|
||||||
"corpid": settings.wechat_corp_id,
|
"corpid": settings.wechat_corp_id,
|
||||||
"corpsecret": settings.wechat_secret,
|
"corpsecret": settings.wechat_secret,
|
||||||
@@ -27,6 +30,7 @@ class WeChatClient:
|
|||||||
self._access_token = data["access_token"]
|
self._access_token = data["access_token"]
|
||||||
self._token_expires_at = now + data.get("expires_in", 7200) - 300
|
self._token_expires_at = now + data.get("expires_in", 7200) - 300
|
||||||
return self._access_token
|
return self._access_token
|
||||||
|
logger.error(f"获取 access_token 失败: {data}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def send_text(self, content: str, to_user: str = "@all") -> bool:
|
async def send_text(self, content: str, to_user: str = "@all") -> bool:
|
||||||
@@ -59,9 +63,10 @@ class WeChatClient:
|
|||||||
) -> bool:
|
) -> bool:
|
||||||
token = await self._get_access_token()
|
token = await self._get_access_token()
|
||||||
if not token:
|
if not token:
|
||||||
|
logger.error("无法获取 access_token,跳过消息发送")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
url = "https://qyapi.weixin.qq.com/cgi-bin/message/send"
|
url = f"{settings.wechat_api_base_url}/cgi-bin/message/send"
|
||||||
params = {"access_token": token}
|
params = {"access_token": token}
|
||||||
body = {
|
body = {
|
||||||
"touser": to_user,
|
"touser": to_user,
|
||||||
@@ -73,4 +78,11 @@ class WeChatClient:
|
|||||||
async with httpx.AsyncClient(timeout=30) as client:
|
async with httpx.AsyncClient(timeout=30) as client:
|
||||||
response = await client.post(url, params=params, json=body)
|
response = await client.post(url, params=params, json=body)
|
||||||
data = response.json()
|
data = response.json()
|
||||||
return data.get("errcode") == 0
|
errcode = data.get("errcode")
|
||||||
|
if errcode == 0:
|
||||||
|
return True
|
||||||
|
logger.error(
|
||||||
|
f"企业微信消息发送失败: errcode={errcode} errmsg={data.get('errmsg')} "
|
||||||
|
f"msgtype={msgtype} touser={to_user}"
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|||||||
+332
-2
@@ -1,7 +1,21 @@
|
|||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.wechat.crypto import WXBizMsgCrypt
|
from app.wechat.crypto import WXBizMsgCrypt
|
||||||
|
from app.wechat.client import WeChatClient
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 防重入:记录最近一次触发爬取的时间戳,60秒内不重复执行
|
||||||
|
_last_crawl_time: float = 0.0
|
||||||
|
_crawl_lock = asyncio.Lock()
|
||||||
|
|
||||||
|
# 防重入:最新公告爬取,60秒内不重复执行
|
||||||
|
_last_latest_time: float = 0.0
|
||||||
|
_latest_lock = asyncio.Lock()
|
||||||
|
|
||||||
|
|
||||||
class WeChatMessageHandler:
|
class WeChatMessageHandler:
|
||||||
@@ -11,6 +25,7 @@ class WeChatMessageHandler:
|
|||||||
sEncodingAESKey=settings.wechat_encoding_aes_key,
|
sEncodingAESKey=settings.wechat_encoding_aes_key,
|
||||||
sReceiveId=settings.wechat_corp_id,
|
sReceiveId=settings.wechat_corp_id,
|
||||||
)
|
)
|
||||||
|
self.client = WeChatClient()
|
||||||
|
|
||||||
def verify_url(
|
def verify_url(
|
||||||
self, msg_signature: str, timestamp: str, nonce: str, echostr: str
|
self, msg_signature: str, timestamp: str, nonce: str, echostr: str
|
||||||
@@ -48,10 +63,325 @@ class WeChatMessageHandler:
|
|||||||
return encrypted
|
return encrypted
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def handle_event(
|
async def handle_event(
|
||||||
self, event: str, event_key: str | None, from_user: str
|
self, event: str, event_key: str | None, from_user: str
|
||||||
) -> str | None:
|
) -> str | None:
|
||||||
|
if event != "click" or not event_key:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def handle_text(self, content: str, from_user: str) -> str | None:
|
if event_key == "today_stats":
|
||||||
|
return await self._handle_today_stats(from_user)
|
||||||
|
elif event_key == "trigger_crawl":
|
||||||
|
return await self._handle_trigger_crawl(from_user)
|
||||||
|
elif event_key == "sync_holidays":
|
||||||
|
return await self._handle_sync_holidays(from_user)
|
||||||
|
elif event_key == "monitor_config":
|
||||||
|
return await self._handle_monitor_config(from_user)
|
||||||
|
elif event_key == "system_status":
|
||||||
|
return await self._handle_system_status(from_user)
|
||||||
|
elif event_key == "workday_status":
|
||||||
|
return await self._handle_workday_status(from_user)
|
||||||
|
elif event_key == "latest_announcements":
|
||||||
|
return await self._handle_latest_announcements(from_user)
|
||||||
|
elif event_key == "pause_scheduler":
|
||||||
|
return await self._handle_pause_scheduler(from_user)
|
||||||
|
elif event_key == "resume_scheduler":
|
||||||
|
return await self._handle_resume_scheduler(from_user)
|
||||||
|
elif event_key == "toggle_ai":
|
||||||
|
return await self._handle_toggle_ai(from_user)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def handle_text(self, content: str, from_user: str) -> str | None:
|
||||||
|
"""文本消息:返回帮助提示"""
|
||||||
|
help_text = (
|
||||||
|
"请使用菜单操作:\n"
|
||||||
|
"---\n"
|
||||||
|
"📋 最新公告 - 获取最新公告\n"
|
||||||
|
"📊 查询 → 监控配置/系统状态/今日工作日\n"
|
||||||
|
"⚙️ 系统管理 → 立即爬取/同步节假日/AI 分析"
|
||||||
|
)
|
||||||
|
ok = await self.client.send_text(help_text, from_user)
|
||||||
|
if not ok:
|
||||||
|
logger.warning(f"发送帮助消息失败, touser={from_user}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _handle_today_stats(self, from_user: str) -> str | None:
|
||||||
|
from app.api.deps import get_db
|
||||||
|
|
||||||
|
try:
|
||||||
|
async for db in get_db():
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from app.models.announcement import Announcement
|
||||||
|
|
||||||
|
total_result = await db.execute(
|
||||||
|
select(func.count()).select_from(Announcement)
|
||||||
|
)
|
||||||
|
total = total_result.scalar() or 0
|
||||||
|
|
||||||
|
today_result = await db.execute(
|
||||||
|
select(func.count()).where(
|
||||||
|
func.date(Announcement.publish_date) == func.current_date()
|
||||||
|
).select_from(Announcement)
|
||||||
|
)
|
||||||
|
today = today_result.scalar() or 0
|
||||||
|
|
||||||
|
text = f"今日新增: {today} 条\n累计公告: {total} 条"
|
||||||
|
await self.client.send_text(text, from_user)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"查询统计失败: {e}")
|
||||||
|
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||||
|
|
||||||
|
async def _handle_trigger_crawl(self, from_user: str) -> str | None:
|
||||||
|
global _last_crawl_time
|
||||||
|
from app.api.deps import get_crawl_service
|
||||||
|
|
||||||
|
# 防重入:企业微信会对同一事件重试多次,60秒内只执行一次
|
||||||
|
async with _crawl_lock:
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - _last_crawl_time < 60:
|
||||||
|
remaining = int(60 - (now - _last_crawl_time))
|
||||||
|
logger.info(f"爬取请求被忽略(防重入),距上次 {now - _last_crawl_time:.1f}s")
|
||||||
|
await self.client.send_text(
|
||||||
|
f"爬取任务进行中,请 {remaining} 秒后再试", from_user
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
_last_crawl_time = now
|
||||||
|
|
||||||
|
await self.client.send_text("开始爬取,请稍候...", from_user)
|
||||||
|
|
||||||
|
try:
|
||||||
|
service = get_crawl_service()
|
||||||
|
results = await service.run_all()
|
||||||
|
total = sum(r.total_count for r in results)
|
||||||
|
stored = sum(
|
||||||
|
r.pipeline_result.stored for r in results
|
||||||
|
if r.pipeline_result
|
||||||
|
)
|
||||||
|
notified = sum(
|
||||||
|
r.pipeline_result.notified for r in results
|
||||||
|
if r.pipeline_result
|
||||||
|
)
|
||||||
|
errors = [r.error_message for r in results if not r.success]
|
||||||
|
msg = f"爬取完成\n抓取: {total} 条\n新增: {stored} 条\n推送: {notified} 条"
|
||||||
|
if errors:
|
||||||
|
msg += f"\n异常: {errors[0][:50]}"
|
||||||
|
await self.client.send_text(msg, from_user)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"手动爬取失败: {e}")
|
||||||
|
await self.client.send_text(f"爬取失败: {e}", from_user)
|
||||||
|
|
||||||
|
async def _handle_sync_holidays(self, from_user: str) -> str | None:
|
||||||
|
from app.services.holiday_service import now_in_china, sync_holidays
|
||||||
|
from app.api.deps import get_db
|
||||||
|
|
||||||
|
try:
|
||||||
|
async for db in get_db():
|
||||||
|
year = now_in_china().year
|
||||||
|
count = await sync_holidays(db, year)
|
||||||
|
await self.client.send_text(
|
||||||
|
f"已同步 {year} 年节假日\n共 {count} 条记录", from_user,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"同步节假日失败: {e}")
|
||||||
|
await self.client.send_text(f"同步失败: {e}", from_user)
|
||||||
|
|
||||||
|
async def _handle_monitor_config(self, from_user: str) -> str | None:
|
||||||
|
import json
|
||||||
|
from app.services.ai_state import get_ai_status_text
|
||||||
|
try:
|
||||||
|
keywords = settings.crawler_keywords
|
||||||
|
sources = json.loads(settings.announcement_sources)
|
||||||
|
source_names = "、".join(v["name"] for v in sources.values())
|
||||||
|
text = (
|
||||||
|
f"📋 监控配置\n"
|
||||||
|
f"---\n"
|
||||||
|
f"监控关键词: {', '.join(keywords)}\n"
|
||||||
|
f"爬取页数: {settings.crawler_max_pages} 页\n"
|
||||||
|
f"定时规则: {settings.scheduler_cron}\n"
|
||||||
|
f"公告来源: {source_names}\n"
|
||||||
|
f"---\n"
|
||||||
|
f"🤖 AI 分析: {get_ai_status_text()}\n"
|
||||||
|
f"AI 模型: {settings.ai_model}\n"
|
||||||
|
f"重点标记: {settings.ai_analysis_title}"
|
||||||
|
)
|
||||||
|
await self.client.send_text(text, from_user)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"查询监控配置失败: {e}")
|
||||||
|
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||||
|
|
||||||
|
async def _handle_system_status(self, from_user: str) -> str | None:
|
||||||
|
from app.api.deps import get_db
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from app.models.announcement import Announcement
|
||||||
|
from app.services.ai_state import get_ai_status_text
|
||||||
|
|
||||||
|
try:
|
||||||
|
async for db in get_db():
|
||||||
|
total_result = await db.execute(
|
||||||
|
select(func.count()).select_from(Announcement)
|
||||||
|
)
|
||||||
|
total = total_result.scalar() or 0
|
||||||
|
|
||||||
|
today_result = await db.execute(
|
||||||
|
select(func.count()).where(
|
||||||
|
func.date(Announcement.publish_date) == func.current_date()
|
||||||
|
).select_from(Announcement)
|
||||||
|
)
|
||||||
|
today = today_result.scalar() or 0
|
||||||
|
|
||||||
|
unsent_result = await db.execute(
|
||||||
|
select(func.count()).where(
|
||||||
|
Announcement.is_sent == False, # noqa: E712
|
||||||
|
Announcement.keyword_matched == True, # noqa: E712
|
||||||
|
).select_from(Announcement)
|
||||||
|
)
|
||||||
|
unsent = unsent_result.scalar() or 0
|
||||||
|
|
||||||
|
# AI 标记统计
|
||||||
|
ai_relevant_result = await db.execute(
|
||||||
|
select(func.count()).where(
|
||||||
|
Announcement.ai_relevant == True, # noqa: E712
|
||||||
|
).select_from(Announcement)
|
||||||
|
)
|
||||||
|
ai_relevant = ai_relevant_result.scalar() or 0
|
||||||
|
|
||||||
|
scheduler_status = "已启用" if settings.scheduler_enabled else "已禁用"
|
||||||
|
text = (
|
||||||
|
f"📊 系统状态\n"
|
||||||
|
f"---\n"
|
||||||
|
f"累计公告: {total} 条\n"
|
||||||
|
f"今日新增: {today} 条\n"
|
||||||
|
f"待推送: {unsent} 条\n"
|
||||||
|
f"---\n"
|
||||||
|
f"定时任务: {scheduler_status}\n"
|
||||||
|
f"定时规则: {settings.scheduler_cron}\n"
|
||||||
|
f"---\n"
|
||||||
|
f"🤖 AI 分析: {get_ai_status_text()}\n"
|
||||||
|
f"AI 标记项目: {ai_relevant} 条"
|
||||||
|
)
|
||||||
|
await self.client.send_text(text, from_user)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"查询系统状态失败: {e}")
|
||||||
|
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||||
|
|
||||||
|
async def _handle_workday_status(self, from_user: str) -> str | None:
|
||||||
|
from app.services.holiday_service import now_in_china, is_workday
|
||||||
|
from app.api.deps import get_db
|
||||||
|
|
||||||
|
try:
|
||||||
|
async for db in get_db():
|
||||||
|
today = now_in_china()
|
||||||
|
workday = await is_workday(db, today)
|
||||||
|
status = "工作日,正常爬取" if workday else "非工作日,跳过爬取"
|
||||||
|
text = f"今天 {today.strftime('%Y-%m-%d %A')}\n{status}"
|
||||||
|
await self.client.send_text(text, from_user)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"查询工作日状态失败: {e}")
|
||||||
|
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||||
|
|
||||||
|
async def _handle_latest_announcements(self, from_user: str) -> str | None:
|
||||||
|
global _last_latest_time
|
||||||
|
from app.api.deps import get_db, get_crawl_service
|
||||||
|
from sqlalchemy import select, asc, or_
|
||||||
|
from app.models.announcement import Announcement
|
||||||
|
|
||||||
|
# 防重入:60秒内整个函数只执行一次(含推送)
|
||||||
|
async with _latest_lock:
|
||||||
|
now = time.monotonic()
|
||||||
|
if (now - _last_latest_time) < 60:
|
||||||
|
logger.info("最新公告请求被忽略(防重入)")
|
||||||
|
return None
|
||||||
|
_last_latest_time = now
|
||||||
|
|
||||||
|
try:
|
||||||
|
await self.client.send_text("正在获取最新公告,请稍候...", from_user)
|
||||||
|
service = get_crawl_service()
|
||||||
|
await service.run_all()
|
||||||
|
|
||||||
|
# 查:广西政采网关键词匹配 + 大化县政府网全部,取最新6条后按时间升序推送
|
||||||
|
async for db in get_db():
|
||||||
|
from sqlalchemy import select, desc, or_
|
||||||
|
result = await db.execute(
|
||||||
|
select(Announcement)
|
||||||
|
.where(or_(
|
||||||
|
Announcement.keyword_matched == True, # noqa: E712
|
||||||
|
Announcement.source_code == "dahuagov",
|
||||||
|
))
|
||||||
|
.order_by(desc(Announcement.publish_date))
|
||||||
|
.limit(6)
|
||||||
|
)
|
||||||
|
items = result.scalars().all()
|
||||||
|
|
||||||
|
if not items:
|
||||||
|
await self.client.send_text("暂无公告", from_user)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# 按时间升序推送,企微里向上滑即为时间正序
|
||||||
|
for ann in sorted(items, key=lambda a: a.publish_date):
|
||||||
|
title = ann.title if len(ann.title) <= 128 else ann.title[:125] + "..."
|
||||||
|
purchase_name = ann.purchase_name or ""
|
||||||
|
if len(purchase_name) > 25:
|
||||||
|
purchase_name = purchase_name[:22] + "..."
|
||||||
|
time_str = ann.publish_date.strftime("%Y-%m-%d %H:%M") if ann.publish_date else "时间未知"
|
||||||
|
source_name = ann.source_name or ann.source_code or ""
|
||||||
|
description = f"{source_name} | {purchase_name} | {time_str}"
|
||||||
|
await self.client.send_textcard(title, description, ann.content_url or "", from_user)
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"查询最新公告失败: {e}")
|
||||||
|
await self.client.send_text("查询失败,请稍后再试", from_user)
|
||||||
|
|
||||||
|
async def _handle_pause_scheduler(self, from_user: str) -> str | None:
|
||||||
|
try:
|
||||||
|
from app.scheduler.jobs import scheduler
|
||||||
|
if scheduler.running:
|
||||||
|
scheduler.pause()
|
||||||
|
await self.client.send_text("定时任务已暂停", from_user)
|
||||||
|
else:
|
||||||
|
await self.client.send_text("定时任务未在运行", from_user)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"暂停定时任务失败: {e}")
|
||||||
|
await self.client.send_text(f"操作失败: {e}", from_user)
|
||||||
|
|
||||||
|
async def _handle_toggle_ai(self, from_user: str) -> str | None:
|
||||||
|
from app.services.ai_state import (
|
||||||
|
get_ai_status_text,
|
||||||
|
get_whitelist,
|
||||||
|
is_ai_enabled,
|
||||||
|
is_whitelisted,
|
||||||
|
set_ai_enabled,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 白名单校验
|
||||||
|
whitelist = get_whitelist()
|
||||||
|
if whitelist and not is_whitelisted(from_user):
|
||||||
|
await self.client.send_text(
|
||||||
|
f"⚠️ 你没有权限操作 AI 分析开关\n"
|
||||||
|
f"当前仅以下用户可操作:\n{', '.join(whitelist)}",
|
||||||
|
from_user,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
current = is_ai_enabled()
|
||||||
|
changed = set_ai_enabled(not current)
|
||||||
|
if not changed:
|
||||||
|
await self.client.send_text(
|
||||||
|
f"AI 分析当前已是「{get_ai_status_text()}」状态,无需切换",
|
||||||
|
from_user,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
await self.client.send_text(
|
||||||
|
f"✅ AI 分析已切换为「{get_ai_status_text()}」\n"
|
||||||
|
f"下次爬取触发时生效",
|
||||||
|
from_user,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _handle_resume_scheduler(self, from_user: str) -> str | None:
|
||||||
|
try:
|
||||||
|
from app.scheduler.jobs import scheduler
|
||||||
|
scheduler.resume()
|
||||||
|
await self.client.send_text("定时任务已恢复", from_user)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"恢复定时任务失败: {e}")
|
||||||
|
await self.client.send_text(f"操作失败: {e}", from_user)
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
import logging
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
MENU = {
|
||||||
|
"button": [
|
||||||
|
{
|
||||||
|
"name": "最新公告",
|
||||||
|
"type": "click",
|
||||||
|
"key": "latest_announcements",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "查询",
|
||||||
|
"sub_button": [
|
||||||
|
{
|
||||||
|
"name": "监控配置",
|
||||||
|
"type": "click",
|
||||||
|
"key": "monitor_config",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "系统状态",
|
||||||
|
"type": "click",
|
||||||
|
"key": "system_status",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "今日工作日",
|
||||||
|
"type": "click",
|
||||||
|
"key": "workday_status",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "系统管理",
|
||||||
|
"sub_button": [
|
||||||
|
{
|
||||||
|
"name": "立即爬取",
|
||||||
|
"type": "click",
|
||||||
|
"key": "trigger_crawl",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "同步节假日",
|
||||||
|
"type": "click",
|
||||||
|
"key": "sync_holidays",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "AI 分析",
|
||||||
|
"type": "click",
|
||||||
|
"key": "toggle_ai",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class MenuManager:
|
||||||
|
def __init__(self, client=None):
|
||||||
|
self.client = client
|
||||||
|
|
||||||
|
async def _get_token(self) -> str | None:
|
||||||
|
from app.wechat.client import WeChatClient
|
||||||
|
c = self.client or WeChatClient()
|
||||||
|
return await c._get_access_token()
|
||||||
|
|
||||||
|
async def create(self) -> bool:
|
||||||
|
token = await self._get_token()
|
||||||
|
if not token:
|
||||||
|
return False
|
||||||
|
|
||||||
|
url = f"{settings.wechat_api_base_url}/cgi-bin/menu/create"
|
||||||
|
params = {"access_token": token, "agentid": int(settings.wechat_agent_id)}
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(timeout=15) as client:
|
||||||
|
response = await client.post(url, params=params, json=MENU)
|
||||||
|
data = response.json()
|
||||||
|
if data.get("errcode") == 0:
|
||||||
|
logger.info("企微菜单创建成功")
|
||||||
|
return True
|
||||||
|
logger.error(f"企微菜单创建失败: {data}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def delete(self) -> bool:
|
||||||
|
token = await self._get_token()
|
||||||
|
if not token:
|
||||||
|
return False
|
||||||
|
|
||||||
|
url = f"{settings.wechat_api_base_url}/cgi-bin/menu/delete"
|
||||||
|
params = {"access_token": token, "agentid": int(settings.wechat_agent_id)}
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(timeout=15) as client:
|
||||||
|
response = await client.get(url, params=params)
|
||||||
|
data = response.json()
|
||||||
|
return data.get("errcode") == 0
|
||||||
|
|
||||||
|
async def get(self) -> dict | None:
|
||||||
|
token = await self._get_token()
|
||||||
|
if not token:
|
||||||
|
return None
|
||||||
|
|
||||||
|
url = f"{settings.wechat_api_base_url}/cgi-bin/menu/get"
|
||||||
|
params = {"access_token": token, "agentid": int(settings.wechat_agent_id)}
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(timeout=15) as client:
|
||||||
|
response = await client.get(url, params=params)
|
||||||
|
data = response.json()
|
||||||
|
if data.get("errcode") == 0:
|
||||||
|
return data
|
||||||
|
return None
|
||||||
+6
-1
@@ -1,5 +1,9 @@
|
|||||||
FROM python:3.12-slim
|
FROM python:3.12-slim
|
||||||
|
|
||||||
|
LABEL maintainer="GX-gp-notify Dev Team" \
|
||||||
|
version="2.0.0" \
|
||||||
|
description="广西政府采购网公告监控系统"
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
@@ -7,7 +11,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY pyproject.toml .
|
COPY pyproject.toml .
|
||||||
RUN pip install --no-cache-dir -e ".[dev]"
|
COPY vendor/ vendor/
|
||||||
|
RUN pip install --no-cache-dir -e ".[dev]" && pip install --no-cache-dir ./vendor/
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|||||||
@@ -3,8 +3,13 @@ services:
|
|||||||
build:
|
build:
|
||||||
context: ..
|
context: ..
|
||||||
dockerfile: docker/Dockerfile
|
dockerfile: docker/Dockerfile
|
||||||
|
tags:
|
||||||
|
- gx-gp-notify:2.0.0
|
||||||
|
- gx-gp-notify:latest
|
||||||
|
image: gx-gp-notify:2.0.0
|
||||||
|
container_name: gx-gp-notify
|
||||||
ports:
|
ports:
|
||||||
- "8000:8000"
|
- "18001:8000"
|
||||||
env_file:
|
env_file:
|
||||||
- ../.env
|
- ../.env
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@@ -1,113 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# 定时搜索脚本启动器
|
|
||||||
# 用于启动广西政府采购网和大化县政府网公告定时搜索任务
|
|
||||||
# 自动激活项目虚拟环境并运行Python脚本
|
|
||||||
|
|
||||||
# 设置脚本遇到错误时退出
|
|
||||||
set -e
|
|
||||||
|
|
||||||
# 获取脚本所在目录的绝对路径
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
PROJECT_ROOT="$SCRIPT_DIR"
|
|
||||||
|
|
||||||
echo "========================================"
|
|
||||||
echo "🕒 政府采购公告定时搜索任务"
|
|
||||||
echo "📂 项目目录: $PROJECT_ROOT"
|
|
||||||
echo "🌐 监控网站:"
|
|
||||||
echo " - 广西政府采购网 (zfcg.gxzf.gov.cn)"
|
|
||||||
echo " - 大化县政府网 (gxdh.gov.cn)"
|
|
||||||
echo "⏰ 开始时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
||||||
echo "========================================"
|
|
||||||
|
|
||||||
# 检查虚拟环境
|
|
||||||
VENV_PATH="$PROJECT_ROOT/venv"
|
|
||||||
if [ ! -d "$VENV_PATH" ]; then
|
|
||||||
echo "❌ 错误: 找不到虚拟环境目录"
|
|
||||||
echo " 预期路径: $VENV_PATH"
|
|
||||||
echo " 请确保虚拟环境已正确创建"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查虚拟环境激活脚本
|
|
||||||
VENV_ACTIVATE="$VENV_PATH/bin/activate"
|
|
||||||
if [ ! -f "$VENV_ACTIVATE" ]; then
|
|
||||||
echo "❌ 错误: 找不到虚拟环境激活脚本"
|
|
||||||
echo " 预期路径: $VENV_ACTIVATE"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ 找到虚拟环境: $VENV_PATH"
|
|
||||||
|
|
||||||
# 激活虚拟环境
|
|
||||||
echo "🔧 激活虚拟环境..."
|
|
||||||
echo " 激活脚本: $VENV_ACTIVATE"
|
|
||||||
source "$VENV_ACTIVATE"
|
|
||||||
|
|
||||||
# 检查Python环境(在虚拟环境中)
|
|
||||||
if ! command -v python &> /dev/null; then
|
|
||||||
echo "❌ 错误: 虚拟环境中未找到Python命令"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查Python版本(在虚拟环境中)
|
|
||||||
PYTHON_VERSION=$(python --version 2>&1 | cut -d' ' -f2 | cut -d. -f1-2)
|
|
||||||
echo "🐍 Python版本 (虚拟环境): $PYTHON_VERSION"
|
|
||||||
|
|
||||||
# 检查项目目录
|
|
||||||
if [ ! -d "$PROJECT_ROOT/gx_gp_monitor" ]; then
|
|
||||||
echo "❌ 错误: 找不到gx_gp_monitor目录"
|
|
||||||
echo " 预期路径: $PROJECT_ROOT/gx_gp_monitor"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 检查cron_crawl.py文件
|
|
||||||
CRON_SCRIPT="$PROJECT_ROOT/gx_gp_monitor/cron_crawl.py"
|
|
||||||
if [ ! -f "$CRON_SCRIPT" ]; then
|
|
||||||
echo "❌ 错误: 找不到cron_crawl.py文件"
|
|
||||||
echo " 预期路径: $CRON_SCRIPT"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ 环境检查通过"
|
|
||||||
|
|
||||||
# 检查当前时间是否在夜间休息时间段 (23:00 - 07:00)
|
|
||||||
CURRENT_HOUR=$(date +%H)
|
|
||||||
CURRENT_MINUTE=$(date +%M)
|
|
||||||
CURRENT_TIME=$((CURRENT_HOUR * 60 + CURRENT_MINUTE))
|
|
||||||
START_TIME=$((23 * 60 + 0)) # 23:00
|
|
||||||
END_TIME=$((7 * 60 + 0)) # 07:00
|
|
||||||
|
|
||||||
if [ $CURRENT_TIME -ge $START_TIME ] || [ $CURRENT_TIME -lt $END_TIME ]; then
|
|
||||||
echo "🌙 当前时间 $(date '+%H:%M') 在夜间休息时间段 (23:00-07:00)"
|
|
||||||
echo "💤 跳过爬取任务,直接结束"
|
|
||||||
echo "========================================"
|
|
||||||
echo "🏁 任务跳过"
|
|
||||||
echo "📅 结束时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
||||||
echo "🔚 退出代码: 0 (跳过)"
|
|
||||||
echo "✅ 任务跳过成功"
|
|
||||||
echo "========================================"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "🚀 启动定时搜索脚本..."
|
|
||||||
|
|
||||||
# 切换到项目根目录
|
|
||||||
cd "$PROJECT_ROOT"
|
|
||||||
|
|
||||||
# 执行定时爬取脚本
|
|
||||||
python "$CRON_SCRIPT"
|
|
||||||
EXIT_CODE=$?
|
|
||||||
|
|
||||||
echo "========================================"
|
|
||||||
echo "🏁 任务完成"
|
|
||||||
echo "📅 结束时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
||||||
echo "🔚 退出代码: $EXIT_CODE"
|
|
||||||
|
|
||||||
if [ $EXIT_CODE -eq 0 ]; then
|
|
||||||
echo "✅ 任务执行成功"
|
|
||||||
else
|
|
||||||
echo "❌ 任务执行失败"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "========================================"
|
|
||||||
exit $EXIT_CODE
|
|
||||||
@@ -53,7 +53,6 @@ def test_pipeline_result_defaults():
|
|||||||
assert result.stored == 0
|
assert result.stored == 0
|
||||||
assert result.filtered == 0
|
assert result.filtered == 0
|
||||||
assert result.notified == 0
|
assert result.notified == 0
|
||||||
assert result.markdown_generated is False
|
|
||||||
|
|
||||||
|
|
||||||
def test_generate_content_hash():
|
def test_generate_content_hash():
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ async def test_gxgp_spider_pipeline_config():
|
|||||||
assert isinstance(config, PipelineConfig)
|
assert isinstance(config, PipelineConfig)
|
||||||
assert config.filter_enabled is True
|
assert config.filter_enabled is True
|
||||||
assert config.notify_mode == "filtered"
|
assert config.notify_mode == "filtered"
|
||||||
assert config.mark_sent is False
|
assert config.mark_sent is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@@ -49,9 +50,16 @@ class MockDahuagovSpider(BaseSpider):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_service():
|
||||||
|
return CrawlService(
|
||||||
|
db_session_factory=AsyncMock(),
|
||||||
|
notification_service=AsyncMock(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_crawl_service_registers_spiders():
|
async def test_crawl_service_registers_spiders():
|
||||||
service = CrawlService()
|
service = _make_service()
|
||||||
service.register(MockGXGPSpider())
|
service.register(MockGXGPSpider())
|
||||||
service.register(MockDahuagovSpider())
|
service.register(MockDahuagovSpider())
|
||||||
assert len(service.spiders) == 2
|
assert len(service.spiders) == 2
|
||||||
@@ -59,7 +67,7 @@ async def test_crawl_service_registers_spiders():
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_crawl_service_run_all():
|
async def test_crawl_service_run_all():
|
||||||
service = CrawlService()
|
service = _make_service()
|
||||||
service.register(MockGXGPSpider())
|
service.register(MockGXGPSpider())
|
||||||
service.register(MockDahuagovSpider())
|
service.register(MockDahuagovSpider())
|
||||||
results = await service.run_all()
|
results = await service.run_all()
|
||||||
@@ -71,7 +79,7 @@ async def test_crawl_service_run_all():
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_crawl_service_run_specific():
|
async def test_crawl_service_run_specific():
|
||||||
service = CrawlService()
|
service = _make_service()
|
||||||
service.register(MockGXGPSpider())
|
service.register(MockGXGPSpider())
|
||||||
service.register(MockDahuagovSpider())
|
service.register(MockDahuagovSpider())
|
||||||
results = await service.run_spider("mock_dahuagov")
|
results = await service.run_spider("mock_dahuagov")
|
||||||
@@ -81,7 +89,7 @@ async def test_crawl_service_run_specific():
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_crawl_service_run_unknown():
|
async def test_crawl_service_run_unknown():
|
||||||
service = CrawlService()
|
service = _make_service()
|
||||||
results = await service.run_spider("nonexistent")
|
results = await service.run_spider("nonexistent")
|
||||||
assert len(results) == 1
|
assert len(results) == 1
|
||||||
assert results[0].success is False
|
assert results[0].success is False
|
||||||
|
|||||||
@@ -45,6 +45,9 @@ async def test_pipeline_filtered_mode():
|
|||||||
pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
|
pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
|
||||||
|
|
||||||
with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
|
with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
|
||||||
|
with patch.object(pipeline, "_exclude_sent", AsyncMock(
|
||||||
|
side_effect=lambda anns: [a for a in anns if "大化" in a["title"]]
|
||||||
|
)):
|
||||||
with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=1)):
|
with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=1)):
|
||||||
pipe_result = await pipeline.process(
|
pipe_result = await pipeline.process(
|
||||||
result.announcements, config
|
result.announcements, config
|
||||||
@@ -84,6 +87,9 @@ async def test_pipeline_all_mode():
|
|||||||
pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
|
pipeline = PostCrawlPipeline(db_session=mock_db, notification_service=mock_notify)
|
||||||
|
|
||||||
with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
|
with patch.object(pipeline, "_save_to_db", AsyncMock(return_value=2)):
|
||||||
|
with patch.object(pipeline, "_exclude_sent", AsyncMock(
|
||||||
|
side_effect=lambda anns: anns
|
||||||
|
)):
|
||||||
with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=2)):
|
with patch.object(pipeline, "_send_notifications", AsyncMock(return_value=2)):
|
||||||
with patch.object(pipeline, "_mark_sent", AsyncMock(return_value=2)):
|
with patch.object(pipeline, "_mark_sent", AsyncMock(return_value=2)):
|
||||||
pipe_result = await pipeline.process(
|
pipe_result = await pipeline.process(
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
984557
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
[uwsgi]
|
|
||||||
# 企业微信回调服务器 uWSGI 配置
|
|
||||||
# 指定 WSGI 应用的入口模块和对象
|
|
||||||
module = app:app
|
|
||||||
|
|
||||||
# 项目基本配置
|
|
||||||
chdir = /home/v6ole/pyproject/GX-gp-notify
|
|
||||||
virtualenv = /home/v6ole/pyproject/GX-gp-notify/venv
|
|
||||||
http = 0.0.0.0:18001 # 指定 uWSGI 监听的 HTTP 地址和端口
|
|
||||||
master = true # 启用主进程模式
|
|
||||||
processes = 2 # 设置进程数
|
|
||||||
threads = 2 # 设置每个进程的线程数
|
|
||||||
enable-threads = true # 启用线程
|
|
||||||
harakiri = 30 # 设置超时时间(秒)
|
|
||||||
http-timeout = 30 # 设置HTTP请求超时
|
|
||||||
max-requests = 5000 # 每个进程处理请求数达到此值后自动重启
|
|
||||||
reload-on-rss = 200 # 当进程占用内存超过此值(MB)时自动重启
|
|
||||||
# 指定日志文件的路径和PID文件的路径
|
|
||||||
logto = ./logs/uwsgi-wechat.log
|
|
||||||
pidfile = ./uwsgi-wechat.pid
|
|
||||||
# 当 uWSGI 退出时,自动清理其创建的套接字和 PID 文件
|
|
||||||
vacuum = true
|
|
||||||
# 添加日期到日志
|
|
||||||
log-date = true
|
|
||||||
|
|
||||||
# 自动重载和内存报告配置
|
|
||||||
; py-autoreload = 1
|
|
||||||
; memory-report = true
|
|
||||||
Vendored
+84
@@ -0,0 +1,84 @@
|
|||||||
|
# LogHive Client SDK
|
||||||
|
|
||||||
|
Python client SDK for sending logs to [LogHive](https://github.com/your-org/loghive).
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install loghive-client
|
||||||
|
```
|
||||||
|
|
||||||
|
Or install from source:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd client
|
||||||
|
pip install .
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### Sync mode (recommended for scripts, Django, Flask)
|
||||||
|
|
||||||
|
```python
|
||||||
|
from loghive_client import LogHiveLogger
|
||||||
|
|
||||||
|
logger = LogHiveLogger(
|
||||||
|
project="my-awesome-app",
|
||||||
|
api_key="your-api-key",
|
||||||
|
endpoint="http://localhost:8000",
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Server started", extra={"port": 8080})
|
||||||
|
logger.error("Database timeout", exc_info=True)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Async mode (for FastAPI, aiohttp, asyncio)
|
||||||
|
|
||||||
|
```python
|
||||||
|
from loghive_client import AsyncLogHiveLogger
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
async with AsyncLogHiveLogger(
|
||||||
|
project="my-api",
|
||||||
|
api_key="your-api-key",
|
||||||
|
endpoint="http://localhost:8000",
|
||||||
|
) as logger:
|
||||||
|
await logger.info("API started")
|
||||||
|
# ...
|
||||||
|
|
||||||
|
asyncio.run(main())
|
||||||
|
```
|
||||||
|
|
||||||
|
### Standard logging integration (zero code change)
|
||||||
|
|
||||||
|
Add the handler to your existing logger:
|
||||||
|
|
||||||
|
```python
|
||||||
|
import logging
|
||||||
|
from loghive_client import LogHiveHandler
|
||||||
|
|
||||||
|
handler = LogHiveHandler("my-project", "api-key", "http://localhost:8000")
|
||||||
|
logging.getLogger().addHandler(handler)
|
||||||
|
|
||||||
|
# All existing logger calls now forward to LogHive
|
||||||
|
logging.info("This goes to LogHive too!")
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
| Param | Default | Description |
|
||||||
|
|-------|---------|-------------|
|
||||||
|
| `project` | (required) | Your project name in LogHive |
|
||||||
|
| `api_key` | (required) | Your project's API key |
|
||||||
|
| `endpoint` | `http://localhost:8000` | LogHive server URL |
|
||||||
|
| `batch_size` | 50 | Max entries per HTTP request |
|
||||||
|
| `flush_interval` | 2.0 | Seconds between flushes |
|
||||||
|
| `max_retries` | 3 | Retries on failure |
|
||||||
|
| `timeout` | 5.0 | HTTP request timeout |
|
||||||
|
|
||||||
|
## Trace ID (request correlation)
|
||||||
|
|
||||||
|
```python
|
||||||
|
logger.set_trace_id("req-abc-123")
|
||||||
|
```
|
||||||
Vendored
+20
@@ -0,0 +1,20 @@
|
|||||||
|
"""LogHive Client — Send logs from your Python projects to LogHive.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
from loghive_client import LogHiveLogger
|
||||||
|
|
||||||
|
logger = LogHiveLogger(
|
||||||
|
project="my-project",
|
||||||
|
api_key="your-api-key",
|
||||||
|
endpoint="http://localhost:8000",
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("User logged in", extra={"user_id": 42})
|
||||||
|
logger.error("Database connection failed", exc_info=True)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from loghive_client.client import LogHiveLogger
|
||||||
|
from loghive_client.async_client import AsyncLogHiveLogger
|
||||||
|
from loghive_client.handler import LogHiveHandler
|
||||||
|
|
||||||
|
__all__ = ["LogHiveLogger", "AsyncLogHiveLogger", "LogHiveHandler"]
|
||||||
+178
@@ -0,0 +1,178 @@
|
|||||||
|
"""Async LogHive client — for use in asyncio-based projects (e.g., FastAPI, aiohttp)."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import traceback
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class AsyncLogHiveLogger:
|
||||||
|
"""Async logger for asyncio applications.
|
||||||
|
|
||||||
|
Uses an async background task to batch and send log entries.
|
||||||
|
Ideal for FastAPI / Starlette / aiohttp projects.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
logger = AsyncLogHiveLogger("my-project", "api-key", "http://localhost:8000")
|
||||||
|
await logger.start()
|
||||||
|
|
||||||
|
await logger.info("Request processed", extra={"path": "/api/users"})
|
||||||
|
|
||||||
|
await logger.stop()
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
project: str,
|
||||||
|
api_key: str,
|
||||||
|
endpoint: str = "http://localhost:8000",
|
||||||
|
batch_size: int = 50,
|
||||||
|
flush_interval: float = 2.0,
|
||||||
|
max_retries: int = 3,
|
||||||
|
timeout: float = 5.0,
|
||||||
|
):
|
||||||
|
self.project = project
|
||||||
|
self.api_key = api_key
|
||||||
|
self.endpoint = endpoint.rstrip("/")
|
||||||
|
self.batch_size = batch_size
|
||||||
|
self.flush_interval = flush_interval
|
||||||
|
self.max_retries = max_retries
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
self._queue: asyncio.Queue = asyncio.Queue()
|
||||||
|
self._task: Optional[asyncio.Task] = None
|
||||||
|
self._client: Optional[httpx.AsyncClient] = None
|
||||||
|
self._stop_event = asyncio.Event()
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
"""Start the background flush task."""
|
||||||
|
if self._task and not self._task.done():
|
||||||
|
return
|
||||||
|
self._client = httpx.AsyncClient(timeout=self.timeout)
|
||||||
|
self._stop_event.clear()
|
||||||
|
self._task = asyncio.create_task(self._flush_loop())
|
||||||
|
logger.debug("AsyncLogHiveLogger started for project '%s'", self.project)
|
||||||
|
|
||||||
|
async def stop(self, flush: bool = True):
|
||||||
|
"""Stop the background task."""
|
||||||
|
self._stop_event.set()
|
||||||
|
if flush:
|
||||||
|
await self._flush_now()
|
||||||
|
if self._task:
|
||||||
|
self._task.cancel()
|
||||||
|
try:
|
||||||
|
await self._task
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
if self._client:
|
||||||
|
await self._client.aclose()
|
||||||
|
logger.debug("AsyncLogHiveLogger stopped for project '%s'", self.project)
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
await self.start()
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *args):
|
||||||
|
await self.stop()
|
||||||
|
|
||||||
|
async def _flush_loop(self):
|
||||||
|
"""Background loop that periodically flushes the queue."""
|
||||||
|
while not self._stop_event.is_set():
|
||||||
|
await self._flush_now()
|
||||||
|
await asyncio.sleep(self.flush_interval)
|
||||||
|
|
||||||
|
async def _flush_now(self):
|
||||||
|
"""Flush all currently queued entries."""
|
||||||
|
entries = []
|
||||||
|
while len(entries) < self.batch_size:
|
||||||
|
try:
|
||||||
|
entry = self._queue.get_nowait()
|
||||||
|
entries.append(entry)
|
||||||
|
except asyncio.QueueEmpty:
|
||||||
|
break
|
||||||
|
|
||||||
|
if not entries:
|
||||||
|
return
|
||||||
|
|
||||||
|
await self._send_batch(entries)
|
||||||
|
|
||||||
|
async def _send_batch(self, entries: List[Dict[str, Any]]):
|
||||||
|
"""Send a batch with retries."""
|
||||||
|
url = urljoin(self.endpoint, "/api/logs/ingest")
|
||||||
|
payload = {"project": self.project, "entries": entries}
|
||||||
|
|
||||||
|
for attempt in range(self.max_retries):
|
||||||
|
try:
|
||||||
|
resp = await self._client.post(
|
||||||
|
url,
|
||||||
|
json=payload,
|
||||||
|
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||||
|
)
|
||||||
|
if resp.status_code == 201:
|
||||||
|
return
|
||||||
|
elif resp.status_code == 401:
|
||||||
|
logger.error("LogHive: Invalid API key — dropping batch")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"LogHive: HTTP %d (attempt %d/%d)",
|
||||||
|
resp.status_code,
|
||||||
|
attempt + 1,
|
||||||
|
self.max_retries,
|
||||||
|
)
|
||||||
|
except httpx.RequestError as e:
|
||||||
|
logger.warning(
|
||||||
|
"LogHive: Connection error (attempt %d/%d): %s",
|
||||||
|
attempt + 1,
|
||||||
|
self.max_retries,
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
|
||||||
|
if attempt < self.max_retries - 1:
|
||||||
|
await asyncio.sleep(2 ** attempt)
|
||||||
|
|
||||||
|
logger.error("LogHive: Failed to send %d entries after %d retries", len(entries), self.max_retries)
|
||||||
|
|
||||||
|
def _enqueue(self, level: str, message: str, **kwargs):
|
||||||
|
"""Enqueue a log entry."""
|
||||||
|
entry = {
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"level": level,
|
||||||
|
"message": message,
|
||||||
|
"logger": kwargs.pop("logger", None) or __name__,
|
||||||
|
"extra": kwargs,
|
||||||
|
}
|
||||||
|
|
||||||
|
exc_info = kwargs.pop("exc_info", None)
|
||||||
|
if exc_info:
|
||||||
|
if isinstance(exc_info, BaseException):
|
||||||
|
entry["exception"] = "".join(
|
||||||
|
traceback.format_exception(type(exc_info), exc_info, exc_info.__traceback__)
|
||||||
|
)
|
||||||
|
elif exc_info is True:
|
||||||
|
entry["exception"] = traceback.format_exc()
|
||||||
|
|
||||||
|
self._queue.put_nowait(entry)
|
||||||
|
|
||||||
|
# ── Public API ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
async def debug(self, message: str, **kwargs):
|
||||||
|
self._enqueue("debug", message, **kwargs)
|
||||||
|
|
||||||
|
async def info(self, message: str, **kwargs):
|
||||||
|
self._enqueue("info", message, **kwargs)
|
||||||
|
|
||||||
|
async def warning(self, message: str, **kwargs):
|
||||||
|
self._enqueue("warning", message, **kwargs)
|
||||||
|
|
||||||
|
async def error(self, message: str, **kwargs):
|
||||||
|
self._enqueue("error", message, **kwargs)
|
||||||
|
|
||||||
|
async def critical(self, message: str, **kwargs):
|
||||||
|
self._enqueue("critical", message, **kwargs)
|
||||||
Vendored
+183
@@ -0,0 +1,183 @@
|
|||||||
|
"""Synchronous LogHive client — uses threading for non-blocking sends."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import traceback
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from queue import Queue, Empty
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class LogHiveLogger:
|
||||||
|
"""Synchronous logger that sends logs to LogHive in the background.
|
||||||
|
|
||||||
|
Uses a background thread with a queue to avoid blocking the main
|
||||||
|
application on network I/O.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
logger = LogHiveLogger("my-project", "api-key-here", "http://localhost:8000")
|
||||||
|
logger.info("Hello, world!")
|
||||||
|
logger.error("Something broke", exc_info=True)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
project: str,
|
||||||
|
api_key: str,
|
||||||
|
endpoint: str = "http://localhost:8000",
|
||||||
|
batch_size: int = 50,
|
||||||
|
flush_interval: float = 2.0,
|
||||||
|
max_retries: int = 3,
|
||||||
|
timeout: float = 5.0,
|
||||||
|
auto_start: bool = True,
|
||||||
|
):
|
||||||
|
self.project = project
|
||||||
|
self.api_key = api_key
|
||||||
|
self.endpoint = endpoint.rstrip("/")
|
||||||
|
self.batch_size = batch_size
|
||||||
|
self.flush_interval = flush_interval
|
||||||
|
self.max_retries = max_retries
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
self._queue: Queue = Queue()
|
||||||
|
self._stop_event = threading.Event()
|
||||||
|
self._thread: Optional[threading.Thread] = None
|
||||||
|
|
||||||
|
if auto_start:
|
||||||
|
self.start()
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
"""Start the background flush thread."""
|
||||||
|
if self._thread and self._thread.is_alive():
|
||||||
|
return
|
||||||
|
self._stop_event.clear()
|
||||||
|
self._thread = threading.Thread(target=self._flush_loop, daemon=True)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def stop(self, flush: bool = True):
|
||||||
|
"""Stop the background thread, optionally flushing remaining logs."""
|
||||||
|
self._stop_event.set()
|
||||||
|
if flush and self._thread:
|
||||||
|
self._flush_now()
|
||||||
|
if self._thread:
|
||||||
|
self._thread.join(timeout=5)
|
||||||
|
|
||||||
|
def _flush_loop(self):
|
||||||
|
"""Background loop that periodically flushes the queue."""
|
||||||
|
while not self._stop_event.is_set():
|
||||||
|
self._flush_now()
|
||||||
|
self._stop_event.wait(self.flush_interval)
|
||||||
|
|
||||||
|
def _flush_now(self):
|
||||||
|
"""Flush all currently queued log entries."""
|
||||||
|
entries = []
|
||||||
|
while len(entries) < self.batch_size:
|
||||||
|
try:
|
||||||
|
entry = self._queue.get_nowait()
|
||||||
|
entries.append(entry)
|
||||||
|
except Empty:
|
||||||
|
break
|
||||||
|
|
||||||
|
if not entries:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._send_batch(entries)
|
||||||
|
|
||||||
|
def _send_batch(self, entries: List[Dict[str, Any]]):
|
||||||
|
"""Send a batch of entries to the LogHive API, with retries."""
|
||||||
|
url = urljoin(self.endpoint, "/api/logs/ingest")
|
||||||
|
payload = {"project": self.project, "entries": entries}
|
||||||
|
|
||||||
|
for attempt in range(self.max_retries):
|
||||||
|
try:
|
||||||
|
resp = httpx.post(
|
||||||
|
url,
|
||||||
|
json=payload,
|
||||||
|
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||||
|
timeout=self.timeout,
|
||||||
|
)
|
||||||
|
if resp.status_code == 201:
|
||||||
|
return
|
||||||
|
elif resp.status_code == 401:
|
||||||
|
logger.error("LogHive: Invalid API key — dropping batch")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"LogHive: HTTP %d (attempt %d/%d)",
|
||||||
|
resp.status_code,
|
||||||
|
attempt + 1,
|
||||||
|
self.max_retries,
|
||||||
|
)
|
||||||
|
except httpx.RequestError as e:
|
||||||
|
logger.warning(
|
||||||
|
"LogHive: Connection error (attempt %d/%d): %s",
|
||||||
|
attempt + 1,
|
||||||
|
self.max_retries,
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
|
||||||
|
if attempt < self.max_retries - 1:
|
||||||
|
time.sleep(2 ** attempt)
|
||||||
|
|
||||||
|
logger.error("LogHive: Failed to send %d entries after %d retries", len(entries), self.max_retries)
|
||||||
|
|
||||||
|
def _enqueue(self, level: str, message: str, **kwargs):
|
||||||
|
"""Enqueue a log entry for async sending."""
|
||||||
|
entry = {
|
||||||
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"level": level,
|
||||||
|
"message": message,
|
||||||
|
"logger": kwargs.pop("logger", None) or __name__,
|
||||||
|
"module": kwargs.pop("module", None),
|
||||||
|
"function": kwargs.pop("function", None),
|
||||||
|
"line_no": kwargs.pop("line_no", None),
|
||||||
|
"trace_id": kwargs.pop("trace_id", None),
|
||||||
|
"extra": kwargs,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Handle exception info
|
||||||
|
exc_info = kwargs.pop("exc_info", None)
|
||||||
|
if exc_info:
|
||||||
|
if isinstance(exc_info, BaseException):
|
||||||
|
entry["exception"] = "".join(
|
||||||
|
traceback.format_exception(type(exc_info), exc_info, exc_info.__traceback__)
|
||||||
|
)
|
||||||
|
elif exc_info is True:
|
||||||
|
entry["exception"] = traceback.format_exc()
|
||||||
|
|
||||||
|
self._queue.put_nowait(entry)
|
||||||
|
|
||||||
|
# ── Public API (matching standard logging levels) ──────────
|
||||||
|
|
||||||
|
def debug(self, message: str, **kwargs):
|
||||||
|
self._enqueue("debug", message, **kwargs)
|
||||||
|
|
||||||
|
def info(self, message: str, **kwargs):
|
||||||
|
self._enqueue("info", message, **kwargs)
|
||||||
|
|
||||||
|
def warning(self, message: str, **kwargs):
|
||||||
|
self._enqueue("warning", message, **kwargs)
|
||||||
|
|
||||||
|
def error(self, message: str, **kwargs):
|
||||||
|
self._enqueue("error", message, **kwargs)
|
||||||
|
|
||||||
|
def critical(self, message: str, **kwargs):
|
||||||
|
self._enqueue("critical", message, **kwargs)
|
||||||
|
|
||||||
|
def log(self, level: str, message: str, **kwargs):
|
||||||
|
"""Log a message with an explicit level string."""
|
||||||
|
self._enqueue(level, message, **kwargs)
|
||||||
|
|
||||||
|
def set_trace_id(self, trace_id: str):
|
||||||
|
"""Set a trace_id for request correlation (used in web frameworks)."""
|
||||||
|
self._current_trace_id = trace_id
|
||||||
|
|
||||||
|
def __del__(self):
|
||||||
|
self.stop(flush=True)
|
||||||
Vendored
+72
@@ -0,0 +1,72 @@
|
|||||||
|
"""Python logging.Handler integration — use LogHive with the stdlib logging module.
|
||||||
|
|
||||||
|
This allows you to replace or augment your existing logging setup with
|
||||||
|
zero code changes (just add a handler to your logger).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from loghive_client.client import LogHiveLogger
|
||||||
|
|
||||||
|
|
||||||
|
class LogHiveHandler(logging.Handler):
|
||||||
|
"""A logging.Handler that sends records to LogHive.
|
||||||
|
|
||||||
|
Use it with Python's standard logging module:
|
||||||
|
import logging
|
||||||
|
from loghive_client import LogHiveHandler
|
||||||
|
|
||||||
|
handler = LogHiveHandler("my-project", "api-key", "http://localhost:8000")
|
||||||
|
logging.getLogger().addHandler(handler)
|
||||||
|
|
||||||
|
All existing logger calls (logger.info, logger.error, etc.) will
|
||||||
|
automatically forward to LogHive.
|
||||||
|
"""
|
||||||
|
|
||||||
|
LEVEL_MAP = {
|
||||||
|
logging.DEBUG: "debug",
|
||||||
|
logging.INFO: "info",
|
||||||
|
logging.WARNING: "warning",
|
||||||
|
logging.ERROR: "error",
|
||||||
|
logging.CRITICAL: "critical",
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
project: str,
|
||||||
|
api_key: str,
|
||||||
|
endpoint: str = "http://localhost:8000",
|
||||||
|
level: int = logging.INFO,
|
||||||
|
):
|
||||||
|
super().__init__(level=level)
|
||||||
|
self._client = LogHiveLogger(
|
||||||
|
project=project,
|
||||||
|
api_key=api_key,
|
||||||
|
endpoint=endpoint,
|
||||||
|
)
|
||||||
|
|
||||||
|
def emit(self, record: logging.LogRecord):
|
||||||
|
"""Send a log record to LogHive."""
|
||||||
|
try:
|
||||||
|
level = self.LEVEL_MAP.get(record.levelno, "info")
|
||||||
|
extra = {
|
||||||
|
"logger": record.name,
|
||||||
|
"module": record.module,
|
||||||
|
"function": record.funcName,
|
||||||
|
"line_no": record.lineno,
|
||||||
|
}
|
||||||
|
if record.exc_info and record.exc_info[0]:
|
||||||
|
import traceback
|
||||||
|
extra["exception"] = "".join(
|
||||||
|
traceback.format_exception(*record.exc_info)
|
||||||
|
)
|
||||||
|
|
||||||
|
self._client._enqueue(level, record.getMessage(), **extra)
|
||||||
|
except Exception:
|
||||||
|
self.handleError(record)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
"""Flush and close."""
|
||||||
|
self._client.stop(flush=True)
|
||||||
|
super().close()
|
||||||
Vendored
+23
@@ -0,0 +1,23 @@
|
|||||||
|
"""Setup script for loghive-client."""
|
||||||
|
|
||||||
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name="loghive-client",
|
||||||
|
version="0.1.0",
|
||||||
|
description="LogHive client SDK — push logs from your Python projects to LogHive",
|
||||||
|
author="LogHive",
|
||||||
|
packages=find_packages(),
|
||||||
|
install_requires=[
|
||||||
|
"httpx>=0.27.0",
|
||||||
|
],
|
||||||
|
python_requires=">=3.10",
|
||||||
|
classifiers=[
|
||||||
|
"Development Status :: 3 - Alpha",
|
||||||
|
"Intended Audience :: Developers",
|
||||||
|
"Programming Language :: Python :: 3",
|
||||||
|
"Programming Language :: Python :: 3.10",
|
||||||
|
"Programming Language :: Python :: 3.11",
|
||||||
|
"Programming Language :: Python :: 3.12",
|
||||||
|
],
|
||||||
|
)
|
||||||
@@ -1,478 +0,0 @@
|
|||||||
```bash
|
|
||||||
C:\Users\Voole>curl ^"http://www.gxdh.gov.cn/xxgk/zdlyxxgk/ggzypzly/zfcgly/cggg/^" ^
|
|
||||||
More? -H ^"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7^" ^
|
|
||||||
More? -H ^"Accept-Language: zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6^" ^
|
|
||||||
More? -H ^"Connection: keep-alive^" ^
|
|
||||||
More? -b ^"Hm_lvt_49d07951332748974d2b32f1fc826392=1761567101; _trs_uv=mh93l70k_3622_kalm^" ^
|
|
||||||
More? -H ^"Referer: http://www.gxdh.gov.cn/xxgk/zdlyxxgk/ggzypzly/zfcgly/cggg/^" ^
|
|
||||||
More? -H ^"Upgrade-Insecure-Requests: 1^" ^
|
|
||||||
More? -H ^"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36 Edg/144.0.0.0^" ^
|
|
||||||
More? --insecure
|
|
||||||
<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<title>
|
|
||||||
采购公告 -
|
|
||||||
广西河池大化瑶族自治县人民政府门户网站
|
|
||||||
</title>
|
|
||||||
<meta name="renderer" content="webkit">
|
|
||||||
<meta http-equiv="x-ua-Compatible" content="IE=Edge,chrome=1">
|
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
|
|
||||||
<meta name="Sitename" content="广西河池大化瑶族自治县人民政府门户网站">
|
|
||||||
<meta name="SiteDomain" content="http://www.gxdh.gov.cn/">
|
|
||||||
<meta name="SiteIDCode" content="4512290003">
|
|
||||||
<meta name="ColumnName" content="采购公告" />
|
|
||||||
<meta name="ColumnDescription" content="发布关于采购公告相关信息" />
|
|
||||||
<meta name="ColumnKeywords" content="采购公告,采购公告">
|
|
||||||
<meta name="ColumnType" content="采购公告">
|
|
||||||
<link rel="stylesheet" href="/cssq/main.css">
|
|
||||||
<link rel="stylesheet" href="/cssq/gxdh/css/main.css">
|
|
||||||
<link rel="stylesheet" href="" id="skin">
|
|
||||||
<link rel="stylesheet" href="/cssq/base.min.css">
|
|
||||||
<script src="/jsq/fenye.js"></script>
|
|
||||||
<link rel="stylesheet" type="text/css" href="http://www.gxdh.gov.cn/material/css/mobile-header.css" />
|
|
||||||
</head>
|
|
||||||
|
|
||||||
<body>
|
|
||||||
<div class="header">
|
|
||||||
<div class="site-top">
|
|
||||||
<div class="wrap">
|
|
||||||
<div class="site-date fn-left"></div>
|
|
||||||
<div class="top-right fn-right">
|
|
||||||
<!--
|
|
||||||
<a href="" target="_blank">登录</a>
|
|
||||||
<a href="" target="_blank">注册</a>
|
|
||||||
-->
|
|
||||||
<!-- <a class="fn-left" href="http://www.pingguo.gov.cn/lxwm/20190422-1790072.shtml" target="_blank"><i class="icon i-weibo-top"></i></a>
|
|
||||||
<a class="fn-left" href="http://www.pingguo.gov.cn/lxwm/20190422-1790072.shtml" target="_blank"><i class="icon i-webchat-top"></i></a> -->
|
|
||||||
<a id="gxzf_t2s" href="javascript:void(0);" onclick="goUserCenter()">用户空间</a>|
|
|
||||||
<script>
|
|
||||||
function goUserCenter() {
|
|
||||||
window.open('http://tyrz.zwfw.gxzf.gov.cn/am/auth/login?goto=aHR0cDovL3R5cnouendmdy5neHpmLmdvdi5jbi9hbS9vYXV0aDIvYXV0aG9yaXplP3NlcnZpY2U9aW5pdFNlcnZpY2UmcmVzcG9uc2VfdHlwZT1jb2RlJmNsaWVudF9pZD1neHp3ZncmY2xpZW50X3NlY3JldD0xMTExMTEmc2NvcGU9YWxsJnJlZGlyZWN0X3VyaT1odHRwJTNBLy96d2Z3Lmd4emYuZ292LmNuL2d4endmdy9tZW1iZXIvbG9naW4vdG9sb2dpbi5kbyUzRmdvdG91cmwlM0RodHRwJTI1M0EvL3p3ZncuZ3h6Zi5nb3YuY24=&lackFlag=NA');
|
|
||||||
var cont = $('meta[name="ColumnType"]').attr('content');
|
|
||||||
if (cont == '首页') {
|
|
||||||
$.getJSON(json, function (res) {
|
|
||||||
$("#sliderNav").empty();
|
|
||||||
var arr = randomArray(res, 7); //json数组,展示个数1
|
|
||||||
appEnd(arr);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
<a id="gxzf_t2s" href="http://www.gxdh.gov.cn/gx-znwd/home/gxdh" target="_blank">智能问答</a>|
|
|
||||||
<a id="gxzf_t2s" href="javascript: void(0); " target="_self">简体版</a>|
|
|
||||||
<a id="gxzf_s2t" href="javascript: void(0); " target="_self">繁体版</a>
|
|
||||||
<!--
|
|
||||||
|<a href="#">English</a>
|
|
||||||
-->
|
|
||||||
|
|
||||||
<div class="ipv6">支持IPv6</div>
|
|
||||||
<div id="gt_btn" class="ipv6 i-wza i-wza-people">无障碍</div>
|
|
||||||
<div class="ipv6 i-zzzq">
|
|
||||||
<a href="http://www.gxdh.gov.cn/zzzq/" target="_blank">长者专区</a>
|
|
||||||
</div>
|
|
||||||
<div class="i-jy"><a href="http://dsjfzj.gxzf.gov.cn/fzlm/gxzfwzjyhdh/index.shtml#10" target="_blank">集</a></div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div class="site-banner">
|
|
||||||
<div class="site-warp">
|
|
||||||
<a class="logo-wrap" href="/" target="_blank">
|
|
||||||
<img id="icon_gh" width="120" src="/imagesq/icon-gh.png" />
|
|
||||||
<img class="Company_name" src="http://www.gxdh.gov.cn/material/images/logo_ash.png" alt=" 广西河池大化瑶族自治县人民政府门户网站" />
|
|
||||||
</a>
|
|
||||||
<div class="site-search fn-right">
|
|
||||||
<div class="site-box">
|
|
||||||
<input class="site-search-txt" placeholder="请输入搜索内容" name="searchWord" type="text" />
|
|
||||||
<input type="hidden" name="siteId" value="121" />
|
|
||||||
<input class="i-site site-search-submit" type="button" value="" onClick="doSearch(this)" />
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div class="nav-list">
|
|
||||||
<ul class="nav">
|
|
||||||
<li class="i-index"><a href="http://www.gxdh.gov.cn/" target="_blank">首页</a></li>
|
|
||||||
<li class="i-city"><a href="http://www.gxdh.gov.cn/zjdh/" target="_blank">走进大化</a></li>
|
|
||||||
<li class="i-info-site"><a href="http://www.gxdh.gov.cn/xxgk/" target="_blank">政府信息公开</a></li>
|
|
||||||
<li class="i-info-site"><a href="http://www.gxdh.gov.cn/ggfw/" target="_blank">公共服务</a></li>
|
|
||||||
<li class="i-int"><a href="https://zwfw.gxzf.gov.cn/eportal/ui?regionCode=3107103&pageId=3cd3d403fb4443d5becb61375b2b093a" target="_blank">网上办事</a></li>
|
|
||||||
<li class="i-info-site"><a href="http://www.gxdh.gov.cn/zwdt/" target="_blank">政务地图</a></li>
|
|
||||||
<li class="i-exc"><a href="http://www.gxdh.gov.cn/zmhd/" target="_blank">互动交流</a></li>
|
|
||||||
<li class="i-data-site"><a href="http://www.gxdh.gov.cn/sjfb/" target="_blank">数据发布</a></li>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
<!--<style type="text/css">
|
|
||||||
.site-warp { background: url('http://www.gxdh.gov.cn/material/images/banner-bg.gif') 140px bottom no-repeat #fff;}
|
|
||||||
</style>-->
|
|
||||||
<style type="text/css">
|
|
||||||
.site-banner,.site-warp{height:405px;background: none; position: relative;}
|
|
||||||
.site-top{background: #eaf3fe66;}
|
|
||||||
.site-search{margin-top: 0px; border: none; width: 338px;}
|
|
||||||
.site-box{box-sizing: border-box; background: #eaf3fe66; padding: 8px; overflow: hidden;}
|
|
||||||
.logo{margin-top: 35px; position: absolute; left: 50%; transform: translateX(-50%);}
|
|
||||||
.nav{width: 1140px; margin: 0 auto;}
|
|
||||||
.nav li{margin: 0 14px;}
|
|
||||||
body{background: url(http://www.gxdh.gov.cn/material/images/dhrm_bg.jpg) top center no-repeat #f1f6fc;}
|
|
||||||
.ipv6{background-color: #4da0e2; color: #fff; font-size: 10px; line-height: 24px;float: right; margin-top: 3px;
|
|
||||||
padding: 0 10px; border-radius: 13px; height: 26px;}
|
|
||||||
.i-wza-people{padding-left: 25px;background-position: 7px 0px;}
|
|
||||||
.logo-wrap{display: block;width: 100%;margin: auto;}
|
|
||||||
img.Company_name.grayscale, img.Company_name {display: block; width: 800px; margin: 0 auto;}
|
|
||||||
img#icon_gh {display: block;width: 120px;margin: 0 auto; margin-top: 45px;}
|
|
||||||
.i-zzzq {
|
|
||||||
background-image: url(/material/images/zzzq/icon-laoren.png);
|
|
||||||
background-position: 10px 5px;
|
|
||||||
background-repeat: no-repeat;
|
|
||||||
background-size: 15px;
|
|
||||||
margin-right: 10px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.i-zzzq a {
|
|
||||||
display: block;
|
|
||||||
width: 60px;
|
|
||||||
text-align: right;
|
|
||||||
color: #FFFFFF;
|
|
||||||
}
|
|
||||||
|
|
||||||
</style>
|
|
||||||
<div class="wrap bg-white">
|
|
||||||
<div class="crumb-nav">
|
|
||||||
当前位置:
|
|
||||||
<a href="../../../../../" target="_self" title="首页" class="CurrChnlCls">首页</a> > <a href="../../../../" target="_self" title="政府信息公开" class="CurrChnlCls">政府信息公开</a> > <a href="../../../" target="_self" title="重点领域信息 公开" class="CurrChnlCls">重点领域信息公开</a> > <a href="../../" target="_self" title="公共资源配置领域" class="CurrChnlCls">公共资源配置领域</a> > <a href="../" target="_self" title="政府采购领域" class="CurrChnlCls">政府采购领域</a> > <a href="./" target="_self" title="采购公告" class="CurrChnlCls">采购公告</a>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<div id="morelist" class="more">
|
|
||||||
|
|
||||||
<div class="r_lmgd lmgd" style="display: none;"
|
|
||||||
data-gd="0">
|
|
||||||
<p>
|
|
||||||
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<ul class="more-list">
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-28
|
|
||||||
</span>
|
|
||||||
<a href="./t27193876.shtml" target="_blank" title="2025年度第二批中央水库移民后期扶持基金项目建设计划的竞争性 磋商招标公告">2025年度第二批中央水库移民后期扶持基金项目建设计划的竞争性磋商招标公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-28
|
|
||||||
</span>
|
|
||||||
<a href="./t27193548.shtml" target="_blank" title="大化镇坡了社区智慧村屯应急预警项目邀请报价的公告">大化镇坡 了社区智慧村屯应急预警项目邀请报价的公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-27
|
|
||||||
</span>
|
|
||||||
<a href="./t27195202.shtml" target="_blank" title="大化瑶族自治县七百弄乡2025年粤桂协作社会帮扶资金项目监理单 位邀请报名公告">大化瑶族自治县七百弄乡2025年粤桂协作社会帮扶资金项目监理单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-27
|
|
||||||
</span>
|
|
||||||
<a href="./t27189570.shtml" target="_blank" title="宝安区援助大化瑶族自治县中医医院附属项目设计单位邀请报名公 告">宝安区援助大化瑶族自治县中医医院附属项目设计单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-26
|
|
||||||
</span>
|
|
||||||
<a href="./t27187394.shtml" target="_blank" title="关于大化瑶族自治县紧密型县域医共体强基工程建设项目建议书及 可行性研究报告编制服务单位邀请报名公告">关于大化瑶族自治县紧密型县域医共体强基工程建设项目建议书及可行性研究报告编制服务单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-23
|
|
||||||
</span>
|
|
||||||
<a href="./t27171613.shtml" target="_blank" title="关于2026年巩固拓展脱贫攻坚成果和乡村振兴项目预算审查服务公 司公开招募的公告">关于2026年巩固拓展脱贫攻坚成果和乡村振兴项目预算审查服务公司公开招募的公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
<div class="r_lmgd lmgd" style="display: none;"
|
|
||||||
data-gd="0">
|
|
||||||
<p>
|
|
||||||
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<ul class="more-list">
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-23
|
|
||||||
</span>
|
|
||||||
<a href="./t27169946.shtml" target="_blank" title="大化瑶族自治县大化镇达悟村东红屯村容村貌改造项目等3个项目设计单位邀请报名公告">大化瑶族自治县大化镇达悟村东红屯村容村貌改造项目等3个项目设计单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-23
|
|
||||||
</span>
|
|
||||||
<a href="./t27168862.shtml" target="_blank" title="大化瑶族自治县七百弄乡2025年粤桂协作社会帮扶资金项目施工单 位邀请报名公告">大化瑶族自治县七百弄乡2025年粤桂协作社会帮扶资金项目施工单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-23
|
|
||||||
</span>
|
|
||||||
<a href="./t27172680.shtml" target="_blank" title="大化瑶族自治县住房和城乡建设局关于2026年生态环境保护专项资 金农村环境综合整治项目(污水处理)工程设计预算、 项目监理、招标代理、预算审核等 服务邀请报名的公告">大化瑶族自治县住房和城乡建设局关于2026年生态环境保护专项资金农村环境综合整治项目(污水处理)工程设计预算、 项目监理、招标代理、预算审核等 服务邀请报名的公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-19
|
|
||||||
</span>
|
|
||||||
<a href="./t27150343.shtml" target="_blank" title="关于择优录用项目水保验收和报备服务单位的公告">关于择优录用 项目水保验收和报备服务单位的公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-12
|
|
||||||
</span>
|
|
||||||
<a href="./t27103117.shtml" target="_blank" title="大化瑶族自治县中医医院排洪管道改造及支护项目监理单位邀请报 名公告">大化瑶族自治县中医医院排洪管道改造及支护项目监理单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2026-01-09
|
|
||||||
</span>
|
|
||||||
<a href="./t27098415.shtml" target="_blank" title="大化县2024年农村人居环境整治新增项目邀请报名公告">大化县2024年农村人居环境整治新增项目邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
<div class="r_lmgd lmgd" style="display: none;"
|
|
||||||
data-gd="0">
|
|
||||||
<p>
|
|
||||||
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<ul class="more-list">
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-12-26
|
|
||||||
</span>
|
|
||||||
<a href="./t27057491.shtml" target="_blank" title="大化瑶族自治县文化广电体育和旅游局关于承办大化红水河亲水徒 步大会暨UTO大化越野跑山赛”项目招标公告">大化瑶族自治县文化广电体育和旅游局关于承办大化红水河亲水徒步大会暨UTO大化越野跑山赛”项目招标公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-12-22
|
|
||||||
</span>
|
|
||||||
<a href="./t27033543.shtml" target="_blank" title="大化瑶族自治县卫生健康局关于大化瑶族自治县中医医院排洪管道 改造及支护项目建设邀请报名公告">大化瑶族自治县卫生健康局关于大化瑶族自治县中医医院排洪管道改造及支护项目建设邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-12-18
|
|
||||||
</span>
|
|
||||||
<a href="./t27023381.shtml" target="_blank" title="大化瑶族自治县大化镇仁良村党群服务中心地质勘测邀请报名公告">大化瑶族自治县大化镇仁良村党群服务中心地质勘测邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-12-12
|
|
||||||
</span>
|
|
||||||
<a href="./t26628863.shtml" target="_blank" title="大化瑶族自治县教育局关于大化瑶族自治县实验中学机器人设备采 购项目邀请报价的公告">大化瑶族自治县教育局关于大化瑶族自治县实验中学机器人设备采购项目邀请报价的公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-12-10
|
|
||||||
</span>
|
|
||||||
<a href="./t26522720.shtml" target="_blank" title="大化瑶族自治县住房和城乡建设局关于组织开展大化瑶族自治县住 宅专项维修资金专户管理银行服务项目招标代理机构的议标报名公告">大化瑶族自治县住房和城乡建设局关于组织开展大化瑶族自治县住宅专项维修资金专户管理银行服务项目招标代理机构的议标报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-12-05
|
|
||||||
</span>
|
|
||||||
<a href="./t26276809.shtml" target="_blank" title="大化瑶族自治县发展和改革局关于2026年巩固拓展脱贫攻坚成果同 乡村振兴项目设计服务公司公开招募公告">大化瑶族自治县发展和改革局关于2026年巩固拓展脱贫攻坚成果同乡村振兴项目设计服务公司公开招募公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
<div class="r_lmgd lmgd" style="display: none;"
|
|
||||||
data-gd="0">
|
|
||||||
<p>
|
|
||||||
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<ul class="more-list">
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-12-01
|
|
||||||
</span>
|
|
||||||
<a href="./t26230612.shtml" target="_blank" title="大化瑶族自治县特殊教育学校特教教具康复设备采购邀请报价公告">大化瑶族自治县特殊教育学校特教教具康复设备采购邀请报价公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-27
|
|
||||||
</span>
|
|
||||||
<a href="./t26214442.shtml" target="_blank" title="大化县贡川乡坡线屯风貌改造提升工程—建筑改造项目、大化县重点乡镇棚户区改造项目北景镇2016年棚户区改造项目(I标段)、 大化县重点乡镇棚户区改造项目大化县都阳镇2016年棚户区改造项目(II标段)结算审核咨询工作招标公告">大化县贡川乡坡线屯风貌改造提升工程—建筑改造项目、大化县重点乡镇棚户区改造项目北景镇2016年棚户区改造项目(I标段)、 大化县重 点乡镇棚户区改造项目大化县都阳镇2016年棚户区改造项目(II标段)结算审核咨询工作招标公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-26
|
|
||||||
</span>
|
|
||||||
<a href="./t26211785.shtml" target="_blank" title="大化县2025年度城市C级危房维修加固 工程项目(大化县工程局住 宅楼五区1栋)维修加固方案编制、预算审核及监理单位招标公告">大化县2025年度城市C级危房维修加固 工程项目(大化县工程局住宅楼五区1栋)维修加固方案编制、预算审核及监理单位招标公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-24
|
|
||||||
</span>
|
|
||||||
<a href="./t26197396.shtml" target="_blank" title="大化镇上旗村望远至下刁、东盘、内角屯道路扩建工程监理单位邀 请报名公告">大化镇上旗村望远至下刁、东盘、内角屯道路扩建工程监理单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-24
|
|
||||||
</span>
|
|
||||||
<a href="./t26197388.shtml" target="_blank" title="大化镇上旗村望远至下刁、东盘、内角屯道路扩建工程施工单位邀 请报名公告">大化镇上旗村望远至下刁、东盘、内角屯道路扩建工程施工单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-24
|
|
||||||
</span>
|
|
||||||
<a href="./t26197374.shtml" target="_blank" title="大化瑶族自治县大化镇凤翔村美凤屯弄卜里甘蔗产业道路硬化项目 等3个项目监理单位邀请报名公告">大化瑶族自治县大化镇凤翔村美凤屯弄卜里甘蔗产业道路硬化项目等3个项目监理单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
<div class="r_lmgd lmgd" style="display: none;"
|
|
||||||
data-gd="0">
|
|
||||||
<p>
|
|
||||||
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
<ul class="more-list">
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-24
|
|
||||||
</span>
|
|
||||||
<a href="./t26197360.shtml" target="_blank" title="大化瑶族自治县大化镇凤翔村美凤屯弄卜里甘蔗产业道路硬化项目 等3个项目施工单位邀请报名公告">大化瑶族自治县大化镇凤翔村美凤屯弄卜里甘蔗产业道路硬化项目等3个项目施工单位邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-17
|
|
||||||
</span>
|
|
||||||
<a href="./t26175508.shtml" target="_blank" title="大化瑶族自治县人力资源和社会保障局关于面向社会公开询价采购2026年基层公共就业服务项目的公告">大化瑶族自治县人力资源和社会保障局关于面向社会公开询价采购2026年基层公共就业服务项目的公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-17
|
|
||||||
</span>
|
|
||||||
<a href="./t26178684.shtml" target="_blank" title="大化瑶族自治县高级中学科教综合楼桩基检测专项服务项目分散采 购询价公告">大化瑶族自治县高级中学科教综合楼桩基检测专项服务项目分散采购询价公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-13
|
|
||||||
</span>
|
|
||||||
<a href="./t26164807.shtml" target="_blank" title="大化瑶族自治县人力资源和社会保障局关于为大化瑶族自治县2026 年度公益性岗位在岗人员购买意外伤害商业保险的询价公告">大化瑶族自治县人力资源和社会保障局关于为大化瑶族自治县2026年度公益性岗位在岗人员购买意外伤害商业保险的询价公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-13
|
|
||||||
</span>
|
|
||||||
<a href="./t26164559.shtml" target="_blank" title="大化瑶族自治县乡镇枢纽农村产业服务中心项目监理服务机构邀请 报名公告">大化瑶族自治县乡镇枢纽农村产业服务中心项目监理服务机构邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
<li><span>
|
|
||||||
2025-11-12
|
|
||||||
</span>
|
|
||||||
<a href="./t26157706.shtml" target="_blank" title="大化瑶族自治县2025年革命老区转移支付增量资金建设项目邀请报 名公告">大化瑶族自治县2025年革命老区转移支付增量资金建设项目邀请报名公告</a>
|
|
||||||
</li>
|
|
||||||
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
<div class="page">
|
|
||||||
<div class="more-page">
|
|
||||||
<script>createPageHTML(22, 0, "index", "shtml", "647");</script>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
|
|
||||||
</div>
|
|
||||||
<div class="footer-site">
|
|
||||||
<div class="wrap">
|
|
||||||
<ul style="width: 100%;">
|
|
||||||
<li class="footer-link-wrong">
|
|
||||||
<script id="_jiucuo_" sitecode='4512290003' src='https://zfwzgl.www.gov.cn/exposure/jiucuo.js'></script>
|
|
||||||
</li>
|
|
||||||
<li class="footer-link-gov"><a href="http://bszs.conac.cn/sitename?method=show&id=27249F7FD0875813E053012819AC8426" target="_blank"><img id="imgConac" src="/imagesq/dzjg.png" data-bd-imgshare-binded="1" vspace="0" hspace="0" border="0" /></a></li>
|
|
||||||
<li class="footer-txt">
|
|
||||||
<p>
|
|
||||||
<a href="http://www.gxdh.gov.cn/map.shtml" target="_blank">网站地图</a>
|
|
||||||
<a href="http://www.gxdh.gov.cn/lxwm/t885224.shtml" target="_BLANK" title="联系我们">联系我们</a></p>
|
|
||||||
<p>主办:大化瑶族自治县人民政府办公室</p>
|
|
||||||
<p>承办:大化瑶族自治县大数据发展局</p>
|
|
||||||
<p>地址:大化瑶族自治县大化镇花锦路2号 邮编:530800</p>
|
|
||||||
<p>联系电话:0778-5872869仅受理网站建设维护相关事宜</p>
|
|
||||||
<p>邮箱:dhdsjj2019@126.com</p>
|
|
||||||
<p>
|
|
||||||
<a href="http://www.beian.gov.cn/portal/registerSystemInfo?recordcode=45122902000008" target="_blank">
|
|
||||||
<img src="/imagesq/ba.png" /> 桂公网安备 45122902000008号
|
|
||||||
</a> <a href="http://beian.miit.gov.cn" target="_blank">桂ICP备14001878号-1
|
|
||||||
</a>网站标识码 :4512290003
|
|
||||||
</p>
|
|
||||||
</li>
|
|
||||||
<!--<li class="footer-link-wrong" style="width: 250px;padding-top: 0px;">
|
|
||||||
<a href="http://www.gxzf.gov.cn/wxhkhd/message.shtml?n=广西河池大化瑶族自治县人民政府门户网站&aid=822&sid=129" target="_blank">
|
|
||||||
<img src="http://www.gxdh.gov.cn/material/images/wzjc.png" />
|
|
||||||
</a>
|
|
||||||
</li>-->
|
|
||||||
<li class="footer-web" style="padding-left: 9px;">
|
|
||||||
<img src="http://www.gxdh.gov.cn/wbwx/material/dhyzzzxzfmhwz1.jpg" width="100px" alt="微信公众号" /><p>微信公众号</p>
|
|
||||||
</li>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<style type="text/css">
|
|
||||||
.footer-txt{width: 552px;}
|
|
||||||
</style>
|
|
||||||
<!--百度统计BEG-->
|
|
||||||
<script>
|
|
||||||
var _hmt = _hmt || [];
|
|
||||||
(function() {
|
|
||||||
var hm = document.createElement("script");
|
|
||||||
hm.src = "https://hm.baidu.com/hm.js?49d07951332748974d2b32f1fc826392";
|
|
||||||
var s = document.getElementsByTagName("script")[0];
|
|
||||||
s.parentNode.insertBefore(hm, s);
|
|
||||||
})();
|
|
||||||
</script>
|
|
||||||
<!--百度统计END-->
|
|
||||||
<script src="/jsq/jquery.min.js"></script>
|
|
||||||
<script src="/jsq/jquery.cookie.js"></script>
|
|
||||||
<script src="/jsq/jquery.s2t.js"></script>
|
|
||||||
<script src="/jsq/jquery.qrcode.min.js"></script>
|
|
||||||
<script src="/jsq/chinese_status.js"></script>
|
|
||||||
<script src="/jsq/checklink.min.js"></script>
|
|
||||||
<script src="/jsq/main.js"></script>
|
|
||||||
<script src="/jsq/wza.js"></script>
|
|
||||||
|
|
||||||
<script src="/jsq/api_gwy.js"></script>
|
|
||||||
<script src="http://www.gxdh.gov.cn/material/js/Access_conf.js"></script>
|
|
||||||
<!--无障碍-->
|
|
||||||
<script type="text/javascript">document.write("<script src='/accessTool/Access_Initial.js?"+Math.random()+"'><\/script>");</script>
|
|
||||||
|
|
||||||
|
|
||||||
<script>
|
|
||||||
var version =Date.parse(new Date());
|
|
||||||
document.write('<script src="/jsq/api_func.js?v='+version+'"><\/script>');
|
|
||||||
SITEID = 129;
|
|
||||||
APPID = 25;
|
|
||||||
</script>
|
|
||||||
|
|
||||||
</script><script id="_trs_ta_js" src="//ta.trs.cn/c/js/ta.js?mpid=3622" async="async" defer="defer"></script>
|
|
||||||
<script src="http://www.gxdh.gov.cn/material/js/M-index.js" type="text/javascript" charset="utf-8"></script>
|
|
||||||
</body>
|
|
||||||
|
|
||||||
<script>
|
|
||||||
document.addEventListener('DOMContentLoaded', function () {
|
|
||||||
const elements = document.querySelectorAll('div[data-gd="0"]');
|
|
||||||
elements.forEach(function (element) {
|
|
||||||
element.style.display = 'none';
|
|
||||||
});
|
|
||||||
});
|
|
||||||
</script>
|
|
||||||
</html>
|
|
||||||
Reference in New Issue
Block a user