chore: 删除旧 Flask 项目文件，迁移到 FastAPI 架构
- 删除 gx_gp_monitor/ 旧项目目录（Flask CLI 架构） - 删除 app.py（旧 WSGI 入口） - 更新 alembic/env.py 为异步引擎（create_async_engine + run_sync）
This commit is contained in:
+17
-18
@@ -1,34 +1,33 @@
|
||||
import asyncio
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from app.models.announcement import Base
|
||||
|
||||
config = context.config
|
||||
from app.config import settings
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline():
|
||||
from app.config import settings
|
||||
url = settings.database_url
|
||||
context.configure(url=url, target_metadata=target_metadata, literal_binds=True)
|
||||
context.configure(url=settings.database_url, target_metadata=target_metadata,
|
||||
literal_binds=True, dialect_opts={"paramstyle": "named"})
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online():
|
||||
from app.config import settings
|
||||
connectable = engine_from_config(
|
||||
{"sqlalchemy.url": settings.database_url},
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
def do_run_migrations(connection):
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def run_migrations_online():
|
||||
connectable = create_async_engine(settings.database_url, echo=True)
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(do_run_migrations)
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
asyncio.run(run_migrations_online())
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
企业微信回调服务器 WSGI 应用入口
|
||||
用于 uWSGI 等部署环境
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# 添加项目路径
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
try:
|
||||
from gx_gp_monitor.wechat.callback_server import create_callback_app
|
||||
|
||||
# 创建 WSGI 应用对象
|
||||
app = create_callback_app()
|
||||
|
||||
except ImportError as e:
|
||||
print(f"导入失败: {e}", file=sys.stderr)
|
||||
print("请确保已安装所有依赖: pip install -r gx_gp_monitor/requirements.txt", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"应用创建失败: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -1,281 +0,0 @@
|
||||
# 广西政府采购网公告监控系统
|
||||
|
||||
广西政府采购网公告爬取和监控的智能系统，支持多种公告类型的自动爬取、智能筛选、数据存储和通知推送。
|
||||
|
||||
## 功能特性
|
||||
|
||||
### 🕷️ 智能爬虫监控
|
||||
- 自动爬取广西政府采购网(zfcg.gxzf.gov.cn)的各类采购公告
|
||||
- 支持10+种公告类型：采购公告、结果公告、合同公告、更正公告、招标文件预公示、单一来源公示、电子卖场公示、履约验收公示、工程类公告、框架协议征集公告等
|
||||
- 智能反爬虫机制：随机User-Agent、代理轮换、请求延迟控制
|
||||
|
||||
### 🎯 多维度智能筛选
|
||||
- **关键词过滤**：支持多关键词精确匹配和模糊匹配
|
||||
- **日期范围筛选**：支持开始日期和结束日期过滤
|
||||
- **自动去重**：基于内容哈希的智能去重机制
|
||||
- **来源筛选**：按公告类型进行筛选
|
||||
|
||||
### 💾 数据持久化存储
|
||||
- **PostgreSQL数据库**：高效的数据存储和查询
|
||||
- **自动清理**：默认90天数据保留，可配置
|
||||
- **性能优化**：连接池、索引优化、批量操作
|
||||
|
||||
### 🔔 高可用性保障
|
||||
- **重试机制**：指数退避重试策略
|
||||
- **超时控制**：连接超时和读取超时设置
|
||||
- **幂等操作**：防止重复处理
|
||||
- **异常处理**：完善的异常捕获和恢复
|
||||
- **结构化日志**：详细的运行日志记录
|
||||
|
||||
### 📱 通知系统
|
||||
- **企业微信通知**：支持文本和Markdown格式消息
|
||||
- **实时推送**：新公告发现即时通知
|
||||
- **系统告警**：异常情况自动告警
|
||||
|
||||
### 📊 Markdown输出
|
||||
- **自动生成**：将公告输出为Markdown格式文件（onu.md）
|
||||
- **结构化展示**：按来源分组、时间排序
|
||||
- **今日高亮**：突出显示今日发布的公告
|
||||
|
||||
## 安装和使用
|
||||
|
||||
### 环境要求
|
||||
- Python 3.8+
|
||||
- PostgreSQL 12+
|
||||
|
||||
### 安装步骤
|
||||
|
||||
1. **克隆项目**
|
||||
```bash
|
||||
cd /path/to/your/workspace
|
||||
# 项目已创建在 gx_gp_monitor 目录下
|
||||
```
|
||||
|
||||
2. **创建虚拟环境**
|
||||
```bash
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate # Linux/Mac
|
||||
# 或
|
||||
venv\Scripts\activate # Windows
|
||||
```
|
||||
|
||||
3. **安装依赖**
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
4. **配置数据库**
|
||||
```bash
|
||||
# 创建PostgreSQL数据库
|
||||
createdb gx-gp-notify
|
||||
|
||||
# 修改配置文件 config/config.yaml 中的数据库连接信息
|
||||
```
|
||||
|
||||
5. **修改配置**
|
||||
```bash
|
||||
# 编辑 config/config.yaml 文件
|
||||
# 设置数据库连接、企业微信配置、关键词等
|
||||
```
|
||||
|
||||
### 使用方法
|
||||
|
||||
#### 命令行使用
|
||||
|
||||
```bash
|
||||
# 查看帮助
|
||||
python main.py --help
|
||||
|
||||
# 执行一次爬取
|
||||
python main.py crawl
|
||||
|
||||
# 带参数爬取
|
||||
python main.py crawl --keywords "大化" "信息化" --max-pages 5
|
||||
|
||||
# 启动调度器（定时任务）
|
||||
python main.py scheduler
|
||||
|
||||
# 数据清理
|
||||
python main.py cleanup --days 30
|
||||
|
||||
# 查看系统状态
|
||||
python main.py status
|
||||
```
|
||||
|
||||
#### 编程接口使用
|
||||
|
||||
```python
|
||||
from gx_gp_monitor.crawler.spider import crawl_announcements
|
||||
from gx_gp_monitor.filters.filters import filter_from_config
|
||||
from gx_gp_monitor.storage.postgresql import save_announcements_to_storage
|
||||
from gx_gp_monitor.storage.md_generator import generate_onu_md
|
||||
from gx_gp_monitor.notification.wechat import send_announcements_notification
|
||||
|
||||
# 执行爬取
|
||||
results = crawl_announcements()
|
||||
|
||||
# 筛选公告
|
||||
filter_obj = filter_from_config()
|
||||
filtered_announcements, stats = filter_obj.filter(all_announcements)
|
||||
|
||||
# 保存到数据库
|
||||
saved_count = save_announcements_to_storage(filtered_announcements)
|
||||
|
||||
# 生成Markdown文件
|
||||
generate_onu_md(filtered_announcements)
|
||||
|
||||
# 发送通知
|
||||
send_announcements_notification(filtered_announcements)
|
||||
```
|
||||
|
||||
## 配置说明
|
||||
|
||||
### 主要配置文件：`config/config.yaml`
|
||||
|
||||
```yaml
|
||||
# 调试模式
|
||||
debug: false
|
||||
|
||||
# 日志配置
|
||||
log_level: INFO
|
||||
log_file: logs/gx_gp_monitor.log
|
||||
|
||||
# 爬虫配置
|
||||
crawler:
|
||||
base_url: "https://zfcg.gxzf.gov.cn"
|
||||
timeout: 30
|
||||
max_retries: 3
|
||||
keyword: ["大化", "信息化"] # 关键词筛选
|
||||
max_pages: 10
|
||||
|
||||
# 数据库配置
|
||||
database:
|
||||
enabled: true
|
||||
host: "localhost"
|
||||
port: 5432
|
||||
name: "gx-gp-notify"
|
||||
user: "your_user"
|
||||
password: "your_password"
|
||||
data_retention_days: 90
|
||||
|
||||
# 企业微信配置
|
||||
wechat_app:
|
||||
enabled: true
|
||||
corp_id: "your_corp_id"
|
||||
agent_id: "your_agent_id"
|
||||
secret: "your_secret"
|
||||
token: "your_token"
|
||||
encoding_aes_key: "your_aes_key"
|
||||
|
||||
# 调度配置
|
||||
scheduler:
|
||||
enabled: true
|
||||
jobs:
|
||||
- name: "daily_crawl"
|
||||
cron: "0 8,14,18 * * *"
|
||||
enabled: true
|
||||
```
|
||||
|
||||
### 公告来源配置
|
||||
|
||||
系统支持以下公告类型：
|
||||
|
||||
- `ZcyAnnouncement1`: 采购公告
|
||||
- `ZcyAnnouncement2`: 结果公告
|
||||
- `ZcyAnnouncement3`: 合同公告
|
||||
- `ZcyAnnouncement4`: 更正公告
|
||||
- `ZcyAnnouncement5`: 招标文件预公示
|
||||
- `ZcyAnnouncement6`: 单一来源公示
|
||||
- `ZcyAnnouncement7`: 电子卖场公示
|
||||
- `ZcyAnnouncement10`: 履约验收公示
|
||||
- `ZcyAnnouncement11`: 工程类公告
|
||||
- `ZcyAnnouncement20`: 框架协议征集公告
|
||||
- `ZcyAnnouncement21`: 框架协议入围结果公告
|
||||
- `ZcyAnnouncement23`: 框架协议成交结果汇总公告
|
||||
- `61-266648`: 采购意向公开
|
||||
|
||||
## 架构设计
|
||||
|
||||
```
|
||||
gx_gp_monitor/
|
||||
├── config/ # 配置管理
|
||||
├── core/ # 核心模块
|
||||
│ ├── models.py # 数据模型
|
||||
│ ├── config_manager.py # 配置管理
|
||||
│ ├── database.py # 数据库操作
|
||||
│ ├── logger.py # 日志管理
|
||||
│ └── reliability.py # 高可用性
|
||||
├── crawler/ # 爬虫模块
|
||||
│ ├── spider.py # 爬虫核心
|
||||
│ └── parsers.py # 数据解析
|
||||
├── filters/ # 筛选模块
|
||||
├── storage/ # 存储模块
|
||||
│ ├── postgresql.py # PostgreSQL存储
|
||||
│ └── md_generator.py # Markdown生成
|
||||
├── notification/ # 通知模块
|
||||
├── scheduler/ # 调度模块
|
||||
└── main.py # 主程序入口
|
||||
```
|
||||
|
||||
## 日志和监控
|
||||
|
||||
### 日志文件
|
||||
- 默认日志文件：`logs/gx_gp_monitor.log`
|
||||
- 日志轮转：10MB大小限制，保留5个备份文件
|
||||
- 日志级别：DEBUG、INFO、WARNING、ERROR、CRITICAL
|
||||
|
||||
### 监控指标
|
||||
- 爬取统计：成功/失败次数、响应时间
|
||||
- 数据统计：公告数量、新增数量、去重统计
|
||||
- 系统状态：内存使用、磁盘空间、健康检查
|
||||
|
||||
## 故障排除
|
||||
|
||||
### 常见问题
|
||||
|
||||
1. **数据库连接失败**
|
||||
- 检查PostgreSQL服务是否运行
|
||||
- 验证数据库连接配置
|
||||
- 确认用户权限
|
||||
|
||||
2. **爬虫请求失败**
|
||||
- 检查网络连接
|
||||
- 验证目标网站是否可访问
|
||||
- 调整请求间隔和重试策略
|
||||
|
||||
3. **企业微信通知失败**
|
||||
- 检查企业微信配置
|
||||
- 验证应用ID和密钥
|
||||
- 确认网络能访问企业微信API
|
||||
|
||||
4. **权限问题**
|
||||
- 确保日志目录和输出文件目录有写入权限
|
||||
- 检查数据库用户权限
|
||||
|
||||
### 调试模式
|
||||
|
||||
启用调试模式获取更详细的日志：
|
||||
|
||||
```yaml
|
||||
debug: true
|
||||
log_level: DEBUG
|
||||
```
|
||||
|
||||
## 许可证
|
||||
|
||||
本项目采用 MIT 许可证。
|
||||
|
||||
## 贡献
|
||||
|
||||
欢迎提交Issue和Pull Request来改进这个项目。
|
||||
|
||||
## 版本历史
|
||||
|
||||
- **v1.0.0** (2024-01-XX)
|
||||
- 初始版本发布
|
||||
- 支持10+种公告类型爬取
|
||||
- 实现智能筛选和去重
|
||||
- PostgreSQL数据存储
|
||||
- 企业微信通知
|
||||
- Markdown文件输出
|
||||
- 定时任务调度
|
||||
@@ -1,7 +0,0 @@
|
||||
"""
|
||||
广西政府采购网公告监控系统
|
||||
广西政府采购网公告爬取和监控的智能系统
|
||||
"""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
__author__ = "GX GP Monitor Team"
|
||||
@@ -1,11 +0,0 @@
|
||||
"""
|
||||
包的main入口，使项目可以直接通过 python -m gx_gp_monitor 运行
|
||||
"""
|
||||
|
||||
try:
|
||||
from .main import main
|
||||
except ImportError:
|
||||
from main import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1 +0,0 @@
|
||||
"""配置管理模块"""
|
||||
@@ -1 +0,0 @@
|
||||
"""核心模块"""
|
||||
@@ -1,394 +0,0 @@
|
||||
"""
|
||||
配置管理模块
|
||||
负责加载、验证和管理系统配置
|
||||
"""
|
||||
|
||||
import os
|
||||
import yaml
|
||||
from typing import Dict, Any, Optional
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class LogLevel(Enum):
|
||||
"""日志级别枚举"""
|
||||
DEBUG = "DEBUG"
|
||||
INFO = "INFO"
|
||||
WARNING = "WARNING"
|
||||
ERROR = "ERROR"
|
||||
CRITICAL = "CRITICAL"
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlerConfig:
|
||||
"""爬虫配置"""
|
||||
base_url: str
|
||||
timeout: int
|
||||
max_retries: int
|
||||
retry_delay: float
|
||||
max_retry_delay: float
|
||||
backoff_factor: float
|
||||
user_agents: list
|
||||
proxies: list
|
||||
request_delay: float
|
||||
request_delay_max: float
|
||||
keyword: list
|
||||
start_date: str
|
||||
end_date: str
|
||||
max_pages: int
|
||||
page_size: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatabaseConfig:
|
||||
"""数据库配置"""
|
||||
enabled: bool
|
||||
type: str
|
||||
host: str
|
||||
port: int
|
||||
name: str
|
||||
user: str
|
||||
password: str
|
||||
pool_size: int
|
||||
max_overflow: int
|
||||
pool_timeout: int
|
||||
pool_recycle: int
|
||||
data_retention_days: int
|
||||
auto_cleanup: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class WeChatConfig:
|
||||
"""企业微信配置"""
|
||||
enabled: bool
|
||||
corp_id: str
|
||||
agent_id: str
|
||||
secret: str
|
||||
token: str
|
||||
encoding_aes_key: str
|
||||
port: int
|
||||
host: str
|
||||
debug: bool
|
||||
use_proxy: bool
|
||||
proxy_api_url: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MarkdownConfig:
|
||||
"""Markdown配置"""
|
||||
enabled: bool
|
||||
output_file: str
|
||||
max_entries: int
|
||||
include_today_highlight: bool
|
||||
template_file: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchedulerConfig:
|
||||
"""调度配置"""
|
||||
enabled: bool
|
||||
timezone: str
|
||||
jobs: list
|
||||
|
||||
|
||||
@dataclass
|
||||
class MonitoringConfig:
|
||||
"""监控配置"""
|
||||
enabled: bool
|
||||
health_check_interval: int
|
||||
alert_on_failure: bool
|
||||
max_consecutive_failures: int
|
||||
metrics_enabled: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemConfig:
|
||||
"""系统配置"""
|
||||
debug: bool
|
||||
log_level: LogLevel
|
||||
log_file: str
|
||||
log_max_size: int
|
||||
log_backup_count: int
|
||||
crawler: CrawlerConfig
|
||||
database: DatabaseConfig
|
||||
wechat_app: WeChatConfig
|
||||
markdown: MarkdownConfig
|
||||
scheduler: Optional[SchedulerConfig]
|
||||
monitoring: MonitoringConfig
|
||||
sources: Dict[str, Dict[str, Any]]
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
"""配置管理器"""
|
||||
|
||||
def __init__(self, config_file: Optional[str] = None):
|
||||
"""
|
||||
初始化配置管理器
|
||||
|
||||
Args:
|
||||
config_file: 配置文件路径，如果为None则使用默认路径
|
||||
"""
|
||||
if config_file is None:
|
||||
# 默认配置文件路径
|
||||
current_dir = Path(__file__).parent.parent
|
||||
self.config_file = current_dir / "config" / "config.yaml"
|
||||
else:
|
||||
self.config_file = Path(config_file)
|
||||
|
||||
self._config_data = {}
|
||||
self._config = None
|
||||
|
||||
def load_config(self) -> SystemConfig:
|
||||
"""
|
||||
加载配置文件
|
||||
|
||||
Returns:
|
||||
SystemConfig: 系统配置对象
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: 配置文件不存在
|
||||
yaml.YAMLError: 配置文件格式错误
|
||||
ValueError: 配置验证失败
|
||||
"""
|
||||
if not self.config_file.exists():
|
||||
raise FileNotFoundError(f"配置文件不存在: {self.config_file}")
|
||||
|
||||
try:
|
||||
with open(self.config_file, 'r', encoding='utf-8') as f:
|
||||
self._config_data = yaml.safe_load(f)
|
||||
except yaml.YAMLError as e:
|
||||
raise yaml.YAMLError(f"配置文件格式错误: {e}")
|
||||
|
||||
# 验证配置
|
||||
self._validate_config()
|
||||
|
||||
# 解析配置
|
||||
self._config = self._parse_config()
|
||||
return self._config
|
||||
|
||||
def _validate_config(self):
|
||||
"""验证配置完整性"""
|
||||
required_keys = [
|
||||
'debug', 'log_level', 'log_file', 'log_max_size', 'log_backup_count',
|
||||
'crawler', 'database', 'wechat_app', 'markdown',
|
||||
'monitoring', 'sources'
|
||||
]
|
||||
|
||||
for key in required_keys:
|
||||
if key not in self._config_data:
|
||||
raise ValueError(f"配置文件缺少必需的配置项: {key}")
|
||||
|
||||
# 验证爬虫配置
|
||||
crawler_required = [
|
||||
'base_url', 'timeout', 'max_retries', 'retry_delay', 'max_retry_delay',
|
||||
'backoff_factor', 'user_agents', 'proxies', 'request_delay',
|
||||
'request_delay_max', 'keyword', 'max_pages', 'page_size'
|
||||
]
|
||||
|
||||
for key in crawler_required:
|
||||
if key not in self._config_data['crawler']:
|
||||
raise ValueError(f"爬虫配置缺少必需项: {key}")
|
||||
|
||||
# 验证数据库配置
|
||||
if self._config_data.get('database', {}).get('enabled', False):
|
||||
db_required = ['type', 'host', 'port', 'name', 'user', 'password']
|
||||
for key in db_required:
|
||||
if key not in self._config_data['database']:
|
||||
raise ValueError(f"数据库配置缺少必需项: {key}")
|
||||
|
||||
# 验证企业微信配置
|
||||
if self._config_data.get('wechat_app', {}).get('enabled', False):
|
||||
wechat_required = ['corp_id', 'agent_id', 'secret', 'token', 'encoding_aes_key']
|
||||
for key in wechat_required:
|
||||
if key not in self._config_data['wechat_app']:
|
||||
raise ValueError(f"企业微信配置缺少必需项: {key}")
|
||||
|
||||
def _parse_config(self) -> SystemConfig:
|
||||
"""解析配置数据"""
|
||||
crawler_data = self._config_data['crawler']
|
||||
crawler = CrawlerConfig(
|
||||
base_url=crawler_data['base_url'],
|
||||
timeout=crawler_data['timeout'],
|
||||
max_retries=crawler_data['max_retries'],
|
||||
retry_delay=crawler_data['retry_delay'],
|
||||
max_retry_delay=crawler_data['max_retry_delay'],
|
||||
backoff_factor=crawler_data['backoff_factor'],
|
||||
user_agents=crawler_data['user_agents'],
|
||||
proxies=crawler_data['proxies'],
|
||||
request_delay=crawler_data['request_delay'],
|
||||
request_delay_max=crawler_data['request_delay_max'],
|
||||
keyword=crawler_data['keyword'],
|
||||
start_date=crawler_data.get('start_date', ''),
|
||||
end_date=crawler_data.get('end_date', ''),
|
||||
max_pages=crawler_data['max_pages'],
|
||||
page_size=crawler_data['page_size']
|
||||
)
|
||||
|
||||
db_data = self._config_data['database']
|
||||
database = DatabaseConfig(
|
||||
enabled=db_data.get('enabled', False),
|
||||
type=db_data.get('type', 'postgresql'),
|
||||
host=db_data.get('host', 'localhost'),
|
||||
port=db_data.get('port', 5432),
|
||||
name=db_data.get('name', ''),
|
||||
user=db_data.get('user', ''),
|
||||
password=db_data.get('password', ''),
|
||||
pool_size=db_data.get('pool_size', 5),
|
||||
max_overflow=db_data.get('max_overflow', 10),
|
||||
pool_timeout=db_data.get('pool_timeout', 30),
|
||||
pool_recycle=db_data.get('pool_recycle', 3600),
|
||||
data_retention_days=db_data.get('data_retention_days', 90),
|
||||
auto_cleanup=db_data.get('auto_cleanup', True)
|
||||
)
|
||||
|
||||
wechat_data = self._config_data['wechat_app']
|
||||
wechat_app = WeChatConfig(
|
||||
enabled=wechat_data.get('enabled', False),
|
||||
corp_id=wechat_data.get('corp_id', ''),
|
||||
agent_id=wechat_data.get('agent_id', ''),
|
||||
secret=wechat_data.get('secret', ''),
|
||||
token=wechat_data.get('token', ''),
|
||||
encoding_aes_key=wechat_data.get('encoding_aes_key', ''),
|
||||
port=wechat_data.get('port', 18001),
|
||||
host=wechat_data.get('host', '0.0.0.0'),
|
||||
debug=wechat_data.get('debug', False),
|
||||
use_proxy=wechat_data.get('use_proxy', False),
|
||||
proxy_api_url=wechat_data.get('proxy_api_url', 'https://api.v6ole.top')
|
||||
)
|
||||
|
||||
md_data = self._config_data['markdown']
|
||||
markdown = MarkdownConfig(
|
||||
enabled=md_data.get('enabled', True),
|
||||
output_file=md_data.get('output_file', 'onu.md'),
|
||||
max_entries=md_data.get('max_entries', 1000),
|
||||
include_today_highlight=md_data.get('include_today_highlight', True),
|
||||
template_file=md_data.get('template_file', 'templates/announcement.md')
|
||||
)
|
||||
|
||||
# scheduler配置为可选
|
||||
scheduler = None
|
||||
if 'scheduler' in self._config_data:
|
||||
scheduler_data = self._config_data['scheduler']
|
||||
scheduler = SchedulerConfig(
|
||||
enabled=scheduler_data.get('enabled', False),
|
||||
timezone=scheduler_data.get('timezone', 'Asia/Shanghai'),
|
||||
jobs=scheduler_data.get('jobs', [])
|
||||
)
|
||||
|
||||
monitoring_data = self._config_data['monitoring']
|
||||
monitoring = MonitoringConfig(
|
||||
enabled=monitoring_data.get('enabled', True),
|
||||
health_check_interval=monitoring_data.get('health_check_interval', 300),
|
||||
alert_on_failure=monitoring_data.get('alert_on_failure', True),
|
||||
max_consecutive_failures=monitoring_data.get('max_consecutive_failures', 3),
|
||||
metrics_enabled=monitoring_data.get('metrics_enabled', True)
|
||||
)
|
||||
|
||||
return SystemConfig(
|
||||
debug=self._config_data['debug'],
|
||||
log_level=LogLevel(self._config_data['log_level']),
|
||||
log_file=self._config_data['log_file'],
|
||||
log_max_size=self._config_data['log_max_size'],
|
||||
log_backup_count=self._config_data['log_backup_count'],
|
||||
crawler=crawler,
|
||||
database=database,
|
||||
wechat_app=wechat_app,
|
||||
markdown=markdown,
|
||||
scheduler=scheduler,
|
||||
monitoring=monitoring,
|
||||
sources=self._config_data['sources']
|
||||
)
|
||||
|
||||
def get_config(self) -> SystemConfig:
|
||||
"""
|
||||
获取配置对象
|
||||
|
||||
Returns:
|
||||
SystemConfig: 系统配置对象
|
||||
|
||||
Raises:
|
||||
RuntimeError: 配置未加载
|
||||
"""
|
||||
if self._config is None:
|
||||
raise RuntimeError("配置未加载，请先调用 load_config()")
|
||||
return self._config
|
||||
|
||||
def reload_config(self) -> SystemConfig:
|
||||
"""
|
||||
重新加载配置
|
||||
|
||||
Returns:
|
||||
SystemConfig: 重新加载的系统配置对象
|
||||
"""
|
||||
self._config = None
|
||||
return self.load_config()
|
||||
|
||||
def get_value(self, key_path: str, default=None) -> Any:
|
||||
"""
|
||||
通过路径获取配置值
|
||||
|
||||
Args:
|
||||
key_path: 配置路径，如 'crawler.timeout' 或 'database.host'
|
||||
default: 默认值
|
||||
|
||||
Returns:
|
||||
配置值或默认值
|
||||
"""
|
||||
keys = key_path.split('.')
|
||||
value = self._config_data
|
||||
|
||||
try:
|
||||
for key in keys:
|
||||
value = value[key]
|
||||
return value
|
||||
except (KeyError, TypeError):
|
||||
return default
|
||||
|
||||
|
||||
# 全局配置管理器实例
|
||||
_config_manager = None
|
||||
|
||||
|
||||
def get_config_manager(config_file: Optional[str] = None) -> ConfigManager:
|
||||
"""
|
||||
获取全局配置管理器实例
|
||||
|
||||
Args:
|
||||
config_file: 配置文件路径
|
||||
|
||||
Returns:
|
||||
ConfigManager: 配置管理器实例
|
||||
"""
|
||||
global _config_manager
|
||||
if _config_manager is None:
|
||||
_config_manager = ConfigManager(config_file)
|
||||
return _config_manager
|
||||
|
||||
|
||||
def load_config(config_file: Optional[str] = None) -> SystemConfig:
|
||||
"""
|
||||
加载系统配置
|
||||
|
||||
Args:
|
||||
config_file: 配置文件路径
|
||||
|
||||
Returns:
|
||||
SystemConfig: 系统配置对象
|
||||
"""
|
||||
manager = get_config_manager(config_file)
|
||||
return manager.load_config()
|
||||
|
||||
|
||||
def get_config() -> SystemConfig:
|
||||
"""
|
||||
获取当前加载的配置
|
||||
|
||||
Returns:
|
||||
SystemConfig: 系统配置对象
|
||||
|
||||
Raises:
|
||||
RuntimeError: 配置未加载
|
||||
"""
|
||||
manager = get_config_manager()
|
||||
return manager.get_config()
|
||||
@@ -1,975 +0,0 @@
|
||||
"""
|
||||
PostgreSQL数据库连接和操作模块
|
||||
提供数据库连接池、CRUD操作、数据清理等功能
|
||||
"""
|
||||
|
||||
import psycopg2
|
||||
from psycopg2 import pool, extras
|
||||
from psycopg2.extras import RealDictCursor
|
||||
from contextlib import contextmanager
|
||||
from typing import List, Dict, Any, Optional, Generator
|
||||
from datetime import datetime, timedelta
|
||||
import threading
|
||||
from dataclasses import asdict
|
||||
|
||||
from .models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
||||
from .config_manager import get_config
|
||||
from .logger import get_logger
|
||||
from .reliability import retry_on_exception, RetryConfig
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class DatabaseConnectionPool:
|
||||
"""数据库连接池管理器"""
|
||||
|
||||
_instance = None
|
||||
_pool = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if self._pool is None:
|
||||
self._pool = None
|
||||
self._config = None
|
||||
|
||||
def init_pool(self, config):
|
||||
"""
|
||||
初始化连接池
|
||||
|
||||
Args:
|
||||
config: 数据库配置
|
||||
"""
|
||||
if self._pool is not None:
|
||||
return
|
||||
|
||||
try:
|
||||
self._config = config
|
||||
self._pool = psycopg2.pool.SimpleConnectionPool(
|
||||
minconn=config.pool_size,
|
||||
maxconn=config.pool_size + config.max_overflow,
|
||||
host=config.host,
|
||||
port=config.port,
|
||||
database=config.name,
|
||||
user=config.user,
|
||||
password=config.password,
|
||||
connect_timeout=config.pool_timeout
|
||||
)
|
||||
logger.info("数据库连接池初始化成功")
|
||||
except Exception as e:
|
||||
logger.error(f"数据库连接池初始化失败: {str(e)}")
|
||||
raise
|
||||
|
||||
def get_connection(self):
|
||||
"""
|
||||
获取数据库连接
|
||||
|
||||
Returns:
|
||||
数据库连接对象
|
||||
|
||||
Raises:
|
||||
Exception: 获取连接失败
|
||||
"""
|
||||
if self._pool is None:
|
||||
raise Exception("数据库连接池未初始化")
|
||||
|
||||
try:
|
||||
conn = self._pool.getconn()
|
||||
# 设置自动提交为False，需要手动提交
|
||||
conn.autocommit = False
|
||||
return conn
|
||||
except Exception as e:
|
||||
logger.error(f"获取数据库连接失败: {str(e)}")
|
||||
raise
|
||||
|
||||
def return_connection(self, conn):
|
||||
"""
|
||||
返回数据库连接到连接池
|
||||
|
||||
Args:
|
||||
conn: 数据库连接对象
|
||||
"""
|
||||
if self._pool and conn:
|
||||
try:
|
||||
self._pool.putconn(conn)
|
||||
except Exception as e:
|
||||
logger.warning(f"返回数据库连接失败: {str(e)}")
|
||||
|
||||
def close_all(self):
|
||||
"""关闭所有连接"""
|
||||
if self._pool:
|
||||
try:
|
||||
self._pool.closeall()
|
||||
logger.info("数据库连接池已关闭")
|
||||
except Exception as e:
|
||||
logger.error(f"关闭数据库连接池失败: {str(e)}")
|
||||
|
||||
|
||||
# 全局连接池实例
|
||||
_connection_pool = DatabaseConnectionPool()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_db_connection():
|
||||
"""
|
||||
获取数据库连接的上下文管理器
|
||||
|
||||
Yields:
|
||||
数据库连接对象
|
||||
"""
|
||||
conn = None
|
||||
try:
|
||||
conn = _connection_pool.get_connection()
|
||||
yield conn
|
||||
except Exception as e:
|
||||
logger.error(f"数据库连接错误: {str(e)}")
|
||||
raise
|
||||
finally:
|
||||
if conn:
|
||||
_connection_pool.return_connection(conn)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_db_cursor(commit: bool = True):
|
||||
"""
|
||||
获取数据库游标的上下文管理器
|
||||
|
||||
Args:
|
||||
commit: 是否自动提交事务
|
||||
|
||||
Yields:
|
||||
数据库游标对象
|
||||
"""
|
||||
with get_db_connection() as conn:
|
||||
cursor = None
|
||||
try:
|
||||
cursor = conn.cursor(cursor_factory=RealDictCursor)
|
||||
yield cursor
|
||||
if commit:
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
logger.error(f"数据库操作错误: {str(e)}")
|
||||
raise
|
||||
finally:
|
||||
if cursor:
|
||||
cursor.close()
|
||||
|
||||
|
||||
class DatabaseManager:
|
||||
"""数据库管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config()
|
||||
if self.config.database.enabled:
|
||||
_connection_pool.init_pool(self.config.database)
|
||||
else:
|
||||
logger.warning("数据库功能已禁用")
|
||||
|
||||
def init_database(self):
|
||||
"""初始化数据库表结构"""
|
||||
if not self.config.database.enabled:
|
||||
return
|
||||
|
||||
logger.info("开始初始化数据库表结构")
|
||||
|
||||
# 创建表的SQL语句
|
||||
create_tables_sql = """
|
||||
-- 定时搜索公告表（关键词匹配专用）
|
||||
CREATE TABLE IF NOT EXISTS auto_announcements (
|
||||
id SERIAL PRIMARY KEY,
|
||||
title VARCHAR(500) NOT NULL,
|
||||
publish_date TIMESTAMP NOT NULL,
|
||||
purchase_name VARCHAR(200),
|
||||
content_url TEXT,
|
||||
source_code VARCHAR(50) NOT NULL,
|
||||
source_name VARCHAR(100) NOT NULL,
|
||||
announcement_type VARCHAR(50) NOT NULL,
|
||||
crawled_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
content_hash VARCHAR(32) UNIQUE,
|
||||
keyword_matched BOOLEAN DEFAULT FALSE,
|
||||
date_filtered BOOLEAN DEFAULT TRUE,
|
||||
is_new BOOLEAN DEFAULT TRUE,
|
||||
is_today BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
-- 手动搜索公告表（全量数据专用）
|
||||
CREATE TABLE IF NOT EXISTS manual_announcements (
|
||||
id SERIAL PRIMARY KEY,
|
||||
title VARCHAR(500) NOT NULL,
|
||||
publish_date TIMESTAMP NOT NULL,
|
||||
purchase_name VARCHAR(200),
|
||||
content_url TEXT,
|
||||
source_code VARCHAR(50) NOT NULL,
|
||||
source_name VARCHAR(100) NOT NULL,
|
||||
announcement_type VARCHAR(50) NOT NULL,
|
||||
crawled_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
content_hash VARCHAR(32),
|
||||
keyword_matched BOOLEAN DEFAULT FALSE,
|
||||
date_filtered BOOLEAN DEFAULT TRUE,
|
||||
is_new BOOLEAN DEFAULT TRUE,
|
||||
is_today BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
-- 原公告表（保留兼容性）
|
||||
CREATE TABLE IF NOT EXISTS announcements (
|
||||
id SERIAL PRIMARY KEY,
|
||||
title VARCHAR(500) NOT NULL,
|
||||
publish_date TIMESTAMP NOT NULL,
|
||||
purchase_name VARCHAR(200),
|
||||
content_url TEXT,
|
||||
source_code VARCHAR(50) NOT NULL,
|
||||
source_name VARCHAR(100) NOT NULL,
|
||||
announcement_type VARCHAR(50) NOT NULL,
|
||||
crawled_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
crawl_mode VARCHAR(20) DEFAULT 'auto',
|
||||
content_hash VARCHAR(32) UNIQUE,
|
||||
keyword_matched BOOLEAN DEFAULT FALSE,
|
||||
date_filtered BOOLEAN DEFAULT TRUE,
|
||||
is_new BOOLEAN DEFAULT TRUE,
|
||||
is_today BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
-- 公告来源表
|
||||
CREATE TABLE IF NOT EXISTS announcement_sources (
|
||||
code VARCHAR(50) PRIMARY KEY,
|
||||
category_id INTEGER NOT NULL,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
type VARCHAR(50) NOT NULL
|
||||
);
|
||||
|
||||
-- 爬取结果表
|
||||
CREATE TABLE IF NOT EXISTS crawl_results (
|
||||
id SERIAL PRIMARY KEY,
|
||||
source_code VARCHAR(50) NOT NULL,
|
||||
status VARCHAR(20) NOT NULL,
|
||||
total_count INTEGER DEFAULT 0,
|
||||
new_count INTEGER DEFAULT 0,
|
||||
error_message TEXT,
|
||||
crawled_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
duration FLOAT DEFAULT 0.0,
|
||||
FOREIGN KEY (source_code) REFERENCES announcement_sources(code)
|
||||
);
|
||||
|
||||
-- 大化县政府网采购公告表（全部推送，不筛选）
|
||||
CREATE TABLE IF NOT EXISTS dahuagov_announcements (
|
||||
id SERIAL PRIMARY KEY,
|
||||
title VARCHAR(500) NOT NULL,
|
||||
publish_date TIMESTAMP NOT NULL,
|
||||
purchase_name VARCHAR(200),
|
||||
content_url TEXT,
|
||||
source_code VARCHAR(50) NOT NULL DEFAULT 'dahuagov',
|
||||
source_name VARCHAR(100) NOT NULL DEFAULT '大化县政府网采购公告',
|
||||
announcement_type VARCHAR(50) NOT NULL DEFAULT 'purchase',
|
||||
crawled_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
content_hash VARCHAR(32) UNIQUE,
|
||||
is_new BOOLEAN DEFAULT TRUE
|
||||
);
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_announcements_publish_date ON announcements(publish_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_announcements_source_code ON announcements(source_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_announcements_content_hash ON announcements(content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_announcements_created_at ON announcements(created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_results_crawled_at ON crawl_results(crawled_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_dahuagov_publish_date ON dahuagov_announcements(publish_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_dahuagov_content_hash ON dahuagov_announcements(content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_dahuagov_created_at ON dahuagov_announcements(created_at DESC);
|
||||
|
||||
-- 创建更新时间触发器
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
DROP TRIGGER IF EXISTS update_announcements_updated_at ON announcements;
|
||||
CREATE TRIGGER update_announcements_updated_at
|
||||
BEFORE UPDATE ON announcements
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
"""
|
||||
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(create_tables_sql)
|
||||
logger.info("数据库表结构初始化完成")
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def save_announcement(self, announcement: Announcement) -> bool:
|
||||
"""
|
||||
保存公告到数据库
|
||||
|
||||
Args:
|
||||
announcement: 公告对象
|
||||
|
||||
Returns:
|
||||
bool: 保存是否成功
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return False
|
||||
|
||||
# 生成内容哈希（如果还没有）
|
||||
if not announcement.content_hash:
|
||||
announcement.generate_content_hash()
|
||||
|
||||
sql = """
|
||||
INSERT INTO announcements (
|
||||
title, publish_date, purchase_name, content_url, source_code, source_name,
|
||||
announcement_type, crawled_at, content_hash, keyword_matched,
|
||||
date_filtered, is_new, is_today
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (content_hash) DO NOTHING
|
||||
"""
|
||||
|
||||
values = (
|
||||
announcement.title,
|
||||
announcement.publish_date,
|
||||
announcement.purchase_name,
|
||||
announcement.content_url,
|
||||
announcement.source_code,
|
||||
announcement.source_name,
|
||||
announcement.announcement_type.value,
|
||||
announcement.crawled_at,
|
||||
announcement.content_hash,
|
||||
announcement.keyword_matched,
|
||||
announcement.date_filtered,
|
||||
announcement.is_new,
|
||||
announcement.is_today
|
||||
)
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, values)
|
||||
affected_rows = cursor.rowcount
|
||||
if affected_rows > 0:
|
||||
logger.debug(f"成功保存公告: {announcement.title[:50]}...")
|
||||
return True
|
||||
else:
|
||||
logger.debug(f"公告已存在，跳过保存: {announcement.title[:50]}...")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"保存公告失败: {str(e)}")
|
||||
return False
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def save_announcements_batch(self, announcements: List[Announcement]) -> int:
|
||||
"""
|
||||
批量保存公告
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
int: 成功保存的数量
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return 0
|
||||
|
||||
if not announcements:
|
||||
return 0
|
||||
|
||||
# 为没有哈希的公告生成哈希
|
||||
for announcement in announcements:
|
||||
if not announcement.content_hash:
|
||||
announcement.generate_content_hash()
|
||||
|
||||
sql = """
|
||||
INSERT INTO announcements (
|
||||
title, publish_date, purchase_name, content_url, source_code, source_name,
|
||||
announcement_type, crawled_at, crawl_mode, content_hash, keyword_matched,
|
||||
date_filtered, is_new, is_today
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (content_hash) DO NOTHING
|
||||
"""
|
||||
|
||||
values = []
|
||||
for announcement in announcements:
|
||||
values.append((
|
||||
announcement.title,
|
||||
announcement.publish_date,
|
||||
announcement.purchase_name,
|
||||
announcement.content_url,
|
||||
announcement.source_code,
|
||||
announcement.source_name,
|
||||
announcement.announcement_type.value,
|
||||
announcement.crawled_at,
|
||||
announcement.crawl_mode,
|
||||
announcement.content_hash,
|
||||
announcement.keyword_matched,
|
||||
announcement.date_filtered,
|
||||
announcement.is_new,
|
||||
announcement.is_today
|
||||
))
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
extras.execute_batch(cursor, sql, values)
|
||||
affected_rows = cursor.rowcount
|
||||
logger.info(f"批量保存公告完成，成功保存 {affected_rows} 条")
|
||||
return affected_rows
|
||||
except Exception as e:
|
||||
logger.error(f"批量保存公告失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
def save_announcements_batch_to_table(self, announcements: List[Announcement], table_name: str) -> int:
|
||||
"""
|
||||
批量保存公告到指定表
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
table_name: 目标表名 ("auto_announcements" 或 "manual_announcements")
|
||||
|
||||
Returns:
|
||||
int: 成功保存的数量
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return 0
|
||||
|
||||
if not announcements:
|
||||
return 0
|
||||
|
||||
# 为没有哈希的公告生成哈希
|
||||
for announcement in announcements:
|
||||
if not announcement.content_hash:
|
||||
announcement.generate_content_hash()
|
||||
|
||||
# 根据表名决定是否使用ON CONFLICT
|
||||
if table_name == "manual_announcements":
|
||||
# 手动搜索表不使用唯一约束（允许重复）
|
||||
sql = f"""
|
||||
INSERT INTO {table_name} (
|
||||
title, publish_date, purchase_name, content_url, source_code, source_name,
|
||||
announcement_type, crawled_at, content_hash, keyword_matched,
|
||||
date_filtered, is_new, is_today
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
"""
|
||||
values = []
|
||||
for announcement in announcements:
|
||||
values.append((
|
||||
announcement.title,
|
||||
announcement.publish_date,
|
||||
announcement.purchase_name,
|
||||
announcement.content_url,
|
||||
announcement.source_code,
|
||||
announcement.source_name,
|
||||
announcement.announcement_type.value,
|
||||
announcement.crawled_at,
|
||||
announcement.content_hash,
|
||||
announcement.keyword_matched,
|
||||
announcement.date_filtered,
|
||||
announcement.is_new,
|
||||
announcement.is_today
|
||||
))
|
||||
else:
|
||||
# 自动搜索表使用唯一约束
|
||||
sql = f"""
|
||||
INSERT INTO {table_name} (
|
||||
title, publish_date, purchase_name, content_url, source_code, source_name,
|
||||
announcement_type, crawled_at, content_hash, keyword_matched,
|
||||
date_filtered, is_new, is_today
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (content_hash) DO NOTHING
|
||||
"""
|
||||
values = []
|
||||
for announcement in announcements:
|
||||
values.append((
|
||||
announcement.title,
|
||||
announcement.publish_date,
|
||||
announcement.purchase_name,
|
||||
announcement.content_url,
|
||||
announcement.source_code,
|
||||
announcement.source_name,
|
||||
announcement.announcement_type.value,
|
||||
announcement.crawled_at,
|
||||
announcement.content_hash,
|
||||
announcement.keyword_matched,
|
||||
announcement.date_filtered,
|
||||
announcement.is_new,
|
||||
announcement.is_today
|
||||
))
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
extras.execute_batch(cursor, sql, values)
|
||||
affected_rows = cursor.rowcount
|
||||
logger.info(f"批量保存公告到{table_name}完成，影响行数: {affected_rows}")
|
||||
return affected_rows
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"批量保存公告到{table_name}失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def get_announcements(self,
|
||||
source_code: Optional[str] = None,
|
||||
start_date: Optional[datetime] = None,
|
||||
end_date: Optional[datetime] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0) -> List[Announcement]:
|
||||
"""
|
||||
查询公告
|
||||
|
||||
Args:
|
||||
source_code: 来源代码过滤
|
||||
start_date: 开始日期
|
||||
end_date: 结束日期
|
||||
limit: 限制数量
|
||||
offset: 偏移量
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 公告列表
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return []
|
||||
|
||||
sql = """
|
||||
SELECT * FROM announcements
|
||||
WHERE 1=1
|
||||
"""
|
||||
params = []
|
||||
|
||||
if source_code:
|
||||
sql += " AND source_code = %s"
|
||||
params.append(source_code)
|
||||
|
||||
if start_date:
|
||||
sql += " AND publish_date >= %s"
|
||||
params.append(start_date)
|
||||
|
||||
if end_date:
|
||||
sql += " AND publish_date <= %s"
|
||||
params.append(end_date)
|
||||
|
||||
sql += " ORDER BY publish_date DESC LIMIT %s OFFSET %s"
|
||||
params.extend([limit, offset])
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, params)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
announcements = []
|
||||
for row in rows:
|
||||
# 转换数据类型
|
||||
row_dict = dict(row)
|
||||
row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type'])
|
||||
announcements.append(Announcement.from_dict(row_dict))
|
||||
|
||||
return announcements
|
||||
except Exception as e:
|
||||
logger.error(f"查询公告失败: {str(e)}")
|
||||
return []
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def save_crawl_result(self, result: CrawlResult) -> bool:
|
||||
"""
|
||||
保存爬取结果
|
||||
|
||||
Args:
|
||||
result: 爬取结果对象
|
||||
|
||||
Returns:
|
||||
bool: 保存是否成功
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return False
|
||||
|
||||
sql = """
|
||||
INSERT INTO crawl_results (
|
||||
source_code, status, total_count, new_count, error_message,
|
||||
crawled_at, duration
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
"""
|
||||
|
||||
values = (
|
||||
result.source.code,
|
||||
result.status.value,
|
||||
result.total_count,
|
||||
result.new_count,
|
||||
result.error_message,
|
||||
result.crawled_at,
|
||||
result.duration
|
||||
)
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, values)
|
||||
logger.debug(f"保存爬取结果: {result.source.name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存爬取结果失败: {str(e)}")
|
||||
return False
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def cleanup_expired_data(self, days: int = 90) -> int:
|
||||
"""
|
||||
清理过期数据
|
||||
|
||||
Args:
|
||||
days: 保留天数
|
||||
|
||||
Returns:
|
||||
int: 清理的记录数
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return 0
|
||||
|
||||
cutoff_date = datetime.now() - timedelta(days=days)
|
||||
|
||||
sql = "DELETE FROM announcements WHERE created_at < %s"
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, (cutoff_date,))
|
||||
deleted_count = cursor.rowcount
|
||||
logger.info(f"清理过期数据完成，删除 {deleted_count} 条记录")
|
||||
return deleted_count
|
||||
except Exception as e:
|
||||
logger.error(f"清理过期数据失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取统计信息
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 统计数据
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return {}
|
||||
|
||||
# 统计所有表的综合信息
|
||||
sql = """
|
||||
SELECT
|
||||
SUM(total_count) as total_announcements,
|
||||
SUM(today_count) as today_announcements,
|
||||
SUM(new_count) as new_announcements,
|
||||
COUNT(DISTINCT source_code) as sources_count,
|
||||
MAX(last_crawl_time) as last_crawl_time
|
||||
FROM (
|
||||
SELECT
|
||||
COUNT(*) as total_count,
|
||||
COUNT(CASE WHEN is_today THEN 1 END) as today_count,
|
||||
COUNT(CASE WHEN is_new THEN 1 END) as new_count,
|
||||
source_code,
|
||||
MAX(crawled_at) as last_crawl_time
|
||||
FROM announcements
|
||||
GROUP BY source_code
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
COUNT(*) as total_count,
|
||||
COUNT(CASE WHEN is_today THEN 1 END) as today_count,
|
||||
COUNT(CASE WHEN is_new THEN 1 END) as new_count,
|
||||
source_code,
|
||||
MAX(crawled_at) as last_crawl_time
|
||||
FROM auto_announcements
|
||||
GROUP BY source_code
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
COUNT(*) as total_count,
|
||||
COUNT(CASE WHEN is_today THEN 1 END) as today_count,
|
||||
COUNT(CASE WHEN is_new THEN 1 END) as new_count,
|
||||
source_code,
|
||||
MAX(crawled_at) as last_crawl_time
|
||||
FROM manual_announcements
|
||||
GROUP BY source_code
|
||||
) as combined_stats
|
||||
"""
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql)
|
||||
result = cursor.fetchone()
|
||||
stats = dict(result) if result else {}
|
||||
|
||||
# 添加各表详细统计
|
||||
detail_sql = """
|
||||
SELECT
|
||||
'announcements' as table_name,
|
||||
COUNT(*) as count,
|
||||
COUNT(DISTINCT source_code) as sources,
|
||||
MAX(crawled_at) as last_crawl
|
||||
FROM announcements
|
||||
UNION ALL
|
||||
SELECT
|
||||
'auto_announcements' as table_name,
|
||||
COUNT(*) as count,
|
||||
COUNT(DISTINCT source_code) as sources,
|
||||
MAX(crawled_at) as last_crawl
|
||||
FROM auto_announcements
|
||||
UNION ALL
|
||||
SELECT
|
||||
'manual_announcements' as table_name,
|
||||
COUNT(*) as count,
|
||||
COUNT(DISTINCT source_code) as sources,
|
||||
MAX(crawled_at) as last_crawl
|
||||
FROM manual_announcements
|
||||
UNION ALL
|
||||
SELECT
|
||||
'dahuagov_announcements' as table_name,
|
||||
COUNT(*) as count,
|
||||
COUNT(DISTINCT source_code) as sources,
|
||||
MAX(crawled_at) as last_crawl
|
||||
FROM dahuagov_announcements
|
||||
"""
|
||||
|
||||
cursor.execute(detail_sql)
|
||||
detail_results = cursor.fetchall()
|
||||
|
||||
stats['table_details'] = {row['table_name']: {
|
||||
'count': row['count'],
|
||||
'sources': row['sources'],
|
||||
'last_crawl': row['last_crawl']
|
||||
} for row in detail_results}
|
||||
|
||||
return stats
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取统计信息失败: {str(e)}")
|
||||
return {}
|
||||
|
||||
def is_announcement_exists(self, content_hash: str) -> bool:
|
||||
"""
|
||||
检查公告是否已存在
|
||||
|
||||
Args:
|
||||
content_hash: 内容哈希
|
||||
|
||||
Returns:
|
||||
bool: 是否存在
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return False
|
||||
|
||||
# 检查所有表中是否存在
|
||||
sql = """
|
||||
SELECT 1 FROM (
|
||||
SELECT content_hash FROM announcements WHERE content_hash = %s
|
||||
UNION ALL
|
||||
SELECT content_hash FROM auto_announcements WHERE content_hash = %s
|
||||
UNION ALL
|
||||
SELECT content_hash FROM manual_announcements WHERE content_hash = %s
|
||||
UNION ALL
|
||||
SELECT content_hash FROM dahuagov_announcements WHERE content_hash = %s
|
||||
) as combined_check LIMIT 1
|
||||
"""
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, (content_hash, content_hash, content_hash, content_hash))
|
||||
return cursor.fetchone() is not None
|
||||
except Exception as e:
|
||||
logger.error(f"检查公告存在性失败: {str(e)}")
|
||||
return False
|
||||
|
||||
def get_recent_announcements(self, hours: int = 24) -> List[Announcement]:
|
||||
"""
|
||||
获取最近的公告
|
||||
|
||||
Args:
|
||||
hours: 最近小时数
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 公告列表
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return []
|
||||
|
||||
cutoff_time = datetime.now() - timedelta(hours=hours)
|
||||
|
||||
sql = """
|
||||
SELECT * FROM announcements
|
||||
WHERE crawled_at >= %s
|
||||
ORDER BY crawled_at DESC
|
||||
"""
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, (cutoff_time,))
|
||||
rows = cursor.fetchall()
|
||||
|
||||
announcements = []
|
||||
for row in rows:
|
||||
row_dict = dict(row)
|
||||
row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type'])
|
||||
announcements.append(Announcement.from_dict(row_dict))
|
||||
|
||||
return announcements
|
||||
except Exception as e:
|
||||
logger.error(f"获取最近公告失败: {str(e)}")
|
||||
return []
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def save_dahuagov_announcements(self, announcements: List[Announcement]) -> int:
|
||||
"""
|
||||
保存大化县政府网公告（全部推送，不筛选关键词）
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
int: 成功保存的新公告数量
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return 0
|
||||
|
||||
if not announcements:
|
||||
return 0
|
||||
|
||||
# 为没有哈希的公告生成哈希
|
||||
for announcement in announcements:
|
||||
if not announcement.content_hash:
|
||||
announcement.generate_content_hash()
|
||||
|
||||
sql = """
|
||||
INSERT INTO dahuagov_announcements (
|
||||
title, publish_date, purchase_name, content_url, source_code, source_name,
|
||||
announcement_type, crawled_at, content_hash, is_new
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (content_hash) DO NOTHING
|
||||
"""
|
||||
|
||||
values = []
|
||||
for announcement in announcements:
|
||||
values.append((
|
||||
announcement.title,
|
||||
announcement.publish_date,
|
||||
announcement.purchase_name,
|
||||
announcement.content_url,
|
||||
announcement.source_code,
|
||||
announcement.source_name,
|
||||
announcement.announcement_type.value,
|
||||
announcement.crawled_at,
|
||||
announcement.content_hash,
|
||||
announcement.is_new
|
||||
))
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
extras.execute_batch(cursor, sql, values)
|
||||
affected_rows = cursor.rowcount
|
||||
logger.info(f"保存大化县公告完成，新增 {affected_rows} 条")
|
||||
return affected_rows
|
||||
except Exception as e:
|
||||
logger.error(f"保存大化县公告失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def get_new_dahuagov_announcements(self) -> List[Announcement]:
|
||||
"""
|
||||
获取大化县未推送的新公告（is_new = TRUE）
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 未推送的公告列表
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return []
|
||||
|
||||
sql = """
|
||||
SELECT * FROM dahuagov_announcements
|
||||
WHERE is_new = TRUE
|
||||
ORDER BY publish_date DESC
|
||||
"""
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
announcements = []
|
||||
for row in rows:
|
||||
row_dict = dict(row)
|
||||
row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type'])
|
||||
announcements.append(Announcement.from_dict(row_dict))
|
||||
|
||||
return announcements
|
||||
except Exception as e:
|
||||
logger.error(f"获取大化县新公告失败: {str(e)}")
|
||||
return []
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def mark_dahuagov_announcements_sent(self, announcements: List[Announcement]) -> int:
|
||||
"""
|
||||
标记大化县公告已发送（is_new = FALSE）
|
||||
|
||||
Args:
|
||||
announcements: 已发送的公告列表
|
||||
|
||||
Returns:
|
||||
int: 更新的记录数
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return 0
|
||||
|
||||
if not announcements:
|
||||
return 0
|
||||
|
||||
# 获取所有公告的哈希值
|
||||
hashes = [ann.content_hash for ann in announcements if ann.content_hash]
|
||||
|
||||
if not hashes:
|
||||
return 0
|
||||
|
||||
sql = """
|
||||
UPDATE dahuagov_announcements
|
||||
SET is_new = FALSE, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE content_hash = ANY(%s)
|
||||
"""
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, (hashes,))
|
||||
affected_rows = cursor.rowcount
|
||||
logger.info(f"标记大化县公告已发送完成，更新 {affected_rows} 条")
|
||||
return affected_rows
|
||||
except Exception as e:
|
||||
logger.error(f"标记大化县公告已发送失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
|
||||
# 全局数据库管理器实例
|
||||
_db_manager = None
|
||||
|
||||
|
||||
def get_database_manager() -> DatabaseManager:
|
||||
"""
|
||||
获取数据库管理器实例
|
||||
|
||||
Returns:
|
||||
DatabaseManager: 数据库管理器实例
|
||||
"""
|
||||
global _db_manager
|
||||
if _db_manager is None:
|
||||
_db_manager = DatabaseManager()
|
||||
return _db_manager
|
||||
|
||||
|
||||
def init_database():
|
||||
"""初始化数据库"""
|
||||
manager = get_database_manager()
|
||||
manager.init_database()
|
||||
|
||||
|
||||
def cleanup_database():
|
||||
"""清理数据库连接"""
|
||||
_connection_pool.close_all()
|
||||
@@ -1,371 +0,0 @@
|
||||
"""
|
||||
统一日志管理模块
|
||||
提供结构化日志记录功能，支持控制台和文件输出
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import logging.handlers
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
from datetime import datetime
|
||||
|
||||
from .config_manager import get_config
|
||||
|
||||
|
||||
class ColoredFormatter(logging.Formatter):
|
||||
"""带颜色的日志格式化器"""
|
||||
|
||||
# ANSI颜色代码
|
||||
COLORS = {
|
||||
'DEBUG': '\033[36m', # 青色
|
||||
'INFO': '\033[32m', # 绿色
|
||||
'WARNING': '\033[33m', # 黄色
|
||||
'ERROR': '\033[31m', # 红色
|
||||
'CRITICAL': '\033[35m', # 紫色
|
||||
}
|
||||
RESET = '\033[0m' # 重置颜色
|
||||
|
||||
def format(self, record):
|
||||
# 检查是否已经包含ANSI颜色代码
|
||||
if '\033[' in record.levelname:
|
||||
# 如果已经着色，直接返回原始格式
|
||||
return super().format(record)
|
||||
|
||||
# 保存原始值
|
||||
original_levelname = record.levelname
|
||||
original_msg = record.msg
|
||||
|
||||
# 添加颜色
|
||||
if record.levelname in self.COLORS:
|
||||
# 为levelname添加颜色
|
||||
record.levelname = f"{self.COLORS[record.levelname]}{record.levelname}{self.RESET}"
|
||||
# 为消息添加颜色
|
||||
record.msg = f"{self.COLORS[original_levelname]}{record.msg}{self.RESET}"
|
||||
|
||||
# 格式化
|
||||
result = super().format(record)
|
||||
|
||||
# 恢复原始值，避免影响其他处理器
|
||||
record.levelname = original_levelname
|
||||
record.msg = original_msg
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class Logger:
|
||||
"""统一日志管理器"""
|
||||
|
||||
_instance = None
|
||||
_initialized = False
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
if not self._initialized:
|
||||
self._loggers = {}
|
||||
self._config = None
|
||||
self._file_handler = None # 全局文件处理器，避免重复创建
|
||||
self._initialized = True
|
||||
|
||||
def init_logger(self, name: str = "gx_gp_monitor", config=None) -> logging.Logger:
|
||||
"""
|
||||
初始化日志器
|
||||
|
||||
Args:
|
||||
name: 日志器名称
|
||||
config: 配置对象，如果为None则从全局配置加载
|
||||
|
||||
Returns:
|
||||
logging.Logger: 配置好的日志器实例
|
||||
"""
|
||||
if name in self._loggers:
|
||||
return self._loggers[name]
|
||||
|
||||
# 获取配置
|
||||
if config is None:
|
||||
try:
|
||||
self._config = get_config()
|
||||
except RuntimeError:
|
||||
# 配置未加载，使用默认配置
|
||||
self._config = self._get_default_config()
|
||||
else:
|
||||
self._config = config
|
||||
|
||||
# 创建日志器
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(getattr(logging, self._config.log_level.value))
|
||||
|
||||
# 防止消息传播到父logger，避免重复记录
|
||||
logger.propagate = False
|
||||
|
||||
# 避免重复添加处理器
|
||||
if logger.handlers:
|
||||
return logger
|
||||
|
||||
# 创建格式化器
|
||||
formatter = logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
|
||||
# 控制台处理器
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setLevel(getattr(logging, self._config.log_level.value))
|
||||
|
||||
# 使用彩色格式化器（如果支持）
|
||||
if sys.platform != 'win32' and 'TERM' in os.environ:
|
||||
colored_formatter = ColoredFormatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
console_handler.setFormatter(colored_formatter)
|
||||
else:
|
||||
console_handler.setFormatter(formatter)
|
||||
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# 文件处理器（如果配置了日志文件）- 使用全局文件处理器避免重复
|
||||
if self._config.log_file and self._file_handler is None:
|
||||
log_dir = Path(self._config.log_file).parent
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._file_handler = logging.handlers.RotatingFileHandler(
|
||||
self._config.log_file,
|
||||
maxBytes=self._config.log_max_size,
|
||||
backupCount=self._config.log_backup_count,
|
||||
encoding='utf-8'
|
||||
)
|
||||
self._file_handler.setLevel(getattr(logging, self._config.log_level.value))
|
||||
self._file_handler.setFormatter(formatter)
|
||||
|
||||
# 为所有logger添加全局文件处理器
|
||||
if self._file_handler is not None:
|
||||
logger.addHandler(self._file_handler)
|
||||
|
||||
self._loggers[name] = logger
|
||||
return logger
|
||||
|
||||
def _get_default_config(self):
|
||||
"""获取默认配置"""
|
||||
from .config_manager import LogLevel
|
||||
|
||||
class DefaultConfig:
|
||||
def __init__(self):
|
||||
self.log_level = LogLevel.INFO
|
||||
self.log_file = "logs/gx_gp_monitor.log"
|
||||
self.log_max_size = 10485760 # 10MB
|
||||
self.log_backup_count = 5
|
||||
|
||||
return DefaultConfig()
|
||||
|
||||
def get_logger(self, name: str = "gx_gp_monitor") -> logging.Logger:
|
||||
"""
|
||||
获取日志器
|
||||
|
||||
Args:
|
||||
name: 日志器名称
|
||||
|
||||
Returns:
|
||||
logging.Logger: 日志器实例
|
||||
"""
|
||||
if name not in self._loggers:
|
||||
return self.init_logger(name)
|
||||
return self._loggers[name]
|
||||
|
||||
def log_crawl_start(self, source_name: str, logger: Optional[logging.Logger] = None):
|
||||
"""记录爬取开始"""
|
||||
if logger is None:
|
||||
logger = self.get_logger()
|
||||
logger.info(f"开始爬取 {source_name}")
|
||||
|
||||
def log_crawl_success(self, source_name: str, count: int, duration: float,
|
||||
logger: Optional[logging.Logger] = None):
|
||||
"""记录爬取成功"""
|
||||
if logger is None:
|
||||
logger = self.get_logger()
|
||||
logger.info(f"{source_name} 爬取完成，共获取 {count} 条公告，耗时 {duration:.2f}秒")
|
||||
|
||||
def log_crawl_error(self, source_name: str, error: str,
|
||||
logger: Optional[logging.Logger] = None):
|
||||
"""记录爬取错误"""
|
||||
if logger is None:
|
||||
logger = self.get_logger()
|
||||
logger.error(f"{source_name} 爬取失败: {error}")
|
||||
|
||||
def log_announcement_filtered(self, reason: str, count: int,
|
||||
logger: Optional[logging.Logger] = None):
|
||||
"""记录公告筛选信息"""
|
||||
if logger is None:
|
||||
logger = self.get_logger()
|
||||
logger.info(f"公告筛选 - {reason}: {count} 条")
|
||||
|
||||
def log_database_operation(self, operation: str, table: str, count: int = 0,
|
||||
logger: Optional[logging.Logger] = None):
|
||||
"""记录数据库操作"""
|
||||
if logger is None:
|
||||
logger = self.get_logger()
|
||||
if count > 0:
|
||||
logger.info(f"数据库操作 - {operation} {table}: {count} 条记录")
|
||||
else:
|
||||
logger.info(f"数据库操作 - {operation} {table}")
|
||||
|
||||
def log_notification_sent(self, channel: str, recipient_count: int,
|
||||
logger: Optional[logging.Logger] = None):
|
||||
"""记录通知发送"""
|
||||
if logger is None:
|
||||
logger = self.get_logger()
|
||||
logger.info(f"通知发送 - {channel}: 向 {recipient_count} 个接收者发送")
|
||||
|
||||
def log_system_metrics(self, metrics: Dict[str, Any],
|
||||
logger: Optional[logging.Logger] = None):
|
||||
"""记录系统指标"""
|
||||
if logger is None:
|
||||
logger = self.get_logger()
|
||||
metrics_str = ", ".join([f"{k}={v}" for k, v in metrics.items()])
|
||||
logger.info(f"系统指标: {metrics_str}")
|
||||
|
||||
def log_performance_warning(self, operation: str, duration: float, threshold: float,
|
||||
logger: Optional[logging.Logger] = None):
|
||||
"""记录性能警告"""
|
||||
if logger is None:
|
||||
logger = self.get_logger()
|
||||
logger.warning(f"性能警告 - {operation} 耗时 {duration:.2f}秒，超过阈值 {threshold:.2f}秒")
|
||||
|
||||
|
||||
# 全局日志管理器实例
|
||||
_logger_manager = Logger()
|
||||
|
||||
|
||||
def get_logger(name: str = "gx_gp_monitor") -> logging.Logger:
|
||||
"""
|
||||
获取日志器
|
||||
|
||||
Args:
|
||||
name: 日志器名称
|
||||
|
||||
Returns:
|
||||
logging.Logger: 日志器实例
|
||||
"""
|
||||
return _logger_manager.get_logger(name)
|
||||
|
||||
|
||||
def init_logger(name: str = "gx_gp_monitor", config=None) -> logging.Logger:
|
||||
"""
|
||||
初始化并获取日志器
|
||||
|
||||
Args:
|
||||
name: 日志器名称
|
||||
config: 配置对象
|
||||
|
||||
Returns:
|
||||
logging.Logger: 日志器实例
|
||||
"""
|
||||
return _logger_manager.init_logger(name, config)
|
||||
|
||||
|
||||
def log_function_call(func_name: str, args: Optional[Dict[str, Any]] = None,
|
||||
logger: Optional[logging.Logger] = None):
|
||||
"""
|
||||
装饰器：记录函数调用
|
||||
|
||||
Args:
|
||||
func_name: 函数名称
|
||||
args: 函数参数
|
||||
logger: 日志器实例
|
||||
"""
|
||||
def decorator(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
nonlocal logger
|
||||
if logger is None:
|
||||
logger = get_logger()
|
||||
|
||||
start_time = datetime.now()
|
||||
logger.debug(f"调用函数: {func_name}")
|
||||
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
logger.debug(f"函数 {func_name} 执行完成，耗时 {duration:.3f}秒")
|
||||
return result
|
||||
except Exception as e:
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
logger.error(f"函数 {func_name} 执行失败，耗时 {duration:.3f}秒: {str(e)}")
|
||||
raise
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
# 便捷函数
|
||||
def log_info(message: str, logger: Optional[logging.Logger] = None):
|
||||
"""记录信息日志"""
|
||||
if logger is None:
|
||||
logger = get_logger()
|
||||
logger.info(message)
|
||||
|
||||
|
||||
def log_warning(message: str, logger: Optional[logging.Logger] = None):
|
||||
"""记录警告日志"""
|
||||
if logger is None:
|
||||
logger = get_logger()
|
||||
logger.warning(message)
|
||||
|
||||
|
||||
def log_error(message: str, logger: Optional[logging.Logger] = None):
|
||||
"""记录错误日志"""
|
||||
if logger is None:
|
||||
logger = get_logger()
|
||||
logger.error(message)
|
||||
|
||||
|
||||
def log_debug(message: str, logger: Optional[logging.Logger] = None):
|
||||
"""记录调试日志"""
|
||||
if logger is None:
|
||||
logger = get_logger()
|
||||
logger.debug(message)
|
||||
|
||||
|
||||
# 便捷的爬取日志记录函数
|
||||
def log_crawl_start(source_name: str):
|
||||
"""记录爬取开始"""
|
||||
_logger_manager.log_crawl_start(source_name)
|
||||
|
||||
|
||||
def log_crawl_success(source_name: str, count: int, duration: float):
|
||||
"""记录爬取成功"""
|
||||
_logger_manager.log_crawl_success(source_name, count, duration)
|
||||
|
||||
|
||||
def log_crawl_error(source_name: str, error: str):
|
||||
"""记录爬取错误"""
|
||||
_logger_manager.log_crawl_error(source_name, error)
|
||||
|
||||
|
||||
def log_announcement_filtered(reason: str, count: int):
|
||||
"""记录公告筛选信息"""
|
||||
_logger_manager.log_announcement_filtered(reason, count)
|
||||
|
||||
|
||||
def log_database_operation(operation: str, table: str, count: int = 0):
|
||||
"""记录数据库操作"""
|
||||
_logger_manager.log_database_operation(operation, table, count)
|
||||
|
||||
|
||||
def log_notification_sent(channel: str, recipient_count: int):
|
||||
"""记录通知发送"""
|
||||
_logger_manager.log_notification_sent(channel, recipient_count)
|
||||
|
||||
|
||||
def log_system_metrics(metrics: Dict[str, Any]):
|
||||
"""记录系统指标"""
|
||||
_logger_manager.log_system_metrics(metrics)
|
||||
|
||||
|
||||
def log_performance_warning(operation: str, duration: float, threshold: float):
|
||||
"""记录性能警告"""
|
||||
_logger_manager.log_performance_warning(operation, duration, threshold)
|
||||
@@ -1,256 +0,0 @@
|
||||
"""
|
||||
数据模型定义
|
||||
定义系统使用的数据结构和模型
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Optional, List, Dict, Any
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class AnnouncementType(Enum):
|
||||
"""公告类型枚举"""
|
||||
PURCHASE = "purchase" # 采购公告
|
||||
RESULT = "result" # 结果公告
|
||||
CONTRACT = "contract" # 合同公告
|
||||
CORRECTION = "correction" # 更正公告
|
||||
PRE_ANNOUNCEMENT = "pre_announcement" # 招标文件预公示
|
||||
SINGLE_SOURCE = "single_source" # 单一来源公示
|
||||
ELECTRONIC_MARKET = "electronic_market" # 电子卖场公示
|
||||
ACCEPTANCE = "acceptance" # 履约验收公示
|
||||
ENGINEERING = "engineering" # 工程类公告
|
||||
FRAMEWORK_AGREEMENT = "framework_agreement" # 框架协议征集公告
|
||||
FRAMEWORK_RESULT = "framework_result" # 框架协议入围结果公告
|
||||
FRAMEWORK_SUMMARY = "framework_summary" # 框架协议成交结果汇总公告
|
||||
INTENTION = "intention" # 采购意向公开
|
||||
|
||||
|
||||
class CrawlStatus(Enum):
|
||||
"""爬取状态枚举"""
|
||||
PENDING = "pending" # 待爬取
|
||||
RUNNING = "running" # 爬取中
|
||||
SUCCESS = "success" # 成功
|
||||
FAILED = "failed" # 失败
|
||||
PARTIAL = "partial" # 部分成功
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnnouncementSource:
|
||||
"""公告来源"""
|
||||
code: str # 来源代码，如 "ZcyAnnouncement1"
|
||||
category_id: int # 分类ID
|
||||
name: str # 显示名称，如 "采购公告"
|
||||
type: AnnouncementType # 公告类型
|
||||
|
||||
|
||||
@dataclass
|
||||
class Announcement:
|
||||
"""公告数据模型"""
|
||||
id: Optional[int] = None # 数据库ID
|
||||
title: str = "" # 公告标题
|
||||
publish_date: datetime = field(default_factory=datetime.now) # 发布时间
|
||||
purchase_name: str = "" # 发布单位
|
||||
content_url: str = "" # 内容链接
|
||||
source_code: str = "" # 来源代码
|
||||
source_name: str = "" # 来源名称
|
||||
announcement_type: AnnouncementType = AnnouncementType.PURCHASE # 公告类型
|
||||
|
||||
# 爬取相关字段
|
||||
crawled_at: Optional[datetime] = None # 爬取时间
|
||||
created_at: Optional[datetime] = None # 创建时间
|
||||
updated_at: Optional[datetime] = None # 更新时间
|
||||
crawl_mode: str = "auto" # 爬取模式：auto(自动)/manual(手动)
|
||||
|
||||
# 去重字段
|
||||
content_hash: Optional[str] = None # 内容哈希，用于去重
|
||||
|
||||
# 筛选相关
|
||||
keyword_matched: bool = False # 是否匹配关键词
|
||||
date_filtered: bool = True # 是否在日期范围内
|
||||
|
||||
# 业务字段
|
||||
is_new: bool = True # 是否为新公告
|
||||
is_today: bool = False # 是否为今日公告
|
||||
|
||||
def __post_init__(self):
|
||||
"""后初始化处理"""
|
||||
if isinstance(self.announcement_type, str):
|
||||
self.announcement_type = AnnouncementType(self.announcement_type)
|
||||
|
||||
if self.publish_date and isinstance(self.publish_date, str):
|
||||
try:
|
||||
self.publish_date = datetime.fromisoformat(self.publish_date.replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
# 如果解析失败，使用当前时间
|
||||
self.publish_date = datetime.now()
|
||||
|
||||
# 判断是否为今日公告
|
||||
today = datetime.now().date()
|
||||
if self.publish_date:
|
||||
self.is_today = self.publish_date.date() == today
|
||||
|
||||
@property
|
||||
def publish_date_str(self) -> str:
|
||||
"""获取发布日期字符串"""
|
||||
return self.publish_date.strftime("%Y-%m-%d") if self.publish_date else ""
|
||||
|
||||
@property
|
||||
def crawled_at_str(self) -> str:
|
||||
"""获取爬取时间字符串"""
|
||||
return self.crawled_at.strftime("%Y-%m-%d %H:%M:%S") if self.crawled_at else ""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
"id": self.id,
|
||||
"title": self.title,
|
||||
"publish_date": self.publish_date.isoformat() if self.publish_date else None,
|
||||
"purchase_name": self.purchase_name,
|
||||
"content_url": self.content_url,
|
||||
"source_code": self.source_code,
|
||||
"source_name": self.source_name,
|
||||
"announcement_type": self.announcement_type.value,
|
||||
"crawled_at": self.crawled_at.isoformat() if self.crawled_at else None,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
"content_hash": self.content_hash,
|
||||
"keyword_matched": self.keyword_matched,
|
||||
"date_filtered": self.date_filtered,
|
||||
"is_new": self.is_new,
|
||||
"is_today": self.is_today
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> 'Announcement':
|
||||
"""从字典创建实例"""
|
||||
# 处理枚举类型
|
||||
if 'announcement_type' in data and isinstance(data['announcement_type'], str):
|
||||
data['announcement_type'] = AnnouncementType(data['announcement_type'])
|
||||
|
||||
# 处理日期时间
|
||||
for date_field in ['publish_date', 'crawled_at', 'created_at', 'updated_at']:
|
||||
if date_field in data and data[date_field] and isinstance(data[date_field], str):
|
||||
try:
|
||||
data[date_field] = datetime.fromisoformat(data[date_field].replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
data[date_field] = None
|
||||
|
||||
return cls(**data)
|
||||
|
||||
def generate_content_hash(self) -> str:
|
||||
"""生成内容哈希用于去重"""
|
||||
import hashlib
|
||||
content = f"{self.title}|{self.publish_date_str}|{self.purchase_name}|{self.content_url}|{self.source_code}"
|
||||
self.content_hash = hashlib.md5(content.encode('utf-8')).hexdigest()
|
||||
return self.content_hash
|
||||
|
||||
def matches_keywords(self, keywords: List[str]) -> bool:
|
||||
"""检查是否匹配关键词"""
|
||||
if not keywords:
|
||||
return True
|
||||
|
||||
search_text = f"{self.title} {self.purchase_name}".lower()
|
||||
for keyword in keywords:
|
||||
if keyword.lower() in search_text:
|
||||
self.keyword_matched = True
|
||||
return True
|
||||
|
||||
self.keyword_matched = False
|
||||
return False
|
||||
|
||||
def in_date_range(self, start_date: Optional[str], end_date: Optional[str]) -> bool:
|
||||
"""检查是否在日期范围内"""
|
||||
if not self.publish_date:
|
||||
self.date_filtered = False
|
||||
return False
|
||||
|
||||
publish_date = self.publish_date.date()
|
||||
|
||||
try:
|
||||
if start_date:
|
||||
start = datetime.fromisoformat(start_date).date()
|
||||
if publish_date < start:
|
||||
self.date_filtered = False
|
||||
return False
|
||||
|
||||
if end_date:
|
||||
end = datetime.fromisoformat(end_date).date()
|
||||
if publish_date > end:
|
||||
self.date_filtered = False
|
||||
return False
|
||||
|
||||
self.date_filtered = True
|
||||
return True
|
||||
except ValueError:
|
||||
# 日期格式错误时，默认通过
|
||||
self.date_filtered = True
|
||||
return True
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlResult:
|
||||
"""爬取结果"""
|
||||
source: AnnouncementSource # 公告来源
|
||||
status: CrawlStatus # 爬取状态
|
||||
total_count: int = 0 # 总公告数
|
||||
new_count: int = 0 # 新增公告数
|
||||
error_message: Optional[str] = None # 错误信息
|
||||
announcements: List[Announcement] = field(default_factory=list) # 公告列表
|
||||
crawled_at: datetime = field(default_factory=datetime.now) # 爬取时间
|
||||
duration: float = 0.0 # 爬取耗时（秒）
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlSession:
|
||||
"""爬取会话"""
|
||||
session_id: str # 会话ID
|
||||
start_time: datetime # 开始时间
|
||||
end_time: Optional[datetime] = None # 结束时间
|
||||
status: CrawlStatus = CrawlStatus.PENDING # 会话状态
|
||||
total_sources: int = 0 # 总来源数
|
||||
completed_sources: int = 0 # 已完成来源数
|
||||
total_announcements: int = 0 # 总公告数
|
||||
new_announcements: int = 0 # 新增公告数
|
||||
results: List[CrawlResult] = field(default_factory=list) # 各来源结果
|
||||
|
||||
@property
|
||||
def duration(self) -> float:
|
||||
"""获取会话持续时间"""
|
||||
if self.end_time and self.start_time:
|
||||
return (self.end_time - self.start_time).total_seconds()
|
||||
elif self.start_time:
|
||||
return (datetime.now() - self.start_time).total_seconds()
|
||||
return 0.0
|
||||
|
||||
@property
|
||||
def progress(self) -> float:
|
||||
"""获取完成进度（0-1）"""
|
||||
if self.total_sources == 0:
|
||||
return 0.0
|
||||
return self.completed_sources / self.total_sources
|
||||
|
||||
|
||||
@dataclass
|
||||
class NotificationMessage:
|
||||
"""通知消息"""
|
||||
title: str # 消息标题
|
||||
content: str # 消息内容
|
||||
message_type: str = "text" # 消息类型：text, markdown, card
|
||||
recipients: List[str] = field(default_factory=lambda: ["@all"]) # 接收者列表
|
||||
attachments: Optional[Dict[str, Any]] = None # 附件信息
|
||||
created_at: datetime = field(default_factory=datetime.now) # 创建时间
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemMetrics:
|
||||
"""系统指标"""
|
||||
timestamp: datetime = field(default_factory=datetime.now) # 时间戳
|
||||
total_announcements: int = 0 # 总公告数
|
||||
today_announcements: int = 0 # 今日公告数
|
||||
new_announcements_today: int = 0 # 今日新增公告数
|
||||
crawl_sessions_today: int = 0 # 今日爬取会话数
|
||||
last_crawl_duration: float = 0.0 # 最后一次爬取耗时
|
||||
database_size: int = 0 # 数据库大小（字节）
|
||||
memory_usage: float = 0.0 # 内存使用率
|
||||
disk_usage: float = 0.0 # 磁盘使用率
|
||||
@@ -1,489 +0,0 @@
|
||||
"""
|
||||
高可用性模块
|
||||
提供重试机制、超时控制、幂等操作、异常处理和恢复功能
|
||||
"""
|
||||
|
||||
import time
|
||||
import random
|
||||
import hashlib
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from typing import Callable, Any, Optional, Type, Union, List
|
||||
from datetime import datetime, timedelta
|
||||
import threading
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from .logger import get_logger
|
||||
from .config_manager import get_config
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class RetryConfig:
|
||||
"""重试配置"""
|
||||
|
||||
def __init__(self,
|
||||
max_retries: int = 3,
|
||||
initial_delay: float = 1.0,
|
||||
max_delay: float = 60.0,
|
||||
backoff_factor: float = 2.0,
|
||||
jitter: bool = True):
|
||||
"""
|
||||
初始化重试配置
|
||||
|
||||
Args:
|
||||
max_retries: 最大重试次数
|
||||
initial_delay: 初始延迟时间（秒）
|
||||
max_delay: 最大延迟时间（秒）
|
||||
backoff_factor: 退避因子
|
||||
jitter: 是否添加随机抖动
|
||||
"""
|
||||
self.max_retries = max_retries
|
||||
self.initial_delay = initial_delay
|
||||
self.max_delay = max_delay
|
||||
self.backoff_factor = backoff_factor
|
||||
self.jitter = jitter
|
||||
|
||||
|
||||
class TimeoutConfig:
|
||||
"""超时配置"""
|
||||
|
||||
def __init__(self,
|
||||
connect_timeout: float = 10.0,
|
||||
read_timeout: float = 30.0,
|
||||
total_timeout: Optional[float] = None):
|
||||
"""
|
||||
初始化超时配置
|
||||
|
||||
Args:
|
||||
connect_timeout: 连接超时时间（秒）
|
||||
read_timeout: 读取超时时间（秒）
|
||||
total_timeout: 总超时时间（秒）
|
||||
"""
|
||||
self.connect_timeout = connect_timeout
|
||||
self.read_timeout = read_timeout
|
||||
self.total_timeout = total_timeout or (connect_timeout + read_timeout)
|
||||
|
||||
|
||||
class CircuitBreakerState:
|
||||
"""熔断器状态"""
|
||||
CLOSED = "closed" # 关闭状态，正常工作
|
||||
OPEN = "open" # 打开状态，快速失败
|
||||
HALF_OPEN = "half_open" # 半开状态，测试恢复
|
||||
|
||||
|
||||
class CircuitBreaker:
|
||||
"""熔断器实现"""
|
||||
|
||||
def __init__(self,
|
||||
failure_threshold: int = 5,
|
||||
recovery_timeout: int = 60,
|
||||
expected_exception: Type[Exception] = Exception):
|
||||
"""
|
||||
初始化熔断器
|
||||
|
||||
Args:
|
||||
failure_threshold: 失败阈值
|
||||
recovery_timeout: 恢复超时时间（秒）
|
||||
expected_exception: 期望的异常类型
|
||||
"""
|
||||
self.failure_threshold = failure_threshold
|
||||
self.recovery_timeout = recovery_timeout
|
||||
self.expected_exception = expected_exception
|
||||
|
||||
self.state = CircuitBreakerState.CLOSED
|
||||
self.failure_count = 0
|
||||
self.last_failure_time = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def __call__(self, func: Callable) -> Callable:
|
||||
"""装饰器实现"""
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return self._execute_with_circuit_breaker(func, *args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
def _execute_with_circuit_breaker(self, func: Callable, *args, **kwargs) -> Any:
|
||||
"""使用熔断器执行函数"""
|
||||
if self.state == CircuitBreakerState.OPEN:
|
||||
if self._should_attempt_reset():
|
||||
self.state = CircuitBreakerState.HALF_OPEN
|
||||
logger.info("熔断器半开，尝试恢复")
|
||||
else:
|
||||
raise CircuitBreakerOpenException("熔断器已打开")
|
||||
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
self._on_success()
|
||||
return result
|
||||
except self.expected_exception as e:
|
||||
self._on_failure()
|
||||
raise
|
||||
|
||||
def _should_attempt_reset(self) -> bool:
|
||||
"""检查是否应该尝试重置"""
|
||||
if self.last_failure_time is None:
|
||||
return True
|
||||
return (datetime.now() - self.last_failure_time).total_seconds() >= self.recovery_timeout
|
||||
|
||||
def _on_success(self):
|
||||
"""成功时的处理"""
|
||||
with self._lock:
|
||||
if self.state == CircuitBreakerState.HALF_OPEN:
|
||||
self.state = CircuitBreakerState.CLOSED
|
||||
self.failure_count = 0
|
||||
logger.info("熔断器关闭，服务恢复正常")
|
||||
|
||||
def _on_failure(self):
|
||||
"""失败时的处理"""
|
||||
with self._lock:
|
||||
self.failure_count += 1
|
||||
self.last_failure_time = datetime.now()
|
||||
|
||||
if self.failure_count >= self.failure_threshold:
|
||||
self.state = CircuitBreakerState.OPEN
|
||||
logger.warning(f"熔断器打开，失败次数达到阈值: {self.failure_count}")
|
||||
|
||||
|
||||
class CircuitBreakerOpenException(Exception):
|
||||
"""熔断器打开异常"""
|
||||
pass
|
||||
|
||||
|
||||
class IdempotencyKey:
|
||||
"""幂等性键生成器"""
|
||||
|
||||
@staticmethod
|
||||
def generate(*args, **kwargs) -> str:
|
||||
"""
|
||||
生成幂等性键
|
||||
|
||||
Args:
|
||||
*args: 位置参数
|
||||
**kwargs: 关键字参数
|
||||
|
||||
Returns:
|
||||
str: 幂等性键
|
||||
"""
|
||||
# 将参数转换为字符串并排序
|
||||
key_parts = []
|
||||
|
||||
# 处理位置参数
|
||||
for i, arg in enumerate(args):
|
||||
key_parts.append(f"arg_{i}:{str(arg)}")
|
||||
|
||||
# 处理关键字参数（排序以保证一致性）
|
||||
for key in sorted(kwargs.keys()):
|
||||
key_parts.append(f"{key}:{str(kwargs[key])}")
|
||||
|
||||
# 生成哈希
|
||||
key_string = "|".join(key_parts)
|
||||
return hashlib.md5(key_string.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
class IdempotencyManager:
|
||||
"""幂等性管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self._executed_keys = set()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def is_executed(self, key: str) -> bool:
|
||||
"""
|
||||
检查操作是否已执行
|
||||
|
||||
Args:
|
||||
key: 幂等性键
|
||||
|
||||
Returns:
|
||||
bool: 是否已执行
|
||||
"""
|
||||
with self._lock:
|
||||
return key in self._executed_keys
|
||||
|
||||
def mark_executed(self, key: str):
|
||||
"""
|
||||
标记操作已执行
|
||||
|
||||
Args:
|
||||
key: 幂等性键
|
||||
"""
|
||||
with self._lock:
|
||||
self._executed_keys.add(key)
|
||||
|
||||
def clear_expired_keys(self, max_age_seconds: int = 3600):
|
||||
"""
|
||||
清理过期的键（简化实现，实际应该使用时间戳）
|
||||
|
||||
Args:
|
||||
max_age_seconds: 最大年龄（秒）
|
||||
"""
|
||||
# 这里简化实现，实际项目中应该记录时间戳
|
||||
pass
|
||||
|
||||
|
||||
def retry_on_exception(retry_config: Optional[RetryConfig] = None,
|
||||
exceptions: tuple = (Exception,),
|
||||
logger: Optional[Any] = None) -> Callable:
|
||||
"""
|
||||
重试装饰器
|
||||
|
||||
Args:
|
||||
retry_config: 重试配置
|
||||
exceptions: 需要重试的异常类型
|
||||
logger: 日志器
|
||||
|
||||
Returns:
|
||||
Callable: 装饰器函数
|
||||
"""
|
||||
if retry_config is None:
|
||||
retry_config = RetryConfig()
|
||||
|
||||
if logger is None:
|
||||
logger = get_logger()
|
||||
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
last_exception = None
|
||||
|
||||
for attempt in range(retry_config.max_retries + 1):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except exceptions as e:
|
||||
last_exception = e
|
||||
|
||||
if attempt < retry_config.max_retries:
|
||||
# 计算延迟时间
|
||||
delay = min(
|
||||
retry_config.initial_delay * (retry_config.backoff_factor ** attempt),
|
||||
retry_config.max_delay
|
||||
)
|
||||
|
||||
# 添加随机抖动
|
||||
if retry_config.jitter:
|
||||
delay = delay * (0.5 + random.random() * 0.5)
|
||||
|
||||
logger.warning(
|
||||
f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{retry_config.max_retries + 1}): {str(e)}，"
|
||||
f"等待 {delay:.2f} 秒后重试"
|
||||
)
|
||||
time.sleep(delay)
|
||||
else:
|
||||
logger.error(
|
||||
f"函数 {func.__name__} 在 {retry_config.max_retries + 1} 次尝试后仍然失败: {str(e)}"
|
||||
)
|
||||
|
||||
raise last_exception
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
def timeout_wrapper(timeout_config: Optional[TimeoutConfig] = None) -> Callable:
|
||||
"""
|
||||
超时装饰器
|
||||
|
||||
Args:
|
||||
timeout_config: 超时配置
|
||||
|
||||
Returns:
|
||||
Callable: 装饰器函数
|
||||
"""
|
||||
if timeout_config is None:
|
||||
timeout_config = TimeoutConfig()
|
||||
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
import signal
|
||||
|
||||
def timeout_handler(signum, frame):
|
||||
raise TimeoutError(f"函数 {func.__name__} 执行超时")
|
||||
|
||||
# 设置信号处理器
|
||||
old_handler = signal.signal(signal.SIGALRM, timeout_handler)
|
||||
signal.alarm(int(timeout_config.total_timeout))
|
||||
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
signal.alarm(0) # 取消闹钟
|
||||
return result
|
||||
finally:
|
||||
signal.signal(signal.SIGALRM, old_handler)
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_with_retry(timeout_config: Optional[TimeoutConfig] = None,
|
||||
retry_config: Optional[RetryConfig] = None):
|
||||
"""
|
||||
创建带有重试机制的HTTP会话
|
||||
|
||||
Args:
|
||||
timeout_config: 超时配置
|
||||
retry_config: 重试配置
|
||||
|
||||
Yields:
|
||||
requests.Session: 配置好的会话对象
|
||||
"""
|
||||
if timeout_config is None:
|
||||
timeout_config = TimeoutConfig()
|
||||
|
||||
if retry_config is None:
|
||||
retry_config = RetryConfig()
|
||||
|
||||
session = requests.Session()
|
||||
|
||||
# 配置重试策略
|
||||
retry_strategy = Retry(
|
||||
total=retry_config.max_retries,
|
||||
backoff_factor=retry_config.backoff_factor,
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
)
|
||||
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
|
||||
# 设置默认超时
|
||||
session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
|
||||
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def safe_execute(func: Callable,
|
||||
fallback: Optional[Callable] = None,
|
||||
exceptions: tuple = (Exception,),
|
||||
logger: Optional[Any] = None) -> Any:
|
||||
"""
|
||||
安全执行函数，提供降级处理
|
||||
|
||||
Args:
|
||||
func: 要执行的函数
|
||||
fallback: 降级函数
|
||||
exceptions: 需要捕获的异常类型
|
||||
logger: 日志器
|
||||
|
||||
Returns:
|
||||
Any: 函数执行结果或降级结果
|
||||
"""
|
||||
if logger is None:
|
||||
logger = get_logger()
|
||||
|
||||
try:
|
||||
return func()
|
||||
except exceptions as e:
|
||||
logger.error(f"函数执行失败: {str(e)}")
|
||||
if fallback:
|
||||
try:
|
||||
logger.info("执行降级函数")
|
||||
return fallback()
|
||||
except Exception as fallback_e:
|
||||
logger.error(f"降级函数也执行失败: {str(fallback_e)}")
|
||||
return None
|
||||
|
||||
|
||||
class HealthChecker:
|
||||
"""健康检查器"""
|
||||
|
||||
def __init__(self, check_interval: int = 300):
|
||||
"""
|
||||
初始化健康检查器
|
||||
|
||||
Args:
|
||||
check_interval: 检查间隔（秒）
|
||||
"""
|
||||
self.check_interval = check_interval
|
||||
self.last_check = None
|
||||
self.is_healthy = True
|
||||
self.consecutive_failures = 0
|
||||
self.max_consecutive_failures = 3
|
||||
|
||||
def check_health(self) -> bool:
|
||||
"""
|
||||
执行健康检查
|
||||
|
||||
Returns:
|
||||
bool: 健康状态
|
||||
"""
|
||||
current_time = datetime.now()
|
||||
|
||||
# 检查是否需要执行检查
|
||||
if (self.last_check and
|
||||
(current_time - self.last_check).total_seconds() < self.check_interval):
|
||||
return self.is_healthy
|
||||
|
||||
self.last_check = current_time
|
||||
|
||||
try:
|
||||
# 执行健康检查逻辑
|
||||
self._perform_health_check()
|
||||
self.is_healthy = True
|
||||
self.consecutive_failures = 0
|
||||
logger.info("健康检查通过")
|
||||
return True
|
||||
except Exception as e:
|
||||
self.consecutive_failures += 1
|
||||
logger.warning(f"健康检查失败 ({self.consecutive_failures}/{self.max_consecutive_failures}): {str(e)}")
|
||||
|
||||
if self.consecutive_failures >= self.max_consecutive_failures:
|
||||
self.is_healthy = False
|
||||
logger.error("连续健康检查失败，系统标记为不健康")
|
||||
|
||||
return False
|
||||
|
||||
def _perform_health_check(self):
|
||||
"""执行具体的健康检查逻辑"""
|
||||
# 这里可以添加数据库连接检查、外部服务检查等
|
||||
config = get_config()
|
||||
|
||||
# 检查数据库连接（如果启用）
|
||||
if config.database.enabled:
|
||||
# 这里应该检查数据库连接
|
||||
pass
|
||||
|
||||
# 检查网络连接
|
||||
try:
|
||||
requests.get("https://www.baidu.com", timeout=5)
|
||||
except:
|
||||
raise Exception("网络连接检查失败")
|
||||
|
||||
|
||||
# 全局实例
|
||||
_circuit_breaker = CircuitBreaker()
|
||||
_idempotency_manager = IdempotencyManager()
|
||||
_health_checker = HealthChecker()
|
||||
|
||||
|
||||
def get_circuit_breaker() -> CircuitBreaker:
|
||||
"""获取全局熔断器实例"""
|
||||
return _circuit_breaker
|
||||
|
||||
|
||||
def get_idempotency_manager() -> IdempotencyManager:
|
||||
"""获取全局幂等性管理器实例"""
|
||||
return _idempotency_manager
|
||||
|
||||
|
||||
def get_health_checker() -> HealthChecker:
|
||||
"""获取全局健康检查器实例"""
|
||||
return _health_checker
|
||||
|
||||
|
||||
def check_system_health() -> bool:
|
||||
"""
|
||||
检查系统健康状态
|
||||
|
||||
Returns:
|
||||
bool: 系统是否健康
|
||||
"""
|
||||
return _health_checker.check_health()
|
||||
@@ -1 +0,0 @@
|
||||
"""爬虫模块"""
|
||||
@@ -1,355 +0,0 @@
|
||||
"""
|
||||
大化瑶族自治县政府采购公告爬虫
|
||||
爬取大化县政府网站的采购公告页面
|
||||
"""
|
||||
|
||||
import time
|
||||
import random
|
||||
from typing import List, Optional, Tuple
|
||||
from datetime import datetime
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from bs4 import BeautifulSoup
|
||||
import requests
|
||||
from fake_useragent import UserAgent
|
||||
|
||||
try:
|
||||
from ..core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
||||
from ..core.reliability import (
|
||||
retry_on_exception, RetryConfig, session_with_retry,
|
||||
TimeoutConfig, safe_execute, check_system_health
|
||||
)
|
||||
except ImportError:
|
||||
from core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
||||
from core.config_manager import get_config
|
||||
from core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
||||
from core.reliability import (
|
||||
retry_on_exception, RetryConfig, session_with_retry,
|
||||
TimeoutConfig, safe_execute, check_system_health
|
||||
)
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class DahuagovSpider:
|
||||
"""大化县政府网站爬虫"""
|
||||
|
||||
# 大化县政府网站配置
|
||||
BASE_URL = "http://www.gxdh.gov.cn"
|
||||
ANNOUNCEMENT_PATH = "/xxgk/zdlyxxgk/ggzypzly/zfcgly/cggg/"
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config()
|
||||
self.ua = UserAgent()
|
||||
self.session = None
|
||||
self.request_count = 0
|
||||
self.error_count = 0
|
||||
|
||||
def _get_random_user_agent(self) -> str:
|
||||
"""获取随机User-Agent"""
|
||||
try:
|
||||
from fake_useragent import UserAgent
|
||||
return self.ua.random
|
||||
except:
|
||||
return random.choice(self.config.crawler.user_agents)
|
||||
|
||||
def init_session(self):
|
||||
"""初始化会话"""
|
||||
if self.session is None:
|
||||
timeout_config = TimeoutConfig(
|
||||
connect_timeout=self.config.crawler.timeout,
|
||||
read_timeout=self.config.crawler.timeout
|
||||
)
|
||||
|
||||
retry_config = RetryConfig(
|
||||
max_retries=self.config.crawler.max_retries,
|
||||
initial_delay=self.config.crawler.retry_delay,
|
||||
max_delay=self.config.crawler.max_retry_delay,
|
||||
backoff_factor=self.config.crawler.backoff_factor
|
||||
)
|
||||
|
||||
self.session = requests.Session()
|
||||
|
||||
adapter = requests.adapters.HTTPAdapter(
|
||||
pool_connections=10,
|
||||
pool_maxsize=20,
|
||||
max_retries=0
|
||||
)
|
||||
self.session.mount('http://', adapter)
|
||||
self.session.mount('https://', adapter)
|
||||
|
||||
self.session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
|
||||
|
||||
return self.session
|
||||
|
||||
def close_session(self):
|
||||
"""关闭会话"""
|
||||
if self.session:
|
||||
self.session.close()
|
||||
self.session = None
|
||||
|
||||
def _fetch_page(self, url: str) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
获取页面内容
|
||||
|
||||
Args:
|
||||
url: 页面URL
|
||||
|
||||
Returns:
|
||||
Tuple[Optional[str], Optional[str]]: (页面内容, 错误信息)
|
||||
"""
|
||||
try:
|
||||
session = self.init_session()
|
||||
user_agent = self._get_random_user_agent()
|
||||
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Connection": "keep-alive",
|
||||
"Referer": self.BASE_URL
|
||||
}
|
||||
|
||||
# 添加随机延迟
|
||||
delay = random.uniform(
|
||||
self.config.crawler.request_delay,
|
||||
self.config.crawler.request_delay_max
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
response = session.get(url, headers=headers, timeout=self.session.timeout)
|
||||
|
||||
self.request_count += 1
|
||||
|
||||
if response.status_code == 200:
|
||||
# 确保使用正确的编码
|
||||
response.encoding = response.apparent_encoding or 'utf-8'
|
||||
return response.text, None
|
||||
else:
|
||||
return None, f"请求失败, 状态码: {response.status_code}"
|
||||
|
||||
except Exception as e:
|
||||
self.error_count += 1
|
||||
logger.error(f"获取页面异常: {str(e)}")
|
||||
return None, f"请求异常: {str(e)}"
|
||||
|
||||
def _parse_page(self, html: str, crawled_at: datetime) -> List[Announcement]:
|
||||
"""
|
||||
解析页面内容
|
||||
|
||||
Args:
|
||||
html: 页面HTML内容
|
||||
crawled_at: 爬取时间
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 解析后的公告列表
|
||||
"""
|
||||
announcements = []
|
||||
|
||||
try:
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# 查找公告列表
|
||||
# 大化县政府网站使用 ul.more-list 结构
|
||||
lists = soup.find_all('ul', class_='more-list')
|
||||
|
||||
if not lists:
|
||||
logger.info("未找到公告列表")
|
||||
return announcements
|
||||
|
||||
for ul in lists:
|
||||
lis = ul.find_all('li')
|
||||
|
||||
for li in lis:
|
||||
try:
|
||||
announcement = self._parse_li_element(li, crawled_at)
|
||||
if announcement:
|
||||
announcements.append(announcement)
|
||||
except Exception as e:
|
||||
logger.warning(f"解析单个公告失败: {str(e)}")
|
||||
continue
|
||||
|
||||
logger.info(f"成功解析 {len(announcements)} 条公告")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"解析页面失败: {str(e)}")
|
||||
|
||||
return announcements
|
||||
|
||||
def _parse_li_element(self, li, crawled_at: datetime) -> Optional[Announcement]:
|
||||
"""
|
||||
解析单个li元素
|
||||
|
||||
Args:
|
||||
li: BeautifulSoup li元素
|
||||
crawled_at: 爬取时间
|
||||
|
||||
Returns:
|
||||
Optional[Announcement]: 解析后的公告对象
|
||||
"""
|
||||
try:
|
||||
# 查找日期 span
|
||||
date_span = li.find('span')
|
||||
if not date_span:
|
||||
return None
|
||||
|
||||
date_text = date_span.get_text(strip=True)
|
||||
if not date_text:
|
||||
return None
|
||||
|
||||
# 解析日期
|
||||
try:
|
||||
publish_date = datetime.strptime(date_text, "%Y-%m-%d")
|
||||
except ValueError:
|
||||
logger.warning(f"日期格式无法解析: {date_text}")
|
||||
return None
|
||||
|
||||
# 查找链接和标题
|
||||
link_tag = li.find('a')
|
||||
if not link_tag:
|
||||
return None
|
||||
|
||||
title = link_tag.get('title', '') or link_tag.get_text(strip=True)
|
||||
if not title:
|
||||
return None
|
||||
|
||||
href = link_tag.get('href', '')
|
||||
if not href:
|
||||
return None
|
||||
|
||||
# 构建完整URL
|
||||
if href.startswith('./') or href.startswith('../'):
|
||||
content_url = urljoin(self.BASE_URL + self.ANNOUNCEMENT_PATH, href)
|
||||
elif href.startswith('/'):
|
||||
content_url = self.BASE_URL + href
|
||||
elif href.startswith('http'):
|
||||
content_url = href
|
||||
else:
|
||||
content_url = urljoin(self.BASE_URL + self.ANNOUNCEMENT_PATH, href)
|
||||
|
||||
# 创建公告对象
|
||||
announcement = Announcement(
|
||||
title=title,
|
||||
publish_date=publish_date,
|
||||
purchase_name="大化瑶族自治县", # 默认采购单位
|
||||
content_url=content_url,
|
||||
source_code="dahuagov",
|
||||
source_name="大化县政府网采购公告",
|
||||
announcement_type=AnnouncementType.PURCHASE,
|
||||
crawled_at=crawled_at,
|
||||
is_new=True
|
||||
)
|
||||
|
||||
# 生成内容哈希用于去重
|
||||
announcement.generate_content_hash()
|
||||
|
||||
return announcement
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"解析li元素失败: {str(e)}")
|
||||
return None
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=2))
|
||||
def crawl(self) -> CrawlResult:
|
||||
"""
|
||||
爬取公告（只爬取第一页）
|
||||
|
||||
Returns:
|
||||
CrawlResult: 爬取结果
|
||||
"""
|
||||
log_crawl_start("大化县政府网采购公告")
|
||||
|
||||
start_time = datetime.now()
|
||||
result = CrawlResult(
|
||||
source=AnnouncementSource(
|
||||
code="dahuagov",
|
||||
category_id=0,
|
||||
name="大化县政府网采购公告",
|
||||
type=AnnouncementType.PURCHASE
|
||||
),
|
||||
status=CrawlStatus.RUNNING,
|
||||
crawled_at=start_time
|
||||
)
|
||||
|
||||
try:
|
||||
# 构建完整URL（只爬取第一页）
|
||||
url = self.BASE_URL + self.ANNOUNCEMENT_PATH
|
||||
|
||||
logger.info(f"开始爬取大化县政府网站: {url}")
|
||||
|
||||
# 获取页面内容
|
||||
html, error_msg = self._fetch_page(url)
|
||||
|
||||
if error_msg:
|
||||
logger.warning(f"获取页面失败: {error_msg}")
|
||||
result.status = CrawlStatus.FAILED
|
||||
result.error_message = error_msg
|
||||
return result
|
||||
|
||||
if not html:
|
||||
logger.info("页面内容为空")
|
||||
result.status = CrawlStatus.SUCCESS
|
||||
result.total_count = 0
|
||||
result.new_count = 0
|
||||
return result
|
||||
|
||||
# 解析页面
|
||||
announcements = self._parse_page(html, start_time)
|
||||
|
||||
# 更新结果
|
||||
result.announcements = announcements
|
||||
result.total_count = len(announcements)
|
||||
result.new_count = len(announcements)
|
||||
result.status = CrawlStatus.SUCCESS
|
||||
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
result.duration = duration
|
||||
|
||||
log_crawl_success("大化县政府网采购公告", len(announcements), duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
result.duration = duration
|
||||
result.status = CrawlStatus.FAILED
|
||||
result.error_message = str(e)
|
||||
|
||||
log_crawl_error("大化县政府网采购公告", str(e))
|
||||
|
||||
return result
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""获取爬虫统计信息"""
|
||||
return {
|
||||
"request_count": self.request_count,
|
||||
"error_count": self.error_count,
|
||||
"error_rate": self.error_count / max(self.request_count, 1),
|
||||
"session_active": self.session is not None
|
||||
}
|
||||
|
||||
|
||||
def create_dahuagov_spider() -> DahuagovSpider:
|
||||
"""
|
||||
创建大化县爬虫实例
|
||||
|
||||
Returns:
|
||||
DahuagovSpider: 爬虫实例
|
||||
"""
|
||||
return DahuagovSpider()
|
||||
|
||||
|
||||
def crawl_dahuagov_announcements() -> List[CrawlResult]:
|
||||
"""
|
||||
便捷函数：爬取大化县公告
|
||||
|
||||
Returns:
|
||||
List[CrawlResult]: 爬取结果列表
|
||||
"""
|
||||
spider = create_dahuagov_spider()
|
||||
|
||||
try:
|
||||
result = spider.crawl()
|
||||
return [result]
|
||||
finally:
|
||||
spider.close_session()
|
||||
@@ -1,301 +0,0 @@
|
||||
"""
|
||||
数据解析器模块
|
||||
负责解析广西政府采购网的API响应数据
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import List, Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
from ..core.models import Announcement, AnnouncementSource, AnnouncementType
|
||||
from ..core.logger import get_logger
|
||||
except ImportError:
|
||||
from core.models import Announcement, AnnouncementSource, AnnouncementType
|
||||
from core.logger import get_logger
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class AnnouncementParser:
|
||||
"""公告数据解析器"""
|
||||
|
||||
@staticmethod
|
||||
def parse_api_response(response_data: Dict[str, Any],
|
||||
source: AnnouncementSource,
|
||||
crawled_at: datetime) -> List[Announcement]:
|
||||
"""
|
||||
解析API响应数据
|
||||
|
||||
Args:
|
||||
response_data: API响应数据
|
||||
source: 公告来源
|
||||
crawled_at: 爬取时间
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 解析后的公告列表
|
||||
"""
|
||||
if not response_data or not isinstance(response_data, dict):
|
||||
logger.warning("API响应数据无效")
|
||||
return []
|
||||
|
||||
try:
|
||||
# 检查响应状态
|
||||
if not response_data.get("success", False):
|
||||
logger.warning(f"API响应失败: {response_data.get('message', '未知错误')}")
|
||||
return []
|
||||
|
||||
# 获取数据部分
|
||||
result = response_data.get("result", {})
|
||||
data = result.get("data", {})
|
||||
records = data.get("data", [])
|
||||
|
||||
if not records:
|
||||
logger.info(f"来源 {source.name} 没有新数据")
|
||||
return []
|
||||
|
||||
announcements = []
|
||||
for record in records:
|
||||
try:
|
||||
announcement = AnnouncementParser._parse_single_record(
|
||||
record, source, crawled_at)
|
||||
if announcement:
|
||||
announcements.append(announcement)
|
||||
except Exception as e:
|
||||
logger.warning(f"解析公告记录失败: {str(e)}, 记录: {record}")
|
||||
continue
|
||||
|
||||
logger.info(f"成功解析 {len(announcements)}/{len(records)} 条公告记录")
|
||||
return announcements
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"解析API响应数据失败: {str(e)}")
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _parse_single_record(record: Dict[str, Any],
|
||||
source: AnnouncementSource,
|
||||
crawled_at: datetime) -> Optional[Announcement]:
|
||||
"""
|
||||
解析单个公告记录
|
||||
|
||||
Args:
|
||||
record: 公告记录数据
|
||||
source: 公告来源
|
||||
crawled_at: 爬取时间
|
||||
|
||||
Returns:
|
||||
Optional[Announcement]: 解析后的公告对象
|
||||
"""
|
||||
try:
|
||||
# 提取基本字段
|
||||
title_raw = record.get("title", "")
|
||||
title = str(title_raw).strip() if title_raw is not None else ""
|
||||
if not title:
|
||||
return None
|
||||
|
||||
# 解析发布时间
|
||||
publish_timestamp = record.get("publishDate")
|
||||
if not publish_timestamp:
|
||||
logger.warning(f"公告缺少发布时间: {title[:50]}...")
|
||||
return None
|
||||
|
||||
try:
|
||||
# 时间戳转换为datetime
|
||||
publish_date = datetime.fromtimestamp(int(publish_timestamp) / 1000)
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.warning(f"发布时间格式错误: {publish_timestamp}, 错误: {str(e)}")
|
||||
return None
|
||||
|
||||
# 提取其他字段
|
||||
purchase_name_raw = record.get("purchaseName", "")
|
||||
purchase_name = str(purchase_name_raw).strip() if purchase_name_raw is not None else ""
|
||||
article_id = record.get("articleId")
|
||||
|
||||
if not article_id:
|
||||
logger.warning(f"公告缺少文章ID: {title[:50]}...")
|
||||
return None
|
||||
|
||||
# 构建内容链接
|
||||
content_url = AnnouncementParser._build_content_url(
|
||||
source.category_id, source.code, article_id)
|
||||
|
||||
# 创建公告对象
|
||||
announcement = Announcement(
|
||||
title=title,
|
||||
publish_date=publish_date,
|
||||
purchase_name=purchase_name,
|
||||
content_url=content_url,
|
||||
source_code=source.code,
|
||||
source_name=source.name,
|
||||
announcement_type=source.type,
|
||||
crawled_at=crawled_at,
|
||||
is_new=True # 默认标记为新公告
|
||||
)
|
||||
|
||||
# 生成内容哈希用于去重
|
||||
announcement.generate_content_hash()
|
||||
|
||||
return announcement
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"解析单个公告记录失败: {str(e)}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _build_content_url(category_id: int, source_code: str, article_id: int) -> str:
|
||||
"""
|
||||
构建公告内容链接
|
||||
|
||||
Args:
|
||||
category_id: 分类ID
|
||||
source_code: 来源代码
|
||||
article_id: 文章ID
|
||||
|
||||
Returns:
|
||||
str: 内容链接
|
||||
"""
|
||||
return f"https://zfcg.gxzf.gov.cn/site/detail?parentId={category_id}&articleId={article_id}"
|
||||
|
||||
@staticmethod
|
||||
def validate_response_structure(response_data: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
验证API响应数据结构
|
||||
|
||||
Args:
|
||||
response_data: API响应数据
|
||||
|
||||
Returns:
|
||||
bool: 结构是否有效
|
||||
"""
|
||||
try:
|
||||
if not isinstance(response_data, dict):
|
||||
return False
|
||||
|
||||
# 检查必需的字段
|
||||
if "success" not in response_data:
|
||||
return False
|
||||
|
||||
if not response_data.get("success", False):
|
||||
return False
|
||||
|
||||
result = response_data.get("result", {})
|
||||
if not isinstance(result, dict):
|
||||
return False
|
||||
|
||||
data = result.get("data", {})
|
||||
if not isinstance(data, dict):
|
||||
return False
|
||||
|
||||
records = data.get("data", [])
|
||||
if not isinstance(records, list):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"验证响应结构失败: {str(e)}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def extract_pagination_info(response_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
提取分页信息
|
||||
|
||||
Args:
|
||||
response_data: API响应数据
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 分页信息
|
||||
"""
|
||||
try:
|
||||
result = response_data.get("result", {})
|
||||
data = result.get("data", {})
|
||||
|
||||
return {
|
||||
"total": data.get("total", 0),
|
||||
"page_no": data.get("pageNo", 1),
|
||||
"page_size": data.get("pageSize", 15),
|
||||
"pages": data.get("pages", 0),
|
||||
"empty": data.get("empty", True),
|
||||
"has_next": data.get("hasNext", False),
|
||||
"has_previous": data.get("hasPrevious", False)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"提取分页信息失败: {str(e)}")
|
||||
return {
|
||||
"total": 0,
|
||||
"page_no": 1,
|
||||
"page_size": 15,
|
||||
"pages": 0,
|
||||
"empty": True,
|
||||
"has_next": False,
|
||||
"has_previous": False
|
||||
}
|
||||
|
||||
|
||||
class SensitiveWordChecker:
|
||||
"""敏感词检查器"""
|
||||
|
||||
@staticmethod
|
||||
def parse_check_response(response_data: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
解析敏感词检查响应
|
||||
|
||||
Args:
|
||||
response_data: 检查响应数据
|
||||
|
||||
Returns:
|
||||
bool: 检查是否通过
|
||||
"""
|
||||
try:
|
||||
if not isinstance(response_data, dict):
|
||||
logger.warning("敏感词检查响应格式无效")
|
||||
return False
|
||||
|
||||
success = response_data.get("success", False)
|
||||
if not success:
|
||||
message = response_data.get("message", "未知错误")
|
||||
logger.warning(f"敏感词检查失败: {message}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"解析敏感词检查响应失败: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
class ErrorResponseParser:
|
||||
"""错误响应解析器"""
|
||||
|
||||
@staticmethod
|
||||
def parse_error(response_data: Dict[str, Any]) -> str:
|
||||
"""
|
||||
解析错误响应
|
||||
|
||||
Args:
|
||||
response_data: 错误响应数据
|
||||
|
||||
Returns:
|
||||
str: 错误信息
|
||||
"""
|
||||
try:
|
||||
if not isinstance(response_data, dict):
|
||||
return "响应格式无效"
|
||||
|
||||
# 尝试提取错误信息
|
||||
error_msg = response_data.get("message") or response_data.get("msg")
|
||||
if error_msg:
|
||||
return str(error_msg)
|
||||
|
||||
# 检查状态码
|
||||
errcode = response_data.get("errcode")
|
||||
if errcode:
|
||||
return f"错误码: {errcode}"
|
||||
|
||||
return "未知错误"
|
||||
|
||||
except Exception as e:
|
||||
return f"解析错误响应失败: {str(e)}"
|
||||
@@ -1,458 +0,0 @@
|
||||
"""
|
||||
爬虫核心模块
|
||||
实现广西政府采购网公告的智能爬取功能
|
||||
"""
|
||||
|
||||
import time
|
||||
import random
|
||||
import json
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from datetime import datetime
|
||||
from urllib.parse import urljoin
|
||||
import requests
|
||||
from fake_useragent import UserAgent
|
||||
|
||||
try:
|
||||
# 尝试相对导入
|
||||
from ..core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
||||
from ..core.reliability import (
|
||||
retry_on_exception, RetryConfig, session_with_retry,
|
||||
TimeoutConfig, safe_execute, check_system_health
|
||||
)
|
||||
from .parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser
|
||||
except ImportError:
|
||||
# 尝试绝对导入
|
||||
from core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
||||
from core.config_manager import get_config
|
||||
from core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
||||
from core.reliability import (
|
||||
retry_on_exception, RetryConfig, session_with_retry,
|
||||
TimeoutConfig, safe_execute, check_system_health
|
||||
)
|
||||
from crawler.parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class GXGPSpider:
|
||||
"""广西政府采购网爬虫"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config()
|
||||
self.ua = UserAgent()
|
||||
|
||||
# API端点
|
||||
self.base_url = self.config.crawler.base_url
|
||||
self.announcement_api = urljoin(self.base_url, "/portal/category")
|
||||
self.sensitive_check_api = urljoin(self.base_url, "/portal/sensitiveWords/check")
|
||||
|
||||
# 会话管理
|
||||
self.session = None
|
||||
|
||||
# 统计信息
|
||||
self.request_count = 0
|
||||
self.error_count = 0
|
||||
|
||||
def init_session(self):
|
||||
"""初始化会话"""
|
||||
if self.session is None:
|
||||
timeout_config = TimeoutConfig(
|
||||
connect_timeout=self.config.crawler.timeout,
|
||||
read_timeout=self.config.crawler.timeout
|
||||
)
|
||||
|
||||
retry_config = RetryConfig(
|
||||
max_retries=self.config.crawler.max_retries,
|
||||
initial_delay=self.config.crawler.retry_delay,
|
||||
max_delay=self.config.crawler.max_retry_delay,
|
||||
backoff_factor=self.config.crawler.backoff_factor
|
||||
)
|
||||
|
||||
self.session = requests.Session()
|
||||
|
||||
# 配置重试和超时
|
||||
adapter = requests.adapters.HTTPAdapter(
|
||||
pool_connections=10,
|
||||
pool_maxsize=20,
|
||||
max_retries=0 # 我们使用自己的重试逻辑
|
||||
)
|
||||
self.session.mount('http://', adapter)
|
||||
self.session.mount('https://', adapter)
|
||||
|
||||
# 设置默认超时
|
||||
self.session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
|
||||
|
||||
return self.session
|
||||
|
||||
def close_session(self):
|
||||
"""关闭会话"""
|
||||
if self.session:
|
||||
self.session.close()
|
||||
self.session = None
|
||||
|
||||
def get_random_user_agent(self) -> str:
|
||||
"""获取随机User-Agent"""
|
||||
try:
|
||||
return self.ua.random
|
||||
except:
|
||||
# fallback到配置的user agents
|
||||
return random.choice(self.config.crawler.user_agents)
|
||||
|
||||
def get_random_proxy(self) -> Optional[Dict[str, str]]:
|
||||
"""获取随机代理"""
|
||||
if not self.config.crawler.proxies:
|
||||
return None
|
||||
|
||||
proxy = random.choice(self.config.crawler.proxies)
|
||||
return {
|
||||
"http": proxy,
|
||||
"https": proxy
|
||||
}
|
||||
|
||||
def check_sensitive_words(self, payload: Dict[str, Any],
|
||||
category_code: str, childrencode: str) -> bool:
|
||||
"""
|
||||
执行敏感词检查
|
||||
|
||||
Args:
|
||||
payload: 请求参数
|
||||
category_code: 分类代码
|
||||
childrencode: 子分类代码
|
||||
|
||||
Returns:
|
||||
bool: 检查是否通过
|
||||
"""
|
||||
try:
|
||||
session = self.init_session()
|
||||
user_agent = self.get_random_user_agent()
|
||||
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": self.base_url,
|
||||
"Referer": f"{self.base_url}/site/category?parentId={category_code}&childrenCode={childrencode}",
|
||||
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
|
||||
}
|
||||
|
||||
proxies = self.get_random_proxy()
|
||||
|
||||
logger.debug(f"执行敏感词检查: {category_code}/{childrencode}")
|
||||
|
||||
response = session.post(
|
||||
self.sensitive_check_api,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
proxies=proxies,
|
||||
timeout=self.session.timeout
|
||||
)
|
||||
|
||||
self.request_count += 1
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
return SensitiveWordChecker.parse_check_response(result)
|
||||
else:
|
||||
logger.warning(f"敏感词检查请求失败, 状态码: {response.status_code}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"敏感词检查异常: {str(e)}")
|
||||
self.error_count += 1
|
||||
return False
|
||||
|
||||
def fetch_announcements_page(self, source: AnnouncementSource,
|
||||
page_no: int = 1) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""
|
||||
获取公告列表页数据
|
||||
|
||||
Args:
|
||||
source: 公告来源
|
||||
page_no: 页码
|
||||
|
||||
Returns:
|
||||
Tuple[Optional[Dict[str, Any]], Optional[str]]: (响应数据, 错误信息)
|
||||
"""
|
||||
try:
|
||||
session = self.init_session()
|
||||
|
||||
# 构建请求参数
|
||||
payload = {
|
||||
"keyword": "", # 关键词筛选，我们在筛选模块处理
|
||||
"publishDateBegin": self.config.crawler.start_date or "",
|
||||
"publishDateEnd": self.config.crawler.end_date or "",
|
||||
"pageNo": page_no,
|
||||
"pageSize": self.config.crawler.page_size,
|
||||
"categoryCode": source.code,
|
||||
"_t": int(time.time() * 1000)
|
||||
}
|
||||
|
||||
# 先执行敏感词检查
|
||||
if not self.check_sensitive_words(payload, str(source.category_id), source.code):
|
||||
return None, "敏感词检查失败"
|
||||
|
||||
# 执行公告数据请求
|
||||
user_agent = self.get_random_user_agent()
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": self.base_url,
|
||||
"Referer": f"{self.base_url}/site/category?parentId={source.category_id}&childrenCode={source.code}",
|
||||
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
|
||||
}
|
||||
|
||||
proxies = self.get_random_proxy()
|
||||
|
||||
logger.debug(f"请求公告数据: {source.name} 第{page_no}页")
|
||||
|
||||
# 添加请求间延迟
|
||||
if page_no > 1:
|
||||
delay = random.uniform(
|
||||
self.config.crawler.request_delay,
|
||||
self.config.crawler.request_delay_max
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
response = session.post(
|
||||
self.announcement_api,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
proxies=proxies,
|
||||
timeout=self.session.timeout
|
||||
)
|
||||
|
||||
self.request_count += 1
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get("success", False):
|
||||
return data, None
|
||||
else:
|
||||
error_msg = ErrorResponseParser.parse_error(data)
|
||||
return None, f"API返回失败: {error_msg}"
|
||||
else:
|
||||
return None, f"请求失败, 状态码: {response.status_code}"
|
||||
|
||||
except requests.exceptions.Timeout as e:
|
||||
self.error_count += 1
|
||||
return None, f"请求超时: {str(e)}"
|
||||
except requests.exceptions.ProxyError as e:
|
||||
self.error_count += 1
|
||||
return None, f"代理错误: {str(e)}"
|
||||
except Exception as e:
|
||||
self.error_count += 1
|
||||
logger.error(f"获取公告数据异常: {str(e)}")
|
||||
return None, f"请求异常: {str(e)}"
|
||||
|
||||
def crawl_source(self, source: AnnouncementSource,
|
||||
max_pages: Optional[int] = None) -> CrawlResult:
|
||||
"""
|
||||
爬取单个来源的公告
|
||||
|
||||
Args:
|
||||
source: 公告来源
|
||||
max_pages: 最大页数限制
|
||||
|
||||
Returns:
|
||||
CrawlResult: 爬取结果
|
||||
"""
|
||||
if max_pages is None:
|
||||
max_pages = self.config.crawler.max_pages
|
||||
|
||||
log_crawl_start(source.name)
|
||||
|
||||
start_time = datetime.now()
|
||||
result = CrawlResult(
|
||||
source=source,
|
||||
status=CrawlStatus.RUNNING,
|
||||
crawled_at=start_time
|
||||
)
|
||||
|
||||
try:
|
||||
page = 1
|
||||
all_announcements = []
|
||||
|
||||
while page <= max_pages:
|
||||
# 获取页面数据
|
||||
response_data, error_msg = self.fetch_announcements_page(source, page)
|
||||
|
||||
if error_msg:
|
||||
logger.warning(f"{source.name} 第{page}页获取失败: {error_msg}")
|
||||
result.status = CrawlStatus.FAILED
|
||||
result.error_message = error_msg
|
||||
break
|
||||
|
||||
if not response_data:
|
||||
logger.info(f"{source.name} 第{page}页无数据")
|
||||
break
|
||||
|
||||
# 解析分页信息
|
||||
pagination = AnnouncementParser.extract_pagination_info(response_data)
|
||||
result.total_count = pagination["total"]
|
||||
|
||||
# 解析公告数据
|
||||
announcements = AnnouncementParser.parse_api_response(
|
||||
response_data, source, start_time)
|
||||
|
||||
if not announcements:
|
||||
logger.info(f"{source.name} 第{page}页解析到0条公告")
|
||||
break
|
||||
|
||||
all_announcements.extend(announcements)
|
||||
|
||||
# 检查是否还有下一页
|
||||
if not pagination["has_next"] or pagination["empty"]:
|
||||
break
|
||||
|
||||
page += 1
|
||||
|
||||
# 更新结果
|
||||
result.announcements = all_announcements
|
||||
result.new_count = len(all_announcements) # 这里的新增数需要在筛选后确定
|
||||
|
||||
if result.status != CrawlStatus.FAILED:
|
||||
result.status = CrawlStatus.SUCCESS
|
||||
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
result.duration = duration
|
||||
|
||||
log_crawl_success(source.name, len(all_announcements), duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
result.duration = duration
|
||||
result.status = CrawlStatus.FAILED
|
||||
result.error_message = str(e)
|
||||
|
||||
log_crawl_error(source.name, str(e))
|
||||
|
||||
return result
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=2))
|
||||
def crawl_all_sources(self, sources: Optional[List[AnnouncementSource]] = None) -> List[CrawlResult]:
|
||||
"""
|
||||
爬取所有来源的公告
|
||||
|
||||
Args:
|
||||
sources: 指定的来源列表，如果为None则使用配置中的所有来源
|
||||
|
||||
Returns:
|
||||
List[CrawlResult]: 所有来源的爬取结果
|
||||
"""
|
||||
# 系统健康检查
|
||||
if not check_system_health():
|
||||
logger.error("系统健康检查失败，跳过爬取")
|
||||
return []
|
||||
|
||||
if sources is None:
|
||||
sources = self._load_sources_from_config()
|
||||
|
||||
logger.info(f"开始爬取 {len(sources)} 个公告来源")
|
||||
|
||||
results = []
|
||||
|
||||
for source in sources:
|
||||
try:
|
||||
result = self.crawl_source(source)
|
||||
results.append(result)
|
||||
|
||||
# 检查是否需要暂停
|
||||
if result.status == CrawlStatus.FAILED:
|
||||
logger.warning(f"来源 {source.name} 爬取失败，继续下一个来源")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"爬取来源 {source.name} 时发生未预期错误: {str(e)}")
|
||||
# 创建失败结果
|
||||
failed_result = CrawlResult(
|
||||
source=source,
|
||||
status=CrawlStatus.FAILED,
|
||||
error_message=str(e),
|
||||
crawled_at=datetime.now()
|
||||
)
|
||||
results.append(failed_result)
|
||||
|
||||
# 统计总结果
|
||||
total_announcements = sum(len(r.announcements) for r in results if r.status == CrawlStatus.SUCCESS)
|
||||
success_count = sum(1 for r in results if r.status == CrawlStatus.SUCCESS)
|
||||
failed_count = len(results) - success_count
|
||||
|
||||
logger.info(
|
||||
f"爬取完成: 共处理 {len(results)} 个来源，"
|
||||
f"成功 {success_count} 个，失败 {failed_count} 个，"
|
||||
f"获取 {total_announcements} 条公告"
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _load_sources_from_config(self) -> List[AnnouncementSource]:
|
||||
"""从配置加载公告来源"""
|
||||
sources = []
|
||||
|
||||
for code, source_config in self.config.sources.items():
|
||||
try:
|
||||
source = AnnouncementSource(
|
||||
code=code,
|
||||
category_id=source_config["category_id"],
|
||||
name=source_config["name"],
|
||||
type=AnnouncementType(source_config["type"])
|
||||
)
|
||||
sources.append(source)
|
||||
except Exception as e:
|
||||
logger.warning(f"加载来源配置失败 {code}: {str(e)}")
|
||||
continue
|
||||
|
||||
return sources
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""获取爬虫统计信息"""
|
||||
return {
|
||||
"request_count": self.request_count,
|
||||
"error_count": self.error_count,
|
||||
"error_rate": self.error_count / max(self.request_count, 1),
|
||||
"session_active": self.session is not None
|
||||
}
|
||||
|
||||
def reset_stats(self):
|
||||
"""重置统计信息"""
|
||||
self.request_count = 0
|
||||
self.error_count = 0
|
||||
|
||||
|
||||
def create_spider() -> GXGPSpider:
|
||||
"""
|
||||
创建爬虫实例
|
||||
|
||||
Returns:
|
||||
GXGPSpider: 爬虫实例
|
||||
"""
|
||||
return GXGPSpider()
|
||||
|
||||
|
||||
def crawl_announcements(keywords: Optional[List[str]] = None,
|
||||
sources: Optional[List[str]] = None) -> List[CrawlResult]:
|
||||
"""
|
||||
便捷函数：爬取公告
|
||||
|
||||
Args:
|
||||
keywords: 关键词过滤（暂时未使用，在筛选模块处理）
|
||||
sources: 来源代码列表
|
||||
|
||||
Returns:
|
||||
List[CrawlResult]: 爬取结果
|
||||
"""
|
||||
spider = create_spider()
|
||||
|
||||
try:
|
||||
# 过滤来源
|
||||
if sources:
|
||||
all_sources = spider._load_sources_from_config()
|
||||
filtered_sources = [s for s in all_sources if s.code in sources]
|
||||
else:
|
||||
filtered_sources = None
|
||||
|
||||
return spider.crawl_all_sources(filtered_sources)
|
||||
finally:
|
||||
spider.close_session()
|
||||
@@ -1,239 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
定时搜索脚本
|
||||
执行搜索、筛选关键词、保存到数据库并发送企业微信卡片通知
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# 添加项目根目录到路径（gx_gp_monitor的父目录）
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
try:
|
||||
from gx_gp_monitor.core.config_manager import load_config, get_config
|
||||
from gx_gp_monitor.core.logger import init_logger, get_logger
|
||||
from gx_gp_monitor.crawler.spider import crawl_announcements
|
||||
from gx_gp_monitor.crawler.dahuagov_spider import crawl_dahuagov_announcements
|
||||
from gx_gp_monitor.filters.filters import filter_from_config
|
||||
from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, save_auto_announcements_to_storage
|
||||
from gx_gp_monitor.notification.wechat import send_announcements_notification, send_system_notification
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
def main():
|
||||
"""主函数：执行定时搜索任务"""
|
||||
try:
|
||||
# 加载配置
|
||||
config = load_config()
|
||||
if not config:
|
||||
logger.error("无法加载配置")
|
||||
return False
|
||||
|
||||
# 初始化日志
|
||||
init_logger(config=config)
|
||||
|
||||
logger.info("=== 开始定时搜索任务 ===")
|
||||
|
||||
# 初始化存储
|
||||
init_storage()
|
||||
|
||||
# 导入数据库模块
|
||||
import gx_gp_monitor.core.database as db_module
|
||||
db_manager = db_module.get_database_manager()
|
||||
|
||||
# 执行搜索（爬取所有公告）
|
||||
logger.info("开始执行定时搜索任务")
|
||||
|
||||
# 收集所有爬取结果
|
||||
all_crawl_results = []
|
||||
|
||||
# 1. 爬取广西政府采购网
|
||||
logger.info("开始爬取广西政府采购网...")
|
||||
gxgp_results = crawl_announcements()
|
||||
if gxgp_results:
|
||||
all_crawl_results.extend(gxgp_results)
|
||||
logger.info(f"广西政府采购网爬取完成，获取 {sum(len(r.announcements) for r in gxgp_results)} 条公告")
|
||||
|
||||
# 2. 爬取大化县政府网采购公告（全部推送，不筛选）
|
||||
logger.info("开始爬取大化县政府网采购公告（全部推送）...")
|
||||
dahua_results = crawl_dahuagov_announcements()
|
||||
if dahua_results:
|
||||
all_crawl_results.extend(dahua_results)
|
||||
logger.info(f"大化县政府网爬取完成，获取 {sum(len(r.announcements) for r in dahua_results)} 条公告")
|
||||
|
||||
if not all_crawl_results:
|
||||
logger.info("爬取完成：无数据")
|
||||
return True
|
||||
|
||||
# 收集所有公告
|
||||
all_announcements = []
|
||||
for result in all_crawl_results:
|
||||
if result.announcements:
|
||||
all_announcements.extend(result.announcements)
|
||||
|
||||
total_crawled = len(all_announcements)
|
||||
logger.info(f"搜索到 {total_crawled} 条原始公告")
|
||||
|
||||
if not all_announcements:
|
||||
logger.info("没有获取到任何公告")
|
||||
return True
|
||||
|
||||
# 分离广西政府采购网和大化县政府网的公告
|
||||
gxgp_all_announcements = [a for a in all_announcements if a.source_code != 'dahuagov']
|
||||
dahua_all_announcements = [a for a in all_announcements if a.source_code == 'dahuagov']
|
||||
|
||||
logger.info(f"广西政府采购网: {len(gxgp_all_announcements)} 条")
|
||||
logger.info(f"大化县政府网: {len(dahua_all_announcements)} 条")
|
||||
|
||||
# ========== 处理广西政府采购网（关键词筛选）==========
|
||||
gxgp_filtered = []
|
||||
if gxgp_all_announcements:
|
||||
# 对广西政府采购网公告进行关键词筛选
|
||||
from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
|
||||
from datetime import date
|
||||
|
||||
keyword_filter = KeywordFilter()
|
||||
gxgp_keyword_filtered = keyword_filter.filter_announcements(
|
||||
gxgp_all_announcements, keywords=config.crawler.keyword)
|
||||
|
||||
# 日期筛选（只处理今天的）
|
||||
date_filter = DateFilter()
|
||||
gxgp_today_filtered = date_filter.filter_announcements(
|
||||
gxgp_keyword_filtered,
|
||||
start_date=date.today(),
|
||||
end_date=date.today()
|
||||
)
|
||||
|
||||
logger.info(f"广西政府采购网关键词筛选后: {len(gxgp_keyword_filtered)} 条")
|
||||
logger.info(f"广西政府采购网今日匹配: {len(gxgp_today_filtered)} 条")
|
||||
|
||||
# 检查是否已存在
|
||||
for ann in gxgp_today_filtered:
|
||||
try:
|
||||
with db_module.get_db_cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM auto_announcements WHERE content_hash = %s LIMIT 1",
|
||||
(ann.content_hash,)
|
||||
)
|
||||
exists = cursor.fetchone() is not None
|
||||
if not exists:
|
||||
gxgp_filtered.append(ann)
|
||||
except Exception as e:
|
||||
logger.warning(f"检查公告是否存在失败: {str(e)}")
|
||||
pass
|
||||
|
||||
logger.info(f"广西政府采购网新增公告: {len(gxgp_filtered)} 条")
|
||||
|
||||
# ========== 处理大化县政府网（全部推送，不筛选）==========
|
||||
dahua_new_announcements = []
|
||||
if dahua_all_announcements:
|
||||
# 大化县公告不需要关键词筛选，直接检查是否已存在于dahuagov_announcements表
|
||||
for ann in dahua_all_announcements:
|
||||
try:
|
||||
with db_module.get_db_cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM dahuagov_announcements WHERE content_hash = %s LIMIT 1",
|
||||
(ann.content_hash,)
|
||||
)
|
||||
exists = cursor.fetchone() is not None
|
||||
if not exists:
|
||||
# 标记为新公告
|
||||
ann.is_new = True
|
||||
dahua_new_announcements.append(ann)
|
||||
except Exception as e:
|
||||
logger.warning(f"检查大化县公告是否存在失败: {str(e)}")
|
||||
pass
|
||||
|
||||
logger.info(f"大化县政府网新增公告: {len(dahua_new_announcements)} 条")
|
||||
|
||||
# 如果没有新增公告，直接结束
|
||||
if not gxgp_filtered and not dahua_new_announcements:
|
||||
logger.info("没有新增公告，任务完成")
|
||||
return True
|
||||
|
||||
# ========== 保存到数据库 ==========
|
||||
# 保存广西政府采购网公告
|
||||
if gxgp_filtered:
|
||||
saved_gxgp = save_auto_announcements_to_storage(gxgp_filtered)
|
||||
logger.info(f"保存广西政府采购网公告: {saved_gxgp} 条")
|
||||
|
||||
# 保存大化县政府网公告到专用表
|
||||
if dahua_new_announcements:
|
||||
saved_dahua = db_manager.save_dahuagov_announcements(dahua_new_announcements)
|
||||
logger.info(f"保存大化县政府网公告: {saved_dahua} 条")
|
||||
|
||||
# ========== 发送企业微信通知 ==========
|
||||
if config.wechat_app.enabled:
|
||||
logger.info("开始发送企业微信卡片通知...")
|
||||
notify_success = True
|
||||
|
||||
# 发送广西政府采购网通知
|
||||
if gxgp_filtered:
|
||||
logger.info(f"发送广西政府采购网通知，共 {len(gxgp_filtered)} 条...")
|
||||
gxgp_success = send_announcements_notification(gxgp_filtered)
|
||||
if gxgp_success:
|
||||
logger.info("广西政府采购网通知发送成功")
|
||||
else:
|
||||
logger.error("广西政府采购网通知发送失败")
|
||||
notify_success = False
|
||||
|
||||
# 发送大化县政府网通知
|
||||
if dahua_new_announcements:
|
||||
logger.info(f"发送大化县政府网通知，共 {len(dahua_new_announcements)} 条...")
|
||||
dahua_success = send_announcements_notification(dahua_new_announcements)
|
||||
if dahua_success:
|
||||
logger.info("大化县政府网通知发送成功")
|
||||
# 标记为已发送
|
||||
db_manager.mark_dahuagov_announcements_sent(dahua_new_announcements)
|
||||
else:
|
||||
logger.error("大化县政府网通知发送失败")
|
||||
notify_success = False
|
||||
|
||||
else:
|
||||
logger.info("企业微信通知未启用，跳过发送")
|
||||
notify_success = True
|
||||
|
||||
# ========== 输出统计信息 ==========
|
||||
print("\n=== 定时搜索任务完成 ===")
|
||||
print(f"总共爬取: {total_crawled} 条公告")
|
||||
print(f"广西政府采购网:")
|
||||
print(f" - 爬取: {len(gxgp_all_announcements)} 条")
|
||||
print(f" - 关键词匹配: {len(gxgp_filtered)} 条")
|
||||
print(f"大化县政府网:")
|
||||
print(f" - 爬取: {len(dahua_all_announcements)} 条")
|
||||
print(f" - 新增推送: {len(dahua_new_announcements)} 条")
|
||||
print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")
|
||||
|
||||
logger.info("=== 定时搜索任务完成 ===")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"定时搜索任务执行失败: {str(e)}")
|
||||
print(f"❌ 定时搜索任务失败: {str(e)}", file=sys.stderr)
|
||||
|
||||
# 尝试发送错误通知
|
||||
try:
|
||||
if config and config.wechat_app.enabled:
|
||||
send_system_notification(
|
||||
"定时搜索任务失败",
|
||||
f"错误信息: {str(e)}"
|
||||
)
|
||||
except Exception as notify_error:
|
||||
logger.error(f"发送错误通知失败: {notify_error}")
|
||||
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = main()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
except ImportError as e:
|
||||
print(f"导入失败: {e}", file=sys.stderr)
|
||||
print("请确保已安装所有依赖: pip install -r gx_gp_monitor/requirements.txt", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"脚本执行失败: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -1 +0,0 @@
|
||||
"""筛选模块"""
|
||||
@@ -1,471 +0,0 @@
|
||||
"""
|
||||
智能筛选模块
|
||||
提供关键词过滤、日期范围筛选、自动去重等功能
|
||||
"""
|
||||
|
||||
from typing import List, Optional, Dict, Any, Set
|
||||
from datetime import datetime, date
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
|
||||
try:
|
||||
from ..core.models import Announcement
|
||||
from ..core.logger import get_logger
|
||||
from ..core.database import get_database_manager
|
||||
except ImportError:
|
||||
from core.models import Announcement
|
||||
from core.logger import get_logger
|
||||
from core.database import get_database_manager
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FilterCriteria:
|
||||
"""筛选条件"""
|
||||
keywords: List[str] = None # 关键词列表
|
||||
start_date: Optional[date] = None # 开始日期
|
||||
end_date: Optional[date] = None # 结束日期
|
||||
sources: List[str] = None # 来源代码列表
|
||||
exclude_today: bool = False # 排除今日公告
|
||||
case_sensitive: bool = False # 关键词匹配是否区分大小写
|
||||
fuzzy_match: bool = False # 是否启用模糊匹配
|
||||
|
||||
def __post_init__(self):
|
||||
if self.keywords is None:
|
||||
self.keywords = []
|
||||
if self.sources is None:
|
||||
self.sources = []
|
||||
|
||||
|
||||
@dataclass
|
||||
class FilterResult:
|
||||
"""筛选结果"""
|
||||
total_count: int = 0 # 总数
|
||||
filtered_count: int = 0 # 筛选后数量
|
||||
keyword_filtered: int = 0 # 关键词筛选数量
|
||||
date_filtered: int = 0 # 日期筛选数量
|
||||
duplicate_filtered: int = 0 # 去重筛选数量
|
||||
source_filtered: int = 0 # 来源筛选数量
|
||||
today_excluded: int = 0 # 排除今日数量
|
||||
|
||||
|
||||
class KeywordFilter:
|
||||
"""关键词筛选器"""
|
||||
|
||||
def __init__(self, case_sensitive: bool = False, fuzzy_match: bool = False):
|
||||
"""
|
||||
初始化关键词筛选器
|
||||
|
||||
Args:
|
||||
case_sensitive: 是否区分大小写
|
||||
fuzzy_match: 是否启用模糊匹配
|
||||
"""
|
||||
self.case_sensitive = case_sensitive
|
||||
self.fuzzy_match = fuzzy_match
|
||||
self._compiled_patterns = {}
|
||||
|
||||
def _compile_patterns(self, keywords: List[str]):
|
||||
"""编译关键词模式"""
|
||||
flags = 0 if self.case_sensitive else re.IGNORECASE
|
||||
|
||||
for keyword in keywords:
|
||||
if keyword not in self._compiled_patterns:
|
||||
if self.fuzzy_match:
|
||||
# 简单的模糊匹配：将关键词中的空格替换为灵活匹配
|
||||
pattern = r'.*'.join(re.escape(word) for word in keyword.split())
|
||||
else:
|
||||
pattern = re.escape(keyword)
|
||||
|
||||
self._compiled_patterns[keyword] = re.compile(pattern, flags)
|
||||
|
||||
def matches(self, text: str, keywords: List[str]) -> bool:
|
||||
"""
|
||||
检查文本是否匹配关键词
|
||||
|
||||
Args:
|
||||
text: 要检查的文本
|
||||
keywords: 关键词列表
|
||||
|
||||
Returns:
|
||||
bool: 是否匹配
|
||||
"""
|
||||
if not keywords or not text:
|
||||
return True # 没有关键词或文本时，认为匹配
|
||||
|
||||
if not self.fuzzy_match:
|
||||
# 精确匹配
|
||||
search_text = text if self.case_sensitive else text.lower()
|
||||
search_keywords = keywords if self.case_sensitive else [k.lower() for k in keywords]
|
||||
|
||||
for keyword in search_keywords:
|
||||
if keyword in search_text:
|
||||
return True
|
||||
return False
|
||||
else:
|
||||
# 正则匹配
|
||||
self._compile_patterns(keywords)
|
||||
for keyword in keywords:
|
||||
pattern = self._compiled_patterns[keyword]
|
||||
if pattern.search(text):
|
||||
return True
|
||||
return False
|
||||
|
||||
def filter_announcements(self, announcements: List[Announcement],
|
||||
keywords: List[str]) -> List[Announcement]:
|
||||
"""
|
||||
筛选匹配关键词的公告
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
keywords: 关键词列表
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 筛选后的公告列表
|
||||
"""
|
||||
if not keywords:
|
||||
# 没有关键词时，所有公告都匹配
|
||||
for announcement in announcements:
|
||||
announcement.keyword_matched = True
|
||||
return announcements
|
||||
|
||||
filtered = []
|
||||
for announcement in announcements:
|
||||
# 构建搜索文本
|
||||
search_text = f"{announcement.title} {announcement.purchase_name}"
|
||||
|
||||
if self.matches(search_text, keywords):
|
||||
announcement.keyword_matched = True
|
||||
filtered.append(announcement)
|
||||
else:
|
||||
announcement.keyword_matched = False
|
||||
|
||||
logger.info(f"关键词筛选: {len(announcements)} -> {len(filtered)} 条公告")
|
||||
return filtered
|
||||
|
||||
|
||||
class DateFilter:
|
||||
"""日期筛选器"""
|
||||
|
||||
def filter_announcements(self, announcements: List[Announcement],
|
||||
start_date: Optional[date] = None,
|
||||
end_date: Optional[date] = None,
|
||||
exclude_today: bool = False) -> List[Announcement]:
|
||||
"""
|
||||
筛选日期范围内的公告
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
start_date: 开始日期
|
||||
end_date: 结束日期
|
||||
exclude_today: 是否排除今日公告
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 筛选后的公告列表
|
||||
"""
|
||||
if not start_date and not end_date and not exclude_today:
|
||||
# 没有日期限制时，所有公告都通过
|
||||
for announcement in announcements:
|
||||
announcement.date_filtered = True
|
||||
return announcements
|
||||
|
||||
filtered = []
|
||||
today = date.today()
|
||||
|
||||
for announcement in announcements:
|
||||
if not announcement.publish_date:
|
||||
# 没有发布日期的公告不通过筛选
|
||||
announcement.date_filtered = False
|
||||
continue
|
||||
|
||||
publish_date = announcement.publish_date.date()
|
||||
|
||||
# 检查日期范围
|
||||
date_in_range = True
|
||||
|
||||
if start_date and publish_date < start_date:
|
||||
date_in_range = False
|
||||
|
||||
if end_date and publish_date > end_date:
|
||||
date_in_range = False
|
||||
|
||||
# 检查是否排除今日
|
||||
if exclude_today and publish_date == today:
|
||||
date_in_range = False
|
||||
|
||||
announcement.date_filtered = date_in_range
|
||||
|
||||
if date_in_range:
|
||||
filtered.append(announcement)
|
||||
|
||||
logger.info(f"日期筛选: {len(announcements)} -> {len(filtered)} 条公告")
|
||||
return filtered
|
||||
|
||||
|
||||
class DuplicateFilter:
|
||||
"""去重筛选器"""
|
||||
|
||||
def __init__(self, use_database: bool = True):
|
||||
"""
|
||||
初始化去重筛选器
|
||||
|
||||
Args:
|
||||
use_database: 是否使用数据库检查重复
|
||||
"""
|
||||
self.use_database = use_database
|
||||
self.db_manager = get_database_manager() if use_database else None
|
||||
|
||||
def filter_announcements(self, announcements: List[Announcement]) -> List[Announcement]:
|
||||
"""
|
||||
去除重复的公告
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 去重后的公告列表
|
||||
"""
|
||||
if not announcements:
|
||||
return announcements
|
||||
|
||||
seen_hashes = set()
|
||||
filtered = []
|
||||
|
||||
for announcement in announcements:
|
||||
# 生成内容哈希
|
||||
if not announcement.content_hash:
|
||||
announcement.generate_content_hash()
|
||||
|
||||
content_hash = announcement.content_hash
|
||||
|
||||
# 检查是否已存在
|
||||
is_duplicate = False
|
||||
|
||||
if self.use_database and self.db_manager:
|
||||
# 数据库检查
|
||||
is_duplicate = self.db_manager.is_announcement_exists(content_hash)
|
||||
else:
|
||||
# 内存检查
|
||||
if content_hash in seen_hashes:
|
||||
is_duplicate = True
|
||||
seen_hashes.add(content_hash)
|
||||
|
||||
if not is_duplicate:
|
||||
filtered.append(announcement)
|
||||
# 注意：这里不设置is_new标志，因为这是在爬取后的筛选阶段
|
||||
|
||||
duplicate_count = len(announcements) - len(filtered)
|
||||
if duplicate_count > 0:
|
||||
logger.info(f"去重筛选: 移除了 {duplicate_count} 条重复公告")
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
class SourceFilter:
|
||||
"""来源筛选器"""
|
||||
|
||||
def filter_announcements(self, announcements: List[Announcement],
|
||||
allowed_sources: List[str]) -> List[Announcement]:
|
||||
"""
|
||||
筛选指定来源的公告
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
allowed_sources: 允许的来源代码列表
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 筛选后的公告列表
|
||||
"""
|
||||
if not allowed_sources:
|
||||
return announcements
|
||||
|
||||
filtered = []
|
||||
for announcement in announcements:
|
||||
if announcement.source_code in allowed_sources:
|
||||
filtered.append(announcement)
|
||||
|
||||
logger.info(f"来源筛选: {len(announcements)} -> {len(filtered)} 条公告")
|
||||
return filtered
|
||||
|
||||
|
||||
class AnnouncementFilter:
|
||||
"""公告智能筛选器"""
|
||||
|
||||
def __init__(self, criteria: FilterCriteria = None):
|
||||
"""
|
||||
初始化公告筛选器
|
||||
|
||||
Args:
|
||||
criteria: 筛选条件
|
||||
"""
|
||||
self.criteria = criteria or FilterCriteria()
|
||||
|
||||
# 初始化各个筛选器
|
||||
self.keyword_filter = KeywordFilter(
|
||||
case_sensitive=self.criteria.case_sensitive,
|
||||
fuzzy_match=self.criteria.fuzzy_match
|
||||
)
|
||||
self.date_filter = DateFilter()
|
||||
self.duplicate_filter = DuplicateFilter()
|
||||
self.source_filter = SourceFilter()
|
||||
|
||||
def filter(self, announcements: List[Announcement]) -> tuple[List[Announcement], FilterResult]:
|
||||
"""
|
||||
执行完整的筛选流程
|
||||
|
||||
Args:
|
||||
announcements: 原始公告列表
|
||||
|
||||
Returns:
|
||||
tuple: (筛选后的公告列表, 筛选结果统计)
|
||||
"""
|
||||
result = FilterResult()
|
||||
result.total_count = len(announcements)
|
||||
|
||||
# 1. 来源筛选
|
||||
if self.criteria.sources:
|
||||
announcements = self.source_filter.filter_announcements(
|
||||
announcements, self.criteria.sources)
|
||||
result.source_filtered = result.total_count - len(announcements)
|
||||
|
||||
# 2. 去重筛选
|
||||
announcements = self.duplicate_filter.filter_announcements(announcements)
|
||||
result.duplicate_filtered = result.total_count - len(announcements) - result.source_filtered
|
||||
|
||||
# 3. 日期筛选
|
||||
announcements = self.date_filter.filter_announcements(
|
||||
announcements,
|
||||
self.criteria.start_date,
|
||||
self.criteria.end_date,
|
||||
self.criteria.exclude_today
|
||||
)
|
||||
result.date_filtered = result.total_count - len(announcements) - result.source_filtered - result.duplicate_filtered
|
||||
|
||||
# 4. 关键词筛选
|
||||
if self.criteria.keywords:
|
||||
announcements = self.keyword_filter.filter_announcements(
|
||||
announcements, self.criteria.keywords)
|
||||
result.keyword_filtered = result.total_count - len(announcements) - result.source_filtered - result.duplicate_filtered - result.date_filtered
|
||||
|
||||
result.filtered_count = len(announcements)
|
||||
|
||||
logger.info(
|
||||
f"筛选完成: 总数 {result.total_count} -> 筛选后 {result.filtered_count} "
|
||||
f"(关键词: {result.keyword_filtered}, 日期: {result.date_filtered}, "
|
||||
f"去重: {result.duplicate_filtered}, 来源: {result.source_filtered})"
|
||||
)
|
||||
|
||||
return announcements, result
|
||||
|
||||
def quick_filter(self, announcement: Announcement) -> bool:
|
||||
"""
|
||||
快速筛选单个公告（用于实时筛选）
|
||||
|
||||
Args:
|
||||
announcement: 公告对象
|
||||
|
||||
Returns:
|
||||
bool: 是否通过筛选
|
||||
"""
|
||||
# 检查来源
|
||||
if self.criteria.sources and announcement.source_code not in self.criteria.sources:
|
||||
return False
|
||||
|
||||
# 检查日期
|
||||
if not announcement.in_date_range(
|
||||
self.criteria.start_date.isoformat() if self.criteria.start_date else None,
|
||||
self.criteria.end_date.isoformat() if self.criteria.end_date else None
|
||||
):
|
||||
return False
|
||||
|
||||
# 检查关键词
|
||||
if self.criteria.keywords:
|
||||
search_text = f"{announcement.title} {announcement.purchase_name}"
|
||||
if not self.keyword_filter.matches(search_text, self.criteria.keywords):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def update_criteria(self, criteria: FilterCriteria):
|
||||
"""
|
||||
更新筛选条件
|
||||
|
||||
Args:
|
||||
criteria: 新的筛选条件
|
||||
"""
|
||||
self.criteria = criteria
|
||||
|
||||
# 重新初始化筛选器
|
||||
self.keyword_filter = KeywordFilter(
|
||||
case_sensitive=self.criteria.case_sensitive,
|
||||
fuzzy_match=self.criteria.fuzzy_match
|
||||
)
|
||||
|
||||
|
||||
def create_default_filter(keywords: List[str] = None,
|
||||
start_date: str = None,
|
||||
end_date: str = None) -> AnnouncementFilter:
|
||||
"""
|
||||
创建默认筛选器
|
||||
|
||||
Args:
|
||||
keywords: 关键词列表
|
||||
start_date: 开始日期字符串 (YYYY-MM-DD)
|
||||
end_date: 结束日期字符串 (YYYY-MM-DD)
|
||||
|
||||
Returns:
|
||||
AnnouncementFilter: 筛选器实例
|
||||
"""
|
||||
criteria = FilterCriteria()
|
||||
|
||||
if keywords:
|
||||
criteria.keywords = keywords
|
||||
|
||||
if start_date:
|
||||
try:
|
||||
criteria.start_date = datetime.fromisoformat(start_date).date()
|
||||
except ValueError:
|
||||
logger.warning(f"无效的开始日期格式: {start_date}")
|
||||
|
||||
if end_date:
|
||||
try:
|
||||
criteria.end_date = datetime.fromisoformat(end_date).date()
|
||||
except ValueError:
|
||||
logger.warning(f"无效的结束日期格式: {end_date}")
|
||||
|
||||
return AnnouncementFilter(criteria)
|
||||
|
||||
|
||||
def filter_from_config() -> AnnouncementFilter:
|
||||
"""
|
||||
从配置文件创建筛选器
|
||||
|
||||
Returns:
|
||||
AnnouncementFilter: 配置化的筛选器
|
||||
"""
|
||||
try:
|
||||
from ..core.config_manager import get_config
|
||||
except ImportError:
|
||||
from core.config_manager import get_config
|
||||
|
||||
config = get_config()
|
||||
|
||||
criteria = FilterCriteria()
|
||||
criteria.keywords = config.crawler.keyword
|
||||
criteria.sources = list(config.sources.keys()) # 默认包含所有来源
|
||||
|
||||
# 日期范围
|
||||
if config.crawler.start_date:
|
||||
try:
|
||||
criteria.start_date = datetime.fromisoformat(config.crawler.start_date).date()
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if config.crawler.end_date:
|
||||
try:
|
||||
criteria.end_date = datetime.fromisoformat(config.crawler.end_date).date()
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return AnnouncementFilter(criteria)
|
||||
@@ -1,505 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
广西政府采购网公告监控系统主程序
|
||||
广西政府采购网公告爬取和监控的智能系统
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
import signal
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
|
||||
# 添加项目根目录到Python路径
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
try:
|
||||
# 尝试相对导入
|
||||
from .core.config_manager import load_config, get_config
|
||||
from .core.logger import init_logger, get_logger
|
||||
from .core.reliability import check_system_health
|
||||
from .crawler.spider import crawl_announcements
|
||||
from .filters.filters import filter_from_config
|
||||
from .storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, save_manual_announcements_by_source_to_storage, cleanup_storage
|
||||
from .storage.md_generator import generate_onu_md
|
||||
from .notification.wechat import send_announcements_notification, send_system_notification
|
||||
from .wechat.callback_server import get_callback_server
|
||||
from .wechat.menu_manager import WeChatMenuManager
|
||||
except ImportError:
|
||||
try:
|
||||
# 尝试绝对导入（直接运行脚本时）
|
||||
from core.config_manager import load_config, get_config
|
||||
from core.logger import init_logger, get_logger
|
||||
from core.reliability import check_system_health
|
||||
from crawler.spider import crawl_announcements
|
||||
from filters.filters import filter_from_config
|
||||
from storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, cleanup_storage
|
||||
from storage.md_generator import generate_onu_md
|
||||
from notification.wechat import send_announcements_notification, send_system_notification
|
||||
# 企业微信模块动态导入，避免循环导入问题
|
||||
wechat_available = True
|
||||
try:
|
||||
from wechat.callback_server import get_callback_server
|
||||
from wechat.menu_manager import WeChatMenuManager
|
||||
except ImportError:
|
||||
wechat_available = False
|
||||
print("企业微信模块不可用", file=sys.stderr)
|
||||
except ImportError as e:
|
||||
print(f"导入错误: {e}", file=sys.stderr)
|
||||
print("请确保依赖已正确安装: pip install -r requirements.txt", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class GXGPMonitorApp:
|
||||
"""广西政府采购网监控系统应用"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = None
|
||||
self.logger = None
|
||||
self.running = False
|
||||
|
||||
def initialize(self, config_file: str = None):
|
||||
"""初始化应用"""
|
||||
try:
|
||||
# 加载配置
|
||||
self.config = load_config(config_file)
|
||||
|
||||
# 初始化日志
|
||||
self.logger = init_logger(config=self.config)
|
||||
|
||||
logger.info("=== 广西政府采购网公告监控系统启动 ===")
|
||||
logger.info(f"版本: 1.0.0")
|
||||
logger.info(f"配置文件: {config_file or '默认配置'}")
|
||||
|
||||
# 系统健康检查
|
||||
if not check_system_health():
|
||||
logger.warning("系统健康检查失败，但继续运行")
|
||||
|
||||
# 初始化存储
|
||||
init_storage()
|
||||
|
||||
logger.info("应用初始化完成")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"应用初始化失败: {str(e)}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None, manual_crawl: bool = False):
|
||||
"""执行搜索任务
|
||||
|
||||
Args:
|
||||
keywords: 关键词列表
|
||||
sources: 来源列表
|
||||
max_pages: 最大页数
|
||||
manual_crawl: 是否为手动搜索（只筛选今天的数据）
|
||||
"""
|
||||
try:
|
||||
logger.info(f"开始执行搜索任务 (手动搜索: {manual_crawl})")
|
||||
|
||||
# 执行爬取
|
||||
crawl_results = crawl_announcements(keywords, sources)
|
||||
|
||||
if not crawl_results:
|
||||
logger.info("搜索完成：无数据")
|
||||
return {"success": True, "results": []}
|
||||
|
||||
# 收集所有公告
|
||||
all_announcements = []
|
||||
for result in crawl_results:
|
||||
if result.announcements:
|
||||
all_announcements.extend(result.announcements)
|
||||
|
||||
logger.info(f"搜索到 {len(all_announcements)} 条原始公告")
|
||||
|
||||
# 保存公告到对应的专用表
|
||||
if not manual_crawl:
|
||||
# 自动爬取：保存到auto_announcements表（关键词匹配专用）
|
||||
all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100)
|
||||
all_saved_count = sum(all_saved_stats.values())
|
||||
logger.info(f"保存自动爬取公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}")
|
||||
else:
|
||||
# 手动搜索：保存到manual_announcements表（全量数据专用）
|
||||
all_saved_stats = save_manual_announcements_by_source_to_storage(all_announcements, max_per_source=100)
|
||||
all_saved_count = sum(all_saved_stats.values())
|
||||
logger.info(f"保存手动搜索公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}")
|
||||
|
||||
# 筛选公告
|
||||
if manual_crawl:
|
||||
# 手动搜索：只筛选今天的公告和用户指定的关键词，不进行去重
|
||||
from .filters.filters import KeywordFilter, DateFilter, SourceFilter
|
||||
from datetime import date
|
||||
|
||||
# 1. 日期筛选：只保留今天的公告
|
||||
date_filter = DateFilter()
|
||||
date_filtered = date_filter.filter_announcements(all_announcements, start_date=date.today(), end_date=date.today())
|
||||
|
||||
# 2. 关键词筛选
|
||||
keyword_filter = KeywordFilter()
|
||||
keyword_filtered = keyword_filter.filter_announcements(date_filtered, keywords=keywords or [])
|
||||
|
||||
# 3. 来源筛选
|
||||
source_filter = SourceFilter()
|
||||
filtered_announcements = source_filter.filter_announcements(keyword_filtered, sources or list(self.config.sources.keys()))
|
||||
|
||||
# 计算统计信息
|
||||
filter_stats = type('FilterResult', (), {
|
||||
"keyword_filtered": len(date_filtered) - len(keyword_filtered),
|
||||
"date_filtered": len(all_announcements) - len(date_filtered),
|
||||
"duplicate_filtered": 0, # 手动搜索不进行去重
|
||||
"source_filtered": len(keyword_filtered) - len(filtered_announcements)
|
||||
})()
|
||||
else:
|
||||
# 自动爬取：筛选出新公告并应用关键词筛选
|
||||
# 1. 筛选出数据库中没有的新公告
|
||||
new_announcements = [ann for ann in all_announcements if ann.is_new]
|
||||
logger.info(f"筛选出 {len(new_announcements)} 条新公告")
|
||||
|
||||
# 2. 对新公告应用关键词筛选等
|
||||
if new_announcements:
|
||||
filter_obj = filter_from_config()
|
||||
filtered_announcements, filter_stats = filter_obj.filter(new_announcements)
|
||||
# 更新统计信息，加上未筛选的新公告数量
|
||||
filter_stats.keyword_filtered += len(new_announcements) - len(filtered_announcements)
|
||||
else:
|
||||
filtered_announcements = []
|
||||
filter_stats = type('FilterResult', (), {
|
||||
"keyword_filtered": 0,
|
||||
"date_filtered": 0,
|
||||
"duplicate_filtered": len(all_announcements),
|
||||
"source_filtered": 0
|
||||
})()
|
||||
|
||||
logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告")
|
||||
|
||||
# 对于手动爬取，不保存筛选后的公告到数据库
|
||||
if not manual_crawl:
|
||||
# 保存筛选后的公告（用于标记关键词匹配等）
|
||||
saved_count = save_announcements_to_storage(filtered_announcements)
|
||||
else:
|
||||
saved_count = 0
|
||||
logger.info("手动爬取模式：跳过筛选后公告的数据库保存")
|
||||
|
||||
# 生成Markdown文件（只在自动爬取时生成）
|
||||
md_success = False
|
||||
if not manual_crawl:
|
||||
md_success = generate_onu_md(filtered_announcements)
|
||||
|
||||
# 发送通知
|
||||
notify_success = False
|
||||
if not manual_crawl and filtered_announcements and self.config.wechat_app.enabled:
|
||||
# 自动爬取时发送卡片消息
|
||||
notify_success = send_announcements_notification(filtered_announcements)
|
||||
elif manual_crawl and filtered_announcements and self.config.wechat_app.enabled:
|
||||
# 手动爬取时不在这里发送消息，由消息处理器负责发送markdown消息
|
||||
notify_success = True # 标记为成功，因为消息会通过其他方式发送
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"total_crawled": len(all_announcements),
|
||||
"filtered": len(filtered_announcements),
|
||||
"saved": saved_count,
|
||||
"markdown_generated": md_success,
|
||||
"notification_sent": notify_success,
|
||||
"filter_stats": {
|
||||
"keyword_filtered": filter_stats.keyword_filtered,
|
||||
"date_filtered": filter_stats.date_filtered,
|
||||
"duplicate_filtered": filter_stats.duplicate_filtered,
|
||||
"source_filtered": filter_stats.source_filtered
|
||||
}
|
||||
}
|
||||
|
||||
# 对于手动爬取，额外返回筛选后的公告列表
|
||||
if manual_crawl:
|
||||
result["filtered_announcements"] = filtered_announcements
|
||||
|
||||
logger.info(f"爬取任务完成: {result}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"爬取任务执行失败: {str(e)}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def run_cleanup(self, days: int = None):
|
||||
"""执行数据清理任务"""
|
||||
try:
|
||||
logger.info("开始执行数据清理任务")
|
||||
|
||||
deleted_count = cleanup_storage(days)
|
||||
|
||||
logger.info(f"数据清理完成：删除 {deleted_count} 条过期数据")
|
||||
|
||||
# 发送通知
|
||||
if deleted_count > 0 and self.config.wechat_app.enabled:
|
||||
send_system_notification(
|
||||
"数据清理完成",
|
||||
f"已清理 {deleted_count} 条过期数据"
|
||||
)
|
||||
|
||||
return {"success": True, "deleted": deleted_count}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"数据清理任务执行失败: {str(e)}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def run_wechat_server(self, host: str = '0.0.0.0', port: int = 18001):
|
||||
"""启动企业微信回调服务器"""
|
||||
if not wechat_available:
|
||||
return {"success": False, "error": "企业微信模块不可用"}
|
||||
|
||||
try:
|
||||
logger.info("启动企业微信回调服务器")
|
||||
|
||||
# 获取回调服务器
|
||||
callback_server = get_callback_server()
|
||||
|
||||
# 启动服务器
|
||||
callback_server.run(host=host, port=port, debug=self.config.debug)
|
||||
|
||||
return {"success": True, "host": host, "port": port}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"启动企业微信回调服务器失败: {str(e)}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def manage_wechat_menu(self, action: str) -> Dict[str, Any]:
|
||||
"""管理企业微信菜单"""
|
||||
if not wechat_available:
|
||||
return {"success": False, "error": "企业微信模块不可用"}
|
||||
|
||||
try:
|
||||
logger.info(f"执行企业微信菜单操作: {action}")
|
||||
|
||||
menu_manager = WeChatMenuManager()
|
||||
|
||||
if action == 'create':
|
||||
success = menu_manager.create_menu()
|
||||
result = {"success": success, "action": "create"}
|
||||
elif action == 'delete':
|
||||
success = menu_manager.delete_menu()
|
||||
result = {"success": success, "action": "delete"}
|
||||
elif action == 'get':
|
||||
menu_info = menu_manager.get_menu()
|
||||
result = {"success": menu_info is not None, "action": "get", "menu": menu_info}
|
||||
elif action == 'test':
|
||||
test_results = menu_manager.test_menu_operations()
|
||||
result = {"success": True, "action": "test", "results": test_results}
|
||||
else:
|
||||
result = {"success": False, "error": f"未知操作: {action}"}
|
||||
|
||||
if result["success"]:
|
||||
logger.info(f"企业微信菜单操作成功: {action}")
|
||||
else:
|
||||
logger.error(f"企业微信菜单操作失败: {action}")
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"企业微信菜单管理异常: {str(e)}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def show_status(self):
|
||||
"""显示系统状态"""
|
||||
try:
|
||||
status = {
|
||||
"system": {
|
||||
"version": "1.0.0",
|
||||
"healthy": check_system_health()
|
||||
},
|
||||
"config": {
|
||||
"debug": self.config.debug,
|
||||
"log_level": self.config.log_level.value
|
||||
},
|
||||
"database": {
|
||||
"enabled": self.config.database.enabled,
|
||||
"type": self.config.database.type
|
||||
},
|
||||
"wechat": {
|
||||
"enabled": self.config.wechat_app.enabled
|
||||
}
|
||||
}
|
||||
|
||||
# 格式化输出
|
||||
print("\n=== 系统状态 ===")
|
||||
print(f"系统健康: {'正常' if status['system']['healthy'] else '异常'}")
|
||||
print(f"调试模式: {'开启' if status['config']['debug'] else '关闭'}")
|
||||
print(f"日志级别: {status['config']['log_level']}")
|
||||
print(f"数据库: {'启用' if status['database']['enabled'] else '禁用'} ({status['database']['type']})")
|
||||
print(f"企业微信: {'启用' if status['wechat']['enabled'] else '禁用'}")
|
||||
|
||||
return status
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取系统状态失败: {str(e)}")
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
def create_argument_parser():
|
||||
"""创建命令行参数解析器"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="广西政府采购网公告监控系统",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
使用示例:
|
||||
python main.py crawl # 执行一次爬取
|
||||
python main.py crawl --keywords "大化" # 爬取指定关键词
|
||||
python main.py cleanup # 执行数据清理
|
||||
python main.py status # 查看系统状态
|
||||
python main.py wechat-server # 启动企业微信回调服务器
|
||||
python main.py wechat-menu --action create # 创建企业微信菜单
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'command',
|
||||
choices=['crawl', 'cleanup', 'status', 'wechat-server', 'wechat-menu'],
|
||||
help='要执行的命令'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--config', '-c',
|
||||
help='配置文件路径'
|
||||
)
|
||||
|
||||
# crawl命令的参数
|
||||
parser.add_argument(
|
||||
'--keywords', '-k',
|
||||
nargs='+',
|
||||
help='关键词过滤（多个关键词用空格分隔）'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--sources', '-s',
|
||||
nargs='+',
|
||||
help='来源代码过滤（多个来源用空格分隔）'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--max-pages',
|
||||
type=int,
|
||||
help='最大爬取页数'
|
||||
)
|
||||
|
||||
# cleanup命令的参数
|
||||
parser.add_argument(
|
||||
'--days', '-d',
|
||||
type=int,
|
||||
help='清理多少天前的过期数据'
|
||||
)
|
||||
|
||||
# wechat-server命令的参数
|
||||
parser.add_argument(
|
||||
'--host', '-H',
|
||||
default='0.0.0.0',
|
||||
help='服务器监听主机地址 (默认: 0.0.0.0)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--port', '-P',
|
||||
type=int,
|
||||
default=18001,
|
||||
help='服务器监听端口 (默认: 18001)'
|
||||
)
|
||||
|
||||
# wechat-menu命令的参数
|
||||
parser.add_argument(
|
||||
'--action', '-a',
|
||||
choices=['create', 'delete', 'get', 'test'],
|
||||
default='create',
|
||||
help='菜单操作类型 (默认: create)'
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
parser = create_argument_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
# 创建应用实例
|
||||
app = GXGPMonitorApp()
|
||||
|
||||
# 初始化应用
|
||||
if not app.initialize(args.config):
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
if args.command == 'crawl':
|
||||
# 执行爬取
|
||||
result = app.run_crawl(
|
||||
keywords=args.keywords,
|
||||
sources=args.sources,
|
||||
max_pages=args.max_pages
|
||||
)
|
||||
|
||||
if result["success"]:
|
||||
print("✅ 爬取任务执行成功")
|
||||
print(f" 爬取公告: {result['total_crawled']}")
|
||||
print(f" 筛选后: {result['filtered']}")
|
||||
print(f" 保存数量: {result['saved']}")
|
||||
if result.get("markdown_generated"):
|
||||
print(" Markdown文件: 已生成")
|
||||
if result.get("notification_sent"):
|
||||
print(" 通知发送: 已发送")
|
||||
else:
|
||||
print(f"❌ 爬取任务执行失败: {result.get('error', '未知错误')}")
|
||||
sys.exit(1)
|
||||
|
||||
elif args.command == 'cleanup':
|
||||
# 执行清理
|
||||
result = app.run_cleanup(days=args.days)
|
||||
if result["success"]:
|
||||
print(f"✅ 数据清理完成，删除 {result['deleted']} 条记录")
|
||||
else:
|
||||
print(f"❌ 数据清理失败: {result.get('error', '未知错误')}")
|
||||
sys.exit(1)
|
||||
|
||||
elif args.command == 'status':
|
||||
# 显示状态
|
||||
app.show_status()
|
||||
|
||||
elif args.command == 'wechat-server':
|
||||
# 启动企业微信回调服务器
|
||||
result = app.run_wechat_server(host=args.host, port=args.port)
|
||||
if result["success"]:
|
||||
print(f"✅ 企业微信回调服务器已启动: {result['host']}:{result['port']}")
|
||||
print(" 回调地址: /api/v1/wechat/callback")
|
||||
else:
|
||||
print(f"❌ 企业微信回调服务器启动失败: {result.get('error', '未知错误')}")
|
||||
sys.exit(1)
|
||||
|
||||
elif args.command == 'wechat-menu':
|
||||
# 企业微信菜单管理
|
||||
result = app.manage_wechat_menu(action=args.action)
|
||||
if result["success"]:
|
||||
if args.action == 'create':
|
||||
print("✅ 企业微信菜单创建成功")
|
||||
elif args.action == 'delete':
|
||||
print("✅ 企业微信菜单删除成功")
|
||||
elif args.action == 'get':
|
||||
print("✅ 企业微信菜单获取成功")
|
||||
if result.get("menu"):
|
||||
print("菜单信息:", json.dumps(result["menu"], indent=2, ensure_ascii=False))
|
||||
elif args.action == 'test':
|
||||
print("✅ 企业微信菜单测试完成")
|
||||
print("测试结果:", result.get("results"))
|
||||
else:
|
||||
error = result.get("error", "未知错误")
|
||||
print(f"❌ 企业微信菜单操作失败: {error}")
|
||||
sys.exit(1)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("收到中断信号，正在退出...")
|
||||
except Exception as e:
|
||||
logger.error(f"程序执行异常: {str(e)}")
|
||||
print(f"❌ 程序执行异常: {str(e)}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1 +0,0 @@
|
||||
"""通知模块"""
|
||||
@@ -1,896 +0,0 @@
|
||||
"""
|
||||
企业微信通知模块
|
||||
提供企业微信消息发送功能，支持文本和Markdown格式
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
import hashlib
|
||||
from typing import Optional, Dict, Any, List
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger
|
||||
from ..core.models import Announcement
|
||||
from ..core.reliability import retry_on_exception, RetryConfig, safe_execute
|
||||
from ..storage.md_generator import AnnouncementMarkdownFormatter
|
||||
except ImportError:
|
||||
from core.config_manager import get_config
|
||||
from core.logger import get_logger
|
||||
from core.models import Announcement
|
||||
from core.reliability import retry_on_exception, RetryConfig, safe_execute
|
||||
from storage.md_generator import AnnouncementMarkdownFormatter
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class WeChatService:
|
||||
"""企业微信服务"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config().wechat_app
|
||||
self._access_token = None
|
||||
self._token_expires_at = 0
|
||||
|
||||
logger.info("企业微信服务初始化完成")
|
||||
|
||||
def _get_access_token(self) -> Optional[str]:
|
||||
"""
|
||||
获取访问令牌
|
||||
|
||||
Returns:
|
||||
Optional[str]: 访问令牌
|
||||
"""
|
||||
current_time = time.time()
|
||||
|
||||
# 检查令牌是否仍然有效
|
||||
if self._access_token and current_time < self._token_expires_at:
|
||||
return self._access_token
|
||||
|
||||
try:
|
||||
# 构建请求URL
|
||||
if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'):
|
||||
url = f"{self.config.proxy_api_url}/cgi-bin/gettoken"
|
||||
else:
|
||||
url = "https://qyapi.weixin.qq.com/cgi-bin/gettoken"
|
||||
|
||||
params = {
|
||||
"corpid": self.config.corp_id,
|
||||
"corpsecret": self.config.secret
|
||||
}
|
||||
|
||||
logger.debug("正在获取企业微信访问令牌")
|
||||
|
||||
response = requests.get(url, params=params, timeout=30)
|
||||
result = response.json()
|
||||
|
||||
if result.get("errcode") == 0:
|
||||
self._access_token = result.get("access_token")
|
||||
# 提前5分钟过期
|
||||
expires_in = result.get("expires_in", 7200) - 300
|
||||
self._token_expires_at = current_time + expires_in
|
||||
|
||||
logger.info("成功获取企业微信访问令牌")
|
||||
return self._access_token
|
||||
else:
|
||||
logger.error(f"获取访问令牌失败: {result}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取访问令牌异常: {str(e)}")
|
||||
return None
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def send_text_message(self, content: str,
|
||||
to_user: str = "@all",
|
||||
to_party: str = "",
|
||||
to_tag: str = "") -> bool:
|
||||
"""
|
||||
发送文本消息
|
||||
|
||||
Args:
|
||||
content: 消息内容
|
||||
to_user: 接收者用户ID，多个用|分隔，@all表示全体
|
||||
to_party: 接收者部门ID，多个用|分隔
|
||||
to_tag: 接收者标签ID，多个用|分隔
|
||||
|
||||
Returns:
|
||||
bool: 发送是否成功
|
||||
"""
|
||||
try:
|
||||
access_token = self._get_access_token()
|
||||
if not access_token:
|
||||
logger.error("无法获取访问令牌，发送失败")
|
||||
return False
|
||||
|
||||
# 构建请求URL
|
||||
if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'):
|
||||
url = f"{self.config.proxy_api_url}/cgi-bin/message/send"
|
||||
else:
|
||||
url = "https://qyapi.weixin.qq.com/cgi-bin/message/send"
|
||||
|
||||
params = {"access_token": access_token}
|
||||
|
||||
data = {
|
||||
"touser": to_user,
|
||||
"toparty": to_party,
|
||||
"totag": to_tag,
|
||||
"msgtype": "text",
|
||||
"agentid": self.config.agent_id,
|
||||
"text": {
|
||||
"content": content
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(f"发送文本消息: {content[:100]}...")
|
||||
|
||||
response = requests.post(url, params=params, json=data, timeout=30)
|
||||
result = response.json()
|
||||
|
||||
if result.get("errcode") == 0:
|
||||
logger.info("文本消息发送成功")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"文本消息发送失败: {result}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送文本消息异常: {str(e)}")
|
||||
return False
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def send_markdown_message(self, content: str,
|
||||
to_user: str = "@all",
|
||||
to_party: str = "",
|
||||
to_tag: str = "") -> bool:
|
||||
"""
|
||||
发送Markdown消息
|
||||
|
||||
Args:
|
||||
content: Markdown格式的消息内容
|
||||
to_user: 接收者用户ID
|
||||
to_party: 接收者部门ID
|
||||
to_tag: 接收者标签ID
|
||||
|
||||
Returns:
|
||||
bool: 发送是否成功
|
||||
"""
|
||||
try:
|
||||
access_token = self._get_access_token()
|
||||
if not access_token:
|
||||
logger.error("无法获取访问令牌，发送失败")
|
||||
return False
|
||||
|
||||
# 构建请求URL
|
||||
if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'):
|
||||
url = f"{self.config.proxy_api_url}/cgi-bin/message/send"
|
||||
else:
|
||||
url = "https://qyapi.weixin.qq.com/cgi-bin/message/send"
|
||||
|
||||
params = {"access_token": access_token}
|
||||
|
||||
data = {
|
||||
"touser": to_user,
|
||||
"toparty": to_party,
|
||||
"totag": to_tag,
|
||||
"msgtype": "markdown",
|
||||
"agentid": self.config.agent_id,
|
||||
"markdown": {
|
||||
"content": content
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug("发送Markdown消息")
|
||||
|
||||
response = requests.post(url, params=params, json=data, timeout=30)
|
||||
result = response.json()
|
||||
|
||||
if result.get("errcode") == 0:
|
||||
logger.info("Markdown消息发送成功")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Markdown消息发送失败: {result}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送Markdown消息异常: {str(e)}")
|
||||
return False
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def send_textcard_message(self, title: str, description: str, url: str,
|
||||
to_user: str = "@all", to_party: str = "", to_tag: str = "",
|
||||
btn_txt: str = "查看详情") -> bool:
|
||||
"""
|
||||
发送文本卡片消息
|
||||
|
||||
Args:
|
||||
title: 标题
|
||||
description: 描述内容（支持HTML）
|
||||
url: 点击跳转的链接
|
||||
to_user: 接收者用户ID
|
||||
to_party: 接收者部门ID
|
||||
to_tag: 接收者标签ID
|
||||
btn_txt: 按钮文字
|
||||
|
||||
Returns:
|
||||
bool: 发送是否成功
|
||||
"""
|
||||
try:
|
||||
access_token = self._get_access_token()
|
||||
if not access_token:
|
||||
logger.error("无法获取访问令牌，发送失败")
|
||||
return False
|
||||
|
||||
# 构建请求URL
|
||||
if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'):
|
||||
url_endpoint = f"{self.config.proxy_api_url}/cgi-bin/message/send"
|
||||
else:
|
||||
url_endpoint = "https://qyapi.weixin.qq.com/cgi-bin/message/send"
|
||||
|
||||
params = {"access_token": access_token}
|
||||
|
||||
data = {
|
||||
"touser": to_user,
|
||||
"toparty": to_party,
|
||||
"totag": to_tag,
|
||||
"msgtype": "textcard",
|
||||
"agentid": self.config.agent_id,
|
||||
"textcard": {
|
||||
"title": title,
|
||||
"description": description,
|
||||
"url": url,
|
||||
"btntxt": btn_txt
|
||||
},
|
||||
"enable_id_trans": 0,
|
||||
"enable_duplicate_check": 0,
|
||||
"duplicate_check_interval": 1800
|
||||
}
|
||||
|
||||
logger.debug(f"发送文本卡片消息: {title}")
|
||||
|
||||
response = requests.post(url_endpoint, params=params, json=data, timeout=30)
|
||||
result = response.json()
|
||||
|
||||
if result.get("errcode") == 0:
|
||||
logger.info("文本卡片消息发送成功")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"文本卡片消息发送失败: {result}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送文本卡片消息异常: {str(e)}")
|
||||
return False
|
||||
|
||||
def send_announcement_notification(self, announcements: List[Announcement],
|
||||
max_count: int = 20) -> bool:
|
||||
"""
|
||||
发送公告通知（每条公告发送一条单独的文本卡片消息）
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
max_count: 最大显示数量
|
||||
|
||||
Returns:
|
||||
bool: 是否至少有一条消息发送成功
|
||||
"""
|
||||
if not announcements:
|
||||
logger.info("没有新公告，跳过通知")
|
||||
return True
|
||||
|
||||
success_count = 0
|
||||
total_count = len(announcements)
|
||||
|
||||
logger.info(f"开始发送 {total_count} 条公告通知，每条单独发送")
|
||||
|
||||
for i, announcement in enumerate(announcements[:max_count], 1):
|
||||
try:
|
||||
logger.debug(f"发送第 {i}/{min(total_count, max_count)} 条公告: {announcement.title[:30]}...")
|
||||
|
||||
# 为每条公告生成单独的文本卡片
|
||||
if self.send_single_announcement_notification(announcement):
|
||||
success_count += 1
|
||||
logger.debug(f"第 {i} 条公告发送成功")
|
||||
else:
|
||||
logger.warning(f"第 {i} 条公告发送失败: {announcement.title[:30]}...")
|
||||
|
||||
# 添加短暂延迟，避免发送过快
|
||||
if i < len(announcements[:max_count]):
|
||||
import time
|
||||
time.sleep(0.5)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送第 {i} 条公告时发生异常: {str(e)}")
|
||||
continue
|
||||
|
||||
logger.info(f"公告通知发送完成: {success_count}/{min(total_count, max_count)} 条成功")
|
||||
|
||||
if total_count > max_count:
|
||||
logger.info(f"还有 {total_count - max_count} 条公告未发送（超过最大数量限制）")
|
||||
|
||||
return success_count > 0
|
||||
|
||||
def send_single_announcement_notification(self, announcement: Announcement) -> bool:
|
||||
"""
|
||||
发送单条公告的通知（文本卡片消息）
|
||||
|
||||
Args:
|
||||
announcement: 单条公告
|
||||
|
||||
Returns:
|
||||
bool: 发送是否成功
|
||||
"""
|
||||
try:
|
||||
# 生成单条公告的文本卡片内容
|
||||
title, description, url = self._generate_single_textcard_notification(announcement)
|
||||
|
||||
# 发送文本卡片消息
|
||||
return self.send_textcard_message(title, description, url, btn_txt="查看详情")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送单条公告通知失败: {str(e)}")
|
||||
return False
|
||||
|
||||
def _generate_single_textcard_notification(self, announcement: Announcement) -> tuple[str, str, str]:
|
||||
"""
|
||||
生成单条公告的文本卡片内容
|
||||
|
||||
新格式示例：
|
||||
---
|
||||
**北海市涠洲岛旅游区管理委员会关于办公桌的网上超市采购项目成交公告**
|
||||
|
||||
工程类公告 | 北海市涠洲岛旅游区管理委员会 | 2026-01-08 09:28
|
||||
---
|
||||
|
||||
Args:
|
||||
announcement: 单条公告
|
||||
|
||||
Returns:
|
||||
tuple[str, str, str]: (标题, 描述HTML, URL)
|
||||
"""
|
||||
# 标题：公告标题（加粗显示，作为卡片标题）
|
||||
announcement_title = announcement.title
|
||||
if len(announcement_title) > 128: # 企业微信卡片标题限制128字符
|
||||
announcement_title = announcement_title[:125] + "..."
|
||||
title = announcement_title
|
||||
|
||||
# 公告类型映射（英文枚举值 -> 中文显示名称）
|
||||
type_mapping = {
|
||||
"PURCHASE": "采购公告",
|
||||
"RESULT": "结果公告",
|
||||
"CONTRACT": "合同公告",
|
||||
"CORRECTION": "更正公告",
|
||||
"PRE_ANNOUNCEMENT": "招标文件预公示",
|
||||
"SINGLE_SOURCE": "单一来源公示",
|
||||
"ELECTRONIC_MARKET": "电子卖场公示",
|
||||
"ACCEPTANCE": "履约验收公示",
|
||||
"ENGINEERING": "工程类公告",
|
||||
"FRAMEWORK_AGREEMENT": "框架协议征集公告",
|
||||
"FRAMEWORK_RESULT": "框架协议入围结果公告",
|
||||
"FRAMEWORK_SUMMARY": "框架协议成交结果汇总公告",
|
||||
"INTENTION": "采购意向公开"
|
||||
}
|
||||
|
||||
# 获取公告类型的中文显示名称
|
||||
announcement_type_enum = str(announcement.announcement_type).split('.')[-1]
|
||||
announcement_type_display = type_mapping.get(announcement_type_enum, announcement_type_enum)
|
||||
|
||||
# 确定来源名称
|
||||
source_name = announcement.purchase_name if announcement.purchase_name else announcement.source_name
|
||||
if len(source_name) > 25: # 限制来源名称长度
|
||||
source_name = source_name[:22] + "..."
|
||||
|
||||
# 根据来源代码添加前缀标识
|
||||
source_prefix = ""
|
||||
if announcement.source_code == 'dahuagov':
|
||||
source_prefix = "【大化县政府网】"
|
||||
else:
|
||||
source_prefix = "【广西政府采购网】"
|
||||
|
||||
# 时间格式化
|
||||
if announcement.publish_date:
|
||||
time_str = announcement.publish_date.strftime("%Y-%m-%d %H:%M")
|
||||
else:
|
||||
time_str = "时间未知"
|
||||
|
||||
# 生成描述：来源标识 | 类型 | 来源单位 | 时间
|
||||
description = f'<div style="font-size: 14px; margin-top: 8px;">{source_prefix}{announcement_type_display} | {source_name} | {time_str}</div>'
|
||||
|
||||
# URL：公告详情链接
|
||||
url = announcement.content_url
|
||||
|
||||
return title, description, url
|
||||
|
||||
def _generate_announcement_notification(self, announcements: List[Announcement],
|
||||
max_count: int) -> str:
|
||||
"""
|
||||
生成公告通知内容（改进版）
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
max_count: 最大显示数量
|
||||
|
||||
Returns:
|
||||
str: Markdown格式的通知内容
|
||||
"""
|
||||
if not announcements:
|
||||
return f"""# 🔔 广西政府采购网公告更新
|
||||
|
||||
**暂无新公告**
|
||||
|
||||
---
|
||||
|
||||
*更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*
|
||||
*点击公告标题查看详情*"""
|
||||
|
||||
# 按日期分组
|
||||
today_announcements = []
|
||||
other_announcements = []
|
||||
today = datetime.now().date()
|
||||
|
||||
for announcement in announcements:
|
||||
if announcement.publish_date and announcement.publish_date.date() == today:
|
||||
today_announcements.append(announcement)
|
||||
else:
|
||||
other_announcements.append(announcement)
|
||||
|
||||
lines = []
|
||||
|
||||
# 标题和概要
|
||||
total_count = len(announcements)
|
||||
|
||||
# 检查是否包含多个来源
|
||||
has_gxgp = any(a.source_code != 'dahuagov' for a in announcements)
|
||||
has_dahua = any(a.source_code == 'dahuagov' for a in announcements)
|
||||
|
||||
if has_gxgp and has_dahua:
|
||||
lines.append("# 🔔 政府采购公告更新（双源监控）")
|
||||
elif has_dahua:
|
||||
lines.append("# 🔔 大化县政府网采购公告更新")
|
||||
else:
|
||||
lines.append("# 🔔 广西政府采购网公告更新")
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"📊 **共发现 {total_count} 条新公告**")
|
||||
lines.append("")
|
||||
|
||||
# 今日公告
|
||||
if today_announcements:
|
||||
lines.append(f"## 🔥 今日公告 ({len(today_announcements)}条)")
|
||||
lines.append("")
|
||||
display_today = today_announcements[:max_count//2]
|
||||
|
||||
for i, announcement in enumerate(display_today, 1):
|
||||
# 改进标题显示：保留更多字符，但确保美观
|
||||
title = announcement.title
|
||||
if len(title) > 50:
|
||||
title = title[:47] + "..."
|
||||
|
||||
# 显示时间
|
||||
time_str = announcement.publish_date.strftime("%H:%M") if announcement.publish_date else "N/A"
|
||||
|
||||
# 添加序号和更好的格式
|
||||
lines.append(f"**{i}.** [{title}]({announcement.content_url})")
|
||||
lines.append(f" ⏰ {time_str} | 📍 {announcement.source_name}")
|
||||
lines.append("")
|
||||
|
||||
if len(today_announcements) > len(display_today):
|
||||
lines.append(f"⚠️ 还有 {len(today_announcements) - len(display_today)} 条今日公告未显示")
|
||||
lines.append("")
|
||||
|
||||
# 其他公告
|
||||
if other_announcements:
|
||||
lines.append(f"## 📄 其他公告 ({len(other_announcements)}条)")
|
||||
lines.append("")
|
||||
remaining_slots = max_count - len(today_announcements) if today_announcements else max_count
|
||||
display_other = other_announcements[:remaining_slots]
|
||||
|
||||
for i, announcement in enumerate(display_other, 1):
|
||||
title = announcement.title
|
||||
if len(title) > 45:
|
||||
title = title[:42] + "..."
|
||||
|
||||
date_str = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A"
|
||||
lines.append(f"**{i}.** [{title}]({announcement.content_url}) - {date_str}")
|
||||
|
||||
if len(other_announcements) > len(display_other):
|
||||
lines.append(f"⚠️ 还有 {len(other_announcements) - len(display_other)} 条历史公告未显示")
|
||||
lines.append("")
|
||||
|
||||
# 统计信息 - 改进版
|
||||
lines.append("## 📈 数据统计")
|
||||
lines.append("")
|
||||
|
||||
# 按来源统计
|
||||
source_stats = {}
|
||||
for announcement in announcements:
|
||||
source = announcement.source_name
|
||||
source_stats[source] = source_stats.get(source, 0) + 1
|
||||
|
||||
# 按类型统计
|
||||
type_stats = {}
|
||||
for announcement in announcements:
|
||||
ann_type = str(announcement.announcement_type).split('.')[-1] # 获取枚举名称
|
||||
type_stats[ann_type] = type_stats.get(ann_type, 0) + 1
|
||||
|
||||
lines.append("**按来源统计：**")
|
||||
for source, count in sorted(source_stats.items()):
|
||||
lines.append(f"• {source}: {count}条")
|
||||
lines.append("")
|
||||
|
||||
lines.append("**按类型统计：**")
|
||||
for ann_type, count in sorted(type_stats.items()):
|
||||
lines.append(f"• {ann_type}: {count}条")
|
||||
lines.append("")
|
||||
|
||||
# 分割线和时间
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append(f"🕒 *更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*")
|
||||
lines.append("💡 *点击公告标题查看详情*")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _generate_textcard_notification(self, announcements: List[Announcement],
|
||||
max_count: int) -> tuple[str, str, str]:
|
||||
"""
|
||||
生成文本卡片格式的通知内容
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
max_count: 最大显示数量
|
||||
|
||||
Returns:
|
||||
tuple[str, str, str]: (标题, 描述HTML, URL)
|
||||
"""
|
||||
# 按日期分组
|
||||
today_announcements = []
|
||||
other_announcements = []
|
||||
today = datetime.now().date()
|
||||
|
||||
for announcement in announcements:
|
||||
if announcement.publish_date and announcement.publish_date.date() == today:
|
||||
today_announcements.append(announcement)
|
||||
else:
|
||||
other_announcements.append(announcement)
|
||||
|
||||
# 生成标题
|
||||
total_count = len(announcements)
|
||||
|
||||
# 检查是否包含多个来源
|
||||
has_gxgp = any(a.source_code != 'dahuagov' for a in announcements)
|
||||
has_dahua = any(a.source_code == 'dahuagov' for a in announcements)
|
||||
|
||||
if has_gxgp and has_dahua:
|
||||
title = f"🔔 政府采购公告更新 ({total_count}条) - 双源监控"
|
||||
elif has_dahua:
|
||||
title = f"🔔 大化县政府网采购公告更新 ({total_count}条)"
|
||||
else:
|
||||
title = f"🔔 广西政府采购网公告更新 ({total_count}条)"
|
||||
|
||||
# 生成描述HTML
|
||||
html_parts = []
|
||||
|
||||
# 总统计
|
||||
html_parts.append('<div class="highlight">📊 发现 {total_count} 条新公告</div>'.format(total_count=total_count))
|
||||
html_parts.append("")
|
||||
|
||||
# 今日公告
|
||||
if today_announcements:
|
||||
html_parts.append('<div class="normal">🔥 今日公告 ({count}条)</div>'.format(count=len(today_announcements)))
|
||||
|
||||
display_today = today_announcements[:max_count//2]
|
||||
for i, announcement in enumerate(display_today, 1):
|
||||
# 标题处理
|
||||
ann_title = announcement.title
|
||||
if len(ann_title) > 35: # 文本卡片标题较短
|
||||
ann_title = ann_title[:32] + "..."
|
||||
|
||||
# 时间和来源
|
||||
time_str = announcement.publish_date.strftime("%H:%M") if announcement.publish_date else "N/A"
|
||||
source = announcement.source_name[:10] # 限制来源名称长度
|
||||
|
||||
html_parts.append('{i}. <a href="{url}">{title}</a>'.format(
|
||||
i=i, url=announcement.content_url, title=ann_title))
|
||||
html_parts.append('<div class="gray">⏰ {time} | 📍 {source}</div>'.format(
|
||||
time=time_str, source=source))
|
||||
|
||||
if len(today_announcements) > len(display_today):
|
||||
remaining = len(today_announcements) - len(display_today)
|
||||
html_parts.append('<div class="gray">还有 {remaining} 条今日公告...</div>'.format(remaining=remaining))
|
||||
|
||||
# 其他公告
|
||||
if other_announcements:
|
||||
html_parts.append("")
|
||||
html_parts.append('<div class="normal">📄 其他公告 ({count}条)</div>'.format(count=len(other_announcements)))
|
||||
|
||||
remaining_slots = max_count - len(today_announcements) if today_announcements else max_count
|
||||
display_other = other_announcements[:remaining_slots]
|
||||
|
||||
for i, announcement in enumerate(display_other, 1):
|
||||
ann_title = announcement.title
|
||||
if len(ann_title) > 30:
|
||||
ann_title = ann_title[:27] + "..."
|
||||
|
||||
date_str = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A"
|
||||
html_parts.append('{i}. <a href="{url}">{title}</a> <span class="gray">({date})</span>'.format(
|
||||
i=i, url=announcement.content_url, title=ann_title, date=date_str))
|
||||
|
||||
if len(other_announcements) > len(display_other):
|
||||
remaining = len(other_announcements) - len(display_other)
|
||||
html_parts.append('<div class="gray">还有 {remaining} 条历史公告...</div>'.format(remaining=remaining))
|
||||
|
||||
# 统计信息
|
||||
html_parts.append("")
|
||||
html_parts.append('<div class="normal">📈 数据统计</div>')
|
||||
|
||||
# 按来源统计
|
||||
source_stats = {}
|
||||
for announcement in announcements:
|
||||
source = announcement.source_name
|
||||
source_stats[source] = source_stats.get(source, 0) + 1
|
||||
|
||||
html_parts.append('<div class="gray">按来源: {stats}</div>'.format(
|
||||
stats=" | ".join([f"{source}:{count}" for source, count in sorted(source_stats.items())])))
|
||||
|
||||
# 时间戳
|
||||
update_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
html_parts.append("")
|
||||
html_parts.append('<div class="gray">🕒 更新时间: {time}</div>'.format(time=update_time))
|
||||
|
||||
description = "\n".join(html_parts)
|
||||
|
||||
# 限制描述长度（企业微信文本卡片description不超过512字符）
|
||||
if len(description) > 500:
|
||||
description = description[:497] + "..."
|
||||
|
||||
# 生成跳转URL（可以跳转到公告列表页面或第一条公告）
|
||||
if announcements:
|
||||
url = announcements[0].content_url # 默认跳转到第一条公告
|
||||
else:
|
||||
url = "https://zfcg.gxzf.gov.cn" # 默认跳转到网站首页
|
||||
|
||||
return title, description, url
|
||||
|
||||
def send_system_notification(self, title: str, content: str,
|
||||
message_type: str = "text") -> bool:
|
||||
"""
|
||||
发送系统通知
|
||||
|
||||
Args:
|
||||
title: 通知标题
|
||||
content: 通知内容
|
||||
message_type: 消息类型 (text/markdown)
|
||||
|
||||
Returns:
|
||||
bool: 发送是否成功
|
||||
"""
|
||||
try:
|
||||
if message_type == "markdown":
|
||||
full_content = f"# {title}\n\n{content}"
|
||||
return self.send_markdown_message(full_content)
|
||||
else:
|
||||
full_content = f"{title}\n\n{content}"
|
||||
return self.send_text_message(full_content)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"发送系统通知失败: {str(e)}")
|
||||
return False
|
||||
|
||||
def send_error_notification(self, error_message: str, error_details: Optional[str] = None) -> bool:
|
||||
"""
|
||||
发送错误通知
|
||||
|
||||
Args:
|
||||
error_message: 错误消息
|
||||
error_details: 错误详情
|
||||
|
||||
Returns:
|
||||
bool: 发送是否成功
|
||||
"""
|
||||
content = f"## ❌ 系统错误\n\n**错误信息**: {error_message}"
|
||||
|
||||
if error_details:
|
||||
content += f"\n\n**错误详情**:\n```\n{error_details}\n```"
|
||||
|
||||
content += f"\n\n*发生时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*"
|
||||
|
||||
return self.send_markdown_message(content)
|
||||
|
||||
def test_connection(self) -> bool:
|
||||
"""
|
||||
测试连接
|
||||
|
||||
Returns:
|
||||
bool: 连接是否正常
|
||||
"""
|
||||
try:
|
||||
token = self._get_access_token()
|
||||
return token is not None
|
||||
except Exception as e:
|
||||
logger.error(f"企业微信连接测试失败: {str(e)}")
|
||||
return False
|
||||
|
||||
def get_service_status(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取服务状态
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 服务状态信息
|
||||
"""
|
||||
return {
|
||||
"service": "wechat",
|
||||
"enabled": self.config.enabled,
|
||||
"corp_id": self.config.corp_id[:10] + "..." if self.config.corp_id else None,
|
||||
"agent_id": self.config.agent_id,
|
||||
"has_token": self._access_token is not None,
|
||||
"token_expires_at": datetime.fromtimestamp(self._token_expires_at).isoformat() if self._token_expires_at > 0 else None,
|
||||
"use_proxy": self.config.use_proxy,
|
||||
"connection_test": self.test_connection() if self.config.enabled else False
|
||||
}
|
||||
|
||||
|
||||
class NotificationManager:
|
||||
"""通知管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self.wechat = WeChatService()
|
||||
self._services = {
|
||||
"wechat": self.wechat
|
||||
}
|
||||
|
||||
def send_announcement_notification(self, announcements: List[Announcement]) -> Dict[str, bool]:
|
||||
"""
|
||||
发送公告通知
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
Dict[str, bool]: 各服务发送结果
|
||||
"""
|
||||
results = {}
|
||||
|
||||
# 企业微信通知
|
||||
if self.wechat.config.enabled:
|
||||
try:
|
||||
results["wechat"] = self.wechat.send_announcement_notification(announcements)
|
||||
except Exception as e:
|
||||
logger.error(f"企业微信通知失败: {str(e)}")
|
||||
results["wechat"] = False
|
||||
else:
|
||||
results["wechat"] = None # 未启用
|
||||
|
||||
return results
|
||||
|
||||
def send_system_notification(self, title: str, content: str) -> Dict[str, bool]:
|
||||
"""
|
||||
发送系统通知
|
||||
|
||||
Args:
|
||||
title: 通知标题
|
||||
content: 通知内容
|
||||
|
||||
Returns:
|
||||
Dict[str, bool]: 发送结果
|
||||
"""
|
||||
results = {}
|
||||
|
||||
if self.wechat.config.enabled:
|
||||
try:
|
||||
results["wechat"] = self.wechat.send_system_notification(title, content, "markdown")
|
||||
except Exception as e:
|
||||
logger.error(f"企业微信系统通知失败: {str(e)}")
|
||||
results["wechat"] = False
|
||||
else:
|
||||
results["wechat"] = None
|
||||
|
||||
return results
|
||||
|
||||
def send_error_notification(self, error_message: str, error_details: Optional[str] = None) -> Dict[str, bool]:
|
||||
"""
|
||||
发送错误通知
|
||||
|
||||
Args:
|
||||
error_message: 错误消息
|
||||
error_details: 错误详情
|
||||
|
||||
Returns:
|
||||
Dict[str, bool]: 发送结果
|
||||
"""
|
||||
results = {}
|
||||
|
||||
if self.wechat.config.enabled:
|
||||
try:
|
||||
results["wechat"] = self.wechat.send_error_notification(error_message, error_details)
|
||||
except Exception as e:
|
||||
logger.error(f"企业微信错误通知失败: {str(e)}")
|
||||
results["wechat"] = False
|
||||
else:
|
||||
results["wechat"] = None
|
||||
|
||||
return results
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取通知服务状态
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 服务状态
|
||||
"""
|
||||
return {
|
||||
"services": {
|
||||
name: service.get_service_status() for name, service in self._services.items()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# 全局通知管理器实例
|
||||
_notification_manager = None
|
||||
|
||||
|
||||
def get_notification_manager() -> NotificationManager:
|
||||
"""
|
||||
获取通知管理器实例
|
||||
|
||||
Returns:
|
||||
NotificationManager: 通知管理器实例
|
||||
"""
|
||||
global _notification_manager
|
||||
if _notification_manager is None:
|
||||
_notification_manager = NotificationManager()
|
||||
return _notification_manager
|
||||
|
||||
|
||||
def send_announcements_notification(announcements: List[Announcement]) -> bool:
|
||||
"""
|
||||
发送公告通知
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
bool: 是否至少有一个服务发送成功
|
||||
"""
|
||||
manager = get_notification_manager()
|
||||
results = manager.send_announcement_notification(announcements)
|
||||
|
||||
# 检查是否有服务发送成功
|
||||
return any(result for result in results.values() if result is True)
|
||||
|
||||
|
||||
def send_system_notification(title: str, content: str) -> bool:
|
||||
"""
|
||||
发送系统通知
|
||||
|
||||
Args:
|
||||
title: 通知标题
|
||||
content: 通知内容
|
||||
|
||||
Returns:
|
||||
bool: 是否至少有一个服务发送成功
|
||||
"""
|
||||
manager = get_notification_manager()
|
||||
results = manager.send_system_notification(title, content)
|
||||
|
||||
return any(result for result in results.values() if result is True)
|
||||
|
||||
|
||||
def send_error_alert(error_message: str, error_details: Optional[str] = None) -> bool:
|
||||
"""
|
||||
发送错误警报
|
||||
|
||||
Args:
|
||||
error_message: 错误消息
|
||||
error_details: 错误详情
|
||||
|
||||
Returns:
|
||||
bool: 是否至少有一个服务发送成功
|
||||
"""
|
||||
manager = get_notification_manager()
|
||||
results = manager.send_error_notification(error_message, error_details)
|
||||
|
||||
return any(result for result in results.values() if result is True)
|
||||
@@ -1,41 +0,0 @@
|
||||
# 广西政府采购网公告监控系统依赖包
|
||||
|
||||
# 核心依赖
|
||||
requests>=2.28.0 # HTTP请求库
|
||||
psycopg2-binary>=2.9.0 # PostgreSQL数据库驱动
|
||||
PyYAML>=6.0 # YAML配置文件解析
|
||||
python-dateutil>=2.8.0 # 日期时间处理
|
||||
|
||||
# 爬虫相关
|
||||
fake-useragent>=1.1.0 # 随机User-Agent生成
|
||||
lxml>=4.9.0 # XML/HTML解析（备用）
|
||||
|
||||
# 调度器
|
||||
schedule>=1.2.0 # 定时任务调度
|
||||
croniter>=1.4.0 # Cron表达式解析
|
||||
|
||||
# 日志和监控
|
||||
logging>=0.4.9.6 # 日志处理（Python内置）
|
||||
colorama>=0.4.6 # 控制台颜色输出（可选，用于彩色日志）
|
||||
|
||||
# 数据处理
|
||||
pandas>=1.5.0 # 数据处理（可选，用于复杂数据分析）
|
||||
openpyxl>=3.0.10 # Excel文件处理（可选）
|
||||
|
||||
# 加密和安全
|
||||
cryptography>=39.0.0 # 加密库（用于微信消息加密）
|
||||
pycryptodome>=3.17.0 # 加密算法库
|
||||
|
||||
# Web框架（企业微信回调服务器）
|
||||
flask>=2.3.0 # Web框架
|
||||
|
||||
# 可选依赖（根据需要安装）
|
||||
# redis>=4.5.0 # Redis缓存（如果需要）
|
||||
# sqlalchemy>=2.0.0 # ORM（如果需要更复杂的数据库操作）
|
||||
# celery>=5.3.0 # 分布式任务队列（如果需要）
|
||||
|
||||
# 开发依赖（仅开发环境需要）
|
||||
# pytest>=7.2.0 # 测试框架
|
||||
# black>=23.0.0 # 代码格式化
|
||||
# flake8>=6.0.0 # 代码检查
|
||||
# mypy>=1.0.0 # 类型检查
|
||||
@@ -1 +0,0 @@
|
||||
"""存储模块"""
|
||||
@@ -1,396 +0,0 @@
|
||||
"""
|
||||
Markdown生成器模块
|
||||
生成公告的Markdown格式输出文件
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
|
||||
try:
|
||||
from ..core.models import Announcement
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger
|
||||
except ImportError:
|
||||
from core.models import Announcement
|
||||
from core.config_manager import get_config
|
||||
from core.logger import get_logger
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class MarkdownGenerator:
|
||||
"""Markdown生成器"""
|
||||
|
||||
def __init__(self, output_file: Optional[str] = None):
|
||||
"""
|
||||
初始化Markdown生成器
|
||||
|
||||
Args:
|
||||
output_file: 输出文件路径
|
||||
"""
|
||||
self.config = get_config()
|
||||
self.output_file = output_file or self.config.markdown.output_file
|
||||
self.max_entries = self.config.markdown.max_entries
|
||||
self.include_today_highlight = self.config.markdown.include_today_highlight
|
||||
|
||||
# 确保输出目录存在
|
||||
output_path = Path(self.output_file)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def generate_markdown(self, announcements: List[Announcement],
|
||||
title: str = "广西政府采购网公告监控",
|
||||
time_period: str = None) -> str:
|
||||
"""
|
||||
生成Markdown内容
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
title: 文档标题
|
||||
|
||||
Returns:
|
||||
str: Markdown格式的文本
|
||||
"""
|
||||
if not announcements:
|
||||
return self._generate_empty_markdown(title, time_period)
|
||||
|
||||
# 按来源分组
|
||||
grouped_announcements = self._group_announcements_by_source(announcements)
|
||||
|
||||
# 生成Markdown
|
||||
lines = []
|
||||
lines.append("# 搜索完成")
|
||||
lines.append("")
|
||||
|
||||
# 解析标题中的关键词
|
||||
keyword = "未知"
|
||||
if "关键词:" in title:
|
||||
keyword_part = title.split("关键词:")[-1].strip()
|
||||
keyword = keyword_part.split()[0] if keyword_part else "未知"
|
||||
|
||||
lines.append(f"📋 关键词搜索: `{keyword}` - 总公告数: `{len(announcements)}`")
|
||||
lines.append("")
|
||||
|
||||
# 使用传入的时间段或默认的更新时间
|
||||
if time_period:
|
||||
lines.append(f"**时间段: {time_period}**")
|
||||
else:
|
||||
lines.append(f"**更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}**")
|
||||
lines.append("")
|
||||
lines.append("")
|
||||
|
||||
# 生成各来源的公告
|
||||
for source_name, source_announcements in grouped_announcements.items():
|
||||
lines.extend(self._generate_source_section(source_name, source_announcements))
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _group_announcements_by_source(self, announcements: List[Announcement]) -> Dict[str, List[Announcement]]:
|
||||
"""按来源分组公告"""
|
||||
grouped = defaultdict(list)
|
||||
|
||||
for announcement in announcements:
|
||||
grouped[announcement.source_name].append(announcement)
|
||||
|
||||
# 对每个组内的公告按时间倒序排列
|
||||
for source_name in grouped:
|
||||
grouped[source_name].sort(key=lambda x: x.publish_date, reverse=True)
|
||||
|
||||
return dict(grouped)
|
||||
|
||||
def _generate_toc(self, grouped_announcements: Dict[str, List[Announcement]]) -> List[str]:
|
||||
"""生成目录"""
|
||||
lines = ["## 目录", ""]
|
||||
|
||||
for source_name, announcements in grouped_announcements.items():
|
||||
# 创建锚点链接
|
||||
anchor = self._create_anchor(source_name)
|
||||
count = len(announcements)
|
||||
lines.append(f"- [{source_name}](#{anchor}) ({count}条)")
|
||||
|
||||
return lines
|
||||
|
||||
def _generate_source_section(self, source_name: str, announcements: List[Announcement]) -> List[str]:
|
||||
"""生成来源章节"""
|
||||
lines = []
|
||||
|
||||
lines.append(f"## {source_name} - **共 {len(announcements)} 条**")
|
||||
lines.append("")
|
||||
|
||||
# 生成公告列表
|
||||
for i, announcement in enumerate(announcements, 1):
|
||||
lines.extend(self._generate_announcement_item(announcement, i))
|
||||
|
||||
return lines
|
||||
|
||||
def _generate_announcement_item(self, announcement: Announcement, index: int) -> List[str]:
|
||||
"""生成单个公告项"""
|
||||
lines = []
|
||||
|
||||
# 公告标题（包含超链接）
|
||||
title_line = f"### {index}. [{announcement.title}]({announcement.content_url})"
|
||||
lines.append(title_line)
|
||||
lines.append("")
|
||||
|
||||
# 公告信息 - 简洁格式
|
||||
info_parts = []
|
||||
|
||||
if announcement.publish_date:
|
||||
publish_date = announcement.publish_date.strftime("%Y-%m-%d")
|
||||
info_parts.append(publish_date)
|
||||
|
||||
if announcement.purchase_name:
|
||||
info_parts.append(announcement.purchase_name)
|
||||
|
||||
info_parts.append(announcement.source_name)
|
||||
|
||||
if info_parts:
|
||||
info_line = " | ".join(info_parts)
|
||||
lines.append(info_line)
|
||||
lines.append("")
|
||||
lines.append("")
|
||||
|
||||
return lines
|
||||
|
||||
def _generate_empty_markdown(self, title: str, time_period: str = None) -> str:
|
||||
"""生成空内容的Markdown"""
|
||||
# 解析标题中的关键词
|
||||
keyword = "未知"
|
||||
if "关键词:" in title:
|
||||
keyword_part = title.split("关键词:")[-1].strip()
|
||||
keyword = keyword_part.split()[0] if keyword_part else "未知"
|
||||
|
||||
lines = [
|
||||
f"📋 关键词搜索: `{keyword}` - 总公告数: `0`",
|
||||
"",
|
||||
f"**时间段: {time_period}**" if time_period else f"**更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}**",
|
||||
"",
|
||||
"",
|
||||
"## 无匹配公告",
|
||||
"",
|
||||
"在指定时间范围内没有找到符合条件的公告。",
|
||||
""
|
||||
]
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _create_anchor(self, text: str) -> str:
|
||||
"""创建锚点链接"""
|
||||
# 移除特殊字符，替换空格为连字符，转为小写
|
||||
import re
|
||||
anchor = re.sub(r'[^\w\s-]', '', text)
|
||||
anchor = re.sub(r'[-\s]+', '-', anchor)
|
||||
return anchor.lower().strip('-')
|
||||
|
||||
def save_to_file(self, announcements: List[Announcement],
|
||||
title: Optional[str] = None) -> bool:
|
||||
"""
|
||||
保存Markdown到文件
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
title: 文档标题
|
||||
|
||||
Returns:
|
||||
bool: 保存是否成功
|
||||
"""
|
||||
try:
|
||||
markdown_content = self.generate_markdown(announcements, title)
|
||||
|
||||
with open(self.output_file, 'w', encoding='utf-8') as f:
|
||||
f.write(markdown_content)
|
||||
|
||||
logger.info(f"Markdown文件已保存到: {self.output_file} (共 {len(announcements)} 条公告)")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存Markdown文件失败: {str(e)}")
|
||||
return False
|
||||
|
||||
def append_to_file(self, new_announcements: List[Announcement]) -> bool:
|
||||
"""
|
||||
追加新公告到现有文件
|
||||
|
||||
Args:
|
||||
new_announcements: 新公告列表
|
||||
|
||||
Returns:
|
||||
bool: 追加是否成功
|
||||
"""
|
||||
if not new_announcements:
|
||||
return True
|
||||
|
||||
try:
|
||||
# 读取现有文件
|
||||
existing_content = ""
|
||||
if os.path.exists(self.output_file):
|
||||
with open(self.output_file, 'r', encoding='utf-8') as f:
|
||||
existing_content = f.read()
|
||||
|
||||
# 如果文件不存在或为空，创建新文件
|
||||
if not existing_content.strip():
|
||||
return self.save_to_file(new_announcements)
|
||||
|
||||
# 解析现有公告（这里简化处理，实际可能需要更复杂的解析）
|
||||
# 为简单起见，我们重新生成完整文件
|
||||
logger.info("重新生成完整Markdown文件")
|
||||
return self.save_to_file(new_announcements)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"追加公告到Markdown文件失败: {str(e)}")
|
||||
return False
|
||||
|
||||
def get_file_stats(self) -> Dict[str, Any]:
|
||||
"""获取文件统计信息"""
|
||||
stats = {
|
||||
"file_exists": False,
|
||||
"file_size": 0,
|
||||
"last_modified": None,
|
||||
"announcement_count": 0
|
||||
}
|
||||
|
||||
try:
|
||||
if os.path.exists(self.output_file):
|
||||
file_stat = os.stat(self.output_file)
|
||||
stats["file_exists"] = True
|
||||
stats["file_size"] = file_stat.st_size
|
||||
stats["last_modified"] = datetime.fromtimestamp(file_stat.st_mtime).isoformat()
|
||||
|
||||
# 尝试统计公告数量（简单计数）
|
||||
with open(self.output_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
# 统计###开头的行（每个公告的标题行）
|
||||
stats["announcement_count"] = content.count("### ")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"获取文件统计信息失败: {str(e)}")
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
class AnnouncementMarkdownFormatter:
|
||||
"""公告Markdown格式化器"""
|
||||
|
||||
@staticmethod
|
||||
def format_announcement_card(announcement: Announcement) -> str:
|
||||
"""格式化单个公告为卡片样式"""
|
||||
lines = []
|
||||
|
||||
# 标题
|
||||
emoji = "🆕" if announcement.is_today else "📄"
|
||||
lines.append(f"### {emoji} {announcement.title}")
|
||||
lines.append("")
|
||||
|
||||
# 链接
|
||||
lines.append(f"[查看详情]({announcement.content_url})")
|
||||
lines.append("")
|
||||
|
||||
# 信息表格
|
||||
lines.append("| 属性 | 值 |")
|
||||
lines.append("|------|-----|")
|
||||
|
||||
if announcement.publish_date:
|
||||
lines.append(f"| 发布时间 | {announcement.publish_date.strftime('%Y-%m-%d %H:%M')} |")
|
||||
|
||||
lines.append(f"| 发布单位 | {announcement.purchase_name or 'N/A'} |")
|
||||
lines.append(f"| 来源 | {announcement.source_name} |")
|
||||
lines.append(f"| 公告类型 | {announcement.announcement_type.value} |")
|
||||
|
||||
if announcement.crawled_at:
|
||||
lines.append(f"| 爬取时间 | {announcement.crawled_at.strftime('%m-%d %H:%M')} |")
|
||||
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@staticmethod
|
||||
def format_announcement_list(announcements: List[Announcement]) -> str:
|
||||
"""格式化公告列表"""
|
||||
if not announcements:
|
||||
return "*暂无公告*"
|
||||
|
||||
lines = []
|
||||
for announcement in announcements:
|
||||
emoji = "🆕" if announcement.is_today else "•"
|
||||
publish_date = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A"
|
||||
line = f"{emoji} [{announcement.title}]({announcement.content_url}) - {publish_date}"
|
||||
lines.append(line)
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@staticmethod
|
||||
def format_notification_message(announcements: List[Announcement],
|
||||
max_count: int = 10) -> str:
|
||||
"""格式化为通知消息"""
|
||||
if not announcements:
|
||||
return "暂无新公告"
|
||||
|
||||
# 只显示前N条
|
||||
display_announcements = announcements[:max_count]
|
||||
remaining_count = len(announcements) - max_count
|
||||
|
||||
lines = [f"🔔 发现 {len(announcements)} 条新公告：", ""]
|
||||
|
||||
for announcement in display_announcements:
|
||||
title = announcement.title[:50] + "..." if len(announcement.title) > 50 else announcement.title
|
||||
publish_date = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A"
|
||||
lines.append(f"• {title} ({publish_date})")
|
||||
|
||||
if remaining_count > 0:
|
||||
lines.append(f"... 还有 {remaining_count} 条公告")
|
||||
|
||||
lines.append("")
|
||||
lines.append("*点击公告标题查看详情*")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def create_markdown_generator(output_file: Optional[str] = None) -> MarkdownGenerator:
|
||||
"""
|
||||
创建Markdown生成器实例
|
||||
|
||||
Args:
|
||||
output_file: 输出文件路径
|
||||
|
||||
Returns:
|
||||
MarkdownGenerator: 生成器实例
|
||||
"""
|
||||
return MarkdownGenerator(output_file)
|
||||
|
||||
|
||||
def generate_onu_md(announcements: List[Announcement]) -> bool:
|
||||
"""
|
||||
生成onu.md文件
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
bool: 生成是否成功
|
||||
"""
|
||||
generator = create_markdown_generator()
|
||||
return generator.save_to_file(announcements, "广西政府采购网公告监控")
|
||||
|
||||
|
||||
def update_onu_md(new_announcements: List[Announcement]) -> bool:
|
||||
"""
|
||||
更新onu.md文件，追加新公告
|
||||
|
||||
Args:
|
||||
new_announcements: 新公告列表
|
||||
|
||||
Returns:
|
||||
bool: 更新是否成功
|
||||
"""
|
||||
generator = create_markdown_generator()
|
||||
|
||||
# 如果文件不存在，创建新文件
|
||||
if not os.path.exists(generator.output_file):
|
||||
return generator.save_to_file(new_announcements)
|
||||
|
||||
# 否则追加新公告
|
||||
return generator.append_to_file(new_announcements)
|
||||
@@ -1,650 +0,0 @@
|
||||
"""
|
||||
PostgreSQL存储模块
|
||||
实现公告数据的PostgreSQL存储和管理
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Any, Optional
|
||||
from datetime import datetime, timedelta
|
||||
import threading
|
||||
from dataclasses import asdict
|
||||
|
||||
try:
|
||||
from ..core.models import Announcement, CrawlResult
|
||||
from ..core.database import get_database_manager, init_database
|
||||
from ..core.logger import get_logger
|
||||
from ..core.reliability import retry_on_exception, RetryConfig, safe_execute
|
||||
except ImportError:
|
||||
from core.models import Announcement, CrawlResult
|
||||
from core.database import get_database_manager, init_database
|
||||
from core.logger import get_logger
|
||||
from core.reliability import retry_on_exception, RetryConfig, safe_execute
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class PostgreSQLStorage:
|
||||
"""PostgreSQL存储管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self.db_manager = get_database_manager()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def save_announcements(self, announcements: List[Announcement]) -> int:
|
||||
"""
|
||||
保存公告列表到数据库（经过筛选的公告）
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
int: 成功保存的数量
|
||||
"""
|
||||
if not announcements:
|
||||
return 0
|
||||
|
||||
logger.info(f"开始保存 {len(announcements)} 条筛选后公告到数据库")
|
||||
|
||||
try:
|
||||
# 批量保存
|
||||
saved_count = self.db_manager.save_announcements_batch(announcements)
|
||||
|
||||
if saved_count > 0:
|
||||
logger.info(f"成功保存 {saved_count} 条筛选后公告到数据库")
|
||||
|
||||
# 标记新公告
|
||||
self._mark_new_announcements(announcements[:saved_count])
|
||||
|
||||
return saved_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存筛选后公告到数据库失败: {str(e)}")
|
||||
# 尝试逐个保存
|
||||
return self._save_announcements_fallback(announcements)
|
||||
|
||||
def save_auto_announcements(self, announcements: List[Announcement]) -> int:
|
||||
"""
|
||||
保存定时搜索公告到专用表（关键词匹配专用）
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
int: 成功保存的数量
|
||||
"""
|
||||
if not announcements:
|
||||
return 0
|
||||
|
||||
logger.info(f"开始保存 {len(announcements)} 条定时搜索公告到专用表")
|
||||
|
||||
try:
|
||||
# 批量保存到auto_announcements表
|
||||
saved_count = self.db_manager.save_announcements_batch_to_table(announcements, "auto_announcements")
|
||||
|
||||
if saved_count > 0:
|
||||
logger.info(f"成功保存 {saved_count} 条定时搜索公告到专用表")
|
||||
|
||||
return saved_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"保存定时搜索公告到专用表失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
def save_manual_announcements_by_source(self, announcements: List[Announcement],
|
||||
max_per_source: int = 100) -> Dict[str, int]:
|
||||
"""
|
||||
按来源保存手动搜索公告到专用表，每个来源保留最新的max_per_source条
|
||||
|
||||
Args:
|
||||
announcements: 所有公告列表（未经关键词筛选）
|
||||
max_per_source: 每个来源最大保留数量
|
||||
|
||||
Returns:
|
||||
Dict[str, int]: 各来源保存的数量
|
||||
"""
|
||||
if not announcements:
|
||||
return {}
|
||||
|
||||
logger.info(f"开始按来源保存 {len(announcements)} 条手动搜索公告到专用表，每个来源最多保留 {max_per_source} 条")
|
||||
|
||||
try:
|
||||
# 按来源分组
|
||||
source_groups = {}
|
||||
for announcement in announcements:
|
||||
source_code = announcement.source_code
|
||||
if source_code not in source_groups:
|
||||
source_groups[source_code] = []
|
||||
source_groups[source_code].append(announcement)
|
||||
|
||||
saved_stats = {}
|
||||
|
||||
for source_code, source_announcements in source_groups.items():
|
||||
# 对每个来源的公告按发布时间排序（最新的在前）
|
||||
sorted_announcements = sorted(
|
||||
source_announcements,
|
||||
key=lambda x: x.publish_date or datetime.min,
|
||||
reverse=True
|
||||
)
|
||||
|
||||
# 为没有哈希的公告生成哈希
|
||||
for announcement in sorted_announcements:
|
||||
if not announcement.content_hash:
|
||||
announcement.generate_content_hash()
|
||||
|
||||
# 批量保存
|
||||
to_save = sorted_announcements[:max_per_source]
|
||||
saved_count = self.db_manager.save_announcements_batch_to_table(to_save, "manual_announcements")
|
||||
saved_stats[source_code] = saved_count
|
||||
|
||||
# 清理该来源超出限制的旧数据
|
||||
if len(sorted_announcements) > max_per_source:
|
||||
self._cleanup_old_announcements_by_source_in_table(source_code, max_per_source, "manual_announcements")
|
||||
|
||||
logger.info(f"来源 {source_code} 保存了 {saved_count} 条手动搜索公告")
|
||||
|
||||
total_saved = sum(saved_stats.values())
|
||||
logger.info(f"按来源保存手动搜索公告完成，总计保存 {total_saved} 条公告")
|
||||
|
||||
return saved_stats
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"按来源保存手动搜索公告失败: {str(e)}")
|
||||
return {}
|
||||
|
||||
def save_all_announcements_by_source(self, announcements: List[Announcement],
|
||||
max_per_source: int = 100) -> Dict[str, int]:
|
||||
"""
|
||||
按来源保存所有公告，每个来源保留最新的max_per_source条
|
||||
|
||||
Args:
|
||||
announcements: 所有公告列表（未经关键词筛选）
|
||||
max_per_source: 每个来源最大保留数量
|
||||
|
||||
Returns:
|
||||
Dict[str, int]: 各来源保存的数量
|
||||
"""
|
||||
if not announcements:
|
||||
return {}
|
||||
|
||||
logger.info(f"开始按来源保存 {len(announcements)} 条公告，每个来源最多保留 {max_per_source} 条")
|
||||
|
||||
try:
|
||||
# 按来源分组
|
||||
source_groups = {}
|
||||
for announcement in announcements:
|
||||
source_code = announcement.source_code
|
||||
if source_code not in source_groups:
|
||||
source_groups[source_code] = []
|
||||
source_groups[source_code].append(announcement)
|
||||
|
||||
saved_stats = {}
|
||||
|
||||
for source_code, source_announcements in source_groups.items():
|
||||
# 对每个来源的公告按发布时间排序（最新的在前）
|
||||
sorted_announcements = sorted(
|
||||
source_announcements,
|
||||
key=lambda x: x.publish_date or x.crawled_at or datetime.min,
|
||||
reverse=True
|
||||
)
|
||||
|
||||
# 取最新的max_per_source条
|
||||
to_save = sorted_announcements[:max_per_source]
|
||||
|
||||
# 为这些公告生成哈希
|
||||
for announcement in to_save:
|
||||
if not announcement.content_hash:
|
||||
announcement.generate_content_hash()
|
||||
|
||||
# 批量保存
|
||||
saved_count = self.db_manager.save_announcements_batch(to_save)
|
||||
|
||||
saved_stats[source_code] = saved_count
|
||||
|
||||
# 清理该来源超出限制的旧数据
|
||||
if len(sorted_announcements) > max_per_source:
|
||||
self._cleanup_old_announcements_by_source(source_code, max_per_source)
|
||||
|
||||
logger.info(f"来源 {source_code} 保存了 {saved_count} 条公告")
|
||||
|
||||
total_saved = sum(saved_stats.values())
|
||||
logger.info(f"按来源保存完成，总计保存 {total_saved} 条公告")
|
||||
|
||||
return saved_stats
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"按来源保存公告失败: {str(e)}")
|
||||
return {}
|
||||
|
||||
def _cleanup_old_announcements_by_source(self, source_code: str, keep_count: int):
|
||||
"""
|
||||
清理指定来源超出限制的旧公告
|
||||
|
||||
Args:
|
||||
source_code: 来源代码
|
||||
keep_count: 保留数量
|
||||
"""
|
||||
try:
|
||||
# 使用窗口函数删除超出限制的记录
|
||||
sql = """
|
||||
DELETE FROM announcements
|
||||
WHERE source_code = %s
|
||||
AND id IN (
|
||||
SELECT id FROM (
|
||||
SELECT id,
|
||||
ROW_NUMBER() OVER (ORDER BY publish_date DESC, crawled_at DESC) as rn
|
||||
FROM announcements
|
||||
WHERE source_code = %s
|
||||
) ranked
|
||||
WHERE rn > %s
|
||||
)
|
||||
"""
|
||||
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, (source_code, source_code, keep_count))
|
||||
deleted_count = cursor.rowcount
|
||||
|
||||
if deleted_count > 0:
|
||||
logger.debug(f"清理来源 {source_code} 的 {deleted_count} 条旧公告")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"清理来源 {source_code} 旧公告失败: {str(e)}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"按来源保存公告失败: {str(e)}")
|
||||
return {}
|
||||
|
||||
def _save_announcements_fallback(self, announcements: List[Announcement]) -> int:
|
||||
"""逐个保存公告的降级方案"""
|
||||
logger.info("使用降级方案逐个保存公告")
|
||||
|
||||
saved_count = 0
|
||||
for announcement in announcements:
|
||||
try:
|
||||
if self.db_manager.save_announcement(announcement):
|
||||
saved_count += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"保存公告失败: {announcement.title[:50]}..., 错误: {str(e)}")
|
||||
continue
|
||||
|
||||
logger.info(f"降级保存完成，成功保存 {saved_count} 条公告")
|
||||
return saved_count
|
||||
|
||||
def _mark_new_announcements(self, announcements: List[Announcement]):
|
||||
"""标记新公告"""
|
||||
# 这里可以添加新公告标记逻辑
|
||||
# 由于我们在爬取时已经标记，这里主要是确保数据库中的标记正确
|
||||
pass
|
||||
|
||||
def _mark_new_announcements_in_table(self, announcements: List[Announcement], table_name: str):
|
||||
"""在指定表中标记新公告"""
|
||||
# 这里可以添加新公告标记逻辑
|
||||
pass
|
||||
|
||||
def _cleanup_old_announcements_by_source_in_table(self, source_code: str, max_per_source: int, table_name: str):
|
||||
"""在指定表中清理来源的旧公告"""
|
||||
try:
|
||||
with self.db_manager.get_db_cursor() as cursor:
|
||||
# 获取该来源当前保存的公告数量
|
||||
cursor.execute(f"""
|
||||
SELECT COUNT(*) FROM {table_name}
|
||||
WHERE source_code = %s
|
||||
""", (source_code,))
|
||||
|
||||
current_count = cursor.fetchone()[0]
|
||||
|
||||
if current_count > max_per_source:
|
||||
# 删除超出数量的旧公告
|
||||
delete_count = current_count - max_per_source
|
||||
cursor.execute(f"""
|
||||
DELETE FROM {table_name}
|
||||
WHERE id IN (
|
||||
SELECT id FROM {table_name}
|
||||
WHERE source_code = %s
|
||||
ORDER BY publish_date DESC, created_at DESC
|
||||
OFFSET %s
|
||||
)
|
||||
""", (source_code, max_per_source))
|
||||
|
||||
logger.info(f"清理了 {cursor.rowcount} 条{table_name}表中来源{source_code}的旧公告")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理{table_name}表中来源{source_code}的旧公告失败: {str(e)}")
|
||||
|
||||
def save_crawl_results(self, results: List[CrawlResult]) -> int:
|
||||
"""
|
||||
保存爬取结果
|
||||
|
||||
Args:
|
||||
results: 爬取结果列表
|
||||
|
||||
Returns:
|
||||
int: 成功保存的数量
|
||||
"""
|
||||
if not results:
|
||||
return 0
|
||||
|
||||
saved_count = 0
|
||||
for result in results:
|
||||
try:
|
||||
if self.db_manager.save_crawl_result(result):
|
||||
saved_count += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"保存爬取结果失败: {result.source.name}, 错误: {str(e)}")
|
||||
continue
|
||||
|
||||
logger.info(f"保存爬取结果完成: {saved_count}/{len(results)}")
|
||||
return saved_count
|
||||
|
||||
def get_recent_announcements(self, hours: int = 24,
|
||||
limit: int = 100) -> List[Announcement]:
|
||||
"""
|
||||
获取最近的公告
|
||||
|
||||
Args:
|
||||
hours: 最近小时数
|
||||
limit: 限制数量
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 公告列表
|
||||
"""
|
||||
try:
|
||||
return self.db_manager.get_recent_announcements(hours)
|
||||
except Exception as e:
|
||||
logger.error(f"获取最近公告失败: {str(e)}")
|
||||
return []
|
||||
|
||||
def cleanup_expired_data(self, days: Optional[int] = None) -> int:
|
||||
"""
|
||||
清理过期数据
|
||||
|
||||
Args:
|
||||
days: 保留天数，如果为None则使用配置默认值
|
||||
|
||||
Returns:
|
||||
int: 清理的记录数
|
||||
"""
|
||||
from ..core.config_manager import get_config
|
||||
|
||||
config = get_config()
|
||||
if days is None:
|
||||
days = config.database.data_retention_days
|
||||
|
||||
logger.info(f"开始清理 {days} 天前的过期数据")
|
||||
|
||||
try:
|
||||
deleted_count = self.db_manager.cleanup_expired_data(days)
|
||||
|
||||
if deleted_count > 0:
|
||||
logger.info(f"成功清理 {deleted_count} 条过期数据")
|
||||
else:
|
||||
logger.info("没有找到需要清理的过期数据")
|
||||
|
||||
return deleted_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理过期数据失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取存储统计信息
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 统计数据
|
||||
"""
|
||||
try:
|
||||
stats = self.db_manager.get_statistics()
|
||||
stats.update({
|
||||
"storage_type": "postgresql",
|
||||
"last_cleanup": datetime.now().isoformat()
|
||||
})
|
||||
return stats
|
||||
except Exception as e:
|
||||
logger.error(f"获取存储统计信息失败: {str(e)}")
|
||||
return {
|
||||
"storage_type": "postgresql",
|
||||
"error": str(e),
|
||||
"total_announcements": 0,
|
||||
"last_cleanup": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
def search_announcements(self, keyword: Optional[str] = None,
|
||||
source_code: Optional[str] = None,
|
||||
start_date: Optional[datetime] = None,
|
||||
end_date: Optional[datetime] = None,
|
||||
limit: int = 50) -> List[Announcement]:
|
||||
"""
|
||||
搜索公告
|
||||
|
||||
Args:
|
||||
keyword: 关键词
|
||||
source_code: 来源代码
|
||||
start_date: 开始日期
|
||||
end_date: 结束日期
|
||||
limit: 限制数量
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 搜索结果
|
||||
"""
|
||||
# 这里可以实现更复杂的搜索逻辑
|
||||
# 目前使用现有的查询方法
|
||||
try:
|
||||
return self.db_manager.get_announcements(
|
||||
source_code=source_code,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
limit=limit
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"搜索公告失败: {str(e)}")
|
||||
return []
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
"""
|
||||
检查存储健康状态
|
||||
|
||||
Returns:
|
||||
bool: 是否健康
|
||||
"""
|
||||
try:
|
||||
# 尝试执行一个简单的查询
|
||||
stats = self.get_statistics()
|
||||
return "error" not in stats
|
||||
except Exception as e:
|
||||
logger.error(f"存储健康检查失败: {str(e)}")
|
||||
return False
|
||||
|
||||
def optimize_storage(self):
|
||||
"""优化存储性能"""
|
||||
# 这里可以添加数据库优化逻辑，如重建索引、清理碎片等
|
||||
logger.info("开始优化存储性能")
|
||||
|
||||
try:
|
||||
# 执行一些基本的优化操作
|
||||
# 注意：实际的优化命令取决于PostgreSQL版本和配置
|
||||
|
||||
# 这里可以添加具体的优化SQL
|
||||
# 例如：VACUUM, REINDEX等
|
||||
|
||||
logger.info("存储优化完成")
|
||||
except Exception as e:
|
||||
logger.error(f"存储优化失败: {str(e)}")
|
||||
|
||||
def backup_data(self, backup_path: Optional[str] = None) -> bool:
|
||||
"""
|
||||
备份数据
|
||||
|
||||
Args:
|
||||
backup_path: 备份文件路径
|
||||
|
||||
Returns:
|
||||
bool: 备份是否成功
|
||||
"""
|
||||
# 这里可以实现数据备份逻辑
|
||||
# 可以使用pg_dump或其他备份工具
|
||||
|
||||
logger.info("开始备份数据")
|
||||
|
||||
try:
|
||||
# 实现备份逻辑
|
||||
# 注意：这需要系统权限来执行pg_dump
|
||||
|
||||
logger.info("数据备份完成")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"数据备份失败: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
class StorageManager:
|
||||
"""存储管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self.postgresql = PostgreSQLStorage()
|
||||
self._current_storage = self.postgresql # 默认使用PostgreSQL
|
||||
|
||||
def save_announcements(self, announcements: List[Announcement]) -> int:
|
||||
"""保存筛选后的公告"""
|
||||
return self._current_storage.save_announcements(announcements)
|
||||
|
||||
def save_all_announcements_by_source(self, announcements: List[Announcement],
|
||||
max_per_source: int = 100) -> Dict[str, int]:
|
||||
"""按来源保存所有公告"""
|
||||
return self._current_storage.save_all_announcements_by_source(announcements, max_per_source)
|
||||
|
||||
def save_auto_announcements(self, announcements: List[Announcement]) -> int:
|
||||
"""保存定时搜索公告到专用表"""
|
||||
return self._current_storage.save_auto_announcements(announcements)
|
||||
|
||||
def save_manual_announcements_by_source(self, announcements: List[Announcement],
|
||||
max_per_source: int = 100) -> Dict[str, int]:
|
||||
"""按来源保存手动搜索公告到专用表"""
|
||||
return self._current_storage.save_manual_announcements_by_source(announcements, max_per_source)
|
||||
|
||||
def save_crawl_results(self, results: List[CrawlResult]) -> int:
|
||||
"""保存爬取结果"""
|
||||
return self._current_storage.save_crawl_results(results)
|
||||
|
||||
def get_recent_announcements(self, hours: int = 24, limit: int = 100) -> List[Announcement]:
|
||||
"""获取最近公告"""
|
||||
return self._current_storage.get_recent_announcements(hours, limit)
|
||||
|
||||
def cleanup_expired_data(self, days: Optional[int] = None) -> int:
|
||||
"""清理过期数据"""
|
||||
return self._current_storage.cleanup_expired_data(days)
|
||||
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""获取统计信息"""
|
||||
return self._current_storage.get_statistics()
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
"""检查健康状态"""
|
||||
return self._current_storage.is_healthy()
|
||||
|
||||
def optimize(self):
|
||||
"""优化存储"""
|
||||
self._current_storage.optimize_storage()
|
||||
|
||||
def backup(self, backup_path: Optional[str] = None) -> bool:
|
||||
"""备份数据"""
|
||||
return self._current_storage.backup_data(backup_path)
|
||||
|
||||
|
||||
# 全局存储管理器实例
|
||||
_storage_manager = None
|
||||
_storage_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_storage_manager() -> StorageManager:
|
||||
"""
|
||||
获取存储管理器实例
|
||||
|
||||
Returns:
|
||||
StorageManager: 存储管理器实例
|
||||
"""
|
||||
global _storage_manager
|
||||
if _storage_manager is None:
|
||||
with _storage_lock:
|
||||
if _storage_manager is None:
|
||||
_storage_manager = StorageManager()
|
||||
return _storage_manager
|
||||
|
||||
|
||||
def init_storage():
|
||||
"""初始化存储"""
|
||||
try:
|
||||
init_database()
|
||||
logger.info("存储初始化完成")
|
||||
except Exception as e:
|
||||
logger.error(f"存储初始化失败: {str(e)}")
|
||||
raise
|
||||
|
||||
|
||||
def save_announcements_to_storage(announcements: List[Announcement]) -> int:
|
||||
"""
|
||||
保存筛选后的公告到存储
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
int: 保存成功的数量
|
||||
"""
|
||||
return get_storage_manager().save_announcements(announcements)
|
||||
|
||||
|
||||
def save_all_announcements_by_source_to_storage(announcements: List[Announcement],
|
||||
max_per_source: int = 100) -> Dict[str, int]:
|
||||
"""
|
||||
按来源保存所有公告到存储
|
||||
|
||||
Args:
|
||||
announcements: 所有公告列表
|
||||
max_per_source: 每个来源最大保留数量
|
||||
|
||||
Returns:
|
||||
Dict[str, int]: 各来源保存的数量
|
||||
"""
|
||||
return get_storage_manager().save_all_announcements_by_source(announcements, max_per_source)
|
||||
|
||||
|
||||
def save_auto_announcements_to_storage(announcements: List[Announcement]) -> int:
|
||||
"""
|
||||
保存定时搜索公告到专用表
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
int: 保存成功的数量
|
||||
"""
|
||||
return get_storage_manager().save_auto_announcements(announcements)
|
||||
|
||||
|
||||
def save_manual_announcements_by_source_to_storage(announcements: List[Announcement],
|
||||
max_per_source: int = 100) -> Dict[str, int]:
|
||||
"""
|
||||
按来源保存手动搜索公告到专用表
|
||||
|
||||
Args:
|
||||
announcements: 所有公告列表
|
||||
max_per_source: 每个来源最大保留数量
|
||||
|
||||
Returns:
|
||||
Dict[str, int]: 各来源保存的数量
|
||||
"""
|
||||
return get_storage_manager().save_manual_announcements_by_source(announcements, max_per_source)
|
||||
|
||||
|
||||
def cleanup_storage(days: Optional[int] = None) -> int:
|
||||
"""
|
||||
清理存储中的过期数据
|
||||
|
||||
Args:
|
||||
days: 保留天数
|
||||
|
||||
Returns:
|
||||
int: 清理的记录数
|
||||
"""
|
||||
return get_storage_manager().cleanup_expired_data(days)
|
||||
@@ -1,283 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- encoding:utf-8 -*-
|
||||
|
||||
""" 对企业微信发送给企业后台的消息加解密示例代码.
|
||||
@copyright: Copyright (c) 1998-2014 Tencent Inc.
|
||||
|
||||
"""
|
||||
# ------------------------------------------------------------------------
|
||||
import logging
|
||||
import base64
|
||||
import random
|
||||
import hashlib
|
||||
import time
|
||||
import struct
|
||||
from Crypto.Cipher import AES
|
||||
import xml.etree.cElementTree as ET
|
||||
import socket
|
||||
|
||||
try:
|
||||
import ierror
|
||||
except ImportError:
|
||||
from . import ierror
|
||||
|
||||
|
||||
"""
|
||||
关于Crypto.Cipher模块，ImportError: No module named 'Crypto'解决方案
|
||||
请到官方网站 https://www.dlitz.net/software/pycrypto/ 下载pycrypto。
|
||||
下载后，按照README中的“Installation”小节的提示进行pycrypto安装。
|
||||
"""
|
||||
|
||||
|
||||
class FormatException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def throw_exception(message, exception_class=FormatException):
|
||||
"""my define raise exception function"""
|
||||
raise exception_class(message)
|
||||
|
||||
|
||||
class SHA1:
|
||||
"""计算企业微信的消息签名接口"""
|
||||
|
||||
def getSHA1(self, token, timestamp, nonce, encrypt):
|
||||
"""用SHA1算法生成安全签名
|
||||
@param token: 票据
|
||||
@param timestamp: 时间戳
|
||||
@param encrypt: 密文
|
||||
@param nonce: 随机字符串
|
||||
@return: 安全签名
|
||||
"""
|
||||
try:
|
||||
sortlist = [token, timestamp, nonce, encrypt]
|
||||
sortlist.sort()
|
||||
sha = hashlib.sha1()
|
||||
sha.update("".join(sortlist).encode())
|
||||
return ierror.WXBizMsgCrypt_OK, sha.hexdigest()
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return ierror.WXBizMsgCrypt_ComputeSignature_Error, None
|
||||
|
||||
|
||||
class XMLParse:
|
||||
"""提供提取消息格式中的密文及生成回复消息格式的接口"""
|
||||
|
||||
# xml消息模板
|
||||
AES_TEXT_RESPONSE_TEMPLATE = """<xml>
|
||||
<Encrypt><![CDATA[%(msg_encrypt)s]]></Encrypt>
|
||||
<MsgSignature><![CDATA[%(msg_signaturet)s]]></MsgSignature>
|
||||
<TimeStamp>%(timestamp)s</TimeStamp>
|
||||
<Nonce><![CDATA[%(nonce)s]]></Nonce>
|
||||
</xml>"""
|
||||
|
||||
def extract(self, xmltext):
|
||||
"""提取出xml数据包中的加密消息
|
||||
@param xmltext: 待提取的xml字符串
|
||||
@return: 提取出的加密消息字符串
|
||||
"""
|
||||
try:
|
||||
xml_tree = ET.fromstring(xmltext)
|
||||
encrypt = xml_tree.find("Encrypt")
|
||||
return ierror.WXBizMsgCrypt_OK, encrypt.text
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return ierror.WXBizMsgCrypt_ParseXml_Error, None
|
||||
|
||||
def generate(self, encrypt, signature, timestamp, nonce):
|
||||
"""生成xml消息
|
||||
@param encrypt: 加密后的消息密文
|
||||
@param signature: 安全签名
|
||||
@param timestamp: 时间戳
|
||||
@param nonce: 随机字符串
|
||||
@return: 生成的xml字符串
|
||||
"""
|
||||
resp_dict = {
|
||||
'msg_encrypt': encrypt,
|
||||
'msg_signaturet': signature,
|
||||
'timestamp': timestamp,
|
||||
'nonce': nonce,
|
||||
}
|
||||
resp_xml = self.AES_TEXT_RESPONSE_TEMPLATE % resp_dict
|
||||
return resp_xml
|
||||
|
||||
|
||||
class PKCS7Encoder():
|
||||
"""提供基于PKCS7算法的加解密接口"""
|
||||
|
||||
block_size = 32
|
||||
|
||||
def encode(self, text):
|
||||
""" 对需要加密的明文进行填充补位
|
||||
@param text: 需要进行填充补位操作的明文
|
||||
@return: 补齐明文字符串
|
||||
"""
|
||||
text_length = len(text)
|
||||
# 计算需要填充的位数
|
||||
amount_to_pad = self.block_size - (text_length % self.block_size)
|
||||
if amount_to_pad == 0:
|
||||
amount_to_pad = self.block_size
|
||||
# 获得补位所用的字符
|
||||
pad = chr(amount_to_pad)
|
||||
return text + (pad * amount_to_pad).encode()
|
||||
|
||||
def decode(self, decrypted):
|
||||
"""删除解密后明文的补位字符
|
||||
@param decrypted: 解密后的明文
|
||||
@return: 删除补位字符后的明文
|
||||
"""
|
||||
pad = ord(decrypted[-1])
|
||||
if pad < 1 or pad > 32:
|
||||
pad = 0
|
||||
return decrypted[:-pad]
|
||||
|
||||
|
||||
class Prpcrypt(object):
|
||||
"""提供接收和推送给企业微信消息的加解密接口"""
|
||||
|
||||
def __init__(self, key):
|
||||
|
||||
# self.key = base64.b64decode(key+"=")
|
||||
self.key = key
|
||||
# 设置加解密模式为AES的CBC模式
|
||||
self.mode = AES.MODE_CBC
|
||||
|
||||
def encrypt(self, text, receiveid):
|
||||
"""对明文进行加密
|
||||
@param text: 需要加密的明文
|
||||
@return: 加密得到的字符串
|
||||
"""
|
||||
# 16位随机字符串添加到明文开头
|
||||
text = text.encode()
|
||||
text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode()
|
||||
|
||||
# 使用自定义的填充方式对明文进行补位填充
|
||||
pkcs7 = PKCS7Encoder()
|
||||
text = pkcs7.encode(text)
|
||||
# 加密
|
||||
cryptor = AES.new(self.key, self.mode, self.key[:16])
|
||||
try:
|
||||
ciphertext = cryptor.encrypt(text)
|
||||
# 使用BASE64对加密后的字符串进行编码
|
||||
return ierror.WXBizMsgCrypt_OK, base64.b64encode(ciphertext)
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return ierror.WXBizMsgCrypt_EncryptAES_Error, None
|
||||
|
||||
def decrypt(self, text, receiveid):
|
||||
"""对解密后的明文进行补位删除
|
||||
@param text: 密文
|
||||
@return: 删除填充补位后的明文
|
||||
"""
|
||||
try:
|
||||
cryptor = AES.new(self.key, self.mode, self.key[:16])
|
||||
# 使用BASE64对密文进行解码，然后AES-CBC解密
|
||||
plain_text = cryptor.decrypt(base64.b64decode(text))
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return ierror.WXBizMsgCrypt_DecryptAES_Error, None
|
||||
try:
|
||||
pad = plain_text[-1]
|
||||
# 去掉补位字符串
|
||||
# pkcs7 = PKCS7Encoder()
|
||||
# plain_text = pkcs7.encode(plain_text)
|
||||
# 去除16位随机字符串
|
||||
content = plain_text[16:-pad]
|
||||
xml_len = socket.ntohl(struct.unpack("I", content[: 4])[0])
|
||||
xml_content = content[4: xml_len + 4]
|
||||
from_receiveid = content[xml_len + 4:]
|
||||
except Exception as e:
|
||||
logger = logging.getLogger()
|
||||
logger.error(e)
|
||||
return ierror.WXBizMsgCrypt_IllegalBuffer, None
|
||||
|
||||
if from_receiveid.decode('utf8') != receiveid:
|
||||
return ierror.WXBizMsgCrypt_ValidateCorpid_Error, None
|
||||
return 0, xml_content
|
||||
|
||||
def get_random_str(self):
|
||||
""" 随机生成16位字符串
|
||||
@return: 16位字符串
|
||||
"""
|
||||
return str(random.randint(1000000000000000, 9999999999999999)).encode()
|
||||
|
||||
|
||||
class WXBizMsgCrypt(object):
|
||||
# 构造函数
|
||||
def __init__(self, sToken, sEncodingAESKey, sReceiveId):
|
||||
try:
|
||||
self.key = base64.b64decode(sEncodingAESKey + "=")
|
||||
assert len(self.key) == 32
|
||||
except:
|
||||
throw_exception("[error]: EncodingAESKey unvalid !", FormatException)
|
||||
# return ierror.WXBizMsgCrypt_IllegalAesKey,None
|
||||
self.m_sToken = sToken
|
||||
self.m_sReceiveId = sReceiveId
|
||||
|
||||
# 验证URL
|
||||
# @param sMsgSignature: 签名串，对应URL参数的msg_signature
|
||||
# @param sTimeStamp: 时间戳，对应URL参数的timestamp
|
||||
# @param sNonce: 随机串，对应URL参数的nonce
|
||||
# @param sEchoStr: 随机串，对应URL参数的echostr
|
||||
# @param sReplyEchoStr: 解密之后的echostr，当return返回0时有效
|
||||
# @return：成功0，失败返回对应的错误码
|
||||
|
||||
def VerifyURL(self, sMsgSignature, sTimeStamp, sNonce, sEchoStr):
|
||||
sha1 = SHA1()
|
||||
ret, signature = sha1.getSHA1(self.m_sToken, sTimeStamp, sNonce, sEchoStr)
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
if not signature == sMsgSignature:
|
||||
return ierror.WXBizMsgCrypt_ValidateSignature_Error, None
|
||||
pc = Prpcrypt(self.key)
|
||||
ret, sReplyEchoStr = pc.decrypt(sEchoStr, self.m_sReceiveId)
|
||||
return ret, sReplyEchoStr
|
||||
|
||||
def EncryptMsg(self, sReplyMsg, sNonce, timestamp=None):
|
||||
# 将企业回复用户的消息加密打包
|
||||
# @param sReplyMsg: 企业号待回复用户的消息，xml格式的字符串
|
||||
# @param sTimeStamp: 时间戳，可以自己生成，也可以用URL参数的timestamp,如为None则自动用当前时间
|
||||
# @param sNonce: 随机串，可以自己生成，也可以用URL参数的nonce
|
||||
# sEncryptMsg: 加密后的可以直接回复用户的密文，包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串,
|
||||
# return：成功0，sEncryptMsg,失败返回对应的错误码None
|
||||
pc = Prpcrypt(self.key)
|
||||
ret, encrypt = pc.encrypt(sReplyMsg, self.m_sReceiveId)
|
||||
encrypt = encrypt.decode('utf8')
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
if timestamp is None:
|
||||
timestamp = str(int(time.time()))
|
||||
# 生成安全签名
|
||||
sha1 = SHA1()
|
||||
ret, signature = sha1.getSHA1(self.m_sToken, timestamp, sNonce, encrypt)
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
xmlParse = XMLParse()
|
||||
return ret, xmlParse.generate(encrypt, signature, timestamp, sNonce)
|
||||
|
||||
def DecryptMsg(self, sPostData, sMsgSignature, sTimeStamp, sNonce):
|
||||
# 检验消息的真实性，并且获取解密后的明文
|
||||
# @param sMsgSignature: 签名串，对应URL参数的msg_signature
|
||||
# @param sTimeStamp: 时间戳，对应URL参数的timestamp
|
||||
# @param sNonce: 随机串，对应URL参数的nonce
|
||||
# @param sPostData: 密文，对应POST请求的数据
|
||||
# xml_content: 解密后的原文，当return返回0时有效
|
||||
# @return: 成功0，失败返回对应的错误码
|
||||
# 验证安全签名
|
||||
xmlParse = XMLParse()
|
||||
ret, encrypt = xmlParse.extract(sPostData)
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
sha1 = SHA1()
|
||||
ret, signature = sha1.getSHA1(self.m_sToken, sTimeStamp, sNonce, encrypt)
|
||||
if ret != 0:
|
||||
return ret, None
|
||||
if not signature == sMsgSignature:
|
||||
return ierror.WXBizMsgCrypt_ValidateSignature_Error, None
|
||||
pc = Prpcrypt(self.key)
|
||||
ret, xml_content = pc.decrypt(encrypt, self.m_sReceiveId)
|
||||
return ret, xml_content
|
||||
@@ -1,10 +0,0 @@
|
||||
"""
|
||||
企业微信交互模块
|
||||
提供企业微信回调服务器、消息处理、菜单管理等功能
|
||||
"""
|
||||
|
||||
from .callback_server import WeChatCallbackServer
|
||||
from .message_handler import WeChatMessageHandler
|
||||
from .menu_manager import WeChatMenuManager
|
||||
|
||||
__all__ = ['WeChatCallbackServer', 'WeChatMessageHandler', 'WeChatMenuManager']
|
||||
@@ -1,275 +0,0 @@
|
||||
"""
|
||||
企业微信回调服务器
|
||||
使用Flask实现企业微信回调消息的接收和处理
|
||||
"""
|
||||
|
||||
import time
|
||||
import xml.etree.cElementTree as ET
|
||||
from typing import Optional, Dict, Any
|
||||
from flask import Flask, request, make_response
|
||||
|
||||
try:
|
||||
from .WXBizMsgCrypt import WXBizMsgCrypt, FormatException
|
||||
from .ierror import WXBizMsgCrypt_OK
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger
|
||||
from .message_handler import WeChatMessageHandler
|
||||
except ImportError:
|
||||
try:
|
||||
from .WXBizMsgCrypt import WXBizMsgCrypt, FormatException
|
||||
from .ierror import WXBizMsgCrypt_OK
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger
|
||||
from .message_handler import WeChatMessageHandler
|
||||
except ImportError as e:
|
||||
raise ImportError(f"企业微信模块导入失败: {e}")
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class WeChatCallbackServer:
|
||||
"""企业微信回调服务器"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config().wechat_app
|
||||
self.app = Flask(__name__)
|
||||
self.message_handler = WeChatMessageHandler()
|
||||
|
||||
# 初始化企业微信消息加解密器
|
||||
self.wxcpt = WXBizMsgCrypt(
|
||||
sToken=self.config.token,
|
||||
sEncodingAESKey=self.config.encoding_aes_key,
|
||||
sReceiveId=self.config.corp_id
|
||||
)
|
||||
|
||||
# 设置路由
|
||||
self._setup_routes()
|
||||
|
||||
logger.info("企业微信回调服务器初始化完成")
|
||||
|
||||
def _setup_routes(self):
|
||||
"""设置路由"""
|
||||
|
||||
@self.app.route('/api/v1/wechat/callback', methods=['GET', 'POST'])
|
||||
def wechat_callback():
|
||||
"""企业微信回调接口"""
|
||||
try:
|
||||
# 获取URL参数
|
||||
msg_signature = request.args.get('msg_signature', '')
|
||||
timestamp = request.args.get('timestamp', '')
|
||||
nonce = request.args.get('nonce', '')
|
||||
|
||||
logger.debug(f"收到企业微信回调请求: method={request.method}")
|
||||
|
||||
if request.method == 'GET':
|
||||
# URL验证
|
||||
return self._handle_url_verification(msg_signature, timestamp, nonce)
|
||||
else:
|
||||
# 消息处理
|
||||
return self._handle_message(msg_signature, timestamp, nonce)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"企业微信回调处理异常: {str(e)}")
|
||||
return make_response("success", 200)
|
||||
|
||||
def _handle_url_verification(self, msg_signature: str, timestamp: str, nonce: str):
|
||||
"""处理URL验证"""
|
||||
try:
|
||||
echostr = request.args.get('echostr', '')
|
||||
|
||||
logger.info("处理企业微信URL验证请求")
|
||||
|
||||
# 验证URL并解密echostr
|
||||
ret, sEchoStr = self.wxcpt.VerifyURL(msg_signature, timestamp, nonce, echostr)
|
||||
|
||||
if ret == WXBizMsgCrypt_OK:
|
||||
logger.info("企业微信URL验证成功")
|
||||
return make_response(sEchoStr.decode('utf-8') if isinstance(sEchoStr, bytes) else sEchoStr)
|
||||
else:
|
||||
logger.error(f"企业微信URL验证失败: {ret}")
|
||||
return make_response("verification failed", 403)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"URL验证异常: {str(e)}")
|
||||
return make_response("verification error", 500)
|
||||
|
||||
def _handle_message(self, msg_signature: str, timestamp: str, nonce: str):
|
||||
"""处理消息"""
|
||||
try:
|
||||
# 获取POST数据 - 企业微信发送的是XML格式
|
||||
post_data = request.get_data(as_text=True)
|
||||
|
||||
logger.debug(f"收到企业微信POST数据: {post_data[:200]}...")
|
||||
|
||||
# 记录详细的调试信息
|
||||
logger.debug(f"msg_signature: {msg_signature}")
|
||||
logger.debug(f"timestamp: {timestamp}")
|
||||
logger.debug(f"nonce: {nonce}")
|
||||
|
||||
# 手动验证签名过程
|
||||
try:
|
||||
from .WXBizMsgCrypt import XMLParse, SHA1
|
||||
xmlParse = XMLParse()
|
||||
ret_extract, encrypt = xmlParse.extract(post_data)
|
||||
if ret_extract == 0:
|
||||
logger.error(f"✅ XML解析成功，提取的encrypt长度: {len(encrypt)}")
|
||||
logger.error(f"提取的encrypt前50字符: {encrypt[:50]}...")
|
||||
sha1 = SHA1()
|
||||
ret_sha1, calculated_signature = sha1.getSHA1(self.config.token, timestamp, nonce, encrypt)
|
||||
if ret_sha1 == 0:
|
||||
logger.error(f"计算的签名: {calculated_signature}")
|
||||
logger.error(f"接收的签名: {msg_signature}")
|
||||
logger.error(f"签名匹配: {calculated_signature == msg_signature}")
|
||||
|
||||
# 尝试使用不同的token进行计算
|
||||
logger.error("尝试使用默认token计算签名...")
|
||||
default_token = "DmvL98cAF6x9CFtQZwqD2emGL8S7HxA"
|
||||
if self.config.token != default_token:
|
||||
ret_test, test_signature = sha1.getSHA1(default_token, timestamp, nonce, encrypt)
|
||||
if ret_test == 0:
|
||||
logger.error(f"默认token计算签名: {test_signature}")
|
||||
logger.error(f"与接收签名匹配: {test_signature == msg_signature}")
|
||||
else:
|
||||
logger.error(f"SHA1计算失败: {ret_sha1}")
|
||||
else:
|
||||
logger.error(f"❌ XML解析失败: {ret_extract}")
|
||||
logger.error("可能的原因：")
|
||||
logger.error("1. POST数据格式不正确")
|
||||
logger.error("2. 缺少Encrypt字段")
|
||||
logger.error("3. XML格式错误")
|
||||
except Exception as e:
|
||||
logger.error(f"签名验证调试异常: {str(e)}")
|
||||
import traceback
|
||||
logger.error(f"详细异常信息: {traceback.format_exc()}")
|
||||
|
||||
# 解密消息
|
||||
ret, xml_content = self.wxcpt.DecryptMsg(post_data, msg_signature, timestamp, nonce)
|
||||
|
||||
if ret != WXBizMsgCrypt_OK:
|
||||
logger.error(f"消息解密失败: {ret}")
|
||||
# 记录更多调试信息
|
||||
logger.error(f"POST数据长度: {len(post_data)}")
|
||||
logger.error(f"POST数据内容: {post_data}")
|
||||
logger.error("💡 可能的原因:")
|
||||
logger.error("1. config.yaml中的token不正确（应为43位）")
|
||||
logger.error("2. config.yaml中的encoding_aes_key不正确")
|
||||
logger.error("3. 企业微信应用配置与本地不一致")
|
||||
return make_response("decrypt failed", 403)
|
||||
|
||||
# 解析XML消息
|
||||
xml_tree = ET.fromstring(xml_content)
|
||||
msg_type = xml_tree.find('MsgType').text
|
||||
|
||||
logger.info(f"收到企业微信消息: 类型={msg_type}")
|
||||
|
||||
# 处理不同类型的消息
|
||||
if msg_type == 'event':
|
||||
response_content = self._handle_event(xml_tree)
|
||||
elif msg_type == 'text':
|
||||
response_content = self._handle_text_message(xml_tree)
|
||||
else:
|
||||
response_content = self._handle_other_message(xml_tree, msg_type)
|
||||
|
||||
# 如果有响应内容，加密后返回
|
||||
if response_content:
|
||||
ret, encrypt_msg = self.wxcpt.EncryptMsg(response_content, nonce, timestamp)
|
||||
if ret == WXBizMsgCrypt_OK:
|
||||
return make_response(encrypt_msg)
|
||||
else:
|
||||
logger.error(f"消息加密失败: {ret}")
|
||||
|
||||
# 返回成功响应
|
||||
return make_response("success", 200)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"消息处理异常: {str(e)}")
|
||||
return make_response("success", 200)
|
||||
|
||||
def _handle_event(self, xml_tree) -> Optional[str]:
|
||||
"""处理事件消息"""
|
||||
try:
|
||||
event = xml_tree.find('Event').text
|
||||
event_key = xml_tree.find('EventKey')
|
||||
event_key = event_key.text if event_key is not None else None
|
||||
from_user = xml_tree.find('FromUserName').text
|
||||
|
||||
logger.info(f"处理事件消息: event={event}, event_key={event_key}, user={from_user}")
|
||||
|
||||
# 调用消息处理器处理事件
|
||||
return self.message_handler.handle_event(event, event_key, from_user)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"事件处理异常: {str(e)}")
|
||||
return None
|
||||
|
||||
def _handle_text_message(self, xml_tree) -> Optional[str]:
|
||||
"""处理文本消息"""
|
||||
try:
|
||||
content = xml_tree.find('Content').text
|
||||
from_user = xml_tree.find('FromUserName').text
|
||||
|
||||
logger.info(f"处理文本消息: content={content[:50]}..., user={from_user}")
|
||||
|
||||
# 调用消息处理器处理文本消息
|
||||
return self.message_handler.handle_text_message(content, from_user)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"文本消息处理异常: {str(e)}")
|
||||
return None
|
||||
|
||||
def _handle_other_message(self, xml_tree, msg_type: str) -> Optional[str]:
|
||||
"""处理其他类型的消息"""
|
||||
try:
|
||||
from_user = xml_tree.find('FromUserName').text
|
||||
logger.info(f"收到其他类型消息: type={msg_type}, user={from_user}")
|
||||
|
||||
# 调用消息处理器处理其他消息
|
||||
return self.message_handler.handle_other_message(msg_type, from_user)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"其他消息处理异常: {str(e)}")
|
||||
return None
|
||||
|
||||
def run(self, host: str = '0.0.0.0', port: int = 18001, debug: bool = False):
|
||||
"""启动服务器"""
|
||||
logger.info(f"启动企业微信回调服务器: {host}:{port}")
|
||||
self.app.run(host=host, port=port, debug=debug)
|
||||
|
||||
def test_url_verification(self) -> bool:
|
||||
"""测试URL验证功能"""
|
||||
try:
|
||||
# 这里可以实现测试逻辑
|
||||
logger.info("企业微信URL验证测试通过")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"URL验证测试失败: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
# 全局回调服务器实例
|
||||
_callback_server = None
|
||||
|
||||
|
||||
def get_callback_server() -> WeChatCallbackServer:
|
||||
"""获取回调服务器实例"""
|
||||
global _callback_server
|
||||
if _callback_server is None:
|
||||
_callback_server = WeChatCallbackServer()
|
||||
return _callback_server
|
||||
|
||||
|
||||
def create_callback_app() -> Flask:
|
||||
"""创建回调应用（用于外部集成）"""
|
||||
# 确保配置已加载（用于uWSGI等部署环境）
|
||||
from ..core.config_manager import load_config, get_config
|
||||
|
||||
# 检查配置是否已加载
|
||||
try:
|
||||
config = get_config()
|
||||
except RuntimeError:
|
||||
# 配置未加载，尝试加载默认配置
|
||||
load_config()
|
||||
|
||||
server = get_callback_server()
|
||||
return server.app
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#########################################################################
|
||||
# Author: jonyqin
|
||||
# Created Time: Thu 11 Sep 2014 01:53:58 PM CST
|
||||
# File Name: ierror.py
|
||||
# Description:定义错误码含义
|
||||
#########################################################################
|
||||
WXBizMsgCrypt_OK = 0
|
||||
WXBizMsgCrypt_ValidateSignature_Error = -40001
|
||||
WXBizMsgCrypt_ParseXml_Error = -40002
|
||||
WXBizMsgCrypt_ComputeSignature_Error = -40003
|
||||
WXBizMsgCrypt_IllegalAesKey = -40004
|
||||
WXBizMsgCrypt_ValidateCorpid_Error = -40005
|
||||
WXBizMsgCrypt_EncryptAES_Error = -40006
|
||||
WXBizMsgCrypt_DecryptAES_Error = -40007
|
||||
WXBizMsgCrypt_IllegalBuffer = -40008
|
||||
WXBizMsgCrypt_EncodeBase64_Error = -40009
|
||||
WXBizMsgCrypt_DecodeBase64_Error = -40010
|
||||
WXBizMsgCrypt_GenReturnXml_Error = -40011
|
||||
@@ -1,330 +0,0 @@
|
||||
"""
|
||||
企业微信菜单管理器
|
||||
负责创建和管理企业微信应用菜单
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
try:
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger
|
||||
from ..notification.wechat import WeChatService
|
||||
except ImportError:
|
||||
try:
|
||||
from core.config_manager import get_config
|
||||
from core.logger import get_logger
|
||||
from notification.wechat import WeChatService
|
||||
except ImportError as e:
|
||||
raise ImportError(f"菜单管理器导入失败: {e}")
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class WeChatMenuManager:
|
||||
"""企业微信菜单管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config().wechat_app
|
||||
self.wechat_service = WeChatService()
|
||||
|
||||
# 菜单配置
|
||||
self.menu_data = {
|
||||
"button": [
|
||||
{
|
||||
"name": "监控操作",
|
||||
"sub_button": [
|
||||
{
|
||||
"type": "click",
|
||||
"name": "立即搜索",
|
||||
"key": "crawl_now"
|
||||
},
|
||||
{
|
||||
"type": "click",
|
||||
"name": "今日统计",
|
||||
"key": "today_stats"
|
||||
},
|
||||
{
|
||||
"type": "click",
|
||||
"name": "关键词搜索",
|
||||
"key": "keyword_search"
|
||||
},
|
||||
{
|
||||
"type": "click",
|
||||
"name": "最新公告",
|
||||
"key": "latest_news"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "系统管理",
|
||||
"sub_button": [
|
||||
{
|
||||
"type": "click",
|
||||
"name": "关键词管理",
|
||||
"key": "keyword_manage"
|
||||
},
|
||||
{
|
||||
"type": "click",
|
||||
"name": "系统状态",
|
||||
"key": "system_status"
|
||||
},
|
||||
{
|
||||
"type": "click",
|
||||
"name": "清理缓存",
|
||||
"key": "clear_cache"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "帮助",
|
||||
"sub_button": [
|
||||
{
|
||||
"type": "click",
|
||||
"name": "使用说明",
|
||||
"key": "help_guide"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
logger.info("企业微信菜单管理器初始化完成")
|
||||
|
||||
def create_menu(self) -> bool:
|
||||
"""
|
||||
创建菜单
|
||||
|
||||
Returns:
|
||||
bool: 创建是否成功
|
||||
"""
|
||||
try:
|
||||
logger.info("开始创建企业微信菜单")
|
||||
|
||||
# 获取访问令牌
|
||||
access_token = self.wechat_service._get_access_token()
|
||||
if not access_token:
|
||||
logger.error("获取访问令牌失败，无法创建菜单")
|
||||
return False
|
||||
|
||||
# 构建请求URL
|
||||
if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'):
|
||||
url = f"{self.config.proxy_api_url}/cgi-bin/menu/create"
|
||||
else:
|
||||
url = "https://qyapi.weixin.qq.com/cgi-bin/menu/create"
|
||||
|
||||
params = {
|
||||
"access_token": access_token,
|
||||
"agentid": self.config.agent_id
|
||||
}
|
||||
|
||||
# 发送创建菜单请求
|
||||
response = requests.post(url, params=params, json=self.menu_data, timeout=30)
|
||||
result = response.json()
|
||||
|
||||
if result.get("errcode") == 0:
|
||||
logger.info("企业微信菜单创建成功")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"企业微信菜单创建失败: {result}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"创建菜单异常: {str(e)}")
|
||||
return False
|
||||
|
||||
def delete_menu(self) -> bool:
|
||||
"""
|
||||
删除菜单
|
||||
|
||||
Returns:
|
||||
bool: 删除是否成功
|
||||
"""
|
||||
try:
|
||||
logger.info("开始删除企业微信菜单")
|
||||
|
||||
# 获取访问令牌
|
||||
access_token = self.wechat_service._get_access_token()
|
||||
if not access_token:
|
||||
logger.error("获取访问令牌失败，无法删除菜单")
|
||||
return False
|
||||
|
||||
# 构建请求URL
|
||||
if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'):
|
||||
url = f"{self.config.proxy_api_url}/cgi-bin/menu/delete"
|
||||
else:
|
||||
url = "https://qyapi.weixin.qq.com/cgi-bin/menu/delete"
|
||||
|
||||
params = {
|
||||
"access_token": access_token,
|
||||
"agentid": self.config.agent_id
|
||||
}
|
||||
|
||||
# 发送删除菜单请求
|
||||
response = requests.get(url, params=params, timeout=30)
|
||||
result = response.json()
|
||||
|
||||
if result.get("errcode") == 0:
|
||||
logger.info("企业微信菜单删除成功")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"企业微信菜单删除失败: {result}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"删除菜单异常: {str(e)}")
|
||||
return False
|
||||
|
||||
def get_menu(self) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
获取当前菜单
|
||||
|
||||
Returns:
|
||||
Optional[Dict[str, Any]]: 菜单信息，失败返回None
|
||||
"""
|
||||
try:
|
||||
logger.info("开始获取企业微信菜单")
|
||||
|
||||
# 获取访问令牌
|
||||
access_token = self.wechat_service._get_access_token()
|
||||
if not access_token:
|
||||
logger.error("获取访问令牌失败，无法获取菜单")
|
||||
return None
|
||||
|
||||
# 构建请求URL
|
||||
if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'):
|
||||
url = f"{self.config.proxy_api_url}/cgi-bin/menu/get"
|
||||
else:
|
||||
url = "https://qyapi.weixin.qq.com/cgi-bin/menu/get"
|
||||
|
||||
params = {
|
||||
"access_token": access_token,
|
||||
"agentid": self.config.agent_id
|
||||
}
|
||||
|
||||
# 发送获取菜单请求
|
||||
response = requests.get(url, params=params, timeout=30)
|
||||
result = response.json()
|
||||
|
||||
if result.get("errcode") == 0:
|
||||
logger.info("企业微信菜单获取成功")
|
||||
return result
|
||||
else:
|
||||
logger.error(f"企业微信菜单获取失败: {result}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取菜单异常: {str(e)}")
|
||||
return None
|
||||
|
||||
def update_menu(self, menu_data: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
更新菜单
|
||||
|
||||
Args:
|
||||
menu_data: 新的菜单数据
|
||||
|
||||
Returns:
|
||||
bool: 更新是否成功
|
||||
"""
|
||||
try:
|
||||
logger.info("开始更新企业微信菜单")
|
||||
|
||||
# 先删除旧菜单
|
||||
if not self.delete_menu():
|
||||
logger.warning("删除旧菜单失败，继续创建新菜单")
|
||||
|
||||
# 更新菜单配置
|
||||
self.menu_data = menu_data
|
||||
|
||||
# 创建新菜单
|
||||
return self.create_menu()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"更新菜单异常: {str(e)}")
|
||||
return False
|
||||
|
||||
def get_menu_info(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取菜单信息（用于调试）
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 菜单信息
|
||||
"""
|
||||
return {
|
||||
"menu_data": self.menu_data,
|
||||
"menu_structure": self._analyze_menu_structure()
|
||||
}
|
||||
|
||||
def _analyze_menu_structure(self) -> Dict[str, Any]:
|
||||
"""分析菜单结构"""
|
||||
try:
|
||||
buttons = self.menu_data.get("button", [])
|
||||
structure = {
|
||||
"total_buttons": len(buttons),
|
||||
"buttons": []
|
||||
}
|
||||
|
||||
for i, button in enumerate(buttons):
|
||||
button_info = {
|
||||
"index": i,
|
||||
"name": button.get("name", ""),
|
||||
"type": button.get("type", "menu"),
|
||||
}
|
||||
|
||||
if "sub_button" in button:
|
||||
button_info["sub_buttons"] = len(button["sub_button"])
|
||||
button_info["sub_button_list"] = [
|
||||
{
|
||||
"name": sub.get("name", ""),
|
||||
"type": sub.get("type", ""),
|
||||
"key": sub.get("key", "")
|
||||
}
|
||||
for sub in button["sub_button"]
|
||||
]
|
||||
else:
|
||||
button_info["key"] = button.get("key", "")
|
||||
|
||||
structure["buttons"].append(button_info)
|
||||
|
||||
return structure
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"分析菜单结构异常: {str(e)}")
|
||||
return {"error": str(e)}
|
||||
|
||||
def test_menu_operations(self) -> Dict[str, bool]:
|
||||
"""
|
||||
测试菜单操作
|
||||
|
||||
Returns:
|
||||
Dict[str, bool]: 测试结果
|
||||
"""
|
||||
results = {
|
||||
"create_menu": False,
|
||||
"get_menu": False,
|
||||
"delete_menu": False
|
||||
}
|
||||
|
||||
try:
|
||||
# 测试获取菜单
|
||||
menu_info = self.get_menu()
|
||||
results["get_menu"] = menu_info is not None
|
||||
|
||||
# 测试创建菜单（如果没有菜单的话）
|
||||
if not menu_info:
|
||||
results["create_menu"] = self.create_menu()
|
||||
else:
|
||||
results["create_menu"] = True # 已经有菜单了
|
||||
|
||||
# 不测试删除，避免影响现有菜单
|
||||
results["delete_menu"] = True
|
||||
|
||||
logger.info(f"菜单操作测试完成: {results}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"菜单操作测试异常: {str(e)}")
|
||||
|
||||
return results
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user