commit 28c57a040a24129411d81ce6759f9bf853d83175 Author: v6ole Date: Wed Jan 7 17:37:09 2026 +0800 手动模式 diff --git a/gx_gp_monitor/README.md b/gx_gp_monitor/README.md new file mode 100644 index 0000000..afe146d --- /dev/null +++ b/gx_gp_monitor/README.md @@ -0,0 +1,281 @@ +# 广西政府采购网公告监控系统 + +广西政府采购网公告爬取和监控的智能系统，支持多种公告类型的自动爬取、智能筛选、数据存储和通知推送。 + +## 功能特性 + +### 🕷️ 智能爬虫监控 +- 自动爬取广西政府采购网(zfcg.gxzf.gov.cn)的各类采购公告 +- 支持10+种公告类型：采购公告、结果公告、合同公告、更正公告、招标文件预公示、单一来源公示、电子卖场公示、履约验收公示、工程类公告、框架协议征集公告等 +- 智能反爬虫机制：随机User-Agent、代理轮换、请求延迟控制 + +### 🎯 多维度智能筛选 +- **关键词过滤**：支持多关键词精确匹配和模糊匹配 +- **日期范围筛选**：支持开始日期和结束日期过滤 +- **自动去重**：基于内容哈希的智能去重机制 +- **来源筛选**：按公告类型进行筛选 + +### 💾 数据持久化存储 +- **PostgreSQL数据库**：高效的数据存储和查询 +- **自动清理**：默认90天数据保留，可配置 +- **性能优化**：连接池、索引优化、批量操作 + +### 🔔 高可用性保障 +- **重试机制**：指数退避重试策略 +- **超时控制**：连接超时和读取超时设置 +- **幂等操作**：防止重复处理 +- **异常处理**：完善的异常捕获和恢复 +- **结构化日志**：详细的运行日志记录 + +### 📱 通知系统 +- **企业微信通知**：支持文本和Markdown格式消息 +- **实时推送**：新公告发现即时通知 +- **系统告警**：异常情况自动告警 + +### 📊 Markdown输出 +- **自动生成**：将公告输出为Markdown格式文件（onu.md） +- **结构化展示**：按来源分组、时间排序 +- **今日高亮**：突出显示今日发布的公告 + +## 安装和使用 + +### 环境要求 +- Python 3.8+ +- PostgreSQL 12+ + +### 安装步骤 + +1. **克隆项目** +```bash +cd /path/to/your/workspace +# 项目已创建在 gx_gp_monitor 目录下 +``` + +2. **创建虚拟环境** +```bash +python3 -m venv venv +source venv/bin/activate # Linux/Mac +# 或 +venv\Scripts\activate # Windows +``` + +3. **安装依赖** +```bash +pip install -r requirements.txt +``` + +4. **配置数据库** +```bash +# 创建PostgreSQL数据库 +createdb gx-gp-notify + +# 修改配置文件 config/config.yaml 中的数据库连接信息 +``` + +5. **修改配置** +```bash +# 编辑 config/config.yaml 文件 +# 设置数据库连接、企业微信配置、关键词等 +``` + +### 使用方法 + +#### 命令行使用 + +```bash +# 查看帮助 +python main.py --help + +# 执行一次爬取 +python main.py crawl + +# 带参数爬取 +python main.py crawl --keywords "大化" "信息化" --max-pages 5 + +# 启动调度器（定时任务） +python main.py scheduler + +# 数据清理 +python main.py cleanup --days 30 + +# 查看系统状态 +python main.py status +``` + +#### 编程接口使用 + +```python +from gx_gp_monitor.crawler.spider import crawl_announcements +from gx_gp_monitor.filters.filters import filter_from_config +from gx_gp_monitor.storage.postgresql import save_announcements_to_storage +from gx_gp_monitor.storage.md_generator import generate_onu_md +from gx_gp_monitor.notification.wechat import send_announcements_notification + +# 执行爬取 +results = crawl_announcements() + +# 筛选公告 +filter_obj = filter_from_config() +filtered_announcements, stats = filter_obj.filter(all_announcements) + +# 保存到数据库 +saved_count = save_announcements_to_storage(filtered_announcements) + +# 生成Markdown文件 +generate_onu_md(filtered_announcements) + +# 发送通知 +send_announcements_notification(filtered_announcements) +``` + +## 配置说明 + +### 主要配置文件：`config/config.yaml` + +```yaml +# 调试模式 +debug: false + +# 日志配置 +log_level: INFO +log_file: logs/gx_gp_monitor.log + +# 爬虫配置 +crawler: + base_url: "https://zfcg.gxzf.gov.cn" + timeout: 30 + max_retries: 3 + keyword: ["大化", "信息化"] # 关键词筛选 + max_pages: 10 + +# 数据库配置 +database: + enabled: true + host: "localhost" + port: 5432 + name: "gx-gp-notify" + user: "your_user" + password: "your_password" + data_retention_days: 90 + +# 企业微信配置 +wechat_app: + enabled: true + corp_id: "your_corp_id" + agent_id: "your_agent_id" + secret: "your_secret" + token: "your_token" + encoding_aes_key: "your_aes_key" + +# 调度配置 +scheduler: + enabled: true + jobs: + - name: "daily_crawl" + cron: "0 8,14,18 * * *" + enabled: true +``` + +### 公告来源配置 + +系统支持以下公告类型： + +- `ZcyAnnouncement1`: 采购公告 +- `ZcyAnnouncement2`: 结果公告 +- `ZcyAnnouncement3`: 合同公告 +- `ZcyAnnouncement4`: 更正公告 +- `ZcyAnnouncement5`: 招标文件预公示 +- `ZcyAnnouncement6`: 单一来源公示 +- `ZcyAnnouncement7`: 电子卖场公示 +- `ZcyAnnouncement10`: 履约验收公示 +- `ZcyAnnouncement11`: 工程类公告 +- `ZcyAnnouncement20`: 框架协议征集公告 +- `ZcyAnnouncement21`: 框架协议入围结果公告 +- `ZcyAnnouncement23`: 框架协议成交结果汇总公告 +- `61-266648`: 采购意向公开 + +## 架构设计 + +``` +gx_gp_monitor/ +├── config/ # 配置管理 +├── core/ # 核心模块 +│ ├── models.py # 数据模型 +│ ├── config_manager.py # 配置管理 +│ ├── database.py # 数据库操作 +│ ├── logger.py # 日志管理 +│ └── reliability.py # 高可用性 +├── crawler/ # 爬虫模块 +│ ├── spider.py # 爬虫核心 +│ └── parsers.py # 数据解析 +├── filters/ # 筛选模块 +├── storage/ # 存储模块 +│ ├── postgresql.py # PostgreSQL存储 +│ └── md_generator.py # Markdown生成 +├── notification/ # 通知模块 +├── scheduler/ # 调度模块 +└── main.py # 主程序入口 +``` + +## 日志和监控 + +### 日志文件 +- 默认日志文件：`logs/gx_gp_monitor.log` +- 日志轮转：10MB大小限制，保留5个备份文件 +- 日志级别：DEBUG、INFO、WARNING、ERROR、CRITICAL + +### 监控指标 +- 爬取统计：成功/失败次数、响应时间 +- 数据统计：公告数量、新增数量、去重统计 +- 系统状态：内存使用、磁盘空间、健康检查 + +## 故障排除 + +### 常见问题 + +1. **数据库连接失败** + - 检查PostgreSQL服务是否运行 + - 验证数据库连接配置 + - 确认用户权限 + +2. **爬虫请求失败** + - 检查网络连接 + - 验证目标网站是否可访问 + - 调整请求间隔和重试策略 + +3. **企业微信通知失败** + - 检查企业微信配置 + - 验证应用ID和密钥 + - 确认网络能访问企业微信API + +4. **权限问题** + - 确保日志目录和输出文件目录有写入权限 + - 检查数据库用户权限 + +### 调试模式 + +启用调试模式获取更详细的日志： + +```yaml +debug: true +log_level: DEBUG +``` + +## 许可证 + +本项目采用 MIT 许可证。 + +## 贡献 + +欢迎提交Issue和Pull Request来改进这个项目。 + +## 版本历史 + +- **v1.0.0** (2024-01-XX) + - 初始版本发布 + - 支持10+种公告类型爬取 + - 实现智能筛选和去重 + - PostgreSQL数据存储 + - 企业微信通知 + - Markdown文件输出 + - 定时任务调度 diff --git a/gx_gp_monitor/__init__.py b/gx_gp_monitor/__init__.py new file mode 100644 index 0000000..72e54cf --- /dev/null +++ b/gx_gp_monitor/__init__.py @@ -0,0 +1,7 @@ +""" +广西政府采购网公告监控系统 +广西政府采购网公告爬取和监控的智能系统 +""" + +__version__ = "1.0.0" +__author__ = "GX GP Monitor Team" diff --git a/gx_gp_monitor/__main__.py b/gx_gp_monitor/__main__.py new file mode 100644 index 0000000..142bcf8 --- /dev/null +++ b/gx_gp_monitor/__main__.py @@ -0,0 +1,8 @@ +""" +包的main入口，使项目可以直接通过 python -m gx_gp_monitor 运行 +""" + +from .main import main + +if __name__ == "__main__": + main() diff --git a/gx_gp_monitor/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..13702db Binary files /dev/null and b/gx_gp_monitor/__pycache__/__init__.cpython-313.pyc differ diff --git a/gx_gp_monitor/__pycache__/__main__.cpython-313.pyc b/gx_gp_monitor/__pycache__/__main__.cpython-313.pyc new file mode 100644 index 0000000..ffa7281 Binary files /dev/null and b/gx_gp_monitor/__pycache__/__main__.cpython-313.pyc differ diff --git a/gx_gp_monitor/__pycache__/main.cpython-313.pyc b/gx_gp_monitor/__pycache__/main.cpython-313.pyc new file mode 100644 index 0000000..fc59984 Binary files /dev/null and b/gx_gp_monitor/__pycache__/main.cpython-313.pyc differ diff --git a/gx_gp_monitor/config/__init__.py b/gx_gp_monitor/config/__init__.py new file mode 100644 index 0000000..b062df6 --- /dev/null +++ b/gx_gp_monitor/config/__init__.py @@ -0,0 +1 @@ +"""配置管理模块""" diff --git a/gx_gp_monitor/config/config.yaml b/gx_gp_monitor/config/config.yaml new file mode 100644 index 0000000..550edbb --- /dev/null +++ b/gx_gp_monitor/config/config.yaml @@ -0,0 +1,135 @@ +# 广西政府采购网公告监控系统配置文件 +# 复制此文件为 config.yaml 并修改相应配置 + +# 调试模式 +debug: false + +# 日志配置 +log_level: INFO +log_file: logs/gx_gp_monitor.log +log_max_size: 10485760 # 10MB +log_backup_count: 5 + +# 爬虫配置 +crawler: + base_url: "https://zfcg.gxzf.gov.cn" + timeout: 30 # 请求超时时间（秒） + max_retries: 3 # 最大重试次数 + retry_delay: 1.0 # 重试初始延迟 + max_retry_delay: 60.0 # 重试最大延迟 + backoff_factor: 2.0 # 退避因子 + user_agents: # User-Agent列表 + - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" + - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + proxies: [] # 代理列表 + request_delay: 1.0 # 请求间延迟 + request_delay_max: 3.0 # 请求间最大延迟 + keyword: ["大化", "信息化"] # 关键词筛选（支持多个关键词） + start_date: "" # 开始日期 (YYYY-MM-DD) + end_date: "" # 结束日期 (YYYY-MM-DD) + max_pages: 10 # 最大页数 + page_size: 100 # 每页大小 + +# 数据库配置 +database: + enabled: true # 是否启用数据库存储 + type: postgresql # 数据库类型 + host: "10.10.10.14" # 数据库主机 + port: 5432 # 数据库端口 + name: "gx-gp-notify" # 数据库名称 + user: "gx-gp-notify" # 数据库用户名 + password: "MA6RBX4F6Bd5DGmw" # 数据库密码 + pool_size: 5 # 连接池大小 + max_overflow: 10 # 最大连接数 + pool_timeout: 30 # 连接超时时间 + pool_recycle: 3600 # 连接回收时间（秒） + data_retention_days: 90 # 数据保留天数 + auto_cleanup: true # 是否自动清理过期数据 + +# 企业微信通知配置 +wechat_app: + enabled: true # 是否启用企业微信通知 + corp_id: "ww69e8e44636f47780" # 企业ID + agent_id: "1000007" # 应用ID + secret: "SmelCwKFoL0E9ATWFzr-w7gsfXBTN72lT1UqnNd0HpI" # 应用Secret + token: "DmvL98cAF6x9CFtQZwqD2emGL8S7HxA" # Token + encoding_aes_key: "yAc4OoSCP92YTefHXYfw27WeG9oF11W9d6nw6QYlU3D" # 消息加密Key + port: 18001 # 服务端口 + host: "0.0.0.0" # 服务主机 + debug: false # 调试模式 + use_proxy: false # 是否使用代理API + proxy_api_url: "https://api.v6ole.top" # 代理API地址 + +# Markdown输出配置 +markdown: + enabled: true # 是否启用Markdown输出 + output_file: "onu.md" # 输出文件路径 + max_entries: 1000 # 最大条目数 + include_today_highlight: true # 是否突出显示今日公告 + template_file: "templates/announcement.md" # 模板文件 + +# 公告来源配置 +sources: + ZcyAnnouncement1: + category_id: 66485 + name: "采购公告" + type: "purchase" + ZcyAnnouncement2: + category_id: 66485 + name: "结果公告" + type: "result" + ZcyAnnouncement3: + category_id: 66485 + name: "合同公告" + type: "contract" + ZcyAnnouncement4: + category_id: 66485 + name: "更正公告" + type: "correction" + ZcyAnnouncement5: + category_id: 66485 + name: "招标文件预公示" + type: "pre_announcement" + ZcyAnnouncement6: + category_id: 66485 + name: "单一来源公示" + type: "single_source" + ZcyAnnouncement7: + category_id: 66485 + name: "电子卖场公示" + type: "electronic_market" + ZcyAnnouncement10: + category_id: 66485 + name: "履约验收公示" + type: "acceptance" + ZcyAnnouncement11: + category_id: 66485 + name: "工程类公告" + type: "engineering" + ZcyAnnouncement20: + category_id: 66485 + name: "框架协议征集公告" + type: "framework_agreement" + ZcyAnnouncement21: + category_id: 66485 + name: "框架协议入围结果公告" + type: "framework_result" + ZcyAnnouncement23: + category_id: 66485 + name: "框架协议成交结果汇总公告" + type: "framework_summary" + "61-266648": + category_id: 66485 + name: "采购意向公开" + type: "intention" + +# 调度配置已移除 - 如需定时任务功能，请重新添加 + +# 监控配置 +monitoring: + enabled: true # 是否启用监控 + health_check_interval: 300 # 健康检查间隔（秒） + alert_on_failure: true # 失败时是否告警 + max_consecutive_failures: 3 # 最大连续失败次数 + metrics_enabled: true # 是否启用指标收集 diff --git a/gx_gp_monitor/core/__init__.py b/gx_gp_monitor/core/__init__.py new file mode 100644 index 0000000..9377aea --- /dev/null +++ b/gx_gp_monitor/core/__init__.py @@ -0,0 +1 @@ +"""核心模块""" diff --git a/gx_gp_monitor/core/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/core/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..5c49209 Binary files /dev/null and b/gx_gp_monitor/core/__pycache__/__init__.cpython-313.pyc differ diff --git a/gx_gp_monitor/core/__pycache__/config_manager.cpython-313.pyc b/gx_gp_monitor/core/__pycache__/config_manager.cpython-313.pyc new file mode 100644 index 0000000..0ceffb1 Binary files /dev/null and b/gx_gp_monitor/core/__pycache__/config_manager.cpython-313.pyc differ diff --git a/gx_gp_monitor/core/__pycache__/database.cpython-313.pyc b/gx_gp_monitor/core/__pycache__/database.cpython-313.pyc new file mode 100644 index 0000000..698a718 Binary files /dev/null and b/gx_gp_monitor/core/__pycache__/database.cpython-313.pyc differ diff --git a/gx_gp_monitor/core/__pycache__/logger.cpython-313.pyc b/gx_gp_monitor/core/__pycache__/logger.cpython-313.pyc new file mode 100644 index 0000000..39ad726 Binary files /dev/null and b/gx_gp_monitor/core/__pycache__/logger.cpython-313.pyc differ diff --git a/gx_gp_monitor/core/__pycache__/models.cpython-313.pyc b/gx_gp_monitor/core/__pycache__/models.cpython-313.pyc new file mode 100644 index 0000000..693591f Binary files /dev/null and b/gx_gp_monitor/core/__pycache__/models.cpython-313.pyc differ diff --git a/gx_gp_monitor/core/__pycache__/reliability.cpython-313.pyc b/gx_gp_monitor/core/__pycache__/reliability.cpython-313.pyc new file mode 100644 index 0000000..7ff2b4b Binary files /dev/null and b/gx_gp_monitor/core/__pycache__/reliability.cpython-313.pyc differ diff --git a/gx_gp_monitor/core/config_manager.py b/gx_gp_monitor/core/config_manager.py new file mode 100644 index 0000000..bbe738d --- /dev/null +++ b/gx_gp_monitor/core/config_manager.py @@ -0,0 +1,394 @@ +""" +配置管理模块 +负责加载、验证和管理系统配置 +""" + +import os +import yaml +from typing import Dict, Any, Optional +from pathlib import Path +from dataclasses import dataclass +from enum import Enum + + +class LogLevel(Enum): + """日志级别枚举""" + DEBUG = "DEBUG" + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + CRITICAL = "CRITICAL" + + +@dataclass +class CrawlerConfig: + """爬虫配置""" + base_url: str + timeout: int + max_retries: int + retry_delay: float + max_retry_delay: float + backoff_factor: float + user_agents: list + proxies: list + request_delay: float + request_delay_max: float + keyword: list + start_date: str + end_date: str + max_pages: int + page_size: int + + +@dataclass +class DatabaseConfig: + """数据库配置""" + enabled: bool + type: str + host: str + port: int + name: str + user: str + password: str + pool_size: int + max_overflow: int + pool_timeout: int + pool_recycle: int + data_retention_days: int + auto_cleanup: bool + + +@dataclass +class WeChatConfig: + """企业微信配置""" + enabled: bool + corp_id: str + agent_id: str + secret: str + token: str + encoding_aes_key: str + port: int + host: str + debug: bool + use_proxy: bool + proxy_api_url: str + + +@dataclass +class MarkdownConfig: + """Markdown配置""" + enabled: bool + output_file: str + max_entries: int + include_today_highlight: bool + template_file: str + + +@dataclass +class SchedulerConfig: + """调度配置""" + enabled: bool + timezone: str + jobs: list + + +@dataclass +class MonitoringConfig: + """监控配置""" + enabled: bool + health_check_interval: int + alert_on_failure: bool + max_consecutive_failures: int + metrics_enabled: bool + + +@dataclass +class SystemConfig: + """系统配置""" + debug: bool + log_level: LogLevel + log_file: str + log_max_size: int + log_backup_count: int + crawler: CrawlerConfig + database: DatabaseConfig + wechat_app: WeChatConfig + markdown: MarkdownConfig + scheduler: Optional[SchedulerConfig] + monitoring: MonitoringConfig + sources: Dict[str, Dict[str, Any]] + + +class ConfigManager: + """配置管理器""" + + def __init__(self, config_file: Optional[str] = None): + """ + 初始化配置管理器 + + Args: + config_file: 配置文件路径，如果为None则使用默认路径 + """ + if config_file is None: + # 默认配置文件路径 + current_dir = Path(__file__).parent.parent + self.config_file = current_dir / "config" / "config.yaml" + else: + self.config_file = Path(config_file) + + self._config_data = {} + self._config = None + + def load_config(self) -> SystemConfig: + """ + 加载配置文件 + + Returns: + SystemConfig: 系统配置对象 + + Raises: + FileNotFoundError: 配置文件不存在 + yaml.YAMLError: 配置文件格式错误 + ValueError: 配置验证失败 + """ + if not self.config_file.exists(): + raise FileNotFoundError(f"配置文件不存在: {self.config_file}") + + try: + with open(self.config_file, 'r', encoding='utf-8') as f: + self._config_data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise yaml.YAMLError(f"配置文件格式错误: {e}") + + # 验证配置 + self._validate_config() + + # 解析配置 + self._config = self._parse_config() + return self._config + + def _validate_config(self): + """验证配置完整性""" + required_keys = [ + 'debug', 'log_level', 'log_file', 'log_max_size', 'log_backup_count', + 'crawler', 'database', 'wechat_app', 'markdown', + 'monitoring', 'sources' + ] + + for key in required_keys: + if key not in self._config_data: + raise ValueError(f"配置文件缺少必需的配置项: {key}") + + # 验证爬虫配置 + crawler_required = [ + 'base_url', 'timeout', 'max_retries', 'retry_delay', 'max_retry_delay', + 'backoff_factor', 'user_agents', 'proxies', 'request_delay', + 'request_delay_max', 'keyword', 'max_pages', 'page_size' + ] + + for key in crawler_required: + if key not in self._config_data['crawler']: + raise ValueError(f"爬虫配置缺少必需项: {key}") + + # 验证数据库配置 + if self._config_data.get('database', {}).get('enabled', False): + db_required = ['type', 'host', 'port', 'name', 'user', 'password'] + for key in db_required: + if key not in self._config_data['database']: + raise ValueError(f"数据库配置缺少必需项: {key}") + + # 验证企业微信配置 + if self._config_data.get('wechat_app', {}).get('enabled', False): + wechat_required = ['corp_id', 'agent_id', 'secret', 'token', 'encoding_aes_key'] + for key in wechat_required: + if key not in self._config_data['wechat_app']: + raise ValueError(f"企业微信配置缺少必需项: {key}") + + def _parse_config(self) -> SystemConfig: + """解析配置数据""" + crawler_data = self._config_data['crawler'] + crawler = CrawlerConfig( + base_url=crawler_data['base_url'], + timeout=crawler_data['timeout'], + max_retries=crawler_data['max_retries'], + retry_delay=crawler_data['retry_delay'], + max_retry_delay=crawler_data['max_retry_delay'], + backoff_factor=crawler_data['backoff_factor'], + user_agents=crawler_data['user_agents'], + proxies=crawler_data['proxies'], + request_delay=crawler_data['request_delay'], + request_delay_max=crawler_data['request_delay_max'], + keyword=crawler_data['keyword'], + start_date=crawler_data.get('start_date', ''), + end_date=crawler_data.get('end_date', ''), + max_pages=crawler_data['max_pages'], + page_size=crawler_data['page_size'] + ) + + db_data = self._config_data['database'] + database = DatabaseConfig( + enabled=db_data.get('enabled', False), + type=db_data.get('type', 'postgresql'), + host=db_data.get('host', 'localhost'), + port=db_data.get('port', 5432), + name=db_data.get('name', ''), + user=db_data.get('user', ''), + password=db_data.get('password', ''), + pool_size=db_data.get('pool_size', 5), + max_overflow=db_data.get('max_overflow', 10), + pool_timeout=db_data.get('pool_timeout', 30), + pool_recycle=db_data.get('pool_recycle', 3600), + data_retention_days=db_data.get('data_retention_days', 90), + auto_cleanup=db_data.get('auto_cleanup', True) + ) + + wechat_data = self._config_data['wechat_app'] + wechat_app = WeChatConfig( + enabled=wechat_data.get('enabled', False), + corp_id=wechat_data.get('corp_id', ''), + agent_id=wechat_data.get('agent_id', ''), + secret=wechat_data.get('secret', ''), + token=wechat_data.get('token', ''), + encoding_aes_key=wechat_data.get('encoding_aes_key', ''), + port=wechat_data.get('port', 18001), + host=wechat_data.get('host', '0.0.0.0'), + debug=wechat_data.get('debug', False), + use_proxy=wechat_data.get('use_proxy', False), + proxy_api_url=wechat_data.get('proxy_api_url', 'https://api.v6ole.top') + ) + + md_data = self._config_data['markdown'] + markdown = MarkdownConfig( + enabled=md_data.get('enabled', True), + output_file=md_data.get('output_file', 'onu.md'), + max_entries=md_data.get('max_entries', 1000), + include_today_highlight=md_data.get('include_today_highlight', True), + template_file=md_data.get('template_file', 'templates/announcement.md') + ) + + # scheduler配置为可选 + scheduler = None + if 'scheduler' in self._config_data: + scheduler_data = self._config_data['scheduler'] + scheduler = SchedulerConfig( + enabled=scheduler_data.get('enabled', False), + timezone=scheduler_data.get('timezone', 'Asia/Shanghai'), + jobs=scheduler_data.get('jobs', []) + ) + + monitoring_data = self._config_data['monitoring'] + monitoring = MonitoringConfig( + enabled=monitoring_data.get('enabled', True), + health_check_interval=monitoring_data.get('health_check_interval', 300), + alert_on_failure=monitoring_data.get('alert_on_failure', True), + max_consecutive_failures=monitoring_data.get('max_consecutive_failures', 3), + metrics_enabled=monitoring_data.get('metrics_enabled', True) + ) + + return SystemConfig( + debug=self._config_data['debug'], + log_level=LogLevel(self._config_data['log_level']), + log_file=self._config_data['log_file'], + log_max_size=self._config_data['log_max_size'], + log_backup_count=self._config_data['log_backup_count'], + crawler=crawler, + database=database, + wechat_app=wechat_app, + markdown=markdown, + scheduler=scheduler, + monitoring=monitoring, + sources=self._config_data['sources'] + ) + + def get_config(self) -> SystemConfig: + """ + 获取配置对象 + + Returns: + SystemConfig: 系统配置对象 + + Raises: + RuntimeError: 配置未加载 + """ + if self._config is None: + raise RuntimeError("配置未加载，请先调用 load_config()") + return self._config + + def reload_config(self) -> SystemConfig: + """ + 重新加载配置 + + Returns: + SystemConfig: 重新加载的系统配置对象 + """ + self._config = None + return self.load_config() + + def get_value(self, key_path: str, default=None) -> Any: + """ + 通过路径获取配置值 + + Args: + key_path: 配置路径，如 'crawler.timeout' 或 'database.host' + default: 默认值 + + Returns: + 配置值或默认值 + """ + keys = key_path.split('.') + value = self._config_data + + try: + for key in keys: + value = value[key] + return value + except (KeyError, TypeError): + return default + + +# 全局配置管理器实例 +_config_manager = None + + +def get_config_manager(config_file: Optional[str] = None) -> ConfigManager: + """ + 获取全局配置管理器实例 + + Args: + config_file: 配置文件路径 + + Returns: + ConfigManager: 配置管理器实例 + """ + global _config_manager + if _config_manager is None: + _config_manager = ConfigManager(config_file) + return _config_manager + + +def load_config(config_file: Optional[str] = None) -> SystemConfig: + """ + 加载系统配置 + + Args: + config_file: 配置文件路径 + + Returns: + SystemConfig: 系统配置对象 + """ + manager = get_config_manager(config_file) + return manager.load_config() + + +def get_config() -> SystemConfig: + """ + 获取当前加载的配置 + + Returns: + SystemConfig: 系统配置对象 + + Raises: + RuntimeError: 配置未加载 + """ + manager = get_config_manager() + return manager.get_config() diff --git a/gx_gp_monitor/core/database.py b/gx_gp_monitor/core/database.py new file mode 100644 index 0000000..cc4bb45 --- /dev/null +++ b/gx_gp_monitor/core/database.py @@ -0,0 +1,612 @@ +""" +PostgreSQL数据库连接和操作模块 +提供数据库连接池、CRUD操作、数据清理等功能 +""" + +import psycopg2 +from psycopg2 import pool, extras +from psycopg2.extras import RealDictCursor +from contextlib import contextmanager +from typing import List, Dict, Any, Optional, Generator +from datetime import datetime, timedelta +import threading +from dataclasses import asdict + +from .models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus +from .config_manager import get_config +from .logger import get_logger +from .reliability import retry_on_exception, RetryConfig + + +logger = get_logger(__name__) + + +class DatabaseConnectionPool: + """数据库连接池管理器""" + + _instance = None + _pool = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if self._pool is None: + self._pool = None + self._config = None + + def init_pool(self, config): + """ + 初始化连接池 + + Args: + config: 数据库配置 + """ + if self._pool is not None: + return + + try: + self._config = config + self._pool = psycopg2.pool.SimpleConnectionPool( + minconn=config.pool_size, + maxconn=config.pool_size + config.max_overflow, + host=config.host, + port=config.port, + database=config.name, + user=config.user, + password=config.password, + connect_timeout=config.pool_timeout + ) + logger.info("数据库连接池初始化成功") + except Exception as e: + logger.error(f"数据库连接池初始化失败: {str(e)}") + raise + + def get_connection(self): + """ + 获取数据库连接 + + Returns: + 数据库连接对象 + + Raises: + Exception: 获取连接失败 + """ + if self._pool is None: + raise Exception("数据库连接池未初始化") + + try: + conn = self._pool.getconn() + # 设置自动提交为False，需要手动提交 + conn.autocommit = False + return conn + except Exception as e: + logger.error(f"获取数据库连接失败: {str(e)}") + raise + + def return_connection(self, conn): + """ + 返回数据库连接到连接池 + + Args: + conn: 数据库连接对象 + """ + if self._pool and conn: + try: + self._pool.putconn(conn) + except Exception as e: + logger.warning(f"返回数据库连接失败: {str(e)}") + + def close_all(self): + """关闭所有连接""" + if self._pool: + try: + self._pool.closeall() + logger.info("数据库连接池已关闭") + except Exception as e: + logger.error(f"关闭数据库连接池失败: {str(e)}") + + +# 全局连接池实例 +_connection_pool = DatabaseConnectionPool() + + +@contextmanager +def get_db_connection(): + """ + 获取数据库连接的上下文管理器 + + Yields: + 数据库连接对象 + """ + conn = None + try: + conn = _connection_pool.get_connection() + yield conn + except Exception as e: + logger.error(f"数据库连接错误: {str(e)}") + raise + finally: + if conn: + _connection_pool.return_connection(conn) + + +@contextmanager +def get_db_cursor(commit: bool = True): + """ + 获取数据库游标的上下文管理器 + + Args: + commit: 是否自动提交事务 + + Yields: + 数据库游标对象 + """ + with get_db_connection() as conn: + cursor = None + try: + cursor = conn.cursor(cursor_factory=RealDictCursor) + yield cursor + if commit: + conn.commit() + except Exception as e: + conn.rollback() + logger.error(f"数据库操作错误: {str(e)}") + raise + finally: + if cursor: + cursor.close() + + +class DatabaseManager: + """数据库管理器""" + + def __init__(self): + self.config = get_config() + if self.config.database.enabled: + _connection_pool.init_pool(self.config.database) + else: + logger.warning("数据库功能已禁用") + + def init_database(self): + """初始化数据库表结构""" + if not self.config.database.enabled: + return + + logger.info("开始初始化数据库表结构") + + # 创建表的SQL语句 + create_tables_sql = """ + -- 公告表 + CREATE TABLE IF NOT EXISTS announcements ( + id SERIAL PRIMARY KEY, + title VARCHAR(500) NOT NULL, + publish_date TIMESTAMP NOT NULL, + purchase_name VARCHAR(200), + content_url TEXT, + source_code VARCHAR(50) NOT NULL, + source_name VARCHAR(100) NOT NULL, + announcement_type VARCHAR(50) NOT NULL, + crawled_at TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + content_hash VARCHAR(32) UNIQUE, + keyword_matched BOOLEAN DEFAULT FALSE, + date_filtered BOOLEAN DEFAULT TRUE, + is_new BOOLEAN DEFAULT TRUE, + is_today BOOLEAN DEFAULT FALSE + ); + + -- 公告来源表 + CREATE TABLE IF NOT EXISTS announcement_sources ( + code VARCHAR(50) PRIMARY KEY, + category_id INTEGER NOT NULL, + name VARCHAR(100) NOT NULL, + type VARCHAR(50) NOT NULL + ); + + -- 爬取结果表 + CREATE TABLE IF NOT EXISTS crawl_results ( + id SERIAL PRIMARY KEY, + source_code VARCHAR(50) NOT NULL, + status VARCHAR(20) NOT NULL, + total_count INTEGER DEFAULT 0, + new_count INTEGER DEFAULT 0, + error_message TEXT, + crawled_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + duration FLOAT DEFAULT 0.0, + FOREIGN KEY (source_code) REFERENCES announcement_sources(code) + ); + + -- 创建索引 + CREATE INDEX IF NOT EXISTS idx_announcements_publish_date ON announcements(publish_date DESC); + CREATE INDEX IF NOT EXISTS idx_announcements_source_code ON announcements(source_code); + CREATE INDEX IF NOT EXISTS idx_announcements_content_hash ON announcements(content_hash); + CREATE INDEX IF NOT EXISTS idx_announcements_created_at ON announcements(created_at DESC); + CREATE INDEX IF NOT EXISTS idx_crawl_results_crawled_at ON crawl_results(crawled_at DESC); + + -- 创建更新时间触发器 + CREATE OR REPLACE FUNCTION update_updated_at_column() + RETURNS TRIGGER AS $$ + BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ language 'plpgsql'; + + DROP TRIGGER IF EXISTS update_announcements_updated_at ON announcements; + CREATE TRIGGER update_announcements_updated_at + BEFORE UPDATE ON announcements + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + """ + + with get_db_cursor() as cursor: + cursor.execute(create_tables_sql) + logger.info("数据库表结构初始化完成") + + @retry_on_exception(RetryConfig(max_retries=3)) + def save_announcement(self, announcement: Announcement) -> bool: + """ + 保存公告到数据库 + + Args: + announcement: 公告对象 + + Returns: + bool: 保存是否成功 + """ + if not self.config.database.enabled: + return False + + # 生成内容哈希（如果还没有） + if not announcement.content_hash: + announcement.generate_content_hash() + + sql = """ + INSERT INTO announcements ( + title, publish_date, purchase_name, content_url, source_code, source_name, + announcement_type, crawled_at, content_hash, keyword_matched, + date_filtered, is_new, is_today + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (content_hash) DO NOTHING + """ + + values = ( + announcement.title, + announcement.publish_date, + announcement.purchase_name, + announcement.content_url, + announcement.source_code, + announcement.source_name, + announcement.announcement_type.value, + announcement.crawled_at, + announcement.content_hash, + announcement.keyword_matched, + announcement.date_filtered, + announcement.is_new, + announcement.is_today + ) + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, values) + affected_rows = cursor.rowcount + if affected_rows > 0: + logger.debug(f"成功保存公告: {announcement.title[:50]}...") + return True + else: + logger.debug(f"公告已存在，跳过保存: {announcement.title[:50]}...") + return False + except Exception as e: + logger.error(f"保存公告失败: {str(e)}") + return False + + @retry_on_exception(RetryConfig(max_retries=3)) + def save_announcements_batch(self, announcements: List[Announcement]) -> int: + """ + 批量保存公告 + + Args: + announcements: 公告列表 + + Returns: + int: 成功保存的数量 + """ + if not self.config.database.enabled: + return 0 + + if not announcements: + return 0 + + # 为没有哈希的公告生成哈希 + for announcement in announcements: + if not announcement.content_hash: + announcement.generate_content_hash() + + sql = """ + INSERT INTO announcements ( + title, publish_date, purchase_name, content_url, source_code, source_name, + announcement_type, crawled_at, content_hash, keyword_matched, + date_filtered, is_new, is_today + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (content_hash) DO NOTHING + """ + + values = [] + for announcement in announcements: + values.append(( + announcement.title, + announcement.publish_date, + announcement.purchase_name, + announcement.content_url, + announcement.source_code, + announcement.source_name, + announcement.announcement_type.value, + announcement.crawled_at, + announcement.content_hash, + announcement.keyword_matched, + announcement.date_filtered, + announcement.is_new, + announcement.is_today + )) + + try: + with get_db_cursor() as cursor: + extras.execute_batch(cursor, sql, values) + affected_rows = cursor.rowcount + logger.info(f"批量保存公告完成，成功保存 {affected_rows} 条") + return affected_rows + except Exception as e: + logger.error(f"批量保存公告失败: {str(e)}") + return 0 + + @retry_on_exception(RetryConfig(max_retries=3)) + def get_announcements(self, + source_code: Optional[str] = None, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + limit: int = 100, + offset: int = 0) -> List[Announcement]: + """ + 查询公告 + + Args: + source_code: 来源代码过滤 + start_date: 开始日期 + end_date: 结束日期 + limit: 限制数量 + offset: 偏移量 + + Returns: + List[Announcement]: 公告列表 + """ + if not self.config.database.enabled: + return [] + + sql = """ + SELECT * FROM announcements + WHERE 1=1 + """ + params = [] + + if source_code: + sql += " AND source_code = %s" + params.append(source_code) + + if start_date: + sql += " AND publish_date >= %s" + params.append(start_date) + + if end_date: + sql += " AND publish_date <= %s" + params.append(end_date) + + sql += " ORDER BY publish_date DESC LIMIT %s OFFSET %s" + params.extend([limit, offset]) + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, params) + rows = cursor.fetchall() + + announcements = [] + for row in rows: + # 转换数据类型 + row_dict = dict(row) + row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type']) + announcements.append(Announcement.from_dict(row_dict)) + + return announcements + except Exception as e: + logger.error(f"查询公告失败: {str(e)}") + return [] + + @retry_on_exception(RetryConfig(max_retries=3)) + def save_crawl_result(self, result: CrawlResult) -> bool: + """ + 保存爬取结果 + + Args: + result: 爬取结果对象 + + Returns: + bool: 保存是否成功 + """ + if not self.config.database.enabled: + return False + + sql = """ + INSERT INTO crawl_results ( + source_code, status, total_count, new_count, error_message, + crawled_at, duration + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + """ + + values = ( + result.source.code, + result.status.value, + result.total_count, + result.new_count, + result.error_message, + result.crawled_at, + result.duration + ) + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, values) + logger.debug(f"保存爬取结果: {result.source.name}") + return True + except Exception as e: + logger.error(f"保存爬取结果失败: {str(e)}") + return False + + @retry_on_exception(RetryConfig(max_retries=3)) + def cleanup_expired_data(self, days: int = 90) -> int: + """ + 清理过期数据 + + Args: + days: 保留天数 + + Returns: + int: 清理的记录数 + """ + if not self.config.database.enabled: + return 0 + + cutoff_date = datetime.now() - timedelta(days=days) + + sql = "DELETE FROM announcements WHERE created_at < %s" + try: + with get_db_cursor() as cursor: + cursor.execute(sql, (cutoff_date,)) + deleted_count = cursor.rowcount + logger.info(f"清理过期数据完成，删除 {deleted_count} 条记录") + return deleted_count + except Exception as e: + logger.error(f"清理过期数据失败: {str(e)}") + return 0 + + @retry_on_exception(RetryConfig(max_retries=3)) + def get_statistics(self) -> Dict[str, Any]: + """ + 获取统计信息 + + Returns: + Dict[str, Any]: 统计数据 + """ + if not self.config.database.enabled: + return {} + + sql = """ + SELECT + COUNT(*) as total_announcements, + COUNT(CASE WHEN is_today THEN 1 END) as today_announcements, + COUNT(CASE WHEN is_new THEN 1 END) as new_announcements, + COUNT(DISTINCT source_code) as sources_count, + MAX(crawled_at) as last_crawl_time + FROM announcements + """ + + try: + with get_db_cursor() as cursor: + cursor.execute(sql) + result = cursor.fetchone() + return dict(result) if result else {} + except Exception as e: + logger.error(f"获取统计信息失败: {str(e)}") + return {} + + def is_announcement_exists(self, content_hash: str) -> bool: + """ + 检查公告是否已存在 + + Args: + content_hash: 内容哈希 + + Returns: + bool: 是否存在 + """ + if not self.config.database.enabled: + return False + + sql = "SELECT 1 FROM announcements WHERE content_hash = %s LIMIT 1" + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, (content_hash,)) + return cursor.fetchone() is not None + except Exception as e: + logger.error(f"检查公告存在性失败: {str(e)}") + return False + + def get_recent_announcements(self, hours: int = 24) -> List[Announcement]: + """ + 获取最近的公告 + + Args: + hours: 最近小时数 + + Returns: + List[Announcement]: 公告列表 + """ + if not self.config.database.enabled: + return [] + + cutoff_time = datetime.now() - timedelta(hours=hours) + + sql = """ + SELECT * FROM announcements + WHERE crawled_at >= %s + ORDER BY crawled_at DESC + """ + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, (cutoff_time,)) + rows = cursor.fetchall() + + announcements = [] + for row in rows: + row_dict = dict(row) + row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type']) + announcements.append(Announcement.from_dict(row_dict)) + + return announcements + except Exception as e: + logger.error(f"获取最近公告失败: {str(e)}") + return [] + + +# 全局数据库管理器实例 +_db_manager = None + + +def get_database_manager() -> DatabaseManager: + """ + 获取数据库管理器实例 + + Returns: + DatabaseManager: 数据库管理器实例 + """ + global _db_manager + if _db_manager is None: + _db_manager = DatabaseManager() + return _db_manager + + +def init_database(): + """初始化数据库""" + manager = get_database_manager() + manager.init_database() + + +def cleanup_database(): + """清理数据库连接""" + _connection_pool.close_all() diff --git a/gx_gp_monitor/core/logger.py b/gx_gp_monitor/core/logger.py new file mode 100644 index 0000000..5c9220d --- /dev/null +++ b/gx_gp_monitor/core/logger.py @@ -0,0 +1,354 @@ +""" +统一日志管理模块 +提供结构化日志记录功能，支持控制台和文件输出 +""" + +import os +import sys +import logging +import logging.handlers +from pathlib import Path +from typing import Optional, Dict, Any +from datetime import datetime + +from .config_manager import get_config + + +class ColoredFormatter(logging.Formatter): + """带颜色的日志格式化器""" + + # ANSI颜色代码 + COLORS = { + 'DEBUG': '\033[36m', # 青色 + 'INFO': '\033[32m', # 绿色 + 'WARNING': '\033[33m', # 黄色 + 'ERROR': '\033[31m', # 红色 + 'CRITICAL': '\033[35m', # 紫色 + } + RESET = '\033[0m' # 重置颜色 + + def format(self, record): + # 检查是否已经包含ANSI颜色代码 + if '\033[' in record.levelname: + # 如果已经着色，直接返回原始格式 + return super().format(record) + + # 添加颜色 + if record.levelname in self.COLORS: + # 为levelname添加颜色 + colored_levelname = f"{self.COLORS[record.levelname]}{record.levelname}{self.RESET}" + # 使用原始levelname来确定消息的颜色 + record.msg = f"{self.COLORS[record.levelname]}{record.msg}{self.RESET}" + record.levelname = colored_levelname + + return super().format(record) + + +class Logger: + """统一日志管理器""" + + _instance = None + _initialized = False + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if not self._initialized: + self._loggers = {} + self._config = None + self._initialized = True + + def init_logger(self, name: str = "gx_gp_monitor", config=None) -> logging.Logger: + """ + 初始化日志器 + + Args: + name: 日志器名称 + config: 配置对象，如果为None则从全局配置加载 + + Returns: + logging.Logger: 配置好的日志器实例 + """ + if name in self._loggers: + return self._loggers[name] + + # 获取配置 + if config is None: + try: + self._config = get_config() + except RuntimeError: + # 配置未加载，使用默认配置 + self._config = self._get_default_config() + else: + self._config = config + + # 创建日志器 + logger = logging.getLogger(name) + logger.setLevel(getattr(logging, self._config.log_level.value)) + + # 避免重复添加处理器 + if logger.handlers: + return logger + + # 创建格式化器 + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + # 控制台处理器 + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(getattr(logging, self._config.log_level.value)) + + # 使用彩色格式化器（如果支持） + if sys.platform != 'win32' and 'TERM' in os.environ: + colored_formatter = ColoredFormatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + console_handler.setFormatter(colored_formatter) + else: + console_handler.setFormatter(formatter) + + logger.addHandler(console_handler) + + # 文件处理器（如果配置了日志文件） + if self._config.log_file: + log_dir = Path(self._config.log_file).parent + log_dir.mkdir(parents=True, exist_ok=True) + + file_handler = logging.handlers.RotatingFileHandler( + self._config.log_file, + maxBytes=self._config.log_max_size, + backupCount=self._config.log_backup_count, + encoding='utf-8' + ) + file_handler.setLevel(getattr(logging, self._config.log_level.value)) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + self._loggers[name] = logger + return logger + + def _get_default_config(self): + """获取默认配置""" + from .config_manager import LogLevel + + class DefaultConfig: + def __init__(self): + self.log_level = LogLevel.INFO + self.log_file = "logs/gx_gp_monitor.log" + self.log_max_size = 10485760 # 10MB + self.log_backup_count = 5 + + return DefaultConfig() + + def get_logger(self, name: str = "gx_gp_monitor") -> logging.Logger: + """ + 获取日志器 + + Args: + name: 日志器名称 + + Returns: + logging.Logger: 日志器实例 + """ + if name not in self._loggers: + return self.init_logger(name) + return self._loggers[name] + + def log_crawl_start(self, source_name: str, logger: Optional[logging.Logger] = None): + """记录爬取开始""" + if logger is None: + logger = self.get_logger() + logger.info(f"开始爬取 {source_name}") + + def log_crawl_success(self, source_name: str, count: int, duration: float, + logger: Optional[logging.Logger] = None): + """记录爬取成功""" + if logger is None: + logger = self.get_logger() + logger.info(f"{source_name} 爬取完成，共获取 {count} 条公告，耗时 {duration:.2f}秒") + + def log_crawl_error(self, source_name: str, error: str, + logger: Optional[logging.Logger] = None): + """记录爬取错误""" + if logger is None: + logger = self.get_logger() + logger.error(f"{source_name} 爬取失败: {error}") + + def log_announcement_filtered(self, reason: str, count: int, + logger: Optional[logging.Logger] = None): + """记录公告筛选信息""" + if logger is None: + logger = self.get_logger() + logger.info(f"公告筛选 - {reason}: {count} 条") + + def log_database_operation(self, operation: str, table: str, count: int = 0, + logger: Optional[logging.Logger] = None): + """记录数据库操作""" + if logger is None: + logger = self.get_logger() + if count > 0: + logger.info(f"数据库操作 - {operation} {table}: {count} 条记录") + else: + logger.info(f"数据库操作 - {operation} {table}") + + def log_notification_sent(self, channel: str, recipient_count: int, + logger: Optional[logging.Logger] = None): + """记录通知发送""" + if logger is None: + logger = self.get_logger() + logger.info(f"通知发送 - {channel}: 向 {recipient_count} 个接收者发送") + + def log_system_metrics(self, metrics: Dict[str, Any], + logger: Optional[logging.Logger] = None): + """记录系统指标""" + if logger is None: + logger = self.get_logger() + metrics_str = ", ".join([f"{k}={v}" for k, v in metrics.items()]) + logger.info(f"系统指标: {metrics_str}") + + def log_performance_warning(self, operation: str, duration: float, threshold: float, + logger: Optional[logging.Logger] = None): + """记录性能警告""" + if logger is None: + logger = self.get_logger() + logger.warning(f"性能警告 - {operation} 耗时 {duration:.2f}秒，超过阈值 {threshold:.2f}秒") + + +# 全局日志管理器实例 +_logger_manager = Logger() + + +def get_logger(name: str = "gx_gp_monitor") -> logging.Logger: + """ + 获取日志器 + + Args: + name: 日志器名称 + + Returns: + logging.Logger: 日志器实例 + """ + return _logger_manager.get_logger(name) + + +def init_logger(name: str = "gx_gp_monitor", config=None) -> logging.Logger: + """ + 初始化并获取日志器 + + Args: + name: 日志器名称 + config: 配置对象 + + Returns: + logging.Logger: 日志器实例 + """ + return _logger_manager.init_logger(name, config) + + +def log_function_call(func_name: str, args: Optional[Dict[str, Any]] = None, + logger: Optional[logging.Logger] = None): + """ + 装饰器：记录函数调用 + + Args: + func_name: 函数名称 + args: 函数参数 + logger: 日志器实例 + """ + def decorator(func): + def wrapper(*args, **kwargs): + nonlocal logger + if logger is None: + logger = get_logger() + + start_time = datetime.now() + logger.debug(f"调用函数: {func_name}") + + try: + result = func(*args, **kwargs) + duration = (datetime.now() - start_time).total_seconds() + logger.debug(f"函数 {func_name} 执行完成，耗时 {duration:.3f}秒") + return result + except Exception as e: + duration = (datetime.now() - start_time).total_seconds() + logger.error(f"函数 {func_name} 执行失败，耗时 {duration:.3f}秒: {str(e)}") + raise + + return wrapper + return decorator + + +# 便捷函数 +def log_info(message: str, logger: Optional[logging.Logger] = None): + """记录信息日志""" + if logger is None: + logger = get_logger() + logger.info(message) + + +def log_warning(message: str, logger: Optional[logging.Logger] = None): + """记录警告日志""" + if logger is None: + logger = get_logger() + logger.warning(message) + + +def log_error(message: str, logger: Optional[logging.Logger] = None): + """记录错误日志""" + if logger is None: + logger = get_logger() + logger.error(message) + + +def log_debug(message: str, logger: Optional[logging.Logger] = None): + """记录调试日志""" + if logger is None: + logger = get_logger() + logger.debug(message) + + +# 便捷的爬取日志记录函数 +def log_crawl_start(source_name: str): + """记录爬取开始""" + _logger_manager.log_crawl_start(source_name) + + +def log_crawl_success(source_name: str, count: int, duration: float): + """记录爬取成功""" + _logger_manager.log_crawl_success(source_name, count, duration) + + +def log_crawl_error(source_name: str, error: str): + """记录爬取错误""" + _logger_manager.log_crawl_error(source_name, error) + + +def log_announcement_filtered(reason: str, count: int): + """记录公告筛选信息""" + _logger_manager.log_announcement_filtered(reason, count) + + +def log_database_operation(operation: str, table: str, count: int = 0): + """记录数据库操作""" + _logger_manager.log_database_operation(operation, table, count) + + +def log_notification_sent(channel: str, recipient_count: int): + """记录通知发送""" + _logger_manager.log_notification_sent(channel, recipient_count) + + +def log_system_metrics(metrics: Dict[str, Any]): + """记录系统指标""" + _logger_manager.log_system_metrics(metrics) + + +def log_performance_warning(operation: str, duration: float, threshold: float): + """记录性能警告""" + _logger_manager.log_performance_warning(operation, duration, threshold) diff --git a/gx_gp_monitor/core/models.py b/gx_gp_monitor/core/models.py new file mode 100644 index 0000000..c8fea67 --- /dev/null +++ b/gx_gp_monitor/core/models.py @@ -0,0 +1,255 @@ +""" +数据模型定义 +定义系统使用的数据结构和模型 +""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, List, Dict, Any +from enum import Enum + + +class AnnouncementType(Enum): + """公告类型枚举""" + PURCHASE = "purchase" # 采购公告 + RESULT = "result" # 结果公告 + CONTRACT = "contract" # 合同公告 + CORRECTION = "correction" # 更正公告 + PRE_ANNOUNCEMENT = "pre_announcement" # 招标文件预公示 + SINGLE_SOURCE = "single_source" # 单一来源公示 + ELECTRONIC_MARKET = "electronic_market" # 电子卖场公示 + ACCEPTANCE = "acceptance" # 履约验收公示 + ENGINEERING = "engineering" # 工程类公告 + FRAMEWORK_AGREEMENT = "framework_agreement" # 框架协议征集公告 + FRAMEWORK_RESULT = "framework_result" # 框架协议入围结果公告 + FRAMEWORK_SUMMARY = "framework_summary" # 框架协议成交结果汇总公告 + INTENTION = "intention" # 采购意向公开 + + +class CrawlStatus(Enum): + """爬取状态枚举""" + PENDING = "pending" # 待爬取 + RUNNING = "running" # 爬取中 + SUCCESS = "success" # 成功 + FAILED = "failed" # 失败 + PARTIAL = "partial" # 部分成功 + + +@dataclass +class AnnouncementSource: + """公告来源""" + code: str # 来源代码，如 "ZcyAnnouncement1" + category_id: int # 分类ID + name: str # 显示名称，如 "采购公告" + type: AnnouncementType # 公告类型 + + +@dataclass +class Announcement: + """公告数据模型""" + id: Optional[int] = None # 数据库ID + title: str = "" # 公告标题 + publish_date: datetime = field(default_factory=datetime.now) # 发布时间 + purchase_name: str = "" # 发布单位 + content_url: str = "" # 内容链接 + source_code: str = "" # 来源代码 + source_name: str = "" # 来源名称 + announcement_type: AnnouncementType = AnnouncementType.PURCHASE # 公告类型 + + # 爬取相关字段 + crawled_at: Optional[datetime] = None # 爬取时间 + created_at: Optional[datetime] = None # 创建时间 + updated_at: Optional[datetime] = None # 更新时间 + + # 去重字段 + content_hash: Optional[str] = None # 内容哈希，用于去重 + + # 筛选相关 + keyword_matched: bool = False # 是否匹配关键词 + date_filtered: bool = True # 是否在日期范围内 + + # 业务字段 + is_new: bool = True # 是否为新公告 + is_today: bool = False # 是否为今日公告 + + def __post_init__(self): + """后初始化处理""" + if isinstance(self.announcement_type, str): + self.announcement_type = AnnouncementType(self.announcement_type) + + if self.publish_date and isinstance(self.publish_date, str): + try: + self.publish_date = datetime.fromisoformat(self.publish_date.replace('Z', '+00:00')) + except ValueError: + # 如果解析失败，使用当前时间 + self.publish_date = datetime.now() + + # 判断是否为今日公告 + today = datetime.now().date() + if self.publish_date: + self.is_today = self.publish_date.date() == today + + @property + def publish_date_str(self) -> str: + """获取发布日期字符串""" + return self.publish_date.strftime("%Y-%m-%d") if self.publish_date else "" + + @property + def crawled_at_str(self) -> str: + """获取爬取时间字符串""" + return self.crawled_at.strftime("%Y-%m-%d %H:%M:%S") if self.crawled_at else "" + + def to_dict(self) -> Dict[str, Any]: + """转换为字典""" + return { + "id": self.id, + "title": self.title, + "publish_date": self.publish_date.isoformat() if self.publish_date else None, + "purchase_name": self.purchase_name, + "content_url": self.content_url, + "source_code": self.source_code, + "source_name": self.source_name, + "announcement_type": self.announcement_type.value, + "crawled_at": self.crawled_at.isoformat() if self.crawled_at else None, + "created_at": self.created_at.isoformat() if self.created_at else None, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + "content_hash": self.content_hash, + "keyword_matched": self.keyword_matched, + "date_filtered": self.date_filtered, + "is_new": self.is_new, + "is_today": self.is_today + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'Announcement': + """从字典创建实例""" + # 处理枚举类型 + if 'announcement_type' in data and isinstance(data['announcement_type'], str): + data['announcement_type'] = AnnouncementType(data['announcement_type']) + + # 处理日期时间 + for date_field in ['publish_date', 'crawled_at', 'created_at', 'updated_at']: + if date_field in data and data[date_field] and isinstance(data[date_field], str): + try: + data[date_field] = datetime.fromisoformat(data[date_field].replace('Z', '+00:00')) + except ValueError: + data[date_field] = None + + return cls(**data) + + def generate_content_hash(self) -> str: + """生成内容哈希用于去重""" + import hashlib + content = f"{self.title}|{self.publish_date_str}|{self.purchase_name}|{self.content_url}|{self.source_code}" + self.content_hash = hashlib.md5(content.encode('utf-8')).hexdigest() + return self.content_hash + + def matches_keywords(self, keywords: List[str]) -> bool: + """检查是否匹配关键词""" + if not keywords: + return True + + search_text = f"{self.title} {self.purchase_name}".lower() + for keyword in keywords: + if keyword.lower() in search_text: + self.keyword_matched = True + return True + + self.keyword_matched = False + return False + + def in_date_range(self, start_date: Optional[str], end_date: Optional[str]) -> bool: + """检查是否在日期范围内""" + if not self.publish_date: + self.date_filtered = False + return False + + publish_date = self.publish_date.date() + + try: + if start_date: + start = datetime.fromisoformat(start_date).date() + if publish_date < start: + self.date_filtered = False + return False + + if end_date: + end = datetime.fromisoformat(end_date).date() + if publish_date > end: + self.date_filtered = False + return False + + self.date_filtered = True + return True + except ValueError: + # 日期格式错误时，默认通过 + self.date_filtered = True + return True + + +@dataclass +class CrawlResult: + """爬取结果""" + source: AnnouncementSource # 公告来源 + status: CrawlStatus # 爬取状态 + total_count: int = 0 # 总公告数 + new_count: int = 0 # 新增公告数 + error_message: Optional[str] = None # 错误信息 + announcements: List[Announcement] = field(default_factory=list) # 公告列表 + crawled_at: datetime = field(default_factory=datetime.now) # 爬取时间 + duration: float = 0.0 # 爬取耗时（秒） + + +@dataclass +class CrawlSession: + """爬取会话""" + session_id: str # 会话ID + start_time: datetime # 开始时间 + end_time: Optional[datetime] = None # 结束时间 + status: CrawlStatus = CrawlStatus.PENDING # 会话状态 + total_sources: int = 0 # 总来源数 + completed_sources: int = 0 # 已完成来源数 + total_announcements: int = 0 # 总公告数 + new_announcements: int = 0 # 新增公告数 + results: List[CrawlResult] = field(default_factory=list) # 各来源结果 + + @property + def duration(self) -> float: + """获取会话持续时间""" + if self.end_time and self.start_time: + return (self.end_time - self.start_time).total_seconds() + elif self.start_time: + return (datetime.now() - self.start_time).total_seconds() + return 0.0 + + @property + def progress(self) -> float: + """获取完成进度（0-1）""" + if self.total_sources == 0: + return 0.0 + return self.completed_sources / self.total_sources + + +@dataclass +class NotificationMessage: + """通知消息""" + title: str # 消息标题 + content: str # 消息内容 + message_type: str = "text" # 消息类型：text, markdown, card + recipients: List[str] = field(default_factory=lambda: ["@all"]) # 接收者列表 + attachments: Optional[Dict[str, Any]] = None # 附件信息 + created_at: datetime = field(default_factory=datetime.now) # 创建时间 + + +@dataclass +class SystemMetrics: + """系统指标""" + timestamp: datetime = field(default_factory=datetime.now) # 时间戳 + total_announcements: int = 0 # 总公告数 + today_announcements: int = 0 # 今日公告数 + new_announcements_today: int = 0 # 今日新增公告数 + crawl_sessions_today: int = 0 # 今日爬取会话数 + last_crawl_duration: float = 0.0 # 最后一次爬取耗时 + database_size: int = 0 # 数据库大小（字节） + memory_usage: float = 0.0 # 内存使用率 + disk_usage: float = 0.0 # 磁盘使用率 diff --git a/gx_gp_monitor/core/reliability.py b/gx_gp_monitor/core/reliability.py new file mode 100644 index 0000000..edf6f47 --- /dev/null +++ b/gx_gp_monitor/core/reliability.py @@ -0,0 +1,489 @@ +""" +高可用性模块 +提供重试机制、超时控制、幂等操作、异常处理和恢复功能 +""" + +import time +import random +import hashlib +from contextlib import contextmanager +from functools import wraps +from typing import Callable, Any, Optional, Type, Union, List +from datetime import datetime, timedelta +import threading +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from .logger import get_logger +from .config_manager import get_config + + +logger = get_logger(__name__) + + +class RetryConfig: + """重试配置""" + + def __init__(self, + max_retries: int = 3, + initial_delay: float = 1.0, + max_delay: float = 60.0, + backoff_factor: float = 2.0, + jitter: bool = True): + """ + 初始化重试配置 + + Args: + max_retries: 最大重试次数 + initial_delay: 初始延迟时间（秒） + max_delay: 最大延迟时间（秒） + backoff_factor: 退避因子 + jitter: 是否添加随机抖动 + """ + self.max_retries = max_retries + self.initial_delay = initial_delay + self.max_delay = max_delay + self.backoff_factor = backoff_factor + self.jitter = jitter + + +class TimeoutConfig: + """超时配置""" + + def __init__(self, + connect_timeout: float = 10.0, + read_timeout: float = 30.0, + total_timeout: Optional[float] = None): + """ + 初始化超时配置 + + Args: + connect_timeout: 连接超时时间（秒） + read_timeout: 读取超时时间（秒） + total_timeout: 总超时时间（秒） + """ + self.connect_timeout = connect_timeout + self.read_timeout = read_timeout + self.total_timeout = total_timeout or (connect_timeout + read_timeout) + + +class CircuitBreakerState: + """熔断器状态""" + CLOSED = "closed" # 关闭状态，正常工作 + OPEN = "open" # 打开状态，快速失败 + HALF_OPEN = "half_open" # 半开状态，测试恢复 + + +class CircuitBreaker: + """熔断器实现""" + + def __init__(self, + failure_threshold: int = 5, + recovery_timeout: int = 60, + expected_exception: Type[Exception] = Exception): + """ + 初始化熔断器 + + Args: + failure_threshold: 失败阈值 + recovery_timeout: 恢复超时时间（秒） + expected_exception: 期望的异常类型 + """ + self.failure_threshold = failure_threshold + self.recovery_timeout = recovery_timeout + self.expected_exception = expected_exception + + self.state = CircuitBreakerState.CLOSED + self.failure_count = 0 + self.last_failure_time = None + self._lock = threading.Lock() + + def __call__(self, func: Callable) -> Callable: + """装饰器实现""" + @wraps(func) + def wrapper(*args, **kwargs): + return self._execute_with_circuit_breaker(func, *args, **kwargs) + return wrapper + + def _execute_with_circuit_breaker(self, func: Callable, *args, **kwargs) -> Any: + """使用熔断器执行函数""" + if self.state == CircuitBreakerState.OPEN: + if self._should_attempt_reset(): + self.state = CircuitBreakerState.HALF_OPEN + logger.info("熔断器半开，尝试恢复") + else: + raise CircuitBreakerOpenException("熔断器已打开") + + try: + result = func(*args, **kwargs) + self._on_success() + return result + except self.expected_exception as e: + self._on_failure() + raise + + def _should_attempt_reset(self) -> bool: + """检查是否应该尝试重置""" + if self.last_failure_time is None: + return True + return (datetime.now() - self.last_failure_time).total_seconds() >= self.recovery_timeout + + def _on_success(self): + """成功时的处理""" + with self._lock: + if self.state == CircuitBreakerState.HALF_OPEN: + self.state = CircuitBreakerState.CLOSED + self.failure_count = 0 + logger.info("熔断器关闭，服务恢复正常") + + def _on_failure(self): + """失败时的处理""" + with self._lock: + self.failure_count += 1 + self.last_failure_time = datetime.now() + + if self.failure_count >= self.failure_threshold: + self.state = CircuitBreakerState.OPEN + logger.warning(f"熔断器打开，失败次数达到阈值: {self.failure_count}") + + +class CircuitBreakerOpenException(Exception): + """熔断器打开异常""" + pass + + +class IdempotencyKey: + """幂等性键生成器""" + + @staticmethod + def generate(*args, **kwargs) -> str: + """ + 生成幂等性键 + + Args: + *args: 位置参数 + **kwargs: 关键字参数 + + Returns: + str: 幂等性键 + """ + # 将参数转换为字符串并排序 + key_parts = [] + + # 处理位置参数 + for i, arg in enumerate(args): + key_parts.append(f"arg_{i}:{str(arg)}") + + # 处理关键字参数（排序以保证一致性） + for key in sorted(kwargs.keys()): + key_parts.append(f"{key}:{str(kwargs[key])}") + + # 生成哈希 + key_string = "|".join(key_parts) + return hashlib.md5(key_string.encode('utf-8')).hexdigest() + + +class IdempotencyManager: + """幂等性管理器""" + + def __init__(self): + self._executed_keys = set() + self._lock = threading.Lock() + + def is_executed(self, key: str) -> bool: + """ + 检查操作是否已执行 + + Args: + key: 幂等性键 + + Returns: + bool: 是否已执行 + """ + with self._lock: + return key in self._executed_keys + + def mark_executed(self, key: str): + """ + 标记操作已执行 + + Args: + key: 幂等性键 + """ + with self._lock: + self._executed_keys.add(key) + + def clear_expired_keys(self, max_age_seconds: int = 3600): + """ + 清理过期的键（简化实现，实际应该使用时间戳） + + Args: + max_age_seconds: 最大年龄（秒） + """ + # 这里简化实现，实际项目中应该记录时间戳 + pass + + +def retry_on_exception(retry_config: Optional[RetryConfig] = None, + exceptions: tuple = (Exception,), + logger: Optional[Any] = None) -> Callable: + """ + 重试装饰器 + + Args: + retry_config: 重试配置 + exceptions: 需要重试的异常类型 + logger: 日志器 + + Returns: + Callable: 装饰器函数 + """ + if retry_config is None: + retry_config = RetryConfig() + + if logger is None: + logger = get_logger() + + def decorator(func: Callable) -> Callable: + @wraps(func) + def wrapper(*args, **kwargs): + last_exception = None + + for attempt in range(retry_config.max_retries + 1): + try: + return func(*args, **kwargs) + except exceptions as e: + last_exception = e + + if attempt < retry_config.max_retries: + # 计算延迟时间 + delay = min( + retry_config.initial_delay * (retry_config.backoff_factor ** attempt), + retry_config.max_delay + ) + + # 添加随机抖动 + if retry_config.jitter: + delay = delay * (0.5 + random.random() * 0.5) + + logger.warning( + f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{retry_config.max_retries + 1}): {str(e)}，" + f"等待 {delay:.2f} 秒后重试" + ) + time.sleep(delay) + else: + logger.error( + f"函数 {func.__name__} 在 {retry_config.max_retries + 1} 次尝试后仍然失败: {str(e)}" + ) + + raise last_exception + + return wrapper + return decorator + + +def timeout_wrapper(timeout_config: Optional[TimeoutConfig] = None) -> Callable: + """ + 超时装饰器 + + Args: + timeout_config: 超时配置 + + Returns: + Callable: 装饰器函数 + """ + if timeout_config is None: + timeout_config = TimeoutConfig() + + def decorator(func: Callable) -> Callable: + @wraps(func) + def wrapper(*args, **kwargs): + import signal + + def timeout_handler(signum, frame): + raise TimeoutError(f"函数 {func.__name__} 执行超时") + + # 设置信号处理器 + old_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(int(timeout_config.total_timeout)) + + try: + result = func(*args, **kwargs) + signal.alarm(0) # 取消闹钟 + return result + finally: + signal.signal(signal.SIGALRM, old_handler) + + return wrapper + return decorator + + +@contextmanager +def session_with_retry(timeout_config: Optional[TimeoutConfig] = None, + retry_config: Optional[RetryConfig] = None): + """ + 创建带有重试机制的HTTP会话 + + Args: + timeout_config: 超时配置 + retry_config: 重试配置 + + Yields: + requests.Session: 配置好的会话对象 + """ + if timeout_config is None: + timeout_config = TimeoutConfig() + + if retry_config is None: + retry_config = RetryConfig() + + session = requests.Session() + + # 配置重试策略 + retry_strategy = Retry( + total=retry_config.max_retries, + backoff_factor=retry_config.backoff_factor, + status_forcelist=[429, 500, 502, 503, 504], + ) + + adapter = HTTPAdapter(max_retries=retry_strategy) + session.mount("http://", adapter) + session.mount("https://", adapter) + + # 设置默认超时 + session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout) + + try: + yield session + finally: + session.close() + + +def safe_execute(func: Callable, + fallback: Optional[Callable] = None, + exceptions: tuple = (Exception,), + logger: Optional[Any] = None) -> Any: + """ + 安全执行函数，提供降级处理 + + Args: + func: 要执行的函数 + fallback: 降级函数 + exceptions: 需要捕获的异常类型 + logger: 日志器 + + Returns: + Any: 函数执行结果或降级结果 + """ + if logger is None: + logger = get_logger() + + try: + return func() + except exceptions as e: + logger.error(f"函数执行失败: {str(e)}") + if fallback: + try: + logger.info("执行降级函数") + return fallback() + except Exception as fallback_e: + logger.error(f"降级函数也执行失败: {str(fallback_e)}") + return None + + +class HealthChecker: + """健康检查器""" + + def __init__(self, check_interval: int = 300): + """ + 初始化健康检查器 + + Args: + check_interval: 检查间隔（秒） + """ + self.check_interval = check_interval + self.last_check = None + self.is_healthy = True + self.consecutive_failures = 0 + self.max_consecutive_failures = 3 + + def check_health(self) -> bool: + """ + 执行健康检查 + + Returns: + bool: 健康状态 + """ + current_time = datetime.now() + + # 检查是否需要执行检查 + if (self.last_check and + (current_time - self.last_check).total_seconds() < self.check_interval): + return self.is_healthy + + self.last_check = current_time + + try: + # 执行健康检查逻辑 + self._perform_health_check() + self.is_healthy = True + self.consecutive_failures = 0 + logger.info("健康检查通过") + return True + except Exception as e: + self.consecutive_failures += 1 + logger.warning(f"健康检查失败 ({self.consecutive_failures}/{self.max_consecutive_failures}): {str(e)}") + + if self.consecutive_failures >= self.max_consecutive_failures: + self.is_healthy = False + logger.error("连续健康检查失败，系统标记为不健康") + + return False + + def _perform_health_check(self): + """执行具体的健康检查逻辑""" + # 这里可以添加数据库连接检查、外部服务检查等 + config = get_config() + + # 检查数据库连接（如果启用） + if config.database.enabled: + # 这里应该检查数据库连接 + pass + + # 检查网络连接 + try: + requests.get("https://www.baidu.com", timeout=5) + except: + raise Exception("网络连接检查失败") + + +# 全局实例 +_circuit_breaker = CircuitBreaker() +_idempotency_manager = IdempotencyManager() +_health_checker = HealthChecker() + + +def get_circuit_breaker() -> CircuitBreaker: + """获取全局熔断器实例""" + return _circuit_breaker + + +def get_idempotency_manager() -> IdempotencyManager: + """获取全局幂等性管理器实例""" + return _idempotency_manager + + +def get_health_checker() -> HealthChecker: + """获取全局健康检查器实例""" + return _health_checker + + +def check_system_health() -> bool: + """ + 检查系统健康状态 + + Returns: + bool: 系统是否健康 + """ + return _health_checker.check_health() diff --git a/gx_gp_monitor/crawler/__init__.py b/gx_gp_monitor/crawler/__init__.py new file mode 100644 index 0000000..1962ccd --- /dev/null +++ b/gx_gp_monitor/crawler/__init__.py @@ -0,0 +1 @@ +"""爬虫模块""" diff --git a/gx_gp_monitor/crawler/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/crawler/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..bb46a23 Binary files /dev/null and b/gx_gp_monitor/crawler/__pycache__/__init__.cpython-313.pyc differ diff --git a/gx_gp_monitor/crawler/__pycache__/parsers.cpython-313.pyc b/gx_gp_monitor/crawler/__pycache__/parsers.cpython-313.pyc new file mode 100644 index 0000000..de4f3a5 Binary files /dev/null and b/gx_gp_monitor/crawler/__pycache__/parsers.cpython-313.pyc differ diff --git a/gx_gp_monitor/crawler/__pycache__/spider.cpython-313.pyc b/gx_gp_monitor/crawler/__pycache__/spider.cpython-313.pyc new file mode 100644 index 0000000..2b29ff7 Binary files /dev/null and b/gx_gp_monitor/crawler/__pycache__/spider.cpython-313.pyc differ diff --git a/gx_gp_monitor/crawler/parsers.py b/gx_gp_monitor/crawler/parsers.py new file mode 100644 index 0000000..8ed8ad5 --- /dev/null +++ b/gx_gp_monitor/crawler/parsers.py @@ -0,0 +1,301 @@ +""" +数据解析器模块 +负责解析广西政府采购网的API响应数据 +""" + +import json +from typing import List, Dict, Any, Optional +from datetime import datetime + +try: + from ..core.models import Announcement, AnnouncementSource, AnnouncementType + from ..core.logger import get_logger +except ImportError: + from core.models import Announcement, AnnouncementSource, AnnouncementType + from core.logger import get_logger + + +logger = get_logger(__name__) + + +class AnnouncementParser: + """公告数据解析器""" + + @staticmethod + def parse_api_response(response_data: Dict[str, Any], + source: AnnouncementSource, + crawled_at: datetime) -> List[Announcement]: + """ + 解析API响应数据 + + Args: + response_data: API响应数据 + source: 公告来源 + crawled_at: 爬取时间 + + Returns: + List[Announcement]: 解析后的公告列表 + """ + if not response_data or not isinstance(response_data, dict): + logger.warning("API响应数据无效") + return [] + + try: + # 检查响应状态 + if not response_data.get("success", False): + logger.warning(f"API响应失败: {response_data.get('message', '未知错误')}") + return [] + + # 获取数据部分 + result = response_data.get("result", {}) + data = result.get("data", {}) + records = data.get("data", []) + + if not records: + logger.info(f"来源 {source.name} 没有新数据") + return [] + + announcements = [] + for record in records: + try: + announcement = AnnouncementParser._parse_single_record( + record, source, crawled_at) + if announcement: + announcements.append(announcement) + except Exception as e: + logger.warning(f"解析公告记录失败: {str(e)}, 记录: {record}") + continue + + logger.info(f"成功解析 {len(announcements)}/{len(records)} 条公告记录") + return announcements + + except Exception as e: + logger.error(f"解析API响应数据失败: {str(e)}") + return [] + + @staticmethod + def _parse_single_record(record: Dict[str, Any], + source: AnnouncementSource, + crawled_at: datetime) -> Optional[Announcement]: + """ + 解析单个公告记录 + + Args: + record: 公告记录数据 + source: 公告来源 + crawled_at: 爬取时间 + + Returns: + Optional[Announcement]: 解析后的公告对象 + """ + try: + # 提取基本字段 + title_raw = record.get("title", "") + title = str(title_raw).strip() if title_raw is not None else "" + if not title: + return None + + # 解析发布时间 + publish_timestamp = record.get("publishDate") + if not publish_timestamp: + logger.warning(f"公告缺少发布时间: {title[:50]}...") + return None + + try: + # 时间戳转换为datetime + publish_date = datetime.fromtimestamp(int(publish_timestamp) / 1000) + except (ValueError, TypeError) as e: + logger.warning(f"发布时间格式错误: {publish_timestamp}, 错误: {str(e)}") + return None + + # 提取其他字段 + purchase_name_raw = record.get("purchaseName", "") + purchase_name = str(purchase_name_raw).strip() if purchase_name_raw is not None else "" + article_id = record.get("articleId") + + if not article_id: + logger.warning(f"公告缺少文章ID: {title[:50]}...") + return None + + # 构建内容链接 + content_url = AnnouncementParser._build_content_url( + source.category_id, source.code, article_id) + + # 创建公告对象 + announcement = Announcement( + title=title, + publish_date=publish_date, + purchase_name=purchase_name, + content_url=content_url, + source_code=source.code, + source_name=source.name, + announcement_type=source.type, + crawled_at=crawled_at, + is_new=True # 默认标记为新公告 + ) + + # 生成内容哈希用于去重 + announcement.generate_content_hash() + + return announcement + + except Exception as e: + logger.error(f"解析单个公告记录失败: {str(e)}") + return None + + @staticmethod + def _build_content_url(category_id: int, source_code: str, article_id: int) -> str: + """ + 构建公告内容链接 + + Args: + category_id: 分类ID + source_code: 来源代码 + article_id: 文章ID + + Returns: + str: 内容链接 + """ + return f"https://zfcg.gxzf.gov.cn/site/detail?parentId={category_id}&articleId={article_id}" + + @staticmethod + def validate_response_structure(response_data: Dict[str, Any]) -> bool: + """ + 验证API响应数据结构 + + Args: + response_data: API响应数据 + + Returns: + bool: 结构是否有效 + """ + try: + if not isinstance(response_data, dict): + return False + + # 检查必需的字段 + if "success" not in response_data: + return False + + if not response_data.get("success", False): + return False + + result = response_data.get("result", {}) + if not isinstance(result, dict): + return False + + data = result.get("data", {}) + if not isinstance(data, dict): + return False + + records = data.get("data", []) + if not isinstance(records, list): + return False + + return True + + except Exception as e: + logger.warning(f"验证响应结构失败: {str(e)}") + return False + + @staticmethod + def extract_pagination_info(response_data: Dict[str, Any]) -> Dict[str, Any]: + """ + 提取分页信息 + + Args: + response_data: API响应数据 + + Returns: + Dict[str, Any]: 分页信息 + """ + try: + result = response_data.get("result", {}) + data = result.get("data", {}) + + return { + "total": data.get("total", 0), + "page_no": data.get("pageNo", 1), + "page_size": data.get("pageSize", 15), + "pages": data.get("pages", 0), + "empty": data.get("empty", True), + "has_next": data.get("hasNext", False), + "has_previous": data.get("hasPrevious", False) + } + except Exception as e: + logger.warning(f"提取分页信息失败: {str(e)}") + return { + "total": 0, + "page_no": 1, + "page_size": 15, + "pages": 0, + "empty": True, + "has_next": False, + "has_previous": False + } + + +class SensitiveWordChecker: + """敏感词检查器""" + + @staticmethod + def parse_check_response(response_data: Dict[str, Any]) -> bool: + """ + 解析敏感词检查响应 + + Args: + response_data: 检查响应数据 + + Returns: + bool: 检查是否通过 + """ + try: + if not isinstance(response_data, dict): + logger.warning("敏感词检查响应格式无效") + return False + + success = response_data.get("success", False) + if not success: + message = response_data.get("message", "未知错误") + logger.warning(f"敏感词检查失败: {message}") + return False + + return True + + except Exception as e: + logger.error(f"解析敏感词检查响应失败: {str(e)}") + return False + + +class ErrorResponseParser: + """错误响应解析器""" + + @staticmethod + def parse_error(response_data: Dict[str, Any]) -> str: + """ + 解析错误响应 + + Args: + response_data: 错误响应数据 + + Returns: + str: 错误信息 + """ + try: + if not isinstance(response_data, dict): + return "响应格式无效" + + # 尝试提取错误信息 + error_msg = response_data.get("message") or response_data.get("msg") + if error_msg: + return str(error_msg) + + # 检查状态码 + errcode = response_data.get("errcode") + if errcode: + return f"错误码: {errcode}" + + return "未知错误" + + except Exception as e: + return f"解析错误响应失败: {str(e)}" diff --git a/gx_gp_monitor/crawler/spider.py b/gx_gp_monitor/crawler/spider.py new file mode 100644 index 0000000..e6ca2b0 --- /dev/null +++ b/gx_gp_monitor/crawler/spider.py @@ -0,0 +1,458 @@ +""" +爬虫核心模块 +实现广西政府采购网公告的智能爬取功能 +""" + +import time +import random +import json +from typing import List, Dict, Any, Optional, Tuple +from datetime import datetime +from urllib.parse import urljoin +import requests +from fake_useragent import UserAgent + +try: + # 尝试相对导入 + from ..core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus + from ..core.config_manager import get_config + from ..core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error + from ..core.reliability import ( + retry_on_exception, RetryConfig, session_with_retry, + TimeoutConfig, safe_execute, check_system_health + ) + from .parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser +except ImportError: + # 尝试绝对导入 + from core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus + from core.config_manager import get_config + from core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error + from core.reliability import ( + retry_on_exception, RetryConfig, session_with_retry, + TimeoutConfig, safe_execute, check_system_health + ) + from crawler.parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser + + +logger = get_logger(__name__) + + +class GXGPSpider: + """广西政府采购网爬虫""" + + def __init__(self): + self.config = get_config() + self.ua = UserAgent() + + # API端点 + self.base_url = self.config.crawler.base_url + self.announcement_api = urljoin(self.base_url, "/portal/category") + self.sensitive_check_api = urljoin(self.base_url, "/portal/sensitiveWords/check") + + # 会话管理 + self.session = None + + # 统计信息 + self.request_count = 0 + self.error_count = 0 + + def init_session(self): + """初始化会话""" + if self.session is None: + timeout_config = TimeoutConfig( + connect_timeout=self.config.crawler.timeout, + read_timeout=self.config.crawler.timeout + ) + + retry_config = RetryConfig( + max_retries=self.config.crawler.max_retries, + initial_delay=self.config.crawler.retry_delay, + max_delay=self.config.crawler.max_retry_delay, + backoff_factor=self.config.crawler.backoff_factor + ) + + self.session = requests.Session() + + # 配置重试和超时 + adapter = requests.adapters.HTTPAdapter( + pool_connections=10, + pool_maxsize=20, + max_retries=0 # 我们使用自己的重试逻辑 + ) + self.session.mount('http://', adapter) + self.session.mount('https://', adapter) + + # 设置默认超时 + self.session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout) + + return self.session + + def close_session(self): + """关闭会话""" + if self.session: + self.session.close() + self.session = None + + def get_random_user_agent(self) -> str: + """获取随机User-Agent""" + try: + return self.ua.random + except: + # fallback到配置的user agents + return random.choice(self.config.crawler.user_agents) + + def get_random_proxy(self) -> Optional[Dict[str, str]]: + """获取随机代理""" + if not self.config.crawler.proxies: + return None + + proxy = random.choice(self.config.crawler.proxies) + return { + "http": proxy, + "https": proxy + } + + def check_sensitive_words(self, payload: Dict[str, Any], + category_code: str, childrencode: str) -> bool: + """ + 执行敏感词检查 + + Args: + payload: 请求参数 + category_code: 分类代码 + childrencode: 子分类代码 + + Returns: + bool: 检查是否通过 + """ + try: + session = self.init_session() + user_agent = self.get_random_user_agent() + + headers = { + "User-Agent": user_agent, + "Content-Type": "application/json;charset=UTF-8", + "Origin": self.base_url, + "Referer": f"{self.base_url}/site/category?parentId={category_code}&childrenCode={childrencode}", + "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" + } + + proxies = self.get_random_proxy() + + logger.debug(f"执行敏感词检查: {category_code}/{childrencode}") + + response = session.post( + self.sensitive_check_api, + json=payload, + headers=headers, + proxies=proxies, + timeout=self.session.timeout + ) + + self.request_count += 1 + + if response.status_code == 200: + result = response.json() + return SensitiveWordChecker.parse_check_response(result) + else: + logger.warning(f"敏感词检查请求失败, 状态码: {response.status_code}") + return False + + except Exception as e: + logger.error(f"敏感词检查异常: {str(e)}") + self.error_count += 1 + return False + + def fetch_announcements_page(self, source: AnnouncementSource, + page_no: int = 1) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + """ + 获取公告列表页数据 + + Args: + source: 公告来源 + page_no: 页码 + + Returns: + Tuple[Optional[Dict[str, Any]], Optional[str]]: (响应数据, 错误信息) + """ + try: + session = self.init_session() + + # 构建请求参数 + payload = { + "keyword": "", # 关键词筛选，我们在筛选模块处理 + "publishDateBegin": self.config.crawler.start_date or "", + "publishDateEnd": self.config.crawler.end_date or "", + "pageNo": page_no, + "pageSize": self.config.crawler.page_size, + "categoryCode": source.code, + "_t": int(time.time() * 1000) + } + + # 先执行敏感词检查 + if not self.check_sensitive_words(payload, str(source.category_id), source.code): + return None, "敏感词检查失败" + + # 执行公告数据请求 + user_agent = self.get_random_user_agent() + headers = { + "User-Agent": user_agent, + "Content-Type": "application/json;charset=UTF-8", + "Origin": self.base_url, + "Referer": f"{self.base_url}/site/category?parentId={source.category_id}&childrenCode={source.code}", + "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" + } + + proxies = self.get_random_proxy() + + logger.debug(f"请求公告数据: {source.name} 第{page_no}页") + + # 添加请求间延迟 + if page_no > 1: + delay = random.uniform( + self.config.crawler.request_delay, + self.config.crawler.request_delay_max + ) + time.sleep(delay) + + response = session.post( + self.announcement_api, + json=payload, + headers=headers, + proxies=proxies, + timeout=self.session.timeout + ) + + self.request_count += 1 + + if response.status_code == 200: + data = response.json() + if data.get("success", False): + return data, None + else: + error_msg = ErrorResponseParser.parse_error(data) + return None, f"API返回失败: {error_msg}" + else: + return None, f"请求失败, 状态码: {response.status_code}" + + except requests.exceptions.Timeout as e: + self.error_count += 1 + return None, f"请求超时: {str(e)}" + except requests.exceptions.ProxyError as e: + self.error_count += 1 + return None, f"代理错误: {str(e)}" + except Exception as e: + self.error_count += 1 + logger.error(f"获取公告数据异常: {str(e)}") + return None, f"请求异常: {str(e)}" + + def crawl_source(self, source: AnnouncementSource, + max_pages: Optional[int] = None) -> CrawlResult: + """ + 爬取单个来源的公告 + + Args: + source: 公告来源 + max_pages: 最大页数限制 + + Returns: + CrawlResult: 爬取结果 + """ + if max_pages is None: + max_pages = self.config.crawler.max_pages + + log_crawl_start(source.name) + + start_time = datetime.now() + result = CrawlResult( + source=source, + status=CrawlStatus.RUNNING, + crawled_at=start_time + ) + + try: + page = 1 + all_announcements = [] + + while page <= max_pages: + # 获取页面数据 + response_data, error_msg = self.fetch_announcements_page(source, page) + + if error_msg: + logger.warning(f"{source.name} 第{page}页获取失败: {error_msg}") + result.status = CrawlStatus.FAILED + result.error_message = error_msg + break + + if not response_data: + logger.info(f"{source.name} 第{page}页无数据") + break + + # 解析分页信息 + pagination = AnnouncementParser.extract_pagination_info(response_data) + result.total_count = pagination["total"] + + # 解析公告数据 + announcements = AnnouncementParser.parse_api_response( + response_data, source, start_time) + + if not announcements: + logger.info(f"{source.name} 第{page}页解析到0条公告") + break + + all_announcements.extend(announcements) + + # 检查是否还有下一页 + if not pagination["has_next"] or pagination["empty"]: + break + + page += 1 + + # 更新结果 + result.announcements = all_announcements + result.new_count = len(all_announcements) # 这里的新增数需要在筛选后确定 + + if result.status != CrawlStatus.FAILED: + result.status = CrawlStatus.SUCCESS + + duration = (datetime.now() - start_time).total_seconds() + result.duration = duration + + log_crawl_success(source.name, len(all_announcements), duration) + + except Exception as e: + duration = (datetime.now() - start_time).total_seconds() + result.duration = duration + result.status = CrawlStatus.FAILED + result.error_message = str(e) + + log_crawl_error(source.name, str(e)) + + return result + + @retry_on_exception(RetryConfig(max_retries=2)) + def crawl_all_sources(self, sources: Optional[List[AnnouncementSource]] = None) -> List[CrawlResult]: + """ + 爬取所有来源的公告 + + Args: + sources: 指定的来源列表，如果为None则使用配置中的所有来源 + + Returns: + List[CrawlResult]: 所有来源的爬取结果 + """ + # 系统健康检查 + if not check_system_health(): + logger.error("系统健康检查失败，跳过爬取") + return [] + + if sources is None: + sources = self._load_sources_from_config() + + logger.info(f"开始爬取 {len(sources)} 个公告来源") + + results = [] + + for source in sources: + try: + result = self.crawl_source(source) + results.append(result) + + # 检查是否需要暂停 + if result.status == CrawlStatus.FAILED: + logger.warning(f"来源 {source.name} 爬取失败，继续下一个来源") + continue + + except Exception as e: + logger.error(f"爬取来源 {source.name} 时发生未预期错误: {str(e)}") + # 创建失败结果 + failed_result = CrawlResult( + source=source, + status=CrawlStatus.FAILED, + error_message=str(e), + crawled_at=datetime.now() + ) + results.append(failed_result) + + # 统计总结果 + total_announcements = sum(len(r.announcements) for r in results if r.status == CrawlStatus.SUCCESS) + success_count = sum(1 for r in results if r.status == CrawlStatus.SUCCESS) + failed_count = len(results) - success_count + + logger.info( + f"爬取完成: 共处理 {len(results)} 个来源，" + f"成功 {success_count} 个，失败 {failed_count} 个，" + f"获取 {total_announcements} 条公告" + ) + + return results + + def _load_sources_from_config(self) -> List[AnnouncementSource]: + """从配置加载公告来源""" + sources = [] + + for code, source_config in self.config.sources.items(): + try: + source = AnnouncementSource( + code=code, + category_id=source_config["category_id"], + name=source_config["name"], + type=AnnouncementType(source_config["type"]) + ) + sources.append(source) + except Exception as e: + logger.warning(f"加载来源配置失败 {code}: {str(e)}") + continue + + return sources + + def get_stats(self) -> Dict[str, Any]: + """获取爬虫统计信息""" + return { + "request_count": self.request_count, + "error_count": self.error_count, + "error_rate": self.error_count / max(self.request_count, 1), + "session_active": self.session is not None + } + + def reset_stats(self): + """重置统计信息""" + self.request_count = 0 + self.error_count = 0 + + +def create_spider() -> GXGPSpider: + """ + 创建爬虫实例 + + Returns: + GXGPSpider: 爬虫实例 + """ + return GXGPSpider() + + +def crawl_announcements(keywords: Optional[List[str]] = None, + sources: Optional[List[str]] = None) -> List[CrawlResult]: + """ + 便捷函数：爬取公告 + + Args: + keywords: 关键词过滤（暂时未使用，在筛选模块处理） + sources: 来源代码列表 + + Returns: + List[CrawlResult]: 爬取结果 + """ + spider = create_spider() + + try: + # 过滤来源 + if sources: + all_sources = spider._load_sources_from_config() + filtered_sources = [s for s in all_sources if s.code in sources] + else: + filtered_sources = None + + return spider.crawl_all_sources(filtered_sources) + finally: + spider.close_session() diff --git a/gx_gp_monitor/filters/__init__.py b/gx_gp_monitor/filters/__init__.py new file mode 100644 index 0000000..3c23a98 --- /dev/null +++ b/gx_gp_monitor/filters/__init__.py @@ -0,0 +1 @@ +"""筛选模块""" diff --git a/gx_gp_monitor/filters/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/filters/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..77102fe Binary files /dev/null and b/gx_gp_monitor/filters/__pycache__/__init__.cpython-313.pyc differ diff --git a/gx_gp_monitor/filters/__pycache__/filters.cpython-313.pyc b/gx_gp_monitor/filters/__pycache__/filters.cpython-313.pyc new file mode 100644 index 0000000..8ea64c8 Binary files /dev/null and b/gx_gp_monitor/filters/__pycache__/filters.cpython-313.pyc differ diff --git a/gx_gp_monitor/filters/filters.py b/gx_gp_monitor/filters/filters.py new file mode 100644 index 0000000..d815381 --- /dev/null +++ b/gx_gp_monitor/filters/filters.py @@ -0,0 +1,471 @@ +""" +智能筛选模块 +提供关键词过滤、日期范围筛选、自动去重等功能 +""" + +from typing import List, Optional, Dict, Any, Set +from datetime import datetime, date +from dataclasses import dataclass +import re + +try: + from ..core.models import Announcement + from ..core.logger import get_logger + from ..core.database import get_database_manager +except ImportError: + from core.models import Announcement + from core.logger import get_logger + from core.database import get_database_manager + + +logger = get_logger(__name__) + + +@dataclass +class FilterCriteria: + """筛选条件""" + keywords: List[str] = None # 关键词列表 + start_date: Optional[date] = None # 开始日期 + end_date: Optional[date] = None # 结束日期 + sources: List[str] = None # 来源代码列表 + exclude_today: bool = False # 排除今日公告 + case_sensitive: bool = False # 关键词匹配是否区分大小写 + fuzzy_match: bool = False # 是否启用模糊匹配 + + def __post_init__(self): + if self.keywords is None: + self.keywords = [] + if self.sources is None: + self.sources = [] + + +@dataclass +class FilterResult: + """筛选结果""" + total_count: int = 0 # 总数 + filtered_count: int = 0 # 筛选后数量 + keyword_filtered: int = 0 # 关键词筛选数量 + date_filtered: int = 0 # 日期筛选数量 + duplicate_filtered: int = 0 # 去重筛选数量 + source_filtered: int = 0 # 来源筛选数量 + today_excluded: int = 0 # 排除今日数量 + + +class KeywordFilter: + """关键词筛选器""" + + def __init__(self, case_sensitive: bool = False, fuzzy_match: bool = False): + """ + 初始化关键词筛选器 + + Args: + case_sensitive: 是否区分大小写 + fuzzy_match: 是否启用模糊匹配 + """ + self.case_sensitive = case_sensitive + self.fuzzy_match = fuzzy_match + self._compiled_patterns = {} + + def _compile_patterns(self, keywords: List[str]): + """编译关键词模式""" + flags = 0 if self.case_sensitive else re.IGNORECASE + + for keyword in keywords: + if keyword not in self._compiled_patterns: + if self.fuzzy_match: + # 简单的模糊匹配：将关键词中的空格替换为灵活匹配 + pattern = r'.*'.join(re.escape(word) for word in keyword.split()) + else: + pattern = re.escape(keyword) + + self._compiled_patterns[keyword] = re.compile(pattern, flags) + + def matches(self, text: str, keywords: List[str]) -> bool: + """ + 检查文本是否匹配关键词 + + Args: + text: 要检查的文本 + keywords: 关键词列表 + + Returns: + bool: 是否匹配 + """ + if not keywords or not text: + return True # 没有关键词或文本时，认为匹配 + + if not self.fuzzy_match: + # 精确匹配 + search_text = text if self.case_sensitive else text.lower() + search_keywords = keywords if self.case_sensitive else [k.lower() for k in keywords] + + for keyword in search_keywords: + if keyword in search_text: + return True + return False + else: + # 正则匹配 + self._compile_patterns(keywords) + for keyword in keywords: + pattern = self._compiled_patterns[keyword] + if pattern.search(text): + return True + return False + + def filter_announcements(self, announcements: List[Announcement], + keywords: List[str]) -> List[Announcement]: + """ + 筛选匹配关键词的公告 + + Args: + announcements: 公告列表 + keywords: 关键词列表 + + Returns: + List[Announcement]: 筛选后的公告列表 + """ + if not keywords: + # 没有关键词时，所有公告都匹配 + for announcement in announcements: + announcement.keyword_matched = True + return announcements + + filtered = [] + for announcement in announcements: + # 构建搜索文本 + search_text = f"{announcement.title} {announcement.purchase_name}" + + if self.matches(search_text, keywords): + announcement.keyword_matched = True + filtered.append(announcement) + else: + announcement.keyword_matched = False + + logger.info(f"关键词筛选: {len(announcements)} -> {len(filtered)} 条公告") + return filtered + + +class DateFilter: + """日期筛选器""" + + def filter_announcements(self, announcements: List[Announcement], + start_date: Optional[date] = None, + end_date: Optional[date] = None, + exclude_today: bool = False) -> List[Announcement]: + """ + 筛选日期范围内的公告 + + Args: + announcements: 公告列表 + start_date: 开始日期 + end_date: 结束日期 + exclude_today: 是否排除今日公告 + + Returns: + List[Announcement]: 筛选后的公告列表 + """ + if not start_date and not end_date and not exclude_today: + # 没有日期限制时，所有公告都通过 + for announcement in announcements: + announcement.date_filtered = True + return announcements + + filtered = [] + today = date.today() + + for announcement in announcements: + if not announcement.publish_date: + # 没有发布日期的公告不通过筛选 + announcement.date_filtered = False + continue + + publish_date = announcement.publish_date.date() + + # 检查日期范围 + date_in_range = True + + if start_date and publish_date < start_date: + date_in_range = False + + if end_date and publish_date > end_date: + date_in_range = False + + # 检查是否排除今日 + if exclude_today and publish_date == today: + date_in_range = False + + announcement.date_filtered = date_in_range + + if date_in_range: + filtered.append(announcement) + + logger.info(f"日期筛选: {len(announcements)} -> {len(filtered)} 条公告") + return filtered + + +class DuplicateFilter: + """去重筛选器""" + + def __init__(self, use_database: bool = True): + """ + 初始化去重筛选器 + + Args: + use_database: 是否使用数据库检查重复 + """ + self.use_database = use_database + self.db_manager = get_database_manager() if use_database else None + + def filter_announcements(self, announcements: List[Announcement]) -> List[Announcement]: + """ + 去除重复的公告 + + Args: + announcements: 公告列表 + + Returns: + List[Announcement]: 去重后的公告列表 + """ + if not announcements: + return announcements + + seen_hashes = set() + filtered = [] + + for announcement in announcements: + # 生成内容哈希 + if not announcement.content_hash: + announcement.generate_content_hash() + + content_hash = announcement.content_hash + + # 检查是否已存在 + is_duplicate = False + + if self.use_database and self.db_manager: + # 数据库检查 + is_duplicate = self.db_manager.is_announcement_exists(content_hash) + else: + # 内存检查 + if content_hash in seen_hashes: + is_duplicate = True + seen_hashes.add(content_hash) + + if not is_duplicate: + filtered.append(announcement) + # 注意：这里不设置is_new标志，因为这是在爬取后的筛选阶段 + + duplicate_count = len(announcements) - len(filtered) + if duplicate_count > 0: + logger.info(f"去重筛选: 移除了 {duplicate_count} 条重复公告") + + return filtered + + +class SourceFilter: + """来源筛选器""" + + def filter_announcements(self, announcements: List[Announcement], + allowed_sources: List[str]) -> List[Announcement]: + """ + 筛选指定来源的公告 + + Args: + announcements: 公告列表 + allowed_sources: 允许的来源代码列表 + + Returns: + List[Announcement]: 筛选后的公告列表 + """ + if not allowed_sources: + return announcements + + filtered = [] + for announcement in announcements: + if announcement.source_code in allowed_sources: + filtered.append(announcement) + + logger.info(f"来源筛选: {len(announcements)} -> {len(filtered)} 条公告") + return filtered + + +class AnnouncementFilter: + """公告智能筛选器""" + + def __init__(self, criteria: FilterCriteria = None): + """ + 初始化公告筛选器 + + Args: + criteria: 筛选条件 + """ + self.criteria = criteria or FilterCriteria() + + # 初始化各个筛选器 + self.keyword_filter = KeywordFilter( + case_sensitive=self.criteria.case_sensitive, + fuzzy_match=self.criteria.fuzzy_match + ) + self.date_filter = DateFilter() + self.duplicate_filter = DuplicateFilter() + self.source_filter = SourceFilter() + + def filter(self, announcements: List[Announcement]) -> tuple[List[Announcement], FilterResult]: + """ + 执行完整的筛选流程 + + Args: + announcements: 原始公告列表 + + Returns: + tuple: (筛选后的公告列表, 筛选结果统计) + """ + result = FilterResult() + result.total_count = len(announcements) + + # 1. 来源筛选 + if self.criteria.sources: + announcements = self.source_filter.filter_announcements( + announcements, self.criteria.sources) + result.source_filtered = result.total_count - len(announcements) + + # 2. 去重筛选 + announcements = self.duplicate_filter.filter_announcements(announcements) + result.duplicate_filtered = result.total_count - len(announcements) - result.source_filtered + + # 3. 日期筛选 + announcements = self.date_filter.filter_announcements( + announcements, + self.criteria.start_date, + self.criteria.end_date, + self.criteria.exclude_today + ) + result.date_filtered = result.total_count - len(announcements) - result.source_filtered - result.duplicate_filtered + + # 4. 关键词筛选 + if self.criteria.keywords: + announcements = self.keyword_filter.filter_announcements( + announcements, self.criteria.keywords) + result.keyword_filtered = result.total_count - len(announcements) - result.source_filtered - result.duplicate_filtered - result.date_filtered + + result.filtered_count = len(announcements) + + logger.info( + f"筛选完成: 总数 {result.total_count} -> 筛选后 {result.filtered_count} " + f"(关键词: {result.keyword_filtered}, 日期: {result.date_filtered}, " + f"去重: {result.duplicate_filtered}, 来源: {result.source_filtered})" + ) + + return announcements, result + + def quick_filter(self, announcement: Announcement) -> bool: + """ + 快速筛选单个公告（用于实时筛选） + + Args: + announcement: 公告对象 + + Returns: + bool: 是否通过筛选 + """ + # 检查来源 + if self.criteria.sources and announcement.source_code not in self.criteria.sources: + return False + + # 检查日期 + if not announcement.in_date_range( + self.criteria.start_date.isoformat() if self.criteria.start_date else None, + self.criteria.end_date.isoformat() if self.criteria.end_date else None + ): + return False + + # 检查关键词 + if self.criteria.keywords: + search_text = f"{announcement.title} {announcement.purchase_name}" + if not self.keyword_filter.matches(search_text, self.criteria.keywords): + return False + + return True + + def update_criteria(self, criteria: FilterCriteria): + """ + 更新筛选条件 + + Args: + criteria: 新的筛选条件 + """ + self.criteria = criteria + + # 重新初始化筛选器 + self.keyword_filter = KeywordFilter( + case_sensitive=self.criteria.case_sensitive, + fuzzy_match=self.criteria.fuzzy_match + ) + + +def create_default_filter(keywords: List[str] = None, + start_date: str = None, + end_date: str = None) -> AnnouncementFilter: + """ + 创建默认筛选器 + + Args: + keywords: 关键词列表 + start_date: 开始日期字符串 (YYYY-MM-DD) + end_date: 结束日期字符串 (YYYY-MM-DD) + + Returns: + AnnouncementFilter: 筛选器实例 + """ + criteria = FilterCriteria() + + if keywords: + criteria.keywords = keywords + + if start_date: + try: + criteria.start_date = datetime.fromisoformat(start_date).date() + except ValueError: + logger.warning(f"无效的开始日期格式: {start_date}") + + if end_date: + try: + criteria.end_date = datetime.fromisoformat(end_date).date() + except ValueError: + logger.warning(f"无效的结束日期格式: {end_date}") + + return AnnouncementFilter(criteria) + + +def filter_from_config() -> AnnouncementFilter: + """ + 从配置文件创建筛选器 + + Returns: + AnnouncementFilter: 配置化的筛选器 + """ + try: + from ..core.config_manager import get_config + except ImportError: + from core.config_manager import get_config + + config = get_config() + + criteria = FilterCriteria() + criteria.keywords = config.crawler.keyword + criteria.sources = list(config.sources.keys()) # 默认包含所有来源 + + # 日期范围 + if config.crawler.start_date: + try: + criteria.start_date = datetime.fromisoformat(config.crawler.start_date).date() + except ValueError: + pass + + if config.crawler.end_date: + try: + criteria.end_date = datetime.fromisoformat(config.crawler.end_date).date() + except ValueError: + pass + + return AnnouncementFilter(criteria) diff --git a/gx_gp_monitor/logs/gx_gp_monitor.log b/gx_gp_monitor/logs/gx_gp_monitor.log new file mode 100644 index 0000000..f655a34 --- /dev/null +++ b/gx_gp_monitor/logs/gx_gp_monitor.log @@ -0,0 +1,799 @@ +2026-01-07 17:16:51 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:16:51 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:16:51 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:16:52 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:16:52 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:16:52 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:16:52 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:16:52 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:16:52 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:16:52 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:16:52 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:20:31 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:20:31 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:20:31 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:20:31 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:20:31 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:20:31 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:20:31 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:20:31 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:20:31 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:20:31 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:20:31 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:21:02 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:21:02 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:21:02 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:21:02 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:21:02 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:21:02 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:21:02 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:21:02 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:21:02 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:21:02 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:21:02 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:22:10 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:22:10 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:22:10 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:22:10 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:22:10 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:22:10 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:22:10 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:22:10 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:22:10 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:22:10 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:22:10 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:22:44 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:22:44 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:22:44 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:22:44 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:22:44 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:22:44 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:22:44 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:22:44 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:22:44 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:22:44 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:22:44 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:22:46 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:22:46 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:22:46 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:22:47 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:22:47 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:22:47 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:22:47 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:22:47 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:22:47 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:22:47 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:22:47 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 采购公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 结果公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 合同公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 更正公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 招标文件预公示 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 单一来源公示 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 电子卖场公示 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 履约验收公示 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 工程类公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 框架协议征集公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 框架协议入围结果公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 框架协议成交结果汇总公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 采购意向公开 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 0 个，失败 13 个，获取 0 条公告 +2026-01-07 17:22:47 - __main__ - INFO - 爬取到 0 条原始公告 +2026-01-07 17:22:47 - __main__ - ERROR - 爬取任务执行失败: attempted relative import beyond top-level package +2026-01-07 17:22:47 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:22:47 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:23:10 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:23:10 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:23:10 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:23:10 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:23:10 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:23:10 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:23:10 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:23:10 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:23:10 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:23:10 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:23:10 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 采购公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 结果公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 合同公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 更正公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 招标文件预公示 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 单一来源公示 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 电子卖场公示 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 履约验收公示 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 工程类公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 框架协议征集公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 框架协议入围结果公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 框架协议成交结果汇总公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 采购意向公开 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 0 个，失败 13 个，获取 0 条公告 +2026-01-07 17:23:10 - __main__ - INFO - 爬取到 0 条原始公告 +2026-01-07 17:23:10 - __main__ - ERROR - 爬取任务执行失败: attempted relative import beyond top-level package +2026-01-07 17:23:10 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:23:10 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:23:34 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:23:34 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:23:34 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:23:35 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:23:35 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:23:35 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:23:35 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:23:35 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:23:35 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:23:35 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:23:35 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:23:35 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:24:05 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:05 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:05 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:05 - crawler.parsers - INFO - 成功解析 97/100 条公告记录 +2026-01-07 17:24:05 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 97 条公告，耗时 30.46秒 +2026-01-07 17:24:05 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:24:05 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:05 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:24:05 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.23秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 78/100 条公告记录 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 78 条公告，耗时 0.17秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 0/100 条公告记录 +2026-01-07 17:24:06 - crawler.spider - INFO - 单一来源公示 第1页解析到0条公告 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 0 条公告，耗时 0.18秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.31秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:24:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.16秒 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:24:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.27秒 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - INFO - 成功解析 62/100 条公告记录 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 62 条公告，耗时 0.20秒 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - INFO - 成功解析 13/100 条公告记录 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 13 条公告，耗时 0.17秒 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - INFO - 成功解析 0/100 条公告记录 +2026-01-07 17:24:08 - crawler.spider - INFO - 框架协议成交结果汇总公告 第1页解析到0条公告 +2026-01-07 17:24:08 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 0 条公告，耗时 0.29秒 +2026-01-07 17:24:08 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:24:08 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:08 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:24:08 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 950 条公告 +2026-01-07 17:24:08 - __main__ - INFO - 爬取到 950 条原始公告 +2026-01-07 17:24:08 - __main__ - ERROR - 爬取任务执行失败: attempted relative import beyond top-level package +2026-01-07 17:24:08 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:24:08 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:24:17 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:24:17 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:24:17 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:24:17 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:24:17 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:24:17 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:24:17 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:24:17 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:24:17 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:24:17 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:24:17 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:26:34 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:26:34 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:26:34 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:26:34 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:26:34 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:26:34 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:26:34 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:26:34 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:26:34 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:26:34 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:26:34 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:26:34 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:26:34 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:34 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 100 条公告，耗时 0.48秒 +2026-01-07 17:26:34 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.23秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 100 条公告，耗时 0.20秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:26:36 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.40秒 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:26:36 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.22秒 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:26:36 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.27秒 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:26:36 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:26:37 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 100 条公告，耗时 0.25秒 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:26:37 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:26:37 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:26:37 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 1300 条公告 +2026-01-07 17:26:37 - __main__ - INFO - 爬取到 1300 条原始公告 +2026-01-07 17:26:37 - __main__ - ERROR - 爬取任务执行失败: attempted relative import beyond top-level package +2026-01-07 17:26:37 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:26:37 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:26:57 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:26:57 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:26:57 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:26:58 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:26:58 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:26:58 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:26:58 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:26:58 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:26:58 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:26:58 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:26:58 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:26:58 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 100 条公告，耗时 0.37秒 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:26:58 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.23秒 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:26:58 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.20秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 100 条公告，耗时 0.16秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.38秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:27:00 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.25秒 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:27:00 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:27:00 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:27:00 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:27:01 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:01 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.25秒 +2026-01-07 17:27:01 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 1300 条公告 +2026-01-07 17:27:01 - __main__ - INFO - 爬取到 1300 条原始公告 +2026-01-07 17:27:01 - filters.filters - INFO - 来源筛选: 1300 -> 1300 条公告 +2026-01-07 17:27:01 - filters.filters - INFO - 关键词筛选: 1300 -> 18 条公告 +2026-01-07 17:27:01 - filters.filters - INFO - 筛选完成: 总数 1300 -> 筛选后 18 (关键词: 1282, 日期: 0, 去重: 0, 来源: 0) +2026-01-07 17:27:01 - __main__ - INFO - 筛选后剩余 18 条公告 +2026-01-07 17:27:01 - storage.postgresql - INFO - 开始保存 18 条公告到数据库 +2026-01-07 17:27:01 - core.database - INFO - 批量保存公告完成，成功保存 1 条 +2026-01-07 17:27:01 - storage.postgresql - INFO - 成功保存 1 条公告到数据库 +2026-01-07 17:27:01 - storage.md_generator - INFO - Markdown文件已保存到: onu.md (共 18 条公告) +2026-01-07 17:27:01 - notification.wechat - INFO - 企业微信服务初始化完成 +2026-01-07 17:27:01 - notification.wechat - INFO - 成功获取企业微信访问令牌 +2026-01-07 17:27:01 - notification.wechat - INFO - Markdown消息发送成功 +2026-01-07 17:27:01 - __main__ - INFO - 爬取任务完成: {'success': True, 'total_crawled': 1300, 'filtered': 18, 'saved': 1, 'markdown_generated': True, 'notification_sent': True, 'filter_stats': {'keyword_filtered': 1282, 'date_filtered': 0, 'duplicate_filtered': 0, 'source_filtered': 0}} +2026-01-07 17:27:01 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:27:01 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:29:05 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:29:05 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:29:05 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:29:06 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:29:06 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:29:06 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:29:06 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:29:06 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:29:06 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:29:06 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:29:06 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:31:58 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:31:58 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:31:58 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:31:58 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:31:58 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:31:58 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:31:58 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:31:58 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:31:58 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:34:32 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:34:32 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:34:32 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:34:32 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:34:32 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:34:32 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:34:32 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:34:32 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:34:32 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:34:35 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:34:35 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:34:35 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:34:35 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:34:35 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:34:35 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:34:35 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:34:35 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:34:35 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:34:35 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:34:35 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:34:35 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:35:05 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:05 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 100 条公告，耗时 30.44秒 +2026-01-07 17:35:05 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:35:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.26秒 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:35:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.20秒 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:35:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.23秒 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:35:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.34秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:35:08 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:35:08 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 100 条公告，耗时 0.25秒 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:35:08 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:35:08 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 1300 条公告 +2026-01-07 17:35:08 - __main__ - INFO - 爬取到 1300 条原始公告 +2026-01-07 17:35:08 - filters.filters - INFO - 来源筛选: 1300 -> 1300 条公告 +2026-01-07 17:35:08 - filters.filters - INFO - 去重筛选: 移除了 18 条重复公告 +2026-01-07 17:35:08 - filters.filters - INFO - 关键词筛选: 1282 -> 0 条公告 +2026-01-07 17:35:08 - filters.filters - INFO - 筛选完成: 总数 1300 -> 筛选后 0 (关键词: 1282, 日期: 0, 去重: 18, 来源: 0) +2026-01-07 17:35:08 - __main__ - INFO - 筛选后剩余 0 条公告 +2026-01-07 17:35:08 - storage.md_generator - INFO - Markdown文件已保存到: onu.md (共 0 条公告) +2026-01-07 17:35:08 - __main__ - INFO - 爬取任务完成: {'success': True, 'total_crawled': 1300, 'filtered': 0, 'saved': 0, 'markdown_generated': True, 'notification_sent': False, 'filter_stats': {'keyword_filtered': 1282, 'date_filtered': 0, 'duplicate_filtered': 18, 'source_filtered': 0}} +2026-01-07 17:35:38 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:35:38 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:35:38 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:35:38 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:35:38 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:35:38 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:35:38 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:35:38 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:35:38 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:35:38 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:35:38 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:35:38 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:36:09 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 100 条公告，耗时 30.49秒 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:36:09 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.20秒 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:36:09 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:36:09 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.27秒 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:36:10 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 100 条公告，耗时 0.28秒 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:36:10 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:36:10 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.36秒 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.27秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 100 条公告，耗时 0.15秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 100 条公告，耗时 0.24秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 100 条公告，耗时 0.16秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:36:11 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 1300 条公告 +2026-01-07 17:36:11 - __main__ - INFO - 爬取到 1300 条原始公告 +2026-01-07 17:36:11 - filters.filters - INFO - 来源筛选: 1300 -> 1300 条公告 +2026-01-07 17:36:12 - filters.filters - INFO - 去重筛选: 移除了 18 条重复公告 +2026-01-07 17:36:12 - filters.filters - INFO - 关键词筛选: 1282 -> 0 条公告 +2026-01-07 17:36:12 - filters.filters - INFO - 筛选完成: 总数 1300 -> 筛选后 0 (关键词: 1282, 日期: 0, 去重: 18, 来源: 0) +2026-01-07 17:36:12 - __main__ - INFO - 筛选后剩余 0 条公告 +2026-01-07 17:36:12 - storage.md_generator - INFO - Markdown文件已保存到: onu.md (共 0 条公告) +2026-01-07 17:36:12 - __main__ - INFO - 爬取任务完成: {'success': True, 'total_crawled': 1300, 'filtered': 0, 'saved': 0, 'markdown_generated': True, 'notification_sent': False, 'filter_stats': {'keyword_filtered': 1282, 'date_filtered': 0, 'duplicate_filtered': 18, 'source_filtered': 0}} diff --git a/gx_gp_monitor/main.py b/gx_gp_monitor/main.py new file mode 100644 index 0000000..ab573c8 --- /dev/null +++ b/gx_gp_monitor/main.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +""" +广西政府采购网公告监控系统主程序 +广西政府采购网公告爬取和监控的智能系统 +""" + +import sys +import argparse +import signal +from pathlib import Path + +# 添加项目根目录到Python路径 +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) + +try: + # 尝试相对导入 + from .core.config_manager import load_config, get_config + from .core.logger import init_logger, get_logger + from .core.reliability import check_system_health + from .crawler.spider import crawl_announcements + from .filters.filters import filter_from_config + from .storage.postgresql import init_storage, save_announcements_to_storage, cleanup_storage + from .storage.md_generator import generate_onu_md + from .notification.wechat import send_announcements_notification, send_system_notification +except ImportError: + try: + # 尝试绝对导入（直接运行脚本时） + from core.config_manager import load_config, get_config + from core.logger import init_logger, get_logger + from core.reliability import check_system_health + from crawler.spider import crawl_announcements + from filters.filters import filter_from_config + from storage.postgresql import init_storage, save_announcements_to_storage, cleanup_storage + from storage.md_generator import generate_onu_md + from notification.wechat import send_announcements_notification, send_system_notification + except ImportError as e: + print(f"导入错误: {e}", file=sys.stderr) + print("请确保依赖已正确安装: pip install -r requirements.txt", file=sys.stderr) + sys.exit(1) + + +logger = get_logger(__name__) + + +class GXGPMonitorApp: + """广西政府采购网监控系统应用""" + + def __init__(self): + self.config = None + self.logger = None + self.running = False + + def initialize(self, config_file: str = None): + """初始化应用""" + try: + # 加载配置 + self.config = load_config(config_file) + + # 初始化日志 + self.logger = init_logger(config=self.config) + + logger.info("=== 广西政府采购网公告监控系统启动 ===") + logger.info(f"版本: 1.0.0") + logger.info(f"配置文件: {config_file or '默认配置'}") + + # 系统健康检查 + if not check_system_health(): + logger.warning("系统健康检查失败，但继续运行") + + # 初始化存储 + init_storage() + + logger.info("应用初始化完成") + return True + + except Exception as e: + print(f"应用初始化失败: {str(e)}", file=sys.stderr) + return False + + def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None): + """执行爬取任务""" + try: + logger.info("开始执行爬取任务") + + # 执行爬取 + crawl_results = crawl_announcements(keywords, sources) + + if not crawl_results: + logger.info("爬取完成：无数据") + return {"success": True, "results": []} + + # 收集所有公告 + all_announcements = [] + for result in crawl_results: + if result.announcements: + all_announcements.extend(result.announcements) + + logger.info(f"爬取到 {len(all_announcements)} 条原始公告") + + # 筛选公告 + filter_obj = filter_from_config() + filtered_announcements, filter_stats = filter_obj.filter(all_announcements) + + logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告") + + # 保存到数据库 + saved_count = save_announcements_to_storage(filtered_announcements) + + # 生成Markdown文件 + md_success = generate_onu_md(filtered_announcements) + + # 发送通知 + notify_success = False + if filtered_announcements and self.config.wechat_app.enabled: + notify_success = send_announcements_notification(filtered_announcements) + + result = { + "success": True, + "total_crawled": len(all_announcements), + "filtered": len(filtered_announcements), + "saved": saved_count, + "markdown_generated": md_success, + "notification_sent": notify_success, + "filter_stats": { + "keyword_filtered": filter_stats.keyword_filtered, + "date_filtered": filter_stats.date_filtered, + "duplicate_filtered": filter_stats.duplicate_filtered, + "source_filtered": filter_stats.source_filtered + } + } + + logger.info(f"爬取任务完成: {result}") + return result + + except Exception as e: + logger.error(f"爬取任务执行失败: {str(e)}") + return {"success": False, "error": str(e)} + + def run_cleanup(self, days: int = None): + """执行数据清理任务""" + try: + logger.info("开始执行数据清理任务") + + deleted_count = cleanup_storage(days) + + logger.info(f"数据清理完成：删除 {deleted_count} 条过期数据") + + # 发送通知 + if deleted_count > 0 and self.config.wechat_app.enabled: + send_system_notification( + "数据清理完成", + f"已清理 {deleted_count} 条过期数据" + ) + + return {"success": True, "deleted": deleted_count} + + except Exception as e: + logger.error(f"数据清理任务执行失败: {str(e)}") + return {"success": False, "error": str(e)} + + + def show_status(self): + """显示系统状态""" + try: + status = { + "system": { + "version": "1.0.0", + "healthy": check_system_health() + }, + "config": { + "debug": self.config.debug, + "log_level": self.config.log_level.value + }, + "database": { + "enabled": self.config.database.enabled, + "type": self.config.database.type + }, + "wechat": { + "enabled": self.config.wechat_app.enabled + } + } + + # 格式化输出 + print("\n=== 系统状态 ===") + print(f"系统健康: {'正常' if status['system']['healthy'] else '异常'}") + print(f"调试模式: {'开启' if status['config']['debug'] else '关闭'}") + print(f"日志级别: {status['config']['log_level']}") + print(f"数据库: {'启用' if status['database']['enabled'] else '禁用'} ({status['database']['type']})") + print(f"企业微信: {'启用' if status['wechat']['enabled'] else '禁用'}") + + return status + + except Exception as e: + logger.error(f"获取系统状态失败: {str(e)}") + return {"error": str(e)} + + +def create_argument_parser(): + """创建命令行参数解析器""" + parser = argparse.ArgumentParser( + description="广西政府采购网公告监控系统", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +使用示例: + python main.py crawl # 执行一次爬取 + python main.py crawl --keywords "大化" # 爬取指定关键词 + python main.py cleanup # 执行数据清理 + python main.py status # 查看系统状态 + """ + ) + + parser.add_argument( + 'command', + choices=['crawl', 'cleanup', 'status'], + help='要执行的命令' + ) + + parser.add_argument( + '--config', '-c', + help='配置文件路径' + ) + + # crawl命令的参数 + parser.add_argument( + '--keywords', '-k', + nargs='+', + help='关键词过滤（多个关键词用空格分隔）' + ) + + parser.add_argument( + '--sources', '-s', + nargs='+', + help='来源代码过滤（多个来源用空格分隔）' + ) + + parser.add_argument( + '--max-pages', + type=int, + help='最大爬取页数' + ) + + # cleanup命令的参数 + parser.add_argument( + '--days', '-d', + type=int, + help='清理多少天前的过期数据' + ) + + return parser + + +def main(): + """主函数""" + parser = create_argument_parser() + args = parser.parse_args() + + # 创建应用实例 + app = GXGPMonitorApp() + + # 初始化应用 + if not app.initialize(args.config): + sys.exit(1) + + try: + if args.command == 'crawl': + # 执行爬取 + result = app.run_crawl( + keywords=args.keywords, + sources=args.sources, + max_pages=args.max_pages + ) + + if result["success"]: + print("✅ 爬取任务执行成功") + print(f" 爬取公告: {result['total_crawled']}") + print(f" 筛选后: {result['filtered']}") + print(f" 保存数量: {result['saved']}") + if result.get("markdown_generated"): + print(" Markdown文件: 已生成") + if result.get("notification_sent"): + print(" 通知发送: 已发送") + else: + print(f"❌ 爬取任务执行失败: {result.get('error', '未知错误')}") + sys.exit(1) + + elif args.command == 'cleanup': + # 执行清理 + result = app.run_cleanup(days=args.days) + if result["success"]: + print(f"✅ 数据清理完成，删除 {result['deleted']} 条记录") + else: + print(f"❌ 数据清理失败: {result.get('error', '未知错误')}") + sys.exit(1) + + elif args.command == 'status': + # 显示状态 + app.show_status() + + except KeyboardInterrupt: + logger.info("收到中断信号，正在退出...") + except Exception as e: + logger.error(f"程序执行异常: {str(e)}") + print(f"❌ 程序执行异常: {str(e)}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/gx_gp_monitor/notification/__init__.py b/gx_gp_monitor/notification/__init__.py new file mode 100644 index 0000000..3ca4a8b --- /dev/null +++ b/gx_gp_monitor/notification/__init__.py @@ -0,0 +1 @@ +"""通知模块""" diff --git a/gx_gp_monitor/notification/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/notification/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..e88f67c Binary files /dev/null and b/gx_gp_monitor/notification/__pycache__/__init__.cpython-313.pyc differ diff --git a/gx_gp_monitor/notification/__pycache__/wechat.cpython-313.pyc b/gx_gp_monitor/notification/__pycache__/wechat.cpython-313.pyc new file mode 100644 index 0000000..74c9e3a Binary files /dev/null and b/gx_gp_monitor/notification/__pycache__/wechat.cpython-313.pyc differ diff --git a/gx_gp_monitor/notification/wechat.py b/gx_gp_monitor/notification/wechat.py new file mode 100644 index 0000000..1eb3ffd --- /dev/null +++ b/gx_gp_monitor/notification/wechat.py @@ -0,0 +1,555 @@ +""" +企业微信通知模块 +提供企业微信消息发送功能，支持文本和Markdown格式 +""" + +import requests +import json +import time +import hashlib +from typing import Optional, Dict, Any, List +from datetime import datetime + +try: + from ..core.config_manager import get_config + from ..core.logger import get_logger + from ..core.models import Announcement + from ..core.reliability import retry_on_exception, RetryConfig, safe_execute + from ..storage.md_generator import AnnouncementMarkdownFormatter +except ImportError: + from core.config_manager import get_config + from core.logger import get_logger + from core.models import Announcement + from core.reliability import retry_on_exception, RetryConfig, safe_execute + from storage.md_generator import AnnouncementMarkdownFormatter + + +logger = get_logger(__name__) + + +class WeChatService: + """企业微信服务""" + + def __init__(self): + self.config = get_config().wechat_app + self._access_token = None + self._token_expires_at = 0 + + logger.info("企业微信服务初始化完成") + + def _get_access_token(self) -> Optional[str]: + """ + 获取访问令牌 + + Returns: + Optional[str]: 访问令牌 + """ + current_time = time.time() + + # 检查令牌是否仍然有效 + if self._access_token and current_time < self._token_expires_at: + return self._access_token + + try: + # 构建请求URL + if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'): + url = f"{self.config.proxy_api_url}/cgi-bin/gettoken" + else: + url = "https://qyapi.weixin.qq.com/cgi-bin/gettoken" + + params = { + "corpid": self.config.corp_id, + "corpsecret": self.config.secret + } + + logger.debug("正在获取企业微信访问令牌") + + response = requests.get(url, params=params, timeout=30) + result = response.json() + + if result.get("errcode") == 0: + self._access_token = result.get("access_token") + # 提前5分钟过期 + expires_in = result.get("expires_in", 7200) - 300 + self._token_expires_at = current_time + expires_in + + logger.info("成功获取企业微信访问令牌") + return self._access_token + else: + logger.error(f"获取访问令牌失败: {result}") + return None + + except Exception as e: + logger.error(f"获取访问令牌异常: {str(e)}") + return None + + @retry_on_exception(RetryConfig(max_retries=3)) + def send_text_message(self, content: str, + to_user: str = "@all", + to_party: str = "", + to_tag: str = "") -> bool: + """ + 发送文本消息 + + Args: + content: 消息内容 + to_user: 接收者用户ID，多个用|分隔，@all表示全体 + to_party: 接收者部门ID，多个用|分隔 + to_tag: 接收者标签ID，多个用|分隔 + + Returns: + bool: 发送是否成功 + """ + try: + access_token = self._get_access_token() + if not access_token: + logger.error("无法获取访问令牌，发送失败") + return False + + # 构建请求URL + if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'): + url = f"{self.config.proxy_api_url}/cgi-bin/message/send" + else: + url = "https://qyapi.weixin.qq.com/cgi-bin/message/send" + + params = {"access_token": access_token} + + data = { + "touser": to_user, + "toparty": to_party, + "totag": to_tag, + "msgtype": "text", + "agentid": self.config.agent_id, + "text": { + "content": content + } + } + + logger.debug(f"发送文本消息: {content[:100]}...") + + response = requests.post(url, params=params, json=data, timeout=30) + result = response.json() + + if result.get("errcode") == 0: + logger.info("文本消息发送成功") + return True + else: + logger.error(f"文本消息发送失败: {result}") + return False + + except Exception as e: + logger.error(f"发送文本消息异常: {str(e)}") + return False + + @retry_on_exception(RetryConfig(max_retries=3)) + def send_markdown_message(self, content: str, + to_user: str = "@all", + to_party: str = "", + to_tag: str = "") -> bool: + """ + 发送Markdown消息 + + Args: + content: Markdown格式的消息内容 + to_user: 接收者用户ID + to_party: 接收者部门ID + to_tag: 接收者标签ID + + Returns: + bool: 发送是否成功 + """ + try: + access_token = self._get_access_token() + if not access_token: + logger.error("无法获取访问令牌，发送失败") + return False + + # 构建请求URL + if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'): + url = f"{self.config.proxy_api_url}/cgi-bin/message/send" + else: + url = "https://qyapi.weixin.qq.com/cgi-bin/message/send" + + params = {"access_token": access_token} + + data = { + "touser": to_user, + "toparty": to_party, + "totag": to_tag, + "msgtype": "markdown", + "agentid": self.config.agent_id, + "markdown": { + "content": content + } + } + + logger.debug("发送Markdown消息") + + response = requests.post(url, params=params, json=data, timeout=30) + result = response.json() + + if result.get("errcode") == 0: + logger.info("Markdown消息发送成功") + return True + else: + logger.error(f"Markdown消息发送失败: {result}") + return False + + except Exception as e: + logger.error(f"发送Markdown消息异常: {str(e)}") + return False + + def send_announcement_notification(self, announcements: List[Announcement], + max_count: int = 20) -> bool: + """ + 发送公告通知 + + Args: + announcements: 公告列表 + max_count: 最大显示数量 + + Returns: + bool: 发送是否成功 + """ + if not announcements: + logger.info("没有新公告，跳过通知") + return True + + try: + # 生成通知内容 + notification_content = self._generate_announcement_notification(announcements, max_count) + + # 发送Markdown消息 + return self.send_markdown_message(notification_content) + + except Exception as e: + logger.error(f"发送公告通知失败: {str(e)}") + return False + + def _generate_announcement_notification(self, announcements: List[Announcement], + max_count: int) -> str: + """ + 生成公告通知内容 + + Args: + announcements: 公告列表 + max_count: 最大显示数量 + + Returns: + str: Markdown格式的通知内容 + """ + # 按日期分组 + today_announcements = [] + other_announcements = [] + + today = datetime.now().date() + + for announcement in announcements: + if announcement.publish_date and announcement.publish_date.date() == today: + today_announcements.append(announcement) + else: + other_announcements.append(announcement) + + lines = [] + + # 标题 + total_count = len(announcements) + lines.append(f"# 🔔 广西政府采购网公告更新") + lines.append("") + lines.append(f"**发现 {total_count} 条新公告**") + lines.append("") + + # 今日公告 + if today_announcements: + lines.append(f"## 📅 今日公告 ({len(today_announcements)}条)") + lines.append("") + display_today = today_announcements[:max_count//2] + for announcement in display_today: + title = announcement.title + if len(title) > 40: + title = title[:40] + "..." + publish_time = announcement.publish_date.strftime("%H:%M") if announcement.publish_date else "N/A" + lines.append(f"• [{title}]({announcement.content_url}) - {publish_time}") + + if len(today_announcements) > len(display_today): + lines.append(f"• ... 还有 {len(today_announcements) - len(display_today)} 条今日公告") + + lines.append("") + + # 其他公告 + if other_announcements: + lines.append(f"## 📄 其他公告 ({len(other_announcements)}条)") + lines.append("") + remaining_slots = max_count - len(today_announcements) if today_announcements else max_count + display_other = other_announcements[:remaining_slots] + + for announcement in display_other: + title = announcement.title + if len(title) > 40: + title = title[:40] + "..." + publish_date = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A" + lines.append(f"• [{title}]({announcement.content_url}) - {publish_date}") + + if len(other_announcements) > len(display_other): + lines.append(f"• ... 还有 {len(other_announcements) - len(display_other)} 条公告") + + lines.append("") + + # 统计信息 + source_stats = {} + for announcement in announcements: + source = announcement.source_name + source_stats[source] = source_stats.get(source, 0) + 1 + + lines.append("## 📊 统计信息") + lines.append("") + for source, count in sorted(source_stats.items()): + lines.append(f"• {source}: {count}条") + + lines.append("") + lines.append("---") + lines.append("") + lines.append(f"*更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*") + lines.append("*点击公告标题查看详情*") + + return "\n".join(lines) + + def send_system_notification(self, title: str, content: str, + message_type: str = "text") -> bool: + """ + 发送系统通知 + + Args: + title: 通知标题 + content: 通知内容 + message_type: 消息类型 (text/markdown) + + Returns: + bool: 发送是否成功 + """ + try: + if message_type == "markdown": + full_content = f"# {title}\n\n{content}" + return self.send_markdown_message(full_content) + else: + full_content = f"{title}\n\n{content}" + return self.send_text_message(full_content) + + except Exception as e: + logger.error(f"发送系统通知失败: {str(e)}") + return False + + def send_error_notification(self, error_message: str, error_details: Optional[str] = None) -> bool: + """ + 发送错误通知 + + Args: + error_message: 错误消息 + error_details: 错误详情 + + Returns: + bool: 发送是否成功 + """ + content = f"## ❌ 系统错误\n\n**错误信息**: {error_message}" + + if error_details: + content += f"\n\n**错误详情**:\n```\n{error_details}\n```" + + content += f"\n\n*发生时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*" + + return self.send_markdown_message(content) + + def test_connection(self) -> bool: + """ + 测试连接 + + Returns: + bool: 连接是否正常 + """ + try: + token = self._get_access_token() + return token is not None + except Exception as e: + logger.error(f"企业微信连接测试失败: {str(e)}") + return False + + def get_service_status(self) -> Dict[str, Any]: + """ + 获取服务状态 + + Returns: + Dict[str, Any]: 服务状态信息 + """ + return { + "service": "wechat", + "enabled": self.config.enabled, + "corp_id": self.config.corp_id[:10] + "..." if self.config.corp_id else None, + "agent_id": self.config.agent_id, + "has_token": self._access_token is not None, + "token_expires_at": datetime.fromtimestamp(self._token_expires_at).isoformat() if self._token_expires_at > 0 else None, + "use_proxy": self.config.use_proxy, + "connection_test": self.test_connection() if self.config.enabled else False + } + + +class NotificationManager: + """通知管理器""" + + def __init__(self): + self.wechat = WeChatService() + self._services = { + "wechat": self.wechat + } + + def send_announcement_notification(self, announcements: List[Announcement]) -> Dict[str, bool]: + """ + 发送公告通知 + + Args: + announcements: 公告列表 + + Returns: + Dict[str, bool]: 各服务发送结果 + """ + results = {} + + # 企业微信通知 + if self.wechat.config.enabled: + try: + results["wechat"] = self.wechat.send_announcement_notification(announcements) + except Exception as e: + logger.error(f"企业微信通知失败: {str(e)}") + results["wechat"] = False + else: + results["wechat"] = None # 未启用 + + return results + + def send_system_notification(self, title: str, content: str) -> Dict[str, bool]: + """ + 发送系统通知 + + Args: + title: 通知标题 + content: 通知内容 + + Returns: + Dict[str, bool]: 发送结果 + """ + results = {} + + if self.wechat.config.enabled: + try: + results["wechat"] = self.wechat.send_system_notification(title, content, "markdown") + except Exception as e: + logger.error(f"企业微信系统通知失败: {str(e)}") + results["wechat"] = False + else: + results["wechat"] = None + + return results + + def send_error_notification(self, error_message: str, error_details: Optional[str] = None) -> Dict[str, bool]: + """ + 发送错误通知 + + Args: + error_message: 错误消息 + error_details: 错误详情 + + Returns: + Dict[str, bool]: 发送结果 + """ + results = {} + + if self.wechat.config.enabled: + try: + results["wechat"] = self.wechat.send_error_notification(error_message, error_details) + except Exception as e: + logger.error(f"企业微信错误通知失败: {str(e)}") + results["wechat"] = False + else: + results["wechat"] = None + + return results + + def get_status(self) -> Dict[str, Any]: + """ + 获取通知服务状态 + + Returns: + Dict[str, Any]: 服务状态 + """ + return { + "services": { + name: service.get_service_status() for name, service in self._services.items() + } + } + + +# 全局通知管理器实例 +_notification_manager = None + + +def get_notification_manager() -> NotificationManager: + """ + 获取通知管理器实例 + + Returns: + NotificationManager: 通知管理器实例 + """ + global _notification_manager + if _notification_manager is None: + _notification_manager = NotificationManager() + return _notification_manager + + +def send_announcements_notification(announcements: List[Announcement]) -> bool: + """ + 发送公告通知 + + Args: + announcements: 公告列表 + + Returns: + bool: 是否至少有一个服务发送成功 + """ + manager = get_notification_manager() + results = manager.send_announcement_notification(announcements) + + # 检查是否有服务发送成功 + return any(result for result in results.values() if result is True) + + +def send_system_notification(title: str, content: str) -> bool: + """ + 发送系统通知 + + Args: + title: 通知标题 + content: 通知内容 + + Returns: + bool: 是否至少有一个服务发送成功 + """ + manager = get_notification_manager() + results = manager.send_system_notification(title, content) + + return any(result for result in results.values() if result is True) + + +def send_error_alert(error_message: str, error_details: Optional[str] = None) -> bool: + """ + 发送错误警报 + + Args: + error_message: 错误消息 + error_details: 错误详情 + + Returns: + bool: 是否至少有一个服务发送成功 + """ + manager = get_notification_manager() + results = manager.send_error_notification(error_message, error_details) + + return any(result for result in results.values() if result is True) diff --git a/gx_gp_monitor/onu.md b/gx_gp_monitor/onu.md new file mode 100644 index 0000000..ecb015a --- /dev/null +++ b/gx_gp_monitor/onu.md @@ -0,0 +1,11 @@ +# 广西政府采购网公告监控 + +**更新时间**: 2026-01-07 17:36:12 + +## 无新公告 + +当前时间范围内没有找到符合条件的公告。 + +--- + +*由广西政府采购网公告监控系统生成* \ No newline at end of file diff --git a/gx_gp_monitor/requirements.txt b/gx_gp_monitor/requirements.txt new file mode 100644 index 0000000..c467090 --- /dev/null +++ b/gx_gp_monitor/requirements.txt @@ -0,0 +1,39 @@ +# 广西政府采购网公告监控系统依赖包 + +# 核心依赖 +requests>=2.28.0 # HTTP请求库 +psycopg2-binary>=2.9.0 # PostgreSQL数据库驱动 +PyYAML>=6.0 # YAML配置文件解析 +python-dateutil>=2.8.0 # 日期时间处理 + +# 爬虫相关 +fake-useragent>=1.1.0 # 随机User-Agent生成 +lxml>=4.9.0 # XML/HTML解析（备用） + +# 调度器 +schedule>=1.2.0 # 定时任务调度 +croniter>=1.4.0 # Cron表达式解析 + +# 日志和监控 +logging>=0.4.9.6 # 日志处理（Python内置） +colorama>=0.4.6 # 控制台颜色输出（可选，用于彩色日志） + +# 数据处理 +pandas>=1.5.0 # 数据处理（可选，用于复杂数据分析） +openpyxl>=3.0.10 # Excel文件处理（可选） + +# 加密和安全 +cryptography>=39.0.0 # 加密库（用于微信消息加密） +pycryptodome>=3.17.0 # 加密算法库 + +# 可选依赖（根据需要安装） +# redis>=4.5.0 # Redis缓存（如果需要） +# sqlalchemy>=2.0.0 # ORM（如果需要更复杂的数据库操作） +# celery>=5.3.0 # 分布式任务队列（如果需要） +# flask>=2.3.0 # Web框架（如果需要Web界面） + +# 开发依赖（仅开发环境需要） +# pytest>=7.2.0 # 测试框架 +# black>=23.0.0 # 代码格式化 +# flake8>=6.0.0 # 代码检查 +# mypy>=1.0.0 # 类型检查 diff --git a/gx_gp_monitor/scheduler/__init__.py b/gx_gp_monitor/scheduler/__init__.py new file mode 100644 index 0000000..d56023a --- /dev/null +++ b/gx_gp_monitor/scheduler/__init__.py @@ -0,0 +1 @@ +"""调度模块""" diff --git a/gx_gp_monitor/scheduler/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/scheduler/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..e6d1dd4 Binary files /dev/null and b/gx_gp_monitor/scheduler/__pycache__/__init__.cpython-313.pyc differ diff --git a/gx_gp_monitor/scheduler/__pycache__/scheduler.cpython-313.pyc b/gx_gp_monitor/scheduler/__pycache__/scheduler.cpython-313.pyc new file mode 100644 index 0000000..b74d244 Binary files /dev/null and b/gx_gp_monitor/scheduler/__pycache__/scheduler.cpython-313.pyc differ diff --git a/gx_gp_monitor/scheduler/scheduler.py b/gx_gp_monitor/scheduler/scheduler.py new file mode 100644 index 0000000..ad43a5b --- /dev/null +++ b/gx_gp_monitor/scheduler/scheduler.py @@ -0,0 +1,372 @@ +""" +调度器模块 +提供定时任务调度功能，支持cron表达式和间隔执行 +""" + +import time +import threading +from datetime import datetime, timedelta +from typing import Callable, Dict, Any, Optional, List +import schedule +from croniter import croniter + +try: + from ..core.config_manager import get_config + from ..core.logger import get_logger +except ImportError: + from core.config_manager import get_config + from core.logger import get_logger + + +logger = get_logger(__name__) + + +class TaskScheduler: + """任务调度器""" + + def __init__(self): + self.config = get_config().scheduler + self._running = False + self._thread = None + self._jobs = {} + self._job_stats = {} + + # 初始化调度 + if self.config.enabled: + self._load_jobs_from_config() + + def _load_jobs_from_config(self): + """从配置加载定时任务""" + for job_config in self.config.jobs: + if job_config.get("enabled", False): + job_name = job_config["name"] + cron_expr = job_config["cron"] + + # 根据任务名称创建对应的任务函数 + if job_name == "daily_crawl": + func = self._create_crawl_job() + elif job_name == "data_cleanup": + func = self._create_cleanup_job() + else: + logger.warning(f"未知的任务类型: {job_name}") + continue + + self.add_cron_job(job_name, cron_expr, func) + + def _create_crawl_job(self) -> Callable: + """创建爬取任务""" + def crawl_job(): + try: + logger.info("开始执行定时爬取任务") + + # 导入这里避免循环导入 + try: + from ..crawler.spider import crawl_announcements + from ..filters.filters import filter_from_config + from ..storage.postgresql import save_announcements_to_storage + from ..storage.md_generator import generate_onu_md + from ..notification.wechat import send_announcements_notification + except ImportError: + from crawler.spider import crawl_announcements + from filters.filters import filter_from_config + from storage.postgresql import save_announcements_to_storage + from storage.md_generator import generate_onu_md + from notification.wechat import send_announcements_notification + + # 执行爬取 + crawl_results = crawl_announcements() + + if not crawl_results: + logger.info("定时爬取任务完成：无数据") + return + + # 收集所有公告 + all_announcements = [] + for result in crawl_results: + if result.announcements: + all_announcements.extend(result.announcements) + + if not all_announcements: + logger.info("定时爬取任务完成：无新公告") + return + + # 筛选公告 + filter_obj = filter_from_config() + filtered_announcements, filter_stats = filter_obj.filter(all_announcements) + + logger.info(f"筛选结果: {len(all_announcements)} -> {len(filtered_announcements)}") + + # 保存到数据库 + saved_count = save_announcements_to_storage(filtered_announcements) + + # 生成Markdown文件 + generate_onu_md(filtered_announcements) + + # 发送通知 + if filtered_announcements: + send_announcements_notification(filtered_announcements) + + logger.info(f"定时爬取任务完成：处理 {len(filtered_announcements)} 条公告，保存 {saved_count} 条") + + except Exception as e: + logger.error(f"定时爬取任务执行失败: {str(e)}") + # 发送错误通知 + from ..notification.wechat import send_error_alert + send_error_alert("定时爬取任务失败", str(e)) + + return crawl_job + + def _create_cleanup_job(self) -> Callable: + """创建数据清理任务""" + def cleanup_job(): + try: + logger.info("开始执行数据清理任务") + + try: + from ..storage.postgresql import cleanup_storage + except ImportError: + from storage.postgresql import cleanup_storage + + # 执行清理 + deleted_count = cleanup_storage() + + logger.info(f"数据清理任务完成：删除 {deleted_count} 条过期数据") + + # 发送通知（如果删除的数据较多） + if deleted_count > 0: + try: + from ..notification.wechat import send_system_notification + except ImportError: + from notification.wechat import send_system_notification + send_system_notification( + "数据清理完成", + f"已清理 {deleted_count} 条过期数据" + ) + + except Exception as e: + logger.error(f"数据清理任务执行失败: {str(e)}") + + return cleanup_job + + def add_cron_job(self, name: str, cron_expr: str, func: Callable) -> bool: + """ + 添加cron定时任务 + + Args: + name: 任务名称 + cron_expr: cron表达式 + func: 任务函数 + + Returns: + bool: 添加是否成功 + """ + try: + # 验证cron表达式 + croniter(cron_expr) + + # 添加到schedule + schedule.every().day.at("00:00").do(func) # 临时设置，会被替换 + + # 存储任务信息 + self._jobs[name] = { + "func": func, + "cron": cron_expr, + "next_run": None, + "last_run": None, + "run_count": 0, + "error_count": 0 + } + + logger.info(f"添加定时任务: {name} ({cron_expr})") + return True + + except Exception as e: + logger.error(f"添加定时任务失败 {name}: {str(e)}") + return False + + def add_interval_job(self, name: str, interval_seconds: int, func: Callable) -> bool: + """ + 添加间隔执行任务 + + Args: + name: 任务名称 + interval_seconds: 执行间隔（秒） + func: 任务函数 + + Returns: + bool: 添加是否成功 + """ + try: + schedule.every(interval_seconds).seconds.do(func) + + self._jobs[name] = { + "func": func, + "interval": interval_seconds, + "next_run": None, + "last_run": None, + "run_count": 0, + "error_count": 0 + } + + logger.info(f"添加间隔任务: {name} ({interval_seconds}秒)") + return True + + except Exception as e: + logger.error(f"添加间隔任务失败 {name}: {str(e)}") + return False + + def remove_job(self, name: str) -> bool: + """ + 移除任务 + + Args: + name: 任务名称 + + Returns: + bool: 移除是否成功 + """ + if name in self._jobs: + # 注意：schedule库没有直接的移除方法 + # 这里只是从我们的记录中移除 + del self._jobs[name] + logger.info(f"移除任务: {name}") + return True + + return False + + def start(self): + """启动调度器""" + if self._running: + logger.warning("调度器已经在运行中") + return + + if not self.config.enabled: + logger.info("调度器已禁用") + return + + self._running = True + self._thread = threading.Thread(target=self._run_scheduler, daemon=True) + self._thread.start() + + logger.info("调度器已启动") + + def stop(self): + """停止调度器""" + if not self._running: + return + + self._running = False + + if self._thread and self._thread.is_alive(): + self._thread.join(timeout=5) + + logger.info("调度器已停止") + + def _run_scheduler(self): + """运行调度器主循环""" + logger.info("调度器主循环开始") + + while self._running: + try: + schedule.run_pending() + time.sleep(1) + except Exception as e: + logger.error(f"调度器运行异常: {str(e)}") + time.sleep(5) # 出错后等待5秒再继续 + + logger.info("调度器主循环结束") + + def run_once(self, job_name: Optional[str] = None): + """ + 手动执行任务一次 + + Args: + job_name: 任务名称，如果为None则执行所有任务 + """ + if job_name: + if job_name in self._jobs: + job_info = self._jobs[job_name] + logger.info(f"手动执行任务: {job_name}") + + try: + job_info["func"]() + job_info["run_count"] += 1 + job_info["last_run"] = datetime.now() + logger.info(f"任务 {job_name} 执行完成") + except Exception as e: + job_info["error_count"] += 1 + logger.error(f"任务 {job_name} 执行失败: {str(e)}") + else: + logger.error(f"任务不存在: {job_name}") + else: + # 执行所有任务 + for name in list(self._jobs.keys()): + self.run_once(name) + + def get_status(self) -> Dict[str, Any]: + """ + 获取调度器状态 + + Returns: + Dict[str, Any]: 状态信息 + """ + jobs_status = {} + for name, job_info in self._jobs.items(): + jobs_status[name] = { + "enabled": True, + "run_count": job_info.get("run_count", 0), + "error_count": job_info.get("error_count", 0), + "last_run": job_info.get("last_run").isoformat() if job_info.get("last_run") else None, + "next_run": job_info.get("next_run").isoformat() if job_info.get("next_run") else None + } + + return { + "enabled": self.config.enabled, + "running": self._running, + "timezone": self.config.timezone, + "jobs": jobs_status + } + + def is_running(self) -> bool: + """检查调度器是否正在运行""" + return self._running + + +# 全局调度器实例 +_scheduler = None + + +def get_scheduler() -> TaskScheduler: + """ + 获取调度器实例 + + Returns: + TaskScheduler: 调度器实例 + """ + global _scheduler + if _scheduler is None: + _scheduler = TaskScheduler() + return _scheduler + + +def start_scheduler(): + """启动调度器""" + scheduler = get_scheduler() + scheduler.start() + + +def stop_scheduler(): + """停止调度器""" + scheduler = get_scheduler() + scheduler.stop() + + +def run_scheduled_jobs(job_name: Optional[str] = None): + """ + 手动执行定时任务 + + Args: + job_name: 任务名称 + """ + scheduler = get_scheduler() + scheduler.run_once(job_name) diff --git a/gx_gp_monitor/storage/__init__.py b/gx_gp_monitor/storage/__init__.py new file mode 100644 index 0000000..e073c6c --- /dev/null +++ b/gx_gp_monitor/storage/__init__.py @@ -0,0 +1 @@ +"""存储模块""" diff --git a/gx_gp_monitor/storage/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/storage/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..8d58e6c Binary files /dev/null and b/gx_gp_monitor/storage/__pycache__/__init__.cpython-313.pyc differ diff --git a/gx_gp_monitor/storage/__pycache__/md_generator.cpython-313.pyc b/gx_gp_monitor/storage/__pycache__/md_generator.cpython-313.pyc new file mode 100644 index 0000000..b7bb1b7 Binary files /dev/null and b/gx_gp_monitor/storage/__pycache__/md_generator.cpython-313.pyc differ diff --git a/gx_gp_monitor/storage/__pycache__/postgresql.cpython-313.pyc b/gx_gp_monitor/storage/__pycache__/postgresql.cpython-313.pyc new file mode 100644 index 0000000..11fe384 Binary files /dev/null and b/gx_gp_monitor/storage/__pycache__/postgresql.cpython-313.pyc differ diff --git a/gx_gp_monitor/storage/md_generator.py b/gx_gp_monitor/storage/md_generator.py new file mode 100644 index 0000000..d4dc18a --- /dev/null +++ b/gx_gp_monitor/storage/md_generator.py @@ -0,0 +1,399 @@ +""" +Markdown生成器模块 +生成公告的Markdown格式输出文件 +""" + +import os +from pathlib import Path +from typing import List, Dict, Any, Optional +from datetime import datetime +from collections import defaultdict + +try: + from ..core.models import Announcement + from ..core.config_manager import get_config + from ..core.logger import get_logger +except ImportError: + from core.models import Announcement + from core.config_manager import get_config + from core.logger import get_logger + + +logger = get_logger(__name__) + + +class MarkdownGenerator: + """Markdown生成器""" + + def __init__(self, output_file: Optional[str] = None): + """ + 初始化Markdown生成器 + + Args: + output_file: 输出文件路径 + """ + self.config = get_config() + self.output_file = output_file or self.config.markdown.output_file + self.max_entries = self.config.markdown.max_entries + self.include_today_highlight = self.config.markdown.include_today_highlight + + # 确保输出目录存在 + output_path = Path(self.output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + def generate_markdown(self, announcements: List[Announcement], + title: str = "广西政府采购网公告监控") -> str: + """ + 生成Markdown内容 + + Args: + announcements: 公告列表 + title: 文档标题 + + Returns: + str: Markdown格式的文本 + """ + if not announcements: + return self._generate_empty_markdown(title) + + # 按来源分组 + grouped_announcements = self._group_announcements_by_source(announcements) + + # 生成Markdown + lines = [] + lines.append(f"# {title}") + lines.append("") + lines.append(f"**更新时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + lines.append(f"**总公告数**: {len(announcements)}") + lines.append("") + + # 生成目录 + lines.extend(self._generate_toc(grouped_announcements)) + lines.append("---") + lines.append("") + + # 生成各来源的公告 + for source_name, source_announcements in grouped_announcements.items(): + lines.extend(self._generate_source_section(source_name, source_announcements)) + lines.append("") + + return "\n".join(lines) + + def _group_announcements_by_source(self, announcements: List[Announcement]) -> Dict[str, List[Announcement]]: + """按来源分组公告""" + grouped = defaultdict(list) + + for announcement in announcements: + grouped[announcement.source_name].append(announcement) + + # 对每个组内的公告按时间倒序排列 + for source_name in grouped: + grouped[source_name].sort(key=lambda x: x.publish_date, reverse=True) + + return dict(grouped) + + def _generate_toc(self, grouped_announcements: Dict[str, List[Announcement]]) -> List[str]: + """生成目录""" + lines = ["## 目录", ""] + + for source_name, announcements in grouped_announcements.items(): + # 创建锚点链接 + anchor = self._create_anchor(source_name) + count = len(announcements) + lines.append(f"- [{source_name}](#{anchor}) ({count}条)") + + return lines + + def _generate_source_section(self, source_name: str, announcements: List[Announcement]) -> List[str]: + """生成来源章节""" + lines = [] + anchor = self._create_anchor(source_name) + + lines.append(f"## {source_name}") + lines.append("") + lines.append(f"**共 {len(announcements)} 条公告**") + lines.append("") + + # 生成公告列表 + for i, announcement in enumerate(announcements, 1): + lines.extend(self._generate_announcement_item(announcement, i)) + + return lines + + def _generate_announcement_item(self, announcement: Announcement, index: int) -> List[str]: + """生成单个公告项""" + lines = [] + + # 公告标题 + title_prefix = "" + if self.include_today_highlight and announcement.is_today: + title_prefix = "🆕 **【今日】**" + + title_line = f"### {index}. {title_prefix}[{announcement.title}]({announcement.content_url})" + lines.append(title_line) + lines.append("") + + # 公告信息 + info_items = [] + + if announcement.publish_date: + publish_date = announcement.publish_date.strftime("%Y-%m-%d") + info_items.append(f"📅 发布时间: {publish_date}") + + if announcement.purchase_name: + info_items.append(f"🏢 发布单位: {announcement.purchase_name}") + + info_items.append(f"📄 来源: {announcement.source_name}") + + if announcement.crawled_at: + crawled_time = announcement.crawled_at.strftime("%m-%d %H:%M") + info_items.append(f"🤖 爬取时间: {crawled_time}") + + if info_items: + lines.append(" | ".join(info_items)) + lines.append("") + + # 如果是新公告，添加标记 + if announcement.is_new: + lines.append("*🚀 新公告*") + lines.append("") + + lines.append("---") + lines.append("") + + return lines + + def _generate_empty_markdown(self, title: str) -> str: + """生成空内容的Markdown""" + lines = [ + f"# {title}", + "", + f"**更新时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + "", + "## 无新公告", + "", + "当前时间范围内没有找到符合条件的公告。", + "", + "---", + "", + f"*由广西政府采购网公告监控系统生成*" + ] + + return "\n".join(lines) + + def _create_anchor(self, text: str) -> str: + """创建锚点链接""" + # 移除特殊字符，替换空格为连字符，转为小写 + import re + anchor = re.sub(r'[^\w\s-]', '', text) + anchor = re.sub(r'[-\s]+', '-', anchor) + return anchor.lower().strip('-') + + def save_to_file(self, announcements: List[Announcement], + title: Optional[str] = None) -> bool: + """ + 保存Markdown到文件 + + Args: + announcements: 公告列表 + title: 文档标题 + + Returns: + bool: 保存是否成功 + """ + try: + markdown_content = self.generate_markdown(announcements, title) + + with open(self.output_file, 'w', encoding='utf-8') as f: + f.write(markdown_content) + + logger.info(f"Markdown文件已保存到: {self.output_file} (共 {len(announcements)} 条公告)") + return True + + except Exception as e: + logger.error(f"保存Markdown文件失败: {str(e)}") + return False + + def append_to_file(self, new_announcements: List[Announcement]) -> bool: + """ + 追加新公告到现有文件 + + Args: + new_announcements: 新公告列表 + + Returns: + bool: 追加是否成功 + """ + if not new_announcements: + return True + + try: + # 读取现有文件 + existing_content = "" + if os.path.exists(self.output_file): + with open(self.output_file, 'r', encoding='utf-8') as f: + existing_content = f.read() + + # 如果文件不存在或为空，创建新文件 + if not existing_content.strip(): + return self.save_to_file(new_announcements) + + # 解析现有公告（这里简化处理，实际可能需要更复杂的解析） + # 为简单起见，我们重新生成完整文件 + logger.info("重新生成完整Markdown文件") + return self.save_to_file(new_announcements) + + except Exception as e: + logger.error(f"追加公告到Markdown文件失败: {str(e)}") + return False + + def get_file_stats(self) -> Dict[str, Any]: + """获取文件统计信息""" + stats = { + "file_exists": False, + "file_size": 0, + "last_modified": None, + "announcement_count": 0 + } + + try: + if os.path.exists(self.output_file): + file_stat = os.stat(self.output_file) + stats["file_exists"] = True + stats["file_size"] = file_stat.st_size + stats["last_modified"] = datetime.fromtimestamp(file_stat.st_mtime).isoformat() + + # 尝试统计公告数量（简单计数） + with open(self.output_file, 'r', encoding='utf-8') as f: + content = f.read() + # 统计###开头的行（每个公告的标题行） + stats["announcement_count"] = content.count("### ") + + except Exception as e: + logger.warning(f"获取文件统计信息失败: {str(e)}") + + return stats + + +class AnnouncementMarkdownFormatter: + """公告Markdown格式化器""" + + @staticmethod + def format_announcement_card(announcement: Announcement) -> str: + """格式化单个公告为卡片样式""" + lines = [] + + # 标题 + emoji = "🆕" if announcement.is_today else "📄" + lines.append(f"### {emoji} {announcement.title}") + lines.append("") + + # 链接 + lines.append(f"[查看详情]({announcement.content_url})") + lines.append("") + + # 信息表格 + lines.append("| 属性 | 值 |") + lines.append("|------|-----|") + + if announcement.publish_date: + lines.append(f"| 发布时间 | {announcement.publish_date.strftime('%Y-%m-%d %H:%M')} |") + + lines.append(f"| 发布单位 | {announcement.purchase_name or 'N/A'} |") + lines.append(f"| 来源 | {announcement.source_name} |") + lines.append(f"| 公告类型 | {announcement.announcement_type.value} |") + + if announcement.crawled_at: + lines.append(f"| 爬取时间 | {announcement.crawled_at.strftime('%m-%d %H:%M')} |") + + lines.append("") + + return "\n".join(lines) + + @staticmethod + def format_announcement_list(announcements: List[Announcement]) -> str: + """格式化公告列表""" + if not announcements: + return "*暂无公告*" + + lines = [] + for announcement in announcements: + emoji = "🆕" if announcement.is_today else "•" + publish_date = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A" + line = f"{emoji} [{announcement.title}]({announcement.content_url}) - {publish_date}" + lines.append(line) + + return "\n".join(lines) + + @staticmethod + def format_notification_message(announcements: List[Announcement], + max_count: int = 10) -> str: + """格式化为通知消息""" + if not announcements: + return "暂无新公告" + + # 只显示前N条 + display_announcements = announcements[:max_count] + remaining_count = len(announcements) - max_count + + lines = [f"🔔 发现 {len(announcements)} 条新公告：", ""] + + for announcement in display_announcements: + title = announcement.title[:50] + "..." if len(announcement.title) > 50 else announcement.title + publish_date = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A" + lines.append(f"• {title} ({publish_date})") + + if remaining_count > 0: + lines.append(f"... 还有 {remaining_count} 条公告") + + lines.append("") + lines.append("*点击公告标题查看详情*") + + return "\n".join(lines) + + +def create_markdown_generator(output_file: Optional[str] = None) -> MarkdownGenerator: + """ + 创建Markdown生成器实例 + + Args: + output_file: 输出文件路径 + + Returns: + MarkdownGenerator: 生成器实例 + """ + return MarkdownGenerator(output_file) + + +def generate_onu_md(announcements: List[Announcement]) -> bool: + """ + 生成onu.md文件 + + Args: + announcements: 公告列表 + + Returns: + bool: 生成是否成功 + """ + generator = create_markdown_generator() + return generator.save_to_file(announcements, "广西政府采购网公告监控") + + +def update_onu_md(new_announcements: List[Announcement]) -> bool: + """ + 更新onu.md文件，追加新公告 + + Args: + new_announcements: 新公告列表 + + Returns: + bool: 更新是否成功 + """ + generator = create_markdown_generator() + + # 如果文件不存在，创建新文件 + if not os.path.exists(generator.output_file): + return generator.save_to_file(new_announcements) + + # 否则追加新公告 + return generator.append_to_file(new_announcements) diff --git a/gx_gp_monitor/storage/postgresql.py b/gx_gp_monitor/storage/postgresql.py new file mode 100644 index 0000000..de761f0 --- /dev/null +++ b/gx_gp_monitor/storage/postgresql.py @@ -0,0 +1,367 @@ +""" +PostgreSQL存储模块 +实现公告数据的PostgreSQL存储和管理 +""" + +from typing import List, Dict, Any, Optional +from datetime import datetime, timedelta +import threading +from dataclasses import asdict + +try: + from ..core.models import Announcement, CrawlResult + from ..core.database import get_database_manager, init_database + from ..core.logger import get_logger + from ..core.reliability import retry_on_exception, RetryConfig, safe_execute +except ImportError: + from core.models import Announcement, CrawlResult + from core.database import get_database_manager, init_database + from core.logger import get_logger + from core.reliability import retry_on_exception, RetryConfig, safe_execute + + +logger = get_logger(__name__) + + +class PostgreSQLStorage: + """PostgreSQL存储管理器""" + + def __init__(self): + self.db_manager = get_database_manager() + self._lock = threading.Lock() + + def save_announcements(self, announcements: List[Announcement]) -> int: + """ + 保存公告列表到数据库 + + Args: + announcements: 公告列表 + + Returns: + int: 成功保存的数量 + """ + if not announcements: + return 0 + + logger.info(f"开始保存 {len(announcements)} 条公告到数据库") + + try: + # 批量保存 + saved_count = self.db_manager.save_announcements_batch(announcements) + + if saved_count > 0: + logger.info(f"成功保存 {saved_count} 条公告到数据库") + + # 标记新公告 + self._mark_new_announcements(announcements[:saved_count]) + + return saved_count + + except Exception as e: + logger.error(f"保存公告到数据库失败: {str(e)}") + # 尝试逐个保存 + return self._save_announcements_fallback(announcements) + + def _save_announcements_fallback(self, announcements: List[Announcement]) -> int: + """逐个保存公告的降级方案""" + logger.info("使用降级方案逐个保存公告") + + saved_count = 0 + for announcement in announcements: + try: + if self.db_manager.save_announcement(announcement): + saved_count += 1 + except Exception as e: + logger.warning(f"保存公告失败: {announcement.title[:50]}..., 错误: {str(e)}") + continue + + logger.info(f"降级保存完成，成功保存 {saved_count} 条公告") + return saved_count + + def _mark_new_announcements(self, announcements: List[Announcement]): + """标记新公告""" + # 这里可以添加新公告标记逻辑 + # 由于我们在爬取时已经标记，这里主要是确保数据库中的标记正确 + pass + + def save_crawl_results(self, results: List[CrawlResult]) -> int: + """ + 保存爬取结果 + + Args: + results: 爬取结果列表 + + Returns: + int: 成功保存的数量 + """ + if not results: + return 0 + + saved_count = 0 + for result in results: + try: + if self.db_manager.save_crawl_result(result): + saved_count += 1 + except Exception as e: + logger.warning(f"保存爬取结果失败: {result.source.name}, 错误: {str(e)}") + continue + + logger.info(f"保存爬取结果完成: {saved_count}/{len(results)}") + return saved_count + + def get_recent_announcements(self, hours: int = 24, + limit: int = 100) -> List[Announcement]: + """ + 获取最近的公告 + + Args: + hours: 最近小时数 + limit: 限制数量 + + Returns: + List[Announcement]: 公告列表 + """ + try: + return self.db_manager.get_recent_announcements(hours) + except Exception as e: + logger.error(f"获取最近公告失败: {str(e)}") + return [] + + def cleanup_expired_data(self, days: Optional[int] = None) -> int: + """ + 清理过期数据 + + Args: + days: 保留天数，如果为None则使用配置默认值 + + Returns: + int: 清理的记录数 + """ + from ..core.config_manager import get_config + + config = get_config() + if days is None: + days = config.database.data_retention_days + + logger.info(f"开始清理 {days} 天前的过期数据") + + try: + deleted_count = self.db_manager.cleanup_expired_data(days) + + if deleted_count > 0: + logger.info(f"成功清理 {deleted_count} 条过期数据") + else: + logger.info("没有找到需要清理的过期数据") + + return deleted_count + + except Exception as e: + logger.error(f"清理过期数据失败: {str(e)}") + return 0 + + def get_statistics(self) -> Dict[str, Any]: + """ + 获取存储统计信息 + + Returns: + Dict[str, Any]: 统计数据 + """ + try: + stats = self.db_manager.get_statistics() + stats.update({ + "storage_type": "postgresql", + "last_cleanup": datetime.now().isoformat() + }) + return stats + except Exception as e: + logger.error(f"获取存储统计信息失败: {str(e)}") + return { + "storage_type": "postgresql", + "error": str(e), + "total_announcements": 0, + "last_cleanup": datetime.now().isoformat() + } + + def search_announcements(self, keyword: Optional[str] = None, + source_code: Optional[str] = None, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + limit: int = 50) -> List[Announcement]: + """ + 搜索公告 + + Args: + keyword: 关键词 + source_code: 来源代码 + start_date: 开始日期 + end_date: 结束日期 + limit: 限制数量 + + Returns: + List[Announcement]: 搜索结果 + """ + # 这里可以实现更复杂的搜索逻辑 + # 目前使用现有的查询方法 + try: + return self.db_manager.get_announcements( + source_code=source_code, + start_date=start_date, + end_date=end_date, + limit=limit + ) + except Exception as e: + logger.error(f"搜索公告失败: {str(e)}") + return [] + + def is_healthy(self) -> bool: + """ + 检查存储健康状态 + + Returns: + bool: 是否健康 + """ + try: + # 尝试执行一个简单的查询 + stats = self.get_statistics() + return "error" not in stats + except Exception as e: + logger.error(f"存储健康检查失败: {str(e)}") + return False + + def optimize_storage(self): + """优化存储性能""" + # 这里可以添加数据库优化逻辑，如重建索引、清理碎片等 + logger.info("开始优化存储性能") + + try: + # 执行一些基本的优化操作 + # 注意：实际的优化命令取决于PostgreSQL版本和配置 + + # 这里可以添加具体的优化SQL + # 例如：VACUUM, REINDEX等 + + logger.info("存储优化完成") + except Exception as e: + logger.error(f"存储优化失败: {str(e)}") + + def backup_data(self, backup_path: Optional[str] = None) -> bool: + """ + 备份数据 + + Args: + backup_path: 备份文件路径 + + Returns: + bool: 备份是否成功 + """ + # 这里可以实现数据备份逻辑 + # 可以使用pg_dump或其他备份工具 + + logger.info("开始备份数据") + + try: + # 实现备份逻辑 + # 注意：这需要系统权限来执行pg_dump + + logger.info("数据备份完成") + return True + + except Exception as e: + logger.error(f"数据备份失败: {str(e)}") + return False + + +class StorageManager: + """存储管理器""" + + def __init__(self): + self.postgresql = PostgreSQLStorage() + self._current_storage = self.postgresql # 默认使用PostgreSQL + + def save_announcements(self, announcements: List[Announcement]) -> int: + """保存公告""" + return self._current_storage.save_announcements(announcements) + + def save_crawl_results(self, results: List[CrawlResult]) -> int: + """保存爬取结果""" + return self._current_storage.save_crawl_results(results) + + def get_recent_announcements(self, hours: int = 24, limit: int = 100) -> List[Announcement]: + """获取最近公告""" + return self._current_storage.get_recent_announcements(hours, limit) + + def cleanup_expired_data(self, days: Optional[int] = None) -> int: + """清理过期数据""" + return self._current_storage.cleanup_expired_data(days) + + def get_statistics(self) -> Dict[str, Any]: + """获取统计信息""" + return self._current_storage.get_statistics() + + def is_healthy(self) -> bool: + """检查健康状态""" + return self._current_storage.is_healthy() + + def optimize(self): + """优化存储""" + self._current_storage.optimize_storage() + + def backup(self, backup_path: Optional[str] = None) -> bool: + """备份数据""" + return self._current_storage.backup_data(backup_path) + + +# 全局存储管理器实例 +_storage_manager = None +_storage_lock = threading.Lock() + + +def get_storage_manager() -> StorageManager: + """ + 获取存储管理器实例 + + Returns: + StorageManager: 存储管理器实例 + """ + global _storage_manager + if _storage_manager is None: + with _storage_lock: + if _storage_manager is None: + _storage_manager = StorageManager() + return _storage_manager + + +def init_storage(): + """初始化存储""" + try: + init_database() + logger.info("存储初始化完成") + except Exception as e: + logger.error(f"存储初始化失败: {str(e)}") + raise + + +def save_announcements_to_storage(announcements: List[Announcement]) -> int: + """ + 保存公告到存储 + + Args: + announcements: 公告列表 + + Returns: + int: 保存成功的数量 + """ + return get_storage_manager().save_announcements(announcements) + + +def cleanup_storage(days: Optional[int] = None) -> int: + """ + 清理存储中的过期数据 + + Args: + days: 保留天数 + + Returns: + int: 清理的记录数 + """ + return get_storage_manager().cleanup_expired_data(days) diff --git a/logs/gx_gp_monitor.log b/logs/gx_gp_monitor.log new file mode 100644 index 0000000..047edc7 --- /dev/null +++ b/logs/gx_gp_monitor.log @@ -0,0 +1,23 @@ +2026-01-07 17:29:10 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:29:10 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:29:10 - gx_gp_monitor.scheduler.scheduler - INFO - 手动执行任务: data_cleanup +2026-01-07 17:29:10 - gx_gp_monitor.scheduler.scheduler - INFO - 开始执行数据清理任务 +2026-01-07 17:29:10 - gx_gp_monitor.core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:29:10 - gx_gp_monitor.core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:29:10 - gx_gp_monitor.storage.postgresql - INFO - 开始清理 90 天前的过期数据 +2026-01-07 17:29:10 - gx_gp_monitor.storage.postgresql - INFO - 开始清理 90 天前的过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.core.database - INFO - 清理过期数据完成，删除 0 条记录 +2026-01-07 17:29:11 - gx_gp_monitor.core.database - INFO - 清理过期数据完成，删除 0 条记录 +2026-01-07 17:29:11 - gx_gp_monitor.storage.postgresql - INFO - 没有找到需要清理的过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.storage.postgresql - INFO - 没有找到需要清理的过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.scheduler.scheduler - INFO - 数据清理任务完成：删除 0 条过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.scheduler.scheduler - INFO - 数据清理任务完成：删除 0 条过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.scheduler.scheduler - INFO - 任务 data_cleanup 执行完成 +2026-01-07 17:29:11 - gx_gp_monitor.scheduler.scheduler - INFO - 任务 data_cleanup 执行完成 +2026-01-07 17:29:14 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:29:14 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:29:21 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:29:21 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:29:21 - gx_gp_monitor.scheduler.scheduler - INFO - 添加间隔任务: custom_task (30秒) +2026-01-07 17:32:05 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:32:05 - gx_gp_monitor.scheduler.scheduler - INFO - 移除任务: data_cleanup diff --git a/old/WeChatService.py b/old/WeChatService.py new file mode 100644 index 0000000..ea1f596 --- /dev/null +++ b/old/WeChatService.py @@ -0,0 +1,625 @@ +import requests +import json +import xml.etree.ElementTree as ET +import hashlib +import time +import random +import string +from config import Config +import logging +from Crypto.Cipher import AES +import base64 +import socket +import struct +import urllib.parse +import redis + +logger = logging.getLogger(__name__) + +class WeChatService: + def __init__(self): + self.corpid = Config.WECHAT_CORPID + self.corpsecret = Config.WECHAT_CORPSECRET + self.agentid = Config.WECHAT_AGENTID + self.token = Config.WECHAT_TOKEN + self.encoding_aes_key = Config.WECHAT_ENCODING_AES_KEY + self.access_token = None + self.token_expires_at = 0 + # 添加代理API配置 + self.use_proxy = getattr(Config, 'USE_WECHAT_PROXY', False) + self.proxy_api_url = getattr(Config, 'WECHAT_PROXY_API_URL', 'https://api.v6ole.top') + + logger.info("WeChatService初始化完成") + + def _get_redis_connection(self): + """获取Redis连接""" + try: + redis_conn = redis.Redis( + host=Config.REDIS_HOST, + port=Config.REDIS_PORT, + db=Config.REDIS_DB, + password=Config.REDIS_PASSWORD, + decode_responses=True, # 自动将响应解码为字符串 + socket_timeout=5, # 设置超时时间 + socket_connect_timeout=5 + ) + # 测试连接 + redis_conn.ping() + return redis_conn + except Exception as e: + logger.error(f"Redis连接失败: {str(e)}") + raise + + def get_access_token(self): + """获取access_token""" + current_time = time.time() + + # 如果access_token未过期，直接返回 + if self.access_token and current_time < self.token_expires_at: + return self.access_token + + try: + # 使用代理API获取access_token + if self.use_proxy: + url = f"{self.proxy_api_url}/cgi-bin/gettoken?corpid={self.corpid}&corpsecret={self.corpsecret}" + logger.info(f"使用代理API获取access_token: {url}") + else: + url = f"https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={self.corpid}&corpsecret={self.corpsecret}" + + response = requests.get(url) + result = response.json() + + if result.get("errcode") == 0: + self.access_token = result.get("access_token") + # 设置过期时间，提前5分钟过期 + expires_in = result.get("expires_in", 7200) - 300 + self.token_expires_at = current_time + expires_in + + # 转换过期时间为可读格式 + expiry_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.token_expires_at)) + logger.info(f"成功获取access_token，过期时间: {expiry_time}") + return self.access_token + elif result.get("errcode") == 60020 and not self.use_proxy: + # 如果是IP限制错误并且未使用代理，尝试使用代理 + logger.info("IP受限，尝试使用代理API获取access_token") + self.use_proxy = True + return self.get_access_token() + else: + logger.error(f"获取access_token失败: {result}") + return None + except Exception as e: + logger.error(f"获取access_token异常: {str(e)}") + return None + + def verify_url(self, signature, timestamp, nonce, echostr): + """验证URL有效性""" + try: + # URL解码echostr + echostr = urllib.parse.unquote(echostr) + + # 1. 将token、timestamp、nonce、echostr四个参数进行字典序排序 + temp_list = [self.token, timestamp, nonce, echostr] + temp_list.sort() + + # 2. 将四个参数字符串拼接成一个字符串进行sha1加密 + temp_str = ''.join(temp_list) + hash_obj = hashlib.sha1(temp_str.encode('utf-8')) + hash_str = hash_obj.hexdigest() + + # 3. 开发者获得加密后的字符串可与signature对比，标识该请求来源于微信 + if hash_str == signature: + # 如果验证成功，需要解密echostr + if self.encoding_aes_key: + return self.decrypt_echostr(echostr) + return echostr + else: + logger.error(f"URL验证失败: signature={signature}, hash_str={hash_str}") + return "URL验证失败" + except Exception as e: + logger.error(f"URL验证异常: {str(e)}") + return "URL验证异常" + + def decrypt_echostr(self, echostr): + """解密echostr""" + try: + # 1. 对密文进行base64解码 + aes_key = base64.b64decode(self.encoding_aes_key + '=') + encrypted = base64.b64decode(echostr) + + # 2. 使用AES解密 + cipher = AES.new(aes_key, AES.MODE_CBC, aes_key[:16]) + decrypted = cipher.decrypt(encrypted) + + # 3. 去除补位字符 + unpad = lambda s: s[:-ord(s[len(s)-1:])] + decrypted = unpad(decrypted) + + # 4. 去除16位随机字符串 + content = decrypted[16:] + xml_len = socket.ntohl(struct.unpack("I", content[:4])[0]) + xml_content = content[4:xml_len+4] + + # 5. 验证企业ID + received_id = content[xml_len+4:].decode('utf-8') + if received_id != self.corpid: + logger.error(f"企业ID验证失败: received={received_id}, expected={self.corpid}") + return "企业ID验证失败" + + return xml_content.decode('utf-8') + except Exception as e: + logger.error(f"解密echostr失败: {str(e)}") + return "解密失败" + + def parse_message(self, xml_data): + """解析接收到的XML消息""" + try: + root = ET.fromstring(xml_data) + msg = {} + for child in root: + msg[child.tag] = child.text + + # 如果消息是加密的，需要解密 + if 'Encrypt' in msg: + logger.info("消息已加密，开始解密") + decrypted = self.decrypt_message(msg['Encrypt']) + logger.info(f"解密后的消息: {decrypted}") + # 解析解密后的XML + decrypted_root = ET.fromstring(decrypted) + msg = {} + for child in decrypted_root: + msg[child.tag] = child.text + + logger.info(f"最终解析的消息: {msg}") + return msg + except Exception as e: + logger.error(f"解析消息失败: {str(e)}") + return None + + def decrypt_message(self, encrypted_msg): + """解密消息""" + try: + # 1. 对密文进行base64解码 + aes_key = base64.b64decode(self.encoding_aes_key + '=') + encrypted = base64.b64decode(encrypted_msg) + + # 2. 使用AES解密 + cipher = AES.new(aes_key, AES.MODE_CBC, aes_key[:16]) + decrypted = cipher.decrypt(encrypted) + + # 3. 去除补位字符 + unpad = lambda s: s[:-ord(s[len(s)-1:])] + decrypted = unpad(decrypted) + + # 4. 去除16位随机字符串 + content = decrypted[16:] + xml_len = socket.ntohl(struct.unpack("I", content[:4])[0]) + xml_content = content[4:xml_len+4] + + # 5. 验证企业ID + received_id = content[xml_len+4:].decode('utf-8') + if received_id != self.corpid: + logger.error(f"企业ID验证失败: received={received_id}, expected={self.corpid}") + raise Exception("企业ID验证失败") + + return xml_content.decode('utf-8') + except Exception as e: + logger.error(f"解密消息失败: {str(e)}") + raise + + def send_text_message(self, content, to_user='@all', to_party='', to_tag=''): + """发送文本消息""" + max_retries = 3 + retry_count = 0 + + while retry_count < max_retries: + try: + access_token = self.get_access_token() + + # 构建消息数据 + data = { + "touser": to_user, + "toparty": to_party, + "totag": to_tag, + "msgtype": "text", + "agentid": self.agentid, + "text": { + "content": content + } + } + + logger.info(f"发送文本消息: {data}") + + # 使用代理API发送消息 + if self.use_proxy: + url = f"{self.proxy_api_url}/cgi-bin/message/send?access_token={access_token}" + logger.info(f"使用代理API发送消息: {url}") + else: + url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={access_token}" + + response = requests.post(url, json=data) + result = response.json() + logger.info(f"发送文本消息响应: {result}") + + # 如果token过期，重新获取并重试 + if result.get('errcode') == 40014: + logger.info("access_token过期，重新获取") + self.access_token = None + self.token_expires_at = 0 + retry_count += 1 + continue + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API发送") + self.use_proxy = True + retry_count += 1 + continue + + # 如果是其他错误，记录并返回 + if result.get('errcode') != 0: + logger.error(f"发送消息失败: {result.get('errmsg')}") + if result.get('errcode') == 301002: # 应用ID不匹配 + logger.error("应用ID不匹配，请检查配置") + break + retry_count += 1 + continue + + return result + except Exception as e: + logger.error(f"发送文本消息失败: {str(e)}") + retry_count += 1 + if retry_count >= max_retries: + raise + time.sleep(1) # 等待1秒后重试 + + return {"errcode": -1, "errmsg": "发送消息失败，已达到最大重试次数"} + + def send_markdown_message(self, content, to_user='@all', to_party='', to_tag=''): + """发送markdown消息""" + try: + access_token = self.get_access_token() + + data = { + "touser": to_user, + "toparty": to_party, + "totag": to_tag, + "msgtype": "markdown", + "agentid": self.agentid, + "markdown": { + "content": content + } + } + + logger.info(f"发送markdown消息: {data}") + + # 使用代理API发送消息 + if self.use_proxy: + url = f"{self.proxy_api_url}/cgi-bin/message/send?access_token={access_token}" + logger.info(f"使用代理API发送消息: {url}") + else: + url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={access_token}" + + response = requests.post(url, json=data) + result = response.json() + logger.info(f"发送markdown消息响应: {result}") + + # 如果token过期，重新获取并重试 + if result.get('errcode') == 40014: + logger.info("access_token过期，重新获取") + self.access_token = None + self.token_expires_at = 0 + return self.send_markdown_message(content, to_user, to_party, to_tag) + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API发送") + self.use_proxy = True + return self.send_markdown_message(content, to_user, to_party, to_tag) + + return result + except Exception as e: + logger.error(f"发送markdown消息失败: {str(e)}") + raise + + def get_user_info(self, userid): + """获取用户信息""" + try: + access_token = self.get_access_token() + + # 使用代理API获取用户信息 + if self.use_proxy: + url = f"{self.proxy_api_url}/cgi-bin/user/get?access_token={access_token}&userid={userid}" + logger.info(f"使用代理API获取用户信息: {url}") + else: + url = f"https://qyapi.weixin.qq.com/cgi-bin/user/get?access_token={access_token}&userid={userid}" + + logger.info(f"获取用户信息: {url}") + response = requests.get(url) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API获取用户信息") + self.use_proxy = True + return self.get_user_info(userid) + + logger.info(f"获取用户信息响应: {result}") + return result + except Exception as e: + logger.error(f"获取用户信息失败: {str(e)}") + return {"errcode": -1, "errmsg": str(e)} + + def generate_temp_token(self, user_id, user_name=None): + """生成临时访问令牌""" + try: + # 生成随机令牌 + token = ''.join(random.choices(string.ascii_letters + string.digits, k=32)) + + # 存储令牌信息到Redis + token_data = { + 'user_id': user_id, + 'name': user_name or '微信用户', + 'created_at': int(time.time()) + } + + # 使用Redis存储令牌，设置过期时间 + key = f"temp_token:{token}" + redis_conn = self._get_redis_connection() + redis_conn.setex( + key, + Config.REDIS_TEMP_TOKEN_EXPIRE, + json.dumps(token_data) + ) + + # 验证令牌是否成功存储 + stored_data = redis_conn.get(key) + if not stored_data: + logger.error("令牌存储失败") + return None + + logger.info(f"成功生成临时令牌: {token}, 存储数据: {stored_data}") + return token + + except Exception as e: + logger.error(f"生成临时令牌失败: {str(e)}") + return None + + def verify_temp_token(self, token): + """验证临时访问令牌""" + try: + # 从Redis中获取令牌信息 + key = f"temp_token:{token}" + redis_conn = self._get_redis_connection() + token_info = redis_conn.get(key) + + if not token_info: + logger.error(f"临时令牌不存在或已过期: {token}") + return None + + # 解析令牌信息 + token_data = json.loads(token_info) + user_id = token_data.get('user_id') + + if not user_id: + logger.error(f"临时令牌中未找到用户ID: {token_info}") + return None + + logger.info(f"临时令牌验证成功: {token}, 用户信息: {token_data}") + return token_data + + except Exception as e: + logger.error(f"验证临时令牌失败: {str(e)}") + return None + + def send_card_message(self, title, description, url, to_user): + """发送卡片消息""" + try: + # 获取access_token + access_token = self.get_access_token() + if not access_token: + logger.error("获取access_token失败") + return None + + # 构建消息内容 + data = { + "touser": to_user, + "toparty": "", + "totag": "", + "msgtype": "textcard", + "agentid": self.agentid, + "textcard": { + "title": title, + "description": description, + "url": url, + "btntxt": "查看详情" + } + } + + logger.info(f"发送卡片消息: {data}") + + # 使用代理API发送消息 + if self.use_proxy: + api_url = f"{self.proxy_api_url}/cgi-bin/message/send?access_token={access_token}" + logger.info(f"使用代理API发送消息: {api_url}") + else: + api_url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={access_token}" + + # 发送消息 + response = requests.post(api_url, json=data) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API发送") + self.use_proxy = True + return self.send_card_message(title, description, url, to_user) + + if result.get('errcode') == 0: + logger.info(f"发送卡片消息成功: {result}") + return result + else: + logger.error(f"发送卡片消息失败: {result}") + return None + + except Exception as e: + logger.error(f"发送卡片消息异常: {str(e)}") + return None + + def create_menu(self): + """创建应用菜单""" + try: + # 获取access_token + access_token = self.get_access_token() + if not access_token: + logger.error("获取access_token失败") + return None + + # 菜单配置 + menu_data = { + "button": [ + { + "name": "设备查询", + "sub_button": [ + { + "type": "click", + "name": "在线统计", + "key": "online" + }, + { + "type": "click", + "name": "设备状态", + "key": "status" + } + ] + }, + { + "name": "设备管理", + "sub_button": [ + { + "type": "click", + "name": "业务下发", + "key": "deploy" + }, + { + "type": "view", + "name": "设备管理", + "url": f"{Config.BASE_URL}/devices" + }, + { + "type": "click", + "name": "序列修复", + "key": "sequence_fix" + } + ] + }, + { + "type": "click", + "name": "帮助", + "key": "help" + } + ] + } + + logger.info(f"创建菜单: {menu_data}") + + # 使用代理API创建菜单 + if self.use_proxy: + api_url = f"{self.proxy_api_url}/cgi-bin/menu/create?access_token={access_token}&agentid={self.agentid}" + logger.info(f"使用代理API创建菜单: {api_url}") + else: + api_url = f"https://qyapi.weixin.qq.com/cgi-bin/menu/create?access_token={access_token}&agentid={self.agentid}" + + # 发送请求 + response = requests.post(api_url, json=menu_data) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API创建菜单") + self.use_proxy = True + return self.create_menu() + + if result.get('errcode') == 0: + logger.info(f"创建菜单成功: {result}") + return result + else: + logger.error(f"创建菜单失败: {result}") + return None + + except Exception as e: + logger.error(f"创建菜单异常: {str(e)}") + return None + + def delete_menu(self): + """删除应用菜单""" + try: + # 获取access_token + access_token = self.get_access_token() + if not access_token: + logger.error("获取access_token失败") + return None + + # 使用代理API删除菜单 + if self.use_proxy: + api_url = f"{self.proxy_api_url}/cgi-bin/menu/delete?access_token={access_token}&agentid={self.agentid}" + logger.info(f"使用代理API删除菜单: {api_url}") + else: + api_url = f"https://qyapi.weixin.qq.com/cgi-bin/menu/delete?access_token={access_token}&agentid={self.agentid}" + + # 发送请求 + response = requests.get(api_url) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API删除菜单") + self.use_proxy = True + return self.delete_menu() + + if result.get('errcode') == 0: + logger.info(f"删除菜单成功: {result}") + return result + else: + logger.error(f"删除菜单失败: {result}") + return None + + except Exception as e: + logger.error(f"删除菜单异常: {str(e)}") + return None + + def get_menu(self): + """获取应用菜单""" + try: + # 获取access_token + access_token = self.get_access_token() + if not access_token: + logger.error("获取access_token失败") + return None + + # 使用代理API获取菜单 + if self.use_proxy: + api_url = f"{self.proxy_api_url}/cgi-bin/menu/get?access_token={access_token}&agentid={self.agentid}" + logger.info(f"使用代理API获取菜单: {api_url}") + else: + api_url = f"https://qyapi.weixin.qq.com/cgi-bin/menu/get?access_token={access_token}&agentid={self.agentid}" + + # 发送请求 + response = requests.get(api_url) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API获取菜单") + self.use_proxy = True + return self.get_menu() + + if result.get('errcode') == 0: + logger.info(f"获取菜单成功: {result}") + return result + else: + logger.error(f"获取菜单失败: {result}") + return None + + except Exception as e: + logger.error(f"获取菜单异常: {str(e)}") + return None \ No newline at end of file diff --git a/old/config.yaml b/old/config.yaml new file mode 100644 index 0000000..4c386d0 --- /dev/null +++ b/old/config.yaml @@ -0,0 +1,104 @@ +# 广西政府采购网公告监控系统配置文件 +# 复制此文件为 config.yaml 并修改相应配置 + +# 调试模式 +debug: false + +# 日志配置 +log_level: INFO +log_file: logs/gx_gp_monitor.log + + +# 爬虫配置 +crawler: + base_url: "https://zfcg.gxzf.gov.cn" + timeout: 30 # 请求超时时间（秒） + max_retries: 3 # 最大重试次数 + retry_delay: 1.0 # 重试初始延迟 + max_retry_delay: 60.0 # 重试最大延迟 + backoff_factor: 2.0 # 退避因子 + user_agents: # User-Agent列表 + - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" + proxies: [] # 代理列表 + request_delay: 1.0 # 请求间延迟 + request_delay_max: 3.0 # 请求间最大延迟 + keyword: ["大化", "信息化"] # 关键词筛选（支持多个关键词） + start_date: "" # 开始日期 (YYYY-MM-DD) + end_date: "" # 结束日期 (YYYY-MM-DD) + max_pages: 10 # 最大页数 + page_size: 100 # 每页大小 + +# 企业微信通知配置 +wechat_app: + enabled: true # 是否启用企业微信通知 + corp_id: "ww69e8e44636f47780" # 企业ID + agent_id: "1000007" # 应用ID + secret: "SmelCwKFoL0E9ATWFzr-w7gsfXBTN72lT1UqnNd0HpI" # 应用Secret + token: "DmvL98cAF6x9CFtQZwqD2emGL8S7HxA" # Token + encoding_aes_key: "yAc4OoSCP92YTefHXYfw27WeG9oF11W9d6nw6QYlU3D" # 消息加密Key + port: 18001 # 服务端口 + host: "0.0.0.0" # 服务主机 + debug: false # 调试模式 +# 数据库配置 + DB_NAME=gx-gp-notify + DB_USER=gx-gp-notify + DB_PASSWORD=MA6RBX4F6Bd5DGmw + DB_HOST=10.10.10.14 + DB_PORT=5432 + + +# 公告来源配置 +sources: + ZcyAnnouncement1: + category_id: 66485 + name: "采购公告" + type: "purchase" + ZcyAnnouncement2: + category_id: 66485 + name: "结果公告" + type: "result" + ZcyAnnouncement3: + category_id: 66485 + name: "合同公告" + type: "contract" + ZcyAnnouncement4: + category_id: 66485 + name: "更正公告" + type: "correction" + ZcyAnnouncement5: + category_id: 66485 + name: "招标文件预公示" + type: "pre_announcement" + ZcyAnnouncement6: + category_id: 66485 + name: "单一来源公示" + type: "single_source" + ZcyAnnouncement7: + category_id: 66485 + name: "电子卖场公示" + type: "electronic_market" + ZcyAnnouncement10: + category_id: 66485 + name: "履约验收公示" + type: "acceptance" + ZcyAnnouncement11: + category_id: 66485 + name: "工程类公告" + type: "engineering" + ZcyAnnouncement20: + category_id: 66485 + name: "框架协议征集公告" + type: "framework_agreement" + ZcyAnnouncement21: + category_id: 66485 + name: "框架协议入围结果公告" + type: "framework_result" + ZcyAnnouncement23: + category_id: 66485 + name: "框架协议成交结果汇总公告" + type: "framework_summary" + "61-266648": + category_id: 66485 + name: "采购意向公开" + type: "intention" diff --git a/old/main/Main_script.py b/old/main/Main_script.py new file mode 100755 index 0000000..89a29f8 --- /dev/null +++ b/old/main/Main_script.py @@ -0,0 +1,44 @@ +import web_crawler as webc + +"""脚本运行指南""" +# 整个程序的启动需点击右上角的绿色三角按钮(或使用快捷组合键Shift+F10快捷启动)，注意！在启动前，请先在按钮的左侧选择“当前文件”再执行。 +# 注意！！！请不要在打开公告数据导入的目标excel文件时启动该爬虫程序，程序无法对正在运行的进程文件进行修改。务必在爬虫程序启动并将数据成功导入目标excel文件后再打开并查看目标excel文件 + +"""脚本结果读取指南""" +# 这个脚本旨在爬取广西政府采购网的多个公告栏目的公告信息。 +#该脚本在运行结束之后会返回以下结果：<某专栏> 新增 xxx 条公告，已保存至excel文件中，建议确认是否有新增公告后再查看excel文件 +#每次爬取之后会把数据导出至指定excel表格，表格将会把数据按时间倒序的方式排列公告数据，同时会把属于今天的公告数据标红。excel文件默认为桌面的政府采购公告.xlsx('D:/Document/政府采购公告.xlsx') + +"""脚本参数修改指南""" +# 若没有创建相应的excel表格文件，无需担心，程序会先检测是否有对应的excel文件，若不存在该文件，程序便会自动生成。 +# excel表格的存取路径可以在utils.py中修改。请点击左侧的"项目"(或使用快捷组合键Alt+1打开项目栏)，选择utils.py并打开。可以在这个文件中修改某些参数。 +# 若需要修改公告信息的筛选条件以及启动爬虫程序的代理列表，请点击左侧的"项目"(或使用快捷组合键Alt+1打开项目栏)，选择utils.py并打开。可以在这个文件中修改这些参数。 +# 我已经设置了一个定时器，在每天的8:40、14:40、18:00这三个时间段，定时器会启动爬虫程序开始爬取网站。如果需要修改时间段，可以在代码的下半部分修改 +# 修改参数以及代码片段后请重新启动程序。 + +# """脚本定时启动""" +# def job(): +# print(f"任务执行于 {datetime.now()}") +# #调用爬取函数 +# webc.web_crawler() +# print(f"任务完成于 {datetime.now()}") +# +# # 安排任务在每天的8:40、14:40、18:00执行 +# # 此处可以修改时间段，只需修改括号内的时间即可，格式参照原先括号里的数即可，如需额外增加时间段，请复制代码：schedule.every().day.at("时间段").do(job)并粘贴到下方，可增加任意数量的时间段 +# schedule.every().day.at("08:37").do(job) +# schedule.every().day.at("11:55").do(job) +# schedule.every().day.at("14:45").do(job) +# schedule.every().day.at("18:00").do(job) +# +# print("脚本定时任务已启动...") +# while True: +# schedule.run_pending() +# time.sleep(1) + +"""脚本单次启动""" +def main(): + print("脚本单次任务已启动...") + webc.web_crawler() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/old/main/utils.py b/old/main/utils.py new file mode 100755 index 0000000..0e559ea --- /dev/null +++ b/old/main/utils.py @@ -0,0 +1,50 @@ +# 定义公告导出的目标excel文件路径 +FILE_PATH = "D:/Document/政府采购公告.xlsx" +# 初始化excel工作表头 +HEADER = ["标题", "发布时间", "发布单位", "内容链接", "来源栏目"] + +# 初始化工作表数量以及名称 +sheet_names = [ + "采购公告", "招标文件预公示", "采购意向公开", "结果公告", "合同公告", + "更正公告", "单一来源公示", "电子卖场公示", "履约验收公示", + "工程类公告", "框架协议征集公告", "框架协议入围结果公告", + "框架协议成交结果汇总公告", "其他" +] + +# === 初始化筛选条件 === +KEYWORD ="大化" #筛选关键词 +START_DATE ="2025-12-01" #筛选最早发布日期 +END_DATE ="2026-01-31" #筛选最晚发布日期 + +# 代理列表 +PROXIES = [ + #若要添加代理，请先测试代理是否能正常连接，否则请将代理置空 + #"http://10.10.1.10:3218", #此为无效代理,仅作示例 + #"http://user:pass@10.10.1.10:8080", #此为无效代理,仅作示例 + # 带认证的代理 + # 添加更多代理... +] + +USER_AGENTS = [ + #此为用户代理,根据实际情况修改 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15", + # 添加更多 User-Agent... +] + +# 公告来源链接 +SOURCE_URL = { + "ZcyAnnouncement2":[66485,"结果公告"],# 结果公告 + "ZcyAnnouncement3":[66485,"合同公告"],# 合同公告 + "ZcyAnnouncement4":[66485,"更正公告"],# 更正公告 + "ZcyAnnouncement6":[66485,"单一来源公示"],# 单一来源公示 + "ZcyAnnouncement7":[66485,"电子卖场公示"],# 电子卖场公示 + "ZcyAnnouncement10":[66485,"履约验收公示"],# 履约验收公示 + "ZcyAnnouncement11":[66485,"工程类公告"],# 工程类公告 + "ZcyAnnouncement20":[66485,"框架协议征集公告"],# 框架协议征集公告 + "ZcyAnnouncement21":[66485,"框架协议入围结果公告"],# 框架协议入围结果公告 + "ZcyAnnouncement23":[66485,"框架协议成交结果汇总公告"],# 框架协议成交结果汇总公告 + "ZcyAnnouncement1":[66485,"采购公告"],# 采购公告 + "ZcyAnnouncement5":[66485,"招标文件预公示"],# 招标文件预公示 + "61-266648":[66485,"采购意向公开"]# 采购意向公开 +} \ No newline at end of file diff --git a/old/main/web_crawler.py b/old/main/web_crawler.py new file mode 100755 index 0000000..f92cc46 --- /dev/null +++ b/old/main/web_crawler.py @@ -0,0 +1,191 @@ +import requests +import time +from datetime import datetime +import random +from fake_useragent import UserAgent +import logging +import write_to_excel as wte +import utils as Utils + +# === 获取筛选条件 === +KEYWORD =Utils.KEYWORD #关键词筛选 +START_DATE =Utils.START_DATE +END_DATE =Utils.END_DATE + +#获取公告来源链接 +SOURCE_URL = Utils.SOURCE_URL + +# === 反扒配置 === + +#获取代理列表 +PROXIES = Utils.PROXIES +USER_AGENTS = Utils.USER_AGENTS + +MAX_RETRIES = 3 # 最大重连次数 +DELAY_MIN, DELAY_MAX = 1, 3 # 随机延迟范围（秒） + +# === 日志配置 === +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + +def get_random_user_agent(): + """随机选择 User-Agent""" + return random.choice(USER_AGENTS) + +def get_random_proxy(): + """随机选择代理""" + if not PROXIES: + return None + proxy = random.choice(PROXIES) + return {"http": proxy, "https": proxy} + +def check_sensitive_words(user_agent,payload,category_code,childrencode): + """敏感词检查请求""" + # === 接口地址 === + url = "https://zfcg.gxzf.gov.cn/portal/sensitiveWords/check" + headers = { + "User-Agent": user_agent, + "Content-Type": "application/json;charset=UTF-8", + "Origin": "https://zfcg.gxzf.gov.cn", + "Referer": f"https://zfcg.gxzf.gov.cn/site/category?parentId={category_code}&childrenCode={childrencode}", + "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" + } + response = requests.post(url, json=payload, headers=headers) + # print("这是敏感词检查请求返回的响应预览",response.json()) + return response.json() + +def get_announcements(category_code, childrencode,page_no=1, session=None): + """请求接口获取公告列表函数(带反扒机制)""" + if session is None: + session = requests.Session() + payload = { + # 接口的请求参数 + "keyword": KEYWORD, #关键词筛选，此参数即为搜索框中的关键词输入值 + "publishDateBegin": START_DATE, #最早日期筛选 + "publishDateEnd": END_DATE, #最晚日期筛选 + "pageNo": page_no, #页码数 + "pageSize": 15, #页容量 + "categoryCode": category_code, # 该参数指定当前查找的公告栏目 + "_t": int(time.time() * 1000) # 动态时间戳 + } + + # 先执行敏感词检查 + user_agent=get_random_user_agent() + check_response = check_sensitive_words(user_agent,payload, category_code, childrencode) + if not check_response.get("success", False): + logging.warning("敏感词检查失败") + return None + + # 再执行公告数据请求 + # === 接口地址 === + api_url = "https://zfcg.gxzf.gov.cn/portal/category" + headers = { + "User-Agent": user_agent, + "Content-Type": "application/json;charset=UTF-8", + "Origin": "https://zfcg.gxzf.gov.cn", + "Referer": f"https://zfcg.gxzf.gov.cn/site/category?parentId={category_code}&childrenCode={childrencode}", + "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" # 如果需要登录 + } + #若添加了至少1条代理，则尝试使用代理 + if PROXIES: + proxy = 'Default_value' + else: + proxy=None + for retry in range(MAX_RETRIES): + try: + # 尝试使用代理，失败则切换为无代理 + if proxy : + proxy = get_random_proxy() + + if proxy: + logging.info(f"使用代理: {proxy['http']}") + else: + logging.info("未使用代理") + + response = session.post( + api_url, + json=payload, + headers=headers, + proxies=proxy, + timeout=10 + ) + if response.status_code == 200: + data = response.json() + if data.get("success",True): + return data + else: + logging.warning(f"接口返回失败: {data.get('error', '未知错误')}") + else: + logging.warning(f"请求失败，状态码: {response.status_code}") + except requests.exceptions.ProxyError as pe: + # 代理错误时记录并跳过，继续无代理请求 + logging.error(f"代理错误: {pe}. 切换为无代理模式") + proxy = None # 下次请求不使用代理 + except Exception as e: + logging.error(f"请求异常: {e}") + + # 重试前等待 + delay = random.uniform(DELAY_MIN, DELAY_MAX) + logging.info(f"第 {retry + 1}/{MAX_RETRIES} 次重试，等待 {delay:.1f} 秒...") + time.sleep(delay) + + return None # 所有重试失败 + +def parse_data(data,category_code,source_name): + """解析公告数据函数""" + results = [] + for item in data["result"]["data"]["data"]: + results.append({ + "标题": item["title"], + "发布时间": datetime.fromtimestamp(int(item["publishDate"]) / 1000).strftime("%Y-%m-%d"), + "发布单位": item["purchaseName"], + "内容链接": f"https://zfcg.gxzf.gov.cn/site/detail?parentId={category_code}&articleId={item['articleId']}", + "来源栏目":source_name + }) + return results + +#爬虫主函数 +def web_crawler(): + session = requests.Session() + # print("这是本地会话存储：",session) + # 初始化引用值为信息公告,先爬取信息公告栏目的公告数据 + childrencode = "ZcyAnnouncement" + # 爬取代码主体，外循环为来源栏目的循环，即遍历需要爬取的所有来源栏目 + for key in SOURCE_URL: + page = 1 + all_results = [] + # print("这是正在爬取的栏目", SOURCE_URL[key][1]) + # print("这是正在爬取的栏目的目录码", str(key)) + # 判断正确的引用值 + # 内循环主体即为在符合筛选条件的公告列表中遍历所有公告，爬取每个公告需要的指定数据。 + while True: + # 这是分割线 + # logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线") + # logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线") + # logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线") + # logging.info(f"正在爬取第 {page} 页...") + response = get_announcements(str(key), childrencode, page,session) + if response["result"]["data"]["empty"] or not response["result"]["data"]["data"]: + break + # logging.info(f"请求后共返回 {response["result"]["data"]["total"]} 条公告") + current_data = parse_data(response, SOURCE_URL[key][0], SOURCE_URL[key][1]) + if not current_data: + break + # logging.info(f"解析后共爬取 {len(current_data)} 条公告") + all_results.extend(current_data) + page += 1 + # 随机延迟 + delay = random.uniform(DELAY_MIN, DELAY_MAX) + time.sleep(delay) + + # print(f"从 {SOURCE_URL[key][1]} 共爬取 {len(all_results)} 条公告") + # 调用excel读写函数，将公告数据筛选后保存到Excel中 + wte.write_to_excel(all_results,SOURCE_URL[key][1]) + # 这是分割线 + print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线") + print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线") + print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线") + + + + + diff --git a/old/main/write_to_excel.py b/old/main/write_to_excel.py new file mode 100755 index 0000000..cfeda12 --- /dev/null +++ b/old/main/write_to_excel.py @@ -0,0 +1,84 @@ +import pandas as pd +from datetime import datetime +from openpyxl.styles import Font, Alignment, PatternFill, Border, Side +import schedule +import time +import utils as Utils + +# 获取excel文件路径和excel的工作表头 +FILE_PATH = Utils.FILE_PATH +HEADER = Utils.HEADER + +# 获取工作表数量以及名称 +sheet_names = Utils.sheet_names + + +def write_to_excel(results,source=""): + # 检查文件是否存在，如果不存在则创建一个新的Excel文件 + try: + df_existing = pd.read_excel(FILE_PATH, sheet_name=None) + except FileNotFoundError: + df_existing = {name: pd.DataFrame(columns=HEADER) for name in sheet_names} + + today_str = datetime.now().strftime('%Y-%m-%d') + aditional_item=0 # 记录本次爬取新增的公告数量 + # 将爬取的公告对号入座填充至excel文件的工作表中 + for result in results: + sheet_name = result["来源栏目"] if result["来源栏目"] in sheet_names else "其他" + df_sheet = df_existing[sheet_name] + + # 检查是否已存在相同内容链接的记录 + if not df_sheet[df_sheet["内容链接"] == result["内容链接"]].empty: + continue + + # 添加新记录 + new_row = pd.DataFrame([result]) + df_sheet = pd.concat([df_sheet, new_row], ignore_index=True) + aditional_item+=1 # 若有新增公告，则加1 + # 按时间降序排列 + df_sheet.sort_values(by='发布时间', ascending=False, inplace=True) + + # 更新现有数据 + df_existing[sheet_name] = df_sheet + + # 写入Excel文件 + with pd.ExcelWriter(FILE_PATH, engine='openpyxl') as writer: + for sheet_name, df in df_existing.items(): + df.to_excel(writer, sheet_name=sheet_name, index=False) + + # 获取工作表对象 + worksheet = writer.sheets[sheet_name] + + # 设置列宽 + for col_idx, column_width in enumerate([40, 15, 20, 30, 20]): + worksheet.column_dimensions[chr(65 + col_idx)].width = column_width + + # 创建红色填充 + red_fill = PatternFill(start_color="FF0000", end_color="FF0000", fill_type="solid") + + # 创建边框样式 + thin_border = Border(left=Side(style='thin'), + right=Side(style='thin'), + top=Side(style='thin'), + bottom=Side(style='thin')) + + # 应用样式 + for row in worksheet.iter_rows(min_row=1, max_col=len(HEADER), max_row=worksheet.max_row): + publish_date = row[1].value if len(row) > 1 else None + # print("这是publish_date",publish_date) + # 若公告发布日期为今天，则将其单元格背景填充至红色 + if publish_date == today_str: + for cell in row: + cell.fill = red_fill + + for idx, cell in enumerate(row): + alignment = Alignment(horizontal='left' if idx < 4 else 'general', vertical='top', wrap_text=True) + cell.alignment = alignment + cell.border = thin_border + print(f"<{source}> 新增 {aditional_item} 条公告，已保存至excel文件中") + + + + + + diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..66e4951 --- /dev/null +++ b/start.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# 广西政府采购网公告监控系统启动脚本 + +echo "=== 广西政府采购网公告监控系统启动脚本 ===" + +# 检查Python环境 +if ! command -v python3 &> /dev/null; then + echo "错误: 未找到python3，请确保Python 3.8+已安装" + exit 1 +fi + +# 检查虚拟环境 +if [ ! -d "venv" ]; then + echo "错误: 未找到虚拟环境，请先运行: python3 -m venv venv" + exit 1 +fi + +# 激活虚拟环境 +echo "激活虚拟环境..." +source venv/bin/activate + +# 检查依赖 +echo "检查依赖..." +python3 -c "import sys; sys.path.insert(0, 'gx_gp_monitor'); from core.config_manager import load_config; print('依赖检查通过')" 2>/dev/null +if [ $? -ne 0 ]; then + echo "错误: 依赖不完整，请运行: pip install -r gx_gp_monitor/requirements.txt" + exit 1 +fi + +# 进入项目目录 +cd gx_gp_monitor + +# 解析命令行参数 +COMMAND=${1:-"status"} +shift + +echo "执行命令: $COMMAND" + +# 执行命令 +case $COMMAND in + "crawl") + python3 main.py crawl "$@" + ;; + "cleanup") + python3 main.py cleanup "$@" + ;; + "status") + python3 main.py status "$@" + ;; + *) + echo "使用方法: $0 {crawl|cleanup|status} [参数]" + echo "" + echo "示例:" + echo " $0 crawl # 执行一次爬取" + echo " $0 crawl --keywords 大化 # 带关键词爬取" + echo " $0 cleanup --days 30 # 清理30天前数据" + echo " $0 status # 查看系统状态" + exit 1 + ;; +esac + +echo "=== 脚本执行完成 ===" diff --git a/test_system.py b/test_system.py new file mode 100644 index 0000000..446a3a0 --- /dev/null +++ b/test_system.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +系统测试脚本 +""" + +import sys +import os +from pathlib import Path + +# 添加项目路径 +project_root = Path(__file__).parent / "gx_gp_monitor" +sys.path.insert(0, str(project_root)) + +def test_imports(): + """测试模块导入""" + print("测试模块导入...") + + try: + from core.config_manager import load_config, get_config + print("✓ core.config_manager 导入成功") + + from core.logger import get_logger + print("✓ core.logger 导入成功") + + from core.models import Announcement, AnnouncementType + print("✓ core.models 导入成功") + + from core.database import get_database_manager + print("✓ core.database 导入成功") + + from crawler.spider import GXGPSpider + print("✓ crawler.spider 导入成功") + + from filters.filters import AnnouncementFilter + print("✓ filters.filters 导入成功") + + from storage.postgresql import PostgreSQLStorage + print("✓ storage.postgresql 导入成功") + + from notification.wechat import WeChatService + print("✓ notification.wechat 导入成功") + + return True + except ImportError as e: + print(f"✗ 导入失败: {e}") + return False + +def test_config(): + """测试配置加载""" + print("\n测试配置加载...") + + try: + from core.config_manager import load_config + + config = load_config() + print("✓ 配置文件加载成功") + print(f" - 调试模式: {config.debug}") + print(f" - 日志级别: {config.log_level.value}") + print(f" - 数据库启用: {config.database.enabled}") + print(f" - 企业微信启用: {config.wechat_app.enabled}") + print(f" - 公告来源数量: {len(config.sources)}") + + return True + except Exception as e: + print(f"✗ 配置加载失败: {e}") + return False + +def test_database_connection(): + """测试数据库连接""" + print("\n测试数据库连接...") + + try: + from core.config_manager import get_config + from core.database import init_database + + config = get_config() + if not config.database.enabled: + print("⚠ 数据库功能已禁用，跳过连接测试") + return True + + # 尝试初始化数据库 + init_database() + print("✓ 数据库连接成功") + return True + + except Exception as e: + print(f"✗ 数据库连接失败: {e}") + return False + +def main(): + """主测试函数""" + print("=== 广西政府采购网公告监控系统测试 ===") + + # 测试导入 + if not test_imports(): + print("\n❌ 模块导入测试失败") + return False + + # 测试配置 + if not test_config(): + print("\n❌ 配置加载测试失败") + return False + + # 测试数据库 + if not test_database_connection(): + print("\n❌ 数据库连接测试失败") + return False + + print("\n✅ 所有测试通过！系统准备就绪。") + return True + +if __name__ == "__main__": + success = main() + sys.exit(0 if success else 1)