From 28c57a040a24129411d81ce6759f9bf853d83175 Mon Sep 17 00:00:00 2001 From: v6ole Date: Wed, 7 Jan 2026 17:37:09 +0800 Subject: [PATCH] =?UTF-8?q?=E6=89=8B=E5=8A=A8=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gx_gp_monitor/README.md | 281 ++++++ gx_gp_monitor/__init__.py | 7 + gx_gp_monitor/__main__.py | 8 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 348 bytes .../__pycache__/__main__.cpython-313.pyc | Bin 0 -> 357 bytes .../__pycache__/main.cpython-313.pyc | Bin 0 -> 15834 bytes gx_gp_monitor/config/__init__.py | 1 + gx_gp_monitor/config/config.yaml | 135 +++ gx_gp_monitor/core/__init__.py | 1 + .../core/__pycache__/__init__.cpython-313.pyc | Bin 0 -> 192 bytes .../config_manager.cpython-313.pyc | Bin 0 -> 14489 bytes .../core/__pycache__/database.cpython-313.pyc | Bin 0 -> 25136 bytes .../core/__pycache__/logger.cpython-313.pyc | Bin 0 -> 15955 bytes .../core/__pycache__/models.cpython-313.pyc | Bin 0 -> 13066 bytes .../__pycache__/reliability.cpython-313.pyc | Bin 0 -> 19852 bytes gx_gp_monitor/core/config_manager.py | 394 +++++++++ gx_gp_monitor/core/database.py | 612 ++++++++++++++ gx_gp_monitor/core/logger.py | 354 ++++++++ gx_gp_monitor/core/models.py | 255 ++++++ gx_gp_monitor/core/reliability.py | 489 +++++++++++ gx_gp_monitor/crawler/__init__.py | 1 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 195 bytes .../__pycache__/parsers.cpython-313.pyc | Bin 0 -> 11226 bytes .../__pycache__/spider.cpython-313.pyc | Bin 0 -> 19508 bytes gx_gp_monitor/crawler/parsers.py | 301 +++++++ gx_gp_monitor/crawler/spider.py | 458 ++++++++++ gx_gp_monitor/filters/__init__.py | 1 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 195 bytes .../__pycache__/filters.cpython-313.pyc | Bin 0 -> 18031 bytes gx_gp_monitor/filters/filters.py | 471 +++++++++++ gx_gp_monitor/logs/gx_gp_monitor.log | 799 ++++++++++++++++++ gx_gp_monitor/main.py | 309 +++++++ gx_gp_monitor/notification/__init__.py | 1 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 200 bytes .../__pycache__/wechat.cpython-313.pyc | Bin 0 -> 22081 bytes gx_gp_monitor/notification/wechat.py | 555 ++++++++++++ gx_gp_monitor/onu.md | 11 + gx_gp_monitor/requirements.txt | 39 + gx_gp_monitor/scheduler/__init__.py | 1 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 197 bytes .../__pycache__/scheduler.cpython-313.pyc | Bin 0 -> 14537 bytes gx_gp_monitor/scheduler/scheduler.py | 372 ++++++++ gx_gp_monitor/storage/__init__.py | 1 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 195 bytes .../__pycache__/md_generator.cpython-313.pyc | Bin 0 -> 18238 bytes .../__pycache__/postgresql.cpython-313.pyc | Bin 0 -> 15029 bytes gx_gp_monitor/storage/md_generator.py | 399 +++++++++ gx_gp_monitor/storage/postgresql.py | 367 ++++++++ logs/gx_gp_monitor.log | 23 + old/WeChatService.py | 625 ++++++++++++++ old/config.yaml | 104 +++ old/main/Main_script.py | 44 + old/main/utils.py | 50 ++ old/main/web_crawler.py | 191 +++++ old/main/write_to_excel.py | 84 ++ start.sh | 63 ++ test_system.py | 114 +++ 57 files changed, 7921 insertions(+) create mode 100644 gx_gp_monitor/README.md create mode 100644 gx_gp_monitor/__init__.py create mode 100644 gx_gp_monitor/__main__.py create mode 100644 gx_gp_monitor/__pycache__/__init__.cpython-313.pyc create mode 100644 gx_gp_monitor/__pycache__/__main__.cpython-313.pyc create mode 100644 gx_gp_monitor/__pycache__/main.cpython-313.pyc create mode 100644 gx_gp_monitor/config/__init__.py create mode 100644 gx_gp_monitor/config/config.yaml create mode 100644 gx_gp_monitor/core/__init__.py create mode 100644 gx_gp_monitor/core/__pycache__/__init__.cpython-313.pyc create mode 100644 gx_gp_monitor/core/__pycache__/config_manager.cpython-313.pyc create mode 100644 gx_gp_monitor/core/__pycache__/database.cpython-313.pyc create mode 100644 gx_gp_monitor/core/__pycache__/logger.cpython-313.pyc create mode 100644 gx_gp_monitor/core/__pycache__/models.cpython-313.pyc create mode 100644 gx_gp_monitor/core/__pycache__/reliability.cpython-313.pyc create mode 100644 gx_gp_monitor/core/config_manager.py create mode 100644 gx_gp_monitor/core/database.py create mode 100644 gx_gp_monitor/core/logger.py create mode 100644 gx_gp_monitor/core/models.py create mode 100644 gx_gp_monitor/core/reliability.py create mode 100644 gx_gp_monitor/crawler/__init__.py create mode 100644 gx_gp_monitor/crawler/__pycache__/__init__.cpython-313.pyc create mode 100644 gx_gp_monitor/crawler/__pycache__/parsers.cpython-313.pyc create mode 100644 gx_gp_monitor/crawler/__pycache__/spider.cpython-313.pyc create mode 100644 gx_gp_monitor/crawler/parsers.py create mode 100644 gx_gp_monitor/crawler/spider.py create mode 100644 gx_gp_monitor/filters/__init__.py create mode 100644 gx_gp_monitor/filters/__pycache__/__init__.cpython-313.pyc create mode 100644 gx_gp_monitor/filters/__pycache__/filters.cpython-313.pyc create mode 100644 gx_gp_monitor/filters/filters.py create mode 100644 gx_gp_monitor/logs/gx_gp_monitor.log create mode 100644 gx_gp_monitor/main.py create mode 100644 gx_gp_monitor/notification/__init__.py create mode 100644 gx_gp_monitor/notification/__pycache__/__init__.cpython-313.pyc create mode 100644 gx_gp_monitor/notification/__pycache__/wechat.cpython-313.pyc create mode 100644 gx_gp_monitor/notification/wechat.py create mode 100644 gx_gp_monitor/onu.md create mode 100644 gx_gp_monitor/requirements.txt create mode 100644 gx_gp_monitor/scheduler/__init__.py create mode 100644 gx_gp_monitor/scheduler/__pycache__/__init__.cpython-313.pyc create mode 100644 gx_gp_monitor/scheduler/__pycache__/scheduler.cpython-313.pyc create mode 100644 gx_gp_monitor/scheduler/scheduler.py create mode 100644 gx_gp_monitor/storage/__init__.py create mode 100644 gx_gp_monitor/storage/__pycache__/__init__.cpython-313.pyc create mode 100644 gx_gp_monitor/storage/__pycache__/md_generator.cpython-313.pyc create mode 100644 gx_gp_monitor/storage/__pycache__/postgresql.cpython-313.pyc create mode 100644 gx_gp_monitor/storage/md_generator.py create mode 100644 gx_gp_monitor/storage/postgresql.py create mode 100644 logs/gx_gp_monitor.log create mode 100644 old/WeChatService.py create mode 100644 old/config.yaml create mode 100755 old/main/Main_script.py create mode 100755 old/main/utils.py create mode 100755 old/main/web_crawler.py create mode 100755 old/main/write_to_excel.py create mode 100755 start.sh create mode 100644 test_system.py diff --git a/gx_gp_monitor/README.md b/gx_gp_monitor/README.md new file mode 100644 index 0000000..afe146d --- /dev/null +++ b/gx_gp_monitor/README.md @@ -0,0 +1,281 @@ +# 广西政府采购网公告监控系统 + +广西政府采购网公告爬取和监控的智能系统，支持多种公告类型的自动爬取、智能筛选、数据存储和通知推送。 + +## 功能特性 + +### 🕷️ 智能爬虫监控 +- 自动爬取广西政府采购网(zfcg.gxzf.gov.cn)的各类采购公告 +- 支持10+种公告类型：采购公告、结果公告、合同公告、更正公告、招标文件预公示、单一来源公示、电子卖场公示、履约验收公示、工程类公告、框架协议征集公告等 +- 智能反爬虫机制：随机User-Agent、代理轮换、请求延迟控制 + +### 🎯 多维度智能筛选 +- **关键词过滤**：支持多关键词精确匹配和模糊匹配 +- **日期范围筛选**：支持开始日期和结束日期过滤 +- **自动去重**：基于内容哈希的智能去重机制 +- **来源筛选**：按公告类型进行筛选 + +### 💾 数据持久化存储 +- **PostgreSQL数据库**：高效的数据存储和查询 +- **自动清理**：默认90天数据保留，可配置 +- **性能优化**：连接池、索引优化、批量操作 + +### 🔔 高可用性保障 +- **重试机制**：指数退避重试策略 +- **超时控制**：连接超时和读取超时设置 +- **幂等操作**：防止重复处理 +- **异常处理**：完善的异常捕获和恢复 +- **结构化日志**：详细的运行日志记录 + +### 📱 通知系统 +- **企业微信通知**：支持文本和Markdown格式消息 +- **实时推送**：新公告发现即时通知 +- **系统告警**：异常情况自动告警 + +### 📊 Markdown输出 +- **自动生成**：将公告输出为Markdown格式文件（onu.md） +- **结构化展示**：按来源分组、时间排序 +- **今日高亮**：突出显示今日发布的公告 + +## 安装和使用 + +### 环境要求 +- Python 3.8+ +- PostgreSQL 12+ + +### 安装步骤 + +1. **克隆项目** +```bash +cd /path/to/your/workspace +# 项目已创建在 gx_gp_monitor 目录下 +``` + +2. **创建虚拟环境** +```bash +python3 -m venv venv +source venv/bin/activate # Linux/Mac +# 或 +venv\Scripts\activate # Windows +``` + +3. **安装依赖** +```bash +pip install -r requirements.txt +``` + +4. **配置数据库** +```bash +# 创建PostgreSQL数据库 +createdb gx-gp-notify + +# 修改配置文件 config/config.yaml 中的数据库连接信息 +``` + +5. **修改配置** +```bash +# 编辑 config/config.yaml 文件 +# 设置数据库连接、企业微信配置、关键词等 +``` + +### 使用方法 + +#### 命令行使用 + +```bash +# 查看帮助 +python main.py --help + +# 执行一次爬取 +python main.py crawl + +# 带参数爬取 +python main.py crawl --keywords "大化" "信息化" --max-pages 5 + +# 启动调度器（定时任务） +python main.py scheduler + +# 数据清理 +python main.py cleanup --days 30 + +# 查看系统状态 +python main.py status +``` + +#### 编程接口使用 + +```python +from gx_gp_monitor.crawler.spider import crawl_announcements +from gx_gp_monitor.filters.filters import filter_from_config +from gx_gp_monitor.storage.postgresql import save_announcements_to_storage +from gx_gp_monitor.storage.md_generator import generate_onu_md +from gx_gp_monitor.notification.wechat import send_announcements_notification + +# 执行爬取 +results = crawl_announcements() + +# 筛选公告 +filter_obj = filter_from_config() +filtered_announcements, stats = filter_obj.filter(all_announcements) + +# 保存到数据库 +saved_count = save_announcements_to_storage(filtered_announcements) + +# 生成Markdown文件 +generate_onu_md(filtered_announcements) + +# 发送通知 +send_announcements_notification(filtered_announcements) +``` + +## 配置说明 + +### 主要配置文件：`config/config.yaml` + +```yaml +# 调试模式 +debug: false + +# 日志配置 +log_level: INFO +log_file: logs/gx_gp_monitor.log + +# 爬虫配置 +crawler: + base_url: "https://zfcg.gxzf.gov.cn" + timeout: 30 + max_retries: 3 + keyword: ["大化", "信息化"] # 关键词筛选 + max_pages: 10 + +# 数据库配置 +database: + enabled: true + host: "localhost" + port: 5432 + name: "gx-gp-notify" + user: "your_user" + password: "your_password" + data_retention_days: 90 + +# 企业微信配置 +wechat_app: + enabled: true + corp_id: "your_corp_id" + agent_id: "your_agent_id" + secret: "your_secret" + token: "your_token" + encoding_aes_key: "your_aes_key" + +# 调度配置 +scheduler: + enabled: true + jobs: + - name: "daily_crawl" + cron: "0 8,14,18 * * *" + enabled: true +``` + +### 公告来源配置 + +系统支持以下公告类型： + +- `ZcyAnnouncement1`: 采购公告 +- `ZcyAnnouncement2`: 结果公告 +- `ZcyAnnouncement3`: 合同公告 +- `ZcyAnnouncement4`: 更正公告 +- `ZcyAnnouncement5`: 招标文件预公示 +- `ZcyAnnouncement6`: 单一来源公示 +- `ZcyAnnouncement7`: 电子卖场公示 +- `ZcyAnnouncement10`: 履约验收公示 +- `ZcyAnnouncement11`: 工程类公告 +- `ZcyAnnouncement20`: 框架协议征集公告 +- `ZcyAnnouncement21`: 框架协议入围结果公告 +- `ZcyAnnouncement23`: 框架协议成交结果汇总公告 +- `61-266648`: 采购意向公开 + +## 架构设计 + +``` +gx_gp_monitor/ +├── config/ # 配置管理 +├── core/ # 核心模块 +│ ├── models.py # 数据模型 +│ ├── config_manager.py # 配置管理 +│ ├── database.py # 数据库操作 +│ ├── logger.py # 日志管理 +│ └── reliability.py # 高可用性 +├── crawler/ # 爬虫模块 +│ ├── spider.py # 爬虫核心 +│ └── parsers.py # 数据解析 +├── filters/ # 筛选模块 +├── storage/ # 存储模块 +│ ├── postgresql.py # PostgreSQL存储 +│ └── md_generator.py # Markdown生成 +├── notification/ # 通知模块 +├── scheduler/ # 调度模块 +└── main.py # 主程序入口 +``` + +## 日志和监控 + +### 日志文件 +- 默认日志文件：`logs/gx_gp_monitor.log` +- 日志轮转：10MB大小限制，保留5个备份文件 +- 日志级别：DEBUG、INFO、WARNING、ERROR、CRITICAL + +### 监控指标 +- 爬取统计：成功/失败次数、响应时间 +- 数据统计：公告数量、新增数量、去重统计 +- 系统状态：内存使用、磁盘空间、健康检查 + +## 故障排除 + +### 常见问题 + +1. **数据库连接失败** + - 检查PostgreSQL服务是否运行 + - 验证数据库连接配置 + - 确认用户权限 + +2. **爬虫请求失败** + - 检查网络连接 + - 验证目标网站是否可访问 + - 调整请求间隔和重试策略 + +3. **企业微信通知失败** + - 检查企业微信配置 + - 验证应用ID和密钥 + - 确认网络能访问企业微信API + +4. **权限问题** + - 确保日志目录和输出文件目录有写入权限 + - 检查数据库用户权限 + +### 调试模式 + +启用调试模式获取更详细的日志： + +```yaml +debug: true +log_level: DEBUG +``` + +## 许可证 + +本项目采用 MIT 许可证。 + +## 贡献 + +欢迎提交Issue和Pull Request来改进这个项目。 + +## 版本历史 + +- **v1.0.0** (2024-01-XX) + - 初始版本发布 + - 支持10+种公告类型爬取 + - 实现智能筛选和去重 + - PostgreSQL数据存储 + - 企业微信通知 + - Markdown文件输出 + - 定时任务调度 diff --git a/gx_gp_monitor/__init__.py b/gx_gp_monitor/__init__.py new file mode 100644 index 0000000..72e54cf --- /dev/null +++ b/gx_gp_monitor/__init__.py @@ -0,0 +1,7 @@ +""" +广西政府采购网公告监控系统 +广西政府采购网公告爬取和监控的智能系统 +""" + +__version__ = "1.0.0" +__author__ = "GX GP Monitor Team" diff --git a/gx_gp_monitor/__main__.py b/gx_gp_monitor/__main__.py new file mode 100644 index 0000000..142bcf8 --- /dev/null +++ b/gx_gp_monitor/__main__.py @@ -0,0 +1,8 @@ +""" +包的main入口，使项目可以直接通过 python -m gx_gp_monitor 运行 +""" + +from .main import main + +if __name__ == "__main__": + main() diff --git a/gx_gp_monitor/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13702dbf9577be86e8173be89161c406a389b937 GIT binary patch literal 348 zcmey&%ge<81U)KonTvt+V-N=h7@>^M0zk%8hG2#whG51b#$cu*rgUaameORPFxS(a z`(G^G|7^=@BsDSDPm}o;dwhIKesX;L zE$;aEvecsD%=|nchbullv9u&3zbHO_CBtWsM{e2ZXXNLm>X(`2=cMWvR2CHFXQd{W z=(|VgrWfeu<(FipRqCf##HSa;=R&N|kB`p;+7%zKS5SG2!zMRBr8Fniu80F@7$|6p h#el>IW=2NFk4(&rEFZWS_{AEyZ}2KMauu-wr2qlciBSLm literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/__pycache__/__main__.cpython-313.pyc b/gx_gp_monitor/__pycache__/__main__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ffa7281c45a3526ebc984c824283d1e0c4734ef3 GIT binary patch literal 357 zcmey&%ge<81ZpaAnNxxEV-N=hn4pZ$20+GChG2#whG52ECT~VB#v(=qh8U(`W~&GW zhC~hqh8RXA22-eV1%`B%bQVq4Qh%TT*VCTX=d)UJ6EpLkwl00zzxe&0o+o?vzg)QU z`RsL1``16&z4ZC)EzkOvzHFHFVt>0rL1jrsex8DEu0nc6e0o8AZhl^7Nq&*Si~SQ` zEbQTW$p|!4lkpY{$iQ11@$n!!KHg80`4)S8d`fK9ZN6y;~7CYR{DN9d*(=;q~@WTsW>qgbgAGhDBr@)n0pZhlH> zPO4oID7Zk5C{_d#AD9^#89y>HF|vH+U|AEG9NK#ahQ&k$B$3?pn zF)zAy)i9zqSDiSTUxYDC(|a==^v*;^%PxA-(<~DJbpDsvu-B6X07?Yed?S# zV4*#CtvL(!*=O%>fBW07@3p^rpx0|Ccp3`6-?g-yqW%p(_(vJZJdjE$>NG`D3`I+5 zX_JJJkhhePlDCYJ!CTfOZ&WY}Gqp#NN9|FrlyI$E;(2t@H>2JoX-SNJ^SPE}J{Nt{ zl090$r{?jk1X}PhQ%jHA+NLFqIZRHYnK6@ZT~lsj z9+TIY&*V22Fa?c;Od*Uq0jONgQ^}O1QF#=dkU`OjJjF~YZOWh+E1d*s*`DQea_ugP zva2a-Pgx$dk{cF`t5e!3&87A%RZ;eOieifP6ayyxo}zfFQVR3d5&;yw==d5bP=IiT z$3jS2c8Z!sve3%N(`}}5C|b%CZ76E);@3_D9BQA$lHzkh6HWM+cW2(eJ9X-A z_}2N^W5;HGer@{BK={=U!vn8RzcVoP)|Kg>jZTkVm>e0M9(p5u>*Qk_^nDmU`F8k6 zCn7D+{c!5+t(hO(;ZSI1k;G<;BqV>v?%i*ZhO|z%y~Eb-cJ1rx3~4$Y-bfy+x?Eje zo73Id>0r1jE)T6u?fV_=FWEeYJYGk)ZNJ0r1Y}su*bh2wc9+ZTbG19V9WJj2iWB#B zIlT_XwvTal^XO$E9YOE$x*29D(eJ$wORHFP>$4#w_v z*xW9kt-Av#&ha>09Se}S+}^HzUF~*nm)jLeBLESy#WfarNRNZ}z`#0uP8hJk`WFBa8J|n|?6AG)-s|z<(G1_X9e(xw*;8j{ZrrFfUxlfL;X8brp=##F&FM?m zC-1&HIr`S@rC(0JbA9R}rr|rMXYTgTTsraR+b5>_-k!X3cly*&t-0rrcfZ?JX)f(H zcfMll?6GybVMW}`tORhi!@uyTBDn|qA#7E6Uz9K^I)PU0QPXPpkJ}ShrKB~H8V$f|p+rjSA|+Z{4>)vmJfwQsz{GdTEs3G{ z&AT=~vz?==zNcq4AG-GyK@xLIi#K!l)+rccJC1~ZkPpe=|A7?}VSn~m&RTPSt@$gOJ4MRkBu69OZ1?)tZw9Ukq z&~a@*4fnJP)Z<9nQuy+gG(H!7(;5^fAPBTBX}z>l$Gt^@K<`7k!+r0Due=dH@iuoj zED|OgN_{wzwQJX!W1gu;#YDLOM)>t1GqmvGWKAFMn>zntt+~QlZY?JaYWCGPr|(>! zdi&Vq=r3x`J{(H?_klr8-g#|$ z^vd+;HI8NZa7H+Sqx4?~HrBt>hv#5n6+u&LX0TavgGJBCAwver;Lc>0KoXL7x%Rmk z5ad)yb8D?;YgyJ^6((dRXyj;=4bh*4Cna9gOa>x@>db}MDhOx*PGk`LM2rqGX zaOa!NW+oTEHF$ix?9Q&kj!^o7BOI+-4xpZ&LDEaj$y4LfCo>8!zBHDxqHp7griV%@ zbLr5|8*NwH{24V^GO0^E*>t>#P1_OBJu|_543-9T7Cvts$_!c>M&jAx&1~A1fNm?` zNGb}Niv6bI31iBc_Nyb-VIz?JsM;&^1h$(*8210FduIxc`o>uU=n6eO#m2P$Bz7R$bmQ>j4?;X~(7DADnK&h<*YoVMKfl)q>><9$H{d zaNq=%v`f&{T=G`AaHKORs!eDg6Ps|jfK|xKC|Yin7EooBOVKJfQClkDSSnkUEdnbO zkWoS(z-G3?p=?#eVso+EMj65ON{Z3~n|cD3iZ+35wJP}5@f5AB6f;MGW#K`_$E9jj zwFr!gE5kI3!w1y&9DhRZDpBu(HV?h4=k=bz<8KvUT$(6L6P0S|IG3(f&Cj46tQQa| zNqqb0yG3AjqOurlPrwAT)RxJ^ih9)1GSL`>85ETXG`DJ6wXM3=xRxv)PxK9?$Tb!` zP-1tvC3(!RpnAXVa|1}%K3t;RaroR&?w+>g3zde71oRJpgy$Vq4jMR zzNl3n-8o{B2*M@?3>|NkdZZvq3@&}Wz98}zi4vm^?ZBzyS3sSdCdR4$K1B2IKJfoL-L)t=ve9zG1Tu zrKWl6-Am!OE}#q}#tSK&zV^;+@9}W|TjAqZC-0m!lZFJ6z%+p7ZHzD_)P}%LaCC(9 z9jMm}pNSp59w+MXqA~-=zXTwQiox?0(tF)ryVFK25x}GtkSNf?=m;ft+nJX-+y`AY z-hSu^C5TM{8`ucmkdA8y_J$n>i-$lwncO6S@B~gauY#LePF8~QFXd=bfLL-f&6Z>$ z-5A^u3@TqC{XA1Cq;$OE1q&u5b2?lMPH;%ck-?ylX3)AIk}{+@=xE<>_uA||Jt39D zW#8*`bTG(Km{pjp#ssz@#nfVg+lX0@2?{n#JSQK4iBil8On}(|DV*&VS>8~B-MPS! z3n{s}kR~!U?!7ODQuyheZ*?(GBT6lqaj^J&Frk|64jaFT;yERo|7K9(_u%M2PKzE+ zWXGYMMTiTfEs)qkO#>qM0CZv$*sJ1JMI@v+-YR3xe*n&0enr&>__JFP#`pqaAOE2wf9Isp( ztla3Y+{kWvZmjZozpk9s*L8ro*AW>+_lW$frf zl_SH#TAmt9f4Z*`$Xt@&*ZhACg+m25mS0^ST=uko+0(&gTi|aZ_i48N`LWzx{qX>n zWuC|_U<<3qa#u}c<*?@Jv8+|&=>_9U%f|~VzbdW@7H{$wZyGPG2o^r&FMMizS#@yP zR{yfC<3*Lhq7D9{4fo>|xrz4;`jq&-hKEZcV`Vd}LkEWTvuUdWy45gEecO5Ugv|Q3 z91z>esRq zOtUvu!0VntvpG?AFWtIXFZ)y_f&8a>1*9Q)hy9Qzwy|^+mj3@&M~RwMpy~rj+!I?X zX_bO}m5JqRtCW`4o&v8EFdvYXF*K4EHIFm;ZXrIx=r`sBX$h>fP0$%=1+5f%1+4%d z(1}#guNrkCkUe;C$PCeMZ6v!15|J5q!Df6WoE!GeO7?6;h?9sVo^)iq> zv9>~d#sgYNw`(KMlAQaMK2^0earMtLTkVfIYi?e4hN9D}S-D6YdFL0m? z&I>2@h`{1H<(d+46TbP6T$veU8OM@U9S*0%3%(xnf*v_q=kxIfrXHZ0EgBL3#uehs zmd%*kf(Z)2h1^!oaa$UyK&)|{ax9T!d#H0G&*PY#masI=*A>c$#qPMeBhboo4H9H{ zi6Vx#I-skW%vwIQaV)E%Z@a)_s)zq>*v)QgWz(Ju=$;4WV9XwTdCXwGxbsTerM6&U zoxiXyShxZHCXB`7NyXzS=8M@E4Q$y{Y|_&Y6|#iHISpkl7}onu>sZ~oM~{5n?F}DE zMjJ*}v*{ZHx=o85dE5a<>sQy;QNOO!tY0VlSYcgXBm1~gg83Q+r1LyNf-~eF!Q%M- zfANI3QdQtv2ltiQC27N1CQply<4Y*;D~sx5oM7M(!pRdsJ_%Edb1%k3%XU&u!BUEz zc-~4dQ#+~0@`G-7sZl8ZLfn~1v>;VJPjJZTtr7s1i9!195t8!Y!&8p0oQ%}3~R+S&`OuQ zUf%p8?j7a9AP>Zpp)K+x%JhE)W%03KXhTFE5L!SUkav_8O9F9gaekti;U9!={WyGf zh;tYbCzVh(HFPO_`{X>?j%$f~@&R7YtdpF6(h9}&Kk)kxB&KwyH0lHekFl)TR{)r& zPW>X>Hyl24e(Ku)p1gZ0eDWsx;IF+GK0g$G^=GqZu34>Cuu(*eABkdcg)QK4_H=c+ zeBB|%KE~edV9Zb&@i+By{5&23z#Q3ccXi;cLRAcwE?g;C*ErqncBf~Zb$+XN5iPm` zzwM<2?R9;?2)uepz-R_u+b=(@V{kY-eYnenc?H484Gji0+uIvHc8j|~A`-KCb`8K1 zk-eC^kCW$Qie^t;pMGa>a^%|7+t;{>BJMq*D{hs;w~vNLMrzHosNj&5767<4o*w;& zsdq2v{hdh6z`%{xOXc<0;`kp^>`P`ZUtFU<^^mYCoB`= zz+Q;Wg6#W$M5VVLrMCbV%6i;lpMj>H-4e2>bMjJ=2uH8qH%ghl@Sz5E?(MH{Kfm-s z>DkhuC7^MmWy6(N&svA9Anm`(%mFVX@f~IbblD)=O{r%}&XmkalM?bL&86=d`nQf} z76dcP{F!CJ%nE;I#YEbY3-M>;*`mfkTGM!1_5&rAY#vWaJJUGOILMr8p3JlcGt2#% zz*Sp#SnP zo-(GR12k%3tiEj;d3H2qM9rqx2XyNfQ8)I%9RFaws=-8ktgYHuNPYa21oEFyHIV3`3*^oec|~ zMf#7`S};eVDA0XND;B|lO7nsz76s)Him(U3Qz_?3E`f3N=vj<-72Bu z@MzN-OnHUL4hjdY-65fM^ToCZpB`Gqla_`@S{k8c!lEq`g_b64xhPx`z$KF&6p!?f z0zIUHftn`3q;t6Fj7q_|3aU}Pzm!zm=Hnd#w*SBot)Gons3`!F?slebC z+{P`L+?t3}I;RZ<1$pM*w?$PU+8hg}Ttb|rxp?0OiV1m(6q7v8S{(e<^yOQe?3;f5 z7gN1Q4lBqt89F=219B??nnA`%yfowU{;~-g! z0wffPir^e|YH(!Q_u4&egK`#yjsZiL3` z49=zw-}(nAWCRrbH{iOSpqT#g5y&2vnv0nps18Xi1l{EABahD*@W!=n?V zJQR|#qv}2L)=h{~5f7f1`@~P#Y%vfK9UTWDFQhnNcltn70|w6EMgkaOeyW6wheULbNeV#|`Zc*KDp^@h{NNu#c^WzFZ+M0_d za#oDxRNYs|GSNz2l6Rr)Y+Eqf>d&^0=N4Ycyp$Qtt@h_u2Xoi>bJy@-=7I}Cagi%vtNtSvx0_ zLVs#eU6ntlDwwmzpR)$)mL%TKH4)EU38hbD6PE>a%i;d8zj^4u@P2>UQy;|z%Nj4O zW}n%`=6#P%+#S%h&8t=g%eGuv{n>st?|C+HS3vhYQhTv_sMw!hIo#yWf11s!XA{>4 zbQ`dyX{cei)?c!2m~JPudq40*j>BXHaol5>2K?1U0#2ikKKKMJ#dgsd?lbe zG_SYV#30f-e+klB(s)UYwB|%;{Yvb4EJUp+rspv!^E`bq;hpDLQ^D_b1;q1M_??Hi zAsVC&3hJIhvsop(XDHn)mEBt-f&8aZ1*8lbQ5H!vmszPPUPa*wt>cd%aB8UZ2ztr!D~cLUhFgUH~4w~fFud(WZzKm`nse-k^9X)Nui>U z>jgV#hkxM_TrpL!`R5s9AQcxr5CA6;nZq$0CA5^5)yQ1vGDXY_9$F4pzKUlhw31fU z$jpmDLDUkhp|y{J(9wE2o;J|N8u_AF66i#LF+CP0sZGt3NGH=NkLfX$P7}eV11#ye zfU;!xi2t+io$%Mn{!cgMAM- znF=!BM9x7m=c;86z?u?$W9G+4IC7`Y{gB%MsA#nBcXweNhJ3%n*>hN3TFQln9hR21 zlUCwzotc|A!oT_<1O~khI1g*03j@$Eg(S;;mAHTSr9dC~=;c4(?h9W&H#zbXp&Z2b z^wnEa7jK9AUYk94>d&{2qjKZ2Go2m|aa?I>Bsv7}dI-{~ci*47)jv7<-t@&I3-EGP zk0L#+EiHxEl2Q^|!YN$mdtv`@+h+FC|ACzk8WfS?WIW2!QoQ1UK0AmTBjU%ud@ejZ z5WajheEdzAII-g&v@JDI-D77!!HTbEI(-;&`V4+Dq0AQh!Hv=N+XA!Oz^`CPXYYVO z(MSX4f5NOmjG&SrVmxrJsE9v4rJZr$wKES2fij4k`4ND84ga2hgQS<5)F%h^1z+e3 z#*@}gq?vWim@uIGc{RO1Hd>(dPV()e$ z6=Dm70?a@(YH`9iN*;DW$3$(ArM+6vCLZD2q=yqCmwAL-%~6T z5Cs$83gg`YZ!B|FO5(`p;6^W|CmmWP^1VdgP#Pmv21*5^Ckji1A9&JfIhs&B{r02a zS4PQ89$IX&MN5^?N`MwsA17w7~tK}4qzzXBpQ zHime9&KJR8;OouQX335+5pLrm7(r<-YTnEw{kKy8&LY%vzk+y)P#@za0$Q1)W8rh4s7UZ`XTgp%%^OcD)TeR#hE5mX zg70V*ty#u@>!{6(j5?F7nURb-pC489&951r`Zko3ZCAU7}3_N9DX^i5~Cp*=vJ#qNYe^SWd)=(gydFX!<^-*gTJ zAB$jOxcBDsMTjmlliQ`Yxp&lYk_&T^Lv@>8@GMD=(4vznwOXSxsxxP4{3Vy{B zK6VF0KZCc3%rQtHVz&bGD=@*eA#Q&knqdEW{?&+@DZZD7;NtM>7kudGH=85%T*w(X z8S0>vcqFij4?cr(a@i4S0zbhBUmFA>W{;hOscu*(+BmS0n$i_8DU|HU?dQzCysl73!g~H1k;L;xJ5APZsz(h{Mv-v>tHAI z8uFppL}A6-B<^xdeCH!vj&Zk1a5=`s#*yD8MS>9^BvBJCgwCw4>6~iPRtR15E@qt zu6P(cmW&A!i-H+|S4fUjgk%s|$24J`6%r5fJXml(N<)t36>}B}Eui*z{vJ$V)B*Fy z^o6|*WhzEY7et~H!ob9~LqlXERm@l4^vCilAwQ z-?V~MR0on){re<#Sv`E<=HZ_n{;24)bz`e`v(;^E%ASDng$bkSjQU6F!GzP=uk;Bg z_Z;6d*fypwh5-hXm->^JjwKiOH2~^COVCt62&x=zxY_vA#*u@c#gA1z&sOeYQ@$54 z?iLW8(tgWh`rep+DNdDbzy=XTq zS>HZ1QM&90lti&+LLGlndtA#V)&7gRZbF*`mtBe^EYuy>v8mC({p6~EdgX*#&l-zA zGz~rbLCTPtjjy;ZyQ}%7<_q-(VN)P7yaQ7t{bkN5iA(r{&WN8v&vR0)@IO4~U@!hU zYeU@DJh;q~Hq}x8R8$4|-(*T4|C_A(CGc^t7C-LQ)hpoRGYUUGlOWV*D@nfE4EcXf z)ihO6|D2c8R3ZDV1Tpi$2Y!v{?Aptqy@1dV zR*DWbw8%&_Zsa#wHpcDthU5qxQg*q(67v$j8vOFm%{Z))A60=d4nJpLxL=mw(Z!!z zK{fZQFz#n2Xk>(pq?mCyyXfj$9w=~z}qO#G$eB}i9~W=E|bU~Xef#HkCfq$l>Sdt+I@&4OX}b9#@`1G z1%5+8uWp=@9aA4sA2S>=^h*XR`~P;#R1lyF=j3{c@k@E#m-5O#$*cY(Ujct#%58qR z4L-k=zwo7e-yFVauWFG|&qkIw}1;^XxSDsOSvnm4Ii?uo RkPpm^jEuJ!go;>z8~{2HHXi^0 literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/core/__pycache__/config_manager.cpython-313.pyc b/gx_gp_monitor/core/__pycache__/config_manager.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ceffb13e69a6943a83d1cc865f74bd92d1a65ba GIT binary patch literal 14489 zcmbt5YjhLml_P0nJw~!*S#Q7OH^N{WVhE1_;SDyJ;K4~+J5hx^HVQ&ABPqf3MV362 zHff-30;Fj$+cr3Dm$+whHsnp)v*%=gt)`Py+6g_|o?q%PMW9T;WMv^Y;VY*Gc$w#@V28z{r zDONvdTBVWO;mw0aQf`29<0`dWx=QH;U#?&oz?$Y^Eeh5Q*z$Q;tAe!v);bScp_6|4iW&Usin8IKEa?i`MHuWeraV*>(O&$qh&D;Z0LY~{p;PnDENP(PC zC?mXvRr55f;k7Kym$6!2$CmMWR>vDyJ#QQ__>Gc&Pi$ll_b3;+Sq_zxHUQVp{A}uj zGdJG8IF_*P>kmQM2Ev8fg{0aaD?OA zN5^>Fm;_)C;qi29ia$yV4!+Mca&icsqCinKRt_lUG7 zTj3CCN7jsVm2Q!CXI<4I2p=i1R-W7^v#{7s_GT@%r+Vc!vANR1_cCDqA>l(2k%bRa zvhZO_7Cscxg@q5XvhZPQj_|Qo-Y`<(w@KyOK)azFDU#L(T~6Z0zy8AH_2({4z5N44 z*mPVpcrd~ZNpxatl#}Q~u{f;3Xp9GlibbMPekA=UC`uv@hS69oLRMQcp$x4(nI7P_9ttKDDcAs= zBgfITcOSoY`K77%FJ1fKf-DGp7vLlvsPEB0cu3L{g@LKEI2VG2leCH0VJ<3Jxo9Xh z6poGrf^b%Vj^OWxiu`J1Emjn7@0OaQfx* z1E+eH6aN!b+v*YYkw&_4p0dfdCtiS2mttS1Ga+faYbx;76ygfA3=P^f(T2!ARmz046kaH~bvPJ~BzaD9qjnC#0n3GwiSVN!HZmd(HUnA`p?E;i{mK*@7Sz2( zQ@o(o9e^(L2(a&YsdLIwS5+(0wOJ=Ql52_7IXu6oJ@Q;*rCD>G_tkz0m@h7MgJ{Re zyA)IML2HDRe9$tGl9j0mQnE5NLrPYr<%1S%9`dqEwL(fhZYu`$gZOXjQX_>70ZWcx z*1}hgSp5}}i5-i>Mpf1^421GJ7P%-q9!ZiR5*rCb&`Xf?m?WCOgehD%w5B90=Fpr= zj>1AF!A8=BWcyLlfgGQ@KmPBJQ{IY^hJi^r0Dh|6s% z``@$T#B?v5xM&uU)0L=eqGx!V53;j}0Pq^j(g;dz!-I%GfRhu)guD%94ekv-kaU8ld5wlxdpmfzDY zvY=sGr3LK?z|0mj;^bLa;!w!T&1x0L6&$`nJvgr5@`K|FZl4!i8>I}~eJEp;oRQ>A zN)DWTz?oSKi-K1+SPsrGxcJ0Tw!n#P9nrvvZI(r6uk1VBJPti0hMTMdWQTX^+(fU$ zz}`kL3k4^n2D03P(~tkRCjh)a^{YZFC~6?*&{6|=+Eq$fAFkD$r&L)~-LGcV%NcNJ zGzj~t2)09>ebzz?{i+=2Vv0_r&$q;?Acqr6o)MLwPCf!j<5cR$Qy0HG_3Vpt#%nZg z=11b|j6QgZ2ENWG(_Me@>1*%&+w_~SPrd)d|GxX|)T{q={gs!mU4FY4&XK9qxohuz zaN~y)H{Ursed(t%k+D}QlldH1CbE4jcqHP7s1I}l9d4Xg@J-4_gFK$iJQkZ!q2i?D z_5%l7&S9`2&P9f08YYzFu|{ABym6c#=rNBD3^EOamkdmuTL~Lwk=t|S-vp{Sp4um< znR3eH5}4X+-rARs{ir?T?GU^j8SiSryE@gAty?Vty0PhWnG z3z%FWYl%`c1n&Akel>AzOK~cv?o#I@gHzQ~1G*f?()6sF)v)whjhc$82J|`FJ`&{1 z<>Yq*NO_Jk$Q6!@JggS~a^Wf;jt~vcIZtjh1dm?poF|6|&iS^16Iu{+o+hfET1Mq( z1!i?v?W5`_AN=)h+MubUhE?@c(OA$vob%|YAjC(a>d&WwkYicDdiIR6+NgT7dRR3~ z$u0SH$^QW6MmcU)yittCJ|YduhAKX@*7=IVX;ZJiHT|m#KpqUoIfZ%$Y>eJmVh5Z^ zL)&>i#?Rh_*DgOd^)F|qUY;O(6*pD;S2pk6Q-bpP_`6f@K7aFv=cZqOoe-LD;%eWGR_vk*)pM-9Qj?v z#w+z#o0eSc{pa4v6=Kt>Q}jpH`p-0!ZPSd2GMF>^8bM$4k-l!GjB;0JTzB z`dJgWThD*vnQx@^Ri8P4{Nsv^f4c@n-+Ja-w<#62WOJTr|9*SAal_@{Wj0;ES+s3Q zo3{LIMuXVD#&KJ{<=E0i{kpt*OQYu3OZAZcO`~zk3f*s(YcRcnz%KjN7R_(-_Q`c;+GE+6ir2Ysb<(R$dMLr`Nc?BapEhGXG` zo4uzP*tx({13WeT8sOQuFi)1o&g40Rv=(vJmf~Dm@Ac(1DX>!>wycxN>9LY{9jnjb zbNVL;!4l}Z0s2n&(~I=I>~s2V{0IBae2Kn;qXB(4LEp9g+C}=#d`{oZrTRWnK2q+t zBpoohvWAhh%G9N2ufOod^%ECCFQAdm4Vqe5zI@`wOHatCn-|^!-DBo!uo?!#`8otX1oa3S5P+mnd=r9Z1jw1k zXfnSPK^p?>OA@&h2at=KBrVCk#Q@I^uy1hNk$8<9xk1mKUyEQJfTRz=oEYJJemHQy zmfry{ekX!m2zDdr29U&_&kr%p;{&%evK}=S6u+by;S$hGq9;j|I?QuJ7-))18i0~j zae*M3cr=MPi5WUr$XFD2D2I)gL0`ZEgNGPl{L_$^YG*6hxw`x3KoegA8sY@?DNTEf zDf4X;Wv@?dh4663+a`G1KDM-7b^6kcyFYCF(2=g+Cpzy-TkpH21*%VtbhR<9cVm^N zqAC-+!FK!DvYOPp_e$jzZMuGk=-ipM?!RFtzU!S4%N#lTf?i1_;F#sqD)6!@yesg{ z+SXdYB`Rs^vXw{?)fAbynv(qOtRv6AG=-COEkfaDJ&RCy*{Ve-s*A>1x*s)*Qq(R= zQOEig=|?@=un0vX+q4KpGuyHVMJu~x5eh%MG)KXM|y)7?V78jtnhuyFU#m0hUEx=b?%^J6zIF!q8TDowGC-Ja`Je`$&x9eER#-MLm1 zH7wAlU8GOU0NzKEr=MA%Plfd;O|hHp&iAP_MNiRem!{am?p=gOZ-EE957xspkPjj( zFv2bz;Xpo=umIJ4NL2`1lyXzp{YbMweKKha;8;>m-B0(I_3Qf${iX%z4j>)dzt+@5 z`R^xpgVPs(a{ZN4iVaI{i)EAdOB$=_j7SvkhgZRH^;^EI?ThT@Fe@@|iE%3#N8u8C zgy-T%Bl5*0#M(o_2wrUyn>V}$41a#M;mhMbx`~s)--9V5rZ!-T*yng78MZ>ShR08o zZhjT9_X0QoRy-QkpKnzY=7cX{$s;$yi13j8h=q*l@P5D?*LAj&KiMt8J18FGgvT9+ z5{c3Hx{eOGVr?g%>9i+equ{=fPA$ZA-a%IrJwbM)6%p$s<^z zJ3>CUfm-jP{hU*o9miPMtL}9LuwI-?&Gxw`JW;;|~e$yYlbqC29XUp?ZC` zsx`f2jZn2VkIB{Wwv2yU@ZO!RU7Bv&DAaDsvvM_NT&;qub;2}xP;{+@rZSFZ!O=Ya z$Yg`)Se11)jIS4*9l7_dI%}mdrJHG@DqR^{qhMnDbY`73XAYk} zJl-~ORCKP$dVJ@r&Q*;+JlQ9D)-04eAbQqieXSRn3ru?1<|`{j-;M$)B5zfd+l?vx zjE{nuceV;nmw^r@0eD_3Bw67}Uc+tKx>+qckoDPhCb#R}>5lQ%iIt+QJzKNneBfLly?oOZn^?0gTTy+c?sVO_bHX83 zbQE$PvEr^m#Z{thMb=q=CU!bDet2?~=v(95HYa5TH?esA-@0x=^z|)qB<5nsv8i+)D-b(uuW`95&@? z&3M`bPuoQIWsB(9m31^`9Ib+*b)sxiEjsSHrK3INsofCqtg4gwPV5o9Yg3l1j^?br zF=KBP?5z`qNf^YMY{QC7!z!U+ReJT_594A3o2^-vsp%AII@2q*UO6DvbZ48pGRE>a5MFNbpXPF4wZ8${>EtizLWvfuAC8;@+`&8MHc{^F!O9r8qc?xjyLIm9VT+yV6}Q^bE4*z*$risuDFbw%?z z_l%i^+<>!#_cG*`(_O9HCCFD9qNP5qua~bZ^$_oaOO(W+Q1OUqs7b{1SxDgZb{{U! zoTng|5VoiKa7O1mQM#7u$ACV>r=r>ix1;G7rT2m#k0Jkl8n5yOur2ZoU>K1+U23?X z17e_2$JKQ(bstL(btNW0eY{xt-@M1WGlzli{@eCB=HIErNmj>CU^ z1OR^e?zu%ropSNK0A1ag4d^O8nu0@nQqm?Dy{MCH||jQp1D_J z0*XC%D0t0O>$MGtf!8g2rM7CkafX7|WzA>!B5AUTNXz7slm8|(Ju60wyNK*0v|(SM z1^~*>Y$e~H_fd})(SW8=B_9WRMX^skIWhIC6SA=7zfzgH^vbpOzgsNzp~6QiIl(Xf zc4Ln0&XS!R{ao&THGEO7@Kt_tKt}CXk#QGmDO*8D=YE~QzYAjsqA5h}qy98bB zRd4GABYL~Q24K8t+B;uO=cf-nNtiz1AwPu)!+Xd@)U`@J9c4vbtaiWrtSx$-51JfD zO*A>AaX)~oA%AfcM_rk_EFmI~;3!T|4J1xbckGmNDThG^(mw6v7!W4k}$IEVs(csw!DmBX!l~hyH@ETRuirfA+!g(7A_ZC`e95 S{(`TWYDVQdwFlGkj`)A1NoBYI literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/core/__pycache__/database.cpython-313.pyc b/gx_gp_monitor/core/__pycache__/database.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..698a718b7b85fb55abb0929cb84173a3dd06967b GIT binary patch literal 25136 zcmch9dwdhundnHP(PJdb!Y^$68o$6I#=OiU<`ww`Hb$nBm5|z9McA@&R>y3O6i2JSys#N3i!+CT31 zozWbPWC=(%yJPFjnK|D%bLO1yeD67PSy?;-$KmR~8Tw8;!~7dwD2Fbdnd{Ur%(IM} z5gE6}t=+B>HE?FPYdW-|wu2Q}0q^U!>pD1*BYAGSzQZ6INM66)*kKY)9lXeQm_>7k zMYMEeiCLt~u-)2W6|EiFVm7(fZnudxXv?^LK}U|51NTgBetT|5o|xB>FXnd?hy@)B z#f2S(Vj(FrZ!hXtBrYO(%l6`q60wBjv$mIZl!;{>^0i>y^7w7N@k6G!HAW;2Dfcd;<^g1i`)x(s~fcLoJGvTx`#CswuZEp3$5kR z))u?9Uti+R?_CV=1qd(Jx)&~DM7z5X(xro1`=ZDmco@HPI20Zf{qC=H&OSRf`}7+# z?>uwu{g-FI|C5=2I6V8z(^uYmarVsVnU_xRvqz3z`RxlSK(oI%^=}6sYj$~BWLUV6 zA!g4$e)Z`0uKxVc%#$atedj&?I=-0Gu9b8n!@~hd@81^|eIdI>vby}fK+8~HxH%$* zhDE&7HyjK@`CebpH|Q7boTTd<3WX&dLP@M6xL-1Mjf97WgT8=d+UgJbMPC?dY9(X8 zFYFHw?e$A0O#A(Tun!v4`$GLt^*W9|v}SPxgTs+vpMS4E7?yI>SMK46*yopQ>U-V$ zNBokxS@b;`aQQ=#KsbKk4*S9pd;@;aANE3@2ZjcftHAIe_FOCFi2ks+-#Z-i`uFwu zNv9>V3-2|P3Vk>%$`Qu|W-bhO{syqUmjNtiM6FvRvTm)YbF(4`KYg#kWm768&MNFN zd6))R4KVLUk#}=2<7T%WQj6OFX_ngvsnu}IK;#TWMN_J#b= zzo5SlM|meq*!4nqPNW!;`3P|JjngpIGtZxK_Qmm=^hpmX3%|KfAla;WmhmtyQjnbV zH8EJCyt}GU%Ltk3C?om~!Y@=9V_G<;;NQb&b=mlT6;BU?}VZWRo~=V7TvZB`y>h@r#nd>kaxJ z^?CtE*uFpr0KI(yUnu1DhHxl^gOX$Up5eXz<&Uf#4)~Xk>>m+_ABOo@zIE5K!I5Rb z;qcJF{^f)Fyn`d&y~DvFSOClWhDHB!SOR4J>PPmA70{AYmVQR-;~gjmm{2bypB`kU z&DJBE4sD8>h3N%(CyJgan$9gavFX^RXm0K0YWwM}r@O|QFIKNV+I)P+Esn`4j_Da& z?vZaE`sReOWHc1d+%iDP@gjBEXCX5@;Dt&@DLY=uJJuSK5Cz|tAj(F_WJHr}2BtB2 zV&MuVL_P+YK}IZvJPau3B{aqmUzD$FBwffK7ziP-AXY;v8NDzuLt(F1D$h7PGQeU0 z58)u;@MIrpJk%I9mrfUzj9VCptcJaQOe;mrQCo^kH$qQYOjyMR8b^*icxsv*J>$sh3Ao()dHz zB^Y9PEt|+fB}-n&u$U>tJy0?9Ye)__XTs}V6svLki{I&1@YUaofQqY~}LFkHz!T8!tDV=%q*Vle=8en`wA%7f% zqGF>Uw-}d-$y(u^5u`LJ%>V(IeetJs@wG*eVx{d4U2z6^#ccsV$pCUBp#{7z5+3dw z-n(}wEN+4l@jgr(kh~sW3Oa-no8eCKQn1EF(YWL$)nr%?$zW~R4k`n%DDvo{D7~Ch zJW;aoV$P-s%chvl0IMT+;fcm$jZ?Wb(cGGgxl2Y{J}#>QZ0>@B(e3kbc>Qm3&wl0I zg0senBFChqNrl7t@US0NZerf!9rf(Eq0(Vz$L$>ClNcyP;Ew@369Ox;nXd*}3cZ4% z&xpVjJ)zn7;n66>9~g-i{{U@uN-3Z%5Ds7_lpUvI!fs6uMdnOsTh6$($V>*c@>#`H z*hmLFkH7Z*@tGH1PQk31(J@NgQ+P~J#bM$oDr2*ZeQ8^Z(|e~~PtZ>?j6?|LiHL^; zxe_X$wHHf%SwUmV#`x8IWxrBubm19!Mw=KGOJdbxg}&E$>y&L<)VA#} zF%}^{3lS76UeBE69n08pZH1$j9k1mepPoj33f;LQiUC*qJV>QbML~s1E=1~-!>vX> zc8~?sqx3P3dYS|>rlpu#6~&~jCCf2@dTdbR&$g6uy~X{TCTFA|$FM8xMdKDm2b%mx-KY>L2{7!7zOqtjWMk*t0RF~RZ&~jq^$;#$b8H^QF8Ak zTVu??SPDM9X^f*-2SG8*G0Q~xmJ93$j`RAn`zA`aP1)L`wswMI9e`pB;UUi~cj%aL z9q+i89k*6E8rbm$4)X9ONQ%hvvge?>fnpG-8@w0Pwf;x|^A@l7>k(fdUXtzg4h)H* za9}9t4-P{ysEGZ;eO|AKtVUj_rtluo@9T$k2{sYXCPh?Ch?*w?fnvlKOtxdP6O(R8 zqTA7to(YXToXw^V}}0ahZ&+_2V=NXUmP4bfKCR}dtXN0?_d0$H7u z1%S?>s!t^frA1&D)L?+x1JNJUB!3Bze9aw5zE(x@GpswJKft<6thWky@)zsn0Hf$2 z>(~K}osD$C+oYFxuKw`5SI$0ph6m1#dDC zX^q^P%h}~q+4a%v`it2sMs?HHoTvCd^H6-*QZi+!k6P-_gfCedVl3SH^rnfihrfjNj4Tp-8*%`1 z0!26oB7ZujlxU@g8XR0;#z{TgY|jGao`)**$;^u^BGv&&uPgvqM2R)% z13>--2m)>ESu4;2Tft|yilD_yrjoyg@ZG*`ZeE~ynr&v-WA%=5?fVNMH(p?Zo8#qt zlfaGJS2Y!L=ZZPVpA*=oD($(dMNONybDOZvd4_Mc8qV_ufbWCW=%JwX+*w%G=?wQ^ zYUj`(kH&j&1sTEp!@2eMQl8;9^qSnpUcNzxX8DJ8%Hk2t8Q}R|ORov%7;0rfEpt*W zYx-Jcz#-*7cd0w8H@nwHTChTH0m)_4oUJzt+)dyrP@dCZndf-QN&gggZf|b;PXBzzCztWr6*@-x&i_7?QodKWfmvESh20_T=$)C%2&y|#ECim*g1a+kXo zCB2W%U5q6UYs5wF60vyDVi%-rMTglTJD;w<3au0Am@-7lOp_=)iYNG+`#+F8%gm?l zYGZYJe}{_-AJ9bnNM``1^m{1?IFuHrqri9n9F_LV+<{tClm(#)Bz9*6;m8@^_uY zIz987gJ>jIOy%>zBFlJSv&dxdShh@ z)vj5w!cH1@dOACk0Y@Uc14E%b-hS{f3*GHITHW1_9Xm4ui+y{*hf0(_+WIPJJ-JaZ zo`k^%?Tv^5p}TcgcXDZnIHN&6?pOCPQwQVFs7%r)uFU*C>T&ai!NpB0t;skBedq)C z_k$jp7=h$2_kngAPJs|wTH73+&TgUE<8rk+yS;SulIuoBaBP!XzuUUXK*FH!p)ay} zm0j>S+rQ#zO@7ed`uC$V#kTP74?w?)UAV2TD3YQPsYIr+K^n!lz&i2DxRSbD<{?_fRnrH< zF$m(Q7gkuiv%7U`t1H7wxLb70h=KIAQ-YdGA^p{aM%}sMIH%;=~yZ&W<@gMV!*&%0TSPtcdtJbf&iK{lqKClrs#i% zW!fKsXd8$M5!yPt9No0miu#n!wspB$+qXKA z*?1E?2NqT!vNu?pKs;A#x5wpl3*D~vtw`V;ZlSuGw%ydawcSat5IdZ$57Z~x6E@A~ zcL_{Fc)5M7mbl;QY)M>HR|^4Oa4>>vc*#g$WH9vgz!DWBXmNGzq`d>FC@vAjUo9Ew zRUJ#6%ftf}F)RMUcW5Z-<(pb@y$hb5Ehwm^J|YPe0EJdZ^EScN^#Js_wb=tT?qXsR z{Y;5^P~=T2W6r#B7y=)_?Lf5-aW9CfDz^h#EoKSf5lphSkvq4X%jn+#}yO^o^|O)AHW15AFgFeE^&72pkHa`3@u+e#>%)2 ztEssceK&9){t`VHG~S_o)4#KQh#1SL=5n$r7<{qy${WRfv3gan&_$4PxcMG5bdbZN zSCs5zb2j*5xuD*mAJoX_gjw5ejtoLo{>u9=&HVg1SrrEFKUJI6-hcI)Zj7tVG?Yah z?y?)Av>FAuN!cPT2Zr%&|2qf<2}+*ZouDswqZrziEEc6G)O`y&7P!}XHPQ1narMu+*=%hL;|PAqeLSjyj|k@{ktQBsX+w>IjGitL*z)G_|J*_hk}jFbu7&?;OQGz>oV2WfXsc|?C=0<@Sw&NpvZ$qul-5KoH6(+^ zOp>XNTB0VZs){K_e4eaWS$2Crf->7H%zoDksW+Q2v*jIm9WXg)V_KL;@w3zs-(0-qN4h zH&L{H(sJ+R+(Hs4S@apP44+@jw{gt*6%KQ21AAeE5iUN|ErI-p_mF%8-=;NuxS56f zA8AR-@okOlN7)r^YuJz0aF7QKQQH`kgeDMOWTC0|KR{?wxUXz8_GogMZHqAq4J^ND zga`%N2jpc3ggyY&ZAGdbP{GU)(TZJ;0s*%FbJD&ptuH~FR?yO>6|}TzL&mjfL&mi( zq=hL5^gdn4xHg@rl~&NwrWL8ujvPEIouH+42`Qa|OMp!yXu;j;`LGRH3iB739dF8` z4dNITyHw&B*P~1yQ-u>$hpF(@OGMl3Qo3`8t~vq~&7 zMEapFKYQrc*Pl3|W_Otcq7apcMo*xDFgXBtD2UE^5*;r$1|EHov7l|9@Mm#9JS~Eg zGXIIPdI^kJU0;ak09crP>9jL~f`6K7N|b#`3SSm6&OrP-NZ?Dgkr98eU;HNC`Fl*h zg~>ro9>e6@n0yD5?_%;eCf~#42~0*YIfTg`Ot42%R$R;V?nZOB_#_q(hudb%k*J;f zFz0{-ymg3u;xSB+Uy6q@c^VT`g~jh<(hNz+2nk`3lt&iMV}K#~nGy{QRg~mRkcbtb zuR)TLW9H3fmrUp8&FPuarK6qGx#d&2b?_g}t(yi$YFibxt@>wM!wn9KV@8>Y<}f*V zqxwIT*a04x21F7NP>e4n8GF=bCz+*D+ftILjoNB6Wa^`~`YBsO)Yd=%mPBn!NQO32 z7q!VES7F++!ShKQku-L~+qbWbmT_GNE%2xe*TlJ~?#}-f8?xh@7 z2~;d=`{0`&3{Q9-oGAI~q^)OqVf8H&VT>n$F+$)2L`!I)HH9&*dAsjy#ajboyC#Y@ zPg=HI&dno?G5@pBT9~^FxqRzd=0fS_)*AN1ViV*)tl?W%8$MjlVtzGA*Ya)mupcoM zZHw8D7ITny5-gN>2q+XGcnjPBk`eG#&T)YuhYFxYL(r zT!6NgGu0R>s0R*aP(^t8cZnB0HFk@$Q>oA(nin( zBgcD8!24TZtjwxNEVDwRgO(l|Bj&NV^&GUIl;JfpG!LI{jC&|&PshzG8IMJ2M>zwh zY?C>qhX>ANz`w-1+?F#LsKQRoF~FI?ED_97eFRpj8iaV(ZR{ym?s_QCPUcL@nUEHg zc0(}Bkp&n>&MNL?iu?~~fuBA3lWTAONLJd_d`YDt0oNgJyDRVh`0A<0K#!Y!_Z6ZV zCYBU1F5)wvJ@J#-7f+HxWsNlo=<#~#q^gK4x<=vp^G9b!e;H?CNSNhS2yH!fE9kaMg*I2$j%3RKg?V5bICX@Tn^w|02l54n zvqi1@gHgODbYLO5t=2Dv&6x@|l7f1n%hduV_@)PwY!2XjayJW|?K|4Lp^FvWl&m5vmy5QW@|wFnQ#YbmYB+b;{c10Fbdg7<8_~@5e<9k{40rmrMiV@Ln&u zNjNBBsP|zpVWU==jXs7qPGIsJCeLH?LrjpXO8OC>=-V6DLUly>M7}GrkR^((+&^q@`xooSUYkcG6PyvC%w{y(MbgGRjVa0X*wa*3rn( z;Ys6S{06TeowB*G^u)K0ee2@F+M~LUh03E@e<&_L+I*$F>QwU!`;TtB+|csDt_i0v z+PeEhv8D7{aBxtn57R`EwL50l7=Izbf}Q76#Q zhdLoCxFOl>o3JX@2}+y-?abZSdJ{HXm049i7GR+5Ci-1mYg7-DqTl7Euop=$zn=tO zZT9=&io76E;3Tu^EhQ6uOVHZz^JgW5gXnqiS$I73BEh0$O%7wQ#SNIGsB?Om550;- zDViK%D@YDxmZCOus%L6GJ0ZH88sz85PW}WfWmMV*s76B~Z?3<{WH(0B1yyn;Ema7%MJd%nO3`Say` z^D@JEJB#^cBwfL`)Uf9pD_R8hg1|u@HjOT^YgL2l3lVS|qKK4o1i2oRp2*?KS8|CnxC%M2 zO>s%KDGoFMJ^m#q>uogvJ*BLmi^&Id*(IJ%xrYJ+l-1BfdB<&BcKW!>j+|AL)fm|Y zZSnG!M<~+2cvAM)sVOUB0P(?HefIg8S6&Bz6oFheNy*zL{qVcj-WZ#C?^%Q*_9*y1 z?-%fl1It@L6vudpd4jkg3*f2<%#9!lN07(N?-(^<&5WM9{`@OK1g4%CujI$ZO<1Ha z>i3}%v^9mqi593Yu|e4R0TD+e6f zncyq?$NC|5JX)|})cmm(#8>z*joNRhcjT0F(zbfO_2#}0Ny z8Q4Ce2cJO>6%9e}MH*{IvNa~1|3PU3zBWtPT)-xhpj3%($GS8tKwo#0zq_OzT1q=m z*CI#UdZ6o}^eo-Ca`5#UrDtHHhtb3%St`TcfMli(>Kr<-9+0z&e&`~|x8q-$U48fD zwKq;*dH?k6V{fKV2;Bdy2R1~h6JQTCP}gP9IH~bH0FS1^j;yiMOI$3?T^?t5?NYno z3&mxtB*igVl9GX&9qv}7D^B#F%W=xxc(qc1K(%;7K-ZEr&^tDSk=qQsVxem6S8;}k|{*;<99fAsR9WJH0-hk+qGd_0og@|j?sLQN{eKADDUK#(qxyeYvg~;HxwhkfH?}G+ zL!s$ek)aZ*bBYX=q&hc3cm9!Y;+TJ0gI~)pH9>Z~fp5|p#y7K&Kc^)r$2VB!iwiX>5py&WNQvz;0sQm`8X>|zCk z6AuB1=WSF+mJ%uCx8#TmIJzE6wpsO5{Yi%+YW4q8yi>N zM#fRas}Pf7+y=PPN%lOMRdrJinwdTL8eOV<#-%F$8V2lk3zYcj600=b0xbYSsU~C~ zkib$6NsCiOk@Rp@z=II_{b0f7rCqRvA6vJB+@aIY*L_ZPO@dpBn##s>6AK$~n?6u= zaG64=3GommLMKx8=TU`DG(Z=A$~#swZ&y1i+3`XXWXCIc#|p!E9gF!DBwfun!LBNs zDw<#yswxih61N9FTFRj5tb&*O(li}t3w=LdNY~L&T}S)x)^*Y>HY%xb;9JpqxVUe{ zpjv{|{~a`{qz9v$MS8Eu5FWTjgO5k=S_r34@m7UF}LAcVYq zkcc^?H6juh+0~^(E-fCxYt-i?7VH$_vv`TqPLPLa|KclniBg6Lu}+NmOH9!IB%-<} zzJtkon4H7pYmmIo$lC!`Pz_vo`KCyOaxJ@^9X&HUt^X4&4 zv~bl$Zgot%mMc0Fh%sZq4%6)mUA0*_NAl8GkV3ZtP2ba7ayOjGz&EsoT>hEd7z3xVzMFW814HgcL&_Kwa|Ne#r`%_%PVG8vonzoK4h*?i zV?I|jwmQbZ>1@MIJl&|T;tJ06#TYo9ZTP?v-LT^p=E)6t?#Q$~3RBhxKYAeQvGAK) z0SRn|KnwZ+XA-+jDva{$DhSwwS|gj&QZ`Pc+b+z!@$!}5K9wR$rifR-29*8CQtQF~ zz-HWbXs^7%gN%5;gB~W3QOZM-geD~12%AFwHvsB_S0g)c!e%En#+Vy5EV^7!bFpB_ zsQI$FU_w_wT1h_`>*c`!)RqtB^>O>Ebo>1%<*?6!#;%pOXq)#LJ#48iV#h>WyYSTy zY$!{1O;_)tK$bNoz1VxSjUHsKSSv4Ds}F9S)|sbtMNwVRC0+4t`d$Z>)Dw9Z{mq9e zvgMJ^g}kdAcKIPFB-@jTDMN2lY;?--i~kcqcVovj3wy>?2cbg|$;y~;M9U^&CQ7l5?-rnCFY?&GK(Sq zCuaQ(J;Uc6-FDGbdQg8$ze}U((A;3;(>^VexAbD}Jrj%#$f4GDLu=El0s5k~#V{9J zOzthzls{K}lYvvb2y(X?cI!0-HyBJ~omotF?m=@*&ubPwH+++U(e3ziru;pIAI>z&s6x7%)#`sIHxm*hL6gb`)d@?v)_WCN)hd zc0*#*gkX|1h)pPRPYR~(29tz@c6U$Gp50R^#z*QcJ|`4YMK&LtB_4M1v_09N>LDXl5R)DrQ7;!@dptgdwy zn+QD_q{5hNF~cV1QqM5Y&|Dfj{}GVM5@dP-RY0+oYzp+7+Ny3%Xr0xnSu9gPzxc<_ zg0i@yVeMiin!B(;9(8pK9Gi-(gpE*PLObI?~hVWSueD)0CvnBW(Hm7ygVi}u@ z-&`(!v3ad|tiDwbv&<)@`J}V};25@$loYTA5e`a{8gEO~=y2ehO8~ymy z?F(mb_kDlNcXsqgr!-@|eYZY(7q0&4!{cMG zUK~4qWbEbhquuX~_70Bz@c7tk$8LS_-uOqaj2?Sma|Z^X8jZBT?6A0>Hp&I{_I+-< z%V}{4%zC@kEy(MfhoD;BW^vox_Dro-koTV2j}dxsVK2tP{cW%4n2*#*0; zm4aPH;SPgcCF2-Y&N8e5_YAA9RB#G5fm1e9i)j}2!ZT`4#cDZqGgV7-38j=#C#0-% zIb58rZG(&Jw7A_i?oKLn>?wd`bm+%-{^8B>o}b-*{d?lv$Gjhoe%K477(MMftjKGr zUfM~rMPgPhW(&n^NvBaEDAw1nZQdv_n;SQ57gSHwnHo1YZWI*tCewD4pk8O%+_ZUJ zUBh7}uVq1}RobU8G)3V)#=pq}K#L(tj2S11s+K!t|Ep1mERN|) z9Q~oAG}8`5kUTq!9O2hT(99@nJS@v=F)D=whwXsP;k0zx1O@NeXX6BAJK+~Wxo&&I zb`vWoO!aJilOXTpJB*CL@HR)gpyX^;7uP1FT16H!M|(*yo2?ED&zsFW1K1c0hAv@E z)zhv{Th)Q3E{CmZ-=Te+>lvHXUA1xN{EmI|oi4Y%{ZLiMv*wO{=1!N>?sjoi(7Ub5 z;p*tHah3ZHaT(B)kTsp3D@9C^q51QWe9=YS)XwT_4`!AJGRudx<+swZPb@pJ+FL%H zX7uZf6G}=qE2zy2X!H8rXTEj%Tf^EpH%n%pUwL+=??A9>O#pr+kNJ!1u9XMZngVN0 z{yNs*v@_VWE6}vd&zk*)-F{vEO}*jF*3(;k%3#Iv0Q~eT{CPD)OFyo;QZqE~n){Q( zzdU@+;m={aHyqy*DqL_~r@xh)-gB_$`Tq9d+{u{NBbbl|HX`QrQ3@0f}SH8o2gO34kS z<%d%9Lg@u}waO%A*M^BCDlxOG{&OZtQ9emQ!WD*lm?F-?j67+A?Hga5fe2$H5vGtt zm@=2*)NB!}YE59(tr`saQUF%$^>Nt5fFEfXFa$F9#r?B%^EjYC0#-ID58CK!6#}TkkX`LYGO^F zqIYOX?@(dA(@aHWZTucesR6nX%z^~7-3c4hX|-|CCx!dhX**~(8)bsr>fpuwh#Jn& zC2mNZj*ukItx^I;)0UqN3COsRHmSF^r#7H9+|U=D-gtVuZ{4tdLHD}j+d|2+dY|ig z&ac*!Jvn9GI0DhSINOKKx&rF(X@hOI04U>#P>#i~h#m%w9W(z9s)GWJ(gc&$!DQ`j zyTxHYY-$#CM_B~Mw-ik{)zlCkm;fj#5O4EB6AiCY7nv<5}0#xS}@rFzmUG5 zdp*oFuMo0^kfF4D{fI7qLPez%kvR(t(Pf~Z+#b$p#Tkbm6}%7%bipn^N$pIDx9(w^ zBmSB;(HjB5Z709Ancs<;`*v*geyq z7L?HBiS`9}qMDudm}X5YfKL;0-?S&}KZUv4K*7jyL~fWbEWiQH@=_`~zA1ZJm+%RG z9QcT=e1*)(#6af8gKUX-dM4E4)1EM1PDQig??sUq3=PxJgHM*JIS5wj0(euj6o>0g zrQx?|PB;r@+k)GeJP|4>VRpuFX*6$EhUagfwx>P>hKa(!+Rcr&4!<=kVj$vk%e*jb zG>@$KJ-`?x5_ulNZE@O%$20+Draw+qv&vLGy%Z>@MBiqG%n-|8ak$YG-fS^l%p9|a zKrW5xOnQ9?0~Q(>B_D`ql?8j~?}nUh0zIe&8LydR3z_5$zxif;!6X?5GsAPo&FpS0m0b4t?RR zTOYlsIlQ>6!opk8oj3A^`G&FzL{WN;I6=~Pr;X<=9Z-8Xwd~3HWu5cO+6-lzYRb0N zl(7&8P+?%8z0)ly4%(g7iv*^r-n6X=%mURu3ukk>c|mP^*3P@luDyp99(Viv zv^=}^5SaFYc8|rn*RyY(%j0wdq|@qZgIO5!xy3+N4i|;PWe{{G004aJInKoiX=aQ* zwAtD%9!EF|A*du02nkRrl0i`O@QS{npn^hlDZp2Bb%>sxpg3T0cx;0DX^XSXVdHoq zArfT}5?MEAvvh6}s{}cJh!>Q+yUpbRJoh;)ZgkrOnG37}oAZF3b2*6*BIsbmXfGN| zTU)pbH4fZvhk;Pv3QCbaf}(S8n;oWPa-rYj?AQPh;U+q40<<#oV2KDR_$AT{3=yD* zF@e}29PVX7DKe9w1SoJgg#^*Bgoufd3{^aMt!8PWQI9tYgVZx4B05Ad49VjPAP+m3?O8n` zV-jm_6dK?7b{}(6H=&{Q#m5`&DXCdG!PMfG}Z@<^~2d4uB$ha4tv-1t_h{(bZ>($GW5apvd_}XLb|n~^x1b6@(fLP z!aW_8R~*c(`YgBV7xK%ROPb-_mEFl9byiSq2&fJ3WWAd;SbBNhrFp?cn*xhA1s632 z7BvnRZNIMG0bS~ndsp?W>R&djE%qMx#jMM@mvV<^ue@Pc5K7q)%AN~j6eWRk4NhmG zKB%h*=qmiini1W~o4P#l)0cBySAA2R6jbK~)H(fC*VS_n?-cAds5QXvMrLmR;xnsG zuL|ZZ2;?mo&RiJGTpq|=-n}uTOY7a%v&~yNnBKiDpj&ziCXzQdkT*A&wo=Tk2 zgf2gz%kOtw*Ui0IP}-Z=li1f5Q0Ik`GrAMMxM!!Sbi*ViORTwBSQ0E;7ARacT)3i- zIi;IWK;f4>8i8*oZ^@;8lUul@R{on+G^A5ZeKZJ(p^MzTONIe92-#Cx=uS|r7(8#5 zLC{bRzMPt6!X}^-f^g9gI`~%#4;qostHr^uu3Rq)taT)6dgnOw zJ!Qr?Y8w1aq5{KFn;sGrjB#ktg$ZLzu@J0tAkE0bLYUDJF{{D3VHyu-!nWn7MyV>H z$WBpS>JEAZMg^Ay8MhgeW=x*MDGF5>`8UJ<5w4m-f(2G0TxHVzscQ@Jk%Ei@7Ds`4Qw2Qo%T$LLC1|DguTT9+Or!PYs9@2 zv1T4^=zO)E66r{?O@PaY>fZxng*gr-fXH!@w@nTofNxRHrAkCai$E+oiQtuxfC5TK z1@zpB=Z3YVpn!sDC4saO?|Lzk3hp=t8v2_8$;EzkvFLegd6$Q?+DJ%~YlTjxc;C34 z2*q)WLi|ByO(B$xL>Qo0dcxKtM9^jNC=Z*Jh6uYXYPojMM)`druE-J2?e2F$XpMf@ z1*T=A2RTxF7t0L;GLC~Xfw4Q=U19EXLkYoJ;y#I9$v_yhm9rdlV04vp3z-tprj@@9 z&81kyO>Jt=>QF|`Df0>Qa7KCe22{b?9&KNzcdNJ7pT2NJy-1{(g4iN(LgeAVfN?pk zC`ZB%irE+zZDwXbFs7M_M=&H85KRUtgcevP8JyMX80Qkmr+k2XW8Hml6xQg$t!0oX zdf_;fqM3d4lE};kPdZkQ{pjrI(RW8r`~d34yH1Urde88symC?deVoAU^FQRkM@9jO zu^*05`dj$+ZA_3;c$|2Qp_Cd!U60iYcC?WFAl=290c^ht65v@3@W-e9xB5E$CACAg zp(p(7clw{S`1jcSJKO#3c7OUaBkH}u1D9e?R(CSZ$r)prK0wP67lv4Kd6a7veH>%rAiCp~OW2;qtb7 zclu0&3g5xOHh+4QzzHrmnIj@XP`E979JVQvA_4j(Jo+ZQ0I6ptDd0%C=2cK|dJw~L zr3Mfjb)^gn+r->D`;mqK5TlQFZn>k9;{Fvo? zu=*qDlivwR{I%1j^eudG1=#hyTYI+pbLaV<^lcs7Ka??~8se^%UCa7Z$4V^UVarxs6FRubyEsffw14f64bojms}fPwcw6cGv_vZeyPSD+6FT8XS+J8?WR z396;AVw!~u4Z$Jk#Vo-IIWem|wd%Lfn#W5$@s~I=_musFeMt3j;+4b?ldl^#y`OxE z8Kk{+-fFMKM|=6VbFU6h8c5u{d=AMEZ2y7AKAEIz2he0@HByVKLau$amrPaM!52D5o_l| z0&Wp%nMPwlTyM7w;n4jN`>VpYrI0`j)6H@9_&Em?^su<~nBaEc7)%a9BFOC^sTA!F zm&MH;!O~-xyo3p;T8eue6Qm=Dn}Iur$-9`8V=^0)3P^~bExEsBKui!D{@36xk?R?S zH&fEPHiy!(yBh8@az*~1HH@O{u5JUZ$n?r5D9o=N{N(vxK7XIQ|9OW@t0=sit5cNp z^Ai+gLs=geTq(GZuXpu~qS{vt-nycCNPR8qlY(Cs1eWi(kKb_HqqdH|D_1BSw2y_z z7vw`}V*aT@%pvmNn|-lzpfXUj;y!_#NTF7)!A7;%Eo6f{$wGy*eX5H|14$6oLdaSG ztfx1V{%Z#TZzvjCCxRLm4$i(@d8smBtP??#sO|I)I#gGGSFTp9r4bx+gvb0-={rB8YH^$v8gN>@7NC`i z#lBJ>Ke*2K>_FuN1)rhfIlg5VR}ZWX6xZCx`iWW8I(j4Nc|Sc!$g-clM#!?C_M4u- zne3-Sv#SPGmy<3f1!k}L9ER9WPm*%N!6HYYpU$m-WJ*{IjXGl3p;>;CYJpUI;<^`+ zy9uQfw;$jj+>$K$^^tfB@++Ld<8Xw@L?1z9W32llR-ld6Mcsn{KweN=7SNV?+lIAs z{mfi~@RYlJ$oezJa3>uOHbQuj_8f(_(et0tEARc)@e$M|Vn8W!YC{f!9&x}ECvftQ zfD75yAR)+FAgyBqC*q#RjX!}k5{V#44^v=0KIEDJthp$VX&atat zC65ojaQoFM2WF9R!5SkVF-K;INTj&;yU?DTq=6WRU{LHSn|Rdq04I_FD@FrW%kVlE zqT4KA4EMeX!Vo@2gd42@5P9%!8Quaz@qr`M9TeqMe9c&4xNF+OidZQntW+(@VSwo60H9q=M92nL{hf+BMLQii zuvIq_i%{e_k-5N)M$1$DFnF@y-7wa3e*En5(Qu?lvP2HcE34ay#W4kn#5AHYBM+)Z zFjc_PREshRyBFk6*FiziX4~WG5E9*R*axvVxPR(wgJZ|~XRS7J?V1EcxJv*JXM01~ zOt}{IZFcWmfs=wIzK`ttlaORnpdPB7}k%-`kY_q+W$mEHq^oXS4s4SngEhSwY3 zX!Jc5&@T(-Z3yIT7|z?&rw*lL;`vqoLGK~&UjM>PpJw|_&7UUuAAi!H|J3!AmQYrH zU*Z=x()5#*JTddH@22H?tJ%NXGTgey-(vOWw_Q)Mg;Fy9YeEIReaWNYf2B^nR!03w zrdg+#|4LVQrz{10a0aLax8a{5;^iEWKM{XP$-=Vq18xN<8v&HWpeB9TyJ1MjI4k34@PHSZNYoC!0UV_T1G z?Nf~~*$?XU|G#;NeoJ^BKfww7DTd1A=@HsRdY3y4z!sPue=OBh= z!*lq4+&Peo5+ta~1puHDW-u9l6&j8-^kt1OIe*I;fCL}Ecv6}H*gM3Bx%%*l5W&k` z2LO!yai1Y1x$=#H{0FQAvIEeEnz_9y@;is7P9xw$Y%z4Fz)-~@VhvR+CazX3as_VC zF6tJO7Gz2SOo?~v2(v&$jf4Qvf1P4yDGVk7@iVO>x%ynI*j1?Dty3-M@ABn#Z>}(foL(&%SX6xOk43e0cMVG z#ZYr(#a{y;5(E7jsd^gvj9`ng2B2zWoGDbrN)nBDd>U1a0S27;2$pXG+}F6p--<=> z93n__I7I9g$6^v0K`$Y8iJ@tAj(+Em0{gCFlof39ujh0ja*5T zZk@qRxqa|Ih7S844!KQGx)1GxV4WCvQeZO@PEz5U6O#l?8ZaUM-7^Q@h*(AYSQPjq zhV^UwL_>jSTB59opeG`h`x#dJ9FrrMpp=*%%OZHn!YQO)6ArXqBXy#;Ft7!D(eqa_Po~T zD(+G6@%gGw8NHvrONrSY1(lZbN<~-lU8RPe*T+s!kcnZR`}le{S4%JHvrbTu4L$Mk zt}DClbZQ=W6EOJM(7d-LIKDSuvUP3_Qo0z83mvE5rOZR^&%56;@gm4D$@bF@)h1 zobrI8Pf3)0Dxz{>U3Gx%V+l*k>;ZM3hG_b^?)~QQ;NLXq#=<~iE0F@=^Cn8qLu>Hd=1qiQ7u5V zUPHA?)H0yjuA!Dm)N-I!Ttl@@RJyCCZ-YVVm!3Jd^u*bvGcPPY|M=qBXD`3~m|pz3 z@{99V&Y!;gtM{%vedfxu-xc|;oPTQR`R^`%|Crc7zl`&AD}|DPKjt3``*~hakB5Tc zfLkHx02hpfCWCHP&<#$-LeYpnEU^6{J|?g|p|O~t>WUlzMz%LHJvk<=82y3oNS~E{ z_<4r$GDHDiRmmxcic=DnQxP@C5)G#&T24cBoR*YuI-=)Fh=J1+BWEBc&PYl*6ESn8 z#KM`0m9vnt39GwIuyjQt(do!oa55N)4Ih~bE*s$`(`5h_AAW7|#P_beaURxU>G@|b zFZ^22O-+-r!+t&}Xh@Ks4#x!DSTqtNei*)fEJ{dl3@0X7rby7|&+Az*@uA2>IOyY} z&_htL2E)KgqLI*;Z_-ch3qpJU*jR8X=7+HehG1kO6bS|iG!ks%#6KClKT7WN`6oz_ zj#RL$BBha7S26tbBy@U2C<#SkFfAC-tBl8O6Ldab1iJG11ijBU84XOsfP8|{=lknv ze^{cJeZKJ!;bUQVeZUH718c&Mv;SkQM5 z4h{8o5APrJ2$nmCdVO6U&)|rsyLX`1Gc1_6{hodOy*_Rbdgv9bz5T#CH0asi?HlMC zI?xO4ySlr3?;P%e7J{MIvv0qrw|58{3AVjMT?4)M3=SReb?qDK75lTSBBha7S25hk z0Cf6Qp=7^j7+#Jes_^;vm_G(jgtM@!ucjU{R*|e;f0(YP$88`^94>ZB>JZc;Z~+js zclLUE@T+QvMm!!`;6~uXbDW^r+qJ*Hw+C3dhKBcd^^;B53crvz`^{LoNe0+CkDZIF zTv|1=FS82hdY4um-6uB1R%;ecvf#|h9-ypUxMl_|idXH`aHy6FPT^P?m2+II7~y>w#e!#X+7BU1j)BYwhA}XP6H#)+7YYb0YM#Kx z!1zc@sK5Ras`wZYKu6KIph!m}AjyIX6kU|TV#|h>ogly=@wldsGJjwzGkVj}{TXX{ z3Z#)1^^Q+ihqz|Qy=KicfWT*k1)o-I&Cm*aHC`>cU$0JFGqA`dUOm+4YIzNfjMq5i zp#7`?oc0z4_(mge>gc+viJ7)5k!luN(@Qlgtr?_R8Lb(y=Cut$G*I}E3a<@%Hc@^R zsf1A)rBRkjxT-vyS;E=#a25$y4LH5j)=F_@#Nn-h-fgs2PMow>A=PSWty1EwqqQnp zvpYaBye_WVtKX{R91i9iY%WS@$l+_&;2VK^9n7|As}jcm?VLy_&EEA~Z9avo=9sN4 zSGT5hi}bE~+NPCrdEMe%0AB;#4HUnDYn126t;>~=L&G&uj*VP1X`4{F*9*pcsb5AT zmFG9}BMFb71UY8}bu1JM2VYeP=0I@V5AM@94xSU#Krl{C9}0)~!#<1>1e5G$sk;Jg z0Urx~%r{NKfB~SbC8uwa#SV@upji~pjM_9Z@o?GeS`_;zegvxTK&!SIs(0Z0q+gQ zw4GtHF$6Rm3H|TXl(Rl~93mfD&PpBqT4a`A{T-P#w@KUb))wlx4A zjx)Pd^~|`!&5BADe{ym5M~kmKzIf~hi$D49mG~oq;@-b#Hg4YBxp}k4Z4~q&h%h0_ zf$))xz#vF30{lSaP6TK%1k*T)PKNmCc$9#6gHI1mh5Zn3>hJc4r-QwOM2Vn^MDG_^ zlzH-3&<4VJmJf!1Zv5%K>HV1~JD zsE7yBm2Jt&w%5kehMSUxn=UnPI~PcA?@w;;Pc;wB_FOPFerhnM4Nmy`iQ`O9x@}jo zZP&Yex}iJS(4AGXrY)D6+R{y(A2)R_Y)$X%Pwwnb?;K9<98QdkChk3y+BueL3dGq9 zmWHcF##(*S_13;|BL-U_QcswEDc#Tp#H++C;edKw(cFwZ+0~G)T{o! zP5}^i2eD}qSz{T{Dp1StH7GoTG6%nfPfjYqOBd%D!afQfP9bYg6T~k~j9WF0UjDNu z--2*%@%V|wg`-O+Ut0R%>BXO&yz<)1mluBipsx9=?ahmL6oNgr*5pXMH_;GJ6QOp^_bw!RvAr9>43pI`ox9Vw^ugb3ENylr&91cBoiw}8 zY)F~6C3HiHJBI+$(BuEbnu&q%(#bcMPyTXs&8+g8xtj0jY#!)r=H$h@8>UGn5kzp- zd8<~s>VfE*)vQ^wwM0L*MG2rQ)}`oG{MP(i^K9(+cOLsr+T4;fwO+YpK__7?;;#uqHh!QT_jkC)hY!FZt@Bq zpqPPz`?7)uC>Ef&3eZpmiWMl0#VBCN%BC@F`0Q7&Ej{t#<%M@Z7Z)F1Am~lV00MMv z9J(VP+v!7u{q1F2r#gVj+4Uqx^)f`bUqLWw@Ete{IcIgy>@bvqZ--$bWLzT`Wu zu?*T+)EB^Wuktk}LBfv1*!T%BBw!0zt3{tNTH;M7TTZlORZ#oXXp0Y@y!XVtSvAza zyqanjhSR$|@Hf{ZJG7gV&YLedZ%GV~X0^zrV@%Gu;dE_BvbG~zf*3txa?EW_*KADI zY|I)EW6Z~x5L23uF(bypm|XMa>4uw=4L7G7b|o8jWvz%WV@%Zx$fE9w`6M&li{wjS#)##A-Oo;IB_WgD>0SWEQOl+v)rKbCJ4(V$U-Itt$cPLmZQ zik*P+gJa8!qZOP2Y0IWGE6SU)lS2Fn9Jv%6cv1N z_PNFL?<}5u{_?LMr;6-m#UQpo*L={3d>w#?9-O$f#E+O>1T^x+Wg~|Wj3EdBaH|B> zSePeApe*R6tsNXu^Z1ora%5YAfwHKcm21>9hyNC^^FIdw3aVp__47?B!}`}c&jk`Y zJ<08Z7w#HPZyZT(9J$oGW1&2;`>y0IL#bA7wkKh2N*bG2G)&EiVzv*C1%GF5m=C1a zZ%?k@{%+^{f#lx1FN_{c@ARg2db3KEwe3<%N4jO#$1S@S0?FN;^zOTpyYEiib8o^M zNbL@$TE^o&sq&`JwbjzOpV0{~{2|JGGzDyq@5#QWzJ^G)f-o0E+EO1 zo&Cwq{=_WW}CPZNa$Rm@zqHuFZ>zpQ zgCLj_*NJXHgsv0af(WUGa)ubXlpa|GP`tnhyTAymMsCPCqRg4-YCOsS;lXjz#|0xp zlZa}uiiJZnmTt_Jemt{u`lY40*B9e2FCKe)`QaxPAO6Mi(`P?>{fS{g;o1x4jz%|k zL1%(G9KAnCXxIYy2|$rq#2fgaACghNSnvVZ$;)fU0tu{`yk`>+v3Q?MF4(gblAnUs zJcdOx%#yh(W2^eKy7si=lw-~~--SkMM{@m+MDs0)%FcA~7prQ(GTB`aX}Fs4vqMItT>RR4IIAEr@w`t5IR;u9 z%dH@c0jEO0{46y=ppbwld~(tNhJ=0t;1!?i$;J)8zN#)y!u($WCz~@}#da-l*p=IQ z->_eHbZaM=6Lf;tj8ZAj;Y+tInhJ%W0(szZ;Zc%hSvG|tR7QNnADIXi95$_?y#OuwvjAq8PpOpwLz8J(kLOBt z8Jqpns+x3FYqF{}UDckfYEM;l%o?+7S&98W&DEFeuC#sA$M#KU9!%Lg)Arkv_S;hS zuDCj5t54ZlGVV?2N7rX+T%aG9D(m1J>*LC{Goz`>?di&$$;zFn%3X04Wp2)6{NOZ-H5b6U7HUX_*2zp*}Ke8C;)q%-i>~J(dzK!Td z5NHuRiQpLo5YJwdAl#2Q1Rd~?KLxe`GhNP%-8r*oRu{KDW=cWLYMbeu-EwsBVr}EO z9dF(G#;wWP+h_XYtts~QOm)r7KzvJzt;tkX&m4%`Qfzh6(uU6ThVJBsZphyzs#;QP zH*CVzZ*nl9OJj<4XPVtNr~>=C7JTX4mJ{tMc0JT<8)pXN!zs2Q}XkAjLZVhc&BrXB!z!X;!IJbBg(Oa6k#aE2teWYW-aK+}QkM^?wsvaQv@{BrfNiHj^CVE=awm=_n6T|?SMHAW| z;Gh{rZz*IV&72;xjux-gTPD$HMv>Br<`F4B<`Oj=%{Ib)l1eXR4mmN4XeM^Z1zV)l z`VhSyvixg!D@_7RsdcjH4fL01QAe!@DnJ(#h-B0A1kV*&mkuQTpd0DK+H!Q z>pG$E1^H@Fn1T3?x$YCYQf$k`)^^wswRWSX%o&rmX3)Wl zHLi2^x14V{lQp|h_fqWcOeNgFm<@iyR*M3MuPrWcp8@6nC~#u3Te=M3)fYSEBSuKr zYB*UGP2N&j2thE-v?ivtbs&mJ6Vuu{Vx=`Ptz8D1!zRP4M}oY`xK zRIp`DvbQ>ix30lExH3{Rp@zFaM)3v+2o(s*^1s03^W^mMyU%|1`i}%XFOu<=f)uY| za=;_dfDMb}d1xrguSt|&QM7r%Iu@Or3I{RCDx+*7#~RTUtSISgP@;rGZVE40@#H@I zVqnR05ALmKTxbXE7ba46(6j0yo7P;^1y-VXYzf#R?b;zix>2mxguNAMmwVslFODkZ z-AM7CzIX%^3b%3^PkqG{koY;5jy-hc{LjRs2Kg}%-D>h0{E(jl5aiDngxzC+&x-{j zK+Vf&5USl3`O>pVwl^7M#lV0jFpQStyN>Ug8$Rtj?aP$gPY#?InD2RI@P)x-`IfY8 zPtvw0W$Q^8dom{Lao=&@rE=%V{_poc<(VIT#rJ|QS-v%G>rUFbQ?}lOu{UEbqY7MH z642~iPZBsq=O{$NAS)Bs*%KE3`t5KCR+vgVdki-HpS}0o;yW+@@!i?Y?O*xhyN^-n z7uQoseg*>}XAu-E8;yJ5_8#1uTDx`-4pH+^TU8iW%YM_H_PYaWtnL?MX*F1V0B-YT$h`J7|oT3 zz1Xn+TrAme>&(8{;rPa+37pEUpQ=q+rABR=YhGdCM+%gnl(o<}iB<1J68uooiV{lV z@NwkD{RPhA3;YFM8KalSAVAU5z(u?WrGtY~EnLObfknp)QfkSfi`Sj>V!#6YrQ$^& zi7JLC(AH}%mPw(#S-oa>ryQP?P3YaFg3W^$7(;O73@rVC7)2~E!|18OU!HmP3Y*9vsVIaYEG(1)?N&MItfR}oDLNFOEK1LV?Ct)rconcf2n5}& zVgKZzfPc5(EHa`^GWCzpA>I7VFbmt?ZWTX+2gjs`>$Rb&5ZnafJ%RzQa{9*(2XC&@MaN)(chQ6SfWWS9p2yj7pWD_OZPt_L6CW5LOR zAl%Izb%D0aXJs3odn8bUV(-@0G{vgEPCzh$S zd)yV*mp^JnS4HX{VC;liWol@t-TguBHHLw4SB{ZJoUlA0O#NQwR0j?iE z34luCMMwQP-CM>tj7i5X2&5lKvAgo^W}XRM*OXvu=C~BwlyNmASo<6ZZ9}F3uvK#v zh+Vb8j#`juh`4Jr_8O38^(SnND4+O#kGq_>;8!p}c18Se070Ip0JD0`AZXxz9^57t zHI6D^PVEFX2>%0s;GvY5#i4GBZdK@RgLaaB{QaK>Tx3O`pHH41yY=_&(1kkwQg{-jhZ zYO)Lfa1TmDmR2p4a6kz>3_^)S5nh%OOB9YQ13)5S)lLZx#ZK4^E703ucUbMD1g+w> aECWCyV6~hQ^a^K|0YC|#E1^Ue{C@#(ty$^- literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/core/__pycache__/reliability.cpython-313.pyc b/gx_gp_monitor/core/__pycache__/reliability.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ff2b4b50ba897761975c3d8b8c2b78569f94e2e GIT binary patch literal 19852 zcmb_^dvsIRneRDzTaqQe-w)Z?j)k$!o5TqZn@0!_lUjB|ii;4oY#fB_9!UgfC*yRI z#v~9(fM605oHi-Wv>~MJBs7qu;m*CQ^UsVFJGF93Zdp@g@S3}(Vhk6W-aqc|+eha} zGJ?tU-VNSopS}0l&u@S4t?Bf1BZqLHYHMHJ5{~;Ly~x27OH2=_IPMtd-~`UWJ5=j= zfoD&Zpkhz8pvF_ZUb9XsXl+zSvtGAOFX&lXyWX%aO-Ng36pZUkf@z&uFt1A&($`r8 z3oF;Hx304a)^!;|276bn&lECQyIDdOp8ECK>vDt~R%TeAyDm@2Lpsf2T%W(LKqz2o z)B3`7MM9B{>ny0`I*S+a(#VDU06TpEXcDed8I5b19_QBUKz{F zLSD9#SI+WskeAzOTcnP)Siy4hkejd6tYmow$SYLxW;s-ywuM;1g=$Apr>V0IpMy9-YYxP9JA1FTFMa^7aAQ^v0iG~JgPB|8f6~#&hko5X?J$#tc16^PJ6=J z97kzq`9hWBK59*<$J;Et)v~t+yxH+)XK#&on~S&E>}_7J%3dGn!qAOVKO2r5dhhDd z(aFKnlcR4&UO#S}JaqWVC$CLCcktSK$0kpH6dC&9kAqKN`{3Eh;~z}Ecv^am{Q8Gi z-`ziXhCZL)t`ZI1F2CE~x63W2QQYnJ_+4nGmEDr6nwIyoFgj*cOYpi0mXKuG@_CWpFH>(F%S99sP8 z@Tc$8V`B}pzyK|S1BbHLXg7-{Hjb67v#CO~8!#i1mUrsegI7O3Bbs)(_BaI$x6kbp z)BC)A{yvw-iFvv9ifL3PB`w=sT|4`GdYnD3E`Pru>YnKHW3F8~W&9jR$B+)r^a@1B zxXl?JJf*PN=7}9#j@vxf6Faa-ukgf<8q$w?G;;l3-mVJ#E8dKeq1Pj)Ux*xdC2`b7 zW3$lfYckpp62{+Tn>;xfdFylxet!Pul{AD)jqKQi^| zAprN}^RGmnA2qhxwL%_dD&$jCKoRYfP=pBJsqwizJ$J3(qxGj%u-U{_jXU~xxf^#c z?)SJG2lfsK{ZF{N{EZK9uJ0YF_xAhydiFNf0Nvq{_ia80u(;(-Tpg*4t(I6MKaV7VS8w$duv&RiZ;lj!TmT=jeaYJ!9 zujHU<8g)2Kjm?EkjV2_`kEV-hPtn%*;twIZJeFu(5le6@+U-Ufj@Pxzjo}!b7=xCIa)_j^sFx%pU_b_i z>1BxSh6F$%H9!Db{NeC-Y6;UX1QLiHf4##X7<$!My!3XUsQ!RIh5#0z9zy^ULX{9e zt7ru>@&ZzvgwRl1Gz)H5xAKXGi? z;)sA9_;-*QKk$j8Og!*2hTCMj_URjwFTNe45-E;z;&f2^z4MVnuiU+M(sXfd2hXRj z+xm`LXvBbpxfIPqWY-9EJZ9wjXvOG&N`S=xFfVCil*6?cHRZ!qz}Xtd4JF}{+Hpf6 zj@U$L>*(Kx8aEDUzAzL?&_nxaSI3S^m3(fNt!TBU36(UESrk=KR83J0MYItDff+jr z!fZ;_QDmct!9MMcKoND|TZ%}A{kq2ReRbjai?8b~+D&{Ipsl9F+R>7pD7|vv6X-pdJ|oymjl;y4ETbch<(QBI78wkh8Rh9< z!qVBB#Q`Q+usW=WGaMO0W^blFOSB}f?i69Y;pl z>E9u^eLMO+-C~B|?&{y|7WT?0L(FpT832B9cRSsCy4;MtN{vQ-%+}zW@C{oOzN1ZE zL3}F*AQl=T3l1J{N^OS-}rLdV6YkC^C3~@dh(j3fP8muc;9Z;gCC5l30{elP6!h`sy78bdX`<2x;(yR4Y8U#(2|cx)9pd-a(4bi_M*?T7oF`6W^Xvx{(1I>-+Viiy&-7c zAR#dg>ex&QRFHxERf`xw$yfM7-`i5z@Ba$)dK7Y;LuLUQQoPEC zznI*A1hoh?5y6xY5+f$LPEe45r<=HP_pSjy#8MvwQ({0wog_btnm%t&zgV0M*PB4p zSIeYDG&%ddPG6v_%kA?qoG0bPP?}wr0J!ZmfDYC^`U-gbBJWNRsU&vDLFIFjdu+Jp zLEVKfd`A)CFqy1}n)Wx17tWa|oEIvbH)fg_wq{RQ?IEju!de%y)?F%{J5kycDs4Kf zIcmI=k$2qmk||j7&=(m?ujI`O&R;W@_i!-d;iyKJmi1+B;qby>?(C7pq1@R+Yr^J? z33ExvTrzGh3uhM&ZM@Bzt_fLdra4twR#Le+^Dj{?GQakbi1}GvVT+c#sLg1pQ(w%@ zY^hOStkELfr65IAq7%fB<}ea>J7@GoFQ0HaQk=F#N}B4d*5kz7>l^}dQ=GY`pK}~T z%mjmzKN*}n^|mBrM?N}w?Y*}p7=$VST8?%IqK}}KKpuZ|9aI5_94_UIdmA>Jg8}i!+DFee2ldTRG}=6R-XOk)bV$Lc_vRn z_xGlRc_!ce2{ePPc7w11brY}+VHAOQAYm;hn(>TE<)F8WXRN%H##ssu zedopReC?wZJ=mPlT%-Q1#@M`2pL8G0(e^#5|)Zw5d)uUW)^P2{~vIZnDiuY zBPk&;2ay795(y(h$5{GFrfIzP$tRJavl4sXWFz{a|!ZL8p>h@qjClwzDXVGIc} z2@1ivxdbm5?K>cG8oj=(+ zv})X3d6}qCMaWt)(mqlBV5s`RG3$d2QWu3v7s;f^@{%Q3vGKRv6Rq1qt=obdUBUA0 z;~8DI(v#t}g>T7MU(7eQRO{_}Y1XST6zO!axw2WWqCBGct@fg^~;R(FbW1 z5sDWGKm{;voS>(O=(1p^h_JLkXI0on(QJxl=rop79!12k`uvEPkPd}K-JGz!F1%#L zb$zzBCR_!T2A-E^)kL*;qP%AIsQpdnO)8Df)@t*IH%B=7 z>L5vJNZlZ_m{8F_G+M*D( z4vqXZaEg+)aT@`RCeFrfA>d=7gKwj8%n0&qX(?aIf*h(!+lP`1g;o-IkS(2Ib0A3YO!_js(-2*U#e_?Ugi=NC{Vv2i8^ zJ4L=pwK2h^Tf1g*> z?{N8cc>1=B>RsK7L>-{8zZ(|49qv8deZ6iN`t(9vG)TjT&H1Plm5!}%0_DX}vPX_e{x!@E&@x6@*%8w#0H-9MYhK|d(jc8u!-@j(4 zIc(0AnEPS>Nb|^o5!cVvqbq-A7}*yro&P)Yg6nE7zvhO4%PAVJIaPO}?zQ^gW!Da^ zRK6Nf4+ln?-|8Bzd8clyW^u53NwD<(-a>aNk&N?U3=3HSf^( zpa1@F(?s!{Q1P76$Hs~mjae5@SXYFsE5@uXK|_m#2MDR&Q*mmR>?*t5{vG|@!ah_N zo}(xgN*1VqB0BuO3PcR)T-IhrRR!9W{D@P%nn1R5$|bK3Nxz7VHC{0CVRS=m9T{yB@6P@ zd341@TLm<)o_Ui@&?iQdcs50dA|*+p)=54`iR|@X!;STFLK~X4YXRs!g`7rH`0q#B8N+aRV==9eu|Fn6Z(KmcZ1xevWnN-q49_|ca}^luZ%KQml1C`9=De&H}JP z4rj??GDUXYL{WVR|Fi3FsTu2a2%-X!>4TPP_1ne;`Xv3BvgSWRjeD(mYF4_9DA0`1 zk)XAjxf{LwH`c__%bF~^Y7e;R;U^o=+YVk{dRW?FC&$-!s)wQ z!p`^_+fuA?QjLG128eaUDqG}LCVsg*WG%!tw-ESazVg|V5_ zi&pYFfG3GFrawqpf3PHy%uaae7_=i$9`8odIC<__V2Ep@;f*)2=4|- zpG*?PnK!0RJS(X%B*mNLp)fi0OL9+0SgILAAVUB-KSX~0tEo?(i90m3GL{g4T7n@6 zClDnqT~?RJ?Gmtd1APK4G=ww6>J;mDXSFY>DI$DW#0)MiogruRL;J*b3>xje3`l$S z^t*GTgnuMDLE9l5p@^7JiT8hvQmJUi3sgW6p(t^5OpUT_-c}CL#uTlCQX&PH1LlEmwDc5bumB)}Mc{%hLvB3&&&;=%W zcOZMgkRgRO5|w8J?3gxQx19E0)2!L}Z&*J=Tk(v_cy-);2FCwn)TY#|+$xY>iTE*u%gb*PCY6jTwg64j7PGT34U?@_QfJ#lzVU0dy~!3JcmTP8 zCuHQ4XKnk`4fA^f(Cu+H__0L}zbGB%0KuIta&pwh3^_I^s-$*M=scgxV&&tq`n~+^?4r>jj;N2Bc!cfrwU#;C32#uFq@R(n?dwSSXqtqAY_>rJ$sqG>4m=YF0u_qvuVSsq=?Wmg2vw%gHpTtVdxE-$l)RJyH~ z^ULIB#_}JyVw(lWor#jgp_0Yt=1eqogqk{nk8cY#Z3~sSSjqfQ$^2=px}s=U8#QyN zTik&E;VS#6<5!D6T>6Wpzg+$sM`*#?Q2p9a)g!~JFBg=ZvYdc}UG+%!>0NK`8gT{} zuKVqhV8^%r>A~OD2OrxUeB3?p`1eAOe=q0{gdPtBH}4Ma-aEeUZ-P(#?KH=IpKn%0 zIc^`{f(N3Sa82Fl%8@N+)1zv>bomXny08e+WMSzk0Aq0h6c zX*O-4rZ#jOYWjBYfyaW63FE%q!6)~O`Su2deZi9N-%_gzi(r5%Ec#2-j`jK4M`v$D z-=I39U!AV~hn42l2JJuUn-KqZgBCdgkvpij*w)kLN|4Y=DFy@t#oGt~B-uJm*LQ!`T06+D-bA2T+C;mo1F$>PeCj_`D#aU=}uJ!+AL zy=~jJn8crT=TLqK%n8)gIDO#kq1AqSzxkMZ77`?kgF_JysSepY=Ik4r4paw zIz!OR6qH`38uHTySMPRRA_oHVeae;--3dCV&Nm;mM&=)gw}JD(U-_-#ETC=`XmOH6 zAv(Ei^A;nqEXnJ}!MCSeP11+PwVrO?j&^^HuO;dx5k(2U;Er#OIa01Bio@uUT@8(2 zUoUilaQqeEI?T|DkDv%NL_#1|8D4jQn0eqQvq8>?QHeO&P*6=g>yA3|$FbJtVhb3` z1BtnmS2^<7X#0ELedoKQo6ao_&U$1lZ(T5B9XrfW*|4GFu7Te1$ivMW+BS+>m&YaS zlBAz`7&DBQiY8LHfDGXyM4}0vcFtITf{U^l^85u-OfNtLlSpO)!CW8@^N#V~eU5Wz zDuZYgQ~9I0@TP`OU(Z8(SX^ubhP2=VVqZ)1cgI^Yw4*P?J4I{flg$;*wVK_OMUosDW>!>xHUwRpD=`W=Z_nh*c z@Q&{MBBSZ{Vcm=z$u9a?dQpppyQnE^sZ>ieL!Ce~*xYHGK`EqZ6OtAy|MO|r(JXDK z948MqRo1izW8?v;Xk`L*P%+Q-RnfPh~n1`3457F)Q8!ZPZI*kY_ElsksJ zkiPU%Va;phLzbu}Tf+W$5cbC-TAP6PlMWRjuvz$(4oq)I7NXdwj9LL!i` zm9ma5sZL6tCup{agwF^I;mYSR$(iu7QtjrB@vI+Wb7uruI)_IB&(PyN>&R$ui5h-W z`oK@o7(7H@i=6)`a_*;-C-+MXxpb!qGB({8x^m&wYwx}Ojp-e~Ip6-~@ zL~wsE06T`S!2u^I++Tu>&#wDK-u@V!OC3kv`}MW=-=zB|eeVL0`fgD4zbU#!(QQOP zrds9?&#rNZ8Km70_?$icLKkk0z)r$5abcf+hu=TY)Y!PsKnWjG(D@LgGe$}@$io4i zlg1;EurA$dVFCupqoQ^fd7#GK=S6+&?220E$qg2foi^gdM7>XHNwi370z(W;KfQY) zT1eNhd`$LXi&jb>WW_#^Tr1f}!at*+1@rWsL1K!-u`EhDmY4O}ac=F3C2zt~_PM2O z%wijS7{;2s;;^M?!cr5m)P(bAP2|rH<-r@^pi(H_V*1Xv7<`EE;?`tT8>e z@#&3+?Zc0aYpUh9*~7i#8Zt2#)J)_r3gs_4`_#mew$PHcV4h>};b%5QHEL}^*pmBv z`elRlQ2PG#gVx(adMWXLn78H+TZ&g2_={zlmId6!%1lZx$XcmY|2M4~X@(p{Ll3kw zbWJJA=bD%hLYC9ZFn#EV)px-kP=$2hqNRKmBt`)qg@a7>p-yR1y_+#rF;L>fDHlRq zZ9t6ZKb8L%Jcckj3iL&M6h0M%d*F~}=JmKmVW=>!`$%Dd)@W`sfXIJ_>UZjY;Fsw?-yUBycuD$r-U!%OkrAJskSlh8t zoIf)8#>vT{SEO#QvR7kkfDme8Jy~_YtYFfoi31JLsY}eb^6OKv#bGK9=p3|?i1qiV zDCvN|ODQiRcxkp$`d=twdVx+#ZAT;;<%MzH;j1Z{eXbt2YzPoOM|G0*eMgYM;kI!G zOM<{<)aHC?&3<9`Q2On#DR&yW;GD3*a>%^j9L%r#`O35W&(@6IAIya-`lA17%8sfu z>5uZnyD$gMx#Jo0zRWK_vhRzGirbg6OQtzKRt(lASpLuj$8UvnnkJ67&+cj2y0-wRks<^vyj`t8U^A4+yL=z?*#mTKVEAnt?ic6pLm z6(Z#f&8$Tt`AR5rC5#eZtOHj;KGx(F3$7Et*G9-@({tl0OW5f-`~Y&G_ZIj|1E>J#wy?RLxe zBz$54Sz}U_$c7hcGtuzET$;qZq%~HIt}hbM*5V5Qt%WPEr%RPZxc_j#DnF{&Er^P6 zt(`7}RWo@)vc4)yiqDDHVaxvnmKMfmyhZpopdbZHF<2?i29E48xIR zRhN#hbZO5hokYpABH_cS!B=57Wh!%t5G9mJgveGa`~;l}Zy{O}Sb!M!La&~GHwif~ z%0p`NU!K2u{*+{mzH;uPE9VYM)mrUV<|8ioYEDplJrvRD6~c&ccNiCIzyR-xAp|GG z5W<&Kfzc?1hDkbdkqSshE)bcEU7HcjT>(LGd!;+KjI6~p6ftF@S;AWq&Jl}}u|#6o zYlJCCzd6X=uy6&o<3D)$2S=V7GJ~sxL-M9Wo5JaZr#79~6iTm~FgJzFO=ISV&iN;n zw}qCsjhWlvo_>7!OUr}s2Q8{OV2x_5X@!@`5MC29*T4WCE~^|`bzl>r890qrUxJXo ze*gMl!JKh(J-)qt$yxxqnO4ZI8dQI7t{%+{nP-0?)ok708Z4c6wrXPj`q2FK!Fd~g z+f1&cJ43BIgBv|TPyhIU58CRnf!)FWCxgX%#?5<~S88p@T06RGV$LI>IggB4AHg+K z_@i3MW12{7Y1?>)<4Se`(a^NQuYDvxd{(ufrGUFwU|cm{eX+7dx2j(KkM&x-BKUD(NB4l9|q)BZB2UC_JhsNFR+_LYA zE>M9X#*WXkA71(R2yu1^AU}m$$)CKmFIN_~o_zAjhV8Dt?m$Bqj5E#n7^2xf;SnHI za`oevuAV9;Vth5teMYW#AA#E(ht1GrMl zOyfy}>?^7{D;Y#n9+C@KD&H#92CVA1tP-a7)^UKT<@=XUm@4rfuou;-)9$}y&N;N< z`3=Fsx#ubXvA5|y*!`EXii0JK$Fi0L%}XSQjM3a>+$ZVHCUuNxOUIDTsld)I9X|3x zAUziRu`8)y7DZJQRZ~<$5u}Egk1_K?Aa-7$X{Gc%7^eb?h$He*bIg_ay2hlPJ>t8` z;Snt=*A@xJhRz>tH*V92o{m|2NjQB zIEh^OEaJ61aY6Dh_Ij*$Dp&gQF!(9Ne9L zaLOoioPBY3UuWz$SZ0nipGKOh-v2^J;l$CF2Q}sY)zMbW9Bsm30u!qY)tZFXVaTk* zhtlQ|ITW*KloDE&Fh8pN2?bvz4zMVwk*??`Ehn)sYWz*lMwGbMsf1OTondg$j1OXt zY!cy0qL|6{6^W$A=o|aZ5vj5fjOf3k9f1M&j9Fz%B*nMahl_Wy=?T|S`(J5V*+?+* zVlJLA-4`<5H)gU2HFjxLAik3)Myq|kbd*M^kJA)Ag-A34VbBlnz}Z%y$%|Ls{YLGQP0-Yi;D)yj-VC` z48U^mK2^F%(LYo4IYkUVT%**#QN$LVPMko1%z#C@wLt4T6TA!$NxyNjO!zZOvG|f+eCc%#(XEA<{3iZ7NAay%4PSDdLv+ishUdGH%^o+}R6w{R z_8S}?w~QtHN|aE1YgrASew{-Ug{5g(3m>jqI65cXF#o!C37sBfb;6qEvo)$?}`@yzb{KPm{t^fc4 literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/core/config_manager.py b/gx_gp_monitor/core/config_manager.py new file mode 100644 index 0000000..bbe738d --- /dev/null +++ b/gx_gp_monitor/core/config_manager.py @@ -0,0 +1,394 @@ +""" +配置管理模块 +负责加载、验证和管理系统配置 +""" + +import os +import yaml +from typing import Dict, Any, Optional +from pathlib import Path +from dataclasses import dataclass +from enum import Enum + + +class LogLevel(Enum): + """日志级别枚举""" + DEBUG = "DEBUG" + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + CRITICAL = "CRITICAL" + + +@dataclass +class CrawlerConfig: + """爬虫配置""" + base_url: str + timeout: int + max_retries: int + retry_delay: float + max_retry_delay: float + backoff_factor: float + user_agents: list + proxies: list + request_delay: float + request_delay_max: float + keyword: list + start_date: str + end_date: str + max_pages: int + page_size: int + + +@dataclass +class DatabaseConfig: + """数据库配置""" + enabled: bool + type: str + host: str + port: int + name: str + user: str + password: str + pool_size: int + max_overflow: int + pool_timeout: int + pool_recycle: int + data_retention_days: int + auto_cleanup: bool + + +@dataclass +class WeChatConfig: + """企业微信配置""" + enabled: bool + corp_id: str + agent_id: str + secret: str + token: str + encoding_aes_key: str + port: int + host: str + debug: bool + use_proxy: bool + proxy_api_url: str + + +@dataclass +class MarkdownConfig: + """Markdown配置""" + enabled: bool + output_file: str + max_entries: int + include_today_highlight: bool + template_file: str + + +@dataclass +class SchedulerConfig: + """调度配置""" + enabled: bool + timezone: str + jobs: list + + +@dataclass +class MonitoringConfig: + """监控配置""" + enabled: bool + health_check_interval: int + alert_on_failure: bool + max_consecutive_failures: int + metrics_enabled: bool + + +@dataclass +class SystemConfig: + """系统配置""" + debug: bool + log_level: LogLevel + log_file: str + log_max_size: int + log_backup_count: int + crawler: CrawlerConfig + database: DatabaseConfig + wechat_app: WeChatConfig + markdown: MarkdownConfig + scheduler: Optional[SchedulerConfig] + monitoring: MonitoringConfig + sources: Dict[str, Dict[str, Any]] + + +class ConfigManager: + """配置管理器""" + + def __init__(self, config_file: Optional[str] = None): + """ + 初始化配置管理器 + + Args: + config_file: 配置文件路径，如果为None则使用默认路径 + """ + if config_file is None: + # 默认配置文件路径 + current_dir = Path(__file__).parent.parent + self.config_file = current_dir / "config" / "config.yaml" + else: + self.config_file = Path(config_file) + + self._config_data = {} + self._config = None + + def load_config(self) -> SystemConfig: + """ + 加载配置文件 + + Returns: + SystemConfig: 系统配置对象 + + Raises: + FileNotFoundError: 配置文件不存在 + yaml.YAMLError: 配置文件格式错误 + ValueError: 配置验证失败 + """ + if not self.config_file.exists(): + raise FileNotFoundError(f"配置文件不存在: {self.config_file}") + + try: + with open(self.config_file, 'r', encoding='utf-8') as f: + self._config_data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise yaml.YAMLError(f"配置文件格式错误: {e}") + + # 验证配置 + self._validate_config() + + # 解析配置 + self._config = self._parse_config() + return self._config + + def _validate_config(self): + """验证配置完整性""" + required_keys = [ + 'debug', 'log_level', 'log_file', 'log_max_size', 'log_backup_count', + 'crawler', 'database', 'wechat_app', 'markdown', + 'monitoring', 'sources' + ] + + for key in required_keys: + if key not in self._config_data: + raise ValueError(f"配置文件缺少必需的配置项: {key}") + + # 验证爬虫配置 + crawler_required = [ + 'base_url', 'timeout', 'max_retries', 'retry_delay', 'max_retry_delay', + 'backoff_factor', 'user_agents', 'proxies', 'request_delay', + 'request_delay_max', 'keyword', 'max_pages', 'page_size' + ] + + for key in crawler_required: + if key not in self._config_data['crawler']: + raise ValueError(f"爬虫配置缺少必需项: {key}") + + # 验证数据库配置 + if self._config_data.get('database', {}).get('enabled', False): + db_required = ['type', 'host', 'port', 'name', 'user', 'password'] + for key in db_required: + if key not in self._config_data['database']: + raise ValueError(f"数据库配置缺少必需项: {key}") + + # 验证企业微信配置 + if self._config_data.get('wechat_app', {}).get('enabled', False): + wechat_required = ['corp_id', 'agent_id', 'secret', 'token', 'encoding_aes_key'] + for key in wechat_required: + if key not in self._config_data['wechat_app']: + raise ValueError(f"企业微信配置缺少必需项: {key}") + + def _parse_config(self) -> SystemConfig: + """解析配置数据""" + crawler_data = self._config_data['crawler'] + crawler = CrawlerConfig( + base_url=crawler_data['base_url'], + timeout=crawler_data['timeout'], + max_retries=crawler_data['max_retries'], + retry_delay=crawler_data['retry_delay'], + max_retry_delay=crawler_data['max_retry_delay'], + backoff_factor=crawler_data['backoff_factor'], + user_agents=crawler_data['user_agents'], + proxies=crawler_data['proxies'], + request_delay=crawler_data['request_delay'], + request_delay_max=crawler_data['request_delay_max'], + keyword=crawler_data['keyword'], + start_date=crawler_data.get('start_date', ''), + end_date=crawler_data.get('end_date', ''), + max_pages=crawler_data['max_pages'], + page_size=crawler_data['page_size'] + ) + + db_data = self._config_data['database'] + database = DatabaseConfig( + enabled=db_data.get('enabled', False), + type=db_data.get('type', 'postgresql'), + host=db_data.get('host', 'localhost'), + port=db_data.get('port', 5432), + name=db_data.get('name', ''), + user=db_data.get('user', ''), + password=db_data.get('password', ''), + pool_size=db_data.get('pool_size', 5), + max_overflow=db_data.get('max_overflow', 10), + pool_timeout=db_data.get('pool_timeout', 30), + pool_recycle=db_data.get('pool_recycle', 3600), + data_retention_days=db_data.get('data_retention_days', 90), + auto_cleanup=db_data.get('auto_cleanup', True) + ) + + wechat_data = self._config_data['wechat_app'] + wechat_app = WeChatConfig( + enabled=wechat_data.get('enabled', False), + corp_id=wechat_data.get('corp_id', ''), + agent_id=wechat_data.get('agent_id', ''), + secret=wechat_data.get('secret', ''), + token=wechat_data.get('token', ''), + encoding_aes_key=wechat_data.get('encoding_aes_key', ''), + port=wechat_data.get('port', 18001), + host=wechat_data.get('host', '0.0.0.0'), + debug=wechat_data.get('debug', False), + use_proxy=wechat_data.get('use_proxy', False), + proxy_api_url=wechat_data.get('proxy_api_url', 'https://api.v6ole.top') + ) + + md_data = self._config_data['markdown'] + markdown = MarkdownConfig( + enabled=md_data.get('enabled', True), + output_file=md_data.get('output_file', 'onu.md'), + max_entries=md_data.get('max_entries', 1000), + include_today_highlight=md_data.get('include_today_highlight', True), + template_file=md_data.get('template_file', 'templates/announcement.md') + ) + + # scheduler配置为可选 + scheduler = None + if 'scheduler' in self._config_data: + scheduler_data = self._config_data['scheduler'] + scheduler = SchedulerConfig( + enabled=scheduler_data.get('enabled', False), + timezone=scheduler_data.get('timezone', 'Asia/Shanghai'), + jobs=scheduler_data.get('jobs', []) + ) + + monitoring_data = self._config_data['monitoring'] + monitoring = MonitoringConfig( + enabled=monitoring_data.get('enabled', True), + health_check_interval=monitoring_data.get('health_check_interval', 300), + alert_on_failure=monitoring_data.get('alert_on_failure', True), + max_consecutive_failures=monitoring_data.get('max_consecutive_failures', 3), + metrics_enabled=monitoring_data.get('metrics_enabled', True) + ) + + return SystemConfig( + debug=self._config_data['debug'], + log_level=LogLevel(self._config_data['log_level']), + log_file=self._config_data['log_file'], + log_max_size=self._config_data['log_max_size'], + log_backup_count=self._config_data['log_backup_count'], + crawler=crawler, + database=database, + wechat_app=wechat_app, + markdown=markdown, + scheduler=scheduler, + monitoring=monitoring, + sources=self._config_data['sources'] + ) + + def get_config(self) -> SystemConfig: + """ + 获取配置对象 + + Returns: + SystemConfig: 系统配置对象 + + Raises: + RuntimeError: 配置未加载 + """ + if self._config is None: + raise RuntimeError("配置未加载，请先调用 load_config()") + return self._config + + def reload_config(self) -> SystemConfig: + """ + 重新加载配置 + + Returns: + SystemConfig: 重新加载的系统配置对象 + """ + self._config = None + return self.load_config() + + def get_value(self, key_path: str, default=None) -> Any: + """ + 通过路径获取配置值 + + Args: + key_path: 配置路径，如 'crawler.timeout' 或 'database.host' + default: 默认值 + + Returns: + 配置值或默认值 + """ + keys = key_path.split('.') + value = self._config_data + + try: + for key in keys: + value = value[key] + return value + except (KeyError, TypeError): + return default + + +# 全局配置管理器实例 +_config_manager = None + + +def get_config_manager(config_file: Optional[str] = None) -> ConfigManager: + """ + 获取全局配置管理器实例 + + Args: + config_file: 配置文件路径 + + Returns: + ConfigManager: 配置管理器实例 + """ + global _config_manager + if _config_manager is None: + _config_manager = ConfigManager(config_file) + return _config_manager + + +def load_config(config_file: Optional[str] = None) -> SystemConfig: + """ + 加载系统配置 + + Args: + config_file: 配置文件路径 + + Returns: + SystemConfig: 系统配置对象 + """ + manager = get_config_manager(config_file) + return manager.load_config() + + +def get_config() -> SystemConfig: + """ + 获取当前加载的配置 + + Returns: + SystemConfig: 系统配置对象 + + Raises: + RuntimeError: 配置未加载 + """ + manager = get_config_manager() + return manager.get_config() diff --git a/gx_gp_monitor/core/database.py b/gx_gp_monitor/core/database.py new file mode 100644 index 0000000..cc4bb45 --- /dev/null +++ b/gx_gp_monitor/core/database.py @@ -0,0 +1,612 @@ +""" +PostgreSQL数据库连接和操作模块 +提供数据库连接池、CRUD操作、数据清理等功能 +""" + +import psycopg2 +from psycopg2 import pool, extras +from psycopg2.extras import RealDictCursor +from contextlib import contextmanager +from typing import List, Dict, Any, Optional, Generator +from datetime import datetime, timedelta +import threading +from dataclasses import asdict + +from .models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus +from .config_manager import get_config +from .logger import get_logger +from .reliability import retry_on_exception, RetryConfig + + +logger = get_logger(__name__) + + +class DatabaseConnectionPool: + """数据库连接池管理器""" + + _instance = None + _pool = None + _lock = threading.Lock() + + def __new__(cls): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if self._pool is None: + self._pool = None + self._config = None + + def init_pool(self, config): + """ + 初始化连接池 + + Args: + config: 数据库配置 + """ + if self._pool is not None: + return + + try: + self._config = config + self._pool = psycopg2.pool.SimpleConnectionPool( + minconn=config.pool_size, + maxconn=config.pool_size + config.max_overflow, + host=config.host, + port=config.port, + database=config.name, + user=config.user, + password=config.password, + connect_timeout=config.pool_timeout + ) + logger.info("数据库连接池初始化成功") + except Exception as e: + logger.error(f"数据库连接池初始化失败: {str(e)}") + raise + + def get_connection(self): + """ + 获取数据库连接 + + Returns: + 数据库连接对象 + + Raises: + Exception: 获取连接失败 + """ + if self._pool is None: + raise Exception("数据库连接池未初始化") + + try: + conn = self._pool.getconn() + # 设置自动提交为False，需要手动提交 + conn.autocommit = False + return conn + except Exception as e: + logger.error(f"获取数据库连接失败: {str(e)}") + raise + + def return_connection(self, conn): + """ + 返回数据库连接到连接池 + + Args: + conn: 数据库连接对象 + """ + if self._pool and conn: + try: + self._pool.putconn(conn) + except Exception as e: + logger.warning(f"返回数据库连接失败: {str(e)}") + + def close_all(self): + """关闭所有连接""" + if self._pool: + try: + self._pool.closeall() + logger.info("数据库连接池已关闭") + except Exception as e: + logger.error(f"关闭数据库连接池失败: {str(e)}") + + +# 全局连接池实例 +_connection_pool = DatabaseConnectionPool() + + +@contextmanager +def get_db_connection(): + """ + 获取数据库连接的上下文管理器 + + Yields: + 数据库连接对象 + """ + conn = None + try: + conn = _connection_pool.get_connection() + yield conn + except Exception as e: + logger.error(f"数据库连接错误: {str(e)}") + raise + finally: + if conn: + _connection_pool.return_connection(conn) + + +@contextmanager +def get_db_cursor(commit: bool = True): + """ + 获取数据库游标的上下文管理器 + + Args: + commit: 是否自动提交事务 + + Yields: + 数据库游标对象 + """ + with get_db_connection() as conn: + cursor = None + try: + cursor = conn.cursor(cursor_factory=RealDictCursor) + yield cursor + if commit: + conn.commit() + except Exception as e: + conn.rollback() + logger.error(f"数据库操作错误: {str(e)}") + raise + finally: + if cursor: + cursor.close() + + +class DatabaseManager: + """数据库管理器""" + + def __init__(self): + self.config = get_config() + if self.config.database.enabled: + _connection_pool.init_pool(self.config.database) + else: + logger.warning("数据库功能已禁用") + + def init_database(self): + """初始化数据库表结构""" + if not self.config.database.enabled: + return + + logger.info("开始初始化数据库表结构") + + # 创建表的SQL语句 + create_tables_sql = """ + -- 公告表 + CREATE TABLE IF NOT EXISTS announcements ( + id SERIAL PRIMARY KEY, + title VARCHAR(500) NOT NULL, + publish_date TIMESTAMP NOT NULL, + purchase_name VARCHAR(200), + content_url TEXT, + source_code VARCHAR(50) NOT NULL, + source_name VARCHAR(100) NOT NULL, + announcement_type VARCHAR(50) NOT NULL, + crawled_at TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + content_hash VARCHAR(32) UNIQUE, + keyword_matched BOOLEAN DEFAULT FALSE, + date_filtered BOOLEAN DEFAULT TRUE, + is_new BOOLEAN DEFAULT TRUE, + is_today BOOLEAN DEFAULT FALSE + ); + + -- 公告来源表 + CREATE TABLE IF NOT EXISTS announcement_sources ( + code VARCHAR(50) PRIMARY KEY, + category_id INTEGER NOT NULL, + name VARCHAR(100) NOT NULL, + type VARCHAR(50) NOT NULL + ); + + -- 爬取结果表 + CREATE TABLE IF NOT EXISTS crawl_results ( + id SERIAL PRIMARY KEY, + source_code VARCHAR(50) NOT NULL, + status VARCHAR(20) NOT NULL, + total_count INTEGER DEFAULT 0, + new_count INTEGER DEFAULT 0, + error_message TEXT, + crawled_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + duration FLOAT DEFAULT 0.0, + FOREIGN KEY (source_code) REFERENCES announcement_sources(code) + ); + + -- 创建索引 + CREATE INDEX IF NOT EXISTS idx_announcements_publish_date ON announcements(publish_date DESC); + CREATE INDEX IF NOT EXISTS idx_announcements_source_code ON announcements(source_code); + CREATE INDEX IF NOT EXISTS idx_announcements_content_hash ON announcements(content_hash); + CREATE INDEX IF NOT EXISTS idx_announcements_created_at ON announcements(created_at DESC); + CREATE INDEX IF NOT EXISTS idx_crawl_results_crawled_at ON crawl_results(crawled_at DESC); + + -- 创建更新时间触发器 + CREATE OR REPLACE FUNCTION update_updated_at_column() + RETURNS TRIGGER AS $$ + BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ language 'plpgsql'; + + DROP TRIGGER IF EXISTS update_announcements_updated_at ON announcements; + CREATE TRIGGER update_announcements_updated_at + BEFORE UPDATE ON announcements + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + """ + + with get_db_cursor() as cursor: + cursor.execute(create_tables_sql) + logger.info("数据库表结构初始化完成") + + @retry_on_exception(RetryConfig(max_retries=3)) + def save_announcement(self, announcement: Announcement) -> bool: + """ + 保存公告到数据库 + + Args: + announcement: 公告对象 + + Returns: + bool: 保存是否成功 + """ + if not self.config.database.enabled: + return False + + # 生成内容哈希（如果还没有） + if not announcement.content_hash: + announcement.generate_content_hash() + + sql = """ + INSERT INTO announcements ( + title, publish_date, purchase_name, content_url, source_code, source_name, + announcement_type, crawled_at, content_hash, keyword_matched, + date_filtered, is_new, is_today + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (content_hash) DO NOTHING + """ + + values = ( + announcement.title, + announcement.publish_date, + announcement.purchase_name, + announcement.content_url, + announcement.source_code, + announcement.source_name, + announcement.announcement_type.value, + announcement.crawled_at, + announcement.content_hash, + announcement.keyword_matched, + announcement.date_filtered, + announcement.is_new, + announcement.is_today + ) + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, values) + affected_rows = cursor.rowcount + if affected_rows > 0: + logger.debug(f"成功保存公告: {announcement.title[:50]}...") + return True + else: + logger.debug(f"公告已存在，跳过保存: {announcement.title[:50]}...") + return False + except Exception as e: + logger.error(f"保存公告失败: {str(e)}") + return False + + @retry_on_exception(RetryConfig(max_retries=3)) + def save_announcements_batch(self, announcements: List[Announcement]) -> int: + """ + 批量保存公告 + + Args: + announcements: 公告列表 + + Returns: + int: 成功保存的数量 + """ + if not self.config.database.enabled: + return 0 + + if not announcements: + return 0 + + # 为没有哈希的公告生成哈希 + for announcement in announcements: + if not announcement.content_hash: + announcement.generate_content_hash() + + sql = """ + INSERT INTO announcements ( + title, publish_date, purchase_name, content_url, source_code, source_name, + announcement_type, crawled_at, content_hash, keyword_matched, + date_filtered, is_new, is_today + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (content_hash) DO NOTHING + """ + + values = [] + for announcement in announcements: + values.append(( + announcement.title, + announcement.publish_date, + announcement.purchase_name, + announcement.content_url, + announcement.source_code, + announcement.source_name, + announcement.announcement_type.value, + announcement.crawled_at, + announcement.content_hash, + announcement.keyword_matched, + announcement.date_filtered, + announcement.is_new, + announcement.is_today + )) + + try: + with get_db_cursor() as cursor: + extras.execute_batch(cursor, sql, values) + affected_rows = cursor.rowcount + logger.info(f"批量保存公告完成，成功保存 {affected_rows} 条") + return affected_rows + except Exception as e: + logger.error(f"批量保存公告失败: {str(e)}") + return 0 + + @retry_on_exception(RetryConfig(max_retries=3)) + def get_announcements(self, + source_code: Optional[str] = None, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + limit: int = 100, + offset: int = 0) -> List[Announcement]: + """ + 查询公告 + + Args: + source_code: 来源代码过滤 + start_date: 开始日期 + end_date: 结束日期 + limit: 限制数量 + offset: 偏移量 + + Returns: + List[Announcement]: 公告列表 + """ + if not self.config.database.enabled: + return [] + + sql = """ + SELECT * FROM announcements + WHERE 1=1 + """ + params = [] + + if source_code: + sql += " AND source_code = %s" + params.append(source_code) + + if start_date: + sql += " AND publish_date >= %s" + params.append(start_date) + + if end_date: + sql += " AND publish_date <= %s" + params.append(end_date) + + sql += " ORDER BY publish_date DESC LIMIT %s OFFSET %s" + params.extend([limit, offset]) + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, params) + rows = cursor.fetchall() + + announcements = [] + for row in rows: + # 转换数据类型 + row_dict = dict(row) + row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type']) + announcements.append(Announcement.from_dict(row_dict)) + + return announcements + except Exception as e: + logger.error(f"查询公告失败: {str(e)}") + return [] + + @retry_on_exception(RetryConfig(max_retries=3)) + def save_crawl_result(self, result: CrawlResult) -> bool: + """ + 保存爬取结果 + + Args: + result: 爬取结果对象 + + Returns: + bool: 保存是否成功 + """ + if not self.config.database.enabled: + return False + + sql = """ + INSERT INTO crawl_results ( + source_code, status, total_count, new_count, error_message, + crawled_at, duration + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + """ + + values = ( + result.source.code, + result.status.value, + result.total_count, + result.new_count, + result.error_message, + result.crawled_at, + result.duration + ) + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, values) + logger.debug(f"保存爬取结果: {result.source.name}") + return True + except Exception as e: + logger.error(f"保存爬取结果失败: {str(e)}") + return False + + @retry_on_exception(RetryConfig(max_retries=3)) + def cleanup_expired_data(self, days: int = 90) -> int: + """ + 清理过期数据 + + Args: + days: 保留天数 + + Returns: + int: 清理的记录数 + """ + if not self.config.database.enabled: + return 0 + + cutoff_date = datetime.now() - timedelta(days=days) + + sql = "DELETE FROM announcements WHERE created_at < %s" + try: + with get_db_cursor() as cursor: + cursor.execute(sql, (cutoff_date,)) + deleted_count = cursor.rowcount + logger.info(f"清理过期数据完成，删除 {deleted_count} 条记录") + return deleted_count + except Exception as e: + logger.error(f"清理过期数据失败: {str(e)}") + return 0 + + @retry_on_exception(RetryConfig(max_retries=3)) + def get_statistics(self) -> Dict[str, Any]: + """ + 获取统计信息 + + Returns: + Dict[str, Any]: 统计数据 + """ + if not self.config.database.enabled: + return {} + + sql = """ + SELECT + COUNT(*) as total_announcements, + COUNT(CASE WHEN is_today THEN 1 END) as today_announcements, + COUNT(CASE WHEN is_new THEN 1 END) as new_announcements, + COUNT(DISTINCT source_code) as sources_count, + MAX(crawled_at) as last_crawl_time + FROM announcements + """ + + try: + with get_db_cursor() as cursor: + cursor.execute(sql) + result = cursor.fetchone() + return dict(result) if result else {} + except Exception as e: + logger.error(f"获取统计信息失败: {str(e)}") + return {} + + def is_announcement_exists(self, content_hash: str) -> bool: + """ + 检查公告是否已存在 + + Args: + content_hash: 内容哈希 + + Returns: + bool: 是否存在 + """ + if not self.config.database.enabled: + return False + + sql = "SELECT 1 FROM announcements WHERE content_hash = %s LIMIT 1" + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, (content_hash,)) + return cursor.fetchone() is not None + except Exception as e: + logger.error(f"检查公告存在性失败: {str(e)}") + return False + + def get_recent_announcements(self, hours: int = 24) -> List[Announcement]: + """ + 获取最近的公告 + + Args: + hours: 最近小时数 + + Returns: + List[Announcement]: 公告列表 + """ + if not self.config.database.enabled: + return [] + + cutoff_time = datetime.now() - timedelta(hours=hours) + + sql = """ + SELECT * FROM announcements + WHERE crawled_at >= %s + ORDER BY crawled_at DESC + """ + + try: + with get_db_cursor() as cursor: + cursor.execute(sql, (cutoff_time,)) + rows = cursor.fetchall() + + announcements = [] + for row in rows: + row_dict = dict(row) + row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type']) + announcements.append(Announcement.from_dict(row_dict)) + + return announcements + except Exception as e: + logger.error(f"获取最近公告失败: {str(e)}") + return [] + + +# 全局数据库管理器实例 +_db_manager = None + + +def get_database_manager() -> DatabaseManager: + """ + 获取数据库管理器实例 + + Returns: + DatabaseManager: 数据库管理器实例 + """ + global _db_manager + if _db_manager is None: + _db_manager = DatabaseManager() + return _db_manager + + +def init_database(): + """初始化数据库""" + manager = get_database_manager() + manager.init_database() + + +def cleanup_database(): + """清理数据库连接""" + _connection_pool.close_all() diff --git a/gx_gp_monitor/core/logger.py b/gx_gp_monitor/core/logger.py new file mode 100644 index 0000000..5c9220d --- /dev/null +++ b/gx_gp_monitor/core/logger.py @@ -0,0 +1,354 @@ +""" +统一日志管理模块 +提供结构化日志记录功能，支持控制台和文件输出 +""" + +import os +import sys +import logging +import logging.handlers +from pathlib import Path +from typing import Optional, Dict, Any +from datetime import datetime + +from .config_manager import get_config + + +class ColoredFormatter(logging.Formatter): + """带颜色的日志格式化器""" + + # ANSI颜色代码 + COLORS = { + 'DEBUG': '\033[36m', # 青色 + 'INFO': '\033[32m', # 绿色 + 'WARNING': '\033[33m', # 黄色 + 'ERROR': '\033[31m', # 红色 + 'CRITICAL': '\033[35m', # 紫色 + } + RESET = '\033[0m' # 重置颜色 + + def format(self, record): + # 检查是否已经包含ANSI颜色代码 + if '\033[' in record.levelname: + # 如果已经着色，直接返回原始格式 + return super().format(record) + + # 添加颜色 + if record.levelname in self.COLORS: + # 为levelname添加颜色 + colored_levelname = f"{self.COLORS[record.levelname]}{record.levelname}{self.RESET}" + # 使用原始levelname来确定消息的颜色 + record.msg = f"{self.COLORS[record.levelname]}{record.msg}{self.RESET}" + record.levelname = colored_levelname + + return super().format(record) + + +class Logger: + """统一日志管理器""" + + _instance = None + _initialized = False + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if not self._initialized: + self._loggers = {} + self._config = None + self._initialized = True + + def init_logger(self, name: str = "gx_gp_monitor", config=None) -> logging.Logger: + """ + 初始化日志器 + + Args: + name: 日志器名称 + config: 配置对象，如果为None则从全局配置加载 + + Returns: + logging.Logger: 配置好的日志器实例 + """ + if name in self._loggers: + return self._loggers[name] + + # 获取配置 + if config is None: + try: + self._config = get_config() + except RuntimeError: + # 配置未加载，使用默认配置 + self._config = self._get_default_config() + else: + self._config = config + + # 创建日志器 + logger = logging.getLogger(name) + logger.setLevel(getattr(logging, self._config.log_level.value)) + + # 避免重复添加处理器 + if logger.handlers: + return logger + + # 创建格式化器 + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + # 控制台处理器 + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(getattr(logging, self._config.log_level.value)) + + # 使用彩色格式化器（如果支持） + if sys.platform != 'win32' and 'TERM' in os.environ: + colored_formatter = ColoredFormatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + console_handler.setFormatter(colored_formatter) + else: + console_handler.setFormatter(formatter) + + logger.addHandler(console_handler) + + # 文件处理器（如果配置了日志文件） + if self._config.log_file: + log_dir = Path(self._config.log_file).parent + log_dir.mkdir(parents=True, exist_ok=True) + + file_handler = logging.handlers.RotatingFileHandler( + self._config.log_file, + maxBytes=self._config.log_max_size, + backupCount=self._config.log_backup_count, + encoding='utf-8' + ) + file_handler.setLevel(getattr(logging, self._config.log_level.value)) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + + self._loggers[name] = logger + return logger + + def _get_default_config(self): + """获取默认配置""" + from .config_manager import LogLevel + + class DefaultConfig: + def __init__(self): + self.log_level = LogLevel.INFO + self.log_file = "logs/gx_gp_monitor.log" + self.log_max_size = 10485760 # 10MB + self.log_backup_count = 5 + + return DefaultConfig() + + def get_logger(self, name: str = "gx_gp_monitor") -> logging.Logger: + """ + 获取日志器 + + Args: + name: 日志器名称 + + Returns: + logging.Logger: 日志器实例 + """ + if name not in self._loggers: + return self.init_logger(name) + return self._loggers[name] + + def log_crawl_start(self, source_name: str, logger: Optional[logging.Logger] = None): + """记录爬取开始""" + if logger is None: + logger = self.get_logger() + logger.info(f"开始爬取 {source_name}") + + def log_crawl_success(self, source_name: str, count: int, duration: float, + logger: Optional[logging.Logger] = None): + """记录爬取成功""" + if logger is None: + logger = self.get_logger() + logger.info(f"{source_name} 爬取完成，共获取 {count} 条公告，耗时 {duration:.2f}秒") + + def log_crawl_error(self, source_name: str, error: str, + logger: Optional[logging.Logger] = None): + """记录爬取错误""" + if logger is None: + logger = self.get_logger() + logger.error(f"{source_name} 爬取失败: {error}") + + def log_announcement_filtered(self, reason: str, count: int, + logger: Optional[logging.Logger] = None): + """记录公告筛选信息""" + if logger is None: + logger = self.get_logger() + logger.info(f"公告筛选 - {reason}: {count} 条") + + def log_database_operation(self, operation: str, table: str, count: int = 0, + logger: Optional[logging.Logger] = None): + """记录数据库操作""" + if logger is None: + logger = self.get_logger() + if count > 0: + logger.info(f"数据库操作 - {operation} {table}: {count} 条记录") + else: + logger.info(f"数据库操作 - {operation} {table}") + + def log_notification_sent(self, channel: str, recipient_count: int, + logger: Optional[logging.Logger] = None): + """记录通知发送""" + if logger is None: + logger = self.get_logger() + logger.info(f"通知发送 - {channel}: 向 {recipient_count} 个接收者发送") + + def log_system_metrics(self, metrics: Dict[str, Any], + logger: Optional[logging.Logger] = None): + """记录系统指标""" + if logger is None: + logger = self.get_logger() + metrics_str = ", ".join([f"{k}={v}" for k, v in metrics.items()]) + logger.info(f"系统指标: {metrics_str}") + + def log_performance_warning(self, operation: str, duration: float, threshold: float, + logger: Optional[logging.Logger] = None): + """记录性能警告""" + if logger is None: + logger = self.get_logger() + logger.warning(f"性能警告 - {operation} 耗时 {duration:.2f}秒，超过阈值 {threshold:.2f}秒") + + +# 全局日志管理器实例 +_logger_manager = Logger() + + +def get_logger(name: str = "gx_gp_monitor") -> logging.Logger: + """ + 获取日志器 + + Args: + name: 日志器名称 + + Returns: + logging.Logger: 日志器实例 + """ + return _logger_manager.get_logger(name) + + +def init_logger(name: str = "gx_gp_monitor", config=None) -> logging.Logger: + """ + 初始化并获取日志器 + + Args: + name: 日志器名称 + config: 配置对象 + + Returns: + logging.Logger: 日志器实例 + """ + return _logger_manager.init_logger(name, config) + + +def log_function_call(func_name: str, args: Optional[Dict[str, Any]] = None, + logger: Optional[logging.Logger] = None): + """ + 装饰器：记录函数调用 + + Args: + func_name: 函数名称 + args: 函数参数 + logger: 日志器实例 + """ + def decorator(func): + def wrapper(*args, **kwargs): + nonlocal logger + if logger is None: + logger = get_logger() + + start_time = datetime.now() + logger.debug(f"调用函数: {func_name}") + + try: + result = func(*args, **kwargs) + duration = (datetime.now() - start_time).total_seconds() + logger.debug(f"函数 {func_name} 执行完成，耗时 {duration:.3f}秒") + return result + except Exception as e: + duration = (datetime.now() - start_time).total_seconds() + logger.error(f"函数 {func_name} 执行失败，耗时 {duration:.3f}秒: {str(e)}") + raise + + return wrapper + return decorator + + +# 便捷函数 +def log_info(message: str, logger: Optional[logging.Logger] = None): + """记录信息日志""" + if logger is None: + logger = get_logger() + logger.info(message) + + +def log_warning(message: str, logger: Optional[logging.Logger] = None): + """记录警告日志""" + if logger is None: + logger = get_logger() + logger.warning(message) + + +def log_error(message: str, logger: Optional[logging.Logger] = None): + """记录错误日志""" + if logger is None: + logger = get_logger() + logger.error(message) + + +def log_debug(message: str, logger: Optional[logging.Logger] = None): + """记录调试日志""" + if logger is None: + logger = get_logger() + logger.debug(message) + + +# 便捷的爬取日志记录函数 +def log_crawl_start(source_name: str): + """记录爬取开始""" + _logger_manager.log_crawl_start(source_name) + + +def log_crawl_success(source_name: str, count: int, duration: float): + """记录爬取成功""" + _logger_manager.log_crawl_success(source_name, count, duration) + + +def log_crawl_error(source_name: str, error: str): + """记录爬取错误""" + _logger_manager.log_crawl_error(source_name, error) + + +def log_announcement_filtered(reason: str, count: int): + """记录公告筛选信息""" + _logger_manager.log_announcement_filtered(reason, count) + + +def log_database_operation(operation: str, table: str, count: int = 0): + """记录数据库操作""" + _logger_manager.log_database_operation(operation, table, count) + + +def log_notification_sent(channel: str, recipient_count: int): + """记录通知发送""" + _logger_manager.log_notification_sent(channel, recipient_count) + + +def log_system_metrics(metrics: Dict[str, Any]): + """记录系统指标""" + _logger_manager.log_system_metrics(metrics) + + +def log_performance_warning(operation: str, duration: float, threshold: float): + """记录性能警告""" + _logger_manager.log_performance_warning(operation, duration, threshold) diff --git a/gx_gp_monitor/core/models.py b/gx_gp_monitor/core/models.py new file mode 100644 index 0000000..c8fea67 --- /dev/null +++ b/gx_gp_monitor/core/models.py @@ -0,0 +1,255 @@ +""" +数据模型定义 +定义系统使用的数据结构和模型 +""" + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional, List, Dict, Any +from enum import Enum + + +class AnnouncementType(Enum): + """公告类型枚举""" + PURCHASE = "purchase" # 采购公告 + RESULT = "result" # 结果公告 + CONTRACT = "contract" # 合同公告 + CORRECTION = "correction" # 更正公告 + PRE_ANNOUNCEMENT = "pre_announcement" # 招标文件预公示 + SINGLE_SOURCE = "single_source" # 单一来源公示 + ELECTRONIC_MARKET = "electronic_market" # 电子卖场公示 + ACCEPTANCE = "acceptance" # 履约验收公示 + ENGINEERING = "engineering" # 工程类公告 + FRAMEWORK_AGREEMENT = "framework_agreement" # 框架协议征集公告 + FRAMEWORK_RESULT = "framework_result" # 框架协议入围结果公告 + FRAMEWORK_SUMMARY = "framework_summary" # 框架协议成交结果汇总公告 + INTENTION = "intention" # 采购意向公开 + + +class CrawlStatus(Enum): + """爬取状态枚举""" + PENDING = "pending" # 待爬取 + RUNNING = "running" # 爬取中 + SUCCESS = "success" # 成功 + FAILED = "failed" # 失败 + PARTIAL = "partial" # 部分成功 + + +@dataclass +class AnnouncementSource: + """公告来源""" + code: str # 来源代码，如 "ZcyAnnouncement1" + category_id: int # 分类ID + name: str # 显示名称，如 "采购公告" + type: AnnouncementType # 公告类型 + + +@dataclass +class Announcement: + """公告数据模型""" + id: Optional[int] = None # 数据库ID + title: str = "" # 公告标题 + publish_date: datetime = field(default_factory=datetime.now) # 发布时间 + purchase_name: str = "" # 发布单位 + content_url: str = "" # 内容链接 + source_code: str = "" # 来源代码 + source_name: str = "" # 来源名称 + announcement_type: AnnouncementType = AnnouncementType.PURCHASE # 公告类型 + + # 爬取相关字段 + crawled_at: Optional[datetime] = None # 爬取时间 + created_at: Optional[datetime] = None # 创建时间 + updated_at: Optional[datetime] = None # 更新时间 + + # 去重字段 + content_hash: Optional[str] = None # 内容哈希，用于去重 + + # 筛选相关 + keyword_matched: bool = False # 是否匹配关键词 + date_filtered: bool = True # 是否在日期范围内 + + # 业务字段 + is_new: bool = True # 是否为新公告 + is_today: bool = False # 是否为今日公告 + + def __post_init__(self): + """后初始化处理""" + if isinstance(self.announcement_type, str): + self.announcement_type = AnnouncementType(self.announcement_type) + + if self.publish_date and isinstance(self.publish_date, str): + try: + self.publish_date = datetime.fromisoformat(self.publish_date.replace('Z', '+00:00')) + except ValueError: + # 如果解析失败，使用当前时间 + self.publish_date = datetime.now() + + # 判断是否为今日公告 + today = datetime.now().date() + if self.publish_date: + self.is_today = self.publish_date.date() == today + + @property + def publish_date_str(self) -> str: + """获取发布日期字符串""" + return self.publish_date.strftime("%Y-%m-%d") if self.publish_date else "" + + @property + def crawled_at_str(self) -> str: + """获取爬取时间字符串""" + return self.crawled_at.strftime("%Y-%m-%d %H:%M:%S") if self.crawled_at else "" + + def to_dict(self) -> Dict[str, Any]: + """转换为字典""" + return { + "id": self.id, + "title": self.title, + "publish_date": self.publish_date.isoformat() if self.publish_date else None, + "purchase_name": self.purchase_name, + "content_url": self.content_url, + "source_code": self.source_code, + "source_name": self.source_name, + "announcement_type": self.announcement_type.value, + "crawled_at": self.crawled_at.isoformat() if self.crawled_at else None, + "created_at": self.created_at.isoformat() if self.created_at else None, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + "content_hash": self.content_hash, + "keyword_matched": self.keyword_matched, + "date_filtered": self.date_filtered, + "is_new": self.is_new, + "is_today": self.is_today + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'Announcement': + """从字典创建实例""" + # 处理枚举类型 + if 'announcement_type' in data and isinstance(data['announcement_type'], str): + data['announcement_type'] = AnnouncementType(data['announcement_type']) + + # 处理日期时间 + for date_field in ['publish_date', 'crawled_at', 'created_at', 'updated_at']: + if date_field in data and data[date_field] and isinstance(data[date_field], str): + try: + data[date_field] = datetime.fromisoformat(data[date_field].replace('Z', '+00:00')) + except ValueError: + data[date_field] = None + + return cls(**data) + + def generate_content_hash(self) -> str: + """生成内容哈希用于去重""" + import hashlib + content = f"{self.title}|{self.publish_date_str}|{self.purchase_name}|{self.content_url}|{self.source_code}" + self.content_hash = hashlib.md5(content.encode('utf-8')).hexdigest() + return self.content_hash + + def matches_keywords(self, keywords: List[str]) -> bool: + """检查是否匹配关键词""" + if not keywords: + return True + + search_text = f"{self.title} {self.purchase_name}".lower() + for keyword in keywords: + if keyword.lower() in search_text: + self.keyword_matched = True + return True + + self.keyword_matched = False + return False + + def in_date_range(self, start_date: Optional[str], end_date: Optional[str]) -> bool: + """检查是否在日期范围内""" + if not self.publish_date: + self.date_filtered = False + return False + + publish_date = self.publish_date.date() + + try: + if start_date: + start = datetime.fromisoformat(start_date).date() + if publish_date < start: + self.date_filtered = False + return False + + if end_date: + end = datetime.fromisoformat(end_date).date() + if publish_date > end: + self.date_filtered = False + return False + + self.date_filtered = True + return True + except ValueError: + # 日期格式错误时，默认通过 + self.date_filtered = True + return True + + +@dataclass +class CrawlResult: + """爬取结果""" + source: AnnouncementSource # 公告来源 + status: CrawlStatus # 爬取状态 + total_count: int = 0 # 总公告数 + new_count: int = 0 # 新增公告数 + error_message: Optional[str] = None # 错误信息 + announcements: List[Announcement] = field(default_factory=list) # 公告列表 + crawled_at: datetime = field(default_factory=datetime.now) # 爬取时间 + duration: float = 0.0 # 爬取耗时（秒） + + +@dataclass +class CrawlSession: + """爬取会话""" + session_id: str # 会话ID + start_time: datetime # 开始时间 + end_time: Optional[datetime] = None # 结束时间 + status: CrawlStatus = CrawlStatus.PENDING # 会话状态 + total_sources: int = 0 # 总来源数 + completed_sources: int = 0 # 已完成来源数 + total_announcements: int = 0 # 总公告数 + new_announcements: int = 0 # 新增公告数 + results: List[CrawlResult] = field(default_factory=list) # 各来源结果 + + @property + def duration(self) -> float: + """获取会话持续时间""" + if self.end_time and self.start_time: + return (self.end_time - self.start_time).total_seconds() + elif self.start_time: + return (datetime.now() - self.start_time).total_seconds() + return 0.0 + + @property + def progress(self) -> float: + """获取完成进度（0-1）""" + if self.total_sources == 0: + return 0.0 + return self.completed_sources / self.total_sources + + +@dataclass +class NotificationMessage: + """通知消息""" + title: str # 消息标题 + content: str # 消息内容 + message_type: str = "text" # 消息类型：text, markdown, card + recipients: List[str] = field(default_factory=lambda: ["@all"]) # 接收者列表 + attachments: Optional[Dict[str, Any]] = None # 附件信息 + created_at: datetime = field(default_factory=datetime.now) # 创建时间 + + +@dataclass +class SystemMetrics: + """系统指标""" + timestamp: datetime = field(default_factory=datetime.now) # 时间戳 + total_announcements: int = 0 # 总公告数 + today_announcements: int = 0 # 今日公告数 + new_announcements_today: int = 0 # 今日新增公告数 + crawl_sessions_today: int = 0 # 今日爬取会话数 + last_crawl_duration: float = 0.0 # 最后一次爬取耗时 + database_size: int = 0 # 数据库大小（字节） + memory_usage: float = 0.0 # 内存使用率 + disk_usage: float = 0.0 # 磁盘使用率 diff --git a/gx_gp_monitor/core/reliability.py b/gx_gp_monitor/core/reliability.py new file mode 100644 index 0000000..edf6f47 --- /dev/null +++ b/gx_gp_monitor/core/reliability.py @@ -0,0 +1,489 @@ +""" +高可用性模块 +提供重试机制、超时控制、幂等操作、异常处理和恢复功能 +""" + +import time +import random +import hashlib +from contextlib import contextmanager +from functools import wraps +from typing import Callable, Any, Optional, Type, Union, List +from datetime import datetime, timedelta +import threading +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from .logger import get_logger +from .config_manager import get_config + + +logger = get_logger(__name__) + + +class RetryConfig: + """重试配置""" + + def __init__(self, + max_retries: int = 3, + initial_delay: float = 1.0, + max_delay: float = 60.0, + backoff_factor: float = 2.0, + jitter: bool = True): + """ + 初始化重试配置 + + Args: + max_retries: 最大重试次数 + initial_delay: 初始延迟时间（秒） + max_delay: 最大延迟时间（秒） + backoff_factor: 退避因子 + jitter: 是否添加随机抖动 + """ + self.max_retries = max_retries + self.initial_delay = initial_delay + self.max_delay = max_delay + self.backoff_factor = backoff_factor + self.jitter = jitter + + +class TimeoutConfig: + """超时配置""" + + def __init__(self, + connect_timeout: float = 10.0, + read_timeout: float = 30.0, + total_timeout: Optional[float] = None): + """ + 初始化超时配置 + + Args: + connect_timeout: 连接超时时间（秒） + read_timeout: 读取超时时间（秒） + total_timeout: 总超时时间（秒） + """ + self.connect_timeout = connect_timeout + self.read_timeout = read_timeout + self.total_timeout = total_timeout or (connect_timeout + read_timeout) + + +class CircuitBreakerState: + """熔断器状态""" + CLOSED = "closed" # 关闭状态，正常工作 + OPEN = "open" # 打开状态，快速失败 + HALF_OPEN = "half_open" # 半开状态，测试恢复 + + +class CircuitBreaker: + """熔断器实现""" + + def __init__(self, + failure_threshold: int = 5, + recovery_timeout: int = 60, + expected_exception: Type[Exception] = Exception): + """ + 初始化熔断器 + + Args: + failure_threshold: 失败阈值 + recovery_timeout: 恢复超时时间（秒） + expected_exception: 期望的异常类型 + """ + self.failure_threshold = failure_threshold + self.recovery_timeout = recovery_timeout + self.expected_exception = expected_exception + + self.state = CircuitBreakerState.CLOSED + self.failure_count = 0 + self.last_failure_time = None + self._lock = threading.Lock() + + def __call__(self, func: Callable) -> Callable: + """装饰器实现""" + @wraps(func) + def wrapper(*args, **kwargs): + return self._execute_with_circuit_breaker(func, *args, **kwargs) + return wrapper + + def _execute_with_circuit_breaker(self, func: Callable, *args, **kwargs) -> Any: + """使用熔断器执行函数""" + if self.state == CircuitBreakerState.OPEN: + if self._should_attempt_reset(): + self.state = CircuitBreakerState.HALF_OPEN + logger.info("熔断器半开，尝试恢复") + else: + raise CircuitBreakerOpenException("熔断器已打开") + + try: + result = func(*args, **kwargs) + self._on_success() + return result + except self.expected_exception as e: + self._on_failure() + raise + + def _should_attempt_reset(self) -> bool: + """检查是否应该尝试重置""" + if self.last_failure_time is None: + return True + return (datetime.now() - self.last_failure_time).total_seconds() >= self.recovery_timeout + + def _on_success(self): + """成功时的处理""" + with self._lock: + if self.state == CircuitBreakerState.HALF_OPEN: + self.state = CircuitBreakerState.CLOSED + self.failure_count = 0 + logger.info("熔断器关闭，服务恢复正常") + + def _on_failure(self): + """失败时的处理""" + with self._lock: + self.failure_count += 1 + self.last_failure_time = datetime.now() + + if self.failure_count >= self.failure_threshold: + self.state = CircuitBreakerState.OPEN + logger.warning(f"熔断器打开，失败次数达到阈值: {self.failure_count}") + + +class CircuitBreakerOpenException(Exception): + """熔断器打开异常""" + pass + + +class IdempotencyKey: + """幂等性键生成器""" + + @staticmethod + def generate(*args, **kwargs) -> str: + """ + 生成幂等性键 + + Args: + *args: 位置参数 + **kwargs: 关键字参数 + + Returns: + str: 幂等性键 + """ + # 将参数转换为字符串并排序 + key_parts = [] + + # 处理位置参数 + for i, arg in enumerate(args): + key_parts.append(f"arg_{i}:{str(arg)}") + + # 处理关键字参数（排序以保证一致性） + for key in sorted(kwargs.keys()): + key_parts.append(f"{key}:{str(kwargs[key])}") + + # 生成哈希 + key_string = "|".join(key_parts) + return hashlib.md5(key_string.encode('utf-8')).hexdigest() + + +class IdempotencyManager: + """幂等性管理器""" + + def __init__(self): + self._executed_keys = set() + self._lock = threading.Lock() + + def is_executed(self, key: str) -> bool: + """ + 检查操作是否已执行 + + Args: + key: 幂等性键 + + Returns: + bool: 是否已执行 + """ + with self._lock: + return key in self._executed_keys + + def mark_executed(self, key: str): + """ + 标记操作已执行 + + Args: + key: 幂等性键 + """ + with self._lock: + self._executed_keys.add(key) + + def clear_expired_keys(self, max_age_seconds: int = 3600): + """ + 清理过期的键（简化实现，实际应该使用时间戳） + + Args: + max_age_seconds: 最大年龄（秒） + """ + # 这里简化实现，实际项目中应该记录时间戳 + pass + + +def retry_on_exception(retry_config: Optional[RetryConfig] = None, + exceptions: tuple = (Exception,), + logger: Optional[Any] = None) -> Callable: + """ + 重试装饰器 + + Args: + retry_config: 重试配置 + exceptions: 需要重试的异常类型 + logger: 日志器 + + Returns: + Callable: 装饰器函数 + """ + if retry_config is None: + retry_config = RetryConfig() + + if logger is None: + logger = get_logger() + + def decorator(func: Callable) -> Callable: + @wraps(func) + def wrapper(*args, **kwargs): + last_exception = None + + for attempt in range(retry_config.max_retries + 1): + try: + return func(*args, **kwargs) + except exceptions as e: + last_exception = e + + if attempt < retry_config.max_retries: + # 计算延迟时间 + delay = min( + retry_config.initial_delay * (retry_config.backoff_factor ** attempt), + retry_config.max_delay + ) + + # 添加随机抖动 + if retry_config.jitter: + delay = delay * (0.5 + random.random() * 0.5) + + logger.warning( + f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{retry_config.max_retries + 1}): {str(e)}，" + f"等待 {delay:.2f} 秒后重试" + ) + time.sleep(delay) + else: + logger.error( + f"函数 {func.__name__} 在 {retry_config.max_retries + 1} 次尝试后仍然失败: {str(e)}" + ) + + raise last_exception + + return wrapper + return decorator + + +def timeout_wrapper(timeout_config: Optional[TimeoutConfig] = None) -> Callable: + """ + 超时装饰器 + + Args: + timeout_config: 超时配置 + + Returns: + Callable: 装饰器函数 + """ + if timeout_config is None: + timeout_config = TimeoutConfig() + + def decorator(func: Callable) -> Callable: + @wraps(func) + def wrapper(*args, **kwargs): + import signal + + def timeout_handler(signum, frame): + raise TimeoutError(f"函数 {func.__name__} 执行超时") + + # 设置信号处理器 + old_handler = signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(int(timeout_config.total_timeout)) + + try: + result = func(*args, **kwargs) + signal.alarm(0) # 取消闹钟 + return result + finally: + signal.signal(signal.SIGALRM, old_handler) + + return wrapper + return decorator + + +@contextmanager +def session_with_retry(timeout_config: Optional[TimeoutConfig] = None, + retry_config: Optional[RetryConfig] = None): + """ + 创建带有重试机制的HTTP会话 + + Args: + timeout_config: 超时配置 + retry_config: 重试配置 + + Yields: + requests.Session: 配置好的会话对象 + """ + if timeout_config is None: + timeout_config = TimeoutConfig() + + if retry_config is None: + retry_config = RetryConfig() + + session = requests.Session() + + # 配置重试策略 + retry_strategy = Retry( + total=retry_config.max_retries, + backoff_factor=retry_config.backoff_factor, + status_forcelist=[429, 500, 502, 503, 504], + ) + + adapter = HTTPAdapter(max_retries=retry_strategy) + session.mount("http://", adapter) + session.mount("https://", adapter) + + # 设置默认超时 + session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout) + + try: + yield session + finally: + session.close() + + +def safe_execute(func: Callable, + fallback: Optional[Callable] = None, + exceptions: tuple = (Exception,), + logger: Optional[Any] = None) -> Any: + """ + 安全执行函数，提供降级处理 + + Args: + func: 要执行的函数 + fallback: 降级函数 + exceptions: 需要捕获的异常类型 + logger: 日志器 + + Returns: + Any: 函数执行结果或降级结果 + """ + if logger is None: + logger = get_logger() + + try: + return func() + except exceptions as e: + logger.error(f"函数执行失败: {str(e)}") + if fallback: + try: + logger.info("执行降级函数") + return fallback() + except Exception as fallback_e: + logger.error(f"降级函数也执行失败: {str(fallback_e)}") + return None + + +class HealthChecker: + """健康检查器""" + + def __init__(self, check_interval: int = 300): + """ + 初始化健康检查器 + + Args: + check_interval: 检查间隔（秒） + """ + self.check_interval = check_interval + self.last_check = None + self.is_healthy = True + self.consecutive_failures = 0 + self.max_consecutive_failures = 3 + + def check_health(self) -> bool: + """ + 执行健康检查 + + Returns: + bool: 健康状态 + """ + current_time = datetime.now() + + # 检查是否需要执行检查 + if (self.last_check and + (current_time - self.last_check).total_seconds() < self.check_interval): + return self.is_healthy + + self.last_check = current_time + + try: + # 执行健康检查逻辑 + self._perform_health_check() + self.is_healthy = True + self.consecutive_failures = 0 + logger.info("健康检查通过") + return True + except Exception as e: + self.consecutive_failures += 1 + logger.warning(f"健康检查失败 ({self.consecutive_failures}/{self.max_consecutive_failures}): {str(e)}") + + if self.consecutive_failures >= self.max_consecutive_failures: + self.is_healthy = False + logger.error("连续健康检查失败，系统标记为不健康") + + return False + + def _perform_health_check(self): + """执行具体的健康检查逻辑""" + # 这里可以添加数据库连接检查、外部服务检查等 + config = get_config() + + # 检查数据库连接（如果启用） + if config.database.enabled: + # 这里应该检查数据库连接 + pass + + # 检查网络连接 + try: + requests.get("https://www.baidu.com", timeout=5) + except: + raise Exception("网络连接检查失败") + + +# 全局实例 +_circuit_breaker = CircuitBreaker() +_idempotency_manager = IdempotencyManager() +_health_checker = HealthChecker() + + +def get_circuit_breaker() -> CircuitBreaker: + """获取全局熔断器实例""" + return _circuit_breaker + + +def get_idempotency_manager() -> IdempotencyManager: + """获取全局幂等性管理器实例""" + return _idempotency_manager + + +def get_health_checker() -> HealthChecker: + """获取全局健康检查器实例""" + return _health_checker + + +def check_system_health() -> bool: + """ + 检查系统健康状态 + + Returns: + bool: 系统是否健康 + """ + return _health_checker.check_health() diff --git a/gx_gp_monitor/crawler/__init__.py b/gx_gp_monitor/crawler/__init__.py new file mode 100644 index 0000000..1962ccd --- /dev/null +++ b/gx_gp_monitor/crawler/__init__.py @@ -0,0 +1 @@ +"""爬虫模块""" diff --git a/gx_gp_monitor/crawler/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/crawler/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb46a2365899c17fdc2e440256d32a1d0bddc659 GIT binary patch literal 195 zcmey&%ge<81U)KonZiK&F^B^Lj8MjB4j^MHLoh=TLpq}-Qz;LS`@Cb#iyFVasgN-WPwEz*yV&jbp@$LkeT-r}&y%}*)KNwq6t0~!i) UPcg_XAD9^#8E-KN6|n$00PChVO8@`> literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/crawler/__pycache__/parsers.cpython-313.pyc b/gx_gp_monitor/crawler/__pycache__/parsers.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de4f3a5fd2d8f92864994de0761fdb3289884256 GIT binary patch literal 11226 zcmbtaYjjiBl|Itd^IDc4@jU&p_J54G(whz5?gZbl?bq= zU1_FmoK92BBoxR>YI`M_D4n!89WtW?+8B~B%>0tZp0VZ*O<8MVVgH$B#&nt?e`fFd zkgjAo^l`m>?m6e~v(LHbp7ZT*@1y1HY?{FJf9sy|cGeQ|Us#YXm6%!PAakDB2t#ZN zo3dNMDBwQXVPE;Xa>(l8neK38>XyR?kfLe>zQ+NSB&b?F&>7tPRJSxgp4 zsBPMALsvGFZ6W>H)ucbCSt0bpQP2a2(>+<+I8tSqVb@_aq0qU|7btaFp ziu4<`#AzagAIHj`70I3}NzbE$WH(hw1{GFgup4@#keH4G?qxlbCTYTY#g)2Y5`>S(zU;SR&kA+=R=?I}fIIXEdb3kx@>YyR)1Tj@vKdpPOc|tP;|KYi6!`*ME>q~q zv*xGHkzTE+ zKz6e!q#a`+4Wt(KnVt0NWddT>rnFHjv~PH*cGW7{U@uGBjAWZKbquSt(QZk+pO3Or z8Aqv;$*AnA%rXs0J!-o;F;{C--X<%<9O&(eUc<_IBeBujsML38AU=&!dLu{@69WkfvXDU9!K!(8uBJgk%9$$-Z``H!${< zNSf>#Ndt^OM^>+v)#q+Ylq@ikT2e*&3M37Hj}7p4dRd>DR#;XatU}?k>t(tPb}f8+ zl~iLPm4LxMeVhL9Yk(kpZcriNYFfXW)A}uwdPv@Fd71l7U2UvV>a})7pVs`~FF_wI z;B`7K(`}50ZKW-6rR?ukOS&YA2z$QOA|k}=Z^b_x07V1N+Bg7~51{99Op zxsbQSe>oX{?Zx;{rUdj0mO`aCKpqqqela!wtMgJnK{P0vEJ9_gr6-t;`bNj*e>g5Q zvT%w9QSs}O!pjN5!HR_9#OQ@CCEZpdM|)YXj}17%{K0t9U}^-T#%Z5*GCr@*!%<-A zIF;8o=;u^E=dhc>4ayZbMi9~+(A+cRb}(+2pBdmZ&XEzfZ-CS9Kjm`sHkYH=0K-v3 zZXc(1GmM|H(hN=mgR{!yV`9bxcN?b_)UaH(Q)ZJ{oKarAL0-)%+$>HGE*jzD+8a*z zhusY)xBG|O4I`&V82{65SD@j*vCW>5%|3s?J9xUmbIRcvaSZ!?-hiKJ5Ed2Fz`qQ` z){mU#@`V?3j(Fj%1ari#l`km4mesEm!xv%y8tlz;pMGMzWoSMp?DQp_;h?z<* zw4Q6dI5=^9#$+Ac7b_~gaN^vFizg?TnW7D&2jZquRG)P&>!N33@a<GC|+ZEn=%{EH22G z+&63JkC`kHgXN3+S;SnmOq9@3LFt8_b3Ku@kA6Z;w|%S$pPD`s={*+t`V*0wZ_XC< z{idM&%Oy1ketF+Wa!a6%z6j^^(2Iv6Wi8iC;jdpU3hN>zZL@~GFugBzqO`u~0c(ek z@_rlXAmsY?wko)NY}R6Gso$?seqwBf{HH2<|8DiC#@hWY>Q5UrSkt1$n%xxSZxXs= z2X&LK?r5ZLZc<>rQ4J{rMxCVa5JUo=hlF_^!ZU;f`+#TSg=}83>1dyi!`kl?Z5mjy zgnjHqI+%7fi{-C@4j(dEY?Mu-B4GEl za=Y!Jz)s`dkn)?-uhVp6Kv>wXF-&6Puc*w#9y>3kFwN84LyxVeD=}*RaQE*f zI`_3&j#~B1R(OhS`9Y$)=jO5{O%9|PT$ao43xG@R2r@%lmN1tNmwy1Bhy`9OPtFM( z0r1XwR>5h!ti$JiHfVv4M0&O2TPBE50aFQL3^+D~Qv+e~jxf0O7~EA1GB_@Kkns;= z*Z|1V@CZkFAyz;??i>ob_am#}^cWZ5vjRWLcN`CThX!E4vX^kdYkK_5X@_@!Q{k&H z=s+@6*ntX%&J}vxJ~snmiH0z>Cm7rT3@&wRHg9&4?1|I!pWFc+vM{H4(wrs+%k03` zo3ImjD10n_iupz@SWk3H21(_faN~?+4=9c8`C4>O-)rp=%hs9l#<}vA zXnD(Qd27V!8a*51~n91GBjYq5Jz9 zQnq=n^wDVPqp{qgx!me#ZuLaR+uf7hQ>DP4%65---`A1y^|7L|ScNsVW@D_pE>>PW z-hHWCx>c`#d*kHBSlL=och5wZ8%GZrB|F;sqWzgnL~7Ty z_G!h{j&Mt)WZ$e||7}w-PovBa*dF+f&+q7{B_B3Ha(zqn{u**4zpWMuAMa?}2)9pb zayzQ1Pq)=}lv6j0RZw%YTn#lht0>IZ(g#YYoAp%(3aEc7P(z;Chl{BdlWs^-n3M`C zkP$7T$$=QC!j2q7X;-4vsD-Fq@NeJ&9slX~=C6JzfXw_4zCZtq4;Ekkb^OO~$uU~C z=K!cje{lD`tDXCJhf)S9tro#W`~2z~cPGBfKTYlkcql?wnyU#w0V?JFo~b_(2#m0; z4Gm`oU7mW+sWXH19{>;kr3Dan6KF`X6u_O0&7vEz|%xFR_N5#7KtJS8qUUqqne zXFUS7=OM!N2gc4o+5>#lPIf`g$(Mk6Wnixm8&gQkH^IBUN}iHMsT8u&D83?+C+JgT z>|Z6*r`l9D8X2cfaQf7! zAn+QkPt~S^@4?@bJj$vI`amJQ__H4^T>fraMBwh#mtil-w~gfb@{!n;k($()%W=Pd z2!b-alK92T^J9OD5svetbO%!ieI`Bpqdd6~h9Jc5g+`Jg?y>3^j8HM1gbx^8QoNmGaqYpj5jPH3cG5ZIMUIj1=pCRl=z>VHo2$=Y$-Y8< z5>&JQ0txJ_qB4ESZBxa?;dhVDnHr;}##nJVSghQViR_u2%}~hT-eYZHb(-IfN(SE? zm@{pRnl|$7QvG;*KJezgIg>SNvf}eJzaF1^-VDr{HbqUF`1ZwLe4b-QFS$gDJXGws zMupq1YNk#_${w3FwB0tDd2lazz#;*BzHsjra!~`xcrhgZSO>}VN=$EP`E(22zLvVt zTGei$KDMYKpMo9e4f5FW3G6?_j#Ydwg=_~Zz(WD)^vkA{$pGeOl%Z@YX=kk>A&Z=^ zBB8bE9->44&|c`|3E3*W7?S5@t4J7=61JRrE$?+DPAUK!z$Noh4~ZS;_9f0Dk`aVd zY_O?p8k-K%EL%4GK~H>4H_u14Q4rJR?=rl|4|?It(DAcl5NZX(xj6a$=O0bRzkB&l zh6y~7>w}HA$pVLR5SC3*ngj{(V*CN;5Q9^|X+|K%+T&lub;9YeVDp}FFJf?kQ)7{Z zxZ?0g;Pg>W3skwseJaqwWkK!;<38#22U)9@c^rkcDD3e0IX$0YK?~Cd514*lGqeDA z;Z%S#U~WWg%)`~n8A!sFT5ud{QBfK^snX#I890l~CJ9W%&?Dn?1%Nqwygnx$QaboZ z>xK-Vw2JIMquM)=ARra%jjw(ZoXums0(eKDcKw=s~m5+)G~1}J-J<5D%G4ARsdYZ0e&2p*1uqa6_J9SA~<+QAtdj&B8> zL+Cd`DaYX$^fGK<$m?_a{7|8FI0pPKlmjOZ+}L4v;DmpG!B57!h76h&29b{U2^qvY z!GnWX2w^ao#ydyMW0-4)gj1nuWNcV=VY$!&mIrmiSthud$AbYk>u~VHW{#i~?_n7+ zhY4Cwwi4F<56FM0D(|TCmXur7MWNadJJlXK^-lc~fnuz>X^DdSoptp~YP@gT5jI4( z?O(>iGDS4`e5<8F1)ptKsLc~cLp32bw0`obC4%KxP4g0knLBk2OKLvT^hmfr+SIko zmqac2V5v*Vt1+nEiV(F#@E)!a?zgrH_lWKBW&VNK4rP|g`DSa?B~wjdDr^hu-rKW8 zpg0Xo6mFs@-fQF^pMbrZp2k?P9LSthEhlTqD7 zKs?o)0oB!`>GCLryUcXa!EeE&iy^%9@529N#lil>Z6gEjC_2w$a(@QRa`NKTxSmWU5)@zx2W zM?8vIMZ?51bAA*V3{!z{y5+(}o*>PfHPa3d55iJK??G?u7$j(va?)6`6sM@%JDc0i zv)P)tqPl2NT}VIGJX5rD^uTAP(pYKTTBdm!)OTh|JI1t9v#OSGaabQov^J%3 zUH(L9&oy>>>(!It6Oobwvxb8*u8VW?{D!th^7A$d((8@%-a6{~BUO8=sTZ$;TuuH-J@}@ry;6T~~k)Dj6b^=m%Cm|~js1$;h1oP0Pbir?MKpS3ikaK1M zgd+U%mpuWOwX11Kc-cPiLnNqlLVq^30Ccs)qe=7R!5{TeHm$6m#8?o(c+Ra8IX8co z0hd1LgrSMRC7<%Iz$>|!c=(WtSA3XCkP-h8=J~?5t>j`WB-c&lm~N%_nyBl$viBOP8#)E# zZy42(im;RQ$HC5NoV-kk6TkO{EX-yjW4Io%@yq2I$i2O_x5SwAE&=L*F=C8ba;(?ntT zE!i|fHvNWFz>yN>8g5Jy23JyBK9Z&m9ce6FZ_zykV^S99C)$o0IUwhh#Es{+#h E2Zr4ay8r+H literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/crawler/__pycache__/spider.cpython-313.pyc b/gx_gp_monitor/crawler/__pycache__/spider.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2b29ff717019bcd18c9866af9ddc7b16dd241c18 GIT binary patch literal 19508 zcmcJ1dvsIRmG{;AZOfKy%WwIC@dMjn8w2JM%o`gUqKgt*K~!POMnRUGd!;0hWQJy% z#xzMWO)@-^6qBUIw0SsRG7S@w#%VJ%OlM||ZtPU*t+UEnwGFH_YgH85ru3iL`&`{C zS%@_G`h5p{?s>n@KEJ)sKDuEt=_xS1HTU_RSVK|2MUQB-3CE3?jG~^WXo{g|87=RY zF*5R%Gjj4(Fbeo8x|LljM%ATe)Lj}z)1_s!T{=c*$2`hzeV3lm6Su0{&}C$d#4YbO zF(!ytcV~5FGud{kjG{HPw%gp5!{l`3GPzwA#?obFtX+9b9^}x_y6*h00;Yht_1(6v zLZ+~*h$-qSW{SH?m=Y3h=q~LlW6HY9ner|>WACb9D!M9}%C0J=3gV5lsk^#s5wobP zhN z`&Yg@G560OUj4y`^Ur-a_uh-wo_zA^JFj2);P~7Vugx9*&Xv>Onm_g4)qnT^GS40R z$GPvE0k{4d6i4;esd(iM9~N;g&pQNxd~T|b4u+XNFKSOpvKl^SO*S#2j)518(obeO}U0-r&HnjieSnmxY|refLAY&;d6| z&6{>Z*Ms3uB8idp40s{Gw=W#>@|L~>-oArw_7EHL4!I9_J^s*vYq&`36blDnmxqDj z;;pnd!1_YI2fg8woF_3t^N>=pv?c&|Fu_>$mRNx{Dt6^?R9XwNPNV7_5xDO4x&a2|7gT zR)Vd7zZ-TkIW8$qY?_nuOJOeT3o$9Nn41_U?!j5&%x9M=ITA z662uE3~6bZNH-yuMojxxbIDV3Nv)!>W}R3DtpJ}?Bk_sLml~w%l)Z<|&BH+^!mh-7(?IE!|EL-T>) zkhl54_MqR}JbY-F3EuDR3pH=u(>OTX7zl=Z1BaRiA9fE8yN7}SUns~lCz{yIk`>l8 z3}n{r_CcK6&0CU7uSpDSfO^i8RS)3R0afXpQ=xvx-~@%Hw8h1^+xQ*QI$lsotK+_Zd^G9Y(Z#TZAv=t_Po zsT3k41*hb(Nr95Xfuu3b&oen^dlO=+5n=kype=J5v=d9muZKzhcyR6q2Z3^=EGWau;8l8 zVd}AzLbR&UssW4FB$~?*!5Vf~*_SUFMoU+6r7N$i6lGZ>y*Kq#$>Lav{Y>|%?r2FfSJE6USe5g)47~mUnXHo!51$ zlB|&(H?ydc>S%GpC&dj@#f>AIp5Ad&O%;?!^A~gZi=+9iTz+e;qAFHe9xEx0Yp8;@ zIHfGe8rd4xQ>L6_JC5#%8p}9i*~vrWi>Hk(F{3#p?C|)CX=5u8^F^)E>J41=hRNRO zCO5aq9jW@p$RZIQkp69uq9iWrm)%If4{q_}fWynXG&>%xZWt&@}G%B%YPLBPw65HFm%!b&ia zV7IswY$;1&LCn`N-y*OQYvI_M1w#ho5i2eq+4RhgxQa5`$cXT2Sc+jLuwZ0x9g>mx zAvm4~hVl+Q#}Q03)l>y#r(9A@rZcS)%lne2VkQtLB4t)yPF0J{qAZM*{_1z%1$OV+ z>0|RRzK4ujBjMEQ)C`Ua^wksS4-NsEIuP{rc~c03Aq_=*$jj<1UtD?X3QHb4l})39FD9Ov{hnFI$&5g8`Wn{?YO#Ih-|$FrMeBGdO$kl`Es)hTXv{k+AbB zLf-3CDHEpw9QrVvG0feNm%;rBdx_9o6~-K5aV3fr)toX-B$hAEJclIm0OVx1fCak= z$jBo{ABkpHa@m#9?0WbQb2)3yKVkZw>12N@jaI6UMy>2xn#tkzv)IA zF^d>e4${ehzEeMF6utv8Nt%}(J`1Vz^GDBJJ^S?h^T+1Db>`}u&&b?Ug#R{!; z?%vY4@|LX`Bw1dGiN9;u1G_e~y?^bktQv6>Z-h;}_SP!*;l4v|BBJyAAdNd5_Vuq_ zvCP}Dvemn^v8A=YrE%G^fu)Tr+uA&h-u>+>`UgBKJp0?*&Z`dVlgHO#D9Z-$S_3x%=WK+XXo6jkdA zWBEk_IG)O{8`&BwER7bna)qr^h08~}0P2?4#q8@5fj6FNL~K5EYUrdpvSicbzR8^t z+7sCyhY< zNa5;fBLdr~vE&nD$(W2YR(xtKnX%aZ64yYETP&`V$2Zn*GE+ZqtOWaGvwm}t;^U(B z&3TG{$y0%wSF?g-9**NduR9RzOFc35kRSaw{s%zNmpF0ZjtR~n3GzbDgvoTqW&*xz zWw`Co^g}5E)9|IslTKNo1hg}hha!M4+d}MrngPKAGA(Jwg~9=1T+#`g=`2nPTxfYC z$ZEw?hE_Cb2wsuS53xKrTa~Un(Fe6>5aWoYWNzE?#EeQK3!7{ZvC$Wu*oci(kVZPU z#TF3eFGDDxB6a50L5wDI%LsY;sM|o$ zm9bSNjuId%Ic@5A4v>ovgqGD}T1SR+oxuAqyIRBn77K*$S6FLEebbC691CbuRuvNP zGy|>Fru0tXK&eTZftD;y3j(iGnZS)Lx5qh9c#~%6EWnLAf`U+HlxApC0yjz`LZ{BL zGA$8!>1=`D>sN`IW~!|D>vSWH zo3s|89eF9c-YCwl&D+*^(}*p`?||&Dw5lt;PKx@!YE}MMY}F32UZ`!?R~v(r z9luj7CHbWb?$i=-tl;#>rEPag)h5q0iV{XI48I0-=>;BKxJjQIIdS#u__eeD1Sov| znKx3Ax^RX9(tZN3+~;5V!Tfup2?$RP)ebvkLvX$jdXo!wmw4gs!b^A;V6-K6xQO4k zZ;4%sfxvzHI_$M`&ptEv-g83zOYGO4J9YKVH)lUQJO9l$>-0S3y9$)PPQz;sdJiG$ z0+Dfezu(6m*aX*X8@z~=vrS$}DP_O%+G}AvmS9*2JT7 zT*wLc_gi|yc;VN%Yx~s?pPPH(`$B~s_Ank>cSPc0WR8W{t3P>S{=`op0|p5K6k)_5 z2oJE-xfO;~MC>Vrh(UF=0*1#ODoD`vMDf6lx)HqA8|W8YIs&6mAkQm_I78us%S0tg zy_rWa?}K3R262$xzWy|}fC*r#Af{3VJdg{;IiPkC@C^i+q2w*4Ag0d|FD;4J4MKZf z#rnP8VP+VMLE@OkO)w8&7;+K30)#*eve>*qU?m8fz;t4WUQ&<%S^sX~YJ>0&ynYu- zlZe6s6T-5>VAYufj^Z&4_&c=lI1?C*htTCm3)w6l*d4g~U>-)d5=PI%>x8S;A$E{C z452JaJw&F0d#}YfZoVo-c6FyjKg5#%2uC^a7q-nZUF$*yBd(=N z%3{^E=eo~!N2^zG)hh&1dbE1mYk|;y^&q_M0dHkU9PBQFK5{sgKGdaRaBP3`dluny#wQOPjvIW+~#{DoAyNZ3`7Qp zqJs=K$V3L%NO9;(a6!!z>URxq8O9&GuxYYjQg_J`DeRdx_Co#Uf|EHV`^Dm^-SvsikuAPUydF`T#?FFE#l0J z{$H?F1?49mJO0?Se?O{3iSo+hD^G483kyRF0{$`U$&QG1(JWs3-1Uk5u1V#Tee>vc zh>29SO?+*lJ5sSeQoKH5U4OZtI8wgyf-SnLhg;PXS=k#Y-5Dv^88Pn!{tFnngtU{( zuD?OaVeO>zTu}STb+1*8dwyIqZi^JQL@X_z8e9G{lg!)5WjB7syoB$&y)KCnT>ehq zg^ITaCiX-M*G(JmxPo5-UI9Z@r~783hL*JH#frP-}6e)ZZU)X3q@H^tKt_G z)!hcgf2d^O{trV1+Doc;XVDa*`YsT)pVU*i0N-k`oIklH8k9mV&f1 z$W5J~ZUvXzN>FK11XRsbK)uJ&(LqsLWgg&40Au4xnP&FniC8rZQz*af6$U`L&PfF`E@Mun@Uy>c~`IbLhZ0OF9of@%C{lEpKv?&9x z^;g=YPL*NrJmIPoYjNP&nPvcsYsA(AU!kKluvCC6@k@MgN`A44DP>erdmaE~0E7}? z0>CebiT>t(XczRO)v0!%FfPp`ISD)PGPDj`FSft~>kq>83&La+wbKN=wE{V%fJ#_B z9%%noSr3f$o_@cy9$dgP2*#-v#|TD7Qpte(fefe=#&8xR`Zr`za6Z0~9ObM(C+AoMamA(ML z@=}Z`fSMW5-NBLb1<0idsQ3*8c>`%VNbgatFjqj@pm_*7L83?V!Nsah36vKAF z`H1kGGUjmxkhzznYB?_FY|BTs&uR-!sxf$4TRodsdSdVQ_C9;xh<+xgBxWg`v6MwE zHJqg;R#0(Wqs-NhsN=?!x@9qob!1b_UKuf#Ue3**&aJxK&>U^p#x-oalt0yQ4~Y0j zP4kwL8Eesr-s8Pv1LqE&Jvg>+qIx1UF%VhPbIEtf6{-5#wDs#V)y>iBcCNa8bjx!+ zv4*y3Yi-P0IA2hWSpgQ0wVzvkcJ)}}cxYnNMEix3h<(enbt@zRWuG!oG}`(}>DEi8 zsnUB!^|3|G6B=&O%2D;Sxe}r)0HbdH#J)LZtr+jPp^#Oq7&ZM76g3xREIP7kXT-j1 z+IlyXec4ReXz# zs$LFiFf-+qfQ6^Zmyc|jHkMq@&O7$l(Z^l|WvLd<-ZGWlGGlKL6sJbj&zWXz_A|!Q z#uu{2i#S_rv|s~QuwklT)5x}1S{W8s+zF}UTMd+*=LQ#C%2$l)@zpWyMLS? zH7<=9mqLR_On-^zK(V)2#3~=vZmQi@PyMT9LmqrC)$6;O6_?sNmv+@Feo?Oi4}-T> zbuvNaO(U#IwhtDbWH12&iW?!YpQmnD=N1q6ez;~#(eJ@?KFxrVH5~Y8>{K&o=s0D4 zKxzOcN+4E{bdh1v!p?tp^mEg@T-W&k=R z+6R{pa))$8C6hT9DkMmUU^0bEg2|aw35cL~(SmV;QA;-fWEBnx*d$}2SL)PW^+0cW zKz3n(@l0OU)A}M{E-carN`2C#q%qtfjW%U8B?SVf7QswF_g7}Zm?5)dN~s5(^udL3{bK-?)XCMS%iN&?2r zA3ZYv;?chfV_3i{Pd_>L#%X-;L0zWX-lRtQ;o!?-2q-ai5=#X^WZj2p}juq{=R97bv(^vNEEy#(S@0pJm^ z1chF?c<#!@*Jm$0HGAQR1o=qeLP9Ywf^R#>MCXnjzw+Ff`4@k5?d5OHzj#KJih_;3*6|ffQ*(o8Q75;8~2dpaCg{W%>R0hC&FbAgcHj}cZF>jhd>VOx$J(WNq zB$}8R7K^+3LF##xo$6UFP=P^egobv!)X5MTc^37`bQP~vQ8TH_D)dHn3UCMm;VAet<4XV(C(L@EycOmkt` zf3N)amDBg`kKTJf2mib8pQ>zswQ9_BE_gN=t!$4}w*O4?XBLPG^>{_6LUyhMtOX;B zCk`3eesH9WEU@E$>&VhnV$9&+LdrQP!!ZNxKx7{grN0bcf`NsB`v|BoNlQsjHErW zX5V;veqes<+n}cw}aA*HtU$-s9|*I)s5qtS9Xl8i!AGmST{^-H(t)Q zj%a?HUplfSF3&YJ5rv(4uCRW*eX6i!R2?%HppwhU2dB+708s%huRXr@RS>l!nx3+> zi`e_#@q6BMop+7*L{{Awsd7(Szws+;$#oTFU2)w+6_njvLX}r?6$d}kOjR75Dmw`3 z>@&7vuGn$*p{ZiWl+7_>nkgtcx#HPJ$A-9q))8Z(!Lh8GSK09uuRJ;yh*PrV`LX<} zQTJHy#F~jF@OR1}pe~YEKVBbK$xZDy6-u2xZUo8qNY>{!8!2n)4N7NfngK-}<8kB3 z$C7Q_J?-2R`P$c~oUX|3y^+#=)8_kT?G3Twikae?XmKM~+&ELYXw36U%}n{?GmoBr z^o75J_W%{Sqb4Xl$3{lvv8cIm%G~&uxCN@X#Ud*BsJgRi(-Qf|OY~b-s6KA*T)w46 z^{*`|@GKx1xbl%~+*kv1`8)tjP$JH(n4b!7Ap;sm28ba-P$1~wB+|;NvQ)w>2l=NA zGol<6zb*-&LWlxGL^4e)0SYLAlp~x@GpV4mXN5qNoH7E6knblZaXYPYNfJv(ju@Bx z0&qlqT(!$0hTRDOf!suckfSFEfP=VF0Ed>m_nf`>T>%u#eP`_I2Op#!!OSR>Ply2A z<-xOMI4YmP5Gk1X1ia*GQphY2vqW(M@j5I=T|m7clps_>1?rPSEQ2V3!EQ2_&>~bu z5P%|n2 zxOMV@pSMreM#{UV%{x9bgHjvpO;Bn>{^9lJ&Pw@5mHG|Es*jdbZ^&2uEMEog1=|Dh z1lb#_V+;;Q zc-ar2@#-6Ag$w<};jCmDpoGNDs~~#}TksH;yiusr z3ODGBA&IbUm!OMOT8FNBv=)H{0tEz9%p|(Y@O3ZV-NWl%y&E0@_lN!HHoDypggyR5 zOt#xS;A7a3-xu%(f)K57yZeKEZa1?OOWKZB4_X9p>_!)^Cq{{u3oRtKyb|@d1=RvV zN4|?Oxby`vCIKplZWz2*5MKM?_%r{F*1w^}qxBhBe4d+(m#+^bv?0c~AAaC~b-T|~ zWH1DBQh|xls)avx0}SA|sb47<{a%@+D*e5^R8{i(bxM^zZd9u_$i~>46n^7ook~Ae zJKi>-n4l-L=i&am<0b`R@%&|~!tvb`RTJz)?fLuS6nJ9|O>qT&*L4ctEnPR!A3q?Y znwMTz+$~dqc&Rf^p(9q?5GNtAwpABuC)+M5F432?AFbm$9C#k@`v$k$6Y219ZTtT~ zlEn|msike#6%LsyKT@?kPN5@KzIa@5>ajQpjji1H2jaL6N!7Mfh6x#GYnf8DUYED4 z3dU`)z*lVNTDrKp?l=V@k$YW{y@TAn2X3P8hJsSJ#W7`kSe66n+agOha19%8Qs9Z# zXjJ->ZDWctdQ5w2-AxL-f5=s+N`>JiBfAquwGaLzBkdQQzD&X>ASANzYKYX&jl3{- z@jZc(z|Vrr{`9FdRyMf_h@>6}P9hnsUQ+u!6qB55c$foESwmj8@M@j83bErb;Vhof z03EX^WByZXBFX}ih1Vvuj#zR^Es&_;LM5AC3)wSuA|M9|m8AOXX#?sh|H7v`@Bqg` z-J@jX&A_PuT!GOEZpx%&?3Cfy5FXPCr7CyHdw>~1E-0W1D0jjyXaoR4WRk}vfds@$ zFb4`t7!fSlL9WNj?eG^+4p0@?@-Iv+shG*bc%18JfBND4Gw;ql`2k#X{O7dh@*?F)7#gcdjnpKfFNQ1;;X-Ve`NmjzC^JRSy)DA6c34Zpw-tdL)MS->JOzk@RW4=zRItOysARs59u z$dDenCHWqf3BI_%5`*~lbv53F_uKi zcYuAlybA5?ylD1fE_-pLcI8y|s>|A}sJ4XDmVBx$`}6Njl)XpxIp7#wwiGG|oS!c4 zY^KgxI_s4`RfF}BG7tP8)vM5M&fOqWfc{)>-BRYaI2Omyx*M&7VDV~zlJHc6!M!2S zb=)ZenI|M%i>Zw8lNNs8{w8vBM1I>sP|?a7@EnBSyMX7Ka1#Y@t&tr7&9WflZGt1m z>t`9k$;x>BKF@xi z-xoTRaC4vKfE)1|HGlBzu`eA6e!% zn4e(a3Undq5zaxJc_xicY1arx>^kNa#KN+m*g3G`3YkoHU9ONRZ|EtR@z<2;*OcKm zRO_cy>t|HyXO#WdRQ6|7bzEtY-6y-Ey#G_>{huk9{zkcPO1bY(a)nC$CkibsrMg$f zDet*nMCq-g+op7-N7R2(KcEGuQzIBw$Z~t&t*2a?O3%o*4}YlVUv{$N8b5W zpfnbzz#p^Mj$6n3&KJTv^f)Gn-L-MDo4ae*AJOx990;q{0-Ppk&Vf&r2R>6a{YL4U mQo0s4=RO&<5ZFaXcZTj0%(t@u literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/crawler/parsers.py b/gx_gp_monitor/crawler/parsers.py new file mode 100644 index 0000000..8ed8ad5 --- /dev/null +++ b/gx_gp_monitor/crawler/parsers.py @@ -0,0 +1,301 @@ +""" +数据解析器模块 +负责解析广西政府采购网的API响应数据 +""" + +import json +from typing import List, Dict, Any, Optional +from datetime import datetime + +try: + from ..core.models import Announcement, AnnouncementSource, AnnouncementType + from ..core.logger import get_logger +except ImportError: + from core.models import Announcement, AnnouncementSource, AnnouncementType + from core.logger import get_logger + + +logger = get_logger(__name__) + + +class AnnouncementParser: + """公告数据解析器""" + + @staticmethod + def parse_api_response(response_data: Dict[str, Any], + source: AnnouncementSource, + crawled_at: datetime) -> List[Announcement]: + """ + 解析API响应数据 + + Args: + response_data: API响应数据 + source: 公告来源 + crawled_at: 爬取时间 + + Returns: + List[Announcement]: 解析后的公告列表 + """ + if not response_data or not isinstance(response_data, dict): + logger.warning("API响应数据无效") + return [] + + try: + # 检查响应状态 + if not response_data.get("success", False): + logger.warning(f"API响应失败: {response_data.get('message', '未知错误')}") + return [] + + # 获取数据部分 + result = response_data.get("result", {}) + data = result.get("data", {}) + records = data.get("data", []) + + if not records: + logger.info(f"来源 {source.name} 没有新数据") + return [] + + announcements = [] + for record in records: + try: + announcement = AnnouncementParser._parse_single_record( + record, source, crawled_at) + if announcement: + announcements.append(announcement) + except Exception as e: + logger.warning(f"解析公告记录失败: {str(e)}, 记录: {record}") + continue + + logger.info(f"成功解析 {len(announcements)}/{len(records)} 条公告记录") + return announcements + + except Exception as e: + logger.error(f"解析API响应数据失败: {str(e)}") + return [] + + @staticmethod + def _parse_single_record(record: Dict[str, Any], + source: AnnouncementSource, + crawled_at: datetime) -> Optional[Announcement]: + """ + 解析单个公告记录 + + Args: + record: 公告记录数据 + source: 公告来源 + crawled_at: 爬取时间 + + Returns: + Optional[Announcement]: 解析后的公告对象 + """ + try: + # 提取基本字段 + title_raw = record.get("title", "") + title = str(title_raw).strip() if title_raw is not None else "" + if not title: + return None + + # 解析发布时间 + publish_timestamp = record.get("publishDate") + if not publish_timestamp: + logger.warning(f"公告缺少发布时间: {title[:50]}...") + return None + + try: + # 时间戳转换为datetime + publish_date = datetime.fromtimestamp(int(publish_timestamp) / 1000) + except (ValueError, TypeError) as e: + logger.warning(f"发布时间格式错误: {publish_timestamp}, 错误: {str(e)}") + return None + + # 提取其他字段 + purchase_name_raw = record.get("purchaseName", "") + purchase_name = str(purchase_name_raw).strip() if purchase_name_raw is not None else "" + article_id = record.get("articleId") + + if not article_id: + logger.warning(f"公告缺少文章ID: {title[:50]}...") + return None + + # 构建内容链接 + content_url = AnnouncementParser._build_content_url( + source.category_id, source.code, article_id) + + # 创建公告对象 + announcement = Announcement( + title=title, + publish_date=publish_date, + purchase_name=purchase_name, + content_url=content_url, + source_code=source.code, + source_name=source.name, + announcement_type=source.type, + crawled_at=crawled_at, + is_new=True # 默认标记为新公告 + ) + + # 生成内容哈希用于去重 + announcement.generate_content_hash() + + return announcement + + except Exception as e: + logger.error(f"解析单个公告记录失败: {str(e)}") + return None + + @staticmethod + def _build_content_url(category_id: int, source_code: str, article_id: int) -> str: + """ + 构建公告内容链接 + + Args: + category_id: 分类ID + source_code: 来源代码 + article_id: 文章ID + + Returns: + str: 内容链接 + """ + return f"https://zfcg.gxzf.gov.cn/site/detail?parentId={category_id}&articleId={article_id}" + + @staticmethod + def validate_response_structure(response_data: Dict[str, Any]) -> bool: + """ + 验证API响应数据结构 + + Args: + response_data: API响应数据 + + Returns: + bool: 结构是否有效 + """ + try: + if not isinstance(response_data, dict): + return False + + # 检查必需的字段 + if "success" not in response_data: + return False + + if not response_data.get("success", False): + return False + + result = response_data.get("result", {}) + if not isinstance(result, dict): + return False + + data = result.get("data", {}) + if not isinstance(data, dict): + return False + + records = data.get("data", []) + if not isinstance(records, list): + return False + + return True + + except Exception as e: + logger.warning(f"验证响应结构失败: {str(e)}") + return False + + @staticmethod + def extract_pagination_info(response_data: Dict[str, Any]) -> Dict[str, Any]: + """ + 提取分页信息 + + Args: + response_data: API响应数据 + + Returns: + Dict[str, Any]: 分页信息 + """ + try: + result = response_data.get("result", {}) + data = result.get("data", {}) + + return { + "total": data.get("total", 0), + "page_no": data.get("pageNo", 1), + "page_size": data.get("pageSize", 15), + "pages": data.get("pages", 0), + "empty": data.get("empty", True), + "has_next": data.get("hasNext", False), + "has_previous": data.get("hasPrevious", False) + } + except Exception as e: + logger.warning(f"提取分页信息失败: {str(e)}") + return { + "total": 0, + "page_no": 1, + "page_size": 15, + "pages": 0, + "empty": True, + "has_next": False, + "has_previous": False + } + + +class SensitiveWordChecker: + """敏感词检查器""" + + @staticmethod + def parse_check_response(response_data: Dict[str, Any]) -> bool: + """ + 解析敏感词检查响应 + + Args: + response_data: 检查响应数据 + + Returns: + bool: 检查是否通过 + """ + try: + if not isinstance(response_data, dict): + logger.warning("敏感词检查响应格式无效") + return False + + success = response_data.get("success", False) + if not success: + message = response_data.get("message", "未知错误") + logger.warning(f"敏感词检查失败: {message}") + return False + + return True + + except Exception as e: + logger.error(f"解析敏感词检查响应失败: {str(e)}") + return False + + +class ErrorResponseParser: + """错误响应解析器""" + + @staticmethod + def parse_error(response_data: Dict[str, Any]) -> str: + """ + 解析错误响应 + + Args: + response_data: 错误响应数据 + + Returns: + str: 错误信息 + """ + try: + if not isinstance(response_data, dict): + return "响应格式无效" + + # 尝试提取错误信息 + error_msg = response_data.get("message") or response_data.get("msg") + if error_msg: + return str(error_msg) + + # 检查状态码 + errcode = response_data.get("errcode") + if errcode: + return f"错误码: {errcode}" + + return "未知错误" + + except Exception as e: + return f"解析错误响应失败: {str(e)}" diff --git a/gx_gp_monitor/crawler/spider.py b/gx_gp_monitor/crawler/spider.py new file mode 100644 index 0000000..e6ca2b0 --- /dev/null +++ b/gx_gp_monitor/crawler/spider.py @@ -0,0 +1,458 @@ +""" +爬虫核心模块 +实现广西政府采购网公告的智能爬取功能 +""" + +import time +import random +import json +from typing import List, Dict, Any, Optional, Tuple +from datetime import datetime +from urllib.parse import urljoin +import requests +from fake_useragent import UserAgent + +try: + # 尝试相对导入 + from ..core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus + from ..core.config_manager import get_config + from ..core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error + from ..core.reliability import ( + retry_on_exception, RetryConfig, session_with_retry, + TimeoutConfig, safe_execute, check_system_health + ) + from .parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser +except ImportError: + # 尝试绝对导入 + from core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus + from core.config_manager import get_config + from core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error + from core.reliability import ( + retry_on_exception, RetryConfig, session_with_retry, + TimeoutConfig, safe_execute, check_system_health + ) + from crawler.parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser + + +logger = get_logger(__name__) + + +class GXGPSpider: + """广西政府采购网爬虫""" + + def __init__(self): + self.config = get_config() + self.ua = UserAgent() + + # API端点 + self.base_url = self.config.crawler.base_url + self.announcement_api = urljoin(self.base_url, "/portal/category") + self.sensitive_check_api = urljoin(self.base_url, "/portal/sensitiveWords/check") + + # 会话管理 + self.session = None + + # 统计信息 + self.request_count = 0 + self.error_count = 0 + + def init_session(self): + """初始化会话""" + if self.session is None: + timeout_config = TimeoutConfig( + connect_timeout=self.config.crawler.timeout, + read_timeout=self.config.crawler.timeout + ) + + retry_config = RetryConfig( + max_retries=self.config.crawler.max_retries, + initial_delay=self.config.crawler.retry_delay, + max_delay=self.config.crawler.max_retry_delay, + backoff_factor=self.config.crawler.backoff_factor + ) + + self.session = requests.Session() + + # 配置重试和超时 + adapter = requests.adapters.HTTPAdapter( + pool_connections=10, + pool_maxsize=20, + max_retries=0 # 我们使用自己的重试逻辑 + ) + self.session.mount('http://', adapter) + self.session.mount('https://', adapter) + + # 设置默认超时 + self.session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout) + + return self.session + + def close_session(self): + """关闭会话""" + if self.session: + self.session.close() + self.session = None + + def get_random_user_agent(self) -> str: + """获取随机User-Agent""" + try: + return self.ua.random + except: + # fallback到配置的user agents + return random.choice(self.config.crawler.user_agents) + + def get_random_proxy(self) -> Optional[Dict[str, str]]: + """获取随机代理""" + if not self.config.crawler.proxies: + return None + + proxy = random.choice(self.config.crawler.proxies) + return { + "http": proxy, + "https": proxy + } + + def check_sensitive_words(self, payload: Dict[str, Any], + category_code: str, childrencode: str) -> bool: + """ + 执行敏感词检查 + + Args: + payload: 请求参数 + category_code: 分类代码 + childrencode: 子分类代码 + + Returns: + bool: 检查是否通过 + """ + try: + session = self.init_session() + user_agent = self.get_random_user_agent() + + headers = { + "User-Agent": user_agent, + "Content-Type": "application/json;charset=UTF-8", + "Origin": self.base_url, + "Referer": f"{self.base_url}/site/category?parentId={category_code}&childrenCode={childrencode}", + "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" + } + + proxies = self.get_random_proxy() + + logger.debug(f"执行敏感词检查: {category_code}/{childrencode}") + + response = session.post( + self.sensitive_check_api, + json=payload, + headers=headers, + proxies=proxies, + timeout=self.session.timeout + ) + + self.request_count += 1 + + if response.status_code == 200: + result = response.json() + return SensitiveWordChecker.parse_check_response(result) + else: + logger.warning(f"敏感词检查请求失败, 状态码: {response.status_code}") + return False + + except Exception as e: + logger.error(f"敏感词检查异常: {str(e)}") + self.error_count += 1 + return False + + def fetch_announcements_page(self, source: AnnouncementSource, + page_no: int = 1) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + """ + 获取公告列表页数据 + + Args: + source: 公告来源 + page_no: 页码 + + Returns: + Tuple[Optional[Dict[str, Any]], Optional[str]]: (响应数据, 错误信息) + """ + try: + session = self.init_session() + + # 构建请求参数 + payload = { + "keyword": "", # 关键词筛选，我们在筛选模块处理 + "publishDateBegin": self.config.crawler.start_date or "", + "publishDateEnd": self.config.crawler.end_date or "", + "pageNo": page_no, + "pageSize": self.config.crawler.page_size, + "categoryCode": source.code, + "_t": int(time.time() * 1000) + } + + # 先执行敏感词检查 + if not self.check_sensitive_words(payload, str(source.category_id), source.code): + return None, "敏感词检查失败" + + # 执行公告数据请求 + user_agent = self.get_random_user_agent() + headers = { + "User-Agent": user_agent, + "Content-Type": "application/json;charset=UTF-8", + "Origin": self.base_url, + "Referer": f"{self.base_url}/site/category?parentId={source.category_id}&childrenCode={source.code}", + "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" + } + + proxies = self.get_random_proxy() + + logger.debug(f"请求公告数据: {source.name} 第{page_no}页") + + # 添加请求间延迟 + if page_no > 1: + delay = random.uniform( + self.config.crawler.request_delay, + self.config.crawler.request_delay_max + ) + time.sleep(delay) + + response = session.post( + self.announcement_api, + json=payload, + headers=headers, + proxies=proxies, + timeout=self.session.timeout + ) + + self.request_count += 1 + + if response.status_code == 200: + data = response.json() + if data.get("success", False): + return data, None + else: + error_msg = ErrorResponseParser.parse_error(data) + return None, f"API返回失败: {error_msg}" + else: + return None, f"请求失败, 状态码: {response.status_code}" + + except requests.exceptions.Timeout as e: + self.error_count += 1 + return None, f"请求超时: {str(e)}" + except requests.exceptions.ProxyError as e: + self.error_count += 1 + return None, f"代理错误: {str(e)}" + except Exception as e: + self.error_count += 1 + logger.error(f"获取公告数据异常: {str(e)}") + return None, f"请求异常: {str(e)}" + + def crawl_source(self, source: AnnouncementSource, + max_pages: Optional[int] = None) -> CrawlResult: + """ + 爬取单个来源的公告 + + Args: + source: 公告来源 + max_pages: 最大页数限制 + + Returns: + CrawlResult: 爬取结果 + """ + if max_pages is None: + max_pages = self.config.crawler.max_pages + + log_crawl_start(source.name) + + start_time = datetime.now() + result = CrawlResult( + source=source, + status=CrawlStatus.RUNNING, + crawled_at=start_time + ) + + try: + page = 1 + all_announcements = [] + + while page <= max_pages: + # 获取页面数据 + response_data, error_msg = self.fetch_announcements_page(source, page) + + if error_msg: + logger.warning(f"{source.name} 第{page}页获取失败: {error_msg}") + result.status = CrawlStatus.FAILED + result.error_message = error_msg + break + + if not response_data: + logger.info(f"{source.name} 第{page}页无数据") + break + + # 解析分页信息 + pagination = AnnouncementParser.extract_pagination_info(response_data) + result.total_count = pagination["total"] + + # 解析公告数据 + announcements = AnnouncementParser.parse_api_response( + response_data, source, start_time) + + if not announcements: + logger.info(f"{source.name} 第{page}页解析到0条公告") + break + + all_announcements.extend(announcements) + + # 检查是否还有下一页 + if not pagination["has_next"] or pagination["empty"]: + break + + page += 1 + + # 更新结果 + result.announcements = all_announcements + result.new_count = len(all_announcements) # 这里的新增数需要在筛选后确定 + + if result.status != CrawlStatus.FAILED: + result.status = CrawlStatus.SUCCESS + + duration = (datetime.now() - start_time).total_seconds() + result.duration = duration + + log_crawl_success(source.name, len(all_announcements), duration) + + except Exception as e: + duration = (datetime.now() - start_time).total_seconds() + result.duration = duration + result.status = CrawlStatus.FAILED + result.error_message = str(e) + + log_crawl_error(source.name, str(e)) + + return result + + @retry_on_exception(RetryConfig(max_retries=2)) + def crawl_all_sources(self, sources: Optional[List[AnnouncementSource]] = None) -> List[CrawlResult]: + """ + 爬取所有来源的公告 + + Args: + sources: 指定的来源列表，如果为None则使用配置中的所有来源 + + Returns: + List[CrawlResult]: 所有来源的爬取结果 + """ + # 系统健康检查 + if not check_system_health(): + logger.error("系统健康检查失败，跳过爬取") + return [] + + if sources is None: + sources = self._load_sources_from_config() + + logger.info(f"开始爬取 {len(sources)} 个公告来源") + + results = [] + + for source in sources: + try: + result = self.crawl_source(source) + results.append(result) + + # 检查是否需要暂停 + if result.status == CrawlStatus.FAILED: + logger.warning(f"来源 {source.name} 爬取失败，继续下一个来源") + continue + + except Exception as e: + logger.error(f"爬取来源 {source.name} 时发生未预期错误: {str(e)}") + # 创建失败结果 + failed_result = CrawlResult( + source=source, + status=CrawlStatus.FAILED, + error_message=str(e), + crawled_at=datetime.now() + ) + results.append(failed_result) + + # 统计总结果 + total_announcements = sum(len(r.announcements) for r in results if r.status == CrawlStatus.SUCCESS) + success_count = sum(1 for r in results if r.status == CrawlStatus.SUCCESS) + failed_count = len(results) - success_count + + logger.info( + f"爬取完成: 共处理 {len(results)} 个来源，" + f"成功 {success_count} 个，失败 {failed_count} 个，" + f"获取 {total_announcements} 条公告" + ) + + return results + + def _load_sources_from_config(self) -> List[AnnouncementSource]: + """从配置加载公告来源""" + sources = [] + + for code, source_config in self.config.sources.items(): + try: + source = AnnouncementSource( + code=code, + category_id=source_config["category_id"], + name=source_config["name"], + type=AnnouncementType(source_config["type"]) + ) + sources.append(source) + except Exception as e: + logger.warning(f"加载来源配置失败 {code}: {str(e)}") + continue + + return sources + + def get_stats(self) -> Dict[str, Any]: + """获取爬虫统计信息""" + return { + "request_count": self.request_count, + "error_count": self.error_count, + "error_rate": self.error_count / max(self.request_count, 1), + "session_active": self.session is not None + } + + def reset_stats(self): + """重置统计信息""" + self.request_count = 0 + self.error_count = 0 + + +def create_spider() -> GXGPSpider: + """ + 创建爬虫实例 + + Returns: + GXGPSpider: 爬虫实例 + """ + return GXGPSpider() + + +def crawl_announcements(keywords: Optional[List[str]] = None, + sources: Optional[List[str]] = None) -> List[CrawlResult]: + """ + 便捷函数：爬取公告 + + Args: + keywords: 关键词过滤（暂时未使用，在筛选模块处理） + sources: 来源代码列表 + + Returns: + List[CrawlResult]: 爬取结果 + """ + spider = create_spider() + + try: + # 过滤来源 + if sources: + all_sources = spider._load_sources_from_config() + filtered_sources = [s for s in all_sources if s.code in sources] + else: + filtered_sources = None + + return spider.crawl_all_sources(filtered_sources) + finally: + spider.close_session() diff --git a/gx_gp_monitor/filters/__init__.py b/gx_gp_monitor/filters/__init__.py new file mode 100644 index 0000000..3c23a98 --- /dev/null +++ b/gx_gp_monitor/filters/__init__.py @@ -0,0 +1 @@ +"""筛选模块""" diff --git a/gx_gp_monitor/filters/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/filters/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77102fe5bd5a7ad63c09cbc8ae3dc761e43807c9 GIT binary patch literal 195 zcmey&%ge<81U)KonZiK&F^B^Lj8MjB4j^MHLoh=TLpq}-Qz;LS`+V)}mkpiIRxErv zce}zQ`1q9!pFz5Ax#(x)=cekHndRrC>K9ZN6y;~7CYR{DN9d*(=;q~@ zWTsW>r&q+M7sTi0=Vg}U7wM;E=9Hus73;^xX95M{a2@oDfh+1a+Nes(Z7ikS zdFwP{?mF>8am%Gr6O@{7DJ>>X=0ll9F5^~^JS*fCD0w!Lw+!+MmAn#?R|I*>mAq1t zR}6V8lstPmh5y{;Zd<>m-=Tw%xl8&T?o#-(_uE?xZbvEg5c3etm63amyA1ES@?l*C z)xRu<@-C++vPP+uq0?4|b}K96q`Hp{QEgf$HAK6d;rpN~G@ZGCO!a} zR}cSa{<*377cX2p{KAzVADMsd3^ZrFhU@Fn3CwPPFeDiEjD`H80q=;wwEG7_g7)sf zqk`7$3%N9cVbB}$h5Uzn0)r_RE#$)0J22u62D`7}>w+xP-GRVpI56NlWoV z>(}?^;Oh4q+)OQ8AuqSz*q7+bUTAp6YGp9Y9L-Gt!eOl|U&!z9kA!^OR?ZKx-+Rpl0|}cTlHSm3XD?s)dADHry6@3%j&g%R z!59pAIWm2pVDJS7aUpy{PH;5L4fui`g4y@Tz({z|=LwAtdLI??2XKvozCh3)@;~eo z@`l3S`qrbcXrY0F0}{(XZxn?qkIS}rFGMG(UaAMK*lo2&PRQ&o{!}5vj!D@eKL7U}(f2@C8PpILG4|93Aj@#8trc z7PP?-CqP9nkbKCCd=rH7LhOmb2yvAw!Uz|QD~3p54vdbDa4Ya035F}j%;g>r&_>8V z;PHk+oc};L^gk->@PBmNjuo~Wo9-!)-4 zRdhT*#?&O7RZwslDMP7;TDH&8&o0pYnt~0mfMTd;0x&iY=ew+exl=@0ksz+Ehn~U+ zSBhv9L3#f4REqZoWKxEDDG03<#x5QK6Ej4xFwh?9nq5L7`V zl<>AVV*vqf#r#cBJBW}oP9?3BvwqfKPuN$^8f=Nm>In0(!6{M+F7<%2EO4WTKkR97 zKP=r+GHDf5z~F@^-Si$ypVC4P;%8O}XnN^FYR5Aa70~Wm(}O)^g*&0IL1?23Fc+j7 zNV4pp^aw2H*7W3MmqV_$lQ=#RlJWQR?u~ZibC^8UggLa7KO5s`17B!qt z(EEY|-Z3BEHLMXBq6XnRmdP)7^RO@Adt{8;A~a>upG3(DAvbIu86EJB1h+IO&miIj zvA1hLozYdjA9!Z=zP0S_k{^{!x4i9~a(>|XV-P3$R8(-qW*f z>)q~cTq|S{Q7kYIjrs$EE(lm3;_kwFInpRxI+2Tz%mv-hh<7+BW!869^ZbsoIT@z#Xpr3D_=9Q z{c`iVU$p*x>&$Shxod)nTWfi1EfiVT{AU4WV+kt@Ovv|0NYHb>P?!s3KqStO@bnKv zp4KSb1TvwQmhT3kbeLAxLX|qhghLg6hnA%v*Fa3@1x)Crd$8ZEAah_qtFFanw4&ML z?9~)f_cB3EBNc?y4tS5F?YC?9s~W@*BdGsBEk0;OnwF?N*vcx; z&PPsOJv(JI_7GB$Bpj-@iogpQI%Hb*)Ws~h60+u(QlAb+1Hwsrrn z@AE`^-2bxu{q6s}6LR>i`y+ckv#*}WNfefTTXnwqOV{$HYo}}Z(pK`^$t}mXOjgGWYh#7A*gy{Ds7@N7fj|DIooc7O zbPxr?`}UHpMn*8BHwX%VO(jW2QLXSV;{^+3~FNB&GcF1@)T3SOmu1ajLf!>uW#1LcfiCJ zSD@LBZ~=@oH8|osYGwb#u=QKm&vBjDh1brC<8c`Ti!>0?$pD6sF60l5_yqG9xF`;y zI|8j^4*d**-a9r1P7FaWI(G!d9~c@Hv?IO%a68dC_dzw0kxl6fAQ)sjiz7^k^o)xj zGj5SsOsr~!wuQ<}E>^1(goc9WAOe=Q0ZV_z)=!ucjm;CL-&pIeYpIHcxk5H-V&@8% zM-0DnxFWmf%3SfXM!u{u>R2;PPuInpcJocUqiegOrQMNEK!anC9eM0jXWY`nTbf8q zv(~zV)ppWy+%gHR*3UJzh^-e;Wvg!#QMs05c}Mc1#kFr~r@CURTBEgh&a`}}y|m+3 z=9%w&^iZ_u{#nBV3CoIzg=1l$pq&%-LvaUmyEGjguKcBqM&uX{B@ljI7dK!5Mra)cakZYMe#js7C zq@!F{a%yXmg3DCPwD!$SBso)b19Osh(d9bZboKOOd{a9X*b+6asj9R0C$*Sv-Y{e0 zo44J>8!~S)iUAma5^&@g7LjOfwJIgYYaZH_{X|Z>b+Sd7W+CSEqg5%@Xfe8TCHvFp zHgb93v@;3DcCg|lBk_CCRT!~A^p+|XVmC)9w-8E1?@ckL_CPd{u8JrKo6JU777$h| zW7}W}+%d{sI{NAW^K@zW1&e_AC84iA%}h{1*h<9=0hIv_qo)C!SD=fEM zw(mr0J$iGMaf9sX;$WnDRjH7M&t1=aDH)(m!+OP<0jy7#4SE^%Q8r}A06oY6eRu&D zN0I^Q?)Ld-pHYzb&yfKs=P*cti{s!;5L+eMm+UPDxCgJ+YiH7n(w)Y#=C$zRcdwm3 zclp9|P*D~~8EV>}CrLWOXw!MJV<=p z+DF$y) z6A~w(v0wrs5E3k9Eq#5}&)Y;nQNI?Ue08><_{3Ut()eBDOC~VDP-Y|wo$jW61Q*S?VDouJ0rV6;VY|%bS+p)PL0IMHze4a3G;$w#i`vf zM>DzkgPyXMO*&!~>&fk$YyGTgK$wt%-kAOrB?4#>JyPPmQeeFPsPBO$+DO-_JkQt?4@#^GF7o?vO0 zUA&TT_&?A-5f1Wx8Sc7)`%o~2K@9@`FxkS&@Ohzvj7z$>n~W{pg_yFAz2Ncb!&)TB zfST!41hJ6qWCSHV-N`8J;Ynjkhne)z$!Ku-m+ztD@Wk}O3+LzGeMvO$U=wiuoajLe zLgO5AxL_PSAn(e#G$MmyVGyS#>1Ray5NuiNr8o?Etd2Vt;4sXm%3QOClK5Whmb0R~}-)b;*?B(#Chiwq0b3F*%H*iFk-+h)11MLu-NgSSJIpx7tpf4nt z21WxR*!K2-sdP|Sj=oI}H;vM>Il^pVXhkUY2Q!Qu*gFFIE+}ZdgM-|Ds6hl}N^Z&y z5@Z|njranjBVSOIm4o;ME}&q7hLtTV!J;I^{jOVmC8Ff4&Sq&f@%cMwKgdG_;F>my+j!@;OCvF-`?v{&LlGM*Y)DkqPV7!t?V{;5YpuUh=$zP`At_xbDH|wz z`Kd#((z>^T(;NTpalUkO#FEG>J@r?yyz);>%Z_y(>5P^(O@AfYv^Bcb9kuP9HTBM= z3^EOeuZFy&ZWq1RmAAC?M8--p#Xxz#jo|zSmSTi*CAbD6A~>%z23;pTx%YDlugRi2 z(qt#3KPviYgx}eBlY+Denb<^zK$M-1i+4eZia-Q$C>cy>I+0ToAQEL02{z`)Mw-E$ z!{rVeT)BeDO^zdoQqZ3ZP%D~9ul;c0-3i4+A{LTh@s6M`Z_pzhVGwO3MYzF$4Dd&= zJW58FvKY{LDcLz9Zi6LDa2x7nhh7b09&vX9ngD*K_aIfyz%O_+#1F*CEvdO?-EGWn#c(^U=Tmh=mWx_sLE%+Td^fGmUp04xT5+B6u2I`ycY_l|c8&BKTQvAeK;bnIKF@<~ zcHs4X6Xb|n?Q%7!5C_)Qy&jImTo`$;QRP^1r+hJ>_IWs7-O1Di6Np*a1l0Q!LyP)%3k<&~8EyWpjT z(_}_CgtY>iAiF8uVA`QJ!X3c~{d%b~Az$|7ib9;&Y(iezu@)kX1Zzq+kSZc63T6oc z#60K}G*+>nTP*?mt%RZ&1GVrLgAc;9#26^4h}&Aub;qh(BFs_q4L#+oOW2)f+D^B{ z?HhRehJ?*=#(dfwx2@%EYp?6HPGf|D^9I1)j+L)&#|?9r6HTlGza&lY`n09%=*_ykG#rQ|P0}3To>5 zKI@llU*0we(4hFke6oYuOi($8*(f9o<3ggz3Top@t;V0LXd=9iU@2D=bkHb^5HZL`~R=$`0$Kwf^5+haSa5O7$kKkg|45j1Z3uW(9@vz`B5^ z*sNl>Ra@63C{LyRznt=xri7(epaK~L;r)tV6Q~;r*Qoc`Jp@e}Fl`%t!4oZ|UWjuq%yYi+pN^+XbvT4my`36deoy@Z^=L=hLM}`gR-c*uQuN52vN?I%b;5A#i^A zTG>@g>5O&KE~9v)^2&uXSI?hy8O1|KWJ?i86qx{j)A$8&GqFU)#4SdpK{kwnm0hK5 zMYghG#K$^T^eQX2qT#LFimGobE6^_2a&p*FG>RMGSyAwz(B+O{G=xzzMz~hu3Cb5R zRfrK{4LO1-ODk+QG>Umpz+;$Id zyXR6{%=T9ibE0~6ym~!fz5Zh1%!9E_oqToYM?>-MK^}h9gE)u^>CsiZZB^XX#@pHw zrji50MHV;4Nx3Nn{+Po{`c@<>djl82V?r7&7?aI8@@y>NP?+q~Tm38rz z8~K$RFE-CS9=o%PU)dG?>b>!=4)O3?ITZDe-lV99=&hRT%B6OzCP`5pn%x>G>d-uZ zMbf4A0S&HN)n&G7vMa_m;h@Sc8}f1IHj!U7*&(i3v}(ij6PJeK9esR9Uvyi4lA<4^ z;qYTc+}_07o1&%7)2%Z@@hv_4mY!&v8;bYRD-tWp;w#qhE7nAd*G`wtbi_CJ@SA(0 zceEsn3d>GW`xT zYMg)X@2-uXk&yj`6PIUxBubY5xEO(5hRg51IDh^J3nzarR``#LN0t&y=-?pdVBUV` z>QBzz<`~(v@mJu(Cb2zZM>qEXjAAbsKg8}K@c#_t2xizEBnA%IAtazL<#It0kZ-o5=Uzg7M3r-cxPp{xdoa55#I7NEG58732Nzti9dz*V6#?uYQ ze^qR3k$NVeLb_2bSi)n33S|pks8k`BtUAvk#WX=nlAZizt>cdk;#V5-0ExCklpjLb zL@Ppj#${dF)rF{&<;I3hd{hWUb?XF{BT{j%VDuVBs8MHsjS|5;jL=IIydM(i3Ugj( zth&0%^3N%_B&!>AHIwaA4p2#w6r^X?T`IaX!r$$V_Ijgtd-*#L+`#PQe%h`pnsOv5 zxXe`Fz)P~#tTRsK-=yFo%1qKBVd#{8w+Fh-I0lK19b*6g2u7u9|HBDloE-))kkvkI z%Jr1CYrC3)4?p`9;!?BN(+|e09aE7d7aUns+)XKmxcAHTpqwZEf*x|k8NpT@0wf8k z!;{ri1=Xj>JNPWsPTX*cEa*|JEcsb-Jd=h)ol5^K1)r$ymp@Vc3)%xa>z4NJliLGV z*{`8*aFx9}f8pJ07fxM0e@>D(r{TnL>Vul4Pfh}OvrC<+ng8+0D{sAidFJogRrf=v z@9L^=Z+G41q|Fi!WLiTOznT&D1_5HixiD*!@gek zk?v?&>rCrRUDUq)H>Mp)EtdTE4bZenEoeGbYvau7sD0aSOdVKjJN>6%6^wtN5pyZ~V z&!hVk_nQm?_%V~33sKE!2yP8U-ld;gQ+4Da7G?#xPuZye?IG2Wu*4`-lu898w-6{) zTyH%pt~G>0)!C`EE03xNKqx!NBZm>Avw(ypT2^sd{jZEzrQI(%Vv9WDz6v>&`W%En zup&+Xf?<9vxR_EDCN`Ca4W`M*#Tn7nE<2%VDn)of*c<7hZ(VKd$`p zJbc;#8ww)6rXg#_Awe}j6Q70;9Timxzzt5e#;=JDk#Y6nC17S&E;1pqqd_OHP0QN!AnS=DpoyCY? z-QQq}ICvtE0%e}Gua^cw@C-RsfwN_zi^hYniKjDgxIcypGc;hC4kFaUw7TX`4K3FV zRKD%`{_&2>mNEiu30wJ@{L}eyTLW)v7~e6=EdOMM{Zz-9?$h0I2l_E$ja|=%HJuEZ|dPU^~5UO@ydOC<-S@GqYD$rJGn-Ta2`7`rFV?&aCNG4@_Hl&q&rD*z>R&A&G^MGZ{=c)I5QzELWn zV!>}sR>Z)96i_Nze+nY|J*96iq2Axv*1m@NRYBV}iu$m`+Fq~yu#U!fjj_E=`{8C9 z;(wxy+sxX3GSLv@$-{1!lYHka=tGZ=p~S&Az#++VsQz#WX&mArM_wkrN3uaB{(lm1 zs0BHmBcJQ#5fXwA^?f5jvMVcyIpUWG9C{W7GpQq=%i~a&;ifREz{rac5t?b1Yp(GdTC$A9sU literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/filters/filters.py b/gx_gp_monitor/filters/filters.py new file mode 100644 index 0000000..d815381 --- /dev/null +++ b/gx_gp_monitor/filters/filters.py @@ -0,0 +1,471 @@ +""" +智能筛选模块 +提供关键词过滤、日期范围筛选、自动去重等功能 +""" + +from typing import List, Optional, Dict, Any, Set +from datetime import datetime, date +from dataclasses import dataclass +import re + +try: + from ..core.models import Announcement + from ..core.logger import get_logger + from ..core.database import get_database_manager +except ImportError: + from core.models import Announcement + from core.logger import get_logger + from core.database import get_database_manager + + +logger = get_logger(__name__) + + +@dataclass +class FilterCriteria: + """筛选条件""" + keywords: List[str] = None # 关键词列表 + start_date: Optional[date] = None # 开始日期 + end_date: Optional[date] = None # 结束日期 + sources: List[str] = None # 来源代码列表 + exclude_today: bool = False # 排除今日公告 + case_sensitive: bool = False # 关键词匹配是否区分大小写 + fuzzy_match: bool = False # 是否启用模糊匹配 + + def __post_init__(self): + if self.keywords is None: + self.keywords = [] + if self.sources is None: + self.sources = [] + + +@dataclass +class FilterResult: + """筛选结果""" + total_count: int = 0 # 总数 + filtered_count: int = 0 # 筛选后数量 + keyword_filtered: int = 0 # 关键词筛选数量 + date_filtered: int = 0 # 日期筛选数量 + duplicate_filtered: int = 0 # 去重筛选数量 + source_filtered: int = 0 # 来源筛选数量 + today_excluded: int = 0 # 排除今日数量 + + +class KeywordFilter: + """关键词筛选器""" + + def __init__(self, case_sensitive: bool = False, fuzzy_match: bool = False): + """ + 初始化关键词筛选器 + + Args: + case_sensitive: 是否区分大小写 + fuzzy_match: 是否启用模糊匹配 + """ + self.case_sensitive = case_sensitive + self.fuzzy_match = fuzzy_match + self._compiled_patterns = {} + + def _compile_patterns(self, keywords: List[str]): + """编译关键词模式""" + flags = 0 if self.case_sensitive else re.IGNORECASE + + for keyword in keywords: + if keyword not in self._compiled_patterns: + if self.fuzzy_match: + # 简单的模糊匹配：将关键词中的空格替换为灵活匹配 + pattern = r'.*'.join(re.escape(word) for word in keyword.split()) + else: + pattern = re.escape(keyword) + + self._compiled_patterns[keyword] = re.compile(pattern, flags) + + def matches(self, text: str, keywords: List[str]) -> bool: + """ + 检查文本是否匹配关键词 + + Args: + text: 要检查的文本 + keywords: 关键词列表 + + Returns: + bool: 是否匹配 + """ + if not keywords or not text: + return True # 没有关键词或文本时，认为匹配 + + if not self.fuzzy_match: + # 精确匹配 + search_text = text if self.case_sensitive else text.lower() + search_keywords = keywords if self.case_sensitive else [k.lower() for k in keywords] + + for keyword in search_keywords: + if keyword in search_text: + return True + return False + else: + # 正则匹配 + self._compile_patterns(keywords) + for keyword in keywords: + pattern = self._compiled_patterns[keyword] + if pattern.search(text): + return True + return False + + def filter_announcements(self, announcements: List[Announcement], + keywords: List[str]) -> List[Announcement]: + """ + 筛选匹配关键词的公告 + + Args: + announcements: 公告列表 + keywords: 关键词列表 + + Returns: + List[Announcement]: 筛选后的公告列表 + """ + if not keywords: + # 没有关键词时，所有公告都匹配 + for announcement in announcements: + announcement.keyword_matched = True + return announcements + + filtered = [] + for announcement in announcements: + # 构建搜索文本 + search_text = f"{announcement.title} {announcement.purchase_name}" + + if self.matches(search_text, keywords): + announcement.keyword_matched = True + filtered.append(announcement) + else: + announcement.keyword_matched = False + + logger.info(f"关键词筛选: {len(announcements)} -> {len(filtered)} 条公告") + return filtered + + +class DateFilter: + """日期筛选器""" + + def filter_announcements(self, announcements: List[Announcement], + start_date: Optional[date] = None, + end_date: Optional[date] = None, + exclude_today: bool = False) -> List[Announcement]: + """ + 筛选日期范围内的公告 + + Args: + announcements: 公告列表 + start_date: 开始日期 + end_date: 结束日期 + exclude_today: 是否排除今日公告 + + Returns: + List[Announcement]: 筛选后的公告列表 + """ + if not start_date and not end_date and not exclude_today: + # 没有日期限制时，所有公告都通过 + for announcement in announcements: + announcement.date_filtered = True + return announcements + + filtered = [] + today = date.today() + + for announcement in announcements: + if not announcement.publish_date: + # 没有发布日期的公告不通过筛选 + announcement.date_filtered = False + continue + + publish_date = announcement.publish_date.date() + + # 检查日期范围 + date_in_range = True + + if start_date and publish_date < start_date: + date_in_range = False + + if end_date and publish_date > end_date: + date_in_range = False + + # 检查是否排除今日 + if exclude_today and publish_date == today: + date_in_range = False + + announcement.date_filtered = date_in_range + + if date_in_range: + filtered.append(announcement) + + logger.info(f"日期筛选: {len(announcements)} -> {len(filtered)} 条公告") + return filtered + + +class DuplicateFilter: + """去重筛选器""" + + def __init__(self, use_database: bool = True): + """ + 初始化去重筛选器 + + Args: + use_database: 是否使用数据库检查重复 + """ + self.use_database = use_database + self.db_manager = get_database_manager() if use_database else None + + def filter_announcements(self, announcements: List[Announcement]) -> List[Announcement]: + """ + 去除重复的公告 + + Args: + announcements: 公告列表 + + Returns: + List[Announcement]: 去重后的公告列表 + """ + if not announcements: + return announcements + + seen_hashes = set() + filtered = [] + + for announcement in announcements: + # 生成内容哈希 + if not announcement.content_hash: + announcement.generate_content_hash() + + content_hash = announcement.content_hash + + # 检查是否已存在 + is_duplicate = False + + if self.use_database and self.db_manager: + # 数据库检查 + is_duplicate = self.db_manager.is_announcement_exists(content_hash) + else: + # 内存检查 + if content_hash in seen_hashes: + is_duplicate = True + seen_hashes.add(content_hash) + + if not is_duplicate: + filtered.append(announcement) + # 注意：这里不设置is_new标志，因为这是在爬取后的筛选阶段 + + duplicate_count = len(announcements) - len(filtered) + if duplicate_count > 0: + logger.info(f"去重筛选: 移除了 {duplicate_count} 条重复公告") + + return filtered + + +class SourceFilter: + """来源筛选器""" + + def filter_announcements(self, announcements: List[Announcement], + allowed_sources: List[str]) -> List[Announcement]: + """ + 筛选指定来源的公告 + + Args: + announcements: 公告列表 + allowed_sources: 允许的来源代码列表 + + Returns: + List[Announcement]: 筛选后的公告列表 + """ + if not allowed_sources: + return announcements + + filtered = [] + for announcement in announcements: + if announcement.source_code in allowed_sources: + filtered.append(announcement) + + logger.info(f"来源筛选: {len(announcements)} -> {len(filtered)} 条公告") + return filtered + + +class AnnouncementFilter: + """公告智能筛选器""" + + def __init__(self, criteria: FilterCriteria = None): + """ + 初始化公告筛选器 + + Args: + criteria: 筛选条件 + """ + self.criteria = criteria or FilterCriteria() + + # 初始化各个筛选器 + self.keyword_filter = KeywordFilter( + case_sensitive=self.criteria.case_sensitive, + fuzzy_match=self.criteria.fuzzy_match + ) + self.date_filter = DateFilter() + self.duplicate_filter = DuplicateFilter() + self.source_filter = SourceFilter() + + def filter(self, announcements: List[Announcement]) -> tuple[List[Announcement], FilterResult]: + """ + 执行完整的筛选流程 + + Args: + announcements: 原始公告列表 + + Returns: + tuple: (筛选后的公告列表, 筛选结果统计) + """ + result = FilterResult() + result.total_count = len(announcements) + + # 1. 来源筛选 + if self.criteria.sources: + announcements = self.source_filter.filter_announcements( + announcements, self.criteria.sources) + result.source_filtered = result.total_count - len(announcements) + + # 2. 去重筛选 + announcements = self.duplicate_filter.filter_announcements(announcements) + result.duplicate_filtered = result.total_count - len(announcements) - result.source_filtered + + # 3. 日期筛选 + announcements = self.date_filter.filter_announcements( + announcements, + self.criteria.start_date, + self.criteria.end_date, + self.criteria.exclude_today + ) + result.date_filtered = result.total_count - len(announcements) - result.source_filtered - result.duplicate_filtered + + # 4. 关键词筛选 + if self.criteria.keywords: + announcements = self.keyword_filter.filter_announcements( + announcements, self.criteria.keywords) + result.keyword_filtered = result.total_count - len(announcements) - result.source_filtered - result.duplicate_filtered - result.date_filtered + + result.filtered_count = len(announcements) + + logger.info( + f"筛选完成: 总数 {result.total_count} -> 筛选后 {result.filtered_count} " + f"(关键词: {result.keyword_filtered}, 日期: {result.date_filtered}, " + f"去重: {result.duplicate_filtered}, 来源: {result.source_filtered})" + ) + + return announcements, result + + def quick_filter(self, announcement: Announcement) -> bool: + """ + 快速筛选单个公告（用于实时筛选） + + Args: + announcement: 公告对象 + + Returns: + bool: 是否通过筛选 + """ + # 检查来源 + if self.criteria.sources and announcement.source_code not in self.criteria.sources: + return False + + # 检查日期 + if not announcement.in_date_range( + self.criteria.start_date.isoformat() if self.criteria.start_date else None, + self.criteria.end_date.isoformat() if self.criteria.end_date else None + ): + return False + + # 检查关键词 + if self.criteria.keywords: + search_text = f"{announcement.title} {announcement.purchase_name}" + if not self.keyword_filter.matches(search_text, self.criteria.keywords): + return False + + return True + + def update_criteria(self, criteria: FilterCriteria): + """ + 更新筛选条件 + + Args: + criteria: 新的筛选条件 + """ + self.criteria = criteria + + # 重新初始化筛选器 + self.keyword_filter = KeywordFilter( + case_sensitive=self.criteria.case_sensitive, + fuzzy_match=self.criteria.fuzzy_match + ) + + +def create_default_filter(keywords: List[str] = None, + start_date: str = None, + end_date: str = None) -> AnnouncementFilter: + """ + 创建默认筛选器 + + Args: + keywords: 关键词列表 + start_date: 开始日期字符串 (YYYY-MM-DD) + end_date: 结束日期字符串 (YYYY-MM-DD) + + Returns: + AnnouncementFilter: 筛选器实例 + """ + criteria = FilterCriteria() + + if keywords: + criteria.keywords = keywords + + if start_date: + try: + criteria.start_date = datetime.fromisoformat(start_date).date() + except ValueError: + logger.warning(f"无效的开始日期格式: {start_date}") + + if end_date: + try: + criteria.end_date = datetime.fromisoformat(end_date).date() + except ValueError: + logger.warning(f"无效的结束日期格式: {end_date}") + + return AnnouncementFilter(criteria) + + +def filter_from_config() -> AnnouncementFilter: + """ + 从配置文件创建筛选器 + + Returns: + AnnouncementFilter: 配置化的筛选器 + """ + try: + from ..core.config_manager import get_config + except ImportError: + from core.config_manager import get_config + + config = get_config() + + criteria = FilterCriteria() + criteria.keywords = config.crawler.keyword + criteria.sources = list(config.sources.keys()) # 默认包含所有来源 + + # 日期范围 + if config.crawler.start_date: + try: + criteria.start_date = datetime.fromisoformat(config.crawler.start_date).date() + except ValueError: + pass + + if config.crawler.end_date: + try: + criteria.end_date = datetime.fromisoformat(config.crawler.end_date).date() + except ValueError: + pass + + return AnnouncementFilter(criteria) diff --git a/gx_gp_monitor/logs/gx_gp_monitor.log b/gx_gp_monitor/logs/gx_gp_monitor.log new file mode 100644 index 0000000..f655a34 --- /dev/null +++ b/gx_gp_monitor/logs/gx_gp_monitor.log @@ -0,0 +1,799 @@ +2026-01-07 17:16:51 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:16:51 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:16:51 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:16:52 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:16:52 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:16:52 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:16:52 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:16:52 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:16:52 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:16:52 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:16:52 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:20:31 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:20:31 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:20:31 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:20:31 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:20:31 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:20:31 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:20:31 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:20:31 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:20:31 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:20:31 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:20:31 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:21:02 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:21:02 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:21:02 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:21:02 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:21:02 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:21:02 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:21:02 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:21:02 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:21:02 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:21:02 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:21:02 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:22:10 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:22:10 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:22:10 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:22:10 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:22:10 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:22:10 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:22:10 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:22:10 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:22:10 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:22:10 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:22:10 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:22:44 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:22:44 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:22:44 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:22:44 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:22:44 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:22:44 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:22:44 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:22:44 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:22:44 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:22:44 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:22:44 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:22:46 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:22:46 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:22:46 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:22:47 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:22:47 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:22:47 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:22:47 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:22:47 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:22:47 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:22:47 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:22:47 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 采购公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 结果公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 合同公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 更正公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 招标文件预公示 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 单一来源公示 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 电子卖场公示 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 履约验收公示 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 工程类公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 框架协议征集公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 框架协议入围结果公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 框架协议成交结果汇总公告 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - ERROR - 爬取来源 采购意向公开 时发生未预期错误: 'Logger' object has no attribute 'log_crawl_start' +2026-01-07 17:22:47 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 0 个，失败 13 个，获取 0 条公告 +2026-01-07 17:22:47 - __main__ - INFO - 爬取到 0 条原始公告 +2026-01-07 17:22:47 - __main__ - ERROR - 爬取任务执行失败: attempted relative import beyond top-level package +2026-01-07 17:22:47 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:22:47 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:23:10 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:23:10 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:23:10 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:23:10 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:23:10 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:23:10 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:23:10 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:23:10 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:23:10 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:23:10 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:23:10 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 采购公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 结果公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 合同公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 更正公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 招标文件预公示 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 单一来源公示 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 电子卖场公示 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 履约验收公示 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 工程类公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 框架协议征集公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 框架协议入围结果公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 框架协议成交结果汇总公告 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - ERROR - 爬取来源 采购意向公开 时发生未预期错误: attempted relative import beyond top-level package +2026-01-07 17:23:10 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 0 个，失败 13 个，获取 0 条公告 +2026-01-07 17:23:10 - __main__ - INFO - 爬取到 0 条原始公告 +2026-01-07 17:23:10 - __main__ - ERROR - 爬取任务执行失败: attempted relative import beyond top-level package +2026-01-07 17:23:10 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:23:10 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:23:34 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:23:34 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:23:34 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:23:35 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:23:35 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:23:35 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:23:35 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:23:35 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:23:35 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:23:35 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:23:35 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:23:35 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:24:05 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:05 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:05 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:05 - crawler.parsers - INFO - 成功解析 97/100 条公告记录 +2026-01-07 17:24:05 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 97 条公告，耗时 30.46秒 +2026-01-07 17:24:05 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:24:05 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:05 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:24:05 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.23秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 78/100 条公告记录 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 78 条公告，耗时 0.17秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 0/100 条公告记录 +2026-01-07 17:24:06 - crawler.spider - INFO - 单一来源公示 第1页解析到0条公告 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 0 条公告，耗时 0.18秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:24:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.31秒 +2026-01-07 17:24:06 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:24:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.16秒 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:24:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.27秒 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - INFO - 成功解析 62/100 条公告记录 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 62 条公告，耗时 0.20秒 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:07 - crawler.parsers - INFO - 成功解析 13/100 条公告记录 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 13 条公告，耗时 0.17秒 +2026-01-07 17:24:07 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - ERROR - 解析单个公告记录失败: 'NoneType' object has no attribute 'strip' +2026-01-07 17:24:08 - crawler.parsers - INFO - 成功解析 0/100 条公告记录 +2026-01-07 17:24:08 - crawler.spider - INFO - 框架协议成交结果汇总公告 第1页解析到0条公告 +2026-01-07 17:24:08 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 0 条公告，耗时 0.29秒 +2026-01-07 17:24:08 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:24:08 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:24:08 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:24:08 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 950 条公告 +2026-01-07 17:24:08 - __main__ - INFO - 爬取到 950 条原始公告 +2026-01-07 17:24:08 - __main__ - ERROR - 爬取任务执行失败: attempted relative import beyond top-level package +2026-01-07 17:24:08 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:24:08 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:24:17 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:24:17 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:24:17 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:24:17 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:24:17 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:24:17 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:24:17 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:24:17 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:24:17 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:24:17 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:24:17 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:26:34 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:26:34 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:26:34 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:26:34 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:26:34 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:26:34 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:26:34 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:26:34 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:26:34 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:26:34 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:26:34 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:26:34 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:26:34 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:34 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 100 条公告，耗时 0.48秒 +2026-01-07 17:26:34 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.23秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 100 条公告，耗时 0.20秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:26:35 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:26:35 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:26:36 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.40秒 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:26:36 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.22秒 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:26:36 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.27秒 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:26:36 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:26:36 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:26:37 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 100 条公告，耗时 0.25秒 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:26:37 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:26:37 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:37 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:26:37 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 1300 条公告 +2026-01-07 17:26:37 - __main__ - INFO - 爬取到 1300 条原始公告 +2026-01-07 17:26:37 - __main__ - ERROR - 爬取任务执行失败: attempted relative import beyond top-level package +2026-01-07 17:26:37 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:26:37 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:26:57 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:26:57 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:26:57 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:26:58 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:26:58 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:26:58 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:26:58 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:26:58 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:26:58 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:26:58 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:26:58 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:26:58 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 100 条公告，耗时 0.37秒 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:26:58 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.23秒 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:26:58 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:26:58 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.20秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 100 条公告，耗时 0.16秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.38秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:26:59 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:26:59 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:27:00 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.25秒 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:27:00 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:27:00 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 100 条公告，耗时 0.17秒 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:27:00 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:27:00 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:27:01 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:27:01 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.25秒 +2026-01-07 17:27:01 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 1300 条公告 +2026-01-07 17:27:01 - __main__ - INFO - 爬取到 1300 条原始公告 +2026-01-07 17:27:01 - filters.filters - INFO - 来源筛选: 1300 -> 1300 条公告 +2026-01-07 17:27:01 - filters.filters - INFO - 关键词筛选: 1300 -> 18 条公告 +2026-01-07 17:27:01 - filters.filters - INFO - 筛选完成: 总数 1300 -> 筛选后 18 (关键词: 1282, 日期: 0, 去重: 0, 来源: 0) +2026-01-07 17:27:01 - __main__ - INFO - 筛选后剩余 18 条公告 +2026-01-07 17:27:01 - storage.postgresql - INFO - 开始保存 18 条公告到数据库 +2026-01-07 17:27:01 - core.database - INFO - 批量保存公告完成，成功保存 1 条 +2026-01-07 17:27:01 - storage.postgresql - INFO - 成功保存 1 条公告到数据库 +2026-01-07 17:27:01 - storage.md_generator - INFO - Markdown文件已保存到: onu.md (共 18 条公告) +2026-01-07 17:27:01 - notification.wechat - INFO - 企业微信服务初始化完成 +2026-01-07 17:27:01 - notification.wechat - INFO - 成功获取企业微信访问令牌 +2026-01-07 17:27:01 - notification.wechat - INFO - Markdown消息发送成功 +2026-01-07 17:27:01 - __main__ - INFO - 爬取任务完成: {'success': True, 'total_crawled': 1300, 'filtered': 18, 'saved': 1, 'markdown_generated': True, 'notification_sent': True, 'filter_stats': {'keyword_filtered': 1282, 'date_filtered': 0, 'duplicate_filtered': 0, 'source_filtered': 0}} +2026-01-07 17:27:01 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:27:01 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:29:05 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:29:05 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:29:05 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:29:06 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:29:06 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:29:06 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:29:06 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:29:06 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:29:06 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:29:06 - scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:29:06 - scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:31:58 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:31:58 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:31:58 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:31:58 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:31:58 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:31:58 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:31:58 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:31:58 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:31:58 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:34:32 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:34:32 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:34:32 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:34:32 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:34:32 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:34:32 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:34:32 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:34:32 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:34:32 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:34:35 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:34:35 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:34:35 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:34:35 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:34:35 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:34:35 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:34:35 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:34:35 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:34:35 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:34:35 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:34:35 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:34:35 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:35:05 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:05 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 100 条公告，耗时 30.44秒 +2026-01-07 17:35:05 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:35:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.26秒 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:35:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.20秒 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:35:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.23秒 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:35:06 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:35:06 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.34秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:35:07 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:35:07 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:35:08 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:35:08 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 100 条公告，耗时 0.25秒 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:35:08 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:35:08 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:35:08 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 1300 条公告 +2026-01-07 17:35:08 - __main__ - INFO - 爬取到 1300 条原始公告 +2026-01-07 17:35:08 - filters.filters - INFO - 来源筛选: 1300 -> 1300 条公告 +2026-01-07 17:35:08 - filters.filters - INFO - 去重筛选: 移除了 18 条重复公告 +2026-01-07 17:35:08 - filters.filters - INFO - 关键词筛选: 1282 -> 0 条公告 +2026-01-07 17:35:08 - filters.filters - INFO - 筛选完成: 总数 1300 -> 筛选后 0 (关键词: 1282, 日期: 0, 去重: 18, 来源: 0) +2026-01-07 17:35:08 - __main__ - INFO - 筛选后剩余 0 条公告 +2026-01-07 17:35:08 - storage.md_generator - INFO - Markdown文件已保存到: onu.md (共 0 条公告) +2026-01-07 17:35:08 - __main__ - INFO - 爬取任务完成: {'success': True, 'total_crawled': 1300, 'filtered': 0, 'saved': 0, 'markdown_generated': True, 'notification_sent': False, 'filter_stats': {'keyword_filtered': 1282, 'date_filtered': 0, 'duplicate_filtered': 18, 'source_filtered': 0}} +2026-01-07 17:35:38 - __main__ - INFO - === 广西政府采购网公告监控系统启动 === +2026-01-07 17:35:38 - __main__ - INFO - 版本: 1.0.0 +2026-01-07 17:35:38 - __main__ - INFO - 配置文件: 默认配置 +2026-01-07 17:35:38 - core.reliability - INFO - 健康检查通过 +2026-01-07 17:35:38 - core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:35:38 - core.database - INFO - 开始初始化数据库表结构 +2026-01-07 17:35:38 - core.database - INFO - 数据库表结构初始化完成 +2026-01-07 17:35:38 - storage.postgresql - INFO - 存储初始化完成 +2026-01-07 17:35:38 - __main__ - INFO - 应用初始化完成 +2026-01-07 17:35:38 - __main__ - INFO - 开始执行爬取任务 +2026-01-07 17:35:38 - crawler.spider - INFO - 开始爬取 13 个公告来源 +2026-01-07 17:35:38 - gx_gp_monitor - INFO - 开始爬取 采购公告 +2026-01-07 17:36:09 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 采购公告 爬取完成，共获取 100 条公告，耗时 30.49秒 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 开始爬取 结果公告 +2026-01-07 17:36:09 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 结果公告 爬取完成，共获取 100 条公告，耗时 0.20秒 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 开始爬取 合同公告 +2026-01-07 17:36:09 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 合同公告 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 开始爬取 更正公告 +2026-01-07 17:36:09 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 更正公告 爬取完成，共获取 100 条公告，耗时 0.27秒 +2026-01-07 17:36:09 - gx_gp_monitor - INFO - 开始爬取 招标文件预公示 +2026-01-07 17:36:10 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 招标文件预公示 爬取完成，共获取 100 条公告，耗时 0.28秒 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 开始爬取 单一来源公示 +2026-01-07 17:36:10 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 单一来源公示 爬取完成，共获取 100 条公告，耗时 0.19秒 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 开始爬取 电子卖场公示 +2026-01-07 17:36:10 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.36秒 +2026-01-07 17:36:10 - gx_gp_monitor - INFO - 开始爬取 履约验收公示 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 履约验收公示 爬取完成，共获取 100 条公告，耗时 0.27秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 工程类公告 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 工程类公告 爬取完成，共获取 100 条公告，耗时 0.21秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 框架协议征集公告 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 框架协议征集公告 爬取完成，共获取 100 条公告，耗时 0.15秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 框架协议入围结果公告 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 框架协议入围结果公告 爬取完成，共获取 100 条公告，耗时 0.24秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 框架协议成交结果汇总公告 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 框架协议成交结果汇总公告 爬取完成，共获取 100 条公告，耗时 0.16秒 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 开始爬取 采购意向公开 +2026-01-07 17:36:11 - crawler.parsers - INFO - 成功解析 100/100 条公告记录 +2026-01-07 17:36:11 - gx_gp_monitor - INFO - 采购意向公开 爬取完成，共获取 100 条公告，耗时 0.18秒 +2026-01-07 17:36:11 - crawler.spider - INFO - 爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 1300 条公告 +2026-01-07 17:36:11 - __main__ - INFO - 爬取到 1300 条原始公告 +2026-01-07 17:36:11 - filters.filters - INFO - 来源筛选: 1300 -> 1300 条公告 +2026-01-07 17:36:12 - filters.filters - INFO - 去重筛选: 移除了 18 条重复公告 +2026-01-07 17:36:12 - filters.filters - INFO - 关键词筛选: 1282 -> 0 条公告 +2026-01-07 17:36:12 - filters.filters - INFO - 筛选完成: 总数 1300 -> 筛选后 0 (关键词: 1282, 日期: 0, 去重: 18, 来源: 0) +2026-01-07 17:36:12 - __main__ - INFO - 筛选后剩余 0 条公告 +2026-01-07 17:36:12 - storage.md_generator - INFO - Markdown文件已保存到: onu.md (共 0 条公告) +2026-01-07 17:36:12 - __main__ - INFO - 爬取任务完成: {'success': True, 'total_crawled': 1300, 'filtered': 0, 'saved': 0, 'markdown_generated': True, 'notification_sent': False, 'filter_stats': {'keyword_filtered': 1282, 'date_filtered': 0, 'duplicate_filtered': 18, 'source_filtered': 0}} diff --git a/gx_gp_monitor/main.py b/gx_gp_monitor/main.py new file mode 100644 index 0000000..ab573c8 --- /dev/null +++ b/gx_gp_monitor/main.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +""" +广西政府采购网公告监控系统主程序 +广西政府采购网公告爬取和监控的智能系统 +""" + +import sys +import argparse +import signal +from pathlib import Path + +# 添加项目根目录到Python路径 +project_root = Path(__file__).parent +sys.path.insert(0, str(project_root)) + +try: + # 尝试相对导入 + from .core.config_manager import load_config, get_config + from .core.logger import init_logger, get_logger + from .core.reliability import check_system_health + from .crawler.spider import crawl_announcements + from .filters.filters import filter_from_config + from .storage.postgresql import init_storage, save_announcements_to_storage, cleanup_storage + from .storage.md_generator import generate_onu_md + from .notification.wechat import send_announcements_notification, send_system_notification +except ImportError: + try: + # 尝试绝对导入（直接运行脚本时） + from core.config_manager import load_config, get_config + from core.logger import init_logger, get_logger + from core.reliability import check_system_health + from crawler.spider import crawl_announcements + from filters.filters import filter_from_config + from storage.postgresql import init_storage, save_announcements_to_storage, cleanup_storage + from storage.md_generator import generate_onu_md + from notification.wechat import send_announcements_notification, send_system_notification + except ImportError as e: + print(f"导入错误: {e}", file=sys.stderr) + print("请确保依赖已正确安装: pip install -r requirements.txt", file=sys.stderr) + sys.exit(1) + + +logger = get_logger(__name__) + + +class GXGPMonitorApp: + """广西政府采购网监控系统应用""" + + def __init__(self): + self.config = None + self.logger = None + self.running = False + + def initialize(self, config_file: str = None): + """初始化应用""" + try: + # 加载配置 + self.config = load_config(config_file) + + # 初始化日志 + self.logger = init_logger(config=self.config) + + logger.info("=== 广西政府采购网公告监控系统启动 ===") + logger.info(f"版本: 1.0.0") + logger.info(f"配置文件: {config_file or '默认配置'}") + + # 系统健康检查 + if not check_system_health(): + logger.warning("系统健康检查失败，但继续运行") + + # 初始化存储 + init_storage() + + logger.info("应用初始化完成") + return True + + except Exception as e: + print(f"应用初始化失败: {str(e)}", file=sys.stderr) + return False + + def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None): + """执行爬取任务""" + try: + logger.info("开始执行爬取任务") + + # 执行爬取 + crawl_results = crawl_announcements(keywords, sources) + + if not crawl_results: + logger.info("爬取完成：无数据") + return {"success": True, "results": []} + + # 收集所有公告 + all_announcements = [] + for result in crawl_results: + if result.announcements: + all_announcements.extend(result.announcements) + + logger.info(f"爬取到 {len(all_announcements)} 条原始公告") + + # 筛选公告 + filter_obj = filter_from_config() + filtered_announcements, filter_stats = filter_obj.filter(all_announcements) + + logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告") + + # 保存到数据库 + saved_count = save_announcements_to_storage(filtered_announcements) + + # 生成Markdown文件 + md_success = generate_onu_md(filtered_announcements) + + # 发送通知 + notify_success = False + if filtered_announcements and self.config.wechat_app.enabled: + notify_success = send_announcements_notification(filtered_announcements) + + result = { + "success": True, + "total_crawled": len(all_announcements), + "filtered": len(filtered_announcements), + "saved": saved_count, + "markdown_generated": md_success, + "notification_sent": notify_success, + "filter_stats": { + "keyword_filtered": filter_stats.keyword_filtered, + "date_filtered": filter_stats.date_filtered, + "duplicate_filtered": filter_stats.duplicate_filtered, + "source_filtered": filter_stats.source_filtered + } + } + + logger.info(f"爬取任务完成: {result}") + return result + + except Exception as e: + logger.error(f"爬取任务执行失败: {str(e)}") + return {"success": False, "error": str(e)} + + def run_cleanup(self, days: int = None): + """执行数据清理任务""" + try: + logger.info("开始执行数据清理任务") + + deleted_count = cleanup_storage(days) + + logger.info(f"数据清理完成：删除 {deleted_count} 条过期数据") + + # 发送通知 + if deleted_count > 0 and self.config.wechat_app.enabled: + send_system_notification( + "数据清理完成", + f"已清理 {deleted_count} 条过期数据" + ) + + return {"success": True, "deleted": deleted_count} + + except Exception as e: + logger.error(f"数据清理任务执行失败: {str(e)}") + return {"success": False, "error": str(e)} + + + def show_status(self): + """显示系统状态""" + try: + status = { + "system": { + "version": "1.0.0", + "healthy": check_system_health() + }, + "config": { + "debug": self.config.debug, + "log_level": self.config.log_level.value + }, + "database": { + "enabled": self.config.database.enabled, + "type": self.config.database.type + }, + "wechat": { + "enabled": self.config.wechat_app.enabled + } + } + + # 格式化输出 + print("\n=== 系统状态 ===") + print(f"系统健康: {'正常' if status['system']['healthy'] else '异常'}") + print(f"调试模式: {'开启' if status['config']['debug'] else '关闭'}") + print(f"日志级别: {status['config']['log_level']}") + print(f"数据库: {'启用' if status['database']['enabled'] else '禁用'} ({status['database']['type']})") + print(f"企业微信: {'启用' if status['wechat']['enabled'] else '禁用'}") + + return status + + except Exception as e: + logger.error(f"获取系统状态失败: {str(e)}") + return {"error": str(e)} + + +def create_argument_parser(): + """创建命令行参数解析器""" + parser = argparse.ArgumentParser( + description="广西政府采购网公告监控系统", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +使用示例: + python main.py crawl # 执行一次爬取 + python main.py crawl --keywords "大化" # 爬取指定关键词 + python main.py cleanup # 执行数据清理 + python main.py status # 查看系统状态 + """ + ) + + parser.add_argument( + 'command', + choices=['crawl', 'cleanup', 'status'], + help='要执行的命令' + ) + + parser.add_argument( + '--config', '-c', + help='配置文件路径' + ) + + # crawl命令的参数 + parser.add_argument( + '--keywords', '-k', + nargs='+', + help='关键词过滤（多个关键词用空格分隔）' + ) + + parser.add_argument( + '--sources', '-s', + nargs='+', + help='来源代码过滤（多个来源用空格分隔）' + ) + + parser.add_argument( + '--max-pages', + type=int, + help='最大爬取页数' + ) + + # cleanup命令的参数 + parser.add_argument( + '--days', '-d', + type=int, + help='清理多少天前的过期数据' + ) + + return parser + + +def main(): + """主函数""" + parser = create_argument_parser() + args = parser.parse_args() + + # 创建应用实例 + app = GXGPMonitorApp() + + # 初始化应用 + if not app.initialize(args.config): + sys.exit(1) + + try: + if args.command == 'crawl': + # 执行爬取 + result = app.run_crawl( + keywords=args.keywords, + sources=args.sources, + max_pages=args.max_pages + ) + + if result["success"]: + print("✅ 爬取任务执行成功") + print(f" 爬取公告: {result['total_crawled']}") + print(f" 筛选后: {result['filtered']}") + print(f" 保存数量: {result['saved']}") + if result.get("markdown_generated"): + print(" Markdown文件: 已生成") + if result.get("notification_sent"): + print(" 通知发送: 已发送") + else: + print(f"❌ 爬取任务执行失败: {result.get('error', '未知错误')}") + sys.exit(1) + + elif args.command == 'cleanup': + # 执行清理 + result = app.run_cleanup(days=args.days) + if result["success"]: + print(f"✅ 数据清理完成，删除 {result['deleted']} 条记录") + else: + print(f"❌ 数据清理失败: {result.get('error', '未知错误')}") + sys.exit(1) + + elif args.command == 'status': + # 显示状态 + app.show_status() + + except KeyboardInterrupt: + logger.info("收到中断信号，正在退出...") + except Exception as e: + logger.error(f"程序执行异常: {str(e)}") + print(f"❌ 程序执行异常: {str(e)}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/gx_gp_monitor/notification/__init__.py b/gx_gp_monitor/notification/__init__.py new file mode 100644 index 0000000..3ca4a8b --- /dev/null +++ b/gx_gp_monitor/notification/__init__.py @@ -0,0 +1 @@ +"""通知模块""" diff --git a/gx_gp_monitor/notification/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/notification/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e88f67ca3e552a8156d27f3149eb6d945bf43a12 GIT binary patch literal 200 zcmey&%ge<81U)KonZiK&F^B^Lj8MjB4j^MHLoh=TLpq}-Qz;LS`?6uy^Z84ktyuVU z?sPv*##`+1@hSPq@$oAeK7(}K^3>1B&rQ`YGt19O)i0yFVY7KW+o?=Waj7T$H!*^MdRc33My}L*yQG?l;)(`6|n(L W205x2Tise@lIkBC@d50!A4?8%CX?+p6*dP>JF#=mMR}v(F zP6I8(0ZJT_5QkFYKmcckf>~H4n$ULWKl}TA z_u(s9&I3Br*>mfi<*buo(`lM{hB<7+Ir(nnN*K zMz_V(VrI>dqh<74GFvRHg~Sb8vRbUHmBcf)WVht7IW0EU){@KSk~HI%yq0`6-$8X+ zDyUBTG8h>?Y@wa1pcqpT#h90=#k3-lmI-MVIjxvcKT^VEbrvsGGu9&NA(|~^vO6<6 zty-uA>2sFSOb$seV{DxcCKvuORmbF!)N-bf$%nLpPF<%%uA^h@nEQ~5t>{sgQ=J8R z%2Q2IFk+QrrYgluRm}8ERm{wpsu+2uWmM~9JE(7I%BUSGXHnP<12ta!_{oc7FHd~( z&c)Nmr}n&j>DU{SBgZFxa>O{ffAHcbKUQW>emF4s+iZ^t7Lf(+C&pU++4hhB{Z^+#h@bB>TNMF5yo*pj?U*<-?KM?kJ zdHcNnkW<6wu-*{6%N_8$y*sx;Gga^Ej2P`Vds~LowDYy)e1l18Wh^fglME zU-{9*v&Xwcge9w`fe`<%J^{gDszc6`mZ42VWl~xUTvV@|%24oyEs%#%!51xkbr8aa zUu~IE+Vq=IOWW(9dMKx+mBmKG&GoT!Ae9YSg zvvzy>`+1!J_k5gzE)Vp2>-%^0vw?@aU7`Bs_S&BQT7Mwq+p(*@XQ#WT-`yAR`$7S> zo+SFZJcy?C!g%WXcku?d+Xn^QZazP0YwDzQoM^BPf<4sve8<7wSbl9JzjlCr-V`q_ zAE3t#Me*_qe9yUPux305A`rdiwfRfU=OIn=UWK+6+=``Yrbc!|E_e z>FMh;Dv6oMwTj3n()A^gBB@DuTPQ}|p!TcpR5KdCI(;8)7DWXyI!j6G%#sXUr}k?) z5CO^4mM6ug-;8z|{yXxe^ftQ`n|@bN?UjTeYW%c|ZiD?uedN8=y0mSDsky*b4N_lB z#_D$1H5aT``jSSeBw(^k4Q458nr8J(W-Va5U)P>r3*9Rzgklv+(~Qg__2-xsjub7F zrD-9{rL|BEN@`*)r#c)`Ne~1Uz{vDBXeGO&QmWz7OJhoIw~S(}eto09$11#csAQoa zC%hJF8!x}`vx)sLU4G~E)RA{Co;q=9-}6QzuvEa*;A*1-9*LWG1Vik7s~t%t`AncT zJ9hyw^SQ%p@9ymSt{z|QcAviK_ zH>0#ujWm(%_jU6|d>C zL_S{I?cE;k;SH?!k+3%y3i29Qf1Z9Q81S=*Tf7z)Ai(k&n`F631Ka5|u!xzwxhu@F zAY!?3D7*%Ck_ADO!uEpw0e{eo#5)-7g<w@&=I{B5>ft!lF^Yv`~;sTG_G`E|OXR zNn!9_SnWO3H49}gIkM};T?Zc@Feh}B*%~vI!r$=1*y85M;^t^YbKF)CvsFiI)ttFH zL6>FNFJ#&FKfdqrSXLGMy_!j`IdpEYep*5wtwrs ztz6-{ar5mL3(Af>@!}K18_uXB1seu7$MZ^$+;`x5^mw^m#ztvq(`(R)YCqutTcHDg_o(#An!+_@~`T#xy4FIw}D zm=2h@lJ(=(4e^5FSb;NA;N+}M%rGA?b7iZ>=8oC9QZfI}7qjeFDRqYZyg7#qv3S`B zU1R0%?-*_83fGRC*PXW&{3W4<0$&A@`kd8ltk|@U`o&y3hU<(?OEtgDC~vCQ{IXsP z@hRlMAS|2uJUiWhZmiD>r~{%&2tWdGDBuQQWO)Nb0Ix#3cpbz-o}Mn0>!e5KKLp8?N9w@?%;tt1TJw!L zGz}bD2OL`K*D*Smc9F`jcd1<(7tQEr;lpV-He+z97J+hCr}G;WtCF2EHTYzVE)@g( zj7I4ovq3YxMhd=>JF_0}<{p=Uu`pSyXn%$)qYcSU$|G}Uqsy3d{H8`zk6L&u z_+x%p3qWTSbYMa06|`cbv61Z&nWKm$s~w`k{LIr6?|ekqrieC>^THdGhd#W#=jls_ zMkWV-cE?80q$f_id~xjWA>oOMfoG;(J_Jd(dwP2>A0N4N;vXlT9=Z7QgHj^|{e zLzo`B1X9_#7~rf04G)>D5(LEd$m-lhSp3XfNdzF-GY2psN&qLV>>BI`yI>nIKmmXS z`Hx~3VnEowflLYbK3k7ziy+|X{s8cBGTgV_HiUDHpNi*T?WYO5$F|`qc^%&rq zvMVqkY@P<2jE7x`->WcK4M7l<5W?QYK;-hci*DQvx427u3YVWz;BIVfC7{wCDp4^n zqcC1v7AsyHDP9~cZW!45ueP%DB9|*|8n-sb3rc`HTXS!Pvv20|??BGJc@ToC;a)DU zVW0_ld_}ytGG15~FD{2pkF7bnCRSV@DXzbeTRQYuLZdFsxssWYYZ=&hEni~x#r325 zF?!54nmO|Qu`aG~{kVAp^8W9=_`P9ftoz*hNWq=g=l(bw!pxVCTUP)xPtU_kBpzNk zI}f)L9$xrW5T*R57H+Je-rZ0R;b+-)3~P*=$~2!XE#Fk6`9+Zy;_3PEhyR}+|A+Bo ziN`poo8rL0yT1Vk&Xjs%EH}x4v!oipfvq>-o!NgC4xDpS9M~puVDeUQ;N0*(0pyDu zShC(;hX>0=h(Y?&%RjsYrY!N!+1Ra=pPsj-sLZo6Qg$OuNZ^p{CWx|47!Vd`=o8K9 zBAa73W7ZuQv|z9WgRK~}L!jVl@~nx;p0G0ZPAoUgzEDR^E|$!|P_RmuFS9T#AXG^y zCNi;B_S@Ja1HmkeBaiTfK5>pR^QcT#5z`hL#8#buTLw~{mVuPcWFUq0xikznj5jMrjhsCb0po%uK`U7`%8ITvUXENfhJ&(vPDgR?vN6wHrWy@CsB;1 zL$>C-Xo(8W&;)2d-AMOFI%ASc6Z%qz;0>b{};nAtR`)`(hgwaVd`CpDtzOrxfrBR^|kiReg?ETB9 z_X=YY#8ClRm1C7~xwQ#A5k%M|hM`I@+xWdKIOW{R8Mu`$0`|Mms~}kt0Tsndr&bUz zapBoc2>4uO74+z#`9P2*am$GqDF~5s2%68@9ltOQQJA&#|A4lGy%6l7{%EJHr6^Br z*|&wWuN*h80-I5S&d4adP&qGFxg}D$<=nPtrI#~1KswB|$85C`TP;_&K5E;*nK#Tt zuo-3*8NpCxK33kSqRy&}jmtG>P34V?HD?!VA)bUf=<*?`(*huSn5w7z8u(X&t8Dmy zlHxP%=tyS_kyUf8OJ$=rgLv)Nxpe8A7806zKs`ub;(G17v_!11L0wH7&Lib<=_EvQ zP}_i+ftmkCrGRspC7EUS>C z7=?oU+&X+ny?MZJ=u#_MmFI@@xKrg)`_07c2*yEwrYrM5F&x>Xog{E%nPmlU3`dq5 z!jZi4iglBfs9S)e>$W4pQSPn5(IU#9MW{K*?1Y-5BP-?cTU}Y{oU$$}lP8H!jNr=5 za#>OM-{ur>X_hrDkXGI?8{8@=VTWuahEbs8kn%`eNoo@y4rG-uP%^*OpFPV6>@F+d zT%ju)!lDL+b9tNNx+7PNYnGOarE%QM+-7N`L?KjOpG&8skEPNGU~Y460vgJs@^Jpk zkp<`&GRfwb$f(2S(qE<>y!EY#-g1vFILskJLHCqro#h$}R zqbePVV}}koCO$fS`Hj<)hfYuY<115p_g?wy@zk@EN8S*caLfl?!;eg1CGYmScH>ig#Jrk##jp&v{GD(9bx29*tfC|Mdv>R znf}9zr(Tjehd%mhtGmNMa7i^Go!E<~Kv@-3bYa}jIxd|$cKMy-aPAIH)sazS=eyOb z9buFXq0`-(+S*z&{Tg9uCXakLb>xH9j<2$-zgtUsa*#Q#u4Z^uP1pg|YA!wX(Zt?U z;(`wEoqBC(^4J@fUU}~FyKheZ;OQD(Wpr9t)TVfiKkyijc3v-U?hkM8^#vbvlP{g8 zzaLDxyrvfp9%)0qP_LIaz&7t7u3(cW#-nqeHw6P>w#)1Gd%$0f7H9B52iTCeo7eh6 zUU0Y54+VUFXHKeOU<(DhJ#a=KN!7FkLJxY`lteJ(LrL(@D~j^kUY{Qvk|s3r_X^r6 zpV{pT_V+@gqyp>5!N~I>BhO;JeID@W`+MBM-T-*S-{Ec+RYMUapO|jZ7k=iZM+HKr-?vB>p9k*7-tThp9 z&B$%PwywO8^mQeJDy>PFschST_L`0=C>pAb=FcB6#SNMJ&HKz;!Mbrnqxdn{Hg2#> z@m<5k!@^X4&7k(YEq^e4aP_eI;JSE8`Ecd% zlA)H7yhusi;HG%%y|LCF_#1j6R<<%ywlY@M7%6K!GjF`Ch3jLX8EYPT#)rbguHoj9 zB_qADx~5276Ia`OCUkD=xwTyBy}!10TrpAQb#S!6YU6BkqgnIbt{mx$*4!4&x=pJ8 z{kMZ$LsPWA`JXD!^+j7cBlVqJ-F@7BeO#%3+#0wb%#}>%)+Yb2P2U~zyZKL>4W1|pK>`sgSHn9n14f{G_O3CR~yNz z9cYSI*2gLvB9#rJO=FGG%C%>*BbDn1whrY-&5kQt1V{swVIlCBw_x0`Pzhe=ODl)B z#;TS_s+LEqR{pwl)xK>5O#lqyCM+G+4ecCeN2*5(Mm1dF(qEgGUC~f;R>Mey7L$P3 zH4exnKzdU8_fPD6>xtK%c>V8U^VUY@t>xCWMdvZx+`G8DTwLDWSUVY9nA03eOk}=tNv(__WIp3|W1jcJLD%`TlF)PeuI>GMcg2@TSFJd*D zwt(pbMu*Ht@Us2@V!U}rxEI-!!u*AcFEA0o{$(fjui&mAaLpt`8$UOhrOa0DTL3b_ zzk^nhj~KzFnpHfs8&4jUR}OC)UNKTN^4N%vD_aS45>FE?jub2&T^KD`$6421sF*ig zFhY;qGja!4wp=Vw8!4zA>5Uew;jC+J#R`Ucz*BQpQO65bQ6Cu_>ojLI<&93wS*I4_ zt$ZdqbSB&S#m{bU$m8n`CYi}lL?bA30wQo}M1?>Pr)dC9WR({LMHd{!N5o+?4Vqf8 z=t91R`9{XH=@YHcDwu1;R&Qv7cIv}V!B9n@pE`8(^1JWO1p1`;1Mrh#q5}&7LdX{v zM9LS0x0{9OFe-W=W&Z9b&pSjoCcPMqH8nzig3wu01F|Lp@N|YKCDyDqKJdT;#@#f& z!g$UIil96Wx`il%vO6G^?Sa5a6M2&LVL|`{v<*Wl7>_7+0E2xP;8Berk|CU?0QCg~ znuRGVO~rh58laz9>~4hl2cSiO`FtvG?O5NKo6FfYpr16^<7OLIxNNj<)O}_XSI`nM zw+u92Fl2Fg^WV0=Tk`iM|1c-!yglN)ool=+>U44Q@8<5lhqJXu4DADYpzyIQXC%ux zGB297gflE5G(P<)^;e+#n>gDB(95W{s)(mTRvblAtez&$>HHjwx&sv=^0;I*cLpg| z`PGf;9$+i@R*+&kjC#0n^1p%#aQO1+SK;hYT0j!ACWSLTaIki4%-Bj)ilk$%q`uIW zL>8OC?!N}QNJ|hk1PVZlz*`K0rdwQ)0t4xF#RRP81+`XOo2*l%EM@hO{@-%PRh3D zbXN>2aP(>FOZJR5Bq=G6WG%YR3{d?_d1jfbjAGNp<+X)YpjO^=juewd!077?Rpjm# zzMY0Jcrv6z@ss58(WPfUoZRzd3L4?XHarl&&;gg*;8eV_s6cRyyiT}T(>jHVL(CGy zIMC!Y@On56|^WNe`AS=Nn#XTT$ER4^ZMj<$g1=s3K z!kMwd*oNGY;z&6$0c7rA1AV9)gSOk(&u938fgR)~oN(4`KUT(r0>Z5#foA96k!^9T zZq$;)LEe#!v@=WWM7;eKv;=r-rR)xnL5@6r;PF^NO{AbEzGmI&H6N{st!ah7!O(QZ zNW;i_uAqLj`tyPn++FQ)d&x1=QB%ymFk)W_hq+^g3nGOJMnTP75x;+X?EX;X{t)-r z&glI+PZ(m3<)1s2kA`9^wnbKKi#l3kj`oP7ecW+xf}*~!+Mt#XgV#k0>*A}vb-MVY z;@GOK@HZ5Sl{G}l8lq*(u4HFejR`6v!w92hRPCzi1i})W8pVntL^(CPX*{(_?h%r} zshUFi;#857hbL97&LX^2g$5J2RAqF-1+4CHFUHMo_ak9Xub7kNcJJ`9!4O(H`~k?; zyWQP^F1MS->nALdB66lrAiao_42U@FLm9Q+-+J{$;Y&ia{YZOGUbgC$w{itbF6|pagY8x)huZueu#OFD$=vQb9 zM{EfSUh#_Rga+OhD(5D&@CGFgQm_ET>Km?M?5c*+ElsE)c7;u8a^aGAZGC(}tz^q2 z6m|qRqYM7!M-myXz6ME$sT|7+&s7$E6oR`t*fRxXP(+WnYB$ z0LKZWW1P>BC|@v@E)ullOVZ3izAE_w5<$dXyee6FFo2gNFPL)&yN^74;9)p>T4Wrc ze`FE%6a5UV1A;CkE0fI#CD7Di3S309F?(e#Xa%3^3LNE>a6dyRniQs!NnNso9-PxE z)HhseV41Y?E9FeiV${uUbeFaRjs4`AKHZZn>7toL=n>B8(|%1Od}GfTQ%Jqqa6dF7 z>SBU5e5Nj@v?&Ku+#t!v;HP;zl2>5v6N5h#_`7hb=F+KyldryFBmhIK2uh(@G-HU@ zznWU%iWLHrf#{1(u`(zU!Z>IfNjI z^dIE6=Sc zPHg0#p%DXVa-9iwl<TMY$bVhjsJJ!UXwQMeDM*gsLF)v;4h_JlR{50%JLnr=hem-NvcowAJ7}p7 zzz!{*S#yz|XPjQ!1%vymn&)nWFXTFigeL9KyMQYWh$(R;o!u=R#t`Wx9ma@f(55YL z?_gi#`U3JNL>UohunB(QtT>a#7WXz{%OF%E_6Ee33#P2=O{Tw#%epG(_IN4%!SryMncDR$g#vG9nb9SX^hWI3>bGH|y#s98h>)o|JcG(tS> z(x20wbG=iQZuK_6UpxgFgdcM7!&T2Js4so(07+D7fe@q#rE%dF><~t9V9THeDl`- zY5A2Uwz)I1xpRE;{ny~4m4)AY|9jsX4Gq$<{6&%cMbZ2vpIetiHsAjT%uHx#)V%2o z1!*>vg_py>j%HUKB<0c~1tB?w(3y#5y9me1|2Crf8ItD~nmQ`Su#x&e4_5=nG zU_dwtPE=5ai3&W1pKoID76!z_M|9qk_=y4PCBXv_L7lho4>YYcCd~QT0$Fc_ILF+> zwRdp$dAN2@gxP+Lg0#dU2v8x!vY-&c>&(V;wsYO*3eVn2N?gGbs1L$R)(0UD`XJZy z5ZBw!_4G%!Kk}u}nn`OM+Bj?*?i#iq1$_`ch0&6MI$(TV@Gn2872s5_E`mV$+bUgP zkLti2@<`zPh~n2*z!0ViEg*=R^2-Ko<>XWpCoIWRb!U@kM4UE zKK|`0QuL<`1*#V^3RK{aLf8;AC5;DueoXhMrx$)11IV7BUP%CloG+R3mEB-)?6088 zUl8h+Lj<5Mm&%$Kwanks9Jl1fEK4JnrK9&nEgSYU!aag-27JS@z(LGrBO?B z)YQz;%>qcuVr!Dxe`B}=7ic=TY?psx2vqN6xYT?NxZDS``*&0t{&tYco`#tTMCJ^{ zlt9XU3JLrI0s(1Mf;4bh1aQfIh7JA#0txO2oB1WC{&yi&-fH$7rV^;5AqMh> zL9P@JfJlDvZ{RooTM)z>f>|X2GhwQpUN0L;16~W#f)^3&mEa|B_-x?CTT?q@IoO8M zSz)a9M2@e$>bdKL~|C0x5HMU^NsE2T&nL`0+ay{b*teI8wa5LBZEyC1qG|ug|mH*Xs-I zVr35rYr!hHLHN--urSy4bxVgd11yWyN8WTtUq75--Gsl6BuJLXWCSDdNsOJsfH0F2 z7(?PO2$clKaS>DH6~k?UmuD^eFOUn2f(rf-f`mq;Qe9DNRPfb* zYT*}DX@WMXHmJU!7kxp`|AMahE&c5X{p~O4jaPCgWA5PQXh!KC-Jf(1X;sE66o!Ag z*VL)P4-6BxYpB9fj>=2ujH+_3V%aqc-iacMYO`u|>6kgvuqi>|$GNI& Optional[str]: + """ + 获取访问令牌 + + Returns: + Optional[str]: 访问令牌 + """ + current_time = time.time() + + # 检查令牌是否仍然有效 + if self._access_token and current_time < self._token_expires_at: + return self._access_token + + try: + # 构建请求URL + if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'): + url = f"{self.config.proxy_api_url}/cgi-bin/gettoken" + else: + url = "https://qyapi.weixin.qq.com/cgi-bin/gettoken" + + params = { + "corpid": self.config.corp_id, + "corpsecret": self.config.secret + } + + logger.debug("正在获取企业微信访问令牌") + + response = requests.get(url, params=params, timeout=30) + result = response.json() + + if result.get("errcode") == 0: + self._access_token = result.get("access_token") + # 提前5分钟过期 + expires_in = result.get("expires_in", 7200) - 300 + self._token_expires_at = current_time + expires_in + + logger.info("成功获取企业微信访问令牌") + return self._access_token + else: + logger.error(f"获取访问令牌失败: {result}") + return None + + except Exception as e: + logger.error(f"获取访问令牌异常: {str(e)}") + return None + + @retry_on_exception(RetryConfig(max_retries=3)) + def send_text_message(self, content: str, + to_user: str = "@all", + to_party: str = "", + to_tag: str = "") -> bool: + """ + 发送文本消息 + + Args: + content: 消息内容 + to_user: 接收者用户ID，多个用|分隔，@all表示全体 + to_party: 接收者部门ID，多个用|分隔 + to_tag: 接收者标签ID，多个用|分隔 + + Returns: + bool: 发送是否成功 + """ + try: + access_token = self._get_access_token() + if not access_token: + logger.error("无法获取访问令牌，发送失败") + return False + + # 构建请求URL + if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'): + url = f"{self.config.proxy_api_url}/cgi-bin/message/send" + else: + url = "https://qyapi.weixin.qq.com/cgi-bin/message/send" + + params = {"access_token": access_token} + + data = { + "touser": to_user, + "toparty": to_party, + "totag": to_tag, + "msgtype": "text", + "agentid": self.config.agent_id, + "text": { + "content": content + } + } + + logger.debug(f"发送文本消息: {content[:100]}...") + + response = requests.post(url, params=params, json=data, timeout=30) + result = response.json() + + if result.get("errcode") == 0: + logger.info("文本消息发送成功") + return True + else: + logger.error(f"文本消息发送失败: {result}") + return False + + except Exception as e: + logger.error(f"发送文本消息异常: {str(e)}") + return False + + @retry_on_exception(RetryConfig(max_retries=3)) + def send_markdown_message(self, content: str, + to_user: str = "@all", + to_party: str = "", + to_tag: str = "") -> bool: + """ + 发送Markdown消息 + + Args: + content: Markdown格式的消息内容 + to_user: 接收者用户ID + to_party: 接收者部门ID + to_tag: 接收者标签ID + + Returns: + bool: 发送是否成功 + """ + try: + access_token = self._get_access_token() + if not access_token: + logger.error("无法获取访问令牌，发送失败") + return False + + # 构建请求URL + if self.config.use_proxy and hasattr(self.config, 'proxy_api_url'): + url = f"{self.config.proxy_api_url}/cgi-bin/message/send" + else: + url = "https://qyapi.weixin.qq.com/cgi-bin/message/send" + + params = {"access_token": access_token} + + data = { + "touser": to_user, + "toparty": to_party, + "totag": to_tag, + "msgtype": "markdown", + "agentid": self.config.agent_id, + "markdown": { + "content": content + } + } + + logger.debug("发送Markdown消息") + + response = requests.post(url, params=params, json=data, timeout=30) + result = response.json() + + if result.get("errcode") == 0: + logger.info("Markdown消息发送成功") + return True + else: + logger.error(f"Markdown消息发送失败: {result}") + return False + + except Exception as e: + logger.error(f"发送Markdown消息异常: {str(e)}") + return False + + def send_announcement_notification(self, announcements: List[Announcement], + max_count: int = 20) -> bool: + """ + 发送公告通知 + + Args: + announcements: 公告列表 + max_count: 最大显示数量 + + Returns: + bool: 发送是否成功 + """ + if not announcements: + logger.info("没有新公告，跳过通知") + return True + + try: + # 生成通知内容 + notification_content = self._generate_announcement_notification(announcements, max_count) + + # 发送Markdown消息 + return self.send_markdown_message(notification_content) + + except Exception as e: + logger.error(f"发送公告通知失败: {str(e)}") + return False + + def _generate_announcement_notification(self, announcements: List[Announcement], + max_count: int) -> str: + """ + 生成公告通知内容 + + Args: + announcements: 公告列表 + max_count: 最大显示数量 + + Returns: + str: Markdown格式的通知内容 + """ + # 按日期分组 + today_announcements = [] + other_announcements = [] + + today = datetime.now().date() + + for announcement in announcements: + if announcement.publish_date and announcement.publish_date.date() == today: + today_announcements.append(announcement) + else: + other_announcements.append(announcement) + + lines = [] + + # 标题 + total_count = len(announcements) + lines.append(f"# 🔔 广西政府采购网公告更新") + lines.append("") + lines.append(f"**发现 {total_count} 条新公告**") + lines.append("") + + # 今日公告 + if today_announcements: + lines.append(f"## 📅 今日公告 ({len(today_announcements)}条)") + lines.append("") + display_today = today_announcements[:max_count//2] + for announcement in display_today: + title = announcement.title + if len(title) > 40: + title = title[:40] + "..." + publish_time = announcement.publish_date.strftime("%H:%M") if announcement.publish_date else "N/A" + lines.append(f"• [{title}]({announcement.content_url}) - {publish_time}") + + if len(today_announcements) > len(display_today): + lines.append(f"• ... 还有 {len(today_announcements) - len(display_today)} 条今日公告") + + lines.append("") + + # 其他公告 + if other_announcements: + lines.append(f"## 📄 其他公告 ({len(other_announcements)}条)") + lines.append("") + remaining_slots = max_count - len(today_announcements) if today_announcements else max_count + display_other = other_announcements[:remaining_slots] + + for announcement in display_other: + title = announcement.title + if len(title) > 40: + title = title[:40] + "..." + publish_date = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A" + lines.append(f"• [{title}]({announcement.content_url}) - {publish_date}") + + if len(other_announcements) > len(display_other): + lines.append(f"• ... 还有 {len(other_announcements) - len(display_other)} 条公告") + + lines.append("") + + # 统计信息 + source_stats = {} + for announcement in announcements: + source = announcement.source_name + source_stats[source] = source_stats.get(source, 0) + 1 + + lines.append("## 📊 统计信息") + lines.append("") + for source, count in sorted(source_stats.items()): + lines.append(f"• {source}: {count}条") + + lines.append("") + lines.append("---") + lines.append("") + lines.append(f"*更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*") + lines.append("*点击公告标题查看详情*") + + return "\n".join(lines) + + def send_system_notification(self, title: str, content: str, + message_type: str = "text") -> bool: + """ + 发送系统通知 + + Args: + title: 通知标题 + content: 通知内容 + message_type: 消息类型 (text/markdown) + + Returns: + bool: 发送是否成功 + """ + try: + if message_type == "markdown": + full_content = f"# {title}\n\n{content}" + return self.send_markdown_message(full_content) + else: + full_content = f"{title}\n\n{content}" + return self.send_text_message(full_content) + + except Exception as e: + logger.error(f"发送系统通知失败: {str(e)}") + return False + + def send_error_notification(self, error_message: str, error_details: Optional[str] = None) -> bool: + """ + 发送错误通知 + + Args: + error_message: 错误消息 + error_details: 错误详情 + + Returns: + bool: 发送是否成功 + """ + content = f"## ❌ 系统错误\n\n**错误信息**: {error_message}" + + if error_details: + content += f"\n\n**错误详情**:\n```\n{error_details}\n```" + + content += f"\n\n*发生时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*" + + return self.send_markdown_message(content) + + def test_connection(self) -> bool: + """ + 测试连接 + + Returns: + bool: 连接是否正常 + """ + try: + token = self._get_access_token() + return token is not None + except Exception as e: + logger.error(f"企业微信连接测试失败: {str(e)}") + return False + + def get_service_status(self) -> Dict[str, Any]: + """ + 获取服务状态 + + Returns: + Dict[str, Any]: 服务状态信息 + """ + return { + "service": "wechat", + "enabled": self.config.enabled, + "corp_id": self.config.corp_id[:10] + "..." if self.config.corp_id else None, + "agent_id": self.config.agent_id, + "has_token": self._access_token is not None, + "token_expires_at": datetime.fromtimestamp(self._token_expires_at).isoformat() if self._token_expires_at > 0 else None, + "use_proxy": self.config.use_proxy, + "connection_test": self.test_connection() if self.config.enabled else False + } + + +class NotificationManager: + """通知管理器""" + + def __init__(self): + self.wechat = WeChatService() + self._services = { + "wechat": self.wechat + } + + def send_announcement_notification(self, announcements: List[Announcement]) -> Dict[str, bool]: + """ + 发送公告通知 + + Args: + announcements: 公告列表 + + Returns: + Dict[str, bool]: 各服务发送结果 + """ + results = {} + + # 企业微信通知 + if self.wechat.config.enabled: + try: + results["wechat"] = self.wechat.send_announcement_notification(announcements) + except Exception as e: + logger.error(f"企业微信通知失败: {str(e)}") + results["wechat"] = False + else: + results["wechat"] = None # 未启用 + + return results + + def send_system_notification(self, title: str, content: str) -> Dict[str, bool]: + """ + 发送系统通知 + + Args: + title: 通知标题 + content: 通知内容 + + Returns: + Dict[str, bool]: 发送结果 + """ + results = {} + + if self.wechat.config.enabled: + try: + results["wechat"] = self.wechat.send_system_notification(title, content, "markdown") + except Exception as e: + logger.error(f"企业微信系统通知失败: {str(e)}") + results["wechat"] = False + else: + results["wechat"] = None + + return results + + def send_error_notification(self, error_message: str, error_details: Optional[str] = None) -> Dict[str, bool]: + """ + 发送错误通知 + + Args: + error_message: 错误消息 + error_details: 错误详情 + + Returns: + Dict[str, bool]: 发送结果 + """ + results = {} + + if self.wechat.config.enabled: + try: + results["wechat"] = self.wechat.send_error_notification(error_message, error_details) + except Exception as e: + logger.error(f"企业微信错误通知失败: {str(e)}") + results["wechat"] = False + else: + results["wechat"] = None + + return results + + def get_status(self) -> Dict[str, Any]: + """ + 获取通知服务状态 + + Returns: + Dict[str, Any]: 服务状态 + """ + return { + "services": { + name: service.get_service_status() for name, service in self._services.items() + } + } + + +# 全局通知管理器实例 +_notification_manager = None + + +def get_notification_manager() -> NotificationManager: + """ + 获取通知管理器实例 + + Returns: + NotificationManager: 通知管理器实例 + """ + global _notification_manager + if _notification_manager is None: + _notification_manager = NotificationManager() + return _notification_manager + + +def send_announcements_notification(announcements: List[Announcement]) -> bool: + """ + 发送公告通知 + + Args: + announcements: 公告列表 + + Returns: + bool: 是否至少有一个服务发送成功 + """ + manager = get_notification_manager() + results = manager.send_announcement_notification(announcements) + + # 检查是否有服务发送成功 + return any(result for result in results.values() if result is True) + + +def send_system_notification(title: str, content: str) -> bool: + """ + 发送系统通知 + + Args: + title: 通知标题 + content: 通知内容 + + Returns: + bool: 是否至少有一个服务发送成功 + """ + manager = get_notification_manager() + results = manager.send_system_notification(title, content) + + return any(result for result in results.values() if result is True) + + +def send_error_alert(error_message: str, error_details: Optional[str] = None) -> bool: + """ + 发送错误警报 + + Args: + error_message: 错误消息 + error_details: 错误详情 + + Returns: + bool: 是否至少有一个服务发送成功 + """ + manager = get_notification_manager() + results = manager.send_error_notification(error_message, error_details) + + return any(result for result in results.values() if result is True) diff --git a/gx_gp_monitor/onu.md b/gx_gp_monitor/onu.md new file mode 100644 index 0000000..ecb015a --- /dev/null +++ b/gx_gp_monitor/onu.md @@ -0,0 +1,11 @@ +# 广西政府采购网公告监控 + +**更新时间**: 2026-01-07 17:36:12 + +## 无新公告 + +当前时间范围内没有找到符合条件的公告。 + +--- + +*由广西政府采购网公告监控系统生成* \ No newline at end of file diff --git a/gx_gp_monitor/requirements.txt b/gx_gp_monitor/requirements.txt new file mode 100644 index 0000000..c467090 --- /dev/null +++ b/gx_gp_monitor/requirements.txt @@ -0,0 +1,39 @@ +# 广西政府采购网公告监控系统依赖包 + +# 核心依赖 +requests>=2.28.0 # HTTP请求库 +psycopg2-binary>=2.9.0 # PostgreSQL数据库驱动 +PyYAML>=6.0 # YAML配置文件解析 +python-dateutil>=2.8.0 # 日期时间处理 + +# 爬虫相关 +fake-useragent>=1.1.0 # 随机User-Agent生成 +lxml>=4.9.0 # XML/HTML解析（备用） + +# 调度器 +schedule>=1.2.0 # 定时任务调度 +croniter>=1.4.0 # Cron表达式解析 + +# 日志和监控 +logging>=0.4.9.6 # 日志处理（Python内置） +colorama>=0.4.6 # 控制台颜色输出（可选，用于彩色日志） + +# 数据处理 +pandas>=1.5.0 # 数据处理（可选，用于复杂数据分析） +openpyxl>=3.0.10 # Excel文件处理（可选） + +# 加密和安全 +cryptography>=39.0.0 # 加密库（用于微信消息加密） +pycryptodome>=3.17.0 # 加密算法库 + +# 可选依赖（根据需要安装） +# redis>=4.5.0 # Redis缓存（如果需要） +# sqlalchemy>=2.0.0 # ORM（如果需要更复杂的数据库操作） +# celery>=5.3.0 # 分布式任务队列（如果需要） +# flask>=2.3.0 # Web框架（如果需要Web界面） + +# 开发依赖（仅开发环境需要） +# pytest>=7.2.0 # 测试框架 +# black>=23.0.0 # 代码格式化 +# flake8>=6.0.0 # 代码检查 +# mypy>=1.0.0 # 类型检查 diff --git a/gx_gp_monitor/scheduler/__init__.py b/gx_gp_monitor/scheduler/__init__.py new file mode 100644 index 0000000..d56023a --- /dev/null +++ b/gx_gp_monitor/scheduler/__init__.py @@ -0,0 +1 @@ +"""调度模块""" diff --git a/gx_gp_monitor/scheduler/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/scheduler/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e6d1dd42fc6c3eeaf3f6b3393211d7de01f5ad0a GIT binary patch literal 197 zcmey&%ge<81U)KonZiK&F^B^Lj8MjB4j^MHLoh=TLpq}-Qz;LS`(i`$(_PD+tyuVU z?sPv*##`+1@hSPq@$oAeK7(}Ka?{Vq&rQ`YGt19O)i0yFVZhg&PYuu%}FiNkB`p;3dP6k6;$5hu*uB_iP{yh0gVN@ Us2Jp$56p~=jJFtsidcXg0D28LH2?qr literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/scheduler/__pycache__/scheduler.cpython-313.pyc b/gx_gp_monitor/scheduler/__pycache__/scheduler.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b74d244c56c2463640050d67f67501ff61533a24 GIT binary patch literal 14537 zcmcIrdsG|OnZKhK(r5qy0>ncgZ2XWPJnCmKb_2%P1{){HbkjhpDj+Z}NO&YUaaw0d z+BC*_fRng}yo~!uNH!1J?P)o+lh{t$&F*EOZ_X-#@w-(=toO)SsSJXN!iHv)~e znQP*B&PGazL+40qPH)QKGk`+p&^Ko`nRt_pbeI;Aj_h@e*atqxLKYE+p^!M(b*lI` zvqN5a%6TGItsKv4iqM&*w65VJ*qO&k(EYV znS{_@CE8~a?U$04`fjpWT}rwcdv>4=Dq*iq{y6%FSE6GRvlFMHFP&g#hes~I`$F`s z=Vwp+?(*r=(W9s06h}{9{qZ}Woq2NhxwmJZe58~2dajyU*{kYX$vQS6`p&{yw*$Y3S|r3+g)0 zAwj=qz~AfjxcUUmu3n$t&Io#Jq}T7}ui@;#2e!xUcXoO`-Mu~1Tc5Y5$IW+QgHq7P z6E{DC#BtI=U}*_1E3bB_c#T616RCA*Al5mw9ceI)Iy%w%_*)u%GdT1etiu5R*^V^G zF|O4+j5LR%Ib27&A_tZTmMVkhAgu3kkw@Pwj(onLC)-{ir0;e4e&Fcb=k5yh!3~a6&^Mj*)<#-oI_Vp;%sjoKD1OM!*mPWgQ=YUEu_@wC& zbRw4sdM6+7czQiOLYmXRk9WJe1g&$w_kN$i;=}3lyZk;O&Fw*M>=N>veO^}=C2)50 z-u^hVy~YR{pS!QyhZANKwpZ@+_PZ+&uJ`u2D+dk@@ZSCIPJiY0dsg=htoC^Qz1@c@ zdmeQ53^@Cdf4zLA+_6NeVgOj)>Ffmpr&GwAx0)5w_abP|w;Q5Caxu^LY+pEURVZ)O z5zUjvh{Xzz^hkaYJh(`1K0Me+VKF>1V>-f_hBpmu3L8t{FT$C^+_Dh2>;h+xsR>tl z6DlP07y}oIGR8(2;c=XJBuNUXm?MYm7$n5Qv@xyn(kF@`qDsgnEo>Js)9zweTtHbY z;!#0LLsRNDl|x%|4arf zJpU6>tgrm~boAvXYHTfbJ)Z;Ff`;<48gNm_f=b)~w_;t$!$OYyNvx87)QONOq&?u` zsbp}jt}Y<-dNA9r<#7-Ki_#$p5vFOM4V29N-~caZx&xk0AFi7X0}o3|_yR0ip2W*3 zMQSNf`bHocB%cUFhBGUtGbaJ}2SddbBL-k`bMffm>6}%OqSA0tb*QL%x@gU@hDfe;ykt5T%D~q(p`ta@MeB|= zTs9Y;XgSsrTv9uA&s0-z@%9<>j*rd7pPI~JQ(4GVHokn)|Bg2GRQ1i=PW2d0oRemP)JAR9hKkGv2){f8?@-iQwWOkA!`_$mk7mT=8(9)wDc zgO<=PXSK$;N&Wx{2b17M&?r=(lc<{19^f=cYnnApS`S0h+Gfy8JSv*jHG^*AQPXr< zGuveJXlx+Oi-;@<+BU5$iFst9J{S^x=pmKFqifTG#8-pVrnRND7ApHIrscX(%Ax)? zU90t0s?=o@ZS7to%2b<9u5aPKvWfCNfG&zL>)Z6LxT;B^O(&HIKyL@0Utm3q?PXF0 ziIN$_!70AW^95LDa-PT7rdGZ~zCjwNOAm9VnV-8rztU5;@5pE~Lhmxy zXcPVDA!Vc;F^C4x78g-%QLo@Fy@IzuA(DAZ??~{L9<K`z2Cd;^{E8FO%NdIH{-ekr?}x;!2)K%Kka?eY8Ib9Q%cALyh? z{Q$mNeXfJ<`Ls^I7xW7+@9J>_L1vHJMgbauHWCNB>KyrTp9VPEno&R+w4oH#OlB*xcuHr(KknJ0aWua zBVbEeQGCvS`Rl8{{Z)+(=DiL*#dX-M4WV^}OK-R@YCj-WpuhaJJ#!bAr1&r+0P*cesPa-7}^hsHLX z;utZ)OO`EYTR&sk@C9fnD?T=_39fCLF*nC63zppZK~8X6`*dAL@Xq^!_ZcNa|2CuT>`?1&lh``2@O8Dh5H zB%1i7a>uw}>9;>npRIdeH}zo9x?{$)Gis{;*klWqZ93yPn|1m-VN*lM)bMr8fMve& z?Si#=G}q(A6k8b#4XxY~&~8xBA{@{F^{#}?T&);Y3d`KVw&P6F&?b*;{$@@$XLX&LZqD#j zY|h48H|JZ|PTHI^<5rTi!-{()?q0W+k@w2zk@0I|zqAERK!%&t)%PBseetAN0QMpa z-HUvOe13PoQwjc5y;s&N))+vOGWz@9h`eQ^Tb`0i74SI zN}YQ*bIH;Q7@4~|Vp$a_EFZsbd{1!A)}W=~bFDfzFJ>g>C11|vP~2mng-^`&F*PQ> zp0kkRWj6@}ReWN(3qC^P>$waHeoVZ3w}4;Rv@>Md`87}j#aF)jU^tKF*6YZquC9>0 zycAOBw7R+~a?Y5J=|WT8YW2AlDon3pFuhuh=_*sbT7912_wy<%%)b^dneTbB~mz95{7CkD$YqE zfdhbJ=-|PBYWSzA(1Du?P3ff2#^6TLE@MwBAD|;&V6OkOTBmc{FvJ9;5hFpcfZ721VJ~--FA19Ni?M4l7~+ z%hvHdz8V(wDs-gO*u?MAk*BV_I!Qmvo|PIKCS~6`%|u^|T;}n20AyIUm3k=RmEQ05 z_SM+L#%D+0j*h$n)N%kRaRaH$@RSU&OyN+@Vk1#3dJ)o(gdsNW(xqM>E0IzYC z&?xHUR6nHZzF4_(R5iA*RJP>;W8WJ<$Iq<>%^wof zU9Lld%HTL7d&VBMNpaA{5RSHl>^E1myePgb47> zl4mHm%$Y~NJHwUE=}3C+aLdt_X|6cT*+QHx%qjAs;psUa6bA#uu%akO53)6^>CKP6AJk%~NxmDPe`KO^K-%Ajoz-OVznNiGx zMJuUbuMKf)!`y}tw_ysz8*EY(@8VVeH}MW`*dAQJ<3ml*vU`SW`8x!>tgf7#D`)GA z)#ob8>+{v;^R^+b;PVO@ju}Y=0X9G645z#e{i6 z7qnE)a8>R}HvObpTY!>ze(u%5SQUGgHdk6HjPu32BPS(K^#H}fGFI))> zHs!pe>b2~w$6njhKF@?g(UG^KM$27^4!E;xT&D?yLdcB({;Iki9EMZicQ&#ycyaRHIx;YLW7qc>rsmit~t;{W;}4# z?G?|PGf>BJN?H%k<0J@LaKiC^9#2rjLuK*&>>-GsVdIc7XkLCnZ@-o4W%RiLUAo{|FpJ$ z)#2hbq2e`gD*$!QFlP;M)^S%5pUW3mcAP$nb@WV}*R!leV~ABF{61}5AMtuHVVCH82Obj~`kFO}ZDR63_x zp>4{?y7M>I1;-?7sq}yg4u9*I7O#Uo6`E-pJw6c;PiD-^Q>UZvzJBHDx5Z59!k7G6 zJPx~tb4D*6$OXlEqGukBPECO(_%IN(+@=mZMtbQaXnuAe61`iL$GhNFFd_mDz^Mr0 zqK4iF)B5_{?tx@SSKM_>Ejq*p$(%ccwG_|i{Q0nIfZqFl3}k~uX5xw&edVVa*~9(A z{V|m`z2u_F5;m3p*;GDbT6)oBKEXc2j#fXH9!n!2H9$1fKW8F_OcD4hF6b-2oHI}l zlM2h`ouCe;^}qE^RsZwB$$dfVw`WXu%JrN`KbDSWT6)P>KJ+6@tgNdg=W5xlY3lR3 z4O>;}_f=X*xA3bWbDk0Nz_=A9W&A+Cn7UfJ+X*y&lkwQt2!S?+{%70e#B|IMc%dF37k9 zwkYV{9`twVZ!>p^xkxoGaV`w)sO8X*u&r}lGgdR5xdiSaVQPF|sHk#8FP(jsd~0&!qJ+VzoX@eMU8sQPj)lUT~Y9To4%ebeTZ69+=(DtKaZyN>M|-!QRpV#Va1 z54L{Dyx(|sW3X&j*t|Pr-i^2i#YF$qdv*G)R`R};ZCIp!e_3^dRsDfg3+Z`Bceuk- zO=1?N?Ks>Wg)4-qr8R|SAxSGpDIg@hyt5cgRdq^K0rZlthTMh%&dL^`kd;3fnM(nC zQ@1D6rt*+UJ>SCs+NCa^CSlMwXiKJDs4a)SLVHg(V({)4;#aFkDlMy3Mlc|a2UwRB zS{8i8HB1VrtVTH3G*oEdmK+>Eb2OmjtXXCB#cDNz2$_6*X z@2jziM3>(?HT%fh>|W7pg#bzcg91wnb!ve?ql+}*))Kv3=pg-zB71#_ctWFZq$`!9BH2Btk>0~v>H4nmf{mXJodwI{)$lk z3b=iI;_$J<5lit&{B3$Z5j)M&6FwO+$x+`eNE;;3L%G z#NCB9fVLXtg4dW9dm-N3#&aCFxy?G^mN(wPhcxGO9tgPl;yIa4XLm2}^Y`_7+#WAv zr#YQn-cF~Jw_;@l7@?xZqg?VR?)0=*(BNSXPfsw^;KJn5!^a=M2;BqxqZpw=%%cGE zxF_+bi1E0!@_4YpKM#?R>!fVe+v#-q{e18J0l(V^ZKFy=$0&LnaI@txLgDbe2hmT- z$C`p08iRJljXaZfbF6%`wqShY9DzsdVWvb|G=9$D$hmRXeGoWc33z3D>R$VVH z(3X!|CKjKveoi16Tc4+8M;pd-#yiI?V=Z$8USm}%t#!2FuLK_A*yymTVKklaUk(Tv zbl0hZ1pNL%&gcZ(YnOK#38fDxC@4!dK!IfZioW&o<#(S*!moLjN{vls;5ae(n^=WM zh<0}Vx(MkM7Np-Z@UKG44rEtk%Rw?{BKqu5TEWHKMbA2quosQFK~1hGZ1R{OA&(dQ zDL7sfJ{qJEdeAbL$1UX&`MNOZwG6mh{g4Ac=b^ozO!ykZXTvvz>q^K34C4XGTNq>CLBYQ6Ve|Gn%th!kRiOuMWZitki0F zh)aY#Al(efMrBo6A@OG}{9BOUheJm^0?x`HStap>3TyI0n!J$*FKBF{RJ{l!_9Ffi zJov{kdH^Gw0bTDU=Ex`HaT}r07YNHDvEHzT literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/scheduler/scheduler.py b/gx_gp_monitor/scheduler/scheduler.py new file mode 100644 index 0000000..ad43a5b --- /dev/null +++ b/gx_gp_monitor/scheduler/scheduler.py @@ -0,0 +1,372 @@ +""" +调度器模块 +提供定时任务调度功能，支持cron表达式和间隔执行 +""" + +import time +import threading +from datetime import datetime, timedelta +from typing import Callable, Dict, Any, Optional, List +import schedule +from croniter import croniter + +try: + from ..core.config_manager import get_config + from ..core.logger import get_logger +except ImportError: + from core.config_manager import get_config + from core.logger import get_logger + + +logger = get_logger(__name__) + + +class TaskScheduler: + """任务调度器""" + + def __init__(self): + self.config = get_config().scheduler + self._running = False + self._thread = None + self._jobs = {} + self._job_stats = {} + + # 初始化调度 + if self.config.enabled: + self._load_jobs_from_config() + + def _load_jobs_from_config(self): + """从配置加载定时任务""" + for job_config in self.config.jobs: + if job_config.get("enabled", False): + job_name = job_config["name"] + cron_expr = job_config["cron"] + + # 根据任务名称创建对应的任务函数 + if job_name == "daily_crawl": + func = self._create_crawl_job() + elif job_name == "data_cleanup": + func = self._create_cleanup_job() + else: + logger.warning(f"未知的任务类型: {job_name}") + continue + + self.add_cron_job(job_name, cron_expr, func) + + def _create_crawl_job(self) -> Callable: + """创建爬取任务""" + def crawl_job(): + try: + logger.info("开始执行定时爬取任务") + + # 导入这里避免循环导入 + try: + from ..crawler.spider import crawl_announcements + from ..filters.filters import filter_from_config + from ..storage.postgresql import save_announcements_to_storage + from ..storage.md_generator import generate_onu_md + from ..notification.wechat import send_announcements_notification + except ImportError: + from crawler.spider import crawl_announcements + from filters.filters import filter_from_config + from storage.postgresql import save_announcements_to_storage + from storage.md_generator import generate_onu_md + from notification.wechat import send_announcements_notification + + # 执行爬取 + crawl_results = crawl_announcements() + + if not crawl_results: + logger.info("定时爬取任务完成：无数据") + return + + # 收集所有公告 + all_announcements = [] + for result in crawl_results: + if result.announcements: + all_announcements.extend(result.announcements) + + if not all_announcements: + logger.info("定时爬取任务完成：无新公告") + return + + # 筛选公告 + filter_obj = filter_from_config() + filtered_announcements, filter_stats = filter_obj.filter(all_announcements) + + logger.info(f"筛选结果: {len(all_announcements)} -> {len(filtered_announcements)}") + + # 保存到数据库 + saved_count = save_announcements_to_storage(filtered_announcements) + + # 生成Markdown文件 + generate_onu_md(filtered_announcements) + + # 发送通知 + if filtered_announcements: + send_announcements_notification(filtered_announcements) + + logger.info(f"定时爬取任务完成：处理 {len(filtered_announcements)} 条公告，保存 {saved_count} 条") + + except Exception as e: + logger.error(f"定时爬取任务执行失败: {str(e)}") + # 发送错误通知 + from ..notification.wechat import send_error_alert + send_error_alert("定时爬取任务失败", str(e)) + + return crawl_job + + def _create_cleanup_job(self) -> Callable: + """创建数据清理任务""" + def cleanup_job(): + try: + logger.info("开始执行数据清理任务") + + try: + from ..storage.postgresql import cleanup_storage + except ImportError: + from storage.postgresql import cleanup_storage + + # 执行清理 + deleted_count = cleanup_storage() + + logger.info(f"数据清理任务完成：删除 {deleted_count} 条过期数据") + + # 发送通知（如果删除的数据较多） + if deleted_count > 0: + try: + from ..notification.wechat import send_system_notification + except ImportError: + from notification.wechat import send_system_notification + send_system_notification( + "数据清理完成", + f"已清理 {deleted_count} 条过期数据" + ) + + except Exception as e: + logger.error(f"数据清理任务执行失败: {str(e)}") + + return cleanup_job + + def add_cron_job(self, name: str, cron_expr: str, func: Callable) -> bool: + """ + 添加cron定时任务 + + Args: + name: 任务名称 + cron_expr: cron表达式 + func: 任务函数 + + Returns: + bool: 添加是否成功 + """ + try: + # 验证cron表达式 + croniter(cron_expr) + + # 添加到schedule + schedule.every().day.at("00:00").do(func) # 临时设置，会被替换 + + # 存储任务信息 + self._jobs[name] = { + "func": func, + "cron": cron_expr, + "next_run": None, + "last_run": None, + "run_count": 0, + "error_count": 0 + } + + logger.info(f"添加定时任务: {name} ({cron_expr})") + return True + + except Exception as e: + logger.error(f"添加定时任务失败 {name}: {str(e)}") + return False + + def add_interval_job(self, name: str, interval_seconds: int, func: Callable) -> bool: + """ + 添加间隔执行任务 + + Args: + name: 任务名称 + interval_seconds: 执行间隔（秒） + func: 任务函数 + + Returns: + bool: 添加是否成功 + """ + try: + schedule.every(interval_seconds).seconds.do(func) + + self._jobs[name] = { + "func": func, + "interval": interval_seconds, + "next_run": None, + "last_run": None, + "run_count": 0, + "error_count": 0 + } + + logger.info(f"添加间隔任务: {name} ({interval_seconds}秒)") + return True + + except Exception as e: + logger.error(f"添加间隔任务失败 {name}: {str(e)}") + return False + + def remove_job(self, name: str) -> bool: + """ + 移除任务 + + Args: + name: 任务名称 + + Returns: + bool: 移除是否成功 + """ + if name in self._jobs: + # 注意：schedule库没有直接的移除方法 + # 这里只是从我们的记录中移除 + del self._jobs[name] + logger.info(f"移除任务: {name}") + return True + + return False + + def start(self): + """启动调度器""" + if self._running: + logger.warning("调度器已经在运行中") + return + + if not self.config.enabled: + logger.info("调度器已禁用") + return + + self._running = True + self._thread = threading.Thread(target=self._run_scheduler, daemon=True) + self._thread.start() + + logger.info("调度器已启动") + + def stop(self): + """停止调度器""" + if not self._running: + return + + self._running = False + + if self._thread and self._thread.is_alive(): + self._thread.join(timeout=5) + + logger.info("调度器已停止") + + def _run_scheduler(self): + """运行调度器主循环""" + logger.info("调度器主循环开始") + + while self._running: + try: + schedule.run_pending() + time.sleep(1) + except Exception as e: + logger.error(f"调度器运行异常: {str(e)}") + time.sleep(5) # 出错后等待5秒再继续 + + logger.info("调度器主循环结束") + + def run_once(self, job_name: Optional[str] = None): + """ + 手动执行任务一次 + + Args: + job_name: 任务名称，如果为None则执行所有任务 + """ + if job_name: + if job_name in self._jobs: + job_info = self._jobs[job_name] + logger.info(f"手动执行任务: {job_name}") + + try: + job_info["func"]() + job_info["run_count"] += 1 + job_info["last_run"] = datetime.now() + logger.info(f"任务 {job_name} 执行完成") + except Exception as e: + job_info["error_count"] += 1 + logger.error(f"任务 {job_name} 执行失败: {str(e)}") + else: + logger.error(f"任务不存在: {job_name}") + else: + # 执行所有任务 + for name in list(self._jobs.keys()): + self.run_once(name) + + def get_status(self) -> Dict[str, Any]: + """ + 获取调度器状态 + + Returns: + Dict[str, Any]: 状态信息 + """ + jobs_status = {} + for name, job_info in self._jobs.items(): + jobs_status[name] = { + "enabled": True, + "run_count": job_info.get("run_count", 0), + "error_count": job_info.get("error_count", 0), + "last_run": job_info.get("last_run").isoformat() if job_info.get("last_run") else None, + "next_run": job_info.get("next_run").isoformat() if job_info.get("next_run") else None + } + + return { + "enabled": self.config.enabled, + "running": self._running, + "timezone": self.config.timezone, + "jobs": jobs_status + } + + def is_running(self) -> bool: + """检查调度器是否正在运行""" + return self._running + + +# 全局调度器实例 +_scheduler = None + + +def get_scheduler() -> TaskScheduler: + """ + 获取调度器实例 + + Returns: + TaskScheduler: 调度器实例 + """ + global _scheduler + if _scheduler is None: + _scheduler = TaskScheduler() + return _scheduler + + +def start_scheduler(): + """启动调度器""" + scheduler = get_scheduler() + scheduler.start() + + +def stop_scheduler(): + """停止调度器""" + scheduler = get_scheduler() + scheduler.stop() + + +def run_scheduled_jobs(job_name: Optional[str] = None): + """ + 手动执行定时任务 + + Args: + job_name: 任务名称 + """ + scheduler = get_scheduler() + scheduler.run_once(job_name) diff --git a/gx_gp_monitor/storage/__init__.py b/gx_gp_monitor/storage/__init__.py new file mode 100644 index 0000000..e073c6c --- /dev/null +++ b/gx_gp_monitor/storage/__init__.py @@ -0,0 +1 @@ +"""存储模块""" diff --git a/gx_gp_monitor/storage/__pycache__/__init__.cpython-313.pyc b/gx_gp_monitor/storage/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d58e6cf7e51161b9f509faaa5cae31347b9aa05 GIT binary patch literal 195 zcmey&%ge<81U)KonZiK&F^B^Lj8MjB4j^MHLoh=TLpq}-Qz;LS`*iJ$r%fxKtyuVU z?sPv*##`+1@hSPq@$oAeK7(}Ka?#Jo&rQ`YGt19O)i0yFVZgt5{c=l`tk9ZK!Nypy@JYH95%W6DWy57c13JJLqYB- T2D#+}Gb1D8Ee4??79a-z<_b4L literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/storage/__pycache__/md_generator.cpython-313.pyc b/gx_gp_monitor/storage/__pycache__/md_generator.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7bb1b7b714948bb66d084b7ccf6eb0a02f9d243 GIT binary patch literal 18238 zcmcJ1d2k!mx@Wic&5LZy`y$77Vkuq{+u59r*x7Jm2i%w&uyciC%Z|a8JS{038Lo%R z;;_XG1k85hu!?tiVD6(XzJ$fVdsX>kW~v+cA-^7|ml}|h{PmPd7^cEM^L?imNlu65 zR=tjV`t<4VEPeVrzw>RKs~H&<3XUhLe?R%=nM9$kl?)prWbDcA2YcbHi-$s6`qI;^bKNtIKKkumMD zb!4y^aL34)_hfctu~{T<*^}L2XYEd^&%TW6%UuIKK2u)-4xY*GbF}Cfdogvd?p~TLWODlSeHqf7BDj-_ci3VkZ%Mg>l$Q)? zDyhC4Bjt5a6nTc^sOgkZ(~L6e3aabCAhlUrK@HNbf^a(w!qVww?;8jl@n3xQ%;Lmk z{DsJ3e4*h{ALOkAzCmwzBs2h& zmvP>p%(l(%4}|^wzEPh)1lN`!U&zxR@DC0T$yXzRp&=jJkKM|LQOaEHBh1i80TWQH zhM`$4qhWPSF{6V?G~nd6Xc)aT&0ad2YO;V+u%p2hibj=1WhgVG2ok z0aL^l4ryJbf?ei?T|U2$^@alM~^3Hmt#Y9nS$Zd-1p+=Ol9(prfE0qs8k>Pu&?$p+#d)+!IlXB`dq+Cc@XdC+` z$8TuKB-crvp(Uly-_Q~Pwqz@qgO+ek$kCf2=b%lM@ypQT*Ys$*GiBQ3xf|cck^`U+ z0CI!INx^&6g?GViOMK#$_*;+1pLp(uH)pYIV~2vR7AG7judvmby7(`B_+I?ehb(uv zG=g!=%fgZk3MSuCSpS~DeO)dyYlBV%gSdVL)2IYCERIgFj(U%Jpb9(e3kn6p{{E5h zfX@>O40w-u4h;_-8iBtMi?2yAh>Z*S(fbC5S(jGO1$`rfQZv%%#sGbSNDTuPb~hdh zjQSc6uMLd&8pn=}v4MMi{h`KP`|F3s>ivPx@Zhn=p`)ImG0$kgKMXwF7=*+-+*^3}mYBPPcXv$a=1qkQB^75jzOXS?(#V%I#!5EwB^$-^6)WGm|Lpy-6&v{# z8)GXNeg%W&)z=JELCI80tYjr$vND=qH(^~c<;G0Kys0>5a`Gl;ByU#7nVeslHeZE- zP(p?`1xbT~J~SK}@d*aj7Yeifeuevi6wwo}uf7f|>NM4pTxsy0fd7*u?e=Kp@yX|#&XnO7DN@Iv?aod1E$3te z&ZTy*1!M=V7`qeRi~49m{8)PIzf%@@e#=-o*He@gkexAh7c6ZN_)vL6)s)K=9tR3Z zNRgLi{E>&_@BS?nx6)QXt20idnfS!>m(E5Au8}3t>I6Y%@$74h(~n+$b;@GtCJRJD zzF>%Lb;=!x!VgG8SmEV&EY~$vPJ#L%2ZrKyFTV7_;&U^L&;RxE^B=g~t`(a&s$HsjA0Twx5(E0;M1QU#X5G5@^JL2;T2H(*T5C|E`E<%BR zK|dVwjRsRY78~>g0WU#l6Lj|mhW#!hTLKNUm6$BUE*xYyzJc@s3Hp&?KcKl)YQ*n_ z?(?LJw4nzPbO5RGV9T&Yy9B$wYMlDNg;aLgLT*7UcNL$z zYQ_=GUB}thB@9$$P0ZQOJKN_vqt0&5R0c)%+zI`HDdUvwq-~O&s+lU7?3*_&lS{f0 zpvAjff$N>x_v-$k?#~S|_cq?WjceF%rbTI%+R1WX&5aEE1iX6y

jQ#T(FWDtNBpE$Yr${ULxO~xl4xp?j&aeegz zS11SO5$diMLjs!|)ver0W~!;TX;fuwW8s4%!@)xy)K=h{J{rXO5~rFrjd({74tTc+ zZ8tLL&#%*lO(TJR??`Y9WW+Xlq1_;^+Hq=T+lO6$=_0QRB`WQGzGH%s^&R%HL7&UO z;yejD7*>b{q@>t-avcn?plqQE<rc=*}sW40FF)-u!gwXN-nmMU{!HBqHi$jG_8 z%^9=R@bGiDo!`#cYNEE=fuytw1fA@>$z6~CSI$&Q=7NPiD4lmvcqY92E@VLW2eGzM zgIua6s-$dYo21RGuTfHeVq`ws<$Z$f243*_zpcIUQHD<7?Z_D`sltZaQJ3 zvKKO`R@XXi^b)8Ep#B zDAdRo%?e)u`u$?-y$Xl*fT=&+!0EWLPshbc~JO}Syo|Gk#9S(9zD$73g;VnwFU3Ge~wqv#hClBl}Si| zZL@P=>$f{%_G;c46%ubwbmkVJvt)W)fL&QUvWa;0v@a{lnk{nPc)@>N{f>Y36T zcX@y=rz=u6OMhL_dUD^yHksLBzuv$XZJ4c#7Il1W+XFD?+z1julEd*KK@{^3_>QJA zI=*HCbW@OujLy)&%Bq`JEfC}3%>`+hRW=1fAy9uFme?e%#zmC>sq01c5>WuZb@Cb8&!uThcYs8W>hFt zXU3+?jOixNlhI>Z(!W9J3zqXu-ezXEiYC~jXrpqk&YRUw;Rcu!5D=Fh${0Y9yvZ1{ zep4^1e9L72wl)Al?L-8XY@Sb^{(kl!M0mV>_HQD9;>0A*JaXFUc7HcM`Te=a7N39P zyYZ*p?(4cLK-uG(2B$#xUe_F`6=>ISldL#~VIf1kXC8LOPdy!<{iCQ{wmPNqQ?E+p zap?1tC!rh{0?l++CDp zc8(y6T?#h2WmI;92t`D=#e`^nD-XZFr);mX_QZSAB|P0U)8emmCE#kX{E&HK3W+vaVzUohpwOhvq@XsR<( z7jw6P=*_v>x#I2frX8e(WijhA(rpu8(j*!sXFH=M9XIQC(dv-*ILDdd7mBAlqD77K zwpDUr&b+M@9?Mo3vzEh8D&~q-&fDrzY2@v9sS=(?=~~R4^s^{_T%M zqg09qjRi4|BCx=M7)G&(YGJgj0T3)cKdmgj@CGQ0fAUQH-M|x2g%9}qeSn%`0V@fzkPP;jn5XJ{Ve{` z%an!vrU`0TW!*WYM~`;$j5^ z7T2k4a2FTaN+|w$sVv>9^(N2<4?r?b0heUiCmoYUWVOtNOy}!`Z5_xdf|x|Fk?w%Lp;6T)^dKy0Q>e6RMsHaoRpTW_*)WN&h?{zR;-&bxk1jv^ z;>8F5cKMl47oU2A1^JEYI&SLyeczG3VEqA6XEgNI_XQ8E6zFgNs=PXuigJhh#3dXIR1$*xE z`A_FhS*N=rc~N`Agh4JbPd7&lQF|klWI0ZqICPv(m*=K2U>EVkpK)zT8-$7P?W4BS*o#$(cwS2btlz} zBO-^Ik_Ae?YM&@Ot9uGSa!!$thfI}J9fe4M#{fL@pvFm6P)_*QwMBH;S5m!-j0sME zSZZLTNRCRyXTX?yQGg&vkJ4MZWLE*8=htu34_UZHOK4`TGL{ z!~UVJFgo1iF^L0;fA|-%J`52=A=Fo#wM1vd-6E;8y6zyA(;rJ};`R40eeg!BbEnI~ zwn8OA7XX{O=w;L&VL@3H3}WzsphF=<(sg$n?e~!Y2SM*+*#HacF2sUPOjcSYv?B^k z*`t#86T=x$auVo4f%XN-NOAQ^)tr*3M_Ue$-Mq zTS0Nn$UU{^$R&^FDLtov9yyX~A|<|tQm%e-ydH@SsBCG|3YhNTd(~hqq418`$e-J@`4=%#B>is8iL~@CNOaw zmJHZ*la5Qc9-<)>0<3~RFA=EWGc0Bmbyrpk(H$!CiyV6VL%0p&E3^ zdjZH65EugJkI)VWs7V?%^-ZRiM=<~kANocCV#$%TeN|Jvii)Y`?qgDHmA~B9&p_4P z3RLCEgOQ+0mofqs=N@C~#TXKDB<-y?K$XF-+ol^bGvu5CRhDoVs#-36_DTG)qy&ZH z(Ni-hDv5zgDTs9gTcHzB(t0HSV7ebr1SqM4RDimJ?7afbvOyRE8-k=O?1c34qfbJU zl2Z8Y6N{%mkXN!eoG{9PVuPY22E$FKh%0cEA!q_YL5C3z=*I&OK#-skdl=dfbYKb| zU;~h4$1ypI$uUfhV{$(xCm@mi0}n!8$R^DImda?vK7e!x`Ahr*LZ+9v78OWR5e?EQ z{tX-XUyy)HE|1Et1i(n>a?C{+Dk=eyzR9wmI&$*JsRvFzFf|Y>Z{W)tVp)xRR^vC0 zqN(awv70Y;N7`pseC24H^DeBYee3Yq!>=9X@~Q!;^4rn1l~*0hb@92bSnf(bcjZDs zNvxojFR0~Qt80i7UEw-qw+VUu)r!9^0mG*HfQo+1pyQpF8Uz|3zjErY)9sjq!^uM#z7u zx3srvzs#y^U!(nUjUMu8o+t&i`e3b#awp(~hS4?X{D4{iX?XQE;1xW9c&LX+sH2#S ze1K8<9-1+r-U8^PVv2*iM!egLJ`QrI8OD}jZ3U5JN(lS{)W$;%qh#EJ9!#VMhe%7% zPX( zAvpjQ>xck{att}TE96}A%w+af%2z)lz-du{!+sc&OTopgFj{UdJ@p}4dk6@EmUrph zv)_MqcJaTxw^LLjKv}S2aGpm57Qsv|g2TssLdJ*}BDqFE<{BLK4G1}@f*#-oc#-a4 zpM)ORr!YB%$wQbZP{{7b%SlKGCf;l$GSa#1?88{$5lkM%1jSki_6m7Jf)R#-qY^_L zG6vbeC^7hiyrW|n?BF4nf_XR?0MTp|%uI-m>;xuVm=H8Xa1@M3ysRIvPlQELQbP^N z3*l?h<3pYxqF|JyH4z$@3Tv4do+?flsjY&I=@i$Y|6nmNI#GE`D;J827s@IZmQ`Ib z>xuT7nQ7il-y9=L%g^k3ao65UQL|q=B3Gms=LG;Tnz7coJE6xnQF!<7; z$z2zkHhftBe*K&w+O%tO2Upy{=QaLlBv719To{Q8DtlF;oaBGBp{fXlWn!_Yp5?)*R)q@ zzsxs5{>v&$dz10YdM)OgNV>+deWmuxt(DuCYyWw<9`c~yA}$bvC5vlb+$O>Q3bDmM z;9~kkQ;4ewL;lci2_b(L4|0DP7KcZ$c|89m>>ZKrWO?AVv%wIWB>e%nZS;5sz&+|= zjo2;jw=8%JC{~9FsvtzAVA1G6JnqEZ%C^9zpdE%FNCt190fqR|1RY8o;(q=Z79!)Z zFG3>A!l6kGI;nU-1Y>>YD;{N{b!`h9%!Z3zlWxt_bY zyASX^zrTi8iNZ`EY)`;A9OrfC`}o~=lRW3?=LUv&&!KC0C*gGI9kaFs$S#ia3^}DV z1BHn@>1-H%e}aP3O#d}JC8`a2*K~7&f>WgV8lDn4RNbm8+5&x1q&jkfcW+8ikY6YP z6T}Of6Ii~`vi=7=Cz>ksMKijYK7LhOf`a^9=o+39ZFG^|F|C_sB30A-&u0Ee!KFCm zINSa3ryR0YeibFH)6~*v^gcbKg{W;^pAn+2Ni4d&6|p9`XXrDFQR5PY5#z5F3QM0J z)EZj~6~t>3N+oJtp|vpo$1M4n16sP1(Oe)Qa zl@@d)THOjKOA;tyE(scc2hWr$yE(^RotOq2hIyu!^o4%F~!ID{p?5a zC(m9y`RL;Ghwx?254hc+tC55Zp)Zh#U;-_Lav)aqE}nVg;>(X;dhgA}KR$e*mPMt2 zxc^Q#*F+;Sf(J$R{3fFU1)Nq{lO5f2A1cdR*g7rxe|j zKzUu;)wnH;d*KPmLkWd)qvDrF@1KkR>G5P~s-qH0**>T%x+iQb`VBPJZMa#zB0${WC?rG-#9&l#fPlZRxDkF&8)oiIMyG~|Z?STEZ>)MPU%i&A zTE{uo&zm+ZaUDwGrGJ<{5nHj9U$K>2e#;!qIojq;?W&ZVW5v9wR=UMGYUWMLlb%O0 z&Xg;ui}c0nck=Z+WA*#^`hDDO_eATxTpic~2e=~Nylqf&+;Ii9ku z$L;f`?xb5XZA{zw`t7m$PQJeL{AjekkE=Vt9q@2P_srY8vM&)`su!|yZ-{~=LCk5J z7(yu7#9H8mU}&fe4rqxVq9HZ~HZ=&q0^NbQhk$(PA=e<=n+CQwEpZV6L@qrP$`XKe zjhd5E%_Nz0O+wpx3}yqE!*E30fR`L_#|U@usg@jY$Aqyy%|?jYq1Avasb)ghA}CBG z9^b5Hh)gvvx-4Q4F8MeR<%N_$`!IsOd-25w!MQ1xyIHVAP+^?$|1W+5bfsp6)(qA|Ai z?^)4UZv(FA{|+63(F;PmAP&8XM4J6?5)xbPj+HmV@9CYh4hVwmxHHz#55K8RX&(Ne z`I4R7upeBY`6ZLTn`)an5vy3uSFGmBn`V~H^u^Y6@@qP|mag*y9OL1-@8NR26We7! zdfwNzatNL*b7Sn}jlti789%ooN_(0O9od91zXq7~M&|DU8+s3|1IABAi1#R#njX3y ztU!LSim2TMc7xbQMh~&Mx*m-B9w#;rm?-h+(W|k)ypuA9)@LD@4LqRgbYL+v=z!Ic z3HDjlX&|UfujUL)TZF0 zfM-*0BaGcMa~kpiPpWVug)$h#_-KL~ zNf=3jhe-sFze0n8V_-NqHUeKDrb-P#A&d2mdf_`jVlM}~ml&f;ga)Nq62(s(d5SNu zMB4w!7l*_BptT459tctj!k31^hF>K*s8!vKSfhmmum-;L0IaD7@u9LIa&P1q=iC5a zxO`JwHZ>Y8u8SPwi`P$NUC6C?T{{h5zcg_b&0KRkXWu>tq1(4k=)WsnJ+b$jb*;0T zVjDa8jh)eTUEJ+VbR9FfH3n-12oZ$GIXB(IiFWrhba! zST}E4ueztd0q+hz+_;Q9;>u)SffM^DOwM33jmcR^1e<6&8TEw@1qN7RmLaB^_mBc* zXZ8b3Ud7}bCa+_Ht*4k*-oi^vP;U7nXC#F)OfRlLOizd_23YpaDkWD8b@TqFWup82{{U0Vw6FvwH zNS`4F{NaYtfg62|X+W|hIwF0BL?@#-DQQsH+DZkcj{O38y_XxL#9WiSa zZ>{30*F>#rIo(>31&G(1n%KYbr3w%t zJOcme18#aj)-08*yGK?Y0AhY($7hZ;oi~Cg4Y)d=ATA!vwjW#YtKM;DTwBXw~1s{!h3IGe`wzAxUUy zn!cjZ(z>e_ine@975+O_7p3aHrOLjgsuDUQz2aM)`&-?rZ*{d-GAT>mvqFWc^J0+nd1#PP@77JwM<%QEa8}q^DaVHoj^N znAG87wi*oRa88uk>2=e0MC$mObqNacv)g~b)Bj5xuu`VntCSW;QTwf~;oo)36GphY Y9z?W!GiTdI>hq0-ZTZ?S95kf=A83So2mk;8 literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/storage/__pycache__/postgresql.cpython-313.pyc b/gx_gp_monitor/storage/__pycache__/postgresql.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..11fe384cfde1c1338c3911e9a7bf6653cd201926 GIT binary patch literal 15029 zcmcIrX>=6Vm44M#)%%9JwJ$V6EW#jx5t~?BEXEQDLtBYs$?i0*Mkp5Da#go&5@#Ya zPAt5{c)>to3^;Zy5(jH0o>=h`1M|l-Kjw6gebtqd%wH9*Vi2mD}m?fhWqA5cbtGU6m^y6jcER565FAj- z7%+D@+g*aI-7UE3x20o7dx=m&0?x(_`M%DwolR^<j0W>gWu z7bxj5^w@DBAa_;^7x2*B`aoI0+r#(F(&dz5&JP%2b{{v3^pu-OXf`1*4<>KUOx~QC zKz7bdpkU53N#~<`$Xa$L*~2uJC+>iuSnrC7@jf9E*tsqJyQAp?Bhw?N(#MZj(`R0} z{PWTDPv1;G_vG~9(dlQ-T>i!5`DN43KYjVksmq6cX1$6tZ{#FyTfZ2WxJ~`xxWxLS z`z7;^!FYcx8X9OcNao&9JQDBU7m+L&_C^Ncq0UA|GKR!nsBjh69g5lg(P%6Y4M+Ax zqH)Q#Q3yReuqz@a2I7$E?Tf^N&`xM~NQ?ycg`%Oph#)!oqy5=rXwr(s24a1X*~m&I zLL@Hi55}Uw$Rpth9gt+(g=rgO(LMcrl3fh#i9lu~oQOxlI2ZMCDv|4VQM&CR0t2*b z2rvR0FbG_Lg`MRCT#vEGME8&HF~iOp>CRdT(oHnonoS29G1D|#zyfW^)FHQFrRk13 zr1hGBt;Za&!#}U~)M5+(Wl}0}25Nz#Ts>yp2q@(N1-YTdjGmGnPm2L_p+BHSph+*b zEB^)RfHGY@CA79ji+LedO4CZUSndeyfih(TWzb7Gv|Ucyt_W1Xs#Lx=l1Wphwe+!(u)?);Z;)zO_`m)g2o95O zEl=gfE;68{b#t^11iOrCEc?!KFx1gX_L0*}XCp5-prB;!-K|lbCB9dPgnIjo3R z@7dqn_eijBFt{%UR2&nUMR_5b2W8e29~@{J1WpPDDU}DMs_eojYu2RZ)k2pdF5nQk zP}MYEwRo6&+I69{?#TVm-#=bDXPA4&dL43!dGTG%(%Vgi#8H$FPB)~^mKii`iLF*2JULt4>eE-TS^)-O6WT9%0Mj5&#u0OgbCb0vNs89hlr3x|hP$ZZ$pWyCLJiJ#b2d)$z2u33h=VnWm z%^KwM5kZIv5-Y|9sU}#YsXd{Af!(3-1C5-3j3m@!pEfMm8w`U;ic3sHL|njC73wgz zq_7#Op^!fdOU;7X;y*z!ME>F>?%FAj_mbK9tnHxf(7x9;jyT2}mM3SgNUqqP^ma~| zcVsx1WfOC;s_sPdvF6FDwp3Ny*|p&)JUd8|fHXn_StM ztl2T)zAIxSGfF4j3sdfeBlnH18S6+c+B)IB^P;CNX|Ma@lB?o6VWHg%_L66}9o&|z zUj6>Yv&?&2##SaPH%-_#U-XnudRkJRmeKphJsUr^Z~SYid!Kl1<1iJ0xFIgwnK~C0Y7ZS_jMRzrWj(;BbR~=X0=lV8#vv}ieOBD z>tc6lc%s$?g^&*zIRZFk>edjYi&OCeutSUc2t@|Y&uxDS>qQ}^!v<>t>FVmxp-W@G z(eoMb@zrC`Tz>bB>6iXEeQG$-0HrQ{@ZsebNA$^s^(fb0g*_t)FILfGj>4{;S8vB` zM24oOriH$%FCM${)?2N<1agnux>D`T(|`v5^FD?e!1xkK8d(5oGz)VeL70mH<@p%} zydx}tOv&_cNQk2B<>USFfrzjW5RU1e4znjrZ(Mc88NQdneqHPuvxM z&G1n1OR*fr`^p@DgW+6*wQT`^u4zfz9RBY6q}Txu+`ro};eClnWvg@;JSOe_uoiC8iTROuIU2B!fD zDj|hRNq{MU3ur3;DWy zpo#0wwqHaM3a|hoxDI(wF<=2m;l2T+94`VX=0q=aq^KL0hu=&;`_kq2UYLI6Wxa}_ zY*njIUqIGMzM+aCEQQ_@sKUyqk<*|8ITSOpNYi<%;uLUNeG=2$Xw`#B7){EcJZ1_e zGUhCYP9>ulO99eP7}&RfC*T--YJI z-#{R%2IZg{$iPxoJG`X`uzY9IvnJ(PlM5_s0awcFbhxs3w0hJrw)1`Q?27ju9g8Nt z+b7JOx5k&%V_QDl{>SZ;YdTYFI+Lq+eDuSKd+$r$9~{5;`^oQxk~4QtxWjqi^4uJM zz2RKFwXK;yw{RuE%J~I6#3g<&EU75*1O5B@<6*tPN1mrpa|R-ZWpe;x`A<=P)szIP zr0aPI5FM2CujV19TNBS&&d*iSn+a9J4T>60-#SstCWc^~)+;}M2RLH-<)JGdJ_k}( zR)h7dK_?GdH7$`IeRlfDA3&w1iaA}7R^QcQhtk8ChWb{eI8rJ;};-6ZZ8} zp3-OTM2Ti8`;r>E2dZkxajW0UzESUYvF9udM9;Z+2ssJ$V zY8enB*6vdEtBY4#%VjUnW7#P0=lhKETgO^9nYE_JehR+DD<3{F{qhOfu+{Sw?rp0N zgv8~;$I`Ezh612krhk0^B-5p_cRORzNP75~qT~MbnadxXx%%GGD`#Fy551qqW3tgp z`vL|A6({|{VdPK&WvuE;3{yv8r_KPDN2eITFo5*IXRx|%v{{o`)_Bz#xR|K%>dPij zzD)n)sp*#wP9OXeSn^i^FMj=)TqUnv%JYSbquI|k$rFwVktTV}s=G?)gdV65mikU5 z4|-7n65;>};GqSl>V7d__x6SdBB5wv5L}~!{Q_9QSVu;TELa$2C2$)wFZW@TJC+>a z;f3ZE+gZjCL}kPfRa4xBgPL22BE>pB0AS)f6b_Vp=Q`PWpf=|Gv%q2 zRqf>|&+_3-nkjv9)5+z@#`Q_>h6%Gj!`UsRQ#HO5JC5y`tl5yN*>Ki7UbAI*`$bpH z(Fey}zN`_wePn&IwLMwWG2z|@I;nS7((b##lNse`WG8DjPq?>S^!OlAHlhud(pw?c z1&^2FRjF(=pJP_HdC9r8We}eCS~ptQ^L4W}a_m2IJj8Q6n;KX>Lcxl{1ecxNOY;qmz#sgk)L z$KJ{V6Fm9o1_#qZADn)Gu)a)-D(EFy2Q`;29&{*<-e7$HU_`PHgv5AIp?m5mlf`;r zmnx%Ed;wq8Jup52`BC!3WAV^Ho`C-zeUm-*1_WLT75|2Ot%7D{paZY(F zj_i1D$E5f6l=t>=Z_DuJDf2lwxKi ztWheotMThG_ zkIA{(l0iM{UY}Y`AbQTtLs(pry#m8hC>~^5{-es3C=&p=wHiuL ziGrRb`Y6h$L3fIWOPBY+(UV5`S!t0gQK<3>c;5#nxQP2Y`sNd>p+W2PuZE^iye@-p z`mxv3@4j>S$v;dFJ(h>}yJN8dAn2oSr4RjDE@q`rOm#GUZKXZcF53dW1cb_BL?Z+t zL)4?HWbGG&dn2KN_}=|eO(D2ubEmLs2LuSLMoZ;IZ^fi{UdlUf+}jAC>Zv+8C*_%! zw9iv5tx63t^DM0k9_l%?faefhzX!bKh}P)g-=;;C+bf?kn`5KZ-0y53G(Jw0;Zdfr~PJ2IzslbLgQ2=5gu$qv@w# zk~egE=#49X|3L!1)5=*`VJ6m*ODi?0FN(rSPO?f5r+J<3y#OjxDw3^n#mh=lpGMAwN@h%94$PJ_FboMq)n42@4#nO7iNgSSMI}Uav zD;9oiUnElrV`M2`rQ_EjnSs=sDfDDdTVq>>}0i^%V+{(APES^ioOnmj}$8=k3w z5iuBw@6B=M;$|V4U;kD~hAl!e1|7*2BN>qy3w_&&_antttx zOYeP9Xu0Z^1H_$N=p`unhb!;AmHyM?MG;0RMsY^ACanodV^mt|d!os5r*d+dlHT32 zNTXMIlysH$wg^!eZlkQ(2?E3m^*U`6(E^le3+YUm6lULq7Q_GqKw?!IiOrudHzJ8u z-Y{oLY{I?>{6MM=mBk`Gm4V#JVv+D54%Hx3<0z2>PaCe{>uN71lta+ScfupIV6+Br zO`sz;7?i9*xM0zn7{Ita7<@1h8c=dv!Qh^LL5vUdM_#ML_ zj)|o~Sv3y_L-DxKzZ)(ii7<-QSYQ?gcz5Qryt-hBh#86U?xM(-Y5m|Awn4Fd6u zy_~NeX&CuYYQgFZf%sYW2EH=4lVvNW7B*jF-F(?dI78qyHMcQ?pHnR>$7)h7TRz8+ zYYlaL=}5)s?9;V4h|;sOKgSmgp_F%?fkLj2;q=23HM-l_}_A8@)ow`pW=YR-y`2|vwb z_S_6shS=hk(QT<_|L62`orAADy6Fai*JpK3zV(`~g0DRJ$S8a2#~A|Av)oO5(YUprL z&iY?m+9^_Ku;(Stu(V13Q)t4(xSzWVkRT)feAIc;I#3ebR-+Nba^D^vM* zp_Jqfh7*E-j%!8C2{n)`->#B50@+k5&mF#YC8`vrAtFTJm8jdss}|vvs3~8=8?IBX zVeVsdt-Rs&P(qjuL5_q%-pBvzD~qfMAb9k_;wr zsR)i5AW)rR2r<&Ey!jLygNv0oGrTA_L@v5Kldgu8t6|c$BIQ~!<`{QvNt(CFQ^Vt` zZ)H{`&~V`?ojNxJMTpY_c^pE{6fv{-6myQkYHvM7)3|Fx(!4>Q;vLW&HQaKJmpDlM zn|AM>67|9_-94OV;TelMPsDvycxvOwQQxU(kmluW=&cS)W)+?D+LUYUSY+JQku-PE zId=-^asAfzwa~w;&KFrg-B%n!&c0%%G&^sm#hkfwiq3LX%C%~&VcfMoXjjCapx0|waPWI!evqQzO0AMgts?D2Ku1Bj8=Y#r34l^!Ixt5N*mL<*0 zzRlSvwx!NWRcAvLjC{DnM5!`wHdzyLQPG(+r(DhBt|dwHlCPM_H$_QxU%;@#TP74G zb+%NeIt{8K^s5<5a3(0pVNcBVNpo$=TsvuQN|~EJHZP{rpxW-Y!b3P0 zy;`nL1r^>t#QVi57aZ-6yVfSnYw4WSo=|uS4`D#n z;tF9u#(#*xk1%)?gCAp%tL6%ShiO#J#Yz01mD`_SVM=f>VC*FfhA_y}Uk_j!2B=_& zA3_B6*Gkl17iY`{{*J8viXW)I;v2L4jHZ?12dbs`z6M#_7*t5{JtoTUkM2(Hx<9o& z_&H6*G8-6GC&f4_rTE5%m+O^M_{6l^QDcPHXAob4)z_VzKeF`HB5Zx#Sj$a((YQ=W zbXv1vHevYJ9<&E2WW5Lxy;meW7Y=U`(CA|z`b&smE99ZBMjw?)9>m{&z@I9b^fll} zS1nF`W(&!D^1h1pBSXD>~w1WwSDyIcS3KIKa8B0LhmXV6Y|T-XD20~f?MZ7nvLgy1AwNg;2Y`zk zkL+&kkYZ&20PF`*9M*RF(*#jAAOyHZMmlk~HbDd&ksQxEFPKZ=mk9C==cMejWkQ0ouIV=5B~C!p#*xX`kFbHhvQGskVd zN!!wtZRzOtaa&uGYm?z=8x;M2U#MPOC|OgK*XPpM@A57y=vS=p%0?cv-0CWZx=K{_ zWT@h+UnQ5SV5$Uo3dR*Fwl2$3wq>KejIwFtO0`H*W~BYVcy38x_!){Th&1_AhW8RnXSWw@JG!Ympm3qK|E z|4iJUkUDVWF*}$~xfP#s8$aRZf6CpJ;_kZUCRXpEt>c#3A>&PBf@LbN5e#n*n&vR; zt`P__yBOlFND}vyyYh(hdFQy>H{{3|?M%(l`)?3|VaCBJ9KIQzMaC1N9I>))rj0F?F X6mcD2^|h6e^JR0}*0bl=F%bSQu#U46 literal 0 HcmV?d00001 diff --git a/gx_gp_monitor/storage/md_generator.py b/gx_gp_monitor/storage/md_generator.py new file mode 100644 index 0000000..d4dc18a --- /dev/null +++ b/gx_gp_monitor/storage/md_generator.py @@ -0,0 +1,399 @@ +""" +Markdown生成器模块 +生成公告的Markdown格式输出文件 +""" + +import os +from pathlib import Path +from typing import List, Dict, Any, Optional +from datetime import datetime +from collections import defaultdict + +try: + from ..core.models import Announcement + from ..core.config_manager import get_config + from ..core.logger import get_logger +except ImportError: + from core.models import Announcement + from core.config_manager import get_config + from core.logger import get_logger + + +logger = get_logger(__name__) + + +class MarkdownGenerator: + """Markdown生成器""" + + def __init__(self, output_file: Optional[str] = None): + """ + 初始化Markdown生成器 + + Args: + output_file: 输出文件路径 + """ + self.config = get_config() + self.output_file = output_file or self.config.markdown.output_file + self.max_entries = self.config.markdown.max_entries + self.include_today_highlight = self.config.markdown.include_today_highlight + + # 确保输出目录存在 + output_path = Path(self.output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + def generate_markdown(self, announcements: List[Announcement], + title: str = "广西政府采购网公告监控") -> str: + """ + 生成Markdown内容 + + Args: + announcements: 公告列表 + title: 文档标题 + + Returns: + str: Markdown格式的文本 + """ + if not announcements: + return self._generate_empty_markdown(title) + + # 按来源分组 + grouped_announcements = self._group_announcements_by_source(announcements) + + # 生成Markdown + lines = [] + lines.append(f"# {title}") + lines.append("") + lines.append(f"**更新时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + lines.append(f"**总公告数**: {len(announcements)}") + lines.append("") + + # 生成目录 + lines.extend(self._generate_toc(grouped_announcements)) + lines.append("---") + lines.append("") + + # 生成各来源的公告 + for source_name, source_announcements in grouped_announcements.items(): + lines.extend(self._generate_source_section(source_name, source_announcements)) + lines.append("") + + return "\n".join(lines) + + def _group_announcements_by_source(self, announcements: List[Announcement]) -> Dict[str, List[Announcement]]: + """按来源分组公告""" + grouped = defaultdict(list) + + for announcement in announcements: + grouped[announcement.source_name].append(announcement) + + # 对每个组内的公告按时间倒序排列 + for source_name in grouped: + grouped[source_name].sort(key=lambda x: x.publish_date, reverse=True) + + return dict(grouped) + + def _generate_toc(self, grouped_announcements: Dict[str, List[Announcement]]) -> List[str]: + """生成目录""" + lines = ["## 目录", ""] + + for source_name, announcements in grouped_announcements.items(): + # 创建锚点链接 + anchor = self._create_anchor(source_name) + count = len(announcements) + lines.append(f"- [{source_name}](#{anchor}) ({count}条)") + + return lines + + def _generate_source_section(self, source_name: str, announcements: List[Announcement]) -> List[str]: + """生成来源章节""" + lines = [] + anchor = self._create_anchor(source_name) + + lines.append(f"## {source_name}") + lines.append("") + lines.append(f"**共 {len(announcements)} 条公告**") + lines.append("") + + # 生成公告列表 + for i, announcement in enumerate(announcements, 1): + lines.extend(self._generate_announcement_item(announcement, i)) + + return lines + + def _generate_announcement_item(self, announcement: Announcement, index: int) -> List[str]: + """生成单个公告项""" + lines = [] + + # 公告标题 + title_prefix = "" + if self.include_today_highlight and announcement.is_today: + title_prefix = "🆕 **【今日】**" + + title_line = f"### {index}. {title_prefix}[{announcement.title}]({announcement.content_url})" + lines.append(title_line) + lines.append("") + + # 公告信息 + info_items = [] + + if announcement.publish_date: + publish_date = announcement.publish_date.strftime("%Y-%m-%d") + info_items.append(f"📅 发布时间: {publish_date}") + + if announcement.purchase_name: + info_items.append(f"🏢 发布单位: {announcement.purchase_name}") + + info_items.append(f"📄 来源: {announcement.source_name}") + + if announcement.crawled_at: + crawled_time = announcement.crawled_at.strftime("%m-%d %H:%M") + info_items.append(f"🤖 爬取时间: {crawled_time}") + + if info_items: + lines.append(" | ".join(info_items)) + lines.append("") + + # 如果是新公告，添加标记 + if announcement.is_new: + lines.append("*🚀 新公告*") + lines.append("") + + lines.append("---") + lines.append("") + + return lines + + def _generate_empty_markdown(self, title: str) -> str: + """生成空内容的Markdown""" + lines = [ + f"# {title}", + "", + f"**更新时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + "", + "## 无新公告", + "", + "当前时间范围内没有找到符合条件的公告。", + "", + "---", + "", + f"*由广西政府采购网公告监控系统生成*" + ] + + return "\n".join(lines) + + def _create_anchor(self, text: str) -> str: + """创建锚点链接""" + # 移除特殊字符，替换空格为连字符，转为小写 + import re + anchor = re.sub(r'[^\w\s-]', '', text) + anchor = re.sub(r'[-\s]+', '-', anchor) + return anchor.lower().strip('-') + + def save_to_file(self, announcements: List[Announcement], + title: Optional[str] = None) -> bool: + """ + 保存Markdown到文件 + + Args: + announcements: 公告列表 + title: 文档标题 + + Returns: + bool: 保存是否成功 + """ + try: + markdown_content = self.generate_markdown(announcements, title) + + with open(self.output_file, 'w', encoding='utf-8') as f: + f.write(markdown_content) + + logger.info(f"Markdown文件已保存到: {self.output_file} (共 {len(announcements)} 条公告)") + return True + + except Exception as e: + logger.error(f"保存Markdown文件失败: {str(e)}") + return False + + def append_to_file(self, new_announcements: List[Announcement]) -> bool: + """ + 追加新公告到现有文件 + + Args: + new_announcements: 新公告列表 + + Returns: + bool: 追加是否成功 + """ + if not new_announcements: + return True + + try: + # 读取现有文件 + existing_content = "" + if os.path.exists(self.output_file): + with open(self.output_file, 'r', encoding='utf-8') as f: + existing_content = f.read() + + # 如果文件不存在或为空，创建新文件 + if not existing_content.strip(): + return self.save_to_file(new_announcements) + + # 解析现有公告（这里简化处理，实际可能需要更复杂的解析） + # 为简单起见，我们重新生成完整文件 + logger.info("重新生成完整Markdown文件") + return self.save_to_file(new_announcements) + + except Exception as e: + logger.error(f"追加公告到Markdown文件失败: {str(e)}") + return False + + def get_file_stats(self) -> Dict[str, Any]: + """获取文件统计信息""" + stats = { + "file_exists": False, + "file_size": 0, + "last_modified": None, + "announcement_count": 0 + } + + try: + if os.path.exists(self.output_file): + file_stat = os.stat(self.output_file) + stats["file_exists"] = True + stats["file_size"] = file_stat.st_size + stats["last_modified"] = datetime.fromtimestamp(file_stat.st_mtime).isoformat() + + # 尝试统计公告数量（简单计数） + with open(self.output_file, 'r', encoding='utf-8') as f: + content = f.read() + # 统计###开头的行（每个公告的标题行） + stats["announcement_count"] = content.count("### ") + + except Exception as e: + logger.warning(f"获取文件统计信息失败: {str(e)}") + + return stats + + +class AnnouncementMarkdownFormatter: + """公告Markdown格式化器""" + + @staticmethod + def format_announcement_card(announcement: Announcement) -> str: + """格式化单个公告为卡片样式""" + lines = [] + + # 标题 + emoji = "🆕" if announcement.is_today else "📄" + lines.append(f"### {emoji} {announcement.title}") + lines.append("") + + # 链接 + lines.append(f"[查看详情]({announcement.content_url})") + lines.append("") + + # 信息表格 + lines.append("| 属性 | 值 |") + lines.append("|------|-----|") + + if announcement.publish_date: + lines.append(f"| 发布时间 | {announcement.publish_date.strftime('%Y-%m-%d %H:%M')} |") + + lines.append(f"| 发布单位 | {announcement.purchase_name or 'N/A'} |") + lines.append(f"| 来源 | {announcement.source_name} |") + lines.append(f"| 公告类型 | {announcement.announcement_type.value} |") + + if announcement.crawled_at: + lines.append(f"| 爬取时间 | {announcement.crawled_at.strftime('%m-%d %H:%M')} |") + + lines.append("") + + return "\n".join(lines) + + @staticmethod + def format_announcement_list(announcements: List[Announcement]) -> str: + """格式化公告列表""" + if not announcements: + return "*暂无公告*" + + lines = [] + for announcement in announcements: + emoji = "🆕" if announcement.is_today else "•" + publish_date = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A" + line = f"{emoji} [{announcement.title}]({announcement.content_url}) - {publish_date}" + lines.append(line) + + return "\n".join(lines) + + @staticmethod + def format_notification_message(announcements: List[Announcement], + max_count: int = 10) -> str: + """格式化为通知消息""" + if not announcements: + return "暂无新公告" + + # 只显示前N条 + display_announcements = announcements[:max_count] + remaining_count = len(announcements) - max_count + + lines = [f"🔔 发现 {len(announcements)} 条新公告：", ""] + + for announcement in display_announcements: + title = announcement.title[:50] + "..." if len(announcement.title) > 50 else announcement.title + publish_date = announcement.publish_date.strftime("%m-%d") if announcement.publish_date else "N/A" + lines.append(f"• {title} ({publish_date})") + + if remaining_count > 0: + lines.append(f"... 还有 {remaining_count} 条公告") + + lines.append("") + lines.append("*点击公告标题查看详情*") + + return "\n".join(lines) + + +def create_markdown_generator(output_file: Optional[str] = None) -> MarkdownGenerator: + """ + 创建Markdown生成器实例 + + Args: + output_file: 输出文件路径 + + Returns: + MarkdownGenerator: 生成器实例 + """ + return MarkdownGenerator(output_file) + + +def generate_onu_md(announcements: List[Announcement]) -> bool: + """ + 生成onu.md文件 + + Args: + announcements: 公告列表 + + Returns: + bool: 生成是否成功 + """ + generator = create_markdown_generator() + return generator.save_to_file(announcements, "广西政府采购网公告监控") + + +def update_onu_md(new_announcements: List[Announcement]) -> bool: + """ + 更新onu.md文件，追加新公告 + + Args: + new_announcements: 新公告列表 + + Returns: + bool: 更新是否成功 + """ + generator = create_markdown_generator() + + # 如果文件不存在，创建新文件 + if not os.path.exists(generator.output_file): + return generator.save_to_file(new_announcements) + + # 否则追加新公告 + return generator.append_to_file(new_announcements) diff --git a/gx_gp_monitor/storage/postgresql.py b/gx_gp_monitor/storage/postgresql.py new file mode 100644 index 0000000..de761f0 --- /dev/null +++ b/gx_gp_monitor/storage/postgresql.py @@ -0,0 +1,367 @@ +""" +PostgreSQL存储模块 +实现公告数据的PostgreSQL存储和管理 +""" + +from typing import List, Dict, Any, Optional +from datetime import datetime, timedelta +import threading +from dataclasses import asdict + +try: + from ..core.models import Announcement, CrawlResult + from ..core.database import get_database_manager, init_database + from ..core.logger import get_logger + from ..core.reliability import retry_on_exception, RetryConfig, safe_execute +except ImportError: + from core.models import Announcement, CrawlResult + from core.database import get_database_manager, init_database + from core.logger import get_logger + from core.reliability import retry_on_exception, RetryConfig, safe_execute + + +logger = get_logger(__name__) + + +class PostgreSQLStorage: + """PostgreSQL存储管理器""" + + def __init__(self): + self.db_manager = get_database_manager() + self._lock = threading.Lock() + + def save_announcements(self, announcements: List[Announcement]) -> int: + """ + 保存公告列表到数据库 + + Args: + announcements: 公告列表 + + Returns: + int: 成功保存的数量 + """ + if not announcements: + return 0 + + logger.info(f"开始保存 {len(announcements)} 条公告到数据库") + + try: + # 批量保存 + saved_count = self.db_manager.save_announcements_batch(announcements) + + if saved_count > 0: + logger.info(f"成功保存 {saved_count} 条公告到数据库") + + # 标记新公告 + self._mark_new_announcements(announcements[:saved_count]) + + return saved_count + + except Exception as e: + logger.error(f"保存公告到数据库失败: {str(e)}") + # 尝试逐个保存 + return self._save_announcements_fallback(announcements) + + def _save_announcements_fallback(self, announcements: List[Announcement]) -> int: + """逐个保存公告的降级方案""" + logger.info("使用降级方案逐个保存公告") + + saved_count = 0 + for announcement in announcements: + try: + if self.db_manager.save_announcement(announcement): + saved_count += 1 + except Exception as e: + logger.warning(f"保存公告失败: {announcement.title[:50]}..., 错误: {str(e)}") + continue + + logger.info(f"降级保存完成，成功保存 {saved_count} 条公告") + return saved_count + + def _mark_new_announcements(self, announcements: List[Announcement]): + """标记新公告""" + # 这里可以添加新公告标记逻辑 + # 由于我们在爬取时已经标记，这里主要是确保数据库中的标记正确 + pass + + def save_crawl_results(self, results: List[CrawlResult]) -> int: + """ + 保存爬取结果 + + Args: + results: 爬取结果列表 + + Returns: + int: 成功保存的数量 + """ + if not results: + return 0 + + saved_count = 0 + for result in results: + try: + if self.db_manager.save_crawl_result(result): + saved_count += 1 + except Exception as e: + logger.warning(f"保存爬取结果失败: {result.source.name}, 错误: {str(e)}") + continue + + logger.info(f"保存爬取结果完成: {saved_count}/{len(results)}") + return saved_count + + def get_recent_announcements(self, hours: int = 24, + limit: int = 100) -> List[Announcement]: + """ + 获取最近的公告 + + Args: + hours: 最近小时数 + limit: 限制数量 + + Returns: + List[Announcement]: 公告列表 + """ + try: + return self.db_manager.get_recent_announcements(hours) + except Exception as e: + logger.error(f"获取最近公告失败: {str(e)}") + return [] + + def cleanup_expired_data(self, days: Optional[int] = None) -> int: + """ + 清理过期数据 + + Args: + days: 保留天数，如果为None则使用配置默认值 + + Returns: + int: 清理的记录数 + """ + from ..core.config_manager import get_config + + config = get_config() + if days is None: + days = config.database.data_retention_days + + logger.info(f"开始清理 {days} 天前的过期数据") + + try: + deleted_count = self.db_manager.cleanup_expired_data(days) + + if deleted_count > 0: + logger.info(f"成功清理 {deleted_count} 条过期数据") + else: + logger.info("没有找到需要清理的过期数据") + + return deleted_count + + except Exception as e: + logger.error(f"清理过期数据失败: {str(e)}") + return 0 + + def get_statistics(self) -> Dict[str, Any]: + """ + 获取存储统计信息 + + Returns: + Dict[str, Any]: 统计数据 + """ + try: + stats = self.db_manager.get_statistics() + stats.update({ + "storage_type": "postgresql", + "last_cleanup": datetime.now().isoformat() + }) + return stats + except Exception as e: + logger.error(f"获取存储统计信息失败: {str(e)}") + return { + "storage_type": "postgresql", + "error": str(e), + "total_announcements": 0, + "last_cleanup": datetime.now().isoformat() + } + + def search_announcements(self, keyword: Optional[str] = None, + source_code: Optional[str] = None, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + limit: int = 50) -> List[Announcement]: + """ + 搜索公告 + + Args: + keyword: 关键词 + source_code: 来源代码 + start_date: 开始日期 + end_date: 结束日期 + limit: 限制数量 + + Returns: + List[Announcement]: 搜索结果 + """ + # 这里可以实现更复杂的搜索逻辑 + # 目前使用现有的查询方法 + try: + return self.db_manager.get_announcements( + source_code=source_code, + start_date=start_date, + end_date=end_date, + limit=limit + ) + except Exception as e: + logger.error(f"搜索公告失败: {str(e)}") + return [] + + def is_healthy(self) -> bool: + """ + 检查存储健康状态 + + Returns: + bool: 是否健康 + """ + try: + # 尝试执行一个简单的查询 + stats = self.get_statistics() + return "error" not in stats + except Exception as e: + logger.error(f"存储健康检查失败: {str(e)}") + return False + + def optimize_storage(self): + """优化存储性能""" + # 这里可以添加数据库优化逻辑，如重建索引、清理碎片等 + logger.info("开始优化存储性能") + + try: + # 执行一些基本的优化操作 + # 注意：实际的优化命令取决于PostgreSQL版本和配置 + + # 这里可以添加具体的优化SQL + # 例如：VACUUM, REINDEX等 + + logger.info("存储优化完成") + except Exception as e: + logger.error(f"存储优化失败: {str(e)}") + + def backup_data(self, backup_path: Optional[str] = None) -> bool: + """ + 备份数据 + + Args: + backup_path: 备份文件路径 + + Returns: + bool: 备份是否成功 + """ + # 这里可以实现数据备份逻辑 + # 可以使用pg_dump或其他备份工具 + + logger.info("开始备份数据") + + try: + # 实现备份逻辑 + # 注意：这需要系统权限来执行pg_dump + + logger.info("数据备份完成") + return True + + except Exception as e: + logger.error(f"数据备份失败: {str(e)}") + return False + + +class StorageManager: + """存储管理器""" + + def __init__(self): + self.postgresql = PostgreSQLStorage() + self._current_storage = self.postgresql # 默认使用PostgreSQL + + def save_announcements(self, announcements: List[Announcement]) -> int: + """保存公告""" + return self._current_storage.save_announcements(announcements) + + def save_crawl_results(self, results: List[CrawlResult]) -> int: + """保存爬取结果""" + return self._current_storage.save_crawl_results(results) + + def get_recent_announcements(self, hours: int = 24, limit: int = 100) -> List[Announcement]: + """获取最近公告""" + return self._current_storage.get_recent_announcements(hours, limit) + + def cleanup_expired_data(self, days: Optional[int] = None) -> int: + """清理过期数据""" + return self._current_storage.cleanup_expired_data(days) + + def get_statistics(self) -> Dict[str, Any]: + """获取统计信息""" + return self._current_storage.get_statistics() + + def is_healthy(self) -> bool: + """检查健康状态""" + return self._current_storage.is_healthy() + + def optimize(self): + """优化存储""" + self._current_storage.optimize_storage() + + def backup(self, backup_path: Optional[str] = None) -> bool: + """备份数据""" + return self._current_storage.backup_data(backup_path) + + +# 全局存储管理器实例 +_storage_manager = None +_storage_lock = threading.Lock() + + +def get_storage_manager() -> StorageManager: + """ + 获取存储管理器实例 + + Returns: + StorageManager: 存储管理器实例 + """ + global _storage_manager + if _storage_manager is None: + with _storage_lock: + if _storage_manager is None: + _storage_manager = StorageManager() + return _storage_manager + + +def init_storage(): + """初始化存储""" + try: + init_database() + logger.info("存储初始化完成") + except Exception as e: + logger.error(f"存储初始化失败: {str(e)}") + raise + + +def save_announcements_to_storage(announcements: List[Announcement]) -> int: + """ + 保存公告到存储 + + Args: + announcements: 公告列表 + + Returns: + int: 保存成功的数量 + """ + return get_storage_manager().save_announcements(announcements) + + +def cleanup_storage(days: Optional[int] = None) -> int: + """ + 清理存储中的过期数据 + + Args: + days: 保留天数 + + Returns: + int: 清理的记录数 + """ + return get_storage_manager().cleanup_expired_data(days) diff --git a/logs/gx_gp_monitor.log b/logs/gx_gp_monitor.log new file mode 100644 index 0000000..047edc7 --- /dev/null +++ b/logs/gx_gp_monitor.log @@ -0,0 +1,23 @@ +2026-01-07 17:29:10 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:29:10 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:29:10 - gx_gp_monitor.scheduler.scheduler - INFO - 手动执行任务: data_cleanup +2026-01-07 17:29:10 - gx_gp_monitor.scheduler.scheduler - INFO - 开始执行数据清理任务 +2026-01-07 17:29:10 - gx_gp_monitor.core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:29:10 - gx_gp_monitor.core.database - INFO - 数据库连接池初始化成功 +2026-01-07 17:29:10 - gx_gp_monitor.storage.postgresql - INFO - 开始清理 90 天前的过期数据 +2026-01-07 17:29:10 - gx_gp_monitor.storage.postgresql - INFO - 开始清理 90 天前的过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.core.database - INFO - 清理过期数据完成，删除 0 条记录 +2026-01-07 17:29:11 - gx_gp_monitor.core.database - INFO - 清理过期数据完成，删除 0 条记录 +2026-01-07 17:29:11 - gx_gp_monitor.storage.postgresql - INFO - 没有找到需要清理的过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.storage.postgresql - INFO - 没有找到需要清理的过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.scheduler.scheduler - INFO - 数据清理任务完成：删除 0 条过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.scheduler.scheduler - INFO - 数据清理任务完成：删除 0 条过期数据 +2026-01-07 17:29:11 - gx_gp_monitor.scheduler.scheduler - INFO - 任务 data_cleanup 执行完成 +2026-01-07 17:29:11 - gx_gp_monitor.scheduler.scheduler - INFO - 任务 data_cleanup 执行完成 +2026-01-07 17:29:14 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:29:14 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:29:21 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: daily_crawl (0 8,14,18 * * *) +2026-01-07 17:29:21 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:29:21 - gx_gp_monitor.scheduler.scheduler - INFO - 添加间隔任务: custom_task (30秒) +2026-01-07 17:32:05 - gx_gp_monitor.scheduler.scheduler - INFO - 添加定时任务: data_cleanup (0 2 * * *) +2026-01-07 17:32:05 - gx_gp_monitor.scheduler.scheduler - INFO - 移除任务: data_cleanup diff --git a/old/WeChatService.py b/old/WeChatService.py new file mode 100644 index 0000000..ea1f596 --- /dev/null +++ b/old/WeChatService.py @@ -0,0 +1,625 @@ +import requests +import json +import xml.etree.ElementTree as ET +import hashlib +import time +import random +import string +from config import Config +import logging +from Crypto.Cipher import AES +import base64 +import socket +import struct +import urllib.parse +import redis + +logger = logging.getLogger(__name__) + +class WeChatService: + def __init__(self): + self.corpid = Config.WECHAT_CORPID + self.corpsecret = Config.WECHAT_CORPSECRET + self.agentid = Config.WECHAT_AGENTID + self.token = Config.WECHAT_TOKEN + self.encoding_aes_key = Config.WECHAT_ENCODING_AES_KEY + self.access_token = None + self.token_expires_at = 0 + # 添加代理API配置 + self.use_proxy = getattr(Config, 'USE_WECHAT_PROXY', False) + self.proxy_api_url = getattr(Config, 'WECHAT_PROXY_API_URL', 'https://api.v6ole.top') + + logger.info("WeChatService初始化完成") + + def _get_redis_connection(self): + """获取Redis连接""" + try: + redis_conn = redis.Redis( + host=Config.REDIS_HOST, + port=Config.REDIS_PORT, + db=Config.REDIS_DB, + password=Config.REDIS_PASSWORD, + decode_responses=True, # 自动将响应解码为字符串 + socket_timeout=5, # 设置超时时间 + socket_connect_timeout=5 + ) + # 测试连接 + redis_conn.ping() + return redis_conn + except Exception as e: + logger.error(f"Redis连接失败: {str(e)}") + raise + + def get_access_token(self): + """获取access_token""" + current_time = time.time() + + # 如果access_token未过期，直接返回 + if self.access_token and current_time < self.token_expires_at: + return self.access_token + + try: + # 使用代理API获取access_token + if self.use_proxy: + url = f"{self.proxy_api_url}/cgi-bin/gettoken?corpid={self.corpid}&corpsecret={self.corpsecret}" + logger.info(f"使用代理API获取access_token: {url}") + else: + url = f"https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={self.corpid}&corpsecret={self.corpsecret}" + + response = requests.get(url) + result = response.json() + + if result.get("errcode") == 0: + self.access_token = result.get("access_token") + # 设置过期时间，提前5分钟过期 + expires_in = result.get("expires_in", 7200) - 300 + self.token_expires_at = current_time + expires_in + + # 转换过期时间为可读格式 + expiry_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.token_expires_at)) + logger.info(f"成功获取access_token，过期时间: {expiry_time}") + return self.access_token + elif result.get("errcode") == 60020 and not self.use_proxy: + # 如果是IP限制错误并且未使用代理，尝试使用代理 + logger.info("IP受限，尝试使用代理API获取access_token") + self.use_proxy = True + return self.get_access_token() + else: + logger.error(f"获取access_token失败: {result}") + return None + except Exception as e: + logger.error(f"获取access_token异常: {str(e)}") + return None + + def verify_url(self, signature, timestamp, nonce, echostr): + """验证URL有效性""" + try: + # URL解码echostr + echostr = urllib.parse.unquote(echostr) + + # 1. 将token、timestamp、nonce、echostr四个参数进行字典序排序 + temp_list = [self.token, timestamp, nonce, echostr] + temp_list.sort() + + # 2. 将四个参数字符串拼接成一个字符串进行sha1加密 + temp_str = ''.join(temp_list) + hash_obj = hashlib.sha1(temp_str.encode('utf-8')) + hash_str = hash_obj.hexdigest() + + # 3. 开发者获得加密后的字符串可与signature对比，标识该请求来源于微信 + if hash_str == signature: + # 如果验证成功，需要解密echostr + if self.encoding_aes_key: + return self.decrypt_echostr(echostr) + return echostr + else: + logger.error(f"URL验证失败: signature={signature}, hash_str={hash_str}") + return "URL验证失败" + except Exception as e: + logger.error(f"URL验证异常: {str(e)}") + return "URL验证异常" + + def decrypt_echostr(self, echostr): + """解密echostr""" + try: + # 1. 对密文进行base64解码 + aes_key = base64.b64decode(self.encoding_aes_key + '=') + encrypted = base64.b64decode(echostr) + + # 2. 使用AES解密 + cipher = AES.new(aes_key, AES.MODE_CBC, aes_key[:16]) + decrypted = cipher.decrypt(encrypted) + + # 3. 去除补位字符 + unpad = lambda s: s[:-ord(s[len(s)-1:])] + decrypted = unpad(decrypted) + + # 4. 去除16位随机字符串 + content = decrypted[16:] + xml_len = socket.ntohl(struct.unpack("I", content[:4])[0]) + xml_content = content[4:xml_len+4] + + # 5. 验证企业ID + received_id = content[xml_len+4:].decode('utf-8') + if received_id != self.corpid: + logger.error(f"企业ID验证失败: received={received_id}, expected={self.corpid}") + return "企业ID验证失败" + + return xml_content.decode('utf-8') + except Exception as e: + logger.error(f"解密echostr失败: {str(e)}") + return "解密失败" + + def parse_message(self, xml_data): + """解析接收到的XML消息""" + try: + root = ET.fromstring(xml_data) + msg = {} + for child in root: + msg[child.tag] = child.text + + # 如果消息是加密的，需要解密 + if 'Encrypt' in msg: + logger.info("消息已加密，开始解密") + decrypted = self.decrypt_message(msg['Encrypt']) + logger.info(f"解密后的消息: {decrypted}") + # 解析解密后的XML + decrypted_root = ET.fromstring(decrypted) + msg = {} + for child in decrypted_root: + msg[child.tag] = child.text + + logger.info(f"最终解析的消息: {msg}") + return msg + except Exception as e: + logger.error(f"解析消息失败: {str(e)}") + return None + + def decrypt_message(self, encrypted_msg): + """解密消息""" + try: + # 1. 对密文进行base64解码 + aes_key = base64.b64decode(self.encoding_aes_key + '=') + encrypted = base64.b64decode(encrypted_msg) + + # 2. 使用AES解密 + cipher = AES.new(aes_key, AES.MODE_CBC, aes_key[:16]) + decrypted = cipher.decrypt(encrypted) + + # 3. 去除补位字符 + unpad = lambda s: s[:-ord(s[len(s)-1:])] + decrypted = unpad(decrypted) + + # 4. 去除16位随机字符串 + content = decrypted[16:] + xml_len = socket.ntohl(struct.unpack("I", content[:4])[0]) + xml_content = content[4:xml_len+4] + + # 5. 验证企业ID + received_id = content[xml_len+4:].decode('utf-8') + if received_id != self.corpid: + logger.error(f"企业ID验证失败: received={received_id}, expected={self.corpid}") + raise Exception("企业ID验证失败") + + return xml_content.decode('utf-8') + except Exception as e: + logger.error(f"解密消息失败: {str(e)}") + raise + + def send_text_message(self, content, to_user='@all', to_party='', to_tag=''): + """发送文本消息""" + max_retries = 3 + retry_count = 0 + + while retry_count < max_retries: + try: + access_token = self.get_access_token() + + # 构建消息数据 + data = { + "touser": to_user, + "toparty": to_party, + "totag": to_tag, + "msgtype": "text", + "agentid": self.agentid, + "text": { + "content": content + } + } + + logger.info(f"发送文本消息: {data}") + + # 使用代理API发送消息 + if self.use_proxy: + url = f"{self.proxy_api_url}/cgi-bin/message/send?access_token={access_token}" + logger.info(f"使用代理API发送消息: {url}") + else: + url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={access_token}" + + response = requests.post(url, json=data) + result = response.json() + logger.info(f"发送文本消息响应: {result}") + + # 如果token过期，重新获取并重试 + if result.get('errcode') == 40014: + logger.info("access_token过期，重新获取") + self.access_token = None + self.token_expires_at = 0 + retry_count += 1 + continue + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API发送") + self.use_proxy = True + retry_count += 1 + continue + + # 如果是其他错误，记录并返回 + if result.get('errcode') != 0: + logger.error(f"发送消息失败: {result.get('errmsg')}") + if result.get('errcode') == 301002: # 应用ID不匹配 + logger.error("应用ID不匹配，请检查配置") + break + retry_count += 1 + continue + + return result + except Exception as e: + logger.error(f"发送文本消息失败: {str(e)}") + retry_count += 1 + if retry_count >= max_retries: + raise + time.sleep(1) # 等待1秒后重试 + + return {"errcode": -1, "errmsg": "发送消息失败，已达到最大重试次数"} + + def send_markdown_message(self, content, to_user='@all', to_party='', to_tag=''): + """发送markdown消息""" + try: + access_token = self.get_access_token() + + data = { + "touser": to_user, + "toparty": to_party, + "totag": to_tag, + "msgtype": "markdown", + "agentid": self.agentid, + "markdown": { + "content": content + } + } + + logger.info(f"发送markdown消息: {data}") + + # 使用代理API发送消息 + if self.use_proxy: + url = f"{self.proxy_api_url}/cgi-bin/message/send?access_token={access_token}" + logger.info(f"使用代理API发送消息: {url}") + else: + url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={access_token}" + + response = requests.post(url, json=data) + result = response.json() + logger.info(f"发送markdown消息响应: {result}") + + # 如果token过期，重新获取并重试 + if result.get('errcode') == 40014: + logger.info("access_token过期，重新获取") + self.access_token = None + self.token_expires_at = 0 + return self.send_markdown_message(content, to_user, to_party, to_tag) + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API发送") + self.use_proxy = True + return self.send_markdown_message(content, to_user, to_party, to_tag) + + return result + except Exception as e: + logger.error(f"发送markdown消息失败: {str(e)}") + raise + + def get_user_info(self, userid): + """获取用户信息""" + try: + access_token = self.get_access_token() + + # 使用代理API获取用户信息 + if self.use_proxy: + url = f"{self.proxy_api_url}/cgi-bin/user/get?access_token={access_token}&userid={userid}" + logger.info(f"使用代理API获取用户信息: {url}") + else: + url = f"https://qyapi.weixin.qq.com/cgi-bin/user/get?access_token={access_token}&userid={userid}" + + logger.info(f"获取用户信息: {url}") + response = requests.get(url) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API获取用户信息") + self.use_proxy = True + return self.get_user_info(userid) + + logger.info(f"获取用户信息响应: {result}") + return result + except Exception as e: + logger.error(f"获取用户信息失败: {str(e)}") + return {"errcode": -1, "errmsg": str(e)} + + def generate_temp_token(self, user_id, user_name=None): + """生成临时访问令牌""" + try: + # 生成随机令牌 + token = ''.join(random.choices(string.ascii_letters + string.digits, k=32)) + + # 存储令牌信息到Redis + token_data = { + 'user_id': user_id, + 'name': user_name or '微信用户', + 'created_at': int(time.time()) + } + + # 使用Redis存储令牌，设置过期时间 + key = f"temp_token:{token}" + redis_conn = self._get_redis_connection() + redis_conn.setex( + key, + Config.REDIS_TEMP_TOKEN_EXPIRE, + json.dumps(token_data) + ) + + # 验证令牌是否成功存储 + stored_data = redis_conn.get(key) + if not stored_data: + logger.error("令牌存储失败") + return None + + logger.info(f"成功生成临时令牌: {token}, 存储数据: {stored_data}") + return token + + except Exception as e: + logger.error(f"生成临时令牌失败: {str(e)}") + return None + + def verify_temp_token(self, token): + """验证临时访问令牌""" + try: + # 从Redis中获取令牌信息 + key = f"temp_token:{token}" + redis_conn = self._get_redis_connection() + token_info = redis_conn.get(key) + + if not token_info: + logger.error(f"临时令牌不存在或已过期: {token}") + return None + + # 解析令牌信息 + token_data = json.loads(token_info) + user_id = token_data.get('user_id') + + if not user_id: + logger.error(f"临时令牌中未找到用户ID: {token_info}") + return None + + logger.info(f"临时令牌验证成功: {token}, 用户信息: {token_data}") + return token_data + + except Exception as e: + logger.error(f"验证临时令牌失败: {str(e)}") + return None + + def send_card_message(self, title, description, url, to_user): + """发送卡片消息""" + try: + # 获取access_token + access_token = self.get_access_token() + if not access_token: + logger.error("获取access_token失败") + return None + + # 构建消息内容 + data = { + "touser": to_user, + "toparty": "", + "totag": "", + "msgtype": "textcard", + "agentid": self.agentid, + "textcard": { + "title": title, + "description": description, + "url": url, + "btntxt": "查看详情" + } + } + + logger.info(f"发送卡片消息: {data}") + + # 使用代理API发送消息 + if self.use_proxy: + api_url = f"{self.proxy_api_url}/cgi-bin/message/send?access_token={access_token}" + logger.info(f"使用代理API发送消息: {api_url}") + else: + api_url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={access_token}" + + # 发送消息 + response = requests.post(api_url, json=data) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API发送") + self.use_proxy = True + return self.send_card_message(title, description, url, to_user) + + if result.get('errcode') == 0: + logger.info(f"发送卡片消息成功: {result}") + return result + else: + logger.error(f"发送卡片消息失败: {result}") + return None + + except Exception as e: + logger.error(f"发送卡片消息异常: {str(e)}") + return None + + def create_menu(self): + """创建应用菜单""" + try: + # 获取access_token + access_token = self.get_access_token() + if not access_token: + logger.error("获取access_token失败") + return None + + # 菜单配置 + menu_data = { + "button": [ + { + "name": "设备查询", + "sub_button": [ + { + "type": "click", + "name": "在线统计", + "key": "online" + }, + { + "type": "click", + "name": "设备状态", + "key": "status" + } + ] + }, + { + "name": "设备管理", + "sub_button": [ + { + "type": "click", + "name": "业务下发", + "key": "deploy" + }, + { + "type": "view", + "name": "设备管理", + "url": f"{Config.BASE_URL}/devices" + }, + { + "type": "click", + "name": "序列修复", + "key": "sequence_fix" + } + ] + }, + { + "type": "click", + "name": "帮助", + "key": "help" + } + ] + } + + logger.info(f"创建菜单: {menu_data}") + + # 使用代理API创建菜单 + if self.use_proxy: + api_url = f"{self.proxy_api_url}/cgi-bin/menu/create?access_token={access_token}&agentid={self.agentid}" + logger.info(f"使用代理API创建菜单: {api_url}") + else: + api_url = f"https://qyapi.weixin.qq.com/cgi-bin/menu/create?access_token={access_token}&agentid={self.agentid}" + + # 发送请求 + response = requests.post(api_url, json=menu_data) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API创建菜单") + self.use_proxy = True + return self.create_menu() + + if result.get('errcode') == 0: + logger.info(f"创建菜单成功: {result}") + return result + else: + logger.error(f"创建菜单失败: {result}") + return None + + except Exception as e: + logger.error(f"创建菜单异常: {str(e)}") + return None + + def delete_menu(self): + """删除应用菜单""" + try: + # 获取access_token + access_token = self.get_access_token() + if not access_token: + logger.error("获取access_token失败") + return None + + # 使用代理API删除菜单 + if self.use_proxy: + api_url = f"{self.proxy_api_url}/cgi-bin/menu/delete?access_token={access_token}&agentid={self.agentid}" + logger.info(f"使用代理API删除菜单: {api_url}") + else: + api_url = f"https://qyapi.weixin.qq.com/cgi-bin/menu/delete?access_token={access_token}&agentid={self.agentid}" + + # 发送请求 + response = requests.get(api_url) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API删除菜单") + self.use_proxy = True + return self.delete_menu() + + if result.get('errcode') == 0: + logger.info(f"删除菜单成功: {result}") + return result + else: + logger.error(f"删除菜单失败: {result}") + return None + + except Exception as e: + logger.error(f"删除菜单异常: {str(e)}") + return None + + def get_menu(self): + """获取应用菜单""" + try: + # 获取access_token + access_token = self.get_access_token() + if not access_token: + logger.error("获取access_token失败") + return None + + # 使用代理API获取菜单 + if self.use_proxy: + api_url = f"{self.proxy_api_url}/cgi-bin/menu/get?access_token={access_token}&agentid={self.agentid}" + logger.info(f"使用代理API获取菜单: {api_url}") + else: + api_url = f"https://qyapi.weixin.qq.com/cgi-bin/menu/get?access_token={access_token}&agentid={self.agentid}" + + # 发送请求 + response = requests.get(api_url) + result = response.json() + + # 如果是IP限制错误并且未使用代理，尝试使用代理 + if result.get('errcode') == 60020 and not self.use_proxy: + logger.info("IP受限，尝试使用代理API获取菜单") + self.use_proxy = True + return self.get_menu() + + if result.get('errcode') == 0: + logger.info(f"获取菜单成功: {result}") + return result + else: + logger.error(f"获取菜单失败: {result}") + return None + + except Exception as e: + logger.error(f"获取菜单异常: {str(e)}") + return None \ No newline at end of file diff --git a/old/config.yaml b/old/config.yaml new file mode 100644 index 0000000..4c386d0 --- /dev/null +++ b/old/config.yaml @@ -0,0 +1,104 @@ +# 广西政府采购网公告监控系统配置文件 +# 复制此文件为 config.yaml 并修改相应配置 + +# 调试模式 +debug: false + +# 日志配置 +log_level: INFO +log_file: logs/gx_gp_monitor.log + + +# 爬虫配置 +crawler: + base_url: "https://zfcg.gxzf.gov.cn" + timeout: 30 # 请求超时时间（秒） + max_retries: 3 # 最大重试次数 + retry_delay: 1.0 # 重试初始延迟 + max_retry_delay: 60.0 # 重试最大延迟 + backoff_factor: 2.0 # 退避因子 + user_agents: # User-Agent列表 + - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15" + proxies: [] # 代理列表 + request_delay: 1.0 # 请求间延迟 + request_delay_max: 3.0 # 请求间最大延迟 + keyword: ["大化", "信息化"] # 关键词筛选（支持多个关键词） + start_date: "" # 开始日期 (YYYY-MM-DD) + end_date: "" # 结束日期 (YYYY-MM-DD) + max_pages: 10 # 最大页数 + page_size: 100 # 每页大小 + +# 企业微信通知配置 +wechat_app: + enabled: true # 是否启用企业微信通知 + corp_id: "ww69e8e44636f47780" # 企业ID + agent_id: "1000007" # 应用ID + secret: "SmelCwKFoL0E9ATWFzr-w7gsfXBTN72lT1UqnNd0HpI" # 应用Secret + token: "DmvL98cAF6x9CFtQZwqD2emGL8S7HxA" # Token + encoding_aes_key: "yAc4OoSCP92YTefHXYfw27WeG9oF11W9d6nw6QYlU3D" # 消息加密Key + port: 18001 # 服务端口 + host: "0.0.0.0" # 服务主机 + debug: false # 调试模式 +# 数据库配置 + DB_NAME=gx-gp-notify + DB_USER=gx-gp-notify + DB_PASSWORD=MA6RBX4F6Bd5DGmw + DB_HOST=10.10.10.14 + DB_PORT=5432 + + +# 公告来源配置 +sources: + ZcyAnnouncement1: + category_id: 66485 + name: "采购公告" + type: "purchase" + ZcyAnnouncement2: + category_id: 66485 + name: "结果公告" + type: "result" + ZcyAnnouncement3: + category_id: 66485 + name: "合同公告" + type: "contract" + ZcyAnnouncement4: + category_id: 66485 + name: "更正公告" + type: "correction" + ZcyAnnouncement5: + category_id: 66485 + name: "招标文件预公示" + type: "pre_announcement" + ZcyAnnouncement6: + category_id: 66485 + name: "单一来源公示" + type: "single_source" + ZcyAnnouncement7: + category_id: 66485 + name: "电子卖场公示" + type: "electronic_market" + ZcyAnnouncement10: + category_id: 66485 + name: "履约验收公示" + type: "acceptance" + ZcyAnnouncement11: + category_id: 66485 + name: "工程类公告" + type: "engineering" + ZcyAnnouncement20: + category_id: 66485 + name: "框架协议征集公告" + type: "framework_agreement" + ZcyAnnouncement21: + category_id: 66485 + name: "框架协议入围结果公告" + type: "framework_result" + ZcyAnnouncement23: + category_id: 66485 + name: "框架协议成交结果汇总公告" + type: "framework_summary" + "61-266648": + category_id: 66485 + name: "采购意向公开" + type: "intention" diff --git a/old/main/Main_script.py b/old/main/Main_script.py new file mode 100755 index 0000000..89a29f8 --- /dev/null +++ b/old/main/Main_script.py @@ -0,0 +1,44 @@ +import web_crawler as webc + +"""脚本运行指南""" +# 整个程序的启动需点击右上角的绿色三角按钮(或使用快捷组合键Shift+F10快捷启动)，注意！在启动前，请先在按钮的左侧选择“当前文件”再执行。 +# 注意！！！请不要在打开公告数据导入的目标excel文件时启动该爬虫程序，程序无法对正在运行的进程文件进行修改。务必在爬虫程序启动并将数据成功导入目标excel文件后再打开并查看目标excel文件 + +"""脚本结果读取指南""" +# 这个脚本旨在爬取广西政府采购网的多个公告栏目的公告信息。 +#该脚本在运行结束之后会返回以下结果：<某专栏> 新增 xxx 条公告，已保存至excel文件中，建议确认是否有新增公告后再查看excel文件 +#每次爬取之后会把数据导出至指定excel表格，表格将会把数据按时间倒序的方式排列公告数据，同时会把属于今天的公告数据标红。excel文件默认为桌面的政府采购公告.xlsx('D:/Document/政府采购公告.xlsx') + +"""脚本参数修改指南""" +# 若没有创建相应的excel表格文件，无需担心，程序会先检测是否有对应的excel文件，若不存在该文件，程序便会自动生成。 +# excel表格的存取路径可以在utils.py中修改。请点击左侧的"项目"(或使用快捷组合键Alt+1打开项目栏)，选择utils.py并打开。可以在这个文件中修改某些参数。 +# 若需要修改公告信息的筛选条件以及启动爬虫程序的代理列表，请点击左侧的"项目"(或使用快捷组合键Alt+1打开项目栏)，选择utils.py并打开。可以在这个文件中修改这些参数。 +# 我已经设置了一个定时器，在每天的8:40、14:40、18:00这三个时间段，定时器会启动爬虫程序开始爬取网站。如果需要修改时间段，可以在代码的下半部分修改 +# 修改参数以及代码片段后请重新启动程序。 + +# """脚本定时启动""" +# def job(): +# print(f"任务执行于 {datetime.now()}") +# #调用爬取函数 +# webc.web_crawler() +# print(f"任务完成于 {datetime.now()}") +# +# # 安排任务在每天的8:40、14:40、18:00执行 +# # 此处可以修改时间段，只需修改括号内的时间即可，格式参照原先括号里的数即可，如需额外增加时间段，请复制代码：schedule.every().day.at("时间段").do(job)并粘贴到下方，可增加任意数量的时间段 +# schedule.every().day.at("08:37").do(job) +# schedule.every().day.at("11:55").do(job) +# schedule.every().day.at("14:45").do(job) +# schedule.every().day.at("18:00").do(job) +# +# print("脚本定时任务已启动...") +# while True: +# schedule.run_pending() +# time.sleep(1) + +"""脚本单次启动""" +def main(): + print("脚本单次任务已启动...") + webc.web_crawler() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/old/main/utils.py b/old/main/utils.py new file mode 100755 index 0000000..0e559ea --- /dev/null +++ b/old/main/utils.py @@ -0,0 +1,50 @@ +# 定义公告导出的目标excel文件路径 +FILE_PATH = "D:/Document/政府采购公告.xlsx" +# 初始化excel工作表头 +HEADER = ["标题", "发布时间", "发布单位", "内容链接", "来源栏目"] + +# 初始化工作表数量以及名称 +sheet_names = [ + "采购公告", "招标文件预公示", "采购意向公开", "结果公告", "合同公告", + "更正公告", "单一来源公示", "电子卖场公示", "履约验收公示", + "工程类公告", "框架协议征集公告", "框架协议入围结果公告", + "框架协议成交结果汇总公告", "其他" +] + +# === 初始化筛选条件 === +KEYWORD ="大化" #筛选关键词 +START_DATE ="2025-12-01" #筛选最早发布日期 +END_DATE ="2026-01-31" #筛选最晚发布日期 + +# 代理列表 +PROXIES = [ + #若要添加代理，请先测试代理是否能正常连接，否则请将代理置空 + #"http://10.10.1.10:3218", #此为无效代理,仅作示例 + #"http://user:pass@10.10.1.10:8080", #此为无效代理,仅作示例 + # 带认证的代理 + # 添加更多代理... +] + +USER_AGENTS = [ + #此为用户代理,根据实际情况修改 + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15", + # 添加更多 User-Agent... +] + +# 公告来源链接 +SOURCE_URL = { + "ZcyAnnouncement2":[66485,"结果公告"],# 结果公告 + "ZcyAnnouncement3":[66485,"合同公告"],# 合同公告 + "ZcyAnnouncement4":[66485,"更正公告"],# 更正公告 + "ZcyAnnouncement6":[66485,"单一来源公示"],# 单一来源公示 + "ZcyAnnouncement7":[66485,"电子卖场公示"],# 电子卖场公示 + "ZcyAnnouncement10":[66485,"履约验收公示"],# 履约验收公示 + "ZcyAnnouncement11":[66485,"工程类公告"],# 工程类公告 + "ZcyAnnouncement20":[66485,"框架协议征集公告"],# 框架协议征集公告 + "ZcyAnnouncement21":[66485,"框架协议入围结果公告"],# 框架协议入围结果公告 + "ZcyAnnouncement23":[66485,"框架协议成交结果汇总公告"],# 框架协议成交结果汇总公告 + "ZcyAnnouncement1":[66485,"采购公告"],# 采购公告 + "ZcyAnnouncement5":[66485,"招标文件预公示"],# 招标文件预公示 + "61-266648":[66485,"采购意向公开"]# 采购意向公开 +} \ No newline at end of file diff --git a/old/main/web_crawler.py b/old/main/web_crawler.py new file mode 100755 index 0000000..f92cc46 --- /dev/null +++ b/old/main/web_crawler.py @@ -0,0 +1,191 @@ +import requests +import time +from datetime import datetime +import random +from fake_useragent import UserAgent +import logging +import write_to_excel as wte +import utils as Utils + +# === 获取筛选条件 === +KEYWORD =Utils.KEYWORD #关键词筛选 +START_DATE =Utils.START_DATE +END_DATE =Utils.END_DATE + +#获取公告来源链接 +SOURCE_URL = Utils.SOURCE_URL + +# === 反扒配置 === + +#获取代理列表 +PROXIES = Utils.PROXIES +USER_AGENTS = Utils.USER_AGENTS + +MAX_RETRIES = 3 # 最大重连次数 +DELAY_MIN, DELAY_MAX = 1, 3 # 随机延迟范围（秒） + +# === 日志配置 === +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + +def get_random_user_agent(): + """随机选择 User-Agent""" + return random.choice(USER_AGENTS) + +def get_random_proxy(): + """随机选择代理""" + if not PROXIES: + return None + proxy = random.choice(PROXIES) + return {"http": proxy, "https": proxy} + +def check_sensitive_words(user_agent,payload,category_code,childrencode): + """敏感词检查请求""" + # === 接口地址 === + url = "https://zfcg.gxzf.gov.cn/portal/sensitiveWords/check" + headers = { + "User-Agent": user_agent, + "Content-Type": "application/json;charset=UTF-8", + "Origin": "https://zfcg.gxzf.gov.cn", + "Referer": f"https://zfcg.gxzf.gov.cn/site/category?parentId={category_code}&childrenCode={childrencode}", + "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" + } + response = requests.post(url, json=payload, headers=headers) + # print("这是敏感词检查请求返回的响应预览",response.json()) + return response.json() + +def get_announcements(category_code, childrencode,page_no=1, session=None): + """请求接口获取公告列表函数(带反扒机制)""" + if session is None: + session = requests.Session() + payload = { + # 接口的请求参数 + "keyword": KEYWORD, #关键词筛选，此参数即为搜索框中的关键词输入值 + "publishDateBegin": START_DATE, #最早日期筛选 + "publishDateEnd": END_DATE, #最晚日期筛选 + "pageNo": page_no, #页码数 + "pageSize": 15, #页容量 + "categoryCode": category_code, # 该参数指定当前查找的公告栏目 + "_t": int(time.time() * 1000) # 动态时间戳 + } + + # 先执行敏感词检查 + user_agent=get_random_user_agent() + check_response = check_sensitive_words(user_agent,payload, category_code, childrencode) + if not check_response.get("success", False): + logging.warning("敏感词检查失败") + return None + + # 再执行公告数据请求 + # === 接口地址 === + api_url = "https://zfcg.gxzf.gov.cn/portal/category" + headers = { + "User-Agent": user_agent, + "Content-Type": "application/json;charset=UTF-8", + "Origin": "https://zfcg.gxzf.gov.cn", + "Referer": f"https://zfcg.gxzf.gov.cn/site/category?parentId={category_code}&childrenCode={childrencode}", + "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" # 如果需要登录 + } + #若添加了至少1条代理，则尝试使用代理 + if PROXIES: + proxy = 'Default_value' + else: + proxy=None + for retry in range(MAX_RETRIES): + try: + # 尝试使用代理，失败则切换为无代理 + if proxy : + proxy = get_random_proxy() + + if proxy: + logging.info(f"使用代理: {proxy['http']}") + else: + logging.info("未使用代理") + + response = session.post( + api_url, + json=payload, + headers=headers, + proxies=proxy, + timeout=10 + ) + if response.status_code == 200: + data = response.json() + if data.get("success",True): + return data + else: + logging.warning(f"接口返回失败: {data.get('error', '未知错误')}") + else: + logging.warning(f"请求失败，状态码: {response.status_code}") + except requests.exceptions.ProxyError as pe: + # 代理错误时记录并跳过，继续无代理请求 + logging.error(f"代理错误: {pe}. 切换为无代理模式") + proxy = None # 下次请求不使用代理 + except Exception as e: + logging.error(f"请求异常: {e}") + + # 重试前等待 + delay = random.uniform(DELAY_MIN, DELAY_MAX) + logging.info(f"第 {retry + 1}/{MAX_RETRIES} 次重试，等待 {delay:.1f} 秒...") + time.sleep(delay) + + return None # 所有重试失败 + +def parse_data(data,category_code,source_name): + """解析公告数据函数""" + results = [] + for item in data["result"]["data"]["data"]: + results.append({ + "标题": item["title"], + "发布时间": datetime.fromtimestamp(int(item["publishDate"]) / 1000).strftime("%Y-%m-%d"), + "发布单位": item["purchaseName"], + "内容链接": f"https://zfcg.gxzf.gov.cn/site/detail?parentId={category_code}&articleId={item['articleId']}", + "来源栏目":source_name + }) + return results + +#爬虫主函数 +def web_crawler(): + session = requests.Session() + # print("这是本地会话存储：",session) + # 初始化引用值为信息公告,先爬取信息公告栏目的公告数据 + childrencode = "ZcyAnnouncement" + # 爬取代码主体，外循环为来源栏目的循环，即遍历需要爬取的所有来源栏目 + for key in SOURCE_URL: + page = 1 + all_results = [] + # print("这是正在爬取的栏目", SOURCE_URL[key][1]) + # print("这是正在爬取的栏目的目录码", str(key)) + # 判断正确的引用值 + # 内循环主体即为在符合筛选条件的公告列表中遍历所有公告，爬取每个公告需要的指定数据。 + while True: + # 这是分割线 + # logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线") + # logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线") + # logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线") + # logging.info(f"正在爬取第 {page} 页...") + response = get_announcements(str(key), childrencode, page,session) + if response["result"]["data"]["empty"] or not response["result"]["data"]["data"]: + break + # logging.info(f"请求后共返回 {response["result"]["data"]["total"]} 条公告") + current_data = parse_data(response, SOURCE_URL[key][0], SOURCE_URL[key][1]) + if not current_data: + break + # logging.info(f"解析后共爬取 {len(current_data)} 条公告") + all_results.extend(current_data) + page += 1 + # 随机延迟 + delay = random.uniform(DELAY_MIN, DELAY_MAX) + time.sleep(delay) + + # print(f"从 {SOURCE_URL[key][1]} 共爬取 {len(all_results)} 条公告") + # 调用excel读写函数，将公告数据筛选后保存到Excel中 + wte.write_to_excel(all_results,SOURCE_URL[key][1]) + # 这是分割线 + print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线") + print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线") + print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线") + + + + + diff --git a/old/main/write_to_excel.py b/old/main/write_to_excel.py new file mode 100755 index 0000000..cfeda12 --- /dev/null +++ b/old/main/write_to_excel.py @@ -0,0 +1,84 @@ +import pandas as pd +from datetime import datetime +from openpyxl.styles import Font, Alignment, PatternFill, Border, Side +import schedule +import time +import utils as Utils + +# 获取excel文件路径和excel的工作表头 +FILE_PATH = Utils.FILE_PATH +HEADER = Utils.HEADER + +# 获取工作表数量以及名称 +sheet_names = Utils.sheet_names + + +def write_to_excel(results,source=""): + # 检查文件是否存在，如果不存在则创建一个新的Excel文件 + try: + df_existing = pd.read_excel(FILE_PATH, sheet_name=None) + except FileNotFoundError: + df_existing = {name: pd.DataFrame(columns=HEADER) for name in sheet_names} + + today_str = datetime.now().strftime('%Y-%m-%d') + aditional_item=0 # 记录本次爬取新增的公告数量 + # 将爬取的公告对号入座填充至excel文件的工作表中 + for result in results: + sheet_name = result["来源栏目"] if result["来源栏目"] in sheet_names else "其他" + df_sheet = df_existing[sheet_name] + + # 检查是否已存在相同内容链接的记录 + if not df_sheet[df_sheet["内容链接"] == result["内容链接"]].empty: + continue + + # 添加新记录 + new_row = pd.DataFrame([result]) + df_sheet = pd.concat([df_sheet, new_row], ignore_index=True) + aditional_item+=1 # 若有新增公告，则加1 + # 按时间降序排列 + df_sheet.sort_values(by='发布时间', ascending=False, inplace=True) + + # 更新现有数据 + df_existing[sheet_name] = df_sheet + + # 写入Excel文件 + with pd.ExcelWriter(FILE_PATH, engine='openpyxl') as writer: + for sheet_name, df in df_existing.items(): + df.to_excel(writer, sheet_name=sheet_name, index=False) + + # 获取工作表对象 + worksheet = writer.sheets[sheet_name] + + # 设置列宽 + for col_idx, column_width in enumerate([40, 15, 20, 30, 20]): + worksheet.column_dimensions[chr(65 + col_idx)].width = column_width + + # 创建红色填充 + red_fill = PatternFill(start_color="FF0000", end_color="FF0000", fill_type="solid") + + # 创建边框样式 + thin_border = Border(left=Side(style='thin'), + right=Side(style='thin'), + top=Side(style='thin'), + bottom=Side(style='thin')) + + # 应用样式 + for row in worksheet.iter_rows(min_row=1, max_col=len(HEADER), max_row=worksheet.max_row): + publish_date = row[1].value if len(row) > 1 else None + # print("这是publish_date",publish_date) + # 若公告发布日期为今天，则将其单元格背景填充至红色 + if publish_date == today_str: + for cell in row: + cell.fill = red_fill + + for idx, cell in enumerate(row): + alignment = Alignment(horizontal='left' if idx < 4 else 'general', vertical='top', wrap_text=True) + cell.alignment = alignment + cell.border = thin_border + print(f"<{source}> 新增 {aditional_item} 条公告，已保存至excel文件中") + + + + + + diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..66e4951 --- /dev/null +++ b/start.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# 广西政府采购网公告监控系统启动脚本 + +echo "=== 广西政府采购网公告监控系统启动脚本 ===" + +# 检查Python环境 +if ! command -v python3 &> /dev/null; then + echo "错误: 未找到python3，请确保Python 3.8+已安装" + exit 1 +fi + +# 检查虚拟环境 +if [ ! -d "venv" ]; then + echo "错误: 未找到虚拟环境，请先运行: python3 -m venv venv" + exit 1 +fi + +# 激活虚拟环境 +echo "激活虚拟环境..." +source venv/bin/activate + +# 检查依赖 +echo "检查依赖..." +python3 -c "import sys; sys.path.insert(0, 'gx_gp_monitor'); from core.config_manager import load_config; print('依赖检查通过')" 2>/dev/null +if [ $? -ne 0 ]; then + echo "错误: 依赖不完整，请运行: pip install -r gx_gp_monitor/requirements.txt" + exit 1 +fi + +# 进入项目目录 +cd gx_gp_monitor + +# 解析命令行参数 +COMMAND=${1:-"status"} +shift + +echo "执行命令: $COMMAND" + +# 执行命令 +case $COMMAND in + "crawl") + python3 main.py crawl "$@" + ;; + "cleanup") + python3 main.py cleanup "$@" + ;; + "status") + python3 main.py status "$@" + ;; + *) + echo "使用方法: $0 {crawl|cleanup|status} [参数]" + echo "" + echo "示例:" + echo " $0 crawl # 执行一次爬取" + echo " $0 crawl --keywords 大化 # 带关键词爬取" + echo " $0 cleanup --days 30 # 清理30天前数据" + echo " $0 status # 查看系统状态" + exit 1 + ;; +esac + +echo "=== 脚本执行完成 ===" diff --git a/test_system.py b/test_system.py new file mode 100644 index 0000000..446a3a0 --- /dev/null +++ b/test_system.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +系统测试脚本 +""" + +import sys +import os +from pathlib import Path + +# 添加项目路径 +project_root = Path(__file__).parent / "gx_gp_monitor" +sys.path.insert(0, str(project_root)) + +def test_imports(): + """测试模块导入""" + print("测试模块导入...") + + try: + from core.config_manager import load_config, get_config + print("✓ core.config_manager 导入成功") + + from core.logger import get_logger + print("✓ core.logger 导入成功") + + from core.models import Announcement, AnnouncementType + print("✓ core.models 导入成功") + + from core.database import get_database_manager + print("✓ core.database 导入成功") + + from crawler.spider import GXGPSpider + print("✓ crawler.spider 导入成功") + + from filters.filters import AnnouncementFilter + print("✓ filters.filters 导入成功") + + from storage.postgresql import PostgreSQLStorage + print("✓ storage.postgresql 导入成功") + + from notification.wechat import WeChatService + print("✓ notification.wechat 导入成功") + + return True + except ImportError as e: + print(f"✗ 导入失败: {e}") + return False + +def test_config(): + """测试配置加载""" + print("\n测试配置加载...") + + try: + from core.config_manager import load_config + + config = load_config() + print("✓ 配置文件加载成功") + print(f" - 调试模式: {config.debug}") + print(f" - 日志级别: {config.log_level.value}") + print(f" - 数据库启用: {config.database.enabled}") + print(f" - 企业微信启用: {config.wechat_app.enabled}") + print(f" - 公告来源数量: {len(config.sources)}") + + return True + except Exception as e: + print(f"✗ 配置加载失败: {e}") + return False + +def test_database_connection(): + """测试数据库连接""" + print("\n测试数据库连接...") + + try: + from core.config_manager import get_config + from core.database import init_database + + config = get_config() + if not config.database.enabled: + print("⚠ 数据库功能已禁用，跳过连接测试") + return True + + # 尝试初始化数据库 + init_database() + print("✓ 数据库连接成功") + return True + + except Exception as e: + print(f"✗ 数据库连接失败: {e}") + return False + +def main(): + """主测试函数""" + print("=== 广西政府采购网公告监控系统测试 ===") + + # 测试导入 + if not test_imports(): + print("\n❌ 模块导入测试失败") + return False + + # 测试配置 + if not test_config(): + print("\n❌ 配置加载测试失败") + return False + + # 测试数据库 + if not test_database_connection(): + print("\n❌ 数据库连接测试失败") + return False + + print("\n✅ 所有测试通过！系统准备就绪。") + return True + +if __name__ == "__main__": + success = main() + sys.exit(0 if success else 1)