#!/usr/bin/env python3 """ 广西政府采购网公告监控系统主程序 广西政府采购网公告爬取和监控的智能系统 """ import sys import argparse import signal from pathlib import Path from typing import Dict, Any # 添加项目根目录到Python路径 project_root = Path(__file__).parent sys.path.insert(0, str(project_root)) try: # 尝试相对导入 from .core.config_manager import load_config, get_config from .core.logger import init_logger, get_logger from .core.reliability import check_system_health from .crawler.spider import crawl_announcements from .filters.filters import filter_from_config from .storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, save_manual_announcements_by_source_to_storage, cleanup_storage from .storage.md_generator import generate_onu_md from .notification.wechat import send_announcements_notification, send_system_notification from .wechat.callback_server import get_callback_server from .wechat.menu_manager import WeChatMenuManager except ImportError: try: # 尝试绝对导入（直接运行脚本时） from core.config_manager import load_config, get_config from core.logger import init_logger, get_logger from core.reliability import check_system_health from crawler.spider import crawl_announcements from filters.filters import filter_from_config from storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, cleanup_storage from storage.md_generator import generate_onu_md from notification.wechat import send_announcements_notification, send_system_notification # 企业微信模块动态导入，避免循环导入问题 wechat_available = True try: from wechat.callback_server import get_callback_server from wechat.menu_manager import WeChatMenuManager except ImportError: wechat_available = False print("企业微信模块不可用", file=sys.stderr) except ImportError as e: print(f"导入错误: {e}", file=sys.stderr) print("请确保依赖已正确安装: pip install -r requirements.txt", file=sys.stderr) sys.exit(1) logger = get_logger(__name__) class GXGPMonitorApp: """广西政府采购网监控系统应用""" def __init__(self): self.config = None self.logger = None self.running = False def initialize(self, config_file: str = None): """初始化应用""" try: # 加载配置 self.config = load_config(config_file) # 初始化日志 self.logger = init_logger(config=self.config) logger.info("=== 广西政府采购网公告监控系统启动 ===") logger.info(f"版本: 1.0.0") logger.info(f"配置文件: {config_file or '默认配置'}") # 系统健康检查 if not check_system_health(): logger.warning("系统健康检查失败，但继续运行") # 初始化存储 init_storage() logger.info("应用初始化完成") return True except Exception as e: print(f"应用初始化失败: {str(e)}", file=sys.stderr) return False def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None, manual_crawl: bool = False): """执行搜索任务 Args: keywords: 关键词列表 sources: 来源列表 max_pages: 最大页数 manual_crawl: 是否为手动搜索（只筛选今天的数据） """ try: logger.info(f"开始执行搜索任务 (手动搜索: {manual_crawl})") # 执行爬取 crawl_results = crawl_announcements(keywords, sources) if not crawl_results: logger.info("搜索完成：无数据") return {"success": True, "results": []} # 收集所有公告 all_announcements = [] for result in crawl_results: if result.announcements: all_announcements.extend(result.announcements) logger.info(f"搜索到 {len(all_announcements)} 条原始公告") # 保存公告到对应的专用表 if not manual_crawl: # 自动爬取：保存到auto_announcements表（关键词匹配专用） all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100) all_saved_count = sum(all_saved_stats.values()) logger.info(f"保存自动爬取公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}") else: # 手动搜索：保存到manual_announcements表（全量数据专用） all_saved_stats = save_manual_announcements_by_source_to_storage(all_announcements, max_per_source=100) all_saved_count = sum(all_saved_stats.values()) logger.info(f"保存手动搜索公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}") # 筛选公告 if manual_crawl: # 手动搜索：只筛选今天的公告和用户指定的关键词，不进行去重 from .filters.filters import KeywordFilter, DateFilter, SourceFilter from datetime import date # 1. 日期筛选：只保留今天的公告 date_filter = DateFilter() date_filtered = date_filter.filter_announcements(all_announcements, start_date=date.today(), end_date=date.today()) # 2. 关键词筛选 keyword_filter = KeywordFilter() keyword_filtered = keyword_filter.filter_announcements(date_filtered, keywords=keywords or []) # 3. 来源筛选 source_filter = SourceFilter() filtered_announcements = source_filter.filter_announcements(keyword_filtered, sources or list(self.config.sources.keys())) # 计算统计信息 filter_stats = type('FilterResult', (), { "keyword_filtered": len(date_filtered) - len(keyword_filtered), "date_filtered": len(all_announcements) - len(date_filtered), "duplicate_filtered": 0, # 手动搜索不进行去重 "source_filtered": len(keyword_filtered) - len(filtered_announcements) })() else: # 自动爬取：筛选出新公告并应用关键词筛选 # 1. 筛选出数据库中没有的新公告 new_announcements = [ann for ann in all_announcements if ann.is_new] logger.info(f"筛选出 {len(new_announcements)} 条新公告") # 2. 对新公告应用关键词筛选等 if new_announcements: filter_obj = filter_from_config() filtered_announcements, filter_stats = filter_obj.filter(new_announcements) # 更新统计信息，加上未筛选的新公告数量 filter_stats.keyword_filtered += len(new_announcements) - len(filtered_announcements) else: filtered_announcements = [] filter_stats = type('FilterResult', (), { "keyword_filtered": 0, "date_filtered": 0, "duplicate_filtered": len(all_announcements), "source_filtered": 0 })() logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告") # 对于手动爬取，不保存筛选后的公告到数据库 if not manual_crawl: # 保存筛选后的公告（用于标记关键词匹配等） saved_count = save_announcements_to_storage(filtered_announcements) else: saved_count = 0 logger.info("手动爬取模式：跳过筛选后公告的数据库保存") # 生成Markdown文件（只在自动爬取时生成） md_success = False if not manual_crawl: md_success = generate_onu_md(filtered_announcements) # 发送通知 notify_success = False if not manual_crawl and filtered_announcements and self.config.wechat_app.enabled: # 自动爬取时发送卡片消息 notify_success = send_announcements_notification(filtered_announcements) elif manual_crawl and filtered_announcements and self.config.wechat_app.enabled: # 手动爬取时不在这里发送消息，由消息处理器负责发送markdown消息 notify_success = True # 标记为成功，因为消息会通过其他方式发送 result = { "success": True, "total_crawled": len(all_announcements), "filtered": len(filtered_announcements), "saved": saved_count, "markdown_generated": md_success, "notification_sent": notify_success, "filter_stats": { "keyword_filtered": filter_stats.keyword_filtered, "date_filtered": filter_stats.date_filtered, "duplicate_filtered": filter_stats.duplicate_filtered, "source_filtered": filter_stats.source_filtered } } # 对于手动爬取，额外返回筛选后的公告列表 if manual_crawl: result["filtered_announcements"] = filtered_announcements logger.info(f"爬取任务完成: {result}") return result except Exception as e: logger.error(f"爬取任务执行失败: {str(e)}") return {"success": False, "error": str(e)} def run_cleanup(self, days: int = None): """执行数据清理任务""" try: logger.info("开始执行数据清理任务") deleted_count = cleanup_storage(days) logger.info(f"数据清理完成：删除 {deleted_count} 条过期数据") # 发送通知 if deleted_count > 0 and self.config.wechat_app.enabled: send_system_notification( "数据清理完成", f"已清理 {deleted_count} 条过期数据" ) return {"success": True, "deleted": deleted_count} except Exception as e: logger.error(f"数据清理任务执行失败: {str(e)}") return {"success": False, "error": str(e)} def run_wechat_server(self, host: str = '0.0.0.0', port: int = 18001): """启动企业微信回调服务器""" if not wechat_available: return {"success": False, "error": "企业微信模块不可用"} try: logger.info("启动企业微信回调服务器") # 获取回调服务器 callback_server = get_callback_server() # 启动服务器 callback_server.run(host=host, port=port, debug=self.config.debug) return {"success": True, "host": host, "port": port} except Exception as e: logger.error(f"启动企业微信回调服务器失败: {str(e)}") return {"success": False, "error": str(e)} def manage_wechat_menu(self, action: str) -> Dict[str, Any]: """管理企业微信菜单""" if not wechat_available: return {"success": False, "error": "企业微信模块不可用"} try: logger.info(f"执行企业微信菜单操作: {action}") menu_manager = WeChatMenuManager() if action == 'create': success = menu_manager.create_menu() result = {"success": success, "action": "create"} elif action == 'delete': success = menu_manager.delete_menu() result = {"success": success, "action": "delete"} elif action == 'get': menu_info = menu_manager.get_menu() result = {"success": menu_info is not None, "action": "get", "menu": menu_info} elif action == 'test': test_results = menu_manager.test_menu_operations() result = {"success": True, "action": "test", "results": test_results} else: result = {"success": False, "error": f"未知操作: {action}"} if result["success"]: logger.info(f"企业微信菜单操作成功: {action}") else: logger.error(f"企业微信菜单操作失败: {action}") return result except Exception as e: logger.error(f"企业微信菜单管理异常: {str(e)}") return {"success": False, "error": str(e)} def show_status(self): """显示系统状态""" try: status = { "system": { "version": "1.0.0", "healthy": check_system_health() }, "config": { "debug": self.config.debug, "log_level": self.config.log_level.value }, "database": { "enabled": self.config.database.enabled, "type": self.config.database.type }, "wechat": { "enabled": self.config.wechat_app.enabled } } # 格式化输出 print("\n=== 系统状态 ===") print(f"系统健康: {'正常' if status['system']['healthy'] else '异常'}") print(f"调试模式: {'开启' if status['config']['debug'] else '关闭'}") print(f"日志级别: {status['config']['log_level']}") print(f"数据库: {'启用' if status['database']['enabled'] else '禁用'} ({status['database']['type']})") print(f"企业微信: {'启用' if status['wechat']['enabled'] else '禁用'}") return status except Exception as e: logger.error(f"获取系统状态失败: {str(e)}") return {"error": str(e)} def create_argument_parser(): """创建命令行参数解析器""" parser = argparse.ArgumentParser( description="广西政府采购网公告监控系统", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" 使用示例: python main.py crawl # 执行一次爬取 python main.py crawl --keywords "大化" # 爬取指定关键词 python main.py cleanup # 执行数据清理 python main.py status # 查看系统状态 python main.py wechat-server # 启动企业微信回调服务器 python main.py wechat-menu --action create # 创建企业微信菜单 """ ) parser.add_argument( 'command', choices=['crawl', 'cleanup', 'status', 'wechat-server', 'wechat-menu'], help='要执行的命令' ) parser.add_argument( '--config', '-c', help='配置文件路径' ) # crawl命令的参数 parser.add_argument( '--keywords', '-k', nargs='+', help='关键词过滤（多个关键词用空格分隔）' ) parser.add_argument( '--sources', '-s', nargs='+', help='来源代码过滤（多个来源用空格分隔）' ) parser.add_argument( '--max-pages', type=int, help='最大爬取页数' ) # cleanup命令的参数 parser.add_argument( '--days', '-d', type=int, help='清理多少天前的过期数据' ) # wechat-server命令的参数 parser.add_argument( '--host', '-H', default='0.0.0.0', help='服务器监听主机地址 (默认: 0.0.0.0)' ) parser.add_argument( '--port', '-P', type=int, default=18001, help='服务器监听端口 (默认: 18001)' ) # wechat-menu命令的参数 parser.add_argument( '--action', '-a', choices=['create', 'delete', 'get', 'test'], default='create', help='菜单操作类型 (默认: create)' ) return parser def main(): """主函数""" parser = create_argument_parser() args = parser.parse_args() # 创建应用实例 app = GXGPMonitorApp() # 初始化应用 if not app.initialize(args.config): sys.exit(1) try: if args.command == 'crawl': # 执行爬取 result = app.run_crawl( keywords=args.keywords, sources=args.sources, max_pages=args.max_pages ) if result["success"]: print("✅ 爬取任务执行成功") print(f" 爬取公告: {result['total_crawled']}") print(f" 筛选后: {result['filtered']}") print(f" 保存数量: {result['saved']}") if result.get("markdown_generated"): print(" Markdown文件: 已生成") if result.get("notification_sent"): print(" 通知发送: 已发送") else: print(f"❌ 爬取任务执行失败: {result.get('error', '未知错误')}") sys.exit(1) elif args.command == 'cleanup': # 执行清理 result = app.run_cleanup(days=args.days) if result["success"]: print(f"✅ 数据清理完成，删除 {result['deleted']} 条记录") else: print(f"❌ 数据清理失败: {result.get('error', '未知错误')}") sys.exit(1) elif args.command == 'status': # 显示状态 app.show_status() elif args.command == 'wechat-server': # 启动企业微信回调服务器 result = app.run_wechat_server(host=args.host, port=args.port) if result["success"]: print(f"✅ 企业微信回调服务器已启动: {result['host']}:{result['port']}") print(" 回调地址: /api/v1/wechat/callback") else: print(f"❌ 企业微信回调服务器启动失败: {result.get('error', '未知错误')}") sys.exit(1) elif args.command == 'wechat-menu': # 企业微信菜单管理 result = app.manage_wechat_menu(action=args.action) if result["success"]: if args.action == 'create': print("✅ 企业微信菜单创建成功") elif args.action == 'delete': print("✅ 企业微信菜单删除成功") elif args.action == 'get': print("✅ 企业微信菜单获取成功") if result.get("menu"): print("菜单信息:", json.dumps(result["menu"], indent=2, ensure_ascii=False)) elif args.action == 'test': print("✅ 企业微信菜单测试完成") print("测试结果:", result.get("results")) else: error = result.get("error", "未知错误") print(f"❌ 企业微信菜单操作失败: {error}") sys.exit(1) except KeyboardInterrupt: logger.info("收到中断信号，正在退出...") except Exception as e: logger.error(f"程序执行异常: {str(e)}") print(f"❌ 程序执行异常: {str(e)}", file=sys.stderr) sys.exit(1) if __name__ == "__main__": main()