315 lines
11 KiB
Python
315 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
广西政府采购网公告监控系统主程序
|
|
广西政府采购网公告爬取和监控的智能系统
|
|
"""
|
|
|
|
import sys
|
|
import argparse
|
|
import signal
|
|
from pathlib import Path
|
|
|
|
# 添加项目根目录到Python路径
|
|
project_root = Path(__file__).parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
try:
|
|
# 尝试相对导入
|
|
from .core.config_manager import load_config, get_config
|
|
from .core.logger import init_logger, get_logger
|
|
from .core.reliability import check_system_health
|
|
from .crawler.spider import crawl_announcements
|
|
from .filters.filters import filter_from_config
|
|
from .storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, cleanup_storage
|
|
from .storage.md_generator import generate_onu_md
|
|
from .notification.wechat import send_announcements_notification, send_system_notification
|
|
except ImportError:
|
|
try:
|
|
# 尝试绝对导入（直接运行脚本时）
|
|
from core.config_manager import load_config, get_config
|
|
from core.logger import init_logger, get_logger
|
|
from core.reliability import check_system_health
|
|
from crawler.spider import crawl_announcements
|
|
from filters.filters import filter_from_config
|
|
from storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, cleanup_storage
|
|
from storage.md_generator import generate_onu_md
|
|
from notification.wechat import send_announcements_notification, send_system_notification
|
|
except ImportError as e:
|
|
print(f"导入错误: {e}", file=sys.stderr)
|
|
print("请确保依赖已正确安装: pip install -r requirements.txt", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
class GXGPMonitorApp:
|
|
"""广西政府采购网监控系统应用"""
|
|
|
|
def __init__(self):
|
|
self.config = None
|
|
self.logger = None
|
|
self.running = False
|
|
|
|
def initialize(self, config_file: str = None):
|
|
"""初始化应用"""
|
|
try:
|
|
# 加载配置
|
|
self.config = load_config(config_file)
|
|
|
|
# 初始化日志
|
|
self.logger = init_logger(config=self.config)
|
|
|
|
logger.info("=== 广西政府采购网公告监控系统启动 ===")
|
|
logger.info(f"版本: 1.0.0")
|
|
logger.info(f"配置文件: {config_file or '默认配置'}")
|
|
|
|
# 系统健康检查
|
|
if not check_system_health():
|
|
logger.warning("系统健康检查失败，但继续运行")
|
|
|
|
# 初始化存储
|
|
init_storage()
|
|
|
|
logger.info("应用初始化完成")
|
|
return True
|
|
|
|
except Exception as e:
|
|
print(f"应用初始化失败: {str(e)}", file=sys.stderr)
|
|
return False
|
|
|
|
def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None):
|
|
"""执行爬取任务"""
|
|
try:
|
|
logger.info("开始执行爬取任务")
|
|
|
|
# 执行爬取
|
|
crawl_results = crawl_announcements(keywords, sources)
|
|
|
|
if not crawl_results:
|
|
logger.info("爬取完成：无数据")
|
|
return {"success": True, "results": []}
|
|
|
|
# 收集所有公告
|
|
all_announcements = []
|
|
for result in crawl_results:
|
|
if result.announcements:
|
|
all_announcements.extend(result.announcements)
|
|
|
|
logger.info(f"爬取到 {len(all_announcements)} 条原始公告")
|
|
|
|
# 先保存所有公告（按来源分组，每源保留最新100条）
|
|
all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100)
|
|
all_saved_count = sum(all_saved_stats.values())
|
|
logger.info(f"保存所有公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}")
|
|
|
|
# 筛选公告
|
|
filter_obj = filter_from_config()
|
|
filtered_announcements, filter_stats = filter_obj.filter(all_announcements)
|
|
|
|
logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告")
|
|
|
|
# 保存筛选后的公告（用于标记关键词匹配等）
|
|
saved_count = save_announcements_to_storage(filtered_announcements)
|
|
|
|
# 生成Markdown文件
|
|
md_success = generate_onu_md(filtered_announcements)
|
|
|
|
# 发送通知
|
|
notify_success = False
|
|
if filtered_announcements and self.config.wechat_app.enabled:
|
|
notify_success = send_announcements_notification(filtered_announcements)
|
|
|
|
result = {
|
|
"success": True,
|
|
"total_crawled": len(all_announcements),
|
|
"filtered": len(filtered_announcements),
|
|
"saved": saved_count,
|
|
"markdown_generated": md_success,
|
|
"notification_sent": notify_success,
|
|
"filter_stats": {
|
|
"keyword_filtered": filter_stats.keyword_filtered,
|
|
"date_filtered": filter_stats.date_filtered,
|
|
"duplicate_filtered": filter_stats.duplicate_filtered,
|
|
"source_filtered": filter_stats.source_filtered
|
|
}
|
|
}
|
|
|
|
logger.info(f"爬取任务完成: {result}")
|
|
return result
|
|
|
|
except Exception as e:
|
|
logger.error(f"爬取任务执行失败: {str(e)}")
|
|
return {"success": False, "error": str(e)}
|
|
|
|
def run_cleanup(self, days: int = None):
|
|
"""执行数据清理任务"""
|
|
try:
|
|
logger.info("开始执行数据清理任务")
|
|
|
|
deleted_count = cleanup_storage(days)
|
|
|
|
logger.info(f"数据清理完成：删除 {deleted_count} 条过期数据")
|
|
|
|
# 发送通知
|
|
if deleted_count > 0 and self.config.wechat_app.enabled:
|
|
send_system_notification(
|
|
"数据清理完成",
|
|
f"已清理 {deleted_count} 条过期数据"
|
|
)
|
|
|
|
return {"success": True, "deleted": deleted_count}
|
|
|
|
except Exception as e:
|
|
logger.error(f"数据清理任务执行失败: {str(e)}")
|
|
return {"success": False, "error": str(e)}
|
|
|
|
|
|
def show_status(self):
|
|
"""显示系统状态"""
|
|
try:
|
|
status = {
|
|
"system": {
|
|
"version": "1.0.0",
|
|
"healthy": check_system_health()
|
|
},
|
|
"config": {
|
|
"debug": self.config.debug,
|
|
"log_level": self.config.log_level.value
|
|
},
|
|
"database": {
|
|
"enabled": self.config.database.enabled,
|
|
"type": self.config.database.type
|
|
},
|
|
"wechat": {
|
|
"enabled": self.config.wechat_app.enabled
|
|
}
|
|
}
|
|
|
|
# 格式化输出
|
|
print("\n=== 系统状态 ===")
|
|
print(f"系统健康: {'正常' if status['system']['healthy'] else '异常'}")
|
|
print(f"调试模式: {'开启' if status['config']['debug'] else '关闭'}")
|
|
print(f"日志级别: {status['config']['log_level']}")
|
|
print(f"数据库: {'启用' if status['database']['enabled'] else '禁用'} ({status['database']['type']})")
|
|
print(f"企业微信: {'启用' if status['wechat']['enabled'] else '禁用'}")
|
|
|
|
return status
|
|
|
|
except Exception as e:
|
|
logger.error(f"获取系统状态失败: {str(e)}")
|
|
return {"error": str(e)}
|
|
|
|
|
|
def create_argument_parser():
|
|
"""创建命令行参数解析器"""
|
|
parser = argparse.ArgumentParser(
|
|
description="广西政府采购网公告监控系统",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
使用示例:
|
|
python main.py crawl # 执行一次爬取
|
|
python main.py crawl --keywords "大化" # 爬取指定关键词
|
|
python main.py cleanup # 执行数据清理
|
|
python main.py status # 查看系统状态
|
|
"""
|
|
)
|
|
|
|
parser.add_argument(
|
|
'command',
|
|
choices=['crawl', 'cleanup', 'status'],
|
|
help='要执行的命令'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--config', '-c',
|
|
help='配置文件路径'
|
|
)
|
|
|
|
# crawl命令的参数
|
|
parser.add_argument(
|
|
'--keywords', '-k',
|
|
nargs='+',
|
|
help='关键词过滤（多个关键词用空格分隔）'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--sources', '-s',
|
|
nargs='+',
|
|
help='来源代码过滤（多个来源用空格分隔）'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--max-pages',
|
|
type=int,
|
|
help='最大爬取页数'
|
|
)
|
|
|
|
# cleanup命令的参数
|
|
parser.add_argument(
|
|
'--days', '-d',
|
|
type=int,
|
|
help='清理多少天前的过期数据'
|
|
)
|
|
|
|
return parser
|
|
|
|
|
|
def main():
|
|
"""主函数"""
|
|
parser = create_argument_parser()
|
|
args = parser.parse_args()
|
|
|
|
# 创建应用实例
|
|
app = GXGPMonitorApp()
|
|
|
|
# 初始化应用
|
|
if not app.initialize(args.config):
|
|
sys.exit(1)
|
|
|
|
try:
|
|
if args.command == 'crawl':
|
|
# 执行爬取
|
|
result = app.run_crawl(
|
|
keywords=args.keywords,
|
|
sources=args.sources,
|
|
max_pages=args.max_pages
|
|
)
|
|
|
|
if result["success"]:
|
|
print("✅ 爬取任务执行成功")
|
|
print(f" 爬取公告: {result['total_crawled']}")
|
|
print(f" 筛选后: {result['filtered']}")
|
|
print(f" 保存数量: {result['saved']}")
|
|
if result.get("markdown_generated"):
|
|
print(" Markdown文件: 已生成")
|
|
if result.get("notification_sent"):
|
|
print(" 通知发送: 已发送")
|
|
else:
|
|
print(f"❌ 爬取任务执行失败: {result.get('error', '未知错误')}")
|
|
sys.exit(1)
|
|
|
|
elif args.command == 'cleanup':
|
|
# 执行清理
|
|
result = app.run_cleanup(days=args.days)
|
|
if result["success"]:
|
|
print(f"✅ 数据清理完成，删除 {result['deleted']} 条记录")
|
|
else:
|
|
print(f"❌ 数据清理失败: {result.get('error', '未知错误')}")
|
|
sys.exit(1)
|
|
|
|
elif args.command == 'status':
|
|
# 显示状态
|
|
app.show_status()
|
|
|
|
except KeyboardInterrupt:
|
|
logger.info("收到中断信号，正在退出...")
|
|
except Exception as e:
|
|
logger.error(f"程序执行异常: {str(e)}")
|
|
print(f"❌ 程序执行异常: {str(e)}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|