手动模式
This commit is contained in:
@@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
广西政府采购网公告监控系统主程序
|
||||
广西政府采购网公告爬取和监控的智能系统
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
import signal
|
||||
from pathlib import Path
|
||||
|
||||
# 添加项目根目录到Python路径
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
try:
|
||||
# 尝试相对导入
|
||||
from .core.config_manager import load_config, get_config
|
||||
from .core.logger import init_logger, get_logger
|
||||
from .core.reliability import check_system_health
|
||||
from .crawler.spider import crawl_announcements
|
||||
from .filters.filters import filter_from_config
|
||||
from .storage.postgresql import init_storage, save_announcements_to_storage, cleanup_storage
|
||||
from .storage.md_generator import generate_onu_md
|
||||
from .notification.wechat import send_announcements_notification, send_system_notification
|
||||
except ImportError:
|
||||
try:
|
||||
# 尝试绝对导入（直接运行脚本时）
|
||||
from core.config_manager import load_config, get_config
|
||||
from core.logger import init_logger, get_logger
|
||||
from core.reliability import check_system_health
|
||||
from crawler.spider import crawl_announcements
|
||||
from filters.filters import filter_from_config
|
||||
from storage.postgresql import init_storage, save_announcements_to_storage, cleanup_storage
|
||||
from storage.md_generator import generate_onu_md
|
||||
from notification.wechat import send_announcements_notification, send_system_notification
|
||||
except ImportError as e:
|
||||
print(f"导入错误: {e}", file=sys.stderr)
|
||||
print("请确保依赖已正确安装: pip install -r requirements.txt", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class GXGPMonitorApp:
|
||||
"""广西政府采购网监控系统应用"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = None
|
||||
self.logger = None
|
||||
self.running = False
|
||||
|
||||
def initialize(self, config_file: str = None):
|
||||
"""初始化应用"""
|
||||
try:
|
||||
# 加载配置
|
||||
self.config = load_config(config_file)
|
||||
|
||||
# 初始化日志
|
||||
self.logger = init_logger(config=self.config)
|
||||
|
||||
logger.info("=== 广西政府采购网公告监控系统启动 ===")
|
||||
logger.info(f"版本: 1.0.0")
|
||||
logger.info(f"配置文件: {config_file or '默认配置'}")
|
||||
|
||||
# 系统健康检查
|
||||
if not check_system_health():
|
||||
logger.warning("系统健康检查失败，但继续运行")
|
||||
|
||||
# 初始化存储
|
||||
init_storage()
|
||||
|
||||
logger.info("应用初始化完成")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"应用初始化失败: {str(e)}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None):
|
||||
"""执行爬取任务"""
|
||||
try:
|
||||
logger.info("开始执行爬取任务")
|
||||
|
||||
# 执行爬取
|
||||
crawl_results = crawl_announcements(keywords, sources)
|
||||
|
||||
if not crawl_results:
|
||||
logger.info("爬取完成：无数据")
|
||||
return {"success": True, "results": []}
|
||||
|
||||
# 收集所有公告
|
||||
all_announcements = []
|
||||
for result in crawl_results:
|
||||
if result.announcements:
|
||||
all_announcements.extend(result.announcements)
|
||||
|
||||
logger.info(f"爬取到 {len(all_announcements)} 条原始公告")
|
||||
|
||||
# 筛选公告
|
||||
filter_obj = filter_from_config()
|
||||
filtered_announcements, filter_stats = filter_obj.filter(all_announcements)
|
||||
|
||||
logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告")
|
||||
|
||||
# 保存到数据库
|
||||
saved_count = save_announcements_to_storage(filtered_announcements)
|
||||
|
||||
# 生成Markdown文件
|
||||
md_success = generate_onu_md(filtered_announcements)
|
||||
|
||||
# 发送通知
|
||||
notify_success = False
|
||||
if filtered_announcements and self.config.wechat_app.enabled:
|
||||
notify_success = send_announcements_notification(filtered_announcements)
|
||||
|
||||
result = {
|
||||
"success": True,
|
||||
"total_crawled": len(all_announcements),
|
||||
"filtered": len(filtered_announcements),
|
||||
"saved": saved_count,
|
||||
"markdown_generated": md_success,
|
||||
"notification_sent": notify_success,
|
||||
"filter_stats": {
|
||||
"keyword_filtered": filter_stats.keyword_filtered,
|
||||
"date_filtered": filter_stats.date_filtered,
|
||||
"duplicate_filtered": filter_stats.duplicate_filtered,
|
||||
"source_filtered": filter_stats.source_filtered
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(f"爬取任务完成: {result}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"爬取任务执行失败: {str(e)}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
def run_cleanup(self, days: int = None):
|
||||
"""执行数据清理任务"""
|
||||
try:
|
||||
logger.info("开始执行数据清理任务")
|
||||
|
||||
deleted_count = cleanup_storage(days)
|
||||
|
||||
logger.info(f"数据清理完成：删除 {deleted_count} 条过期数据")
|
||||
|
||||
# 发送通知
|
||||
if deleted_count > 0 and self.config.wechat_app.enabled:
|
||||
send_system_notification(
|
||||
"数据清理完成",
|
||||
f"已清理 {deleted_count} 条过期数据"
|
||||
)
|
||||
|
||||
return {"success": True, "deleted": deleted_count}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"数据清理任务执行失败: {str(e)}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
|
||||
def show_status(self):
|
||||
"""显示系统状态"""
|
||||
try:
|
||||
status = {
|
||||
"system": {
|
||||
"version": "1.0.0",
|
||||
"healthy": check_system_health()
|
||||
},
|
||||
"config": {
|
||||
"debug": self.config.debug,
|
||||
"log_level": self.config.log_level.value
|
||||
},
|
||||
"database": {
|
||||
"enabled": self.config.database.enabled,
|
||||
"type": self.config.database.type
|
||||
},
|
||||
"wechat": {
|
||||
"enabled": self.config.wechat_app.enabled
|
||||
}
|
||||
}
|
||||
|
||||
# 格式化输出
|
||||
print("\n=== 系统状态 ===")
|
||||
print(f"系统健康: {'正常' if status['system']['healthy'] else '异常'}")
|
||||
print(f"调试模式: {'开启' if status['config']['debug'] else '关闭'}")
|
||||
print(f"日志级别: {status['config']['log_level']}")
|
||||
print(f"数据库: {'启用' if status['database']['enabled'] else '禁用'} ({status['database']['type']})")
|
||||
print(f"企业微信: {'启用' if status['wechat']['enabled'] else '禁用'}")
|
||||
|
||||
return status
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取系统状态失败: {str(e)}")
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
def create_argument_parser():
|
||||
"""创建命令行参数解析器"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="广西政府采购网公告监控系统",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
使用示例:
|
||||
python main.py crawl # 执行一次爬取
|
||||
python main.py crawl --keywords "大化" # 爬取指定关键词
|
||||
python main.py cleanup # 执行数据清理
|
||||
python main.py status # 查看系统状态
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'command',
|
||||
choices=['crawl', 'cleanup', 'status'],
|
||||
help='要执行的命令'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--config', '-c',
|
||||
help='配置文件路径'
|
||||
)
|
||||
|
||||
# crawl命令的参数
|
||||
parser.add_argument(
|
||||
'--keywords', '-k',
|
||||
nargs='+',
|
||||
help='关键词过滤（多个关键词用空格分隔）'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--sources', '-s',
|
||||
nargs='+',
|
||||
help='来源代码过滤（多个来源用空格分隔）'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--max-pages',
|
||||
type=int,
|
||||
help='最大爬取页数'
|
||||
)
|
||||
|
||||
# cleanup命令的参数
|
||||
parser.add_argument(
|
||||
'--days', '-d',
|
||||
type=int,
|
||||
help='清理多少天前的过期数据'
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
parser = create_argument_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
# 创建应用实例
|
||||
app = GXGPMonitorApp()
|
||||
|
||||
# 初始化应用
|
||||
if not app.initialize(args.config):
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
if args.command == 'crawl':
|
||||
# 执行爬取
|
||||
result = app.run_crawl(
|
||||
keywords=args.keywords,
|
||||
sources=args.sources,
|
||||
max_pages=args.max_pages
|
||||
)
|
||||
|
||||
if result["success"]:
|
||||
print("✅ 爬取任务执行成功")
|
||||
print(f" 爬取公告: {result['total_crawled']}")
|
||||
print(f" 筛选后: {result['filtered']}")
|
||||
print(f" 保存数量: {result['saved']}")
|
||||
if result.get("markdown_generated"):
|
||||
print(" Markdown文件: 已生成")
|
||||
if result.get("notification_sent"):
|
||||
print(" 通知发送: 已发送")
|
||||
else:
|
||||
print(f"❌ 爬取任务执行失败: {result.get('error', '未知错误')}")
|
||||
sys.exit(1)
|
||||
|
||||
elif args.command == 'cleanup':
|
||||
# 执行清理
|
||||
result = app.run_cleanup(days=args.days)
|
||||
if result["success"]:
|
||||
print(f"✅ 数据清理完成，删除 {result['deleted']} 条记录")
|
||||
else:
|
||||
print(f"❌ 数据清理失败: {result.get('error', '未知错误')}")
|
||||
sys.exit(1)
|
||||
|
||||
elif args.command == 'status':
|
||||
# 显示状态
|
||||
app.show_status()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("收到中断信号，正在退出...")
|
||||
except Exception as e:
|
||||
logger.error(f"程序执行异常: {str(e)}")
|
||||
print(f"❌ 程序执行异常: {str(e)}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user