136 lines
5.3 KiB
Python
Executable File
136 lines
5.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
定时爬取脚本
|
|
执行爬取、筛选关键词、保存到数据库并发送企业微信卡片通知
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# 添加项目根目录到路径（gx_gp_monitor的父目录）
|
|
project_root = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
try:
|
|
from gx_gp_monitor.core.config_manager import load_config, get_config
|
|
from gx_gp_monitor.core.logger import init_logger, get_logger
|
|
from gx_gp_monitor.crawler.spider import crawl_announcements
|
|
from gx_gp_monitor.filters.filters import filter_from_config
|
|
from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage
|
|
from gx_gp_monitor.notification.wechat import send_announcements_notification, send_system_notification
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
def main():
|
|
"""主函数：执行定时爬取任务"""
|
|
try:
|
|
# 加载配置
|
|
config = load_config()
|
|
if not config:
|
|
logger.error("无法加载配置")
|
|
return False
|
|
|
|
# 初始化日志
|
|
init_logger(config=config)
|
|
|
|
logger.info("=== 开始定时爬取任务 ===")
|
|
|
|
# 初始化存储
|
|
init_storage()
|
|
|
|
# 执行爬取
|
|
logger.info("开始执行爬取...")
|
|
crawl_results = crawl_announcements()
|
|
|
|
if not crawl_results:
|
|
logger.info("爬取完成：无数据")
|
|
return True
|
|
|
|
# 收集所有公告
|
|
all_announcements = []
|
|
for result in crawl_results:
|
|
if result.announcements:
|
|
all_announcements.extend(result.announcements)
|
|
|
|
total_crawled = len(all_announcements)
|
|
logger.info(f"爬取到 {total_crawled} 条原始公告")
|
|
|
|
# 保存所有公告（按来源分组，每源保留最新100条）
|
|
all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100)
|
|
all_saved_count = sum(all_saved_stats.values())
|
|
logger.info(f"保存所有公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}")
|
|
|
|
# 筛选出新公告（数据库中没有的）
|
|
new_announcements = [ann for ann in all_announcements if ann.is_new]
|
|
logger.info(f"筛选出 {len(new_announcements)} 条新公告")
|
|
|
|
if not new_announcements:
|
|
logger.info("没有新的公告，任务完成")
|
|
return True
|
|
|
|
# 对新公告应用关键词筛选
|
|
filter_obj = filter_from_config()
|
|
filtered_announcements, filter_stats = filter_obj.filter(new_announcements)
|
|
|
|
logger.info(f"关键词筛选后剩余 {len(filtered_announcements)} 条公告")
|
|
|
|
if not filtered_announcements:
|
|
logger.info("没有匹配关键词的公告，任务完成")
|
|
return True
|
|
|
|
# 保存筛选后的公告（用于标记关键词匹配等）
|
|
saved_count = save_announcements_to_storage(filtered_announcements)
|
|
logger.info(f"保存筛选后公告完成：{saved_count} 条")
|
|
|
|
# 发送企业微信卡片通知
|
|
if config.wechat_app.enabled:
|
|
logger.info("开始发送企业微信卡片通知...")
|
|
notify_success = send_announcements_notification(filtered_announcements)
|
|
|
|
if notify_success:
|
|
logger.info("企业微信卡片通知发送成功")
|
|
else:
|
|
logger.error("企业微信卡片通知发送失败")
|
|
else:
|
|
logger.info("企业微信通知未启用，跳过发送")
|
|
|
|
# 输出统计信息
|
|
print("\n=== 定时爬取任务完成 ===")
|
|
print(f"总共爬取: {total_crawled} 条公告")
|
|
print(f"新增公告: {len(new_announcements)} 条")
|
|
print(f"关键词筛选: {len(filtered_announcements)} 条")
|
|
print(f"保存到数据库: {saved_count} 条")
|
|
print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")
|
|
|
|
logger.info("=== 定时爬取任务完成 ===")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"定时爬取任务执行失败: {str(e)}")
|
|
print(f"❌ 定时爬取任务失败: {str(e)}", file=sys.stderr)
|
|
|
|
# 尝试发送错误通知
|
|
try:
|
|
if config and config.wechat_app.enabled:
|
|
send_system_notification(
|
|
"定时爬取任务失败",
|
|
f"错误信息: {str(e)}"
|
|
)
|
|
except Exception as notify_error:
|
|
logger.error(f"发送错误通知失败: {notify_error}")
|
|
|
|
return False
|
|
|
|
if __name__ == "__main__":
|
|
success = main()
|
|
sys.exit(0 if success else 1)
|
|
|
|
except ImportError as e:
|
|
print(f"导入失败: {e}", file=sys.stderr)
|
|
print("请确保已安装所有依赖: pip install -r gx_gp_monitor/requirements.txt", file=sys.stderr)
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
print(f"脚本执行失败: {e}", file=sys.stderr)
|
|
sys.exit(1)
|