157 lines
6.0 KiB
Python
Executable File
157 lines
6.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
定时搜索脚本
|
|
执行搜索、筛选关键词、保存到数据库并发送企业微信卡片通知
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# 添加项目根目录到路径（gx_gp_monitor的父目录）
|
|
project_root = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
try:
|
|
from gx_gp_monitor.core.config_manager import load_config, get_config
|
|
from gx_gp_monitor.core.logger import init_logger, get_logger
|
|
from gx_gp_monitor.crawler.spider import crawl_announcements
|
|
from gx_gp_monitor.filters.filters import filter_from_config
|
|
from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, save_auto_announcements_to_storage
|
|
from gx_gp_monitor.notification.wechat import send_announcements_notification, send_system_notification
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
def main():
|
|
"""主函数：执行定时搜索任务"""
|
|
try:
|
|
# 加载配置
|
|
config = load_config()
|
|
if not config:
|
|
logger.error("无法加载配置")
|
|
return False
|
|
|
|
# 初始化日志
|
|
init_logger(config=config)
|
|
|
|
logger.info("=== 开始定时搜索任务 ===")
|
|
|
|
# 初始化存储
|
|
init_storage()
|
|
|
|
# 执行搜索（爬取所有公告，然后进行关键词筛选）
|
|
logger.info("开始执行定时搜索任务")
|
|
crawl_results = crawl_announcements()
|
|
|
|
if not crawl_results:
|
|
logger.info("爬取完成：无数据")
|
|
return True
|
|
|
|
# 收集所有公告
|
|
all_announcements = []
|
|
for result in crawl_results:
|
|
if result.announcements:
|
|
all_announcements.extend(result.announcements)
|
|
|
|
total_crawled = len(all_announcements)
|
|
logger.info(f"搜索到 {total_crawled} 条原始公告")
|
|
|
|
if not all_announcements:
|
|
logger.info("没有获取到任何公告")
|
|
return True
|
|
|
|
# 对所有公告进行关键词筛选
|
|
from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
|
|
from datetime import date
|
|
|
|
# 1. 关键词筛选
|
|
keyword_filter = KeywordFilter()
|
|
keyword_filtered = keyword_filter.filter_announcements(all_announcements, keywords=config.crawler.keyword)
|
|
|
|
# 2. 日期筛选（只保留今天的）
|
|
date_filter = DateFilter()
|
|
today_announcements = date_filter.filter_announcements(
|
|
keyword_filtered,
|
|
start_date=date.today(),
|
|
end_date=date.today()
|
|
)
|
|
|
|
logger.info(f"关键词筛选后剩余 {len(keyword_filtered)} 条公告")
|
|
logger.info(f"筛选出今天 {len(today_announcements)} 条匹配公告")
|
|
|
|
if not today_announcements:
|
|
logger.info("今天没有匹配关键词的公告")
|
|
return True
|
|
|
|
# 构造筛选统计信息
|
|
filter_stats = type('FilterResult', (), {
|
|
"keyword_filtered": len(all_announcements) - len(keyword_filtered),
|
|
"date_filtered": len(keyword_filtered) - len(today_announcements),
|
|
"duplicate_filtered": 0,
|
|
"source_filtered": 0
|
|
})()
|
|
|
|
filtered_announcements = today_announcements
|
|
|
|
logger.info(f"关键词筛选后剩余 {len(filtered_announcements)} 条公告")
|
|
|
|
if not filtered_announcements:
|
|
logger.info("没有匹配关键词的公告，任务完成")
|
|
return True
|
|
|
|
# 保存筛选后的公告到定时搜索专用表
|
|
saved_count = save_auto_announcements_to_storage(filtered_announcements)
|
|
logger.info(f"保存定时搜索公告完成：{saved_count} 条")
|
|
|
|
# 发送企业微信卡片通知
|
|
if config.wechat_app.enabled:
|
|
logger.info("开始发送企业微信卡片通知...")
|
|
notify_success = send_announcements_notification(filtered_announcements)
|
|
|
|
if notify_success:
|
|
logger.info("企业微信卡片通知发送成功")
|
|
else:
|
|
logger.error("企业微信卡片通知发送失败")
|
|
else:
|
|
logger.info("企业微信通知未启用，跳过发送")
|
|
|
|
# 输出统计信息
|
|
print("\n=== 定时搜索任务完成 ===")
|
|
print(f"总共爬取: {total_crawled} 条公告")
|
|
print(f"关键词筛选: {len(keyword_filtered)} 条")
|
|
print(f"今日匹配公告: {len(today_announcements)} 条")
|
|
print(f"筛选后公告: {len(filtered_announcements)} 条")
|
|
print(f"保存到数据库: {saved_count} 条")
|
|
print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")
|
|
|
|
logger.info("=== 定时搜索任务完成 ===")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"定时搜索任务执行失败: {str(e)}")
|
|
print(f"❌ 定时搜索任务失败: {str(e)}", file=sys.stderr)
|
|
|
|
# 尝试发送错误通知
|
|
try:
|
|
if config and config.wechat_app.enabled:
|
|
send_system_notification(
|
|
"定时搜索任务失败",
|
|
f"错误信息: {str(e)}"
|
|
)
|
|
except Exception as notify_error:
|
|
logger.error(f"发送错误通知失败: {notify_error}")
|
|
|
|
return False
|
|
|
|
if __name__ == "__main__":
|
|
success = main()
|
|
sys.exit(0 if success else 1)
|
|
|
|
except ImportError as e:
|
|
print(f"导入失败: {e}", file=sys.stderr)
|
|
print("请确保已安装所有依赖: pip install -r gx_gp_monitor/requirements.txt", file=sys.stderr)
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
print(f"脚本执行失败: {e}", file=sys.stderr)
|
|
sys.exit(1)
|