182 lines
7.5 KiB
Python
Executable File
182 lines
7.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
定时搜索脚本
|
|
执行搜索、筛选关键词、保存到数据库并发送企业微信卡片通知
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# 添加项目根目录到路径（gx_gp_monitor的父目录）
|
|
project_root = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
try:
|
|
from gx_gp_monitor.core.config_manager import load_config, get_config
|
|
from gx_gp_monitor.core.logger import init_logger, get_logger
|
|
from gx_gp_monitor.crawler.spider import crawl_announcements
|
|
from gx_gp_monitor.filters.filters import filter_from_config
|
|
from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, save_auto_announcements_to_storage
|
|
from gx_gp_monitor.notification.wechat import send_announcements_notification, send_system_notification
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
def main():
|
|
"""主函数：执行定时搜索任务"""
|
|
try:
|
|
# 加载配置
|
|
config = load_config()
|
|
if not config:
|
|
logger.error("无法加载配置")
|
|
return False
|
|
|
|
# 初始化日志
|
|
init_logger(config=config)
|
|
|
|
logger.info("=== 开始定时搜索任务 ===")
|
|
|
|
# 初始化存储
|
|
init_storage()
|
|
|
|
# 执行搜索（爬取所有公告，筛选出新增的关键词匹配公告）
|
|
logger.info("开始执行定时搜索任务")
|
|
crawl_results = crawl_announcements()
|
|
|
|
if not crawl_results:
|
|
logger.info("爬取完成：无数据")
|
|
return True
|
|
|
|
# 收集所有公告
|
|
all_announcements = []
|
|
for result in crawl_results:
|
|
if result.announcements:
|
|
all_announcements.extend(result.announcements)
|
|
|
|
total_crawled = len(all_announcements)
|
|
logger.info(f"搜索到 {total_crawled} 条原始公告")
|
|
|
|
if not all_announcements:
|
|
logger.info("没有获取到任何公告")
|
|
return True
|
|
|
|
# 对所有公告进行关键词筛选
|
|
from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
|
|
from datetime import date
|
|
|
|
keyword_filter = KeywordFilter()
|
|
keyword_filtered = keyword_filter.filter_announcements(all_announcements, keywords=config.crawler.keyword)
|
|
|
|
# 对关键词筛选结果进行日期筛选（只处理今天的）
|
|
date_filter = DateFilter()
|
|
today_keyword_announcements = date_filter.filter_announcements(
|
|
keyword_filtered,
|
|
start_date=date.today(),
|
|
end_date=date.today()
|
|
)
|
|
|
|
logger.info(f"关键词筛选后剩余 {len(keyword_filtered)} 条公告")
|
|
logger.info(f"筛选出今天关键词匹配 {len(today_keyword_announcements)} 条公告")
|
|
|
|
if not today_keyword_announcements:
|
|
logger.info("今天没有关键词匹配的公告")
|
|
return True
|
|
|
|
# 检查auto_announcements表，筛选出真正新增的公告
|
|
truly_new_announcements = []
|
|
for ann in today_keyword_announcements:
|
|
try:
|
|
# 直接导入，避免相对导入问题
|
|
import gx_gp_monitor.core.database as db_module
|
|
with db_module.get_db_cursor() as cursor:
|
|
cursor.execute(
|
|
"SELECT 1 FROM auto_announcements WHERE content_hash = %s LIMIT 1",
|
|
(ann.content_hash,)
|
|
)
|
|
exists = cursor.fetchone() is not None
|
|
if not exists:
|
|
truly_new_announcements.append(ann)
|
|
except Exception as e:
|
|
logger.warning(f"检查公告是否存在失败: {str(e)}")
|
|
# 如果检查失败，为了安全起见，不添加到新公告列表
|
|
pass
|
|
|
|
logger.info(f"从今天关键词匹配公告中筛选出 {len(truly_new_announcements)} 条auto_announcements表中不存在的新公告")
|
|
|
|
if not truly_new_announcements:
|
|
logger.info("没有真正新增的关键词匹配公告，任务完成")
|
|
return True
|
|
|
|
# 构造筛选统计信息
|
|
filter_stats = type('FilterResult', (), {
|
|
"keyword_filtered": len(all_announcements) - len(keyword_filtered),
|
|
"date_filtered": len(keyword_filtered) - len(today_keyword_announcements),
|
|
"duplicate_filtered": len(today_keyword_announcements) - len(truly_new_announcements),
|
|
"source_filtered": 0
|
|
})()
|
|
|
|
filtered_announcements = truly_new_announcements
|
|
|
|
logger.info(f"关键词筛选后剩余 {len(filtered_announcements)} 条公告")
|
|
|
|
if not filtered_announcements:
|
|
logger.info("没有匹配关键词的公告，任务完成")
|
|
return True
|
|
|
|
# 保存筛选后的公告到定时搜索专用表
|
|
saved_count = save_auto_announcements_to_storage(filtered_announcements)
|
|
logger.info(f"保存定时搜索公告完成：{saved_count} 条")
|
|
|
|
# 发送企业微信卡片通知
|
|
if config.wechat_app.enabled:
|
|
logger.info("开始发送企业微信卡片通知...")
|
|
notify_success = send_announcements_notification(filtered_announcements)
|
|
|
|
if notify_success:
|
|
logger.info("企业微信卡片通知发送成功")
|
|
else:
|
|
logger.error("企业微信卡片通知发送失败")
|
|
else:
|
|
logger.info("企业微信通知未启用，跳过发送")
|
|
|
|
# 输出统计信息
|
|
print("\n=== 定时搜索任务完成 ===")
|
|
print(f"总共爬取: {total_crawled} 条公告")
|
|
print(f"关键词筛选: {len(keyword_filtered)} 条")
|
|
print(f"今日关键词匹配: {len(today_keyword_announcements)} 条")
|
|
print(f"真正新增公告: {len(truly_new_announcements)} 条")
|
|
print(f"筛选后公告: {len(filtered_announcements)} 条")
|
|
print(f"保存到数据库: {saved_count} 条")
|
|
print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")
|
|
|
|
logger.info("=== 定时搜索任务完成 ===")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"定时搜索任务执行失败: {str(e)}")
|
|
print(f"❌ 定时搜索任务失败: {str(e)}", file=sys.stderr)
|
|
|
|
# 尝试发送错误通知
|
|
try:
|
|
if config and config.wechat_app.enabled:
|
|
send_system_notification(
|
|
"定时搜索任务失败",
|
|
f"错误信息: {str(e)}"
|
|
)
|
|
except Exception as notify_error:
|
|
logger.error(f"发送错误通知失败: {notify_error}")
|
|
|
|
return False
|
|
|
|
if __name__ == "__main__":
|
|
success = main()
|
|
sys.exit(0 if success else 1)
|
|
|
|
except ImportError as e:
|
|
print(f"导入失败: {e}", file=sys.stderr)
|
|
print("请确保已安装所有依赖: pip install -r gx_gp_monitor/requirements.txt", file=sys.stderr)
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
print(f"脚本执行失败: {e}", file=sys.stderr)
|
|
sys.exit(1)
|