v6ole
c548a8b5bd
feat(core): 添加大化县政府网采购公告数据表和相关功能 - 创建 dahuagov_announcements 表用于存储大化县政府网采购公告 - 添加相关索引以提高查询性能 - 实现 save_dahuagov_announcements、get_new_dahuagov_announcements 和 mark_dahuagov_announcements_sent 方法 - 修改统计查询以包含大化县公告数据 - 更新内容哈希检查逻辑以支持新表 feat(cron): 集成大化县政府网采购公告爬取功能 - 导入大化县政府网爬虫模块 - 修改定时任务流程以同时爬取广西政府采购网和大化县政府网 - 对不同来源公告采用不同处理策略： - 广西政府采购网：关键词筛选后推送 - 大化县政府网：全部推送，不过滤关键词 - 分别处理和统计两个来源的公告数据 - 实现独立的通知发送和状态更新机制 feat(notification): 优化企业微信通知显示大化县来源标识 - 为不同来源公告添加前缀标识（【大化县政府网】或【广西政府采购网】） - 根据公告来源动态调整通知标题： - 单一来源显示具体来源 - 双来源显示"双源监控"标识 - 改进通知卡片的来源区分度，便于用户识别公告来源 ```
240 lines
11 KiB
Python
Executable File
240 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
定时搜索脚本
|
|
执行搜索、筛选关键词、保存到数据库并发送企业微信卡片通知
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# 添加项目根目录到路径（gx_gp_monitor的父目录）
|
|
project_root = Path(__file__).parent.parent
|
|
sys.path.insert(0, str(project_root))
|
|
|
|
try:
|
|
from gx_gp_monitor.core.config_manager import load_config, get_config
|
|
from gx_gp_monitor.core.logger import init_logger, get_logger
|
|
from gx_gp_monitor.crawler.spider import crawl_announcements
|
|
from gx_gp_monitor.crawler.dahuagov_spider import crawl_dahuagov_announcements
|
|
from gx_gp_monitor.filters.filters import filter_from_config
|
|
from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, save_auto_announcements_to_storage
|
|
from gx_gp_monitor.notification.wechat import send_announcements_notification, send_system_notification
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
def main():
|
|
"""主函数：执行定时搜索任务"""
|
|
try:
|
|
# 加载配置
|
|
config = load_config()
|
|
if not config:
|
|
logger.error("无法加载配置")
|
|
return False
|
|
|
|
# 初始化日志
|
|
init_logger(config=config)
|
|
|
|
logger.info("=== 开始定时搜索任务 ===")
|
|
|
|
# 初始化存储
|
|
init_storage()
|
|
|
|
# 导入数据库模块
|
|
import gx_gp_monitor.core.database as db_module
|
|
db_manager = db_module.get_database_manager()
|
|
|
|
# 执行搜索（爬取所有公告）
|
|
logger.info("开始执行定时搜索任务")
|
|
|
|
# 收集所有爬取结果
|
|
all_crawl_results = []
|
|
|
|
# 1. 爬取广西政府采购网
|
|
logger.info("开始爬取广西政府采购网...")
|
|
gxgp_results = crawl_announcements()
|
|
if gxgp_results:
|
|
all_crawl_results.extend(gxgp_results)
|
|
logger.info(f"广西政府采购网爬取完成，获取 {sum(len(r.announcements) for r in gxgp_results)} 条公告")
|
|
|
|
# 2. 爬取大化县政府网采购公告（全部推送，不筛选）
|
|
logger.info("开始爬取大化县政府网采购公告（全部推送）...")
|
|
dahua_results = crawl_dahuagov_announcements()
|
|
if dahua_results:
|
|
all_crawl_results.extend(dahua_results)
|
|
logger.info(f"大化县政府网爬取完成，获取 {sum(len(r.announcements) for r in dahua_results)} 条公告")
|
|
|
|
if not all_crawl_results:
|
|
logger.info("爬取完成：无数据")
|
|
return True
|
|
|
|
# 收集所有公告
|
|
all_announcements = []
|
|
for result in all_crawl_results:
|
|
if result.announcements:
|
|
all_announcements.extend(result.announcements)
|
|
|
|
total_crawled = len(all_announcements)
|
|
logger.info(f"搜索到 {total_crawled} 条原始公告")
|
|
|
|
if not all_announcements:
|
|
logger.info("没有获取到任何公告")
|
|
return True
|
|
|
|
# 分离广西政府采购网和大化县政府网的公告
|
|
gxgp_all_announcements = [a for a in all_announcements if a.source_code != 'dahuagov']
|
|
dahua_all_announcements = [a for a in all_announcements if a.source_code == 'dahuagov']
|
|
|
|
logger.info(f"广西政府采购网: {len(gxgp_all_announcements)} 条")
|
|
logger.info(f"大化县政府网: {len(dahua_all_announcements)} 条")
|
|
|
|
# ========== 处理广西政府采购网（关键词筛选）==========
|
|
gxgp_filtered = []
|
|
if gxgp_all_announcements:
|
|
# 对广西政府采购网公告进行关键词筛选
|
|
from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
|
|
from datetime import date
|
|
|
|
keyword_filter = KeywordFilter()
|
|
gxgp_keyword_filtered = keyword_filter.filter_announcements(
|
|
gxgp_all_announcements, keywords=config.crawler.keyword)
|
|
|
|
# 日期筛选（只处理今天的）
|
|
date_filter = DateFilter()
|
|
gxgp_today_filtered = date_filter.filter_announcements(
|
|
gxgp_keyword_filtered,
|
|
start_date=date.today(),
|
|
end_date=date.today()
|
|
)
|
|
|
|
logger.info(f"广西政府采购网关键词筛选后: {len(gxgp_keyword_filtered)} 条")
|
|
logger.info(f"广西政府采购网今日匹配: {len(gxgp_today_filtered)} 条")
|
|
|
|
# 检查是否已存在
|
|
for ann in gxgp_today_filtered:
|
|
try:
|
|
with db_module.get_db_cursor() as cursor:
|
|
cursor.execute(
|
|
"SELECT 1 FROM auto_announcements WHERE content_hash = %s LIMIT 1",
|
|
(ann.content_hash,)
|
|
)
|
|
exists = cursor.fetchone() is not None
|
|
if not exists:
|
|
gxgp_filtered.append(ann)
|
|
except Exception as e:
|
|
logger.warning(f"检查公告是否存在失败: {str(e)}")
|
|
pass
|
|
|
|
logger.info(f"广西政府采购网新增公告: {len(gxgp_filtered)} 条")
|
|
|
|
# ========== 处理大化县政府网（全部推送，不筛选）==========
|
|
dahua_new_announcements = []
|
|
if dahua_all_announcements:
|
|
# 大化县公告不需要关键词筛选，直接检查是否已存在于dahuagov_announcements表
|
|
for ann in dahua_all_announcements:
|
|
try:
|
|
with db_module.get_db_cursor() as cursor:
|
|
cursor.execute(
|
|
"SELECT 1 FROM dahuagov_announcements WHERE content_hash = %s LIMIT 1",
|
|
(ann.content_hash,)
|
|
)
|
|
exists = cursor.fetchone() is not None
|
|
if not exists:
|
|
# 标记为新公告
|
|
ann.is_new = True
|
|
dahua_new_announcements.append(ann)
|
|
except Exception as e:
|
|
logger.warning(f"检查大化县公告是否存在失败: {str(e)}")
|
|
pass
|
|
|
|
logger.info(f"大化县政府网新增公告: {len(dahua_new_announcements)} 条")
|
|
|
|
# 如果没有新增公告，直接结束
|
|
if not gxgp_filtered and not dahua_new_announcements:
|
|
logger.info("没有新增公告，任务完成")
|
|
return True
|
|
|
|
# ========== 保存到数据库 ==========
|
|
# 保存广西政府采购网公告
|
|
if gxgp_filtered:
|
|
saved_gxgp = save_auto_announcements_to_storage(gxgp_filtered)
|
|
logger.info(f"保存广西政府采购网公告: {saved_gxgp} 条")
|
|
|
|
# 保存大化县政府网公告到专用表
|
|
if dahua_new_announcements:
|
|
saved_dahua = db_manager.save_dahuagov_announcements(dahua_new_announcements)
|
|
logger.info(f"保存大化县政府网公告: {saved_dahua} 条")
|
|
|
|
# ========== 发送企业微信通知 ==========
|
|
if config.wechat_app.enabled:
|
|
logger.info("开始发送企业微信卡片通知...")
|
|
notify_success = True
|
|
|
|
# 发送广西政府采购网通知
|
|
if gxgp_filtered:
|
|
logger.info(f"发送广西政府采购网通知，共 {len(gxgp_filtered)} 条...")
|
|
gxgp_success = send_announcements_notification(gxgp_filtered)
|
|
if gxgp_success:
|
|
logger.info("广西政府采购网通知发送成功")
|
|
else:
|
|
logger.error("广西政府采购网通知发送失败")
|
|
notify_success = False
|
|
|
|
# 发送大化县政府网通知
|
|
if dahua_new_announcements:
|
|
logger.info(f"发送大化县政府网通知，共 {len(dahua_new_announcements)} 条...")
|
|
dahua_success = send_announcements_notification(dahua_new_announcements)
|
|
if dahua_success:
|
|
logger.info("大化县政府网通知发送成功")
|
|
# 标记为已发送
|
|
db_manager.mark_dahuagov_announcements_sent(dahua_new_announcements)
|
|
else:
|
|
logger.error("大化县政府网通知发送失败")
|
|
notify_success = False
|
|
|
|
else:
|
|
logger.info("企业微信通知未启用，跳过发送")
|
|
notify_success = True
|
|
|
|
# ========== 输出统计信息 ==========
|
|
print("\n=== 定时搜索任务完成 ===")
|
|
print(f"总共爬取: {total_crawled} 条公告")
|
|
print(f"广西政府采购网:")
|
|
print(f" - 爬取: {len(gxgp_all_announcements)} 条")
|
|
print(f" - 关键词匹配: {len(gxgp_filtered)} 条")
|
|
print(f"大化县政府网:")
|
|
print(f" - 爬取: {len(dahua_all_announcements)} 条")
|
|
print(f" - 新增推送: {len(dahua_new_announcements)} 条")
|
|
print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")
|
|
|
|
logger.info("=== 定时搜索任务完成 ===")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"定时搜索任务执行失败: {str(e)}")
|
|
print(f"❌ 定时搜索任务失败: {str(e)}", file=sys.stderr)
|
|
|
|
# 尝试发送错误通知
|
|
try:
|
|
if config and config.wechat_app.enabled:
|
|
send_system_notification(
|
|
"定时搜索任务失败",
|
|
f"错误信息: {str(e)}"
|
|
)
|
|
except Exception as notify_error:
|
|
logger.error(f"发送错误通知失败: {notify_error}")
|
|
|
|
return False
|
|
|
|
if __name__ == "__main__":
|
|
success = main()
|
|
sys.exit(0 if success else 1)
|
|
|
|
except ImportError as e:
|
|
print(f"导入失败: {e}", file=sys.stderr)
|
|
print("请确保已安装所有依赖: pip install -r gx_gp_monitor/requirements.txt", file=sys.stderr)
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
print(f"脚本执行失败: {e}", file=sys.stderr)
|
|
sys.exit(1)
|