```
feat(core): 添加大化县政府网采购公告数据表和相关功能 - 创建 dahuagov_announcements 表用于存储大化县政府网采购公告 - 添加相关索引以提高查询性能 - 实现 save_dahuagov_announcements、get_new_dahuagov_announcements 和 mark_dahuagov_announcements_sent 方法 - 修改统计查询以包含大化县公告数据 - 更新内容哈希检查逻辑以支持新表 feat(cron): 集成大化县政府网采购公告爬取功能 - 导入大化县政府网爬虫模块 - 修改定时任务流程以同时爬取广西政府采购网和大化县政府网 - 对不同来源公告采用不同处理策略： - 广西政府采购网：关键词筛选后推送 - 大化县政府网：全部推送，不过滤关键词 - 分别处理和统计两个来源的公告数据 - 实现独立的通知发送和状态更新机制 feat(notification): 优化企业微信通知显示大化县来源标识 - 为不同来源公告添加前缀标识（【大化县政府网】或【广西政府采购网】） - 根据公告来源动态调整通知标题： - 单一来源显示具体来源 - 双来源显示"双源监控"标识 - 改进通知卡片的来源区分度，便于用户识别公告来源 ```
This commit is contained in:
+134
-76
@@ -16,6 +16,7 @@ try:
|
||||
from gx_gp_monitor.core.config_manager import load_config, get_config
|
||||
from gx_gp_monitor.core.logger import init_logger, get_logger
|
||||
from gx_gp_monitor.crawler.spider import crawl_announcements
|
||||
from gx_gp_monitor.crawler.dahuagov_spider import crawl_dahuagov_announcements
|
||||
from gx_gp_monitor.filters.filters import filter_from_config
|
||||
from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, save_auto_announcements_to_storage
|
||||
from gx_gp_monitor.notification.wechat import send_announcements_notification, send_system_notification
|
||||
@@ -39,17 +40,37 @@ try:
|
||||
# 初始化存储
|
||||
init_storage()
|
||||
|
||||
# 执行搜索（爬取所有公告，筛选出新增的关键词匹配公告）
|
||||
logger.info("开始执行定时搜索任务")
|
||||
crawl_results = crawl_announcements()
|
||||
# 导入数据库模块
|
||||
import gx_gp_monitor.core.database as db_module
|
||||
db_manager = db_module.get_database_manager()
|
||||
|
||||
if not crawl_results:
|
||||
# 执行搜索（爬取所有公告）
|
||||
logger.info("开始执行定时搜索任务")
|
||||
|
||||
# 收集所有爬取结果
|
||||
all_crawl_results = []
|
||||
|
||||
# 1. 爬取广西政府采购网
|
||||
logger.info("开始爬取广西政府采购网...")
|
||||
gxgp_results = crawl_announcements()
|
||||
if gxgp_results:
|
||||
all_crawl_results.extend(gxgp_results)
|
||||
logger.info(f"广西政府采购网爬取完成，获取 {sum(len(r.announcements) for r in gxgp_results)} 条公告")
|
||||
|
||||
# 2. 爬取大化县政府网采购公告（全部推送，不筛选）
|
||||
logger.info("开始爬取大化县政府网采购公告（全部推送）...")
|
||||
dahua_results = crawl_dahuagov_announcements()
|
||||
if dahua_results:
|
||||
all_crawl_results.extend(dahua_results)
|
||||
logger.info(f"大化县政府网爬取完成，获取 {sum(len(r.announcements) for r in dahua_results)} 条公告")
|
||||
|
||||
if not all_crawl_results:
|
||||
logger.info("爬取完成：无数据")
|
||||
return True
|
||||
|
||||
# 收集所有公告
|
||||
all_announcements = []
|
||||
for result in crawl_results:
|
||||
for result in all_crawl_results:
|
||||
if result.announcements:
|
||||
all_announcements.extend(result.announcements)
|
||||
|
||||
@@ -60,93 +81,130 @@ try:
|
||||
logger.info("没有获取到任何公告")
|
||||
return True
|
||||
|
||||
# 对所有公告进行关键词筛选
|
||||
from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
|
||||
from datetime import date
|
||||
# 分离广西政府采购网和大化县政府网的公告
|
||||
gxgp_all_announcements = [a for a in all_announcements if a.source_code != 'dahuagov']
|
||||
dahua_all_announcements = [a for a in all_announcements if a.source_code == 'dahuagov']
|
||||
|
||||
keyword_filter = KeywordFilter()
|
||||
keyword_filtered = keyword_filter.filter_announcements(all_announcements, keywords=config.crawler.keyword)
|
||||
logger.info(f"广西政府采购网: {len(gxgp_all_announcements)} 条")
|
||||
logger.info(f"大化县政府网: {len(dahua_all_announcements)} 条")
|
||||
|
||||
# 对关键词筛选结果进行日期筛选（只处理今天的）
|
||||
date_filter = DateFilter()
|
||||
today_keyword_announcements = date_filter.filter_announcements(
|
||||
keyword_filtered,
|
||||
start_date=date.today(),
|
||||
end_date=date.today()
|
||||
)
|
||||
# ========== 处理广西政府采购网（关键词筛选）==========
|
||||
gxgp_filtered = []
|
||||
if gxgp_all_announcements:
|
||||
# 对广西政府采购网公告进行关键词筛选
|
||||
from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
|
||||
from datetime import date
|
||||
|
||||
logger.info(f"关键词筛选后剩余 {len(keyword_filtered)} 条公告")
|
||||
logger.info(f"筛选出今天关键词匹配 {len(today_keyword_announcements)} 条公告")
|
||||
keyword_filter = KeywordFilter()
|
||||
gxgp_keyword_filtered = keyword_filter.filter_announcements(
|
||||
gxgp_all_announcements, keywords=config.crawler.keyword)
|
||||
|
||||
if not today_keyword_announcements:
|
||||
logger.info("今天没有关键词匹配的公告")
|
||||
# 日期筛选（只处理今天的）
|
||||
date_filter = DateFilter()
|
||||
gxgp_today_filtered = date_filter.filter_announcements(
|
||||
gxgp_keyword_filtered,
|
||||
start_date=date.today(),
|
||||
end_date=date.today()
|
||||
)
|
||||
|
||||
logger.info(f"广西政府采购网关键词筛选后: {len(gxgp_keyword_filtered)} 条")
|
||||
logger.info(f"广西政府采购网今日匹配: {len(gxgp_today_filtered)} 条")
|
||||
|
||||
# 检查是否已存在
|
||||
for ann in gxgp_today_filtered:
|
||||
try:
|
||||
with db_module.get_db_cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM auto_announcements WHERE content_hash = %s LIMIT 1",
|
||||
(ann.content_hash,)
|
||||
)
|
||||
exists = cursor.fetchone() is not None
|
||||
if not exists:
|
||||
gxgp_filtered.append(ann)
|
||||
except Exception as e:
|
||||
logger.warning(f"检查公告是否存在失败: {str(e)}")
|
||||
pass
|
||||
|
||||
logger.info(f"广西政府采购网新增公告: {len(gxgp_filtered)} 条")
|
||||
|
||||
# ========== 处理大化县政府网（全部推送，不筛选）==========
|
||||
dahua_new_announcements = []
|
||||
if dahua_all_announcements:
|
||||
# 大化县公告不需要关键词筛选，直接检查是否已存在于dahuagov_announcements表
|
||||
for ann in dahua_all_announcements:
|
||||
try:
|
||||
with db_module.get_db_cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM dahuagov_announcements WHERE content_hash = %s LIMIT 1",
|
||||
(ann.content_hash,)
|
||||
)
|
||||
exists = cursor.fetchone() is not None
|
||||
if not exists:
|
||||
# 标记为新公告
|
||||
ann.is_new = True
|
||||
dahua_new_announcements.append(ann)
|
||||
except Exception as e:
|
||||
logger.warning(f"检查大化县公告是否存在失败: {str(e)}")
|
||||
pass
|
||||
|
||||
logger.info(f"大化县政府网新增公告: {len(dahua_new_announcements)} 条")
|
||||
|
||||
# 如果没有新增公告，直接结束
|
||||
if not gxgp_filtered and not dahua_new_announcements:
|
||||
logger.info("没有新增公告，任务完成")
|
||||
return True
|
||||
|
||||
# 检查auto_announcements表，筛选出真正新增的公告
|
||||
truly_new_announcements = []
|
||||
for ann in today_keyword_announcements:
|
||||
try:
|
||||
# 直接导入，避免相对导入问题
|
||||
import gx_gp_monitor.core.database as db_module
|
||||
with db_module.get_db_cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM auto_announcements WHERE content_hash = %s LIMIT 1",
|
||||
(ann.content_hash,)
|
||||
)
|
||||
exists = cursor.fetchone() is not None
|
||||
if not exists:
|
||||
truly_new_announcements.append(ann)
|
||||
except Exception as e:
|
||||
logger.warning(f"检查公告是否存在失败: {str(e)}")
|
||||
# 如果检查失败，为了安全起见，不添加到新公告列表
|
||||
pass
|
||||
# ========== 保存到数据库 ==========
|
||||
# 保存广西政府采购网公告
|
||||
if gxgp_filtered:
|
||||
saved_gxgp = save_auto_announcements_to_storage(gxgp_filtered)
|
||||
logger.info(f"保存广西政府采购网公告: {saved_gxgp} 条")
|
||||
|
||||
logger.info(f"从今天关键词匹配公告中筛选出 {len(truly_new_announcements)} 条auto_announcements表中不存在的新公告")
|
||||
# 保存大化县政府网公告到专用表
|
||||
if dahua_new_announcements:
|
||||
saved_dahua = db_manager.save_dahuagov_announcements(dahua_new_announcements)
|
||||
logger.info(f"保存大化县政府网公告: {saved_dahua} 条")
|
||||
|
||||
if not truly_new_announcements:
|
||||
logger.info("没有真正新增的关键词匹配公告，任务完成")
|
||||
return True
|
||||
|
||||
# 构造筛选统计信息
|
||||
filter_stats = type('FilterResult', (), {
|
||||
"keyword_filtered": len(all_announcements) - len(keyword_filtered),
|
||||
"date_filtered": len(keyword_filtered) - len(today_keyword_announcements),
|
||||
"duplicate_filtered": len(today_keyword_announcements) - len(truly_new_announcements),
|
||||
"source_filtered": 0
|
||||
})()
|
||||
|
||||
filtered_announcements = truly_new_announcements
|
||||
|
||||
logger.info(f"关键词筛选后剩余 {len(filtered_announcements)} 条公告")
|
||||
|
||||
if not filtered_announcements:
|
||||
logger.info("没有匹配关键词的公告，任务完成")
|
||||
return True
|
||||
|
||||
# 保存筛选后的公告到定时搜索专用表
|
||||
saved_count = save_auto_announcements_to_storage(filtered_announcements)
|
||||
logger.info(f"保存定时搜索公告完成：{saved_count} 条")
|
||||
|
||||
# 发送企业微信卡片通知
|
||||
# ========== 发送企业微信通知 ==========
|
||||
if config.wechat_app.enabled:
|
||||
logger.info("开始发送企业微信卡片通知...")
|
||||
notify_success = send_announcements_notification(filtered_announcements)
|
||||
notify_success = True
|
||||
|
||||
# 发送广西政府采购网通知
|
||||
if gxgp_filtered:
|
||||
logger.info(f"发送广西政府采购网通知，共 {len(gxgp_filtered)} 条...")
|
||||
gxgp_success = send_announcements_notification(gxgp_filtered)
|
||||
if gxgp_success:
|
||||
logger.info("广西政府采购网通知发送成功")
|
||||
else:
|
||||
logger.error("广西政府采购网通知发送失败")
|
||||
notify_success = False
|
||||
|
||||
# 发送大化县政府网通知
|
||||
if dahua_new_announcements:
|
||||
logger.info(f"发送大化县政府网通知，共 {len(dahua_new_announcements)} 条...")
|
||||
dahua_success = send_announcements_notification(dahua_new_announcements)
|
||||
if dahua_success:
|
||||
logger.info("大化县政府网通知发送成功")
|
||||
# 标记为已发送
|
||||
db_manager.mark_dahuagov_announcements_sent(dahua_new_announcements)
|
||||
else:
|
||||
logger.error("大化县政府网通知发送失败")
|
||||
notify_success = False
|
||||
|
||||
if notify_success:
|
||||
logger.info("企业微信卡片通知发送成功")
|
||||
else:
|
||||
logger.error("企业微信卡片通知发送失败")
|
||||
else:
|
||||
logger.info("企业微信通知未启用，跳过发送")
|
||||
notify_success = True
|
||||
|
||||
# 输出统计信息
|
||||
# ========== 输出统计信息 ==========
|
||||
print("\n=== 定时搜索任务完成 ===")
|
||||
print(f"总共爬取: {total_crawled} 条公告")
|
||||
print(f"关键词筛选: {len(keyword_filtered)} 条")
|
||||
print(f"今日关键词匹配: {len(today_keyword_announcements)} 条")
|
||||
print(f"真正新增公告: {len(truly_new_announcements)} 条")
|
||||
print(f"筛选后公告: {len(filtered_announcements)} 条")
|
||||
print(f"保存到数据库: {saved_count} 条")
|
||||
print(f"广西政府采购网:")
|
||||
print(f" - 爬取: {len(gxgp_all_announcements)} 条")
|
||||
print(f" - 关键词匹配: {len(gxgp_filtered)} 条")
|
||||
print(f"大化县政府网:")
|
||||
print(f" - 爬取: {len(dahua_all_announcements)} 条")
|
||||
print(f" - 新增推送: {len(dahua_new_announcements)} 条")
|
||||
print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")
|
||||
|
||||
logger.info("=== 定时搜索任务完成 ===")
|
||||
|
||||
Reference in New Issue
Block a user