feat(core): 添加大化县政府网采购公告数据表和相关功能

- 创建 dahuagov_announcements 表用于存储大化县政府网采购公告
- 添加相关索引以提高查询性能
- 实现 save_dahuagov_announcements、get_new_dahuagov_announcements
  和 mark_dahuagov_announcements_sent 方法
- 修改统计查询以包含大化县公告数据
- 更新内容哈希检查逻辑以支持新表

feat(cron): 集成大化县政府网采购公告爬取功能

- 导入大化县政府网爬虫模块
- 修改定时任务流程以同时爬取广西政府采购网和大化县政府网
- 对不同来源公告采用不同处理策略:
  - 广西政府采购网:关键词筛选后推送
  - 大化县政府网:全部推送,不过滤关键词
- 分别处理和统计两个来源的公告数据
- 实现独立的通知发送和状态更新机制

feat(notification): 优化企业微信通知显示大化县来源标识

- 为不同来源公告添加前缀标识(【大化县政府网】或【广西政府采购网】)
- 根据公告来源动态调整通知标题:
  - 单一来源显示具体来源
  - 双来源显示"双源监控"标识
- 改进通知卡片的来源区分度,便于用户识别公告来源
```
This commit is contained in:
2026-05-09 10:55:00 +08:00
parent bd8df98e9a
commit c548a8b5bd
17 changed files with 128615 additions and 22108 deletions
+157 -1
View File
@@ -264,12 +264,32 @@ class DatabaseManager:
FOREIGN KEY (source_code) REFERENCES announcement_sources(code)
);
-- 大化县政府网采购公告表(全部推送,不筛选)
CREATE TABLE IF NOT EXISTS dahuagov_announcements (
id SERIAL PRIMARY KEY,
title VARCHAR(500) NOT NULL,
publish_date TIMESTAMP NOT NULL,
purchase_name VARCHAR(200),
content_url TEXT,
source_code VARCHAR(50) NOT NULL DEFAULT 'dahuagov',
source_name VARCHAR(100) NOT NULL DEFAULT '大化县政府网采购公告',
announcement_type VARCHAR(50) NOT NULL DEFAULT 'purchase',
crawled_at TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
content_hash VARCHAR(32) UNIQUE,
is_new BOOLEAN DEFAULT TRUE
);
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_announcements_publish_date ON announcements(publish_date DESC);
CREATE INDEX IF NOT EXISTS idx_announcements_source_code ON announcements(source_code);
CREATE INDEX IF NOT EXISTS idx_announcements_content_hash ON announcements(content_hash);
CREATE INDEX IF NOT EXISTS idx_announcements_created_at ON announcements(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_crawl_results_crawled_at ON crawl_results(crawled_at DESC);
CREATE INDEX IF NOT EXISTS idx_dahuagov_publish_date ON dahuagov_announcements(publish_date DESC);
CREATE INDEX IF NOT EXISTS idx_dahuagov_content_hash ON dahuagov_announcements(content_hash);
CREATE INDEX IF NOT EXISTS idx_dahuagov_created_at ON dahuagov_announcements(created_at DESC);
-- 创建更新时间触发器
CREATE OR REPLACE FUNCTION update_updated_at_column()
@@ -704,6 +724,13 @@ class DatabaseManager:
COUNT(DISTINCT source_code) as sources,
MAX(crawled_at) as last_crawl
FROM manual_announcements
UNION ALL
SELECT
'dahuagov_announcements' as table_name,
COUNT(*) as count,
COUNT(DISTINCT source_code) as sources,
MAX(crawled_at) as last_crawl
FROM dahuagov_announcements
"""
cursor.execute(detail_sql)
@@ -742,12 +769,14 @@ class DatabaseManager:
SELECT content_hash FROM auto_announcements WHERE content_hash = %s
UNION ALL
SELECT content_hash FROM manual_announcements WHERE content_hash = %s
UNION ALL
SELECT content_hash FROM dahuagov_announcements WHERE content_hash = %s
) as combined_check LIMIT 1
"""
try:
with get_db_cursor() as cursor:
cursor.execute(sql, (content_hash, content_hash, content_hash))
cursor.execute(sql, (content_hash, content_hash, content_hash, content_hash))
return cursor.fetchone() is not None
except Exception as e:
logger.error(f"检查公告存在性失败: {str(e)}")
@@ -790,6 +819,133 @@ class DatabaseManager:
logger.error(f"获取最近公告失败: {str(e)}")
return []
@retry_on_exception(RetryConfig(max_retries=3))
def save_dahuagov_announcements(self, announcements: List[Announcement]) -> int:
"""
保存大化县政府网公告(全部推送,不筛选关键词)
Args:
announcements: 公告列表
Returns:
int: 成功保存的新公告数量
"""
if not self.config.database.enabled:
return 0
if not announcements:
return 0
# 为没有哈希的公告生成哈希
for announcement in announcements:
if not announcement.content_hash:
announcement.generate_content_hash()
sql = """
INSERT INTO dahuagov_announcements (
title, publish_date, purchase_name, content_url, source_code, source_name,
announcement_type, crawled_at, content_hash, is_new
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (content_hash) DO NOTHING
"""
values = []
for announcement in announcements:
values.append((
announcement.title,
announcement.publish_date,
announcement.purchase_name,
announcement.content_url,
announcement.source_code,
announcement.source_name,
announcement.announcement_type.value,
announcement.crawled_at,
announcement.content_hash,
announcement.is_new
))
try:
with get_db_cursor() as cursor:
extras.execute_batch(cursor, sql, values)
affected_rows = cursor.rowcount
logger.info(f"保存大化县公告完成,新增 {affected_rows}")
return affected_rows
except Exception as e:
logger.error(f"保存大化县公告失败: {str(e)}")
return 0
@retry_on_exception(RetryConfig(max_retries=3))
def get_new_dahuagov_announcements(self) -> List[Announcement]:
"""
获取大化县未推送的新公告(is_new = TRUE
Returns:
List[Announcement]: 未推送的公告列表
"""
if not self.config.database.enabled:
return []
sql = """
SELECT * FROM dahuagov_announcements
WHERE is_new = TRUE
ORDER BY publish_date DESC
"""
try:
with get_db_cursor() as cursor:
cursor.execute(sql)
rows = cursor.fetchall()
announcements = []
for row in rows:
row_dict = dict(row)
row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type'])
announcements.append(Announcement.from_dict(row_dict))
return announcements
except Exception as e:
logger.error(f"获取大化县新公告失败: {str(e)}")
return []
@retry_on_exception(RetryConfig(max_retries=3))
def mark_dahuagov_announcements_sent(self, announcements: List[Announcement]) -> int:
"""
标记大化县公告已发送(is_new = FALSE
Args:
announcements: 已发送的公告列表
Returns:
int: 更新的记录数
"""
if not self.config.database.enabled:
return 0
if not announcements:
return 0
# 获取所有公告的哈希值
hashes = [ann.content_hash for ann in announcements if ann.content_hash]
if not hashes:
return 0
sql = """
UPDATE dahuagov_announcements
SET is_new = FALSE, updated_at = CURRENT_TIMESTAMP
WHERE content_hash = ANY(%s)
"""
try:
with get_db_cursor() as cursor:
cursor.execute(sql, (hashes,))
affected_rows = cursor.rowcount
logger.info(f"标记大化县公告已发送完成,更新 {affected_rows}")
return affected_rows
except Exception as e:
logger.error(f"标记大化县公告已发送失败: {str(e)}")
return 0
# 全局数据库管理器实例
_db_manager = None
+355
View File
@@ -0,0 +1,355 @@
"""
大化瑶族自治县政府采购公告爬虫
爬取大化县政府网站的采购公告页面
"""
import time
import random
from typing import List, Optional, Tuple
from datetime import datetime
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup
import requests
from fake_useragent import UserAgent
try:
from ..core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
from ..core.config_manager import get_config
from ..core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
from ..core.reliability import (
retry_on_exception, RetryConfig, session_with_retry,
TimeoutConfig, safe_execute, check_system_health
)
except ImportError:
from core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
from core.config_manager import get_config
from core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
from core.reliability import (
retry_on_exception, RetryConfig, session_with_retry,
TimeoutConfig, safe_execute, check_system_health
)
logger = get_logger(__name__)
class DahuagovSpider:
"""大化县政府网站爬虫"""
# 大化县政府网站配置
BASE_URL = "http://www.gxdh.gov.cn"
ANNOUNCEMENT_PATH = "/xxgk/zdlyxxgk/ggzypzly/zfcgly/cggg/"
def __init__(self):
self.config = get_config()
self.ua = UserAgent()
self.session = None
self.request_count = 0
self.error_count = 0
def _get_random_user_agent(self) -> str:
"""获取随机User-Agent"""
try:
from fake_useragent import UserAgent
return self.ua.random
except:
return random.choice(self.config.crawler.user_agents)
def init_session(self):
"""初始化会话"""
if self.session is None:
timeout_config = TimeoutConfig(
connect_timeout=self.config.crawler.timeout,
read_timeout=self.config.crawler.timeout
)
retry_config = RetryConfig(
max_retries=self.config.crawler.max_retries,
initial_delay=self.config.crawler.retry_delay,
max_delay=self.config.crawler.max_retry_delay,
backoff_factor=self.config.crawler.backoff_factor
)
self.session = requests.Session()
adapter = requests.adapters.HTTPAdapter(
pool_connections=10,
pool_maxsize=20,
max_retries=0
)
self.session.mount('http://', adapter)
self.session.mount('https://', adapter)
self.session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
return self.session
def close_session(self):
"""关闭会话"""
if self.session:
self.session.close()
self.session = None
def _fetch_page(self, url: str) -> Tuple[Optional[str], Optional[str]]:
"""
获取页面内容
Args:
url: 页面URL
Returns:
Tuple[Optional[str], Optional[str]]: (页面内容, 错误信息)
"""
try:
session = self.init_session()
user_agent = self._get_random_user_agent()
headers = {
"User-Agent": user_agent,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Connection": "keep-alive",
"Referer": self.BASE_URL
}
# 添加随机延迟
delay = random.uniform(
self.config.crawler.request_delay,
self.config.crawler.request_delay_max
)
time.sleep(delay)
response = session.get(url, headers=headers, timeout=self.session.timeout)
self.request_count += 1
if response.status_code == 200:
# 确保使用正确的编码
response.encoding = response.apparent_encoding or 'utf-8'
return response.text, None
else:
return None, f"请求失败, 状态码: {response.status_code}"
except Exception as e:
self.error_count += 1
logger.error(f"获取页面异常: {str(e)}")
return None, f"请求异常: {str(e)}"
def _parse_page(self, html: str, crawled_at: datetime) -> List[Announcement]:
"""
解析页面内容
Args:
html: 页面HTML内容
crawled_at: 爬取时间
Returns:
List[Announcement]: 解析后的公告列表
"""
announcements = []
try:
soup = BeautifulSoup(html, 'html.parser')
# 查找公告列表
# 大化县政府网站使用 ul.more-list 结构
lists = soup.find_all('ul', class_='more-list')
if not lists:
logger.info("未找到公告列表")
return announcements
for ul in lists:
lis = ul.find_all('li')
for li in lis:
try:
announcement = self._parse_li_element(li, crawled_at)
if announcement:
announcements.append(announcement)
except Exception as e:
logger.warning(f"解析单个公告失败: {str(e)}")
continue
logger.info(f"成功解析 {len(announcements)} 条公告")
except Exception as e:
logger.error(f"解析页面失败: {str(e)}")
return announcements
def _parse_li_element(self, li, crawled_at: datetime) -> Optional[Announcement]:
"""
解析单个li元素
Args:
li: BeautifulSoup li元素
crawled_at: 爬取时间
Returns:
Optional[Announcement]: 解析后的公告对象
"""
try:
# 查找日期 span
date_span = li.find('span')
if not date_span:
return None
date_text = date_span.get_text(strip=True)
if not date_text:
return None
# 解析日期
try:
publish_date = datetime.strptime(date_text, "%Y-%m-%d")
except ValueError:
logger.warning(f"日期格式无法解析: {date_text}")
return None
# 查找链接和标题
link_tag = li.find('a')
if not link_tag:
return None
title = link_tag.get('title', '') or link_tag.get_text(strip=True)
if not title:
return None
href = link_tag.get('href', '')
if not href:
return None
# 构建完整URL
if href.startswith('./') or href.startswith('../'):
content_url = urljoin(self.BASE_URL + self.ANNOUNCEMENT_PATH, href)
elif href.startswith('/'):
content_url = self.BASE_URL + href
elif href.startswith('http'):
content_url = href
else:
content_url = urljoin(self.BASE_URL + self.ANNOUNCEMENT_PATH, href)
# 创建公告对象
announcement = Announcement(
title=title,
publish_date=publish_date,
purchase_name="大化瑶族自治县", # 默认采购单位
content_url=content_url,
source_code="dahuagov",
source_name="大化县政府网采购公告",
announcement_type=AnnouncementType.PURCHASE,
crawled_at=crawled_at,
is_new=True
)
# 生成内容哈希用于去重
announcement.generate_content_hash()
return announcement
except Exception as e:
logger.warning(f"解析li元素失败: {str(e)}")
return None
@retry_on_exception(RetryConfig(max_retries=2))
def crawl(self) -> CrawlResult:
"""
爬取公告(只爬取第一页)
Returns:
CrawlResult: 爬取结果
"""
log_crawl_start("大化县政府网采购公告")
start_time = datetime.now()
result = CrawlResult(
source=AnnouncementSource(
code="dahuagov",
category_id=0,
name="大化县政府网采购公告",
type=AnnouncementType.PURCHASE
),
status=CrawlStatus.RUNNING,
crawled_at=start_time
)
try:
# 构建完整URL(只爬取第一页)
url = self.BASE_URL + self.ANNOUNCEMENT_PATH
logger.info(f"开始爬取大化县政府网站: {url}")
# 获取页面内容
html, error_msg = self._fetch_page(url)
if error_msg:
logger.warning(f"获取页面失败: {error_msg}")
result.status = CrawlStatus.FAILED
result.error_message = error_msg
return result
if not html:
logger.info("页面内容为空")
result.status = CrawlStatus.SUCCESS
result.total_count = 0
result.new_count = 0
return result
# 解析页面
announcements = self._parse_page(html, start_time)
# 更新结果
result.announcements = announcements
result.total_count = len(announcements)
result.new_count = len(announcements)
result.status = CrawlStatus.SUCCESS
duration = (datetime.now() - start_time).total_seconds()
result.duration = duration
log_crawl_success("大化县政府网采购公告", len(announcements), duration)
except Exception as e:
duration = (datetime.now() - start_time).total_seconds()
result.duration = duration
result.status = CrawlStatus.FAILED
result.error_message = str(e)
log_crawl_error("大化县政府网采购公告", str(e))
return result
def get_stats(self) -> dict:
"""获取爬虫统计信息"""
return {
"request_count": self.request_count,
"error_count": self.error_count,
"error_rate": self.error_count / max(self.request_count, 1),
"session_active": self.session is not None
}
def create_dahuagov_spider() -> DahuagovSpider:
"""
创建大化县爬虫实例
Returns:
DahuagovSpider: 爬虫实例
"""
return DahuagovSpider()
def crawl_dahuagov_announcements() -> List[CrawlResult]:
"""
便捷函数:爬取大化县公告
Returns:
List[CrawlResult]: 爬取结果列表
"""
spider = create_dahuagov_spider()
try:
result = spider.crawl()
return [result]
finally:
spider.close_session()
+134 -76
View File
@@ -16,6 +16,7 @@ try:
from gx_gp_monitor.core.config_manager import load_config, get_config
from gx_gp_monitor.core.logger import init_logger, get_logger
from gx_gp_monitor.crawler.spider import crawl_announcements
from gx_gp_monitor.crawler.dahuagov_spider import crawl_dahuagov_announcements
from gx_gp_monitor.filters.filters import filter_from_config
from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, save_auto_announcements_to_storage
from gx_gp_monitor.notification.wechat import send_announcements_notification, send_system_notification
@@ -39,17 +40,37 @@ try:
# 初始化存储
init_storage()
# 执行搜索(爬取所有公告,筛选出新增的关键词匹配公告)
logger.info("开始执行定时搜索任务")
crawl_results = crawl_announcements()
# 导入数据库模块
import gx_gp_monitor.core.database as db_module
db_manager = db_module.get_database_manager()
if not crawl_results:
# 执行搜索(爬取所有公告)
logger.info("开始执行定时搜索任务")
# 收集所有爬取结果
all_crawl_results = []
# 1. 爬取广西政府采购网
logger.info("开始爬取广西政府采购网...")
gxgp_results = crawl_announcements()
if gxgp_results:
all_crawl_results.extend(gxgp_results)
logger.info(f"广西政府采购网爬取完成,获取 {sum(len(r.announcements) for r in gxgp_results)} 条公告")
# 2. 爬取大化县政府网采购公告(全部推送,不筛选)
logger.info("开始爬取大化县政府网采购公告(全部推送)...")
dahua_results = crawl_dahuagov_announcements()
if dahua_results:
all_crawl_results.extend(dahua_results)
logger.info(f"大化县政府网爬取完成,获取 {sum(len(r.announcements) for r in dahua_results)} 条公告")
if not all_crawl_results:
logger.info("爬取完成:无数据")
return True
# 收集所有公告
all_announcements = []
for result in crawl_results:
for result in all_crawl_results:
if result.announcements:
all_announcements.extend(result.announcements)
@@ -60,93 +81,130 @@ try:
logger.info("没有获取到任何公告")
return True
# 对所有公告进行关键词筛选
from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
from datetime import date
# 分离广西政府采购网和大化县政府网的公告
gxgp_all_announcements = [a for a in all_announcements if a.source_code != 'dahuagov']
dahua_all_announcements = [a for a in all_announcements if a.source_code == 'dahuagov']
keyword_filter = KeywordFilter()
keyword_filtered = keyword_filter.filter_announcements(all_announcements, keywords=config.crawler.keyword)
logger.info(f"广西政府采购网: {len(gxgp_all_announcements)}")
logger.info(f"大化县政府网: {len(dahua_all_announcements)}")
# 对关键词筛选结果进行日期筛选(只处理今天的)
date_filter = DateFilter()
today_keyword_announcements = date_filter.filter_announcements(
keyword_filtered,
start_date=date.today(),
end_date=date.today()
)
# ========== 处理广西政府采购网(关键词筛选)==========
gxgp_filtered = []
if gxgp_all_announcements:
# 对广西政府采购网公告进行关键词筛选
from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
from datetime import date
logger.info(f"关键词筛选后剩余 {len(keyword_filtered)} 条公告")
logger.info(f"筛选出今天关键词匹配 {len(today_keyword_announcements)} 条公告")
keyword_filter = KeywordFilter()
gxgp_keyword_filtered = keyword_filter.filter_announcements(
gxgp_all_announcements, keywords=config.crawler.keyword)
if not today_keyword_announcements:
logger.info("今天没有关键词匹配的公告")
# 日期筛选(只处理今天的)
date_filter = DateFilter()
gxgp_today_filtered = date_filter.filter_announcements(
gxgp_keyword_filtered,
start_date=date.today(),
end_date=date.today()
)
logger.info(f"广西政府采购网关键词筛选后: {len(gxgp_keyword_filtered)}")
logger.info(f"广西政府采购网今日匹配: {len(gxgp_today_filtered)}")
# 检查是否已存在
for ann in gxgp_today_filtered:
try:
with db_module.get_db_cursor() as cursor:
cursor.execute(
"SELECT 1 FROM auto_announcements WHERE content_hash = %s LIMIT 1",
(ann.content_hash,)
)
exists = cursor.fetchone() is not None
if not exists:
gxgp_filtered.append(ann)
except Exception as e:
logger.warning(f"检查公告是否存在失败: {str(e)}")
pass
logger.info(f"广西政府采购网新增公告: {len(gxgp_filtered)}")
# ========== 处理大化县政府网(全部推送,不筛选)==========
dahua_new_announcements = []
if dahua_all_announcements:
# 大化县公告不需要关键词筛选,直接检查是否已存在于dahuagov_announcements表
for ann in dahua_all_announcements:
try:
with db_module.get_db_cursor() as cursor:
cursor.execute(
"SELECT 1 FROM dahuagov_announcements WHERE content_hash = %s LIMIT 1",
(ann.content_hash,)
)
exists = cursor.fetchone() is not None
if not exists:
# 标记为新公告
ann.is_new = True
dahua_new_announcements.append(ann)
except Exception as e:
logger.warning(f"检查大化县公告是否存在失败: {str(e)}")
pass
logger.info(f"大化县政府网新增公告: {len(dahua_new_announcements)}")
# 如果没有新增公告,直接结束
if not gxgp_filtered and not dahua_new_announcements:
logger.info("没有新增公告,任务完成")
return True
# 检查auto_announcements表,筛选出真正新增的公告
truly_new_announcements = []
for ann in today_keyword_announcements:
try:
# 直接导入,避免相对导入问题
import gx_gp_monitor.core.database as db_module
with db_module.get_db_cursor() as cursor:
cursor.execute(
"SELECT 1 FROM auto_announcements WHERE content_hash = %s LIMIT 1",
(ann.content_hash,)
)
exists = cursor.fetchone() is not None
if not exists:
truly_new_announcements.append(ann)
except Exception as e:
logger.warning(f"检查公告是否存在失败: {str(e)}")
# 如果检查失败,为了安全起见,不添加到新公告列表
pass
# ========== 保存到数据库 ==========
# 保存广西政府采购网公告
if gxgp_filtered:
saved_gxgp = save_auto_announcements_to_storage(gxgp_filtered)
logger.info(f"保存广西政府采购网公告: {saved_gxgp}")
logger.info(f"从今天关键词匹配公告中筛选出 {len(truly_new_announcements)} 条auto_announcements表中不存在的新公告")
# 保存大化县政府网公告到专用表
if dahua_new_announcements:
saved_dahua = db_manager.save_dahuagov_announcements(dahua_new_announcements)
logger.info(f"保存大化县政府网公告: {saved_dahua}")
if not truly_new_announcements:
logger.info("没有真正新增的关键词匹配公告,任务完成")
return True
# 构造筛选统计信息
filter_stats = type('FilterResult', (), {
"keyword_filtered": len(all_announcements) - len(keyword_filtered),
"date_filtered": len(keyword_filtered) - len(today_keyword_announcements),
"duplicate_filtered": len(today_keyword_announcements) - len(truly_new_announcements),
"source_filtered": 0
})()
filtered_announcements = truly_new_announcements
logger.info(f"关键词筛选后剩余 {len(filtered_announcements)} 条公告")
if not filtered_announcements:
logger.info("没有匹配关键词的公告,任务完成")
return True
# 保存筛选后的公告到定时搜索专用表
saved_count = save_auto_announcements_to_storage(filtered_announcements)
logger.info(f"保存定时搜索公告完成:{saved_count}")
# 发送企业微信卡片通知
# ========== 发送企业微信通知 ==========
if config.wechat_app.enabled:
logger.info("开始发送企业微信卡片通知...")
notify_success = send_announcements_notification(filtered_announcements)
notify_success = True
# 发送广西政府采购网通知
if gxgp_filtered:
logger.info(f"发送广西政府采购网通知,共 {len(gxgp_filtered)} 条...")
gxgp_success = send_announcements_notification(gxgp_filtered)
if gxgp_success:
logger.info("广西政府采购网通知发送成功")
else:
logger.error("广西政府采购网通知发送失败")
notify_success = False
# 发送大化县政府网通知
if dahua_new_announcements:
logger.info(f"发送大化县政府网通知,共 {len(dahua_new_announcements)} 条...")
dahua_success = send_announcements_notification(dahua_new_announcements)
if dahua_success:
logger.info("大化县政府网通知发送成功")
# 标记为已发送
db_manager.mark_dahuagov_announcements_sent(dahua_new_announcements)
else:
logger.error("大化县政府网通知发送失败")
notify_success = False
if notify_success:
logger.info("企业微信卡片通知发送成功")
else:
logger.error("企业微信卡片通知发送失败")
else:
logger.info("企业微信通知未启用,跳过发送")
notify_success = True
# 输出统计信息
# ========== 输出统计信息 ==========
print("\n=== 定时搜索任务完成 ===")
print(f"总共爬取: {total_crawled} 条公告")
print(f"关键词筛选: {len(keyword_filtered)}")
print(f"今日关键词匹配: {len(today_keyword_announcements)}")
print(f"真正新增公告: {len(truly_new_announcements)}")
print(f"筛选后公告: {len(filtered_announcements)}")
print(f"保存到数据库: {saved_count}")
print(f"广西政府采购网:")
print(f" - 爬取: {len(gxgp_all_announcements)}")
print(f" - 关键词匹配: {len(gxgp_filtered)}")
print(f"大化县政府网:")
print(f" - 爬取: {len(dahua_all_announcements)}")
print(f" - 新增推送: {len(dahua_new_announcements)}")
print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")
logger.info("=== 定时搜索任务完成 ===")
+32 -4
View File
@@ -383,14 +383,21 @@ class WeChatService:
if len(source_name) > 25: # 限制来源名称长度
source_name = source_name[:22] + "..."
# 根据来源代码添加前缀标识
source_prefix = ""
if announcement.source_code == 'dahuagov':
source_prefix = "【大化县政府网】"
else:
source_prefix = "【广西政府采购网】"
# 时间格式化
if announcement.publish_date:
time_str = announcement.publish_date.strftime("%Y-%m-%d %H:%M")
else:
time_str = "时间未知"
# 生成描述:类型 | 来源 | 时间(使用默认颜色)
description = f'<div style="font-size: 14px; margin-top: 8px;">{announcement_type_display} | {source_name} | {time_str}</div>'
# 生成描述:来源标识 | 类型 | 来源单位 | 时间
description = f'<div style="font-size: 14px; margin-top: 8px;">{source_prefix}{announcement_type_display} | {source_name} | {time_str}</div>'
# URL:公告详情链接
url = announcement.content_url
@@ -434,7 +441,18 @@ class WeChatService:
# 标题和概要
total_count = len(announcements)
lines.append("# 🔔 广西政府采购网公告更新")
# 检查是否包含多个来源
has_gxgp = any(a.source_code != 'dahuagov' for a in announcements)
has_dahua = any(a.source_code == 'dahuagov' for a in announcements)
if has_gxgp and has_dahua:
lines.append("# 🔔 政府采购公告更新(双源监控)")
elif has_dahua:
lines.append("# 🔔 大化县政府网采购公告更新")
else:
lines.append("# 🔔 广西政府采购网公告更新")
lines.append("")
lines.append(f"📊 **共发现 {total_count} 条新公告**")
lines.append("")
@@ -541,7 +559,17 @@ class WeChatService:
# 生成标题
total_count = len(announcements)
title = f"🔔 广西政府采购网公告更新 ({total_count}条)"
# 检查是否包含多个来源
has_gxgp = any(a.source_code != 'dahuagov' for a in announcements)
has_dahua = any(a.source_code == 'dahuagov' for a in announcements)
if has_gxgp and has_dahua:
title = f"🔔 政府采购公告更新 ({total_count}条) - 双源监控"
elif has_dahua:
title = f"🔔 大化县政府网采购公告更新 ({total_count}条)"
else:
title = f"🔔 广西政府采购网公告更新 ({total_count}条)"
# 生成描述HTML
html_parts = []