```
feat(core): 添加大化县政府网采购公告数据表和相关功能 - 创建 dahuagov_announcements 表用于存储大化县政府网采购公告 - 添加相关索引以提高查询性能 - 实现 save_dahuagov_announcements、get_new_dahuagov_announcements 和 mark_dahuagov_announcements_sent 方法 - 修改统计查询以包含大化县公告数据 - 更新内容哈希检查逻辑以支持新表 feat(cron): 集成大化县政府网采购公告爬取功能 - 导入大化县政府网爬虫模块 - 修改定时任务流程以同时爬取广西政府采购网和大化县政府网 - 对不同来源公告采用不同处理策略: - 广西政府采购网:关键词筛选后推送 - 大化县政府网:全部推送,不过滤关键词 - 分别处理和统计两个来源的公告数据 - 实现独立的通知发送和状态更新机制 feat(notification): 优化企业微信通知显示大化县来源标识 - 为不同来源公告添加前缀标识(【大化县政府网】或【广西政府采购网】) - 根据公告来源动态调整通知标题: - 单一来源显示具体来源 - 双来源显示"双源监控"标识 - 改进通知卡片的来源区分度,便于用户识别公告来源 ```
This commit is contained in:
Binary file not shown.
@@ -264,12 +264,32 @@ class DatabaseManager:
|
||||
FOREIGN KEY (source_code) REFERENCES announcement_sources(code)
|
||||
);
|
||||
|
||||
-- 大化县政府网采购公告表(全部推送,不筛选)
|
||||
CREATE TABLE IF NOT EXISTS dahuagov_announcements (
|
||||
id SERIAL PRIMARY KEY,
|
||||
title VARCHAR(500) NOT NULL,
|
||||
publish_date TIMESTAMP NOT NULL,
|
||||
purchase_name VARCHAR(200),
|
||||
content_url TEXT,
|
||||
source_code VARCHAR(50) NOT NULL DEFAULT 'dahuagov',
|
||||
source_name VARCHAR(100) NOT NULL DEFAULT '大化县政府网采购公告',
|
||||
announcement_type VARCHAR(50) NOT NULL DEFAULT 'purchase',
|
||||
crawled_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
content_hash VARCHAR(32) UNIQUE,
|
||||
is_new BOOLEAN DEFAULT TRUE
|
||||
);
|
||||
|
||||
-- 创建索引
|
||||
CREATE INDEX IF NOT EXISTS idx_announcements_publish_date ON announcements(publish_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_announcements_source_code ON announcements(source_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_announcements_content_hash ON announcements(content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_announcements_created_at ON announcements(created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_crawl_results_crawled_at ON crawl_results(crawled_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_dahuagov_publish_date ON dahuagov_announcements(publish_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_dahuagov_content_hash ON dahuagov_announcements(content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_dahuagov_created_at ON dahuagov_announcements(created_at DESC);
|
||||
|
||||
-- 创建更新时间触发器
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
@@ -704,6 +724,13 @@ class DatabaseManager:
|
||||
COUNT(DISTINCT source_code) as sources,
|
||||
MAX(crawled_at) as last_crawl
|
||||
FROM manual_announcements
|
||||
UNION ALL
|
||||
SELECT
|
||||
'dahuagov_announcements' as table_name,
|
||||
COUNT(*) as count,
|
||||
COUNT(DISTINCT source_code) as sources,
|
||||
MAX(crawled_at) as last_crawl
|
||||
FROM dahuagov_announcements
|
||||
"""
|
||||
|
||||
cursor.execute(detail_sql)
|
||||
@@ -742,12 +769,14 @@ class DatabaseManager:
|
||||
SELECT content_hash FROM auto_announcements WHERE content_hash = %s
|
||||
UNION ALL
|
||||
SELECT content_hash FROM manual_announcements WHERE content_hash = %s
|
||||
UNION ALL
|
||||
SELECT content_hash FROM dahuagov_announcements WHERE content_hash = %s
|
||||
) as combined_check LIMIT 1
|
||||
"""
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, (content_hash, content_hash, content_hash))
|
||||
cursor.execute(sql, (content_hash, content_hash, content_hash, content_hash))
|
||||
return cursor.fetchone() is not None
|
||||
except Exception as e:
|
||||
logger.error(f"检查公告存在性失败: {str(e)}")
|
||||
@@ -790,6 +819,133 @@ class DatabaseManager:
|
||||
logger.error(f"获取最近公告失败: {str(e)}")
|
||||
return []
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def save_dahuagov_announcements(self, announcements: List[Announcement]) -> int:
|
||||
"""
|
||||
保存大化县政府网公告(全部推送,不筛选关键词)
|
||||
|
||||
Args:
|
||||
announcements: 公告列表
|
||||
|
||||
Returns:
|
||||
int: 成功保存的新公告数量
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return 0
|
||||
|
||||
if not announcements:
|
||||
return 0
|
||||
|
||||
# 为没有哈希的公告生成哈希
|
||||
for announcement in announcements:
|
||||
if not announcement.content_hash:
|
||||
announcement.generate_content_hash()
|
||||
|
||||
sql = """
|
||||
INSERT INTO dahuagov_announcements (
|
||||
title, publish_date, purchase_name, content_url, source_code, source_name,
|
||||
announcement_type, crawled_at, content_hash, is_new
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (content_hash) DO NOTHING
|
||||
"""
|
||||
|
||||
values = []
|
||||
for announcement in announcements:
|
||||
values.append((
|
||||
announcement.title,
|
||||
announcement.publish_date,
|
||||
announcement.purchase_name,
|
||||
announcement.content_url,
|
||||
announcement.source_code,
|
||||
announcement.source_name,
|
||||
announcement.announcement_type.value,
|
||||
announcement.crawled_at,
|
||||
announcement.content_hash,
|
||||
announcement.is_new
|
||||
))
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
extras.execute_batch(cursor, sql, values)
|
||||
affected_rows = cursor.rowcount
|
||||
logger.info(f"保存大化县公告完成,新增 {affected_rows} 条")
|
||||
return affected_rows
|
||||
except Exception as e:
|
||||
logger.error(f"保存大化县公告失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def get_new_dahuagov_announcements(self) -> List[Announcement]:
|
||||
"""
|
||||
获取大化县未推送的新公告(is_new = TRUE)
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 未推送的公告列表
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return []
|
||||
|
||||
sql = """
|
||||
SELECT * FROM dahuagov_announcements
|
||||
WHERE is_new = TRUE
|
||||
ORDER BY publish_date DESC
|
||||
"""
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
announcements = []
|
||||
for row in rows:
|
||||
row_dict = dict(row)
|
||||
row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type'])
|
||||
announcements.append(Announcement.from_dict(row_dict))
|
||||
|
||||
return announcements
|
||||
except Exception as e:
|
||||
logger.error(f"获取大化县新公告失败: {str(e)}")
|
||||
return []
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=3))
|
||||
def mark_dahuagov_announcements_sent(self, announcements: List[Announcement]) -> int:
|
||||
"""
|
||||
标记大化县公告已发送(is_new = FALSE)
|
||||
|
||||
Args:
|
||||
announcements: 已发送的公告列表
|
||||
|
||||
Returns:
|
||||
int: 更新的记录数
|
||||
"""
|
||||
if not self.config.database.enabled:
|
||||
return 0
|
||||
|
||||
if not announcements:
|
||||
return 0
|
||||
|
||||
# 获取所有公告的哈希值
|
||||
hashes = [ann.content_hash for ann in announcements if ann.content_hash]
|
||||
|
||||
if not hashes:
|
||||
return 0
|
||||
|
||||
sql = """
|
||||
UPDATE dahuagov_announcements
|
||||
SET is_new = FALSE, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE content_hash = ANY(%s)
|
||||
"""
|
||||
|
||||
try:
|
||||
with get_db_cursor() as cursor:
|
||||
cursor.execute(sql, (hashes,))
|
||||
affected_rows = cursor.rowcount
|
||||
logger.info(f"标记大化县公告已发送完成,更新 {affected_rows} 条")
|
||||
return affected_rows
|
||||
except Exception as e:
|
||||
logger.error(f"标记大化县公告已发送失败: {str(e)}")
|
||||
return 0
|
||||
|
||||
|
||||
# 全局数据库管理器实例
|
||||
_db_manager = None
|
||||
|
||||
Reference in New Issue
Block a user