v6ole/GX-gp-notify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

删除无用文件

Browse Source
This commit is contained in:
v6ole
2026-01-09 15:17:42 +08:00
parent f00068f804
commit b37e8ce581
37 changed files with 644 additions and 14407 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gx_gp_monitor/core/__pycache__/database.cpython-313.pyc
BIN
View File
Binary file not shown.
gx_gp_monitor/core/__pycache__/models.cpython-313.pyc
BIN
View File
Binary file not shown.
gx_gp_monitor/core/database.py
+218 -11
View File
@@ -182,7 +182,47 @@ class DatabaseManager:
# 创建表的SQL语句
create_tables_sql = """
-- 公告表
-- 定时搜索公告表（关键词匹配专用）
CREATE TABLE IF NOT EXISTS auto_announcements (
id SERIAL PRIMARY KEY,
title VARCHAR(500) NOT NULL,
publish_date TIMESTAMP NOT NULL,
purchase_name VARCHAR(200),
content_url TEXT,
source_code VARCHAR(50) NOT NULL,
source_name VARCHAR(100) NOT NULL,
announcement_type VARCHAR(50) NOT NULL,
crawled_at TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
content_hash VARCHAR(32) UNIQUE,
keyword_matched BOOLEAN DEFAULT FALSE,
date_filtered BOOLEAN DEFAULT TRUE,
is_new BOOLEAN DEFAULT TRUE,
is_today BOOLEAN DEFAULT FALSE
);
-- 手动搜索公告表（全量数据专用）
CREATE TABLE IF NOT EXISTS manual_announcements (
id SERIAL PRIMARY KEY,
title VARCHAR(500) NOT NULL,
publish_date TIMESTAMP NOT NULL,
purchase_name VARCHAR(200),
content_url TEXT,
source_code VARCHAR(50) NOT NULL,
source_name VARCHAR(100) NOT NULL,
announcement_type VARCHAR(50) NOT NULL,
crawled_at TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
content_hash VARCHAR(32),
keyword_matched BOOLEAN DEFAULT FALSE,
date_filtered BOOLEAN DEFAULT TRUE,
is_new BOOLEAN DEFAULT TRUE,
is_today BOOLEAN DEFAULT FALSE
);
-- 原公告表（保留兼容性）
CREATE TABLE IF NOT EXISTS announcements (
id SERIAL PRIMARY KEY,
title VARCHAR(500) NOT NULL,
@@ -195,6 +235,7 @@ class DatabaseManager:
crawled_at TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
crawl_mode VARCHAR(20) DEFAULT 'auto',
content_hash VARCHAR(32) UNIQUE,
keyword_matched BOOLEAN DEFAULT FALSE,
date_filtered BOOLEAN DEFAULT TRUE,
@@ -331,9 +372,9 @@ class DatabaseManager:
sql = """
INSERT INTO announcements (
title, publish_date, purchase_name, content_url, source_code, source_name,
announcement_type, crawled_at, content_hash, keyword_matched,
announcement_type, crawled_at, crawl_mode, content_hash, keyword_matched,
date_filtered, is_new, is_today
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (content_hash) DO NOTHING
"""
@@ -348,6 +389,7 @@ class DatabaseManager:
announcement.source_name,
announcement.announcement_type.value,
announcement.crawled_at,
announcement.crawl_mode,
announcement.content_hash,
announcement.keyword_matched,
announcement.date_filtered,
@@ -365,6 +407,94 @@ class DatabaseManager:
logger.error(f"批量保存公告失败: {str(e)}")
return 0
def save_announcements_batch_to_table(self, announcements: List[Announcement], table_name: str) -> int:
"""
批量保存公告到指定表
Args:
announcements: 公告列表
table_name: 目标表名 ("auto_announcements" 或 "manual_announcements")
Returns:
int: 成功保存的数量
"""
if not self.config.database.enabled:
return 0
if not announcements:
return 0
# 为没有哈希的公告生成哈希
for announcement in announcements:
if not announcement.content_hash:
announcement.generate_content_hash()
# 根据表名决定是否使用ON CONFLICT
if table_name == "manual_announcements":
# 手动搜索表不使用唯一约束（允许重复）
sql = f"""
INSERT INTO {table_name} (
title, publish_date, purchase_name, content_url, source_code, source_name,
announcement_type, crawled_at, content_hash, keyword_matched,
date_filtered, is_new, is_today
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
values = []
for announcement in announcements:
values.append((
announcement.title,
announcement.publish_date,
announcement.purchase_name,
announcement.content_url,
announcement.source_code,
announcement.source_name,
announcement.announcement_type.value,
announcement.crawled_at,
announcement.content_hash,
announcement.keyword_matched,
announcement.date_filtered,
announcement.is_new,
announcement.is_today
))
else:
# 自动搜索表使用唯一约束
sql = f"""
INSERT INTO {table_name} (
title, publish_date, purchase_name, content_url, source_code, source_name,
announcement_type, crawled_at, content_hash, keyword_matched,
date_filtered, is_new, is_today
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (content_hash) DO NOTHING
"""
values = []
for announcement in announcements:
values.append((
announcement.title,
announcement.publish_date,
announcement.purchase_name,
announcement.content_url,
announcement.source_code,
announcement.source_name,
announcement.announcement_type.value,
announcement.crawled_at,
announcement.content_hash,
announcement.keyword_matched,
announcement.date_filtered,
announcement.is_new,
announcement.is_today
))
try:
with get_db_cursor() as cursor:
extras.execute_batch(cursor, sql, values)
affected_rows = cursor.rowcount
logger.info(f"批量保存公告到{table_name}完成，影响行数: {affected_rows}")
return affected_rows
except Exception as e:
logger.error(f"批量保存公告到{table_name}失败: {str(e)}")
return 0
@retry_on_exception(RetryConfig(max_retries=3))
def get_announcements(self,
source_code: Optional[str] = None,
@@ -504,21 +634,89 @@ class DatabaseManager:
if not self.config.database.enabled:
return {}
# 统计所有表的综合信息
sql = """
SELECT
COUNT(*) as total_announcements,
COUNT(CASE WHEN is_today THEN 1 END) as today_announcements,
COUNT(CASE WHEN is_new THEN 1 END) as new_announcements,
SUM(total_count) as total_announcements,
SUM(today_count) as today_announcements,
SUM(new_count) as new_announcements,
COUNT(DISTINCT source_code) as sources_count,
MAX(crawled_at) as last_crawl_time
FROM announcements
MAX(last_crawl_time) as last_crawl_time
FROM (
SELECT
COUNT(*) as total_count,
COUNT(CASE WHEN is_today THEN 1 END) as today_count,
COUNT(CASE WHEN is_new THEN 1 END) as new_count,
source_code,
MAX(crawled_at) as last_crawl_time
FROM announcements
GROUP BY source_code
UNION ALL
SELECT
COUNT(*) as total_count,
COUNT(CASE WHEN is_today THEN 1 END) as today_count,
COUNT(CASE WHEN is_new THEN 1 END) as new_count,
source_code,
MAX(crawled_at) as last_crawl_time
FROM auto_announcements
GROUP BY source_code
UNION ALL
SELECT
COUNT(*) as total_count,
COUNT(CASE WHEN is_today THEN 1 END) as today_count,
COUNT(CASE WHEN is_new THEN 1 END) as new_count,
source_code,
MAX(crawled_at) as last_crawl_time
FROM manual_announcements
GROUP BY source_code
) as combined_stats
"""
try:
with get_db_cursor() as cursor:
cursor.execute(sql)
result = cursor.fetchone()
return dict(result) if result else {}
stats = dict(result) if result else {}
# 添加各表详细统计
detail_sql = """
SELECT
'announcements' as table_name,
COUNT(*) as count,
COUNT(DISTINCT source_code) as sources,
MAX(crawled_at) as last_crawl
FROM announcements
UNION ALL
SELECT
'auto_announcements' as table_name,
COUNT(*) as count,
COUNT(DISTINCT source_code) as sources,
MAX(crawled_at) as last_crawl
FROM auto_announcements
UNION ALL
SELECT
'manual_announcements' as table_name,
COUNT(*) as count,
COUNT(DISTINCT source_code) as sources,
MAX(crawled_at) as last_crawl
FROM manual_announcements
"""
cursor.execute(detail_sql)
detail_results = cursor.fetchall()
stats['table_details'] = {row['table_name']: {
'count': row['count'],
'sources': row['sources'],
'last_crawl': row['last_crawl']
} for row in detail_results}
return stats
except Exception as e:
logger.error(f"获取统计信息失败: {str(e)}")
return {}
@@ -536,11 +734,20 @@ class DatabaseManager:
if not self.config.database.enabled:
return False
sql = "SELECT 1 FROM announcements WHERE content_hash = %s LIMIT 1"
# 检查所有表中是否存在
sql = """
SELECT 1 FROM (
SELECT content_hash FROM announcements WHERE content_hash = %s
UNION ALL
SELECT content_hash FROM auto_announcements WHERE content_hash = %s
UNION ALL
SELECT content_hash FROM manual_announcements WHERE content_hash = %s
) as combined_check LIMIT 1
"""
try:
with get_db_cursor() as cursor:
cursor.execute(sql, (content_hash,))
cursor.execute(sql, (content_hash, content_hash, content_hash))
return cursor.fetchone() is not None
except Exception as e:
logger.error(f"检查公告存在性失败: {str(e)}")
gx_gp_monitor/core/models.py
+1
View File
@@ -60,6 +60,7 @@ class Announcement:
crawled_at: Optional[datetime] = None # 爬取时间
created_at: Optional[datetime] = None # 创建时间
updated_at: Optional[datetime] = None # 更新时间
crawl_mode: str = "auto" # 爬取模式：auto(自动)/manual(手动)
# 去重字段
content_hash: Optional[str] = None # 内容哈希，用于去重