Remove obsolete scripts related to WeChat menu creation, server functionality, and cron job scheduling. Update project structure by deleting unnecessary files to streamline the codebase.
This commit is contained in:
+36
-11
@@ -39,7 +39,7 @@ try:
|
||||
# 初始化存储
|
||||
init_storage()
|
||||
|
||||
# 执行搜索（爬取所有公告，然后进行关键词筛选）
|
||||
# 执行搜索（爬取所有公告，筛选出新增的关键词匹配公告）
|
||||
logger.info("开始执行定时搜索任务")
|
||||
crawl_results = crawl_announcements()
|
||||
|
||||
@@ -64,34 +64,58 @@ try:
|
||||
from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
|
||||
from datetime import date
|
||||
|
||||
# 1. 关键词筛选
|
||||
keyword_filter = KeywordFilter()
|
||||
keyword_filtered = keyword_filter.filter_announcements(all_announcements, keywords=config.crawler.keyword)
|
||||
|
||||
# 2. 日期筛选（只保留今天的）
|
||||
# 对关键词筛选结果进行日期筛选（只处理今天的）
|
||||
date_filter = DateFilter()
|
||||
today_announcements = date_filter.filter_announcements(
|
||||
today_keyword_announcements = date_filter.filter_announcements(
|
||||
keyword_filtered,
|
||||
start_date=date.today(),
|
||||
end_date=date.today()
|
||||
)
|
||||
|
||||
logger.info(f"关键词筛选后剩余 {len(keyword_filtered)} 条公告")
|
||||
logger.info(f"筛选出今天 {len(today_announcements)} 条匹配公告")
|
||||
logger.info(f"筛选出今天关键词匹配 {len(today_keyword_announcements)} 条公告")
|
||||
|
||||
if not today_announcements:
|
||||
logger.info("今天没有匹配关键词的公告")
|
||||
if not today_keyword_announcements:
|
||||
logger.info("今天没有关键词匹配的公告")
|
||||
return True
|
||||
|
||||
# 检查auto_announcements表，筛选出真正新增的公告
|
||||
truly_new_announcements = []
|
||||
for ann in today_keyword_announcements:
|
||||
try:
|
||||
# 直接导入，避免相对导入问题
|
||||
import gx_gp_monitor.core.database as db_module
|
||||
with db_module.get_db_cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM auto_announcements WHERE content_hash = %s LIMIT 1",
|
||||
(ann.content_hash,)
|
||||
)
|
||||
exists = cursor.fetchone() is not None
|
||||
if not exists:
|
||||
truly_new_announcements.append(ann)
|
||||
except Exception as e:
|
||||
logger.warning(f"检查公告是否存在失败: {str(e)}")
|
||||
# 如果检查失败，为了安全起见，不添加到新公告列表
|
||||
pass
|
||||
|
||||
logger.info(f"从今天关键词匹配公告中筛选出 {len(truly_new_announcements)} 条auto_announcements表中不存在的新公告")
|
||||
|
||||
if not truly_new_announcements:
|
||||
logger.info("没有真正新增的关键词匹配公告，任务完成")
|
||||
return True
|
||||
|
||||
# 构造筛选统计信息
|
||||
filter_stats = type('FilterResult', (), {
|
||||
"keyword_filtered": len(all_announcements) - len(keyword_filtered),
|
||||
"date_filtered": len(keyword_filtered) - len(today_announcements),
|
||||
"duplicate_filtered": 0,
|
||||
"date_filtered": len(keyword_filtered) - len(today_keyword_announcements),
|
||||
"duplicate_filtered": len(today_keyword_announcements) - len(truly_new_announcements),
|
||||
"source_filtered": 0
|
||||
})()
|
||||
|
||||
filtered_announcements = today_announcements
|
||||
filtered_announcements = truly_new_announcements
|
||||
|
||||
logger.info(f"关键词筛选后剩余 {len(filtered_announcements)} 条公告")
|
||||
|
||||
@@ -119,7 +143,8 @@ try:
|
||||
print("\n=== 定时搜索任务完成 ===")
|
||||
print(f"总共爬取: {total_crawled} 条公告")
|
||||
print(f"关键词筛选: {len(keyword_filtered)} 条")
|
||||
print(f"今日匹配公告: {len(today_announcements)} 条")
|
||||
print(f"今日关键词匹配: {len(today_keyword_announcements)} 条")
|
||||
print(f"真正新增公告: {len(truly_new_announcements)} 条")
|
||||
print(f"筛选后公告: {len(filtered_announcements)} 条")
|
||||
print(f"保存到数据库: {saved_count} 条")
|
||||
print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""调度模块"""
|
||||
@@ -1,372 +0,0 @@
|
||||
"""
|
||||
调度器模块
|
||||
提供定时任务调度功能，支持cron表达式和间隔执行
|
||||
"""
|
||||
|
||||
import time
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Callable, Dict, Any, Optional, List
|
||||
import schedule
|
||||
from croniter import croniter
|
||||
|
||||
try:
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger
|
||||
except ImportError:
|
||||
from core.config_manager import get_config
|
||||
from core.logger import get_logger
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class TaskScheduler:
|
||||
"""任务调度器"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config().scheduler
|
||||
self._running = False
|
||||
self._thread = None
|
||||
self._jobs = {}
|
||||
self._job_stats = {}
|
||||
|
||||
# 初始化调度
|
||||
if self.config.enabled:
|
||||
self._load_jobs_from_config()
|
||||
|
||||
def _load_jobs_from_config(self):
|
||||
"""从配置加载定时任务"""
|
||||
for job_config in self.config.jobs:
|
||||
if job_config.get("enabled", False):
|
||||
job_name = job_config["name"]
|
||||
cron_expr = job_config["cron"]
|
||||
|
||||
# 根据任务名称创建对应的任务函数
|
||||
if job_name == "daily_crawl":
|
||||
func = self._create_crawl_job()
|
||||
elif job_name == "data_cleanup":
|
||||
func = self._create_cleanup_job()
|
||||
else:
|
||||
logger.warning(f"未知的任务类型: {job_name}")
|
||||
continue
|
||||
|
||||
self.add_cron_job(job_name, cron_expr, func)
|
||||
|
||||
def _create_crawl_job(self) -> Callable:
|
||||
"""创建爬取任务"""
|
||||
def crawl_job():
|
||||
try:
|
||||
logger.info("开始执行定时爬取任务")
|
||||
|
||||
# 导入这里避免循环导入
|
||||
try:
|
||||
from ..crawler.spider import crawl_announcements
|
||||
from ..filters.filters import filter_from_config
|
||||
from ..storage.postgresql import save_announcements_to_storage
|
||||
from ..storage.md_generator import generate_onu_md
|
||||
from ..notification.wechat import send_announcements_notification
|
||||
except ImportError:
|
||||
from crawler.spider import crawl_announcements
|
||||
from filters.filters import filter_from_config
|
||||
from storage.postgresql import save_announcements_to_storage
|
||||
from storage.md_generator import generate_onu_md
|
||||
from notification.wechat import send_announcements_notification
|
||||
|
||||
# 执行爬取
|
||||
crawl_results = crawl_announcements()
|
||||
|
||||
if not crawl_results:
|
||||
logger.info("定时爬取任务完成：无数据")
|
||||
return
|
||||
|
||||
# 收集所有公告
|
||||
all_announcements = []
|
||||
for result in crawl_results:
|
||||
if result.announcements:
|
||||
all_announcements.extend(result.announcements)
|
||||
|
||||
if not all_announcements:
|
||||
logger.info("定时爬取任务完成：无新公告")
|
||||
return
|
||||
|
||||
# 筛选公告
|
||||
filter_obj = filter_from_config()
|
||||
filtered_announcements, filter_stats = filter_obj.filter(all_announcements)
|
||||
|
||||
logger.info(f"筛选结果: {len(all_announcements)} -> {len(filtered_announcements)}")
|
||||
|
||||
# 保存到数据库
|
||||
saved_count = save_announcements_to_storage(filtered_announcements)
|
||||
|
||||
# 生成Markdown文件
|
||||
generate_onu_md(filtered_announcements)
|
||||
|
||||
# 发送通知
|
||||
if filtered_announcements:
|
||||
send_announcements_notification(filtered_announcements)
|
||||
|
||||
logger.info(f"定时爬取任务完成：处理 {len(filtered_announcements)} 条公告，保存 {saved_count} 条")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"定时爬取任务执行失败: {str(e)}")
|
||||
# 发送错误通知
|
||||
from ..notification.wechat import send_error_alert
|
||||
send_error_alert("定时爬取任务失败", str(e))
|
||||
|
||||
return crawl_job
|
||||
|
||||
def _create_cleanup_job(self) -> Callable:
|
||||
"""创建数据清理任务"""
|
||||
def cleanup_job():
|
||||
try:
|
||||
logger.info("开始执行数据清理任务")
|
||||
|
||||
try:
|
||||
from ..storage.postgresql import cleanup_storage
|
||||
except ImportError:
|
||||
from storage.postgresql import cleanup_storage
|
||||
|
||||
# 执行清理
|
||||
deleted_count = cleanup_storage()
|
||||
|
||||
logger.info(f"数据清理任务完成：删除 {deleted_count} 条过期数据")
|
||||
|
||||
# 发送通知（如果删除的数据较多）
|
||||
if deleted_count > 0:
|
||||
try:
|
||||
from ..notification.wechat import send_system_notification
|
||||
except ImportError:
|
||||
from notification.wechat import send_system_notification
|
||||
send_system_notification(
|
||||
"数据清理完成",
|
||||
f"已清理 {deleted_count} 条过期数据"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"数据清理任务执行失败: {str(e)}")
|
||||
|
||||
return cleanup_job
|
||||
|
||||
def add_cron_job(self, name: str, cron_expr: str, func: Callable) -> bool:
|
||||
"""
|
||||
添加cron定时任务
|
||||
|
||||
Args:
|
||||
name: 任务名称
|
||||
cron_expr: cron表达式
|
||||
func: 任务函数
|
||||
|
||||
Returns:
|
||||
bool: 添加是否成功
|
||||
"""
|
||||
try:
|
||||
# 验证cron表达式
|
||||
croniter(cron_expr)
|
||||
|
||||
# 添加到schedule
|
||||
schedule.every().day.at("00:00").do(func) # 临时设置，会被替换
|
||||
|
||||
# 存储任务信息
|
||||
self._jobs[name] = {
|
||||
"func": func,
|
||||
"cron": cron_expr,
|
||||
"next_run": None,
|
||||
"last_run": None,
|
||||
"run_count": 0,
|
||||
"error_count": 0
|
||||
}
|
||||
|
||||
logger.info(f"添加定时任务: {name} ({cron_expr})")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"添加定时任务失败 {name}: {str(e)}")
|
||||
return False
|
||||
|
||||
def add_interval_job(self, name: str, interval_seconds: int, func: Callable) -> bool:
|
||||
"""
|
||||
添加间隔执行任务
|
||||
|
||||
Args:
|
||||
name: 任务名称
|
||||
interval_seconds: 执行间隔（秒）
|
||||
func: 任务函数
|
||||
|
||||
Returns:
|
||||
bool: 添加是否成功
|
||||
"""
|
||||
try:
|
||||
schedule.every(interval_seconds).seconds.do(func)
|
||||
|
||||
self._jobs[name] = {
|
||||
"func": func,
|
||||
"interval": interval_seconds,
|
||||
"next_run": None,
|
||||
"last_run": None,
|
||||
"run_count": 0,
|
||||
"error_count": 0
|
||||
}
|
||||
|
||||
logger.info(f"添加间隔任务: {name} ({interval_seconds}秒)")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"添加间隔任务失败 {name}: {str(e)}")
|
||||
return False
|
||||
|
||||
def remove_job(self, name: str) -> bool:
|
||||
"""
|
||||
移除任务
|
||||
|
||||
Args:
|
||||
name: 任务名称
|
||||
|
||||
Returns:
|
||||
bool: 移除是否成功
|
||||
"""
|
||||
if name in self._jobs:
|
||||
# 注意：schedule库没有直接的移除方法
|
||||
# 这里只是从我们的记录中移除
|
||||
del self._jobs[name]
|
||||
logger.info(f"移除任务: {name}")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def start(self):
|
||||
"""启动调度器"""
|
||||
if self._running:
|
||||
logger.warning("调度器已经在运行中")
|
||||
return
|
||||
|
||||
if not self.config.enabled:
|
||||
logger.info("调度器已禁用")
|
||||
return
|
||||
|
||||
self._running = True
|
||||
self._thread = threading.Thread(target=self._run_scheduler, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
logger.info("调度器已启动")
|
||||
|
||||
def stop(self):
|
||||
"""停止调度器"""
|
||||
if not self._running:
|
||||
return
|
||||
|
||||
self._running = False
|
||||
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=5)
|
||||
|
||||
logger.info("调度器已停止")
|
||||
|
||||
def _run_scheduler(self):
|
||||
"""运行调度器主循环"""
|
||||
logger.info("调度器主循环开始")
|
||||
|
||||
while self._running:
|
||||
try:
|
||||
schedule.run_pending()
|
||||
time.sleep(1)
|
||||
except Exception as e:
|
||||
logger.error(f"调度器运行异常: {str(e)}")
|
||||
time.sleep(5) # 出错后等待5秒再继续
|
||||
|
||||
logger.info("调度器主循环结束")
|
||||
|
||||
def run_once(self, job_name: Optional[str] = None):
|
||||
"""
|
||||
手动执行任务一次
|
||||
|
||||
Args:
|
||||
job_name: 任务名称，如果为None则执行所有任务
|
||||
"""
|
||||
if job_name:
|
||||
if job_name in self._jobs:
|
||||
job_info = self._jobs[job_name]
|
||||
logger.info(f"手动执行任务: {job_name}")
|
||||
|
||||
try:
|
||||
job_info["func"]()
|
||||
job_info["run_count"] += 1
|
||||
job_info["last_run"] = datetime.now()
|
||||
logger.info(f"任务 {job_name} 执行完成")
|
||||
except Exception as e:
|
||||
job_info["error_count"] += 1
|
||||
logger.error(f"任务 {job_name} 执行失败: {str(e)}")
|
||||
else:
|
||||
logger.error(f"任务不存在: {job_name}")
|
||||
else:
|
||||
# 执行所有任务
|
||||
for name in list(self._jobs.keys()):
|
||||
self.run_once(name)
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取调度器状态
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 状态信息
|
||||
"""
|
||||
jobs_status = {}
|
||||
for name, job_info in self._jobs.items():
|
||||
jobs_status[name] = {
|
||||
"enabled": True,
|
||||
"run_count": job_info.get("run_count", 0),
|
||||
"error_count": job_info.get("error_count", 0),
|
||||
"last_run": job_info.get("last_run").isoformat() if job_info.get("last_run") else None,
|
||||
"next_run": job_info.get("next_run").isoformat() if job_info.get("next_run") else None
|
||||
}
|
||||
|
||||
return {
|
||||
"enabled": self.config.enabled,
|
||||
"running": self._running,
|
||||
"timezone": self.config.timezone,
|
||||
"jobs": jobs_status
|
||||
}
|
||||
|
||||
def is_running(self) -> bool:
|
||||
"""检查调度器是否正在运行"""
|
||||
return self._running
|
||||
|
||||
|
||||
# 全局调度器实例
|
||||
_scheduler = None
|
||||
|
||||
|
||||
def get_scheduler() -> TaskScheduler:
|
||||
"""
|
||||
获取调度器实例
|
||||
|
||||
Returns:
|
||||
TaskScheduler: 调度器实例
|
||||
"""
|
||||
global _scheduler
|
||||
if _scheduler is None:
|
||||
_scheduler = TaskScheduler()
|
||||
return _scheduler
|
||||
|
||||
|
||||
def start_scheduler():
|
||||
"""启动调度器"""
|
||||
scheduler = get_scheduler()
|
||||
scheduler.start()
|
||||
|
||||
|
||||
def stop_scheduler():
|
||||
"""停止调度器"""
|
||||
scheduler = get_scheduler()
|
||||
scheduler.stop()
|
||||
|
||||
|
||||
def run_scheduled_jobs(job_name: Optional[str] = None):
|
||||
"""
|
||||
手动执行定时任务
|
||||
|
||||
Args:
|
||||
job_name: 任务名称
|
||||
"""
|
||||
scheduler = get_scheduler()
|
||||
scheduler.run_once(job_name)
|
||||
Reference in New Issue
Block a user