手动模式
This commit is contained in:
@@ -0,0 +1,489 @@
|
||||
"""
|
||||
高可用性模块
|
||||
提供重试机制、超时控制、幂等操作、异常处理和恢复功能
|
||||
"""
|
||||
|
||||
import time
|
||||
import random
|
||||
import hashlib
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from typing import Callable, Any, Optional, Type, Union, List
|
||||
from datetime import datetime, timedelta
|
||||
import threading
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from .logger import get_logger
|
||||
from .config_manager import get_config
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class RetryConfig:
|
||||
"""重试配置"""
|
||||
|
||||
def __init__(self,
|
||||
max_retries: int = 3,
|
||||
initial_delay: float = 1.0,
|
||||
max_delay: float = 60.0,
|
||||
backoff_factor: float = 2.0,
|
||||
jitter: bool = True):
|
||||
"""
|
||||
初始化重试配置
|
||||
|
||||
Args:
|
||||
max_retries: 最大重试次数
|
||||
initial_delay: 初始延迟时间（秒）
|
||||
max_delay: 最大延迟时间（秒）
|
||||
backoff_factor: 退避因子
|
||||
jitter: 是否添加随机抖动
|
||||
"""
|
||||
self.max_retries = max_retries
|
||||
self.initial_delay = initial_delay
|
||||
self.max_delay = max_delay
|
||||
self.backoff_factor = backoff_factor
|
||||
self.jitter = jitter
|
||||
|
||||
|
||||
class TimeoutConfig:
|
||||
"""超时配置"""
|
||||
|
||||
def __init__(self,
|
||||
connect_timeout: float = 10.0,
|
||||
read_timeout: float = 30.0,
|
||||
total_timeout: Optional[float] = None):
|
||||
"""
|
||||
初始化超时配置
|
||||
|
||||
Args:
|
||||
connect_timeout: 连接超时时间（秒）
|
||||
read_timeout: 读取超时时间（秒）
|
||||
total_timeout: 总超时时间（秒）
|
||||
"""
|
||||
self.connect_timeout = connect_timeout
|
||||
self.read_timeout = read_timeout
|
||||
self.total_timeout = total_timeout or (connect_timeout + read_timeout)
|
||||
|
||||
|
||||
class CircuitBreakerState:
|
||||
"""熔断器状态"""
|
||||
CLOSED = "closed" # 关闭状态，正常工作
|
||||
OPEN = "open" # 打开状态，快速失败
|
||||
HALF_OPEN = "half_open" # 半开状态，测试恢复
|
||||
|
||||
|
||||
class CircuitBreaker:
|
||||
"""熔断器实现"""
|
||||
|
||||
def __init__(self,
|
||||
failure_threshold: int = 5,
|
||||
recovery_timeout: int = 60,
|
||||
expected_exception: Type[Exception] = Exception):
|
||||
"""
|
||||
初始化熔断器
|
||||
|
||||
Args:
|
||||
failure_threshold: 失败阈值
|
||||
recovery_timeout: 恢复超时时间（秒）
|
||||
expected_exception: 期望的异常类型
|
||||
"""
|
||||
self.failure_threshold = failure_threshold
|
||||
self.recovery_timeout = recovery_timeout
|
||||
self.expected_exception = expected_exception
|
||||
|
||||
self.state = CircuitBreakerState.CLOSED
|
||||
self.failure_count = 0
|
||||
self.last_failure_time = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def __call__(self, func: Callable) -> Callable:
|
||||
"""装饰器实现"""
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return self._execute_with_circuit_breaker(func, *args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
def _execute_with_circuit_breaker(self, func: Callable, *args, **kwargs) -> Any:
|
||||
"""使用熔断器执行函数"""
|
||||
if self.state == CircuitBreakerState.OPEN:
|
||||
if self._should_attempt_reset():
|
||||
self.state = CircuitBreakerState.HALF_OPEN
|
||||
logger.info("熔断器半开，尝试恢复")
|
||||
else:
|
||||
raise CircuitBreakerOpenException("熔断器已打开")
|
||||
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
self._on_success()
|
||||
return result
|
||||
except self.expected_exception as e:
|
||||
self._on_failure()
|
||||
raise
|
||||
|
||||
def _should_attempt_reset(self) -> bool:
|
||||
"""检查是否应该尝试重置"""
|
||||
if self.last_failure_time is None:
|
||||
return True
|
||||
return (datetime.now() - self.last_failure_time).total_seconds() >= self.recovery_timeout
|
||||
|
||||
def _on_success(self):
|
||||
"""成功时的处理"""
|
||||
with self._lock:
|
||||
if self.state == CircuitBreakerState.HALF_OPEN:
|
||||
self.state = CircuitBreakerState.CLOSED
|
||||
self.failure_count = 0
|
||||
logger.info("熔断器关闭，服务恢复正常")
|
||||
|
||||
def _on_failure(self):
|
||||
"""失败时的处理"""
|
||||
with self._lock:
|
||||
self.failure_count += 1
|
||||
self.last_failure_time = datetime.now()
|
||||
|
||||
if self.failure_count >= self.failure_threshold:
|
||||
self.state = CircuitBreakerState.OPEN
|
||||
logger.warning(f"熔断器打开，失败次数达到阈值: {self.failure_count}")
|
||||
|
||||
|
||||
class CircuitBreakerOpenException(Exception):
|
||||
"""熔断器打开异常"""
|
||||
pass
|
||||
|
||||
|
||||
class IdempotencyKey:
|
||||
"""幂等性键生成器"""
|
||||
|
||||
@staticmethod
|
||||
def generate(*args, **kwargs) -> str:
|
||||
"""
|
||||
生成幂等性键
|
||||
|
||||
Args:
|
||||
*args: 位置参数
|
||||
**kwargs: 关键字参数
|
||||
|
||||
Returns:
|
||||
str: 幂等性键
|
||||
"""
|
||||
# 将参数转换为字符串并排序
|
||||
key_parts = []
|
||||
|
||||
# 处理位置参数
|
||||
for i, arg in enumerate(args):
|
||||
key_parts.append(f"arg_{i}:{str(arg)}")
|
||||
|
||||
# 处理关键字参数（排序以保证一致性）
|
||||
for key in sorted(kwargs.keys()):
|
||||
key_parts.append(f"{key}:{str(kwargs[key])}")
|
||||
|
||||
# 生成哈希
|
||||
key_string = "|".join(key_parts)
|
||||
return hashlib.md5(key_string.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
class IdempotencyManager:
|
||||
"""幂等性管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self._executed_keys = set()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def is_executed(self, key: str) -> bool:
|
||||
"""
|
||||
检查操作是否已执行
|
||||
|
||||
Args:
|
||||
key: 幂等性键
|
||||
|
||||
Returns:
|
||||
bool: 是否已执行
|
||||
"""
|
||||
with self._lock:
|
||||
return key in self._executed_keys
|
||||
|
||||
def mark_executed(self, key: str):
|
||||
"""
|
||||
标记操作已执行
|
||||
|
||||
Args:
|
||||
key: 幂等性键
|
||||
"""
|
||||
with self._lock:
|
||||
self._executed_keys.add(key)
|
||||
|
||||
def clear_expired_keys(self, max_age_seconds: int = 3600):
|
||||
"""
|
||||
清理过期的键（简化实现，实际应该使用时间戳）
|
||||
|
||||
Args:
|
||||
max_age_seconds: 最大年龄（秒）
|
||||
"""
|
||||
# 这里简化实现，实际项目中应该记录时间戳
|
||||
pass
|
||||
|
||||
|
||||
def retry_on_exception(retry_config: Optional[RetryConfig] = None,
|
||||
exceptions: tuple = (Exception,),
|
||||
logger: Optional[Any] = None) -> Callable:
|
||||
"""
|
||||
重试装饰器
|
||||
|
||||
Args:
|
||||
retry_config: 重试配置
|
||||
exceptions: 需要重试的异常类型
|
||||
logger: 日志器
|
||||
|
||||
Returns:
|
||||
Callable: 装饰器函数
|
||||
"""
|
||||
if retry_config is None:
|
||||
retry_config = RetryConfig()
|
||||
|
||||
if logger is None:
|
||||
logger = get_logger()
|
||||
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
last_exception = None
|
||||
|
||||
for attempt in range(retry_config.max_retries + 1):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except exceptions as e:
|
||||
last_exception = e
|
||||
|
||||
if attempt < retry_config.max_retries:
|
||||
# 计算延迟时间
|
||||
delay = min(
|
||||
retry_config.initial_delay * (retry_config.backoff_factor ** attempt),
|
||||
retry_config.max_delay
|
||||
)
|
||||
|
||||
# 添加随机抖动
|
||||
if retry_config.jitter:
|
||||
delay = delay * (0.5 + random.random() * 0.5)
|
||||
|
||||
logger.warning(
|
||||
f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{retry_config.max_retries + 1}): {str(e)}，"
|
||||
f"等待 {delay:.2f} 秒后重试"
|
||||
)
|
||||
time.sleep(delay)
|
||||
else:
|
||||
logger.error(
|
||||
f"函数 {func.__name__} 在 {retry_config.max_retries + 1} 次尝试后仍然失败: {str(e)}"
|
||||
)
|
||||
|
||||
raise last_exception
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
def timeout_wrapper(timeout_config: Optional[TimeoutConfig] = None) -> Callable:
|
||||
"""
|
||||
超时装饰器
|
||||
|
||||
Args:
|
||||
timeout_config: 超时配置
|
||||
|
||||
Returns:
|
||||
Callable: 装饰器函数
|
||||
"""
|
||||
if timeout_config is None:
|
||||
timeout_config = TimeoutConfig()
|
||||
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
import signal
|
||||
|
||||
def timeout_handler(signum, frame):
|
||||
raise TimeoutError(f"函数 {func.__name__} 执行超时")
|
||||
|
||||
# 设置信号处理器
|
||||
old_handler = signal.signal(signal.SIGALRM, timeout_handler)
|
||||
signal.alarm(int(timeout_config.total_timeout))
|
||||
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
signal.alarm(0) # 取消闹钟
|
||||
return result
|
||||
finally:
|
||||
signal.signal(signal.SIGALRM, old_handler)
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_with_retry(timeout_config: Optional[TimeoutConfig] = None,
|
||||
retry_config: Optional[RetryConfig] = None):
|
||||
"""
|
||||
创建带有重试机制的HTTP会话
|
||||
|
||||
Args:
|
||||
timeout_config: 超时配置
|
||||
retry_config: 重试配置
|
||||
|
||||
Yields:
|
||||
requests.Session: 配置好的会话对象
|
||||
"""
|
||||
if timeout_config is None:
|
||||
timeout_config = TimeoutConfig()
|
||||
|
||||
if retry_config is None:
|
||||
retry_config = RetryConfig()
|
||||
|
||||
session = requests.Session()
|
||||
|
||||
# 配置重试策略
|
||||
retry_strategy = Retry(
|
||||
total=retry_config.max_retries,
|
||||
backoff_factor=retry_config.backoff_factor,
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
)
|
||||
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
|
||||
# 设置默认超时
|
||||
session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
|
||||
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def safe_execute(func: Callable,
|
||||
fallback: Optional[Callable] = None,
|
||||
exceptions: tuple = (Exception,),
|
||||
logger: Optional[Any] = None) -> Any:
|
||||
"""
|
||||
安全执行函数，提供降级处理
|
||||
|
||||
Args:
|
||||
func: 要执行的函数
|
||||
fallback: 降级函数
|
||||
exceptions: 需要捕获的异常类型
|
||||
logger: 日志器
|
||||
|
||||
Returns:
|
||||
Any: 函数执行结果或降级结果
|
||||
"""
|
||||
if logger is None:
|
||||
logger = get_logger()
|
||||
|
||||
try:
|
||||
return func()
|
||||
except exceptions as e:
|
||||
logger.error(f"函数执行失败: {str(e)}")
|
||||
if fallback:
|
||||
try:
|
||||
logger.info("执行降级函数")
|
||||
return fallback()
|
||||
except Exception as fallback_e:
|
||||
logger.error(f"降级函数也执行失败: {str(fallback_e)}")
|
||||
return None
|
||||
|
||||
|
||||
class HealthChecker:
|
||||
"""健康检查器"""
|
||||
|
||||
def __init__(self, check_interval: int = 300):
|
||||
"""
|
||||
初始化健康检查器
|
||||
|
||||
Args:
|
||||
check_interval: 检查间隔（秒）
|
||||
"""
|
||||
self.check_interval = check_interval
|
||||
self.last_check = None
|
||||
self.is_healthy = True
|
||||
self.consecutive_failures = 0
|
||||
self.max_consecutive_failures = 3
|
||||
|
||||
def check_health(self) -> bool:
|
||||
"""
|
||||
执行健康检查
|
||||
|
||||
Returns:
|
||||
bool: 健康状态
|
||||
"""
|
||||
current_time = datetime.now()
|
||||
|
||||
# 检查是否需要执行检查
|
||||
if (self.last_check and
|
||||
(current_time - self.last_check).total_seconds() < self.check_interval):
|
||||
return self.is_healthy
|
||||
|
||||
self.last_check = current_time
|
||||
|
||||
try:
|
||||
# 执行健康检查逻辑
|
||||
self._perform_health_check()
|
||||
self.is_healthy = True
|
||||
self.consecutive_failures = 0
|
||||
logger.info("健康检查通过")
|
||||
return True
|
||||
except Exception as e:
|
||||
self.consecutive_failures += 1
|
||||
logger.warning(f"健康检查失败 ({self.consecutive_failures}/{self.max_consecutive_failures}): {str(e)}")
|
||||
|
||||
if self.consecutive_failures >= self.max_consecutive_failures:
|
||||
self.is_healthy = False
|
||||
logger.error("连续健康检查失败，系统标记为不健康")
|
||||
|
||||
return False
|
||||
|
||||
def _perform_health_check(self):
|
||||
"""执行具体的健康检查逻辑"""
|
||||
# 这里可以添加数据库连接检查、外部服务检查等
|
||||
config = get_config()
|
||||
|
||||
# 检查数据库连接（如果启用）
|
||||
if config.database.enabled:
|
||||
# 这里应该检查数据库连接
|
||||
pass
|
||||
|
||||
# 检查网络连接
|
||||
try:
|
||||
requests.get("https://www.baidu.com", timeout=5)
|
||||
except:
|
||||
raise Exception("网络连接检查失败")
|
||||
|
||||
|
||||
# 全局实例
|
||||
_circuit_breaker = CircuitBreaker()
|
||||
_idempotency_manager = IdempotencyManager()
|
||||
_health_checker = HealthChecker()
|
||||
|
||||
|
||||
def get_circuit_breaker() -> CircuitBreaker:
|
||||
"""获取全局熔断器实例"""
|
||||
return _circuit_breaker
|
||||
|
||||
|
||||
def get_idempotency_manager() -> IdempotencyManager:
|
||||
"""获取全局幂等性管理器实例"""
|
||||
return _idempotency_manager
|
||||
|
||||
|
||||
def get_health_checker() -> HealthChecker:
|
||||
"""获取全局健康检查器实例"""
|
||||
return _health_checker
|
||||
|
||||
|
||||
def check_system_health() -> bool:
|
||||
"""
|
||||
检查系统健康状态
|
||||
|
||||
Returns:
|
||||
bool: 系统是否健康
|
||||
"""
|
||||
return _health_checker.check_health()
|
||||
Reference in New Issue
Block a user