490 lines
14 KiB
Python
490 lines
14 KiB
Python
"""
|
||
高可用性模块
|
||
提供重试机制、超时控制、幂等操作、异常处理和恢复功能
|
||
"""
|
||
|
||
import time
|
||
import random
|
||
import hashlib
|
||
from contextlib import contextmanager
|
||
from functools import wraps
|
||
from typing import Callable, Any, Optional, Type, Union, List
|
||
from datetime import datetime, timedelta
|
||
import threading
|
||
import requests
|
||
from requests.adapters import HTTPAdapter
|
||
from urllib3.util.retry import Retry
|
||
|
||
from .logger import get_logger
|
||
from .config_manager import get_config
|
||
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
|
||
class RetryConfig:
|
||
"""重试配置"""
|
||
|
||
def __init__(self,
|
||
max_retries: int = 3,
|
||
initial_delay: float = 1.0,
|
||
max_delay: float = 60.0,
|
||
backoff_factor: float = 2.0,
|
||
jitter: bool = True):
|
||
"""
|
||
初始化重试配置
|
||
|
||
Args:
|
||
max_retries: 最大重试次数
|
||
initial_delay: 初始延迟时间（秒）
|
||
max_delay: 最大延迟时间（秒）
|
||
backoff_factor: 退避因子
|
||
jitter: 是否添加随机抖动
|
||
"""
|
||
self.max_retries = max_retries
|
||
self.initial_delay = initial_delay
|
||
self.max_delay = max_delay
|
||
self.backoff_factor = backoff_factor
|
||
self.jitter = jitter
|
||
|
||
|
||
class TimeoutConfig:
|
||
"""超时配置"""
|
||
|
||
def __init__(self,
|
||
connect_timeout: float = 10.0,
|
||
read_timeout: float = 30.0,
|
||
total_timeout: Optional[float] = None):
|
||
"""
|
||
初始化超时配置
|
||
|
||
Args:
|
||
connect_timeout: 连接超时时间（秒）
|
||
read_timeout: 读取超时时间（秒）
|
||
total_timeout: 总超时时间（秒）
|
||
"""
|
||
self.connect_timeout = connect_timeout
|
||
self.read_timeout = read_timeout
|
||
self.total_timeout = total_timeout or (connect_timeout + read_timeout)
|
||
|
||
|
||
class CircuitBreakerState:
|
||
"""熔断器状态"""
|
||
CLOSED = "closed" # 关闭状态，正常工作
|
||
OPEN = "open" # 打开状态，快速失败
|
||
HALF_OPEN = "half_open" # 半开状态，测试恢复
|
||
|
||
|
||
class CircuitBreaker:
|
||
"""熔断器实现"""
|
||
|
||
def __init__(self,
|
||
failure_threshold: int = 5,
|
||
recovery_timeout: int = 60,
|
||
expected_exception: Type[Exception] = Exception):
|
||
"""
|
||
初始化熔断器
|
||
|
||
Args:
|
||
failure_threshold: 失败阈值
|
||
recovery_timeout: 恢复超时时间（秒）
|
||
expected_exception: 期望的异常类型
|
||
"""
|
||
self.failure_threshold = failure_threshold
|
||
self.recovery_timeout = recovery_timeout
|
||
self.expected_exception = expected_exception
|
||
|
||
self.state = CircuitBreakerState.CLOSED
|
||
self.failure_count = 0
|
||
self.last_failure_time = None
|
||
self._lock = threading.Lock()
|
||
|
||
def __call__(self, func: Callable) -> Callable:
|
||
"""装饰器实现"""
|
||
@wraps(func)
|
||
def wrapper(*args, **kwargs):
|
||
return self._execute_with_circuit_breaker(func, *args, **kwargs)
|
||
return wrapper
|
||
|
||
def _execute_with_circuit_breaker(self, func: Callable, *args, **kwargs) -> Any:
|
||
"""使用熔断器执行函数"""
|
||
if self.state == CircuitBreakerState.OPEN:
|
||
if self._should_attempt_reset():
|
||
self.state = CircuitBreakerState.HALF_OPEN
|
||
logger.info("熔断器半开，尝试恢复")
|
||
else:
|
||
raise CircuitBreakerOpenException("熔断器已打开")
|
||
|
||
try:
|
||
result = func(*args, **kwargs)
|
||
self._on_success()
|
||
return result
|
||
except self.expected_exception as e:
|
||
self._on_failure()
|
||
raise
|
||
|
||
def _should_attempt_reset(self) -> bool:
|
||
"""检查是否应该尝试重置"""
|
||
if self.last_failure_time is None:
|
||
return True
|
||
return (datetime.now() - self.last_failure_time).total_seconds() >= self.recovery_timeout
|
||
|
||
def _on_success(self):
|
||
"""成功时的处理"""
|
||
with self._lock:
|
||
if self.state == CircuitBreakerState.HALF_OPEN:
|
||
self.state = CircuitBreakerState.CLOSED
|
||
self.failure_count = 0
|
||
logger.info("熔断器关闭，服务恢复正常")
|
||
|
||
def _on_failure(self):
|
||
"""失败时的处理"""
|
||
with self._lock:
|
||
self.failure_count += 1
|
||
self.last_failure_time = datetime.now()
|
||
|
||
if self.failure_count >= self.failure_threshold:
|
||
self.state = CircuitBreakerState.OPEN
|
||
logger.warning(f"熔断器打开，失败次数达到阈值: {self.failure_count}")
|
||
|
||
|
||
class CircuitBreakerOpenException(Exception):
|
||
"""熔断器打开异常"""
|
||
pass
|
||
|
||
|
||
class IdempotencyKey:
|
||
"""幂等性键生成器"""
|
||
|
||
@staticmethod
|
||
def generate(*args, **kwargs) -> str:
|
||
"""
|
||
生成幂等性键
|
||
|
||
Args:
|
||
*args: 位置参数
|
||
**kwargs: 关键字参数
|
||
|
||
Returns:
|
||
str: 幂等性键
|
||
"""
|
||
# 将参数转换为字符串并排序
|
||
key_parts = []
|
||
|
||
# 处理位置参数
|
||
for i, arg in enumerate(args):
|
||
key_parts.append(f"arg_{i}:{str(arg)}")
|
||
|
||
# 处理关键字参数（排序以保证一致性）
|
||
for key in sorted(kwargs.keys()):
|
||
key_parts.append(f"{key}:{str(kwargs[key])}")
|
||
|
||
# 生成哈希
|
||
key_string = "|".join(key_parts)
|
||
return hashlib.md5(key_string.encode('utf-8')).hexdigest()
|
||
|
||
|
||
class IdempotencyManager:
|
||
"""幂等性管理器"""
|
||
|
||
def __init__(self):
|
||
self._executed_keys = set()
|
||
self._lock = threading.Lock()
|
||
|
||
def is_executed(self, key: str) -> bool:
|
||
"""
|
||
检查操作是否已执行
|
||
|
||
Args:
|
||
key: 幂等性键
|
||
|
||
Returns:
|
||
bool: 是否已执行
|
||
"""
|
||
with self._lock:
|
||
return key in self._executed_keys
|
||
|
||
def mark_executed(self, key: str):
|
||
"""
|
||
标记操作已执行
|
||
|
||
Args:
|
||
key: 幂等性键
|
||
"""
|
||
with self._lock:
|
||
self._executed_keys.add(key)
|
||
|
||
def clear_expired_keys(self, max_age_seconds: int = 3600):
|
||
"""
|
||
清理过期的键（简化实现，实际应该使用时间戳）
|
||
|
||
Args:
|
||
max_age_seconds: 最大年龄（秒）
|
||
"""
|
||
# 这里简化实现，实际项目中应该记录时间戳
|
||
pass
|
||
|
||
|
||
def retry_on_exception(retry_config: Optional[RetryConfig] = None,
|
||
exceptions: tuple = (Exception,),
|
||
logger: Optional[Any] = None) -> Callable:
|
||
"""
|
||
重试装饰器
|
||
|
||
Args:
|
||
retry_config: 重试配置
|
||
exceptions: 需要重试的异常类型
|
||
logger: 日志器
|
||
|
||
Returns:
|
||
Callable: 装饰器函数
|
||
"""
|
||
if retry_config is None:
|
||
retry_config = RetryConfig()
|
||
|
||
if logger is None:
|
||
logger = get_logger()
|
||
|
||
def decorator(func: Callable) -> Callable:
|
||
@wraps(func)
|
||
def wrapper(*args, **kwargs):
|
||
last_exception = None
|
||
|
||
for attempt in range(retry_config.max_retries + 1):
|
||
try:
|
||
return func(*args, **kwargs)
|
||
except exceptions as e:
|
||
last_exception = e
|
||
|
||
if attempt < retry_config.max_retries:
|
||
# 计算延迟时间
|
||
delay = min(
|
||
retry_config.initial_delay * (retry_config.backoff_factor ** attempt),
|
||
retry_config.max_delay
|
||
)
|
||
|
||
# 添加随机抖动
|
||
if retry_config.jitter:
|
||
delay = delay * (0.5 + random.random() * 0.5)
|
||
|
||
logger.warning(
|
||
f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{retry_config.max_retries + 1}): {str(e)}，"
|
||
f"等待 {delay:.2f} 秒后重试"
|
||
)
|
||
time.sleep(delay)
|
||
else:
|
||
logger.error(
|
||
f"函数 {func.__name__} 在 {retry_config.max_retries + 1} 次尝试后仍然失败: {str(e)}"
|
||
)
|
||
|
||
raise last_exception
|
||
|
||
return wrapper
|
||
return decorator
|
||
|
||
|
||
def timeout_wrapper(timeout_config: Optional[TimeoutConfig] = None) -> Callable:
|
||
"""
|
||
超时装饰器
|
||
|
||
Args:
|
||
timeout_config: 超时配置
|
||
|
||
Returns:
|
||
Callable: 装饰器函数
|
||
"""
|
||
if timeout_config is None:
|
||
timeout_config = TimeoutConfig()
|
||
|
||
def decorator(func: Callable) -> Callable:
|
||
@wraps(func)
|
||
def wrapper(*args, **kwargs):
|
||
import signal
|
||
|
||
def timeout_handler(signum, frame):
|
||
raise TimeoutError(f"函数 {func.__name__} 执行超时")
|
||
|
||
# 设置信号处理器
|
||
old_handler = signal.signal(signal.SIGALRM, timeout_handler)
|
||
signal.alarm(int(timeout_config.total_timeout))
|
||
|
||
try:
|
||
result = func(*args, **kwargs)
|
||
signal.alarm(0) # 取消闹钟
|
||
return result
|
||
finally:
|
||
signal.signal(signal.SIGALRM, old_handler)
|
||
|
||
return wrapper
|
||
return decorator
|
||
|
||
|
||
@contextmanager
|
||
def session_with_retry(timeout_config: Optional[TimeoutConfig] = None,
|
||
retry_config: Optional[RetryConfig] = None):
|
||
"""
|
||
创建带有重试机制的HTTP会话
|
||
|
||
Args:
|
||
timeout_config: 超时配置
|
||
retry_config: 重试配置
|
||
|
||
Yields:
|
||
requests.Session: 配置好的会话对象
|
||
"""
|
||
if timeout_config is None:
|
||
timeout_config = TimeoutConfig()
|
||
|
||
if retry_config is None:
|
||
retry_config = RetryConfig()
|
||
|
||
session = requests.Session()
|
||
|
||
# 配置重试策略
|
||
retry_strategy = Retry(
|
||
total=retry_config.max_retries,
|
||
backoff_factor=retry_config.backoff_factor,
|
||
status_forcelist=[429, 500, 502, 503, 504],
|
||
)
|
||
|
||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||
session.mount("http://", adapter)
|
||
session.mount("https://", adapter)
|
||
|
||
# 设置默认超时
|
||
session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
|
||
|
||
try:
|
||
yield session
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
def safe_execute(func: Callable,
|
||
fallback: Optional[Callable] = None,
|
||
exceptions: tuple = (Exception,),
|
||
logger: Optional[Any] = None) -> Any:
|
||
"""
|
||
安全执行函数，提供降级处理
|
||
|
||
Args:
|
||
func: 要执行的函数
|
||
fallback: 降级函数
|
||
exceptions: 需要捕获的异常类型
|
||
logger: 日志器
|
||
|
||
Returns:
|
||
Any: 函数执行结果或降级结果
|
||
"""
|
||
if logger is None:
|
||
logger = get_logger()
|
||
|
||
try:
|
||
return func()
|
||
except exceptions as e:
|
||
logger.error(f"函数执行失败: {str(e)}")
|
||
if fallback:
|
||
try:
|
||
logger.info("执行降级函数")
|
||
return fallback()
|
||
except Exception as fallback_e:
|
||
logger.error(f"降级函数也执行失败: {str(fallback_e)}")
|
||
return None
|
||
|
||
|
||
class HealthChecker:
|
||
"""健康检查器"""
|
||
|
||
def __init__(self, check_interval: int = 300):
|
||
"""
|
||
初始化健康检查器
|
||
|
||
Args:
|
||
check_interval: 检查间隔（秒）
|
||
"""
|
||
self.check_interval = check_interval
|
||
self.last_check = None
|
||
self.is_healthy = True
|
||
self.consecutive_failures = 0
|
||
self.max_consecutive_failures = 3
|
||
|
||
def check_health(self) -> bool:
|
||
"""
|
||
执行健康检查
|
||
|
||
Returns:
|
||
bool: 健康状态
|
||
"""
|
||
current_time = datetime.now()
|
||
|
||
# 检查是否需要执行检查
|
||
if (self.last_check and
|
||
(current_time - self.last_check).total_seconds() < self.check_interval):
|
||
return self.is_healthy
|
||
|
||
self.last_check = current_time
|
||
|
||
try:
|
||
# 执行健康检查逻辑
|
||
self._perform_health_check()
|
||
self.is_healthy = True
|
||
self.consecutive_failures = 0
|
||
logger.info("健康检查通过")
|
||
return True
|
||
except Exception as e:
|
||
self.consecutive_failures += 1
|
||
logger.warning(f"健康检查失败 ({self.consecutive_failures}/{self.max_consecutive_failures}): {str(e)}")
|
||
|
||
if self.consecutive_failures >= self.max_consecutive_failures:
|
||
self.is_healthy = False
|
||
logger.error("连续健康检查失败，系统标记为不健康")
|
||
|
||
return False
|
||
|
||
def _perform_health_check(self):
|
||
"""执行具体的健康检查逻辑"""
|
||
# 这里可以添加数据库连接检查、外部服务检查等
|
||
config = get_config()
|
||
|
||
# 检查数据库连接（如果启用）
|
||
if config.database.enabled:
|
||
# 这里应该检查数据库连接
|
||
pass
|
||
|
||
# 检查网络连接
|
||
try:
|
||
requests.get("https://www.baidu.com", timeout=5)
|
||
except:
|
||
raise Exception("网络连接检查失败")
|
||
|
||
|
||
# 全局实例
|
||
_circuit_breaker = CircuitBreaker()
|
||
_idempotency_manager = IdempotencyManager()
|
||
_health_checker = HealthChecker()
|
||
|
||
|
||
def get_circuit_breaker() -> CircuitBreaker:
|
||
"""获取全局熔断器实例"""
|
||
return _circuit_breaker
|
||
|
||
|
||
def get_idempotency_manager() -> IdempotencyManager:
|
||
"""获取全局幂等性管理器实例"""
|
||
return _idempotency_manager
|
||
|
||
|
||
def get_health_checker() -> HealthChecker:
|
||
"""获取全局健康检查器实例"""
|
||
return _health_checker
|
||
|
||
|
||
def check_system_health() -> bool:
|
||
"""
|
||
检查系统健康状态
|
||
|
||
Returns:
|
||
bool: 系统是否健康
|
||
"""
|
||
return _health_checker.check_health()
|