feat: deliver alert events through persistent outbox
This commit is contained in:
@@ -0,0 +1,427 @@
|
||||
"""Transactional alert-event and notification-outbox workflow tests."""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
|
||||
from app.config import Settings
|
||||
from app.models import (
|
||||
AlertEvent,
|
||||
AlertTypeEnum,
|
||||
Base,
|
||||
Device,
|
||||
DeviceTypeEnum,
|
||||
NotificationOutbox,
|
||||
NotificationStatus,
|
||||
PingRecord,
|
||||
)
|
||||
from app.services.alerter import Alerter
|
||||
from app.services.fping_runner import ProbeResult
|
||||
from app.services.pinger import DeviceStateChange, Pinger
|
||||
from app.services.scheduler import PingScheduler
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db_session() -> AsyncSession:
|
||||
"""Provide a complete isolated persistence boundary for alert tests."""
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(Base.metadata.create_all)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
yield session
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def offline_change(db_session: AsyncSession) -> DeviceStateChange:
|
||||
"""Persist a device and expose a real offline state-change value."""
|
||||
observed_at = datetime(2026, 8, 4, 9, 30)
|
||||
device = Device(
|
||||
name="核心交换机",
|
||||
ip="10.0.0.8",
|
||||
device_type=DeviceTypeEnum.switch,
|
||||
location="一楼机房",
|
||||
project_name="园区网",
|
||||
current_status="offline",
|
||||
last_ping_time=observed_at,
|
||||
)
|
||||
db_session.add(device)
|
||||
await db_session.commit()
|
||||
return DeviceStateChange(
|
||||
device=device,
|
||||
old_status="online",
|
||||
new_status="offline",
|
||||
consecutive_failures=2,
|
||||
event_type="offline",
|
||||
reason="2 consecutive full-loss rounds",
|
||||
)
|
||||
|
||||
|
||||
async def test_transition_creates_event_and_pending_outbox_in_one_transaction(
|
||||
db_session: AsyncSession,
|
||||
offline_change: DeviceStateChange,
|
||||
):
|
||||
"""Removing either insert breaks the durable event-delivery contract."""
|
||||
event = await Alerter().record_transition(offline_change, db_session)
|
||||
|
||||
outbox = await db_session.scalar(
|
||||
select(NotificationOutbox).where(
|
||||
NotificationOutbox.alert_event_id == event.id
|
||||
)
|
||||
)
|
||||
|
||||
assert outbox is not None
|
||||
assert outbox.status == NotificationStatus.pending
|
||||
assert "核心交换机" in outbox.message_content
|
||||
assert "10.0.0.8" in outbox.message_content
|
||||
assert "一楼机房" in outbox.message_content
|
||||
assert "园区网" in outbox.message_content
|
||||
assert "2 consecutive full-loss rounds" in outbox.message_content
|
||||
assert "2026-08-04 09:30:00" in outbox.message_content
|
||||
|
||||
|
||||
async def test_event_and_outbox_rollback_together(
|
||||
db_session: AsyncSession,
|
||||
offline_change: DeviceStateChange,
|
||||
):
|
||||
"""A caller rollback cannot retain an event without its notification."""
|
||||
await Alerter().record_transition(offline_change, db_session)
|
||||
|
||||
await db_session.rollback()
|
||||
|
||||
assert await db_session.scalar(select(AlertEvent)) is None
|
||||
assert await db_session.scalar(select(NotificationOutbox)) is None
|
||||
|
||||
|
||||
async def test_offline_escalation_closes_and_links_open_degraded_event(
|
||||
db_session: AsyncSession,
|
||||
offline_change: DeviceStateChange,
|
||||
):
|
||||
"""Escalating degraded to offline must not leave two open incidents."""
|
||||
device = offline_change.device
|
||||
device.last_ping_time = datetime(2026, 8, 4, 9, 20)
|
||||
degraded = await Alerter().record_transition(
|
||||
DeviceStateChange(
|
||||
device=device,
|
||||
old_status="online",
|
||||
new_status="degraded",
|
||||
consecutive_failures=0,
|
||||
event_type="degraded",
|
||||
reason="window packet loss 20.00%",
|
||||
),
|
||||
db_session,
|
||||
)
|
||||
await db_session.commit()
|
||||
device.last_ping_time = datetime(2026, 8, 4, 9, 30)
|
||||
|
||||
offline = await Alerter().record_transition(
|
||||
DeviceStateChange(
|
||||
device=device,
|
||||
old_status="degraded",
|
||||
new_status="offline",
|
||||
consecutive_failures=2,
|
||||
event_type="offline",
|
||||
reason="2 consecutive full-loss rounds",
|
||||
),
|
||||
db_session,
|
||||
)
|
||||
|
||||
assert degraded.is_resolved is True
|
||||
assert degraded.end_at == datetime(2026, 8, 4, 9, 30)
|
||||
assert degraded.duration_minutes == 10
|
||||
assert degraded.related_event_id == offline.id
|
||||
assert offline.related_event_id == degraded.id
|
||||
|
||||
|
||||
async def test_recovery_closes_open_fault_and_notifies_with_duration(
|
||||
db_session: AsyncSession,
|
||||
offline_change: DeviceStateChange,
|
||||
):
|
||||
"""Recovery closes one active incident and carries its duration to operators."""
|
||||
opened = await Alerter().record_transition(offline_change, db_session)
|
||||
await db_session.commit()
|
||||
device = offline_change.device
|
||||
device.last_ping_time = datetime(2026, 8, 4, 10, 1)
|
||||
|
||||
recovered = await Alerter().record_transition(
|
||||
DeviceStateChange(
|
||||
device=device,
|
||||
old_status="offline",
|
||||
new_status="online",
|
||||
consecutive_failures=0,
|
||||
event_type="recovered",
|
||||
reason="3 consecutive clean rounds",
|
||||
),
|
||||
db_session,
|
||||
)
|
||||
outbox = await db_session.scalar(
|
||||
select(NotificationOutbox).where(
|
||||
NotificationOutbox.alert_event_id == recovered.id
|
||||
)
|
||||
)
|
||||
|
||||
assert opened.is_resolved is True
|
||||
assert opened.end_at == datetime(2026, 8, 4, 10, 1)
|
||||
assert opened.duration_minutes == 31
|
||||
assert recovered.alert_type == AlertTypeEnum.recovered
|
||||
assert recovered.is_resolved is True
|
||||
assert recovered.duration_minutes == 31
|
||||
assert recovered.related_event_id == opened.id
|
||||
assert "持续时间：31 分钟" in outbox.message_content
|
||||
|
||||
|
||||
async def test_probe_state_event_and_outbox_share_the_caller_transaction(
|
||||
monkeypatch,
|
||||
):
|
||||
"""A failed caller commit cannot persist a state transition without its event."""
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
observed_at = datetime.now() - timedelta(seconds=30)
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(Base.metadata.create_all)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as setup:
|
||||
device = Device(
|
||||
name="edge-atomic",
|
||||
ip="10.0.0.18",
|
||||
device_type=DeviceTypeEnum.switch,
|
||||
current_status="online",
|
||||
is_enabled=True,
|
||||
offline_consecutive_rounds=2,
|
||||
)
|
||||
setup.add(device)
|
||||
await setup.flush()
|
||||
device_id = device.id
|
||||
setup.add(
|
||||
PingRecord(
|
||||
device_id=device_id,
|
||||
is_alive=False,
|
||||
response_time_ms=None,
|
||||
round_num=1,
|
||||
sent_count=3,
|
||||
received_count=0,
|
||||
packet_loss_percent=100.0,
|
||||
average_rtt_ms=None,
|
||||
is_valid=True,
|
||||
created_at=observed_at,
|
||||
)
|
||||
)
|
||||
await setup.commit()
|
||||
|
||||
async def fake_run_fping_count(*args, **kwargs):
|
||||
return {
|
||||
"10.0.0.18": ProbeResult(
|
||||
"10.0.0.18",
|
||||
3,
|
||||
0,
|
||||
None,
|
||||
True,
|
||||
)
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.pinger.run_fping_count",
|
||||
fake_run_fping_count,
|
||||
)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
changes = await Pinger().run_one_round(session)
|
||||
await Alerter().record_transition(changes[0], session)
|
||||
await session.rollback()
|
||||
|
||||
async with AsyncSession(engine, expire_on_commit=False) as verification:
|
||||
persisted_device = await verification.get(Device, device_id)
|
||||
records = list(
|
||||
(
|
||||
await verification.execute(
|
||||
select(PingRecord).where(PingRecord.device_id == device_id)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assert persisted_device.current_status == "online"
|
||||
assert len(records) == 1
|
||||
assert await verification.scalar(select(AlertEvent)) is None
|
||||
assert await verification.scalar(select(NotificationOutbox)) is None
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
class RecordingSession:
|
||||
"""Small session boundary exposing transaction and close ordering."""
|
||||
|
||||
def __init__(self, events: list[str]):
|
||||
self.events = events
|
||||
self.closed = asyncio.Event()
|
||||
|
||||
async def __aenter__(self):
|
||||
self.events.append("session-enter")
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
self.events.append("session-exit")
|
||||
self.closed.set()
|
||||
|
||||
async def commit(self):
|
||||
self.events.append("commit")
|
||||
|
||||
async def rollback(self):
|
||||
self.events.append("rollback")
|
||||
|
||||
|
||||
class RecordingPinger:
|
||||
def __init__(self, events: list[str], changes=None):
|
||||
self.events = events
|
||||
self.changes = list(changes or ["transition"])
|
||||
|
||||
async def run_one_round(self, db):
|
||||
self.events.append("probe")
|
||||
return self.changes
|
||||
|
||||
|
||||
class RecordingAlerter:
|
||||
def __init__(self, events: list[str]):
|
||||
self.events = events
|
||||
|
||||
async def record_transition(self, change, db):
|
||||
self.events.append(f"alert:{change}")
|
||||
|
||||
|
||||
class RecordingDispatcher:
|
||||
def __init__(self, events: list[str]):
|
||||
self.events = events
|
||||
|
||||
async def dispatch_due(self, db, now):
|
||||
assert isinstance(now, datetime)
|
||||
self.events.append("dispatch")
|
||||
|
||||
|
||||
async def test_scheduler_persists_transitions_before_dispatching_due_messages():
|
||||
"""Changing scheduler order cannot expose uncommitted outbox rows to dispatch."""
|
||||
events: list[str] = []
|
||||
session = RecordingSession(events)
|
||||
scheduler = PingScheduler(
|
||||
pinger=RecordingPinger(events),
|
||||
alerter=RecordingAlerter(events),
|
||||
dispatcher=RecordingDispatcher(events),
|
||||
session_factory=lambda: session,
|
||||
)
|
||||
|
||||
await scheduler._run_cycle()
|
||||
|
||||
assert events == [
|
||||
"session-enter",
|
||||
"probe",
|
||||
"alert:transition",
|
||||
"commit",
|
||||
"dispatch",
|
||||
"commit",
|
||||
"session-exit",
|
||||
]
|
||||
|
||||
|
||||
async def test_scheduler_lock_prevents_overlapping_cycles():
|
||||
"""Even concurrent triggers cannot overlap probe or notification sessions."""
|
||||
events: list[str] = []
|
||||
|
||||
class ConcurrencyPinger:
|
||||
active = 0
|
||||
maximum = 0
|
||||
|
||||
async def run_one_round(self, db):
|
||||
self.active += 1
|
||||
self.maximum = max(self.maximum, self.active)
|
||||
await asyncio.sleep(0.01)
|
||||
self.active -= 1
|
||||
return []
|
||||
|
||||
pinger = ConcurrencyPinger()
|
||||
scheduler = PingScheduler(
|
||||
pinger=pinger,
|
||||
alerter=RecordingAlerter(events),
|
||||
dispatcher=RecordingDispatcher(events),
|
||||
session_factory=lambda: RecordingSession(events),
|
||||
)
|
||||
|
||||
await asyncio.gather(scheduler._run_cycle(), scheduler._run_cycle())
|
||||
|
||||
assert pinger.maximum == 1
|
||||
|
||||
|
||||
async def test_scheduler_stop_waits_for_active_session_to_close():
|
||||
"""Cancellation cannot return while a probe session remains open."""
|
||||
events: list[str] = []
|
||||
session = RecordingSession(events)
|
||||
probe_started = asyncio.Event()
|
||||
never_complete = asyncio.Event()
|
||||
|
||||
class BlockingPinger:
|
||||
async def run_one_round(self, db):
|
||||
probe_started.set()
|
||||
await never_complete.wait()
|
||||
|
||||
scheduler = PingScheduler(
|
||||
pinger=BlockingPinger(),
|
||||
alerter=RecordingAlerter(events),
|
||||
dispatcher=RecordingDispatcher(events),
|
||||
session_factory=lambda: session,
|
||||
interval_seconds=1,
|
||||
)
|
||||
scheduler.start()
|
||||
first_task = scheduler._task
|
||||
scheduler.start()
|
||||
|
||||
await probe_started.wait()
|
||||
await scheduler.stop()
|
||||
|
||||
assert first_task is not None
|
||||
assert session.closed.is_set()
|
||||
assert scheduler._task is None
|
||||
assert events[-2:] == ["rollback", "session-exit"]
|
||||
|
||||
|
||||
async def test_scheduler_leaves_pending_outbox_untouched_when_delivery_disabled():
|
||||
"""Disabling WeCom preserves queued notifications for a later enablement."""
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
class EmptyPinger:
|
||||
async def run_one_round(self, db):
|
||||
return []
|
||||
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(Base.metadata.create_all)
|
||||
async with session_factory() as setup:
|
||||
setup.add(
|
||||
NotificationOutbox(
|
||||
alert_event_id=42,
|
||||
message_content="保留待发送事件",
|
||||
)
|
||||
)
|
||||
await setup.commit()
|
||||
|
||||
runtime_settings = Settings(wecom_notification_enabled=False)
|
||||
scheduler = PingScheduler(
|
||||
pinger=EmptyPinger(),
|
||||
alerter=Alerter(runtime_settings),
|
||||
session_factory=session_factory,
|
||||
runtime_settings=runtime_settings,
|
||||
)
|
||||
await scheduler._run_cycle()
|
||||
|
||||
async with session_factory() as verification:
|
||||
outbox = await verification.scalar(select(NotificationOutbox))
|
||||
assert outbox.status == NotificationStatus.pending
|
||||
assert outbox.attempt_count == 0
|
||||
await scheduler.stop()
|
||||
finally:
|
||||
await engine.dispose()
|
||||
Reference in New Issue
Block a user