"""Transactional alert-event and notification-outbox workflow tests.""" import asyncio from datetime import datetime, timedelta import pytest from sqlalchemy import select from sqlalchemy.ext.asyncio import ( AsyncSession, async_sessionmaker, create_async_engine, ) from app.config import Settings from app.models import ( AlertEvent, AlertTypeEnum, Base, Device, DeviceTypeEnum, NotificationOutbox, NotificationStatus, PingRecord, ) from app.services.alerter import Alerter from app.services.fping_runner import ProbeResult from app.services.pinger import DeviceStateChange, Pinger from app.services.scheduler import PingScheduler @pytest.fixture async def db_session() -> AsyncSession: """Provide a complete isolated persistence boundary for alert tests.""" engine = create_async_engine("sqlite+aiosqlite:///:memory:") try: async with engine.begin() as connection: await connection.run_sync(Base.metadata.create_all) async with AsyncSession(engine, expire_on_commit=False) as session: yield session finally: await engine.dispose() @pytest.fixture async def offline_change(db_session: AsyncSession) -> DeviceStateChange: """Persist a device and expose a real offline state-change value.""" observed_at = datetime(2026, 8, 4, 9, 30) device = Device( name="核心交换机", ip="10.0.0.8", device_type=DeviceTypeEnum.switch, location="一楼机房", project_name="园区网", current_status="offline", last_ping_time=observed_at, ) db_session.add(device) await db_session.commit() return DeviceStateChange( device=device, old_status="online", new_status="offline", consecutive_failures=2, event_type="offline", reason="2 consecutive full-loss rounds", ) async def test_transition_creates_event_and_pending_outbox_in_one_transaction( db_session: AsyncSession, offline_change: DeviceStateChange, ): """Removing either insert breaks the durable event-delivery contract.""" event = await Alerter().record_transition(offline_change, db_session) outbox = await db_session.scalar( select(NotificationOutbox).where( NotificationOutbox.alert_event_id == event.id ) ) assert outbox is not None assert outbox.status == NotificationStatus.pending assert "核心交换机" in outbox.message_content assert "10.0.0.8" in outbox.message_content assert "一楼机房" in outbox.message_content assert "园区网" in outbox.message_content assert "2 consecutive full-loss rounds" in outbox.message_content assert "2026-08-04 09:30:00" in outbox.message_content async def test_event_and_outbox_rollback_together( db_session: AsyncSession, offline_change: DeviceStateChange, ): """A caller rollback cannot retain an event without its notification.""" await Alerter().record_transition(offline_change, db_session) await db_session.rollback() assert await db_session.scalar(select(AlertEvent)) is None assert await db_session.scalar(select(NotificationOutbox)) is None async def test_offline_escalation_closes_and_links_open_degraded_event( db_session: AsyncSession, offline_change: DeviceStateChange, ): """Escalating degraded to offline must not leave two open incidents.""" device = offline_change.device device.last_ping_time = datetime(2026, 8, 4, 9, 20) degraded = await Alerter().record_transition( DeviceStateChange( device=device, old_status="online", new_status="degraded", consecutive_failures=0, event_type="degraded", reason="window packet loss 20.00%", ), db_session, ) await db_session.commit() device.last_ping_time = datetime(2026, 8, 4, 9, 30) offline = await Alerter().record_transition( DeviceStateChange( device=device, old_status="degraded", new_status="offline", consecutive_failures=2, event_type="offline", reason="2 consecutive full-loss rounds", ), db_session, ) assert degraded.is_resolved is True assert degraded.end_at == datetime(2026, 8, 4, 9, 30) assert degraded.duration_minutes == 10 assert degraded.related_event_id == offline.id assert offline.related_event_id == degraded.id async def test_recovery_closes_open_fault_and_notifies_with_duration( db_session: AsyncSession, offline_change: DeviceStateChange, ): """Recovery closes one active incident and carries its duration to operators.""" opened = await Alerter().record_transition(offline_change, db_session) await db_session.commit() device = offline_change.device device.last_ping_time = datetime(2026, 8, 4, 10, 1) recovered = await Alerter().record_transition( DeviceStateChange( device=device, old_status="offline", new_status="online", consecutive_failures=0, event_type="recovered", reason="3 consecutive clean rounds", ), db_session, ) outbox = await db_session.scalar( select(NotificationOutbox).where( NotificationOutbox.alert_event_id == recovered.id ) ) assert opened.is_resolved is True assert opened.end_at == datetime(2026, 8, 4, 10, 1) assert opened.duration_minutes == 31 assert recovered.alert_type == AlertTypeEnum.recovered assert recovered.is_resolved is True assert recovered.duration_minutes == 31 assert recovered.related_event_id == opened.id assert "持续时间:31 分钟" in outbox.message_content async def test_probe_state_event_and_outbox_share_the_caller_transaction( monkeypatch, ): """A failed caller commit cannot persist a state transition without its event.""" engine = create_async_engine("sqlite+aiosqlite:///:memory:") observed_at = datetime.now() - timedelta(seconds=30) try: async with engine.begin() as connection: await connection.run_sync(Base.metadata.create_all) async with AsyncSession(engine, expire_on_commit=False) as setup: device = Device( name="edge-atomic", ip="10.0.0.18", device_type=DeviceTypeEnum.switch, current_status="online", is_enabled=True, offline_consecutive_rounds=2, ) setup.add(device) await setup.flush() device_id = device.id setup.add( PingRecord( device_id=device_id, is_alive=False, response_time_ms=None, round_num=1, sent_count=3, received_count=0, packet_loss_percent=100.0, average_rtt_ms=None, is_valid=True, created_at=observed_at, ) ) await setup.commit() async def fake_run_fping_count(*args, **kwargs): return { "10.0.0.18": ProbeResult( "10.0.0.18", 3, 0, None, True, ) } monkeypatch.setattr( "app.services.pinger.run_fping_count", fake_run_fping_count, ) async with AsyncSession(engine, expire_on_commit=False) as session: changes = await Pinger().run_one_round(session) await Alerter().record_transition(changes[0], session) await session.rollback() async with AsyncSession(engine, expire_on_commit=False) as verification: persisted_device = await verification.get(Device, device_id) records = list( ( await verification.execute( select(PingRecord).where(PingRecord.device_id == device_id) ) ) .scalars() .all() ) assert persisted_device.current_status == "online" assert len(records) == 1 assert await verification.scalar(select(AlertEvent)) is None assert await verification.scalar(select(NotificationOutbox)) is None finally: await engine.dispose() class RecordingSession: """Small session boundary exposing transaction and close ordering.""" def __init__(self, events: list[str]): self.events = events self.closed = asyncio.Event() async def __aenter__(self): self.events.append("session-enter") return self async def __aexit__(self, exc_type, exc_value, traceback): self.events.append("session-exit") self.closed.set() async def commit(self): self.events.append("commit") async def rollback(self): self.events.append("rollback") class RecordingPinger: def __init__(self, events: list[str], changes=None): self.events = events self.changes = list(changes or ["transition"]) async def run_one_round(self, db): self.events.append("probe") return self.changes class RecordingAlerter: def __init__(self, events: list[str]): self.events = events async def record_transition(self, change, db): self.events.append(f"alert:{change}") class RecordingDispatcher: def __init__(self, events: list[str]): self.events = events async def dispatch_due(self, db, now): assert isinstance(now, datetime) self.events.append("dispatch") async def test_scheduler_persists_transitions_before_dispatching_due_messages(): """Changing scheduler order cannot expose uncommitted outbox rows to dispatch.""" events: list[str] = [] session = RecordingSession(events) scheduler = PingScheduler( pinger=RecordingPinger(events), alerter=RecordingAlerter(events), dispatcher=RecordingDispatcher(events), session_factory=lambda: session, ) await scheduler._run_cycle() assert events == [ "session-enter", "probe", "alert:transition", "commit", "dispatch", "commit", "session-exit", ] async def test_scheduler_lock_prevents_overlapping_cycles(): """Even concurrent triggers cannot overlap probe or notification sessions.""" events: list[str] = [] class ConcurrencyPinger: active = 0 maximum = 0 async def run_one_round(self, db): self.active += 1 self.maximum = max(self.maximum, self.active) await asyncio.sleep(0.01) self.active -= 1 return [] pinger = ConcurrencyPinger() scheduler = PingScheduler( pinger=pinger, alerter=RecordingAlerter(events), dispatcher=RecordingDispatcher(events), session_factory=lambda: RecordingSession(events), ) await asyncio.gather(scheduler._run_cycle(), scheduler._run_cycle()) assert pinger.maximum == 1 async def test_scheduler_stop_waits_for_active_session_to_close(): """Cancellation cannot return while a probe session remains open.""" events: list[str] = [] session = RecordingSession(events) probe_started = asyncio.Event() never_complete = asyncio.Event() class BlockingPinger: async def run_one_round(self, db): probe_started.set() await never_complete.wait() scheduler = PingScheduler( pinger=BlockingPinger(), alerter=RecordingAlerter(events), dispatcher=RecordingDispatcher(events), session_factory=lambda: session, interval_seconds=1, ) scheduler.start() first_task = scheduler._task scheduler.start() await probe_started.wait() await scheduler.stop() assert first_task is not None assert session.closed.is_set() assert scheduler._task is None assert events[-2:] == ["rollback", "session-exit"] async def test_scheduler_leaves_pending_outbox_untouched_when_delivery_disabled(): """Disabling WeCom preserves queued notifications for a later enablement.""" engine = create_async_engine("sqlite+aiosqlite:///:memory:") session_factory = async_sessionmaker(engine, expire_on_commit=False) class EmptyPinger: async def run_one_round(self, db): return [] try: async with engine.begin() as connection: await connection.run_sync(Base.metadata.create_all) async with session_factory() as setup: setup.add( NotificationOutbox( alert_event_id=42, message_content="保留待发送事件", ) ) await setup.commit() runtime_settings = Settings(wecom_notification_enabled=False) scheduler = PingScheduler( pinger=EmptyPinger(), alerter=Alerter(runtime_settings), session_factory=session_factory, runtime_settings=runtime_settings, ) await scheduler._run_cycle() async with session_factory() as verification: outbox = await verification.scalar(select(NotificationOutbox)) assert outbox.status == NotificationStatus.pending assert outbox.attempt_count == 0 await scheduler.stop() finally: await engine.dispose()