PingWatch/backend/tests/test_alert_workflow.py

"""Transactional alert-event and notification-outbox workflow tests."""

import asyncio
from datetime import datetime, timedelta

import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import (
    AsyncSession,
    async_sessionmaker,
    create_async_engine,
)

from app.config import Settings
from app.models import (
    AlertEvent,
    AlertTypeEnum,
    Base,
    Device,
    DeviceTypeEnum,
    NotificationOutbox,
    NotificationStatus,
    PingRecord,
)
from app.services.alerter import Alerter
from app.services.fping_runner import ProbeResult
from app.services.pinger import DeviceStateChange, Pinger
from app.services.scheduler import PingScheduler


@pytest.fixture
async def db_session() -> AsyncSession:
    """Provide a complete isolated persistence boundary for alert tests."""
    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
    try:
        async with engine.begin() as connection:
            await connection.run_sync(Base.metadata.create_all)
        async with AsyncSession(engine, expire_on_commit=False) as session:
            yield session
    finally:
        await engine.dispose()


@pytest.fixture
async def offline_change(db_session: AsyncSession) -> DeviceStateChange:
    """Persist a device and expose a real offline state-change value."""
    observed_at = datetime(2026, 8, 4, 9, 30)
    device = Device(
        name="核心交换机",
        ip="10.0.0.8",
        device_type=DeviceTypeEnum.switch,
        location="一楼机房",
        project_name="园区网",
        current_status="offline",
        last_ping_time=observed_at,
    )
    db_session.add(device)
    await db_session.commit()
    return DeviceStateChange(
        device=device,
        old_status="online",
        new_status="offline",
        consecutive_failures=2,
        event_type="offline",
        reason="2 consecutive full-loss rounds",
    )


async def test_transition_creates_event_and_pending_outbox_in_one_transaction(
    db_session: AsyncSession,
    offline_change: DeviceStateChange,
):
    """Removing either insert breaks the durable event-delivery contract."""
    event = await Alerter().record_transition(offline_change, db_session)

    outbox = await db_session.scalar(
        select(NotificationOutbox).where(
            NotificationOutbox.alert_event_id == event.id
        )
    )

    assert outbox is not None
    assert outbox.status == NotificationStatus.pending
    assert "核心交换机" in outbox.message_content
    assert "10.0.0.8" in outbox.message_content
    assert "一楼机房" in outbox.message_content
    assert "园区网" in outbox.message_content
    assert "2 consecutive full-loss rounds" in outbox.message_content
    assert "2026-08-04 09:30:00" in outbox.message_content


async def test_event_and_outbox_rollback_together(
    db_session: AsyncSession,
    offline_change: DeviceStateChange,
):
    """A caller rollback cannot retain an event without its notification."""
    await Alerter().record_transition(offline_change, db_session)

    await db_session.rollback()

    assert await db_session.scalar(select(AlertEvent)) is None
    assert await db_session.scalar(select(NotificationOutbox)) is None


async def test_offline_escalation_closes_and_links_open_degraded_event(
    db_session: AsyncSession,
    offline_change: DeviceStateChange,
):
    """Escalating degraded to offline must not leave two open incidents."""
    device = offline_change.device
    device.last_ping_time = datetime(2026, 8, 4, 9, 20)
    degraded = await Alerter().record_transition(
        DeviceStateChange(
            device=device,
            old_status="online",
            new_status="degraded",
            consecutive_failures=0,
            event_type="degraded",
            reason="window packet loss 20.00%",
        ),
        db_session,
    )
    await db_session.commit()
    device.last_ping_time = datetime(2026, 8, 4, 9, 30)

    offline = await Alerter().record_transition(
        DeviceStateChange(
            device=device,
            old_status="degraded",
            new_status="offline",
            consecutive_failures=2,
            event_type="offline",
            reason="2 consecutive full-loss rounds",
        ),
        db_session,
    )

    assert degraded.is_resolved is True
    assert degraded.end_at == datetime(2026, 8, 4, 9, 30)
    assert degraded.duration_minutes == 10
    assert degraded.related_event_id == offline.id
    assert offline.related_event_id == degraded.id


async def test_recovery_closes_open_fault_and_notifies_with_duration(
    db_session: AsyncSession,
    offline_change: DeviceStateChange,
):
    """Recovery closes one active incident and carries its duration to operators."""
    opened = await Alerter().record_transition(offline_change, db_session)
    await db_session.commit()
    device = offline_change.device
    device.last_ping_time = datetime(2026, 8, 4, 10, 1)

    recovered = await Alerter().record_transition(
        DeviceStateChange(
            device=device,
            old_status="offline",
            new_status="online",
            consecutive_failures=0,
            event_type="recovered",
            reason="3 consecutive clean rounds",
        ),
        db_session,
    )
    outbox = await db_session.scalar(
        select(NotificationOutbox).where(
            NotificationOutbox.alert_event_id == recovered.id
        )
    )

    assert opened.is_resolved is True
    assert opened.end_at == datetime(2026, 8, 4, 10, 1)
    assert opened.duration_minutes == 31
    assert recovered.alert_type == AlertTypeEnum.recovered
    assert recovered.is_resolved is True
    assert recovered.duration_minutes == 31
    assert recovered.related_event_id == opened.id
    assert "持续时间：31 分钟" in outbox.message_content


async def test_probe_state_event_and_outbox_share_the_caller_transaction(
    monkeypatch,
):
    """A failed caller commit cannot persist a state transition without its event."""
    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
    observed_at = datetime.now() - timedelta(seconds=30)
    try:
        async with engine.begin() as connection:
            await connection.run_sync(Base.metadata.create_all)
        async with AsyncSession(engine, expire_on_commit=False) as setup:
            device = Device(
                name="edge-atomic",
                ip="10.0.0.18",
                device_type=DeviceTypeEnum.switch,
                current_status="online",
                is_enabled=True,
                offline_consecutive_rounds=2,
            )
            setup.add(device)
            await setup.flush()
            device_id = device.id
            setup.add(
                PingRecord(
                    device_id=device_id,
                    is_alive=False,
                    response_time_ms=None,
                    round_num=1,
                    sent_count=3,
                    received_count=0,
                    packet_loss_percent=100.0,
                    average_rtt_ms=None,
                    is_valid=True,
                    created_at=observed_at,
                )
            )
            await setup.commit()

        async def fake_run_fping_count(*args, **kwargs):
            return {
                "10.0.0.18": ProbeResult(
                    "10.0.0.18",
                    3,
                    0,
                    None,
                    True,
                )
            }

        monkeypatch.setattr(
            "app.services.pinger.run_fping_count",
            fake_run_fping_count,
        )
        async with AsyncSession(engine, expire_on_commit=False) as session:
            changes = await Pinger().run_one_round(session)
            await Alerter().record_transition(changes[0], session)
            await session.rollback()

        async with AsyncSession(engine, expire_on_commit=False) as verification:
            persisted_device = await verification.get(Device, device_id)
            records = list(
                (
                    await verification.execute(
                        select(PingRecord).where(PingRecord.device_id == device_id)
                    )
                )
                .scalars()
                .all()
            )
            assert persisted_device.current_status == "online"
            assert len(records) == 1
            assert await verification.scalar(select(AlertEvent)) is None
            assert await verification.scalar(select(NotificationOutbox)) is None
    finally:
        await engine.dispose()


class RecordingSession:
    """Small session boundary exposing transaction and close ordering."""

    def __init__(self, events: list[str]):
        self.events = events
        self.closed = asyncio.Event()

    async def __aenter__(self):
        self.events.append("session-enter")
        return self

    async def __aexit__(self, exc_type, exc_value, traceback):
        self.events.append("session-exit")
        self.closed.set()

    async def commit(self):
        self.events.append("commit")

    async def rollback(self):
        self.events.append("rollback")


class RecordingPinger:
    def __init__(self, events: list[str], changes=None):
        self.events = events
        self.changes = list(changes or ["transition"])

    async def run_one_round(self, db):
        self.events.append("probe")
        return self.changes


class RecordingAlerter:
    def __init__(self, events: list[str]):
        self.events = events

    async def record_transition(self, change, db):
        self.events.append(f"alert:{change}")


class RecordingDispatcher:
    def __init__(self, events: list[str]):
        self.events = events

    async def dispatch_due(self, db, now):
        assert isinstance(now, datetime)
        self.events.append("dispatch")


async def test_scheduler_persists_transitions_before_dispatching_due_messages():
    """Changing scheduler order cannot expose uncommitted outbox rows to dispatch."""
    events: list[str] = []
    session = RecordingSession(events)
    scheduler = PingScheduler(
        pinger=RecordingPinger(events),
        alerter=RecordingAlerter(events),
        dispatcher=RecordingDispatcher(events),
        session_factory=lambda: session,
    )

    await scheduler._run_cycle()

    assert events == [
        "session-enter",
        "probe",
        "alert:transition",
        "commit",
        "dispatch",
        "commit",
        "session-exit",
    ]


async def test_scheduler_lock_prevents_overlapping_cycles():
    """Even concurrent triggers cannot overlap probe or notification sessions."""
    events: list[str] = []

    class ConcurrencyPinger:
        active = 0
        maximum = 0

        async def run_one_round(self, db):
            self.active += 1
            self.maximum = max(self.maximum, self.active)
            await asyncio.sleep(0.01)
            self.active -= 1
            return []

    pinger = ConcurrencyPinger()
    scheduler = PingScheduler(
        pinger=pinger,
        alerter=RecordingAlerter(events),
        dispatcher=RecordingDispatcher(events),
        session_factory=lambda: RecordingSession(events),
    )

    await asyncio.gather(scheduler._run_cycle(), scheduler._run_cycle())

    assert pinger.maximum == 1


async def test_scheduler_stop_waits_for_active_session_to_close():
    """Cancellation cannot return while a probe session remains open."""
    events: list[str] = []
    session = RecordingSession(events)
    probe_started = asyncio.Event()
    never_complete = asyncio.Event()

    class BlockingPinger:
        async def run_one_round(self, db):
            probe_started.set()
            await never_complete.wait()

    scheduler = PingScheduler(
        pinger=BlockingPinger(),
        alerter=RecordingAlerter(events),
        dispatcher=RecordingDispatcher(events),
        session_factory=lambda: session,
        interval_seconds=1,
    )
    scheduler.start()
    first_task = scheduler._task
    scheduler.start()

    await probe_started.wait()
    await scheduler.stop()

    assert first_task is not None
    assert session.closed.is_set()
    assert scheduler._task is None
    assert events[-2:] == ["rollback", "session-exit"]


async def test_scheduler_leaves_pending_outbox_untouched_when_delivery_disabled():
    """Disabling WeCom preserves queued notifications for a later enablement."""
    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
    session_factory = async_sessionmaker(engine, expire_on_commit=False)

    class EmptyPinger:
        async def run_one_round(self, db):
            return []

    try:
        async with engine.begin() as connection:
            await connection.run_sync(Base.metadata.create_all)
        async with session_factory() as setup:
            setup.add(
                NotificationOutbox(
                    alert_event_id=42,
                    message_content="保留待发送事件",
                )
            )
            await setup.commit()

        runtime_settings = Settings(wecom_notification_enabled=False)
        scheduler = PingScheduler(
            pinger=EmptyPinger(),
            alerter=Alerter(runtime_settings),
            session_factory=session_factory,
            runtime_settings=runtime_settings,
        )
        await scheduler._run_cycle()

        async with session_factory() as verification:
            outbox = await verification.scalar(select(NotificationOutbox))
            assert outbox.status == NotificationStatus.pending
            assert outbox.attempt_count == 0
        await scheduler.stop()
    finally:
        await engine.dispose()