feat: deliver alert events through persistent outbox
This commit is contained in:
@@ -0,0 +1,327 @@
|
||||
"""Persistent notification dispatch and retry behavior tests."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
|
||||
|
||||
from app.config import Settings
|
||||
from app.models import (
|
||||
AlertEvent,
|
||||
AlertTypeEnum,
|
||||
Base,
|
||||
NotificationOutbox,
|
||||
NotificationStatus,
|
||||
)
|
||||
from app.services.notification_dispatcher import (
|
||||
DeliveryResult,
|
||||
MAX_EVENTS_PER_MESSAGE,
|
||||
NotificationDispatcher,
|
||||
WeComClient,
|
||||
)
|
||||
|
||||
|
||||
class FakeWeComClient:
|
||||
"""Return one controlled outcome without any external network access."""
|
||||
|
||||
def __init__(self, *results: DeliveryResult):
|
||||
self._results = list(results)
|
||||
self.contents: list[str] = []
|
||||
|
||||
async def send_text(self, content: str) -> DeliveryResult:
|
||||
self.contents.append(content)
|
||||
return self._results.pop(0)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db_session() -> AsyncSession:
|
||||
"""Provide an isolated outbox database."""
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(Base.metadata.create_all)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
yield session
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_retryable_failure_reschedules(
|
||||
db_session: AsyncSession,
|
||||
):
|
||||
"""A transient failure must remain pending with exponential retry evidence."""
|
||||
frozen_time = datetime(2026, 8, 4, 10, 0)
|
||||
pending_message = NotificationOutbox(
|
||||
alert_event_id=1,
|
||||
message_content="设备离线",
|
||||
)
|
||||
db_session.add(pending_message)
|
||||
await db_session.commit()
|
||||
client = FakeWeComClient(DeliveryResult(False, "timeout", True))
|
||||
|
||||
await NotificationDispatcher(client).dispatch_due(
|
||||
db_session,
|
||||
frozen_time,
|
||||
)
|
||||
|
||||
assert pending_message.status == NotificationStatus.pending
|
||||
assert pending_message.attempt_count == 1
|
||||
assert pending_message.next_attempt_at == datetime(2026, 8, 4, 10, 0, 30)
|
||||
|
||||
|
||||
async def test_success_marks_outbox_and_event_delivered(
|
||||
db_session: AsyncSession,
|
||||
):
|
||||
"""A successful transport result updates both delivery evidence records."""
|
||||
now = datetime(2026, 8, 4, 10, 0)
|
||||
event = AlertEvent(
|
||||
device_id=7,
|
||||
alert_type=AlertTypeEnum.offline,
|
||||
message="full loss",
|
||||
start_at=now,
|
||||
is_resolved=False,
|
||||
)
|
||||
db_session.add(event)
|
||||
await db_session.flush()
|
||||
message = NotificationOutbox(
|
||||
alert_event_id=event.id,
|
||||
message_content="设备 7 离线",
|
||||
)
|
||||
db_session.add(message)
|
||||
await db_session.commit()
|
||||
|
||||
summary = await NotificationDispatcher(
|
||||
FakeWeComClient(DeliveryResult(True)),
|
||||
_enabled_settings(),
|
||||
).dispatch_due(db_session, now)
|
||||
|
||||
assert summary.sent == 1
|
||||
assert message.status == NotificationStatus.sent
|
||||
assert message.sent_at == now
|
||||
assert message.attempt_count == 1
|
||||
assert event.notification_sent is True
|
||||
assert event.notification_attempts == 1
|
||||
assert event.last_notification_error is None
|
||||
|
||||
|
||||
async def test_non_retryable_or_exhausted_failure_is_terminal(
|
||||
db_session: AsyncSession,
|
||||
):
|
||||
"""The configured attempt ceiling prevents an infinite retry loop."""
|
||||
now = datetime(2026, 8, 4, 10, 0)
|
||||
message = NotificationOutbox(
|
||||
alert_event_id=99,
|
||||
message_content="bad recipient",
|
||||
attempt_count=1,
|
||||
)
|
||||
db_session.add(message)
|
||||
await db_session.commit()
|
||||
runtime_settings = _enabled_settings(wecom_notification_max_attempts=2)
|
||||
|
||||
summary = await NotificationDispatcher(
|
||||
FakeWeComClient(DeliveryResult(False, "HTTP 503", True)),
|
||||
runtime_settings,
|
||||
).dispatch_due(db_session, now)
|
||||
|
||||
assert summary.failed == 1
|
||||
assert message.status == NotificationStatus.failed
|
||||
assert message.attempt_count == 2
|
||||
assert message.next_attempt_at is None
|
||||
|
||||
|
||||
async def test_only_due_messages_are_dispatched_in_fixed_safe_batches(
|
||||
db_session: AsyncSession,
|
||||
):
|
||||
"""Large bursts are split while future retries remain untouched."""
|
||||
now = datetime(2026, 8, 4, 10, 0)
|
||||
due = [
|
||||
NotificationOutbox(alert_event_id=index, message_content=f"事件 {index}")
|
||||
for index in range(MAX_EVENTS_PER_MESSAGE + 1)
|
||||
]
|
||||
future = NotificationOutbox(
|
||||
alert_event_id=100,
|
||||
message_content="未来重试",
|
||||
next_attempt_at=now + timedelta(minutes=1),
|
||||
)
|
||||
db_session.add_all([*due, future])
|
||||
await db_session.commit()
|
||||
client = FakeWeComClient(DeliveryResult(True), DeliveryResult(True))
|
||||
|
||||
summary = await NotificationDispatcher(
|
||||
client,
|
||||
_enabled_settings(),
|
||||
).dispatch_due(db_session, now)
|
||||
|
||||
assert summary.sent == MAX_EVENTS_PER_MESSAGE + 1
|
||||
assert len(client.contents) == 2
|
||||
assert future.status == NotificationStatus.pending
|
||||
assert future.attempt_count == 0
|
||||
|
||||
|
||||
def _enabled_settings(**overrides) -> Settings:
|
||||
values = {
|
||||
"wecom_notification_enabled": True,
|
||||
"WECOM_CORP_ID": "test-corp",
|
||||
"WECOM_AGENT_ID": 1000001,
|
||||
"WECOM_APP_SECRET": "test-secret",
|
||||
}
|
||||
values.update(overrides)
|
||||
return Settings(**values)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("to_party", "expected_recipient", "unexpected_key"),
|
||||
[("", ("touser", "@all"), "toparty"), ("2|3", ("toparty", "2|3"), "touser")],
|
||||
)
|
||||
async def test_wecom_uses_application_scope_unless_department_is_explicit(
|
||||
to_party,
|
||||
expected_recipient,
|
||||
unexpected_key,
|
||||
):
|
||||
"""Recipient payloads preserve the app visibility boundary by default."""
|
||||
payloads: list[dict] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/gettoken"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"errcode": 0, "access_token": "token-a", "expires_in": 7200},
|
||||
)
|
||||
payloads.append(json.loads(request.content))
|
||||
return httpx.Response(200, json={"errcode": 0, "errmsg": "ok"})
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
||||
client = WeComClient(
|
||||
_enabled_settings(WECOM_TO_PARTY=to_party),
|
||||
http_client=http,
|
||||
)
|
||||
result = await client.send_text("controlled test message")
|
||||
|
||||
key, value = expected_recipient
|
||||
assert result.success is True
|
||||
assert payloads[0][key] == value
|
||||
assert unexpected_key not in payloads[0]
|
||||
assert payloads[0]["agentid"] == 1000001
|
||||
|
||||
|
||||
async def test_wecom_caches_token_until_expiry_margin():
|
||||
"""Repeated messages do not request a new token before its safe expiry."""
|
||||
calls = {"token": 0, "message": 0}
|
||||
current_time = [datetime(2026, 8, 4, 10, 0)]
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/gettoken"):
|
||||
calls["token"] += 1
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"errcode": 0, "access_token": "token-a", "expires_in": 120},
|
||||
)
|
||||
calls["message"] += 1
|
||||
return httpx.Response(200, json={"errcode": 0, "errmsg": "ok"})
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
||||
client = WeComClient(
|
||||
_enabled_settings(),
|
||||
http_client=http,
|
||||
now=lambda: current_time[0],
|
||||
)
|
||||
await client.send_text("first")
|
||||
current_time[0] += timedelta(seconds=30)
|
||||
await client.send_text("second")
|
||||
|
||||
assert calls == {"token": 1, "message": 2}
|
||||
|
||||
|
||||
async def test_wecom_refreshes_token_after_safety_margin():
|
||||
"""A token inside the safety margin cannot be reused for a new message."""
|
||||
token_calls = 0
|
||||
current_time = [datetime(2026, 8, 4, 10, 0)]
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
nonlocal token_calls
|
||||
if request.url.path.endswith("/gettoken"):
|
||||
token_calls += 1
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"errcode": 0,
|
||||
"access_token": f"token-{token_calls}",
|
||||
"expires_in": 120,
|
||||
},
|
||||
)
|
||||
return httpx.Response(200, json={"errcode": 0, "errmsg": "ok"})
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
||||
client = WeComClient(
|
||||
_enabled_settings(),
|
||||
http_client=http,
|
||||
now=lambda: current_time[0],
|
||||
)
|
||||
await client.send_text("first")
|
||||
current_time[0] += timedelta(seconds=61)
|
||||
await client.send_text("second")
|
||||
|
||||
assert token_calls == 2
|
||||
|
||||
|
||||
async def test_wecom_rejects_non_object_json_without_raising():
|
||||
"""Malformed external response shapes return sanitized retry evidence."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=["unexpected", "shape"])
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
||||
result = await WeComClient(
|
||||
_enabled_settings(),
|
||||
http_client=http,
|
||||
).send_text("controlled")
|
||||
|
||||
assert result == DeliveryResult(False, "invalid token response", True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", [429, 500, 503])
|
||||
async def test_wecom_marks_rate_limit_and_server_errors_retryable(status_code):
|
||||
"""Transient HTTP classes reach the outbox retry path without response leakage."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/gettoken"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"errcode": 0, "access_token": "token-a", "expires_in": 7200},
|
||||
)
|
||||
return httpx.Response(status_code, text="sensitive-response-body")
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
||||
result = await WeComClient(
|
||||
_enabled_settings(),
|
||||
http_client=http,
|
||||
).send_text("sensitive-message-content")
|
||||
|
||||
assert result.success is False
|
||||
assert result.retryable is True
|
||||
assert result.error == f"HTTP {status_code}"
|
||||
|
||||
|
||||
async def test_wecom_transport_failure_is_sanitized_and_retryable(caplog):
|
||||
"""Logs and returned evidence never expose secrets, tokens, bodies, or content."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/gettoken"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"errcode": 0, "access_token": "token-a", "expires_in": 7200},
|
||||
)
|
||||
raise httpx.ConnectError("response-sensitive", request=request)
|
||||
|
||||
caplog.set_level(logging.WARNING)
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
||||
result = await WeComClient(
|
||||
_enabled_settings(WECOM_APP_SECRET="secret-sensitive"),
|
||||
http_client=http,
|
||||
).send_text("message-sensitive")
|
||||
|
||||
assert result == DeliveryResult(False, "transport error", True)
|
||||
assert "secret-sensitive" not in caplog.text
|
||||
assert "token-a" not in caplog.text
|
||||
assert "response-sensitive" not in caplog.text
|
||||
assert "message-sensitive" not in caplog.text
|
||||
Reference in New Issue
Block a user