328 lines
11 KiB
Python
328 lines
11 KiB
Python
"""Persistent notification dispatch and retry behavior tests."""
|
|
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
|
|
import httpx
|
|
import pytest
|
|
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
|
|
|
|
from app.config import Settings
|
|
from app.models import (
|
|
AlertEvent,
|
|
AlertTypeEnum,
|
|
Base,
|
|
NotificationOutbox,
|
|
NotificationStatus,
|
|
)
|
|
from app.services.notification_dispatcher import (
|
|
DeliveryResult,
|
|
MAX_EVENTS_PER_MESSAGE,
|
|
NotificationDispatcher,
|
|
WeComClient,
|
|
)
|
|
|
|
|
|
class FakeWeComClient:
|
|
"""Return one controlled outcome without any external network access."""
|
|
|
|
def __init__(self, *results: DeliveryResult):
|
|
self._results = list(results)
|
|
self.contents: list[str] = []
|
|
|
|
async def send_text(self, content: str) -> DeliveryResult:
|
|
self.contents.append(content)
|
|
return self._results.pop(0)
|
|
|
|
|
|
@pytest.fixture
|
|
async def db_session() -> AsyncSession:
|
|
"""Provide an isolated outbox database."""
|
|
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
|
try:
|
|
async with engine.begin() as connection:
|
|
await connection.run_sync(Base.metadata.create_all)
|
|
async with AsyncSession(engine, expire_on_commit=False) as session:
|
|
yield session
|
|
finally:
|
|
await engine.dispose()
|
|
|
|
|
|
async def test_retryable_failure_reschedules(
|
|
db_session: AsyncSession,
|
|
):
|
|
"""A transient failure must remain pending with exponential retry evidence."""
|
|
frozen_time = datetime(2026, 8, 4, 10, 0)
|
|
pending_message = NotificationOutbox(
|
|
alert_event_id=1,
|
|
message_content="设备离线",
|
|
)
|
|
db_session.add(pending_message)
|
|
await db_session.commit()
|
|
client = FakeWeComClient(DeliveryResult(False, "timeout", True))
|
|
|
|
await NotificationDispatcher(client).dispatch_due(
|
|
db_session,
|
|
frozen_time,
|
|
)
|
|
|
|
assert pending_message.status == NotificationStatus.pending
|
|
assert pending_message.attempt_count == 1
|
|
assert pending_message.next_attempt_at == datetime(2026, 8, 4, 10, 0, 30)
|
|
|
|
|
|
async def test_success_marks_outbox_and_event_delivered(
|
|
db_session: AsyncSession,
|
|
):
|
|
"""A successful transport result updates both delivery evidence records."""
|
|
now = datetime(2026, 8, 4, 10, 0)
|
|
event = AlertEvent(
|
|
device_id=7,
|
|
alert_type=AlertTypeEnum.offline,
|
|
message="full loss",
|
|
start_at=now,
|
|
is_resolved=False,
|
|
)
|
|
db_session.add(event)
|
|
await db_session.flush()
|
|
message = NotificationOutbox(
|
|
alert_event_id=event.id,
|
|
message_content="设备 7 离线",
|
|
)
|
|
db_session.add(message)
|
|
await db_session.commit()
|
|
|
|
summary = await NotificationDispatcher(
|
|
FakeWeComClient(DeliveryResult(True)),
|
|
_enabled_settings(),
|
|
).dispatch_due(db_session, now)
|
|
|
|
assert summary.sent == 1
|
|
assert message.status == NotificationStatus.sent
|
|
assert message.sent_at == now
|
|
assert message.attempt_count == 1
|
|
assert event.notification_sent is True
|
|
assert event.notification_attempts == 1
|
|
assert event.last_notification_error is None
|
|
|
|
|
|
async def test_non_retryable_or_exhausted_failure_is_terminal(
|
|
db_session: AsyncSession,
|
|
):
|
|
"""The configured attempt ceiling prevents an infinite retry loop."""
|
|
now = datetime(2026, 8, 4, 10, 0)
|
|
message = NotificationOutbox(
|
|
alert_event_id=99,
|
|
message_content="bad recipient",
|
|
attempt_count=1,
|
|
)
|
|
db_session.add(message)
|
|
await db_session.commit()
|
|
runtime_settings = _enabled_settings(wecom_notification_max_attempts=2)
|
|
|
|
summary = await NotificationDispatcher(
|
|
FakeWeComClient(DeliveryResult(False, "HTTP 503", True)),
|
|
runtime_settings,
|
|
).dispatch_due(db_session, now)
|
|
|
|
assert summary.failed == 1
|
|
assert message.status == NotificationStatus.failed
|
|
assert message.attempt_count == 2
|
|
assert message.next_attempt_at is None
|
|
|
|
|
|
async def test_only_due_messages_are_dispatched_in_fixed_safe_batches(
|
|
db_session: AsyncSession,
|
|
):
|
|
"""Large bursts are split while future retries remain untouched."""
|
|
now = datetime(2026, 8, 4, 10, 0)
|
|
due = [
|
|
NotificationOutbox(alert_event_id=index, message_content=f"事件 {index}")
|
|
for index in range(MAX_EVENTS_PER_MESSAGE + 1)
|
|
]
|
|
future = NotificationOutbox(
|
|
alert_event_id=100,
|
|
message_content="未来重试",
|
|
next_attempt_at=now + timedelta(minutes=1),
|
|
)
|
|
db_session.add_all([*due, future])
|
|
await db_session.commit()
|
|
client = FakeWeComClient(DeliveryResult(True), DeliveryResult(True))
|
|
|
|
summary = await NotificationDispatcher(
|
|
client,
|
|
_enabled_settings(),
|
|
).dispatch_due(db_session, now)
|
|
|
|
assert summary.sent == MAX_EVENTS_PER_MESSAGE + 1
|
|
assert len(client.contents) == 2
|
|
assert future.status == NotificationStatus.pending
|
|
assert future.attempt_count == 0
|
|
|
|
|
|
def _enabled_settings(**overrides) -> Settings:
|
|
values = {
|
|
"wecom_notification_enabled": True,
|
|
"WECOM_CORP_ID": "test-corp",
|
|
"WECOM_AGENT_ID": 1000001,
|
|
"WECOM_APP_SECRET": "test-secret",
|
|
}
|
|
values.update(overrides)
|
|
return Settings(**values)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("to_party", "expected_recipient", "unexpected_key"),
|
|
[("", ("touser", "@all"), "toparty"), ("2|3", ("toparty", "2|3"), "touser")],
|
|
)
|
|
async def test_wecom_uses_application_scope_unless_department_is_explicit(
|
|
to_party,
|
|
expected_recipient,
|
|
unexpected_key,
|
|
):
|
|
"""Recipient payloads preserve the app visibility boundary by default."""
|
|
payloads: list[dict] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path.endswith("/gettoken"):
|
|
return httpx.Response(
|
|
200,
|
|
json={"errcode": 0, "access_token": "token-a", "expires_in": 7200},
|
|
)
|
|
payloads.append(json.loads(request.content))
|
|
return httpx.Response(200, json={"errcode": 0, "errmsg": "ok"})
|
|
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
|
client = WeComClient(
|
|
_enabled_settings(WECOM_TO_PARTY=to_party),
|
|
http_client=http,
|
|
)
|
|
result = await client.send_text("controlled test message")
|
|
|
|
key, value = expected_recipient
|
|
assert result.success is True
|
|
assert payloads[0][key] == value
|
|
assert unexpected_key not in payloads[0]
|
|
assert payloads[0]["agentid"] == 1000001
|
|
|
|
|
|
async def test_wecom_caches_token_until_expiry_margin():
|
|
"""Repeated messages do not request a new token before its safe expiry."""
|
|
calls = {"token": 0, "message": 0}
|
|
current_time = [datetime(2026, 8, 4, 10, 0)]
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path.endswith("/gettoken"):
|
|
calls["token"] += 1
|
|
return httpx.Response(
|
|
200,
|
|
json={"errcode": 0, "access_token": "token-a", "expires_in": 120},
|
|
)
|
|
calls["message"] += 1
|
|
return httpx.Response(200, json={"errcode": 0, "errmsg": "ok"})
|
|
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
|
client = WeComClient(
|
|
_enabled_settings(),
|
|
http_client=http,
|
|
now=lambda: current_time[0],
|
|
)
|
|
await client.send_text("first")
|
|
current_time[0] += timedelta(seconds=30)
|
|
await client.send_text("second")
|
|
|
|
assert calls == {"token": 1, "message": 2}
|
|
|
|
|
|
async def test_wecom_refreshes_token_after_safety_margin():
|
|
"""A token inside the safety margin cannot be reused for a new message."""
|
|
token_calls = 0
|
|
current_time = [datetime(2026, 8, 4, 10, 0)]
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
nonlocal token_calls
|
|
if request.url.path.endswith("/gettoken"):
|
|
token_calls += 1
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"errcode": 0,
|
|
"access_token": f"token-{token_calls}",
|
|
"expires_in": 120,
|
|
},
|
|
)
|
|
return httpx.Response(200, json={"errcode": 0, "errmsg": "ok"})
|
|
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
|
client = WeComClient(
|
|
_enabled_settings(),
|
|
http_client=http,
|
|
now=lambda: current_time[0],
|
|
)
|
|
await client.send_text("first")
|
|
current_time[0] += timedelta(seconds=61)
|
|
await client.send_text("second")
|
|
|
|
assert token_calls == 2
|
|
|
|
|
|
async def test_wecom_rejects_non_object_json_without_raising():
|
|
"""Malformed external response shapes return sanitized retry evidence."""
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, json=["unexpected", "shape"])
|
|
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
|
result = await WeComClient(
|
|
_enabled_settings(),
|
|
http_client=http,
|
|
).send_text("controlled")
|
|
|
|
assert result == DeliveryResult(False, "invalid token response", True)
|
|
|
|
|
|
@pytest.mark.parametrize("status_code", [429, 500, 503])
|
|
async def test_wecom_marks_rate_limit_and_server_errors_retryable(status_code):
|
|
"""Transient HTTP classes reach the outbox retry path without response leakage."""
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path.endswith("/gettoken"):
|
|
return httpx.Response(
|
|
200,
|
|
json={"errcode": 0, "access_token": "token-a", "expires_in": 7200},
|
|
)
|
|
return httpx.Response(status_code, text="sensitive-response-body")
|
|
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
|
result = await WeComClient(
|
|
_enabled_settings(),
|
|
http_client=http,
|
|
).send_text("sensitive-message-content")
|
|
|
|
assert result.success is False
|
|
assert result.retryable is True
|
|
assert result.error == f"HTTP {status_code}"
|
|
|
|
|
|
async def test_wecom_transport_failure_is_sanitized_and_retryable(caplog):
|
|
"""Logs and returned evidence never expose secrets, tokens, bodies, or content."""
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path.endswith("/gettoken"):
|
|
return httpx.Response(
|
|
200,
|
|
json={"errcode": 0, "access_token": "token-a", "expires_in": 7200},
|
|
)
|
|
raise httpx.ConnectError("response-sensitive", request=request)
|
|
|
|
caplog.set_level(logging.WARNING)
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
|
|
result = await WeComClient(
|
|
_enabled_settings(WECOM_APP_SECRET="secret-sensitive"),
|
|
http_client=http,
|
|
).send_text("message-sensitive")
|
|
|
|
assert result == DeliveryResult(False, "transport error", True)
|
|
assert "secret-sensitive" not in caplog.text
|
|
assert "token-a" not in caplog.text
|
|
assert "response-sensitive" not in caplog.text
|
|
assert "message-sensitive" not in caplog.text
|