2a392b87f8
- Add rate_limit.py middleware with per-client throttling and cleanup task - Add pool_monitor.py for database connection pool health monitoring - Add custom exceptions module (GameEngineError, DatabaseError, etc.) - Add config settings for eviction intervals, session timeouts, memory limits - Add unit tests for rate limiting and pool monitoring 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
389 lines
13 KiB
Python
389 lines
13 KiB
Python
"""
|
|
Tests for database connection pool monitoring.
|
|
|
|
Verifies that the PoolMonitor correctly:
|
|
- Reports pool statistics (checked in/out, overflow)
|
|
- Classifies health status (healthy/warning/critical)
|
|
- Tracks history of stats over time
|
|
- Logs warnings when usage exceeds threshold
|
|
|
|
Author: Claude
|
|
Date: 2025-11-27
|
|
"""
|
|
|
|
import pytest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from app.monitoring.pool_monitor import PoolMonitor, PoolStats
|
|
|
|
|
|
# ============================================================================
|
|
# TEST FIXTURES
|
|
# ============================================================================
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_engine():
|
|
"""
|
|
Create a mock SQLAlchemy engine with pool stats.
|
|
|
|
Default configuration: 15 available, 5 in use, no overflow.
|
|
"""
|
|
engine = MagicMock()
|
|
pool = MagicMock()
|
|
pool.checkedin.return_value = 15
|
|
pool.checkedout.return_value = 5
|
|
pool.overflow.return_value = 0
|
|
engine.pool = pool
|
|
return engine
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_settings():
|
|
"""Mock settings with pool configuration."""
|
|
settings = MagicMock()
|
|
settings.db_pool_size = 20
|
|
settings.db_max_overflow = 10
|
|
return settings
|
|
|
|
|
|
# ============================================================================
|
|
# POOL STATS TESTS
|
|
# ============================================================================
|
|
|
|
|
|
class TestPoolMonitorStats:
|
|
"""Tests for PoolMonitor.get_stats()."""
|
|
|
|
def test_get_stats_returns_pool_stats(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify get_stats() returns PoolStats with correct values.
|
|
|
|
The returned PoolStats should reflect the current pool state
|
|
from the engine's pool attribute.
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
stats = monitor.get_stats()
|
|
|
|
assert isinstance(stats, PoolStats)
|
|
assert stats.checkedout == 5
|
|
assert stats.checkedin == 15
|
|
assert stats.overflow == 0
|
|
assert stats.pool_size == 20
|
|
assert stats.max_overflow == 10
|
|
assert stats.total_capacity == 30
|
|
|
|
def test_get_stats_calculates_usage_percent(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify usage percentage is calculated correctly.
|
|
|
|
Usage = checkedout / total_capacity
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
stats = monitor.get_stats()
|
|
|
|
# 5 out of 30 = 16.67%
|
|
assert stats.usage_percent == pytest.approx(5 / 30, rel=0.01)
|
|
|
|
def test_get_stats_handles_zero_capacity(self, mock_engine):
|
|
"""
|
|
Verify get_stats() handles zero capacity without division error.
|
|
|
|
Edge case: if both pool_size and max_overflow are 0.
|
|
"""
|
|
settings = MagicMock()
|
|
settings.db_pool_size = 0
|
|
settings.db_max_overflow = 0
|
|
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
stats = monitor.get_stats()
|
|
|
|
assert stats.usage_percent == 0
|
|
assert stats.total_capacity == 0
|
|
|
|
def test_get_stats_records_history(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify get_stats() records each call in history.
|
|
|
|
History is used for monitoring dashboards and alerts.
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
|
|
# Make multiple calls
|
|
for _ in range(5):
|
|
monitor.get_stats()
|
|
|
|
history = monitor.get_history(limit=10)
|
|
assert len(history) == 5
|
|
|
|
def test_get_stats_limits_history_size(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify history is limited to prevent memory growth.
|
|
|
|
Older entries should be removed when max_history is exceeded.
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine, max_history=5)
|
|
|
|
# Make more calls than max_history
|
|
for _ in range(10):
|
|
monitor.get_stats()
|
|
|
|
history = monitor.get_history(limit=10)
|
|
assert len(history) == 5
|
|
|
|
|
|
# ============================================================================
|
|
# HEALTH STATUS TESTS
|
|
# ============================================================================
|
|
|
|
|
|
class TestPoolMonitorHealth:
|
|
"""Tests for PoolMonitor.get_health_status()."""
|
|
|
|
def test_health_status_healthy(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify health status is 'healthy' when usage < 75%.
|
|
|
|
At 5/30 (16.7%) usage, status should be healthy.
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
health = monitor.get_health_status()
|
|
|
|
assert health["status"] == "healthy"
|
|
assert health["in_use"] == 5
|
|
assert health["available"] == 15
|
|
|
|
def test_health_status_warning(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify health status is 'warning' when usage is 75-90%.
|
|
|
|
At 24/30 (80%) usage, status should be warning.
|
|
"""
|
|
mock_engine.pool.checkedout.return_value = 24
|
|
mock_engine.pool.checkedin.return_value = 6
|
|
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
health = monitor.get_health_status()
|
|
|
|
assert health["status"] == "warning"
|
|
assert health["usage_percent"] == pytest.approx(80, rel=1)
|
|
|
|
def test_health_status_critical(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify health status is 'critical' when usage >= 90%.
|
|
|
|
At 28/30 (93%) usage, status should be critical.
|
|
"""
|
|
mock_engine.pool.checkedout.return_value = 28
|
|
mock_engine.pool.checkedin.return_value = 2
|
|
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
health = monitor.get_health_status()
|
|
|
|
assert health["status"] == "critical"
|
|
assert health["usage_percent"] >= 90
|
|
|
|
def test_health_status_includes_timestamp(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify health status includes ISO timestamp.
|
|
|
|
Timestamp is used for monitoring and alerting correlation.
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
health = monitor.get_health_status()
|
|
|
|
assert "timestamp" in health
|
|
assert isinstance(health["timestamp"], str)
|
|
assert "T" in health["timestamp"] # ISO format
|
|
|
|
def test_health_status_includes_all_fields(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify health status includes all required fields.
|
|
|
|
All fields are needed for complete monitoring visibility.
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
health = monitor.get_health_status()
|
|
|
|
required_fields = [
|
|
"status",
|
|
"pool_size",
|
|
"max_overflow",
|
|
"available",
|
|
"in_use",
|
|
"overflow_active",
|
|
"total_capacity",
|
|
"usage_percent",
|
|
"timestamp",
|
|
]
|
|
|
|
for field in required_fields:
|
|
assert field in health, f"Missing field: {field}"
|
|
|
|
|
|
# ============================================================================
|
|
# HISTORY TESTS
|
|
# ============================================================================
|
|
|
|
|
|
class TestPoolMonitorHistory:
|
|
"""Tests for PoolMonitor.get_history()."""
|
|
|
|
def test_get_history_returns_recent_stats(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify get_history() returns recent stat snapshots.
|
|
|
|
History should include checkedout, usage_percent, and timestamp.
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
|
|
# Generate some history
|
|
for _ in range(3):
|
|
monitor.get_stats()
|
|
|
|
history = monitor.get_history(limit=5)
|
|
|
|
assert len(history) == 3
|
|
for entry in history:
|
|
assert "checkedout" in entry
|
|
assert "usage_percent" in entry
|
|
assert "timestamp" in entry
|
|
|
|
def test_get_history_respects_limit(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify get_history() respects the limit parameter.
|
|
|
|
Should return at most 'limit' entries.
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
|
|
for _ in range(10):
|
|
monitor.get_stats()
|
|
|
|
history = monitor.get_history(limit=3)
|
|
assert len(history) == 3
|
|
|
|
def test_get_history_returns_empty_when_no_stats(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify get_history() returns empty list when no stats collected.
|
|
|
|
Fresh monitor should have no history.
|
|
"""
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
history = monitor.get_history()
|
|
|
|
assert history == []
|
|
|
|
|
|
# ============================================================================
|
|
# ALERT THRESHOLD TESTS
|
|
# ============================================================================
|
|
|
|
|
|
class TestPoolMonitorAlerts:
|
|
"""Tests for alert threshold behavior."""
|
|
|
|
def test_logs_warning_when_threshold_exceeded(self, mock_engine, mock_settings, caplog):
|
|
"""
|
|
Verify warning is logged when usage exceeds alert threshold.
|
|
|
|
Default threshold is 80%. At 85% usage, warning should be logged.
|
|
"""
|
|
mock_engine.pool.checkedout.return_value = 26 # 86.7%
|
|
mock_engine.pool.checkedin.return_value = 4
|
|
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine, alert_threshold=0.8)
|
|
|
|
import logging
|
|
with caplog.at_level(logging.WARNING):
|
|
monitor.get_stats()
|
|
|
|
assert "Connection pool usage high" in caplog.text
|
|
|
|
def test_logs_info_when_overflow_active(self, mock_engine, mock_settings, caplog):
|
|
"""
|
|
Verify info is logged when overflow connections are in use.
|
|
|
|
Overflow indicates pool is at capacity and using extra connections.
|
|
"""
|
|
mock_engine.pool.overflow.return_value = 3
|
|
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine)
|
|
|
|
import logging
|
|
with caplog.at_level(logging.INFO):
|
|
monitor.get_stats()
|
|
|
|
assert "Pool overflow active" in caplog.text
|
|
|
|
def test_no_warning_when_below_threshold(self, mock_engine, mock_settings, caplog):
|
|
"""
|
|
Verify no warning when usage is below threshold.
|
|
|
|
At 50% usage with 80% threshold, no warning should be logged.
|
|
"""
|
|
mock_engine.pool.checkedout.return_value = 10 # 33%
|
|
mock_engine.pool.checkedin.return_value = 20
|
|
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = PoolMonitor(mock_engine, alert_threshold=0.8)
|
|
|
|
import logging
|
|
with caplog.at_level(logging.WARNING):
|
|
monitor.get_stats()
|
|
|
|
assert "Connection pool usage high" not in caplog.text
|
|
|
|
|
|
# ============================================================================
|
|
# INIT FUNCTION TESTS
|
|
# ============================================================================
|
|
|
|
|
|
class TestInitPoolMonitor:
|
|
"""Tests for init_pool_monitor function."""
|
|
|
|
def test_init_pool_monitor_creates_global_instance(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify init_pool_monitor() creates global instance.
|
|
|
|
The global instance is used by health endpoints.
|
|
"""
|
|
from app.monitoring.pool_monitor import init_pool_monitor, pool_monitor
|
|
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = init_pool_monitor(mock_engine)
|
|
|
|
assert monitor is not None
|
|
assert isinstance(monitor, PoolMonitor)
|
|
|
|
def test_init_pool_monitor_returns_monitor(self, mock_engine, mock_settings):
|
|
"""
|
|
Verify init_pool_monitor() returns the created monitor.
|
|
|
|
Return value allows caller to use the monitor directly.
|
|
"""
|
|
from app.monitoring.pool_monitor import init_pool_monitor
|
|
|
|
with patch("app.monitoring.pool_monitor.get_settings", return_value=mock_settings):
|
|
monitor = init_pool_monitor(mock_engine)
|
|
|
|
# Should be able to use the returned monitor
|
|
stats = monitor.get_stats()
|
|
assert stats is not None
|