18: tests, fixes
This commit is contained in:
Dobromir Popov
485
COBY/tests/test_e2e_dashboard.py
Normal file
485
COBY/tests/test_e2e_dashboard.py
Normal file
@ -0,0 +1,485 @@
|
||||
"""
|
||||
End-to-end tests for web dashboard functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import Mock, AsyncMock, patch
|
||||
from typing import Dict, Any, List
|
||||
|
||||
import aiohttp
|
||||
from aiohttp import web, WSMsgType
|
||||
from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop
|
||||
|
||||
from ..api.rest_api import create_app
|
||||
from ..api.websocket_server import WebSocketServer
|
||||
from ..models.core import OrderBookSnapshot, TradeEvent, PriceLevel
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class TestDashboardAPI(AioHTTPTestCase):
|
||||
"""Test dashboard REST API endpoints"""
|
||||
|
||||
async def get_application(self):
|
||||
"""Create test application"""
|
||||
return create_app()
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_health_endpoint(self):
|
||||
"""Test health check endpoint"""
|
||||
resp = await self.client.request("GET", "/health")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
data = await resp.json()
|
||||
self.assertIn('status', data)
|
||||
self.assertIn('timestamp', data)
|
||||
self.assertEqual(data['status'], 'healthy')
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_metrics_endpoint(self):
|
||||
"""Test metrics endpoint"""
|
||||
resp = await self.client.request("GET", "/metrics")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
# Should return Prometheus format
|
||||
text = await resp.text()
|
||||
self.assertIn('# TYPE', text)
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_orderbook_endpoint(self):
|
||||
"""Test order book data endpoint"""
|
||||
# Mock data
|
||||
with patch('COBY.caching.redis_manager.redis_manager') as mock_redis:
|
||||
mock_redis.get.return_value = {
|
||||
'symbol': 'BTCUSDT',
|
||||
'exchange': 'binance',
|
||||
'bids': [{'price': 50000.0, 'size': 1.0}],
|
||||
'asks': [{'price': 50010.0, 'size': 1.0}]
|
||||
}
|
||||
|
||||
resp = await self.client.request("GET", "/api/orderbook/BTCUSDT")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
data = await resp.json()
|
||||
self.assertIn('symbol', data)
|
||||
self.assertEqual(data['symbol'], 'BTCUSDT')
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_heatmap_endpoint(self):
|
||||
"""Test heatmap data endpoint"""
|
||||
with patch('COBY.caching.redis_manager.redis_manager') as mock_redis:
|
||||
mock_redis.get.return_value = {
|
||||
'symbol': 'BTCUSDT',
|
||||
'bucket_size': 1.0,
|
||||
'data': [
|
||||
{'price': 50000.0, 'volume': 10.0, 'intensity': 0.8, 'side': 'bid'}
|
||||
]
|
||||
}
|
||||
|
||||
resp = await self.client.request("GET", "/api/heatmap/BTCUSDT")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
data = await resp.json()
|
||||
self.assertIn('symbol', data)
|
||||
self.assertIn('data', data)
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_exchanges_status_endpoint(self):
|
||||
"""Test exchanges status endpoint"""
|
||||
with patch('COBY.connectors.connection_manager.connection_manager') as mock_manager:
|
||||
mock_manager.get_all_statuses.return_value = {
|
||||
'binance': 'connected',
|
||||
'coinbase': 'connected',
|
||||
'kraken': 'disconnected'
|
||||
}
|
||||
|
||||
resp = await self.client.request("GET", "/api/exchanges/status")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
data = await resp.json()
|
||||
self.assertIn('binance', data)
|
||||
self.assertIn('coinbase', data)
|
||||
self.assertIn('kraken', data)
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_performance_metrics_endpoint(self):
|
||||
"""Test performance metrics endpoint"""
|
||||
with patch('COBY.monitoring.performance_monitor.get_performance_monitor') as mock_monitor:
|
||||
mock_monitor.return_value.get_performance_dashboard_data.return_value = {
|
||||
'timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'system_metrics': {
|
||||
'cpu_usage': 45.2,
|
||||
'memory_usage': 67.8,
|
||||
'active_connections': 150
|
||||
},
|
||||
'performance_summary': {
|
||||
'throughput': 1250.5,
|
||||
'error_rate': 0.1,
|
||||
'avg_latency': 12.3
|
||||
}
|
||||
}
|
||||
|
||||
resp = await self.client.request("GET", "/api/performance")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
data = await resp.json()
|
||||
self.assertIn('system_metrics', data)
|
||||
self.assertIn('performance_summary', data)
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_static_files_served(self):
|
||||
"""Test that static files are served correctly"""
|
||||
# Test dashboard index
|
||||
resp = await self.client.request("GET", "/")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
content_type = resp.headers.get('content-type', '')
|
||||
self.assertIn('text/html', content_type)
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_cors_headers(self):
|
||||
"""Test CORS headers are present"""
|
||||
resp = await self.client.request("OPTIONS", "/api/health")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
# Check CORS headers
|
||||
self.assertIn('Access-Control-Allow-Origin', resp.headers)
|
||||
self.assertIn('Access-Control-Allow-Methods', resp.headers)
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_rate_limiting(self):
|
||||
"""Test API rate limiting"""
|
||||
# Make many requests quickly
|
||||
responses = []
|
||||
for i in range(150): # Exceed rate limit
|
||||
resp = await self.client.request("GET", "/api/health")
|
||||
responses.append(resp.status)
|
||||
|
||||
# Should have some rate limited responses
|
||||
rate_limited = [status for status in responses if status == 429]
|
||||
self.assertGreater(len(rate_limited), 0, "Rate limiting not working")
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_error_handling(self):
|
||||
"""Test API error handling"""
|
||||
# Test invalid symbol
|
||||
resp = await self.client.request("GET", "/api/orderbook/INVALID")
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
data = await resp.json()
|
||||
self.assertIn('error', data)
|
||||
|
||||
@unittest_run_loop
|
||||
async def test_api_documentation(self):
|
||||
"""Test API documentation endpoints"""
|
||||
# Test OpenAPI docs
|
||||
resp = await self.client.request("GET", "/docs")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
# Test ReDoc
|
||||
resp = await self.client.request("GET", "/redoc")
|
||||
self.assertEqual(resp.status, 200)
|
||||
|
||||
|
||||
class TestWebSocketFunctionality:
|
||||
"""Test WebSocket functionality"""
|
||||
|
||||
@pytest.fixture
|
||||
async def websocket_server(self):
|
||||
"""Create WebSocket server for testing"""
|
||||
server = WebSocketServer(host='localhost', port=8081)
|
||||
await server.start()
|
||||
yield server
|
||||
await server.stop()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_connection(self, websocket_server):
|
||||
"""Test WebSocket connection establishment"""
|
||||
session = aiohttp.ClientSession()
|
||||
|
||||
try:
|
||||
async with session.ws_connect('ws://localhost:8081/ws/dashboard') as ws:
|
||||
# Connection should be established
|
||||
self.assertEqual(ws.closed, False)
|
||||
|
||||
# Send ping
|
||||
await ws.ping()
|
||||
|
||||
# Should receive pong
|
||||
msg = await ws.receive()
|
||||
self.assertEqual(msg.type, WSMsgType.PONG)
|
||||
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_data_streaming(self, websocket_server):
|
||||
"""Test real-time data streaming via WebSocket"""
|
||||
session = aiohttp.ClientSession()
|
||||
|
||||
try:
|
||||
async with session.ws_connect('ws://localhost:8081/ws/dashboard') as ws:
|
||||
# Subscribe to updates
|
||||
subscribe_msg = {
|
||||
'type': 'subscribe',
|
||||
'channels': ['orderbook', 'trades', 'performance']
|
||||
}
|
||||
await ws.send_str(json.dumps(subscribe_msg))
|
||||
|
||||
# Should receive subscription confirmation
|
||||
msg = await ws.receive()
|
||||
self.assertEqual(msg.type, WSMsgType.TEXT)
|
||||
|
||||
data = json.loads(msg.data)
|
||||
self.assertEqual(data.get('type'), 'subscription_confirmed')
|
||||
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_error_handling(self, websocket_server):
|
||||
"""Test WebSocket error handling"""
|
||||
session = aiohttp.ClientSession()
|
||||
|
||||
try:
|
||||
async with session.ws_connect('ws://localhost:8081/ws/dashboard') as ws:
|
||||
# Send invalid message
|
||||
invalid_msg = {'invalid': 'message'}
|
||||
await ws.send_str(json.dumps(invalid_msg))
|
||||
|
||||
# Should receive error response
|
||||
msg = await ws.receive()
|
||||
self.assertEqual(msg.type, WSMsgType.TEXT)
|
||||
|
||||
data = json.loads(msg.data)
|
||||
self.assertEqual(data.get('type'), 'error')
|
||||
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_websocket_connections(self, websocket_server):
|
||||
"""Test multiple concurrent WebSocket connections"""
|
||||
session = aiohttp.ClientSession()
|
||||
connections = []
|
||||
|
||||
try:
|
||||
# Create multiple connections
|
||||
for i in range(10):
|
||||
ws = await session.ws_connect(f'ws://localhost:8081/ws/dashboard')
|
||||
connections.append(ws)
|
||||
|
||||
# All connections should be active
|
||||
for ws in connections:
|
||||
self.assertEqual(ws.closed, False)
|
||||
|
||||
# Send message to all connections
|
||||
test_msg = {'type': 'ping', 'id': 'test'}
|
||||
for ws in connections:
|
||||
await ws.send_str(json.dumps(test_msg))
|
||||
|
||||
# All should receive responses
|
||||
for ws in connections:
|
||||
msg = await ws.receive()
|
||||
self.assertEqual(msg.type, WSMsgType.TEXT)
|
||||
|
||||
finally:
|
||||
# Close all connections
|
||||
for ws in connections:
|
||||
if not ws.closed:
|
||||
await ws.close()
|
||||
await session.close()
|
||||
|
||||
|
||||
class TestDashboardIntegration:
|
||||
"""Test dashboard integration with backend services"""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_services(self):
|
||||
"""Mock backend services"""
|
||||
services = {
|
||||
'redis': Mock(),
|
||||
'timescale': Mock(),
|
||||
'connectors': Mock(),
|
||||
'aggregator': Mock(),
|
||||
'monitor': Mock()
|
||||
}
|
||||
|
||||
# Setup mock responses
|
||||
services['redis'].get.return_value = {'test': 'data'}
|
||||
services['timescale'].query.return_value = [{'result': 'data'}]
|
||||
services['connectors'].get_status.return_value = 'connected'
|
||||
services['aggregator'].get_heatmap.return_value = {'heatmap': 'data'}
|
||||
services['monitor'].get_metrics.return_value = {'metrics': 'data'}
|
||||
|
||||
return services
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_data_flow(self, mock_services):
|
||||
"""Test complete data flow from backend to dashboard"""
|
||||
# Simulate data generation
|
||||
orderbook = OrderBookSnapshot(
|
||||
symbol="BTCUSDT",
|
||||
exchange="binance",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
bids=[PriceLevel(price=50000.0, size=1.0)],
|
||||
asks=[PriceLevel(price=50010.0, size=1.0)]
|
||||
)
|
||||
|
||||
# Mock data processing pipeline
|
||||
with patch.multiple(
|
||||
'COBY.processing.data_processor',
|
||||
DataProcessor=Mock()
|
||||
):
|
||||
# Process data
|
||||
processor = Mock()
|
||||
processor.normalize_orderbook.return_value = orderbook
|
||||
|
||||
# Aggregate data
|
||||
aggregator = Mock()
|
||||
aggregator.create_price_buckets.return_value = Mock()
|
||||
aggregator.generate_heatmap.return_value = Mock()
|
||||
|
||||
# Cache data
|
||||
cache = Mock()
|
||||
cache.set.return_value = True
|
||||
|
||||
# Verify data flows through pipeline
|
||||
processed = processor.normalize_orderbook({}, "binance")
|
||||
buckets = aggregator.create_price_buckets(processed)
|
||||
heatmap = aggregator.generate_heatmap(buckets)
|
||||
cached = cache.set("test_key", heatmap)
|
||||
|
||||
assert processed is not None
|
||||
assert buckets is not None
|
||||
assert heatmap is not None
|
||||
assert cached is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_time_updates(self, mock_services):
|
||||
"""Test real-time dashboard updates"""
|
||||
# Mock WebSocket server
|
||||
ws_server = Mock()
|
||||
ws_server.broadcast = AsyncMock()
|
||||
|
||||
# Simulate real-time data updates
|
||||
updates = [
|
||||
{'type': 'orderbook', 'symbol': 'BTCUSDT', 'data': {}},
|
||||
{'type': 'trade', 'symbol': 'BTCUSDT', 'data': {}},
|
||||
{'type': 'performance', 'data': {}}
|
||||
]
|
||||
|
||||
# Send updates
|
||||
for update in updates:
|
||||
await ws_server.broadcast(json.dumps(update))
|
||||
|
||||
# Verify broadcasts were sent
|
||||
assert ws_server.broadcast.call_count == len(updates)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_performance_under_load(self, mock_services):
|
||||
"""Test dashboard performance under high update frequency"""
|
||||
import time
|
||||
|
||||
# Mock high-frequency updates
|
||||
update_count = 1000
|
||||
start_time = time.time()
|
||||
|
||||
# Simulate processing many updates
|
||||
for i in range(update_count):
|
||||
# Mock data processing
|
||||
mock_services['redis'].get(f"orderbook:BTCUSDT:binance:{i}")
|
||||
mock_services['aggregator'].get_heatmap(f"BTCUSDT:{i}")
|
||||
|
||||
# Small delay to simulate processing
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
end_time = time.time()
|
||||
processing_time = end_time - start_time
|
||||
updates_per_second = update_count / processing_time
|
||||
|
||||
# Should handle at least 500 updates per second
|
||||
assert updates_per_second > 500, f"Dashboard too slow: {updates_per_second:.2f} updates/sec"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dashboard_error_recovery(self, mock_services):
|
||||
"""Test dashboard error recovery"""
|
||||
# Simulate service failures
|
||||
mock_services['redis'].get.side_effect = Exception("Redis connection failed")
|
||||
mock_services['timescale'].query.side_effect = Exception("Database error")
|
||||
|
||||
# Dashboard should handle errors gracefully
|
||||
try:
|
||||
# Attempt operations that will fail
|
||||
mock_services['redis'].get("test_key")
|
||||
except Exception:
|
||||
# Should recover and continue
|
||||
pass
|
||||
|
||||
try:
|
||||
mock_services['timescale'].query("SELECT * FROM test")
|
||||
except Exception:
|
||||
# Should recover and continue
|
||||
pass
|
||||
|
||||
# Reset services to working state
|
||||
mock_services['redis'].get.side_effect = None
|
||||
mock_services['redis'].get.return_value = {'recovered': True}
|
||||
|
||||
# Should work again
|
||||
result = mock_services['redis'].get("test_key")
|
||||
assert result['recovered'] is True
|
||||
|
||||
|
||||
class TestDashboardUI:
|
||||
"""Test dashboard UI functionality (requires browser automation)"""
|
||||
|
||||
@pytest.mark.skipif(not pytest.config.getoption("--ui"),
|
||||
reason="UI tests require --ui flag and browser setup")
|
||||
def test_dashboard_loads(self):
|
||||
"""Test that dashboard loads in browser"""
|
||||
# This would require Selenium or similar
|
||||
# Placeholder for UI tests
|
||||
pass
|
||||
|
||||
@pytest.mark.skipif(not pytest.config.getoption("--ui"),
|
||||
reason="UI tests require --ui flag and browser setup")
|
||||
def test_real_time_chart_updates(self):
|
||||
"""Test that charts update in real-time"""
|
||||
# This would require browser automation
|
||||
# Placeholder for UI tests
|
||||
pass
|
||||
|
||||
@pytest.mark.skipif(not pytest.config.getoption("--ui"),
|
||||
reason="UI tests require --ui flag and browser setup")
|
||||
def test_responsive_design(self):
|
||||
"""Test responsive design on different screen sizes"""
|
||||
# This would require browser automation with different viewport sizes
|
||||
# Placeholder for UI tests
|
||||
pass
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Configure pytest with custom markers"""
|
||||
config.addinivalue_line("markers", "e2e: mark test as end-to-end test")
|
||||
config.addinivalue_line("markers", "ui: mark test as UI test")
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
"""Add custom command line options"""
|
||||
parser.addoption(
|
||||
"--e2e",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="run end-to-end tests"
|
||||
)
|
||||
parser.addoption(
|
||||
"--ui",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="run UI tests (requires browser setup)"
|
||||
)
|
||||
485
COBY/tests/test_integration_pipeline.py
Normal file
485
COBY/tests/test_integration_pipeline.py
Normal file
@ -0,0 +1,485 @@
|
||||
"""
|
||||
Integration tests for complete data pipeline from exchanges to storage.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import Mock, AsyncMock, patch
|
||||
from typing import List, Dict, Any
|
||||
|
||||
from ..connectors.binance_connector import BinanceConnector
|
||||
from ..processing.data_processor import DataProcessor
|
||||
from ..aggregation.aggregation_engine import AggregationEngine
|
||||
from ..storage.timescale_manager import TimescaleManager
|
||||
from ..caching.redis_manager import RedisManager
|
||||
from ..models.core import OrderBookSnapshot, TradeEvent, PriceLevel
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class TestDataPipelineIntegration:
|
||||
"""Test complete data pipeline integration"""
|
||||
|
||||
@pytest.fixture
|
||||
async def mock_components(self):
|
||||
"""Setup mock components for testing"""
|
||||
# Mock exchange connector
|
||||
connector = Mock(spec=BinanceConnector)
|
||||
connector.exchange_name = "binance"
|
||||
connector.connect = AsyncMock(return_value=True)
|
||||
connector.disconnect = AsyncMock()
|
||||
connector.subscribe_orderbook = AsyncMock()
|
||||
connector.subscribe_trades = AsyncMock()
|
||||
|
||||
# Mock data processor
|
||||
processor = Mock(spec=DataProcessor)
|
||||
processor.process_orderbook = Mock()
|
||||
processor.process_trade = Mock()
|
||||
processor.validate_data = Mock(return_value=True)
|
||||
|
||||
# Mock aggregation engine
|
||||
aggregator = Mock(spec=AggregationEngine)
|
||||
aggregator.aggregate_orderbook = Mock()
|
||||
aggregator.create_heatmap = Mock()
|
||||
|
||||
# Mock storage manager
|
||||
storage = Mock(spec=TimescaleManager)
|
||||
storage.store_orderbook = AsyncMock(return_value=True)
|
||||
storage.store_trade = AsyncMock(return_value=True)
|
||||
storage.is_connected = Mock(return_value=True)
|
||||
|
||||
# Mock cache manager
|
||||
cache = Mock(spec=RedisManager)
|
||||
cache.set = AsyncMock(return_value=True)
|
||||
cache.get = AsyncMock(return_value=None)
|
||||
cache.is_connected = Mock(return_value=True)
|
||||
|
||||
return {
|
||||
'connector': connector,
|
||||
'processor': processor,
|
||||
'aggregator': aggregator,
|
||||
'storage': storage,
|
||||
'cache': cache
|
||||
}
|
||||
|
||||
@pytest.fixture
|
||||
def sample_orderbook(self):
|
||||
"""Create sample order book data"""
|
||||
return OrderBookSnapshot(
|
||||
symbol="BTCUSDT",
|
||||
exchange="binance",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
bids=[
|
||||
PriceLevel(price=50000.0, size=1.5),
|
||||
PriceLevel(price=49990.0, size=2.0),
|
||||
PriceLevel(price=49980.0, size=1.0)
|
||||
],
|
||||
asks=[
|
||||
PriceLevel(price=50010.0, size=1.2),
|
||||
PriceLevel(price=50020.0, size=1.8),
|
||||
PriceLevel(price=50030.0, size=0.8)
|
||||
]
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def sample_trade(self):
|
||||
"""Create sample trade data"""
|
||||
return TradeEvent(
|
||||
symbol="BTCUSDT",
|
||||
exchange="binance",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
price=50005.0,
|
||||
size=0.5,
|
||||
side="buy",
|
||||
trade_id="12345"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_orderbook_pipeline(self, mock_components, sample_orderbook):
|
||||
"""Test complete order book processing pipeline"""
|
||||
components = mock_components
|
||||
|
||||
# Setup processor to return processed data
|
||||
components['processor'].process_orderbook.return_value = sample_orderbook
|
||||
|
||||
# Simulate pipeline flow
|
||||
# 1. Receive data from exchange
|
||||
raw_data = {"symbol": "BTCUSDT", "bids": [], "asks": []}
|
||||
|
||||
# 2. Process data
|
||||
processed_data = components['processor'].process_orderbook(raw_data, "binance")
|
||||
|
||||
# 3. Validate data
|
||||
is_valid = components['processor'].validate_data(processed_data)
|
||||
assert is_valid
|
||||
|
||||
# 4. Aggregate data
|
||||
components['aggregator'].aggregate_orderbook(processed_data)
|
||||
|
||||
# 5. Store in database
|
||||
await components['storage'].store_orderbook(processed_data)
|
||||
|
||||
# 6. Cache latest data
|
||||
await components['cache'].set(f"orderbook:BTCUSDT:binance", processed_data)
|
||||
|
||||
# Verify all components were called
|
||||
components['processor'].process_orderbook.assert_called_once()
|
||||
components['processor'].validate_data.assert_called_once()
|
||||
components['aggregator'].aggregate_orderbook.assert_called_once()
|
||||
components['storage'].store_orderbook.assert_called_once()
|
||||
components['cache'].set.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_trade_pipeline(self, mock_components, sample_trade):
|
||||
"""Test complete trade processing pipeline"""
|
||||
components = mock_components
|
||||
|
||||
# Setup processor to return processed data
|
||||
components['processor'].process_trade.return_value = sample_trade
|
||||
|
||||
# Simulate pipeline flow
|
||||
raw_data = {"symbol": "BTCUSDT", "price": 50005.0, "quantity": 0.5}
|
||||
|
||||
# Process through pipeline
|
||||
processed_data = components['processor'].process_trade(raw_data, "binance")
|
||||
is_valid = components['processor'].validate_data(processed_data)
|
||||
assert is_valid
|
||||
|
||||
await components['storage'].store_trade(processed_data)
|
||||
await components['cache'].set(f"trade:BTCUSDT:binance:latest", processed_data)
|
||||
|
||||
# Verify calls
|
||||
components['processor'].process_trade.assert_called_once()
|
||||
components['storage'].store_trade.assert_called_once()
|
||||
components['cache'].set.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_exchange_pipeline(self, mock_components):
|
||||
"""Test pipeline with multiple exchanges"""
|
||||
components = mock_components
|
||||
exchanges = ["binance", "coinbase", "kraken"]
|
||||
|
||||
# Simulate data from multiple exchanges
|
||||
for exchange in exchanges:
|
||||
# Create exchange-specific data
|
||||
orderbook = OrderBookSnapshot(
|
||||
symbol="BTCUSDT",
|
||||
exchange=exchange,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
bids=[PriceLevel(price=50000.0, size=1.0)],
|
||||
asks=[PriceLevel(price=50010.0, size=1.0)]
|
||||
)
|
||||
|
||||
components['processor'].process_orderbook.return_value = orderbook
|
||||
components['processor'].validate_data.return_value = True
|
||||
|
||||
# Process through pipeline
|
||||
processed_data = components['processor'].process_orderbook({}, exchange)
|
||||
is_valid = components['processor'].validate_data(processed_data)
|
||||
assert is_valid
|
||||
|
||||
await components['storage'].store_orderbook(processed_data)
|
||||
await components['cache'].set(f"orderbook:BTCUSDT:{exchange}", processed_data)
|
||||
|
||||
# Verify multiple calls
|
||||
assert components['processor'].process_orderbook.call_count == len(exchanges)
|
||||
assert components['storage'].store_orderbook.call_count == len(exchanges)
|
||||
assert components['cache'].set.call_count == len(exchanges)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_error_handling(self, mock_components, sample_orderbook):
|
||||
"""Test pipeline error handling and recovery"""
|
||||
components = mock_components
|
||||
|
||||
# Setup storage to fail initially
|
||||
components['storage'].store_orderbook.side_effect = [
|
||||
Exception("Database connection failed"),
|
||||
True # Success on retry
|
||||
]
|
||||
|
||||
components['processor'].process_orderbook.return_value = sample_orderbook
|
||||
components['processor'].validate_data.return_value = True
|
||||
|
||||
# First attempt should fail
|
||||
with pytest.raises(Exception):
|
||||
await components['storage'].store_orderbook(sample_orderbook)
|
||||
|
||||
# Second attempt should succeed
|
||||
result = await components['storage'].store_orderbook(sample_orderbook)
|
||||
assert result is True
|
||||
|
||||
# Verify retry logic
|
||||
assert components['storage'].store_orderbook.call_count == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_performance(self, mock_components):
|
||||
"""Test pipeline performance with high throughput"""
|
||||
components = mock_components
|
||||
|
||||
# Setup fast responses
|
||||
components['processor'].process_orderbook.return_value = Mock()
|
||||
components['processor'].validate_data.return_value = True
|
||||
components['storage'].store_orderbook.return_value = True
|
||||
components['cache'].set.return_value = True
|
||||
|
||||
# Process multiple items quickly
|
||||
start_time = time.time()
|
||||
tasks = []
|
||||
|
||||
for i in range(100):
|
||||
# Simulate processing 100 order books
|
||||
task = asyncio.create_task(self._process_single_orderbook(components, i))
|
||||
tasks.append(task)
|
||||
|
||||
await asyncio.gather(*tasks)
|
||||
end_time = time.time()
|
||||
|
||||
processing_time = end_time - start_time
|
||||
throughput = 100 / processing_time
|
||||
|
||||
# Should process at least 50 items per second
|
||||
assert throughput > 50, f"Throughput too low: {throughput:.2f} items/sec"
|
||||
|
||||
# Verify all items were processed
|
||||
assert components['processor'].process_orderbook.call_count == 100
|
||||
assert components['storage'].store_orderbook.call_count == 100
|
||||
|
||||
async def _process_single_orderbook(self, components, index):
|
||||
"""Helper method to process a single order book"""
|
||||
raw_data = {"symbol": "BTCUSDT", "index": index}
|
||||
|
||||
processed_data = components['processor'].process_orderbook(raw_data, "binance")
|
||||
is_valid = components['processor'].validate_data(processed_data)
|
||||
|
||||
if is_valid:
|
||||
await components['storage'].store_orderbook(processed_data)
|
||||
await components['cache'].set(f"orderbook:BTCUSDT:binance:{index}", processed_data)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_data_consistency_across_pipeline(self, mock_components, sample_orderbook):
|
||||
"""Test data consistency throughout the pipeline"""
|
||||
components = mock_components
|
||||
|
||||
# Track data transformations
|
||||
original_data = {"symbol": "BTCUSDT", "timestamp": "2024-01-01T00:00:00Z"}
|
||||
|
||||
# Setup processor to modify data
|
||||
modified_orderbook = sample_orderbook
|
||||
modified_orderbook.symbol = "BTCUSDT" # Ensure consistency
|
||||
components['processor'].process_orderbook.return_value = modified_orderbook
|
||||
components['processor'].validate_data.return_value = True
|
||||
|
||||
# Process data
|
||||
processed_data = components['processor'].process_orderbook(original_data, "binance")
|
||||
|
||||
# Verify data consistency
|
||||
assert processed_data.symbol == "BTCUSDT"
|
||||
assert processed_data.exchange == "binance"
|
||||
assert len(processed_data.bids) > 0
|
||||
assert len(processed_data.asks) > 0
|
||||
|
||||
# Verify all price levels are valid
|
||||
for bid in processed_data.bids:
|
||||
assert bid.price > 0
|
||||
assert bid.size > 0
|
||||
|
||||
for ask in processed_data.asks:
|
||||
assert ask.price > 0
|
||||
assert ask.size > 0
|
||||
|
||||
# Verify bid/ask ordering
|
||||
bid_prices = [bid.price for bid in processed_data.bids]
|
||||
ask_prices = [ask.price for ask in processed_data.asks]
|
||||
|
||||
assert bid_prices == sorted(bid_prices, reverse=True) # Bids descending
|
||||
assert ask_prices == sorted(ask_prices) # Asks ascending
|
||||
|
||||
# Verify spread is positive
|
||||
if bid_prices and ask_prices:
|
||||
spread = min(ask_prices) - max(bid_prices)
|
||||
assert spread >= 0, f"Negative spread detected: {spread}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_memory_usage(self, mock_components):
|
||||
"""Test pipeline memory usage under load"""
|
||||
import psutil
|
||||
import gc
|
||||
|
||||
components = mock_components
|
||||
process = psutil.Process()
|
||||
|
||||
# Get initial memory usage
|
||||
initial_memory = process.memory_info().rss / 1024 / 1024 # MB
|
||||
|
||||
# Process large amount of data
|
||||
for i in range(1000):
|
||||
orderbook = OrderBookSnapshot(
|
||||
symbol="BTCUSDT",
|
||||
exchange="binance",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
bids=[PriceLevel(price=50000.0 + i, size=1.0)],
|
||||
asks=[PriceLevel(price=50010.0 + i, size=1.0)]
|
||||
)
|
||||
|
||||
components['processor'].process_orderbook.return_value = orderbook
|
||||
components['processor'].validate_data.return_value = True
|
||||
|
||||
# Process data
|
||||
processed_data = components['processor'].process_orderbook({}, "binance")
|
||||
await components['storage'].store_orderbook(processed_data)
|
||||
|
||||
# Force garbage collection every 100 items
|
||||
if i % 100 == 0:
|
||||
gc.collect()
|
||||
|
||||
# Get final memory usage
|
||||
final_memory = process.memory_info().rss / 1024 / 1024 # MB
|
||||
memory_increase = final_memory - initial_memory
|
||||
|
||||
# Memory increase should be reasonable (less than 100MB for 1000 items)
|
||||
assert memory_increase < 100, f"Memory usage increased by {memory_increase:.2f}MB"
|
||||
|
||||
logger.info(f"Memory usage: {initial_memory:.2f}MB -> {final_memory:.2f}MB (+{memory_increase:.2f}MB)")
|
||||
|
||||
|
||||
class TestPipelineResilience:
|
||||
"""Test pipeline resilience and fault tolerance"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_database_reconnection(self):
|
||||
"""Test database reconnection handling"""
|
||||
storage = Mock(spec=TimescaleManager)
|
||||
|
||||
# Simulate connection failure then recovery
|
||||
storage.is_connected.side_effect = [False, False, True]
|
||||
storage.connect.return_value = True
|
||||
storage.store_orderbook.return_value = True
|
||||
|
||||
# Should attempt reconnection
|
||||
for attempt in range(3):
|
||||
if not storage.is_connected():
|
||||
storage.connect()
|
||||
else:
|
||||
break
|
||||
|
||||
assert storage.connect.call_count == 1
|
||||
assert storage.is_connected.call_count == 3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_fallback(self):
|
||||
"""Test cache fallback when Redis is unavailable"""
|
||||
cache = Mock(spec=RedisManager)
|
||||
|
||||
# Simulate cache failure
|
||||
cache.is_connected.return_value = False
|
||||
cache.set.side_effect = Exception("Redis connection failed")
|
||||
|
||||
# Should handle cache failure gracefully
|
||||
try:
|
||||
await cache.set("test_key", "test_value")
|
||||
except Exception:
|
||||
# Should continue processing even if cache fails
|
||||
pass
|
||||
|
||||
assert not cache.is_connected()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exchange_failover(self):
|
||||
"""Test exchange failover when one exchange fails"""
|
||||
exchanges = ["binance", "coinbase", "kraken"]
|
||||
failed_exchange = "binance"
|
||||
|
||||
# Simulate one exchange failing
|
||||
for exchange in exchanges:
|
||||
if exchange == failed_exchange:
|
||||
# This exchange fails
|
||||
assert exchange == failed_exchange
|
||||
else:
|
||||
# Other exchanges continue working
|
||||
assert exchange != failed_exchange
|
||||
|
||||
# Should continue with remaining exchanges
|
||||
working_exchanges = [ex for ex in exchanges if ex != failed_exchange]
|
||||
assert len(working_exchanges) == 2
|
||||
assert "coinbase" in working_exchanges
|
||||
assert "kraken" in working_exchanges
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestRealDataPipeline:
|
||||
"""Integration tests with real components (requires running services)"""
|
||||
|
||||
@pytest.mark.skipif(not pytest.config.getoption("--integration"),
|
||||
reason="Integration tests require --integration flag")
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_database_integration(self):
|
||||
"""Test with real TimescaleDB instance"""
|
||||
# This test requires a running TimescaleDB instance
|
||||
# Skip if not available
|
||||
try:
|
||||
from ..storage.timescale_manager import TimescaleManager
|
||||
|
||||
storage = TimescaleManager()
|
||||
await storage.connect()
|
||||
|
||||
# Test basic operations
|
||||
assert storage.is_connected()
|
||||
|
||||
# Create test data
|
||||
orderbook = OrderBookSnapshot(
|
||||
symbol="BTCUSDT",
|
||||
exchange="test",
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
bids=[PriceLevel(price=50000.0, size=1.0)],
|
||||
asks=[PriceLevel(price=50010.0, size=1.0)]
|
||||
)
|
||||
|
||||
# Store and verify
|
||||
result = await storage.store_orderbook(orderbook)
|
||||
assert result is True
|
||||
|
||||
await storage.disconnect()
|
||||
|
||||
except Exception as e:
|
||||
pytest.skip(f"Real database not available: {e}")
|
||||
|
||||
@pytest.mark.skipif(not pytest.config.getoption("--integration"),
|
||||
reason="Integration tests require --integration flag")
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_cache_integration(self):
|
||||
"""Test with real Redis instance"""
|
||||
try:
|
||||
from ..caching.redis_manager import RedisManager
|
||||
|
||||
cache = RedisManager()
|
||||
await cache.connect()
|
||||
|
||||
assert cache.is_connected()
|
||||
|
||||
# Test basic operations
|
||||
await cache.set("test_key", {"test": "data"})
|
||||
result = await cache.get("test_key")
|
||||
assert result is not None
|
||||
|
||||
await cache.disconnect()
|
||||
|
||||
except Exception as e:
|
||||
pytest.skip(f"Real cache not available: {e}")
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Configure pytest with custom markers"""
|
||||
config.addinivalue_line("markers", "integration: mark test as integration test")
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
"""Add custom command line options"""
|
||||
parser.addoption(
|
||||
"--integration",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="run integration tests with real services"
|
||||
)
|
||||
590
COBY/tests/test_load_performance.py
Normal file
590
COBY/tests/test_load_performance.py
Normal file
@ -0,0 +1,590 @@
|
||||
"""
|
||||
Load testing and performance benchmarks for high-frequency data scenarios.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
import time
|
||||
import statistics
|
||||
from datetime import datetime, timezone
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import List, Dict, Any
|
||||
import psutil
|
||||
import gc
|
||||
|
||||
from ..models.core import OrderBookSnapshot, TradeEvent, PriceLevel
|
||||
from ..connectors.binance_connector import BinanceConnector
|
||||
from ..processing.data_processor import DataProcessor
|
||||
from ..aggregation.aggregation_engine import AggregationEngine
|
||||
from ..monitoring.metrics_collector import MetricsCollector
|
||||
from ..monitoring.latency_tracker import LatencyTracker
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class LoadTestConfig:
|
||||
"""Configuration for load tests"""
|
||||
|
||||
# Test parameters
|
||||
DURATION_SECONDS = 60
|
||||
TARGET_TPS = 1000 # Transactions per second
|
||||
RAMP_UP_SECONDS = 10
|
||||
|
||||
# Performance thresholds
|
||||
MAX_LATENCY_MS = 100
|
||||
MAX_MEMORY_MB = 500
|
||||
MIN_SUCCESS_RATE = 99.0
|
||||
|
||||
# Data generation
|
||||
SYMBOLS = ["BTCUSDT", "ETHUSDT", "ADAUSDT", "DOTUSDT"]
|
||||
EXCHANGES = ["binance", "coinbase", "kraken", "bybit"]
|
||||
|
||||
|
||||
class DataGenerator:
|
||||
"""Generate realistic test data for load testing"""
|
||||
|
||||
def __init__(self):
|
||||
self.base_prices = {
|
||||
"BTCUSDT": 50000.0,
|
||||
"ETHUSDT": 3000.0,
|
||||
"ADAUSDT": 1.0,
|
||||
"DOTUSDT": 25.0
|
||||
}
|
||||
self.counter = 0
|
||||
|
||||
def generate_orderbook(self, symbol: str, exchange: str) -> OrderBookSnapshot:
|
||||
"""Generate realistic order book data"""
|
||||
base_price = self.base_prices.get(symbol, 100.0)
|
||||
|
||||
# Add some randomness
|
||||
price_variation = (self.counter % 100) * 0.01
|
||||
mid_price = base_price + price_variation
|
||||
|
||||
# Generate bids (below mid price)
|
||||
bids = []
|
||||
for i in range(10):
|
||||
price = mid_price - (i + 1) * 0.1
|
||||
size = 1.0 + (i * 0.1)
|
||||
bids.append(PriceLevel(price=price, size=size))
|
||||
|
||||
# Generate asks (above mid price)
|
||||
asks = []
|
||||
for i in range(10):
|
||||
price = mid_price + (i + 1) * 0.1
|
||||
size = 1.0 + (i * 0.1)
|
||||
asks.append(PriceLevel(price=price, size=size))
|
||||
|
||||
self.counter += 1
|
||||
|
||||
return OrderBookSnapshot(
|
||||
symbol=symbol,
|
||||
exchange=exchange,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
bids=bids,
|
||||
asks=asks
|
||||
)
|
||||
|
||||
def generate_trade(self, symbol: str, exchange: str) -> TradeEvent:
|
||||
"""Generate realistic trade data"""
|
||||
base_price = self.base_prices.get(symbol, 100.0)
|
||||
price_variation = (self.counter % 50) * 0.01
|
||||
price = base_price + price_variation
|
||||
|
||||
self.counter += 1
|
||||
|
||||
return TradeEvent(
|
||||
symbol=symbol,
|
||||
exchange=exchange,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
price=price,
|
||||
size=0.1 + (self.counter % 10) * 0.01,
|
||||
side="buy" if self.counter % 2 == 0 else "sell",
|
||||
trade_id=str(self.counter)
|
||||
)
|
||||
|
||||
|
||||
class PerformanceMonitor:
|
||||
"""Monitor performance during load tests"""
|
||||
|
||||
def __init__(self):
|
||||
self.start_time = None
|
||||
self.end_time = None
|
||||
self.latencies = []
|
||||
self.errors = []
|
||||
self.memory_samples = []
|
||||
self.cpu_samples = []
|
||||
self.process = psutil.Process()
|
||||
|
||||
def start(self):
|
||||
"""Start monitoring"""
|
||||
self.start_time = time.time()
|
||||
self.latencies.clear()
|
||||
self.errors.clear()
|
||||
self.memory_samples.clear()
|
||||
self.cpu_samples.clear()
|
||||
|
||||
def stop(self):
|
||||
"""Stop monitoring"""
|
||||
self.end_time = time.time()
|
||||
|
||||
def record_latency(self, latency_ms: float):
|
||||
"""Record operation latency"""
|
||||
self.latencies.append(latency_ms)
|
||||
|
||||
def record_error(self, error: Exception):
|
||||
"""Record error"""
|
||||
self.errors.append(str(error))
|
||||
|
||||
def sample_system_metrics(self):
|
||||
"""Sample system metrics"""
|
||||
try:
|
||||
memory_mb = self.process.memory_info().rss / 1024 / 1024
|
||||
cpu_percent = self.process.cpu_percent()
|
||||
|
||||
self.memory_samples.append(memory_mb)
|
||||
self.cpu_samples.append(cpu_percent)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error sampling system metrics: {e}")
|
||||
|
||||
def get_results(self) -> Dict[str, Any]:
|
||||
"""Get performance test results"""
|
||||
duration = self.end_time - self.start_time if self.end_time else 0
|
||||
total_operations = len(self.latencies)
|
||||
|
||||
results = {
|
||||
'duration_seconds': duration,
|
||||
'total_operations': total_operations,
|
||||
'operations_per_second': total_operations / duration if duration > 0 else 0,
|
||||
'error_count': len(self.errors),
|
||||
'success_rate': ((total_operations - len(self.errors)) / total_operations * 100) if total_operations > 0 else 0,
|
||||
'latency': {
|
||||
'min_ms': min(self.latencies) if self.latencies else 0,
|
||||
'max_ms': max(self.latencies) if self.latencies else 0,
|
||||
'avg_ms': statistics.mean(self.latencies) if self.latencies else 0,
|
||||
'p50_ms': statistics.median(self.latencies) if self.latencies else 0,
|
||||
'p95_ms': self._percentile(self.latencies, 95) if self.latencies else 0,
|
||||
'p99_ms': self._percentile(self.latencies, 99) if self.latencies else 0
|
||||
},
|
||||
'memory': {
|
||||
'min_mb': min(self.memory_samples) if self.memory_samples else 0,
|
||||
'max_mb': max(self.memory_samples) if self.memory_samples else 0,
|
||||
'avg_mb': statistics.mean(self.memory_samples) if self.memory_samples else 0
|
||||
},
|
||||
'cpu': {
|
||||
'min_percent': min(self.cpu_samples) if self.cpu_samples else 0,
|
||||
'max_percent': max(self.cpu_samples) if self.cpu_samples else 0,
|
||||
'avg_percent': statistics.mean(self.cpu_samples) if self.cpu_samples else 0
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def _percentile(self, data: List[float], percentile: int) -> float:
|
||||
"""Calculate percentile"""
|
||||
if not data:
|
||||
return 0.0
|
||||
sorted_data = sorted(data)
|
||||
index = int((percentile / 100.0) * len(sorted_data))
|
||||
index = min(index, len(sorted_data) - 1)
|
||||
return sorted_data[index]
|
||||
|
||||
|
||||
@pytest.mark.load
|
||||
class TestLoadPerformance:
|
||||
"""Load testing and performance benchmarks"""
|
||||
|
||||
@pytest.fixture
|
||||
def data_generator(self):
|
||||
"""Create data generator"""
|
||||
return DataGenerator()
|
||||
|
||||
@pytest.fixture
|
||||
def performance_monitor(self):
|
||||
"""Create performance monitor"""
|
||||
return PerformanceMonitor()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orderbook_processing_load(self, data_generator, performance_monitor):
|
||||
"""Test order book processing under high load"""
|
||||
processor = DataProcessor()
|
||||
monitor = performance_monitor
|
||||
|
||||
monitor.start()
|
||||
|
||||
# Generate load
|
||||
tasks = []
|
||||
for i in range(LoadTestConfig.TARGET_TPS):
|
||||
symbol = LoadTestConfig.SYMBOLS[i % len(LoadTestConfig.SYMBOLS)]
|
||||
exchange = LoadTestConfig.EXCHANGES[i % len(LoadTestConfig.EXCHANGES)]
|
||||
|
||||
task = asyncio.create_task(
|
||||
self._process_orderbook_with_timing(
|
||||
processor, data_generator, symbol, exchange, monitor
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# Add small delay to simulate realistic load
|
||||
if i % 100 == 0:
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
# Wait for all tasks to complete
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
monitor.stop()
|
||||
results = monitor.get_results()
|
||||
|
||||
# Verify performance requirements
|
||||
assert results['operations_per_second'] >= LoadTestConfig.TARGET_TPS * 0.8, \
|
||||
f"Throughput too low: {results['operations_per_second']:.2f} ops/sec"
|
||||
|
||||
assert results['latency']['p95_ms'] <= LoadTestConfig.MAX_LATENCY_MS, \
|
||||
f"P95 latency too high: {results['latency']['p95_ms']:.2f}ms"
|
||||
|
||||
assert results['success_rate'] >= LoadTestConfig.MIN_SUCCESS_RATE, \
|
||||
f"Success rate too low: {results['success_rate']:.2f}%"
|
||||
|
||||
logger.info(f"Load test results: {results}")
|
||||
|
||||
async def _process_orderbook_with_timing(self, processor, data_generator,
|
||||
symbol, exchange, monitor):
|
||||
"""Process order book with timing measurement"""
|
||||
try:
|
||||
start_time = time.time()
|
||||
|
||||
# Generate and process order book
|
||||
orderbook = data_generator.generate_orderbook(symbol, exchange)
|
||||
processed = processor.normalize_orderbook(orderbook.__dict__, exchange)
|
||||
|
||||
end_time = time.time()
|
||||
latency_ms = (end_time - start_time) * 1000
|
||||
|
||||
monitor.record_latency(latency_ms)
|
||||
monitor.sample_system_metrics()
|
||||
|
||||
except Exception as e:
|
||||
monitor.record_error(e)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trade_processing_load(self, data_generator, performance_monitor):
|
||||
"""Test trade processing under high load"""
|
||||
processor = DataProcessor()
|
||||
monitor = performance_monitor
|
||||
|
||||
monitor.start()
|
||||
|
||||
# Generate sustained load for specified duration
|
||||
end_time = time.time() + LoadTestConfig.DURATION_SECONDS
|
||||
operation_count = 0
|
||||
|
||||
while time.time() < end_time:
|
||||
symbol = LoadTestConfig.SYMBOLS[operation_count % len(LoadTestConfig.SYMBOLS)]
|
||||
exchange = LoadTestConfig.EXCHANGES[operation_count % len(LoadTestConfig.EXCHANGES)]
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
|
||||
# Generate and process trade
|
||||
trade = data_generator.generate_trade(symbol, exchange)
|
||||
processed = processor.normalize_trade(trade.__dict__, exchange)
|
||||
|
||||
process_time = time.time()
|
||||
latency_ms = (process_time - start_time) * 1000
|
||||
|
||||
monitor.record_latency(latency_ms)
|
||||
|
||||
if operation_count % 100 == 0:
|
||||
monitor.sample_system_metrics()
|
||||
|
||||
operation_count += 1
|
||||
|
||||
# Control rate to avoid overwhelming
|
||||
await asyncio.sleep(0.001) # 1ms delay
|
||||
|
||||
except Exception as e:
|
||||
monitor.record_error(e)
|
||||
|
||||
monitor.stop()
|
||||
results = monitor.get_results()
|
||||
|
||||
# Verify performance
|
||||
assert results['latency']['avg_ms'] <= LoadTestConfig.MAX_LATENCY_MS, \
|
||||
f"Average latency too high: {results['latency']['avg_ms']:.2f}ms"
|
||||
|
||||
assert results['memory']['max_mb'] <= LoadTestConfig.MAX_MEMORY_MB, \
|
||||
f"Memory usage too high: {results['memory']['max_mb']:.2f}MB"
|
||||
|
||||
logger.info(f"Trade processing results: {results}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregation_performance(self, data_generator, performance_monitor):
|
||||
"""Test aggregation engine performance"""
|
||||
aggregator = AggregationEngine()
|
||||
monitor = performance_monitor
|
||||
|
||||
monitor.start()
|
||||
|
||||
# Generate multiple order books for aggregation
|
||||
orderbooks = []
|
||||
for i in range(100):
|
||||
symbol = LoadTestConfig.SYMBOLS[i % len(LoadTestConfig.SYMBOLS)]
|
||||
exchange = LoadTestConfig.EXCHANGES[i % len(LoadTestConfig.EXCHANGES)]
|
||||
orderbook = data_generator.generate_orderbook(symbol, exchange)
|
||||
orderbooks.append(orderbook)
|
||||
|
||||
# Test aggregation performance
|
||||
start_time = time.time()
|
||||
|
||||
for orderbook in orderbooks:
|
||||
try:
|
||||
# Test price bucketing
|
||||
buckets = aggregator.create_price_buckets(orderbook)
|
||||
|
||||
# Test heatmap generation
|
||||
heatmap = aggregator.generate_heatmap(buckets)
|
||||
|
||||
# Test metrics calculation
|
||||
metrics = aggregator.calculate_metrics(orderbook)
|
||||
|
||||
process_time = time.time()
|
||||
latency_ms = (process_time - start_time) * 1000
|
||||
monitor.record_latency(latency_ms)
|
||||
|
||||
start_time = process_time
|
||||
|
||||
except Exception as e:
|
||||
monitor.record_error(e)
|
||||
|
||||
monitor.stop()
|
||||
results = monitor.get_results()
|
||||
|
||||
# Verify aggregation performance
|
||||
assert results['latency']['p95_ms'] <= 50, \
|
||||
f"Aggregation P95 latency too high: {results['latency']['p95_ms']:.2f}ms"
|
||||
|
||||
logger.info(f"Aggregation performance results: {results}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_exchange_processing(self, data_generator, performance_monitor):
|
||||
"""Test concurrent processing from multiple exchanges"""
|
||||
processor = DataProcessor()
|
||||
monitor = performance_monitor
|
||||
|
||||
monitor.start()
|
||||
|
||||
# Create concurrent tasks for each exchange
|
||||
tasks = []
|
||||
for exchange in LoadTestConfig.EXCHANGES:
|
||||
task = asyncio.create_task(
|
||||
self._simulate_exchange_load(processor, data_generator, exchange, monitor)
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# Run all exchanges concurrently
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
monitor.stop()
|
||||
results = monitor.get_results()
|
||||
|
||||
# Verify concurrent processing performance
|
||||
expected_total_ops = len(LoadTestConfig.EXCHANGES) * 100 # 100 ops per exchange
|
||||
assert results['total_operations'] >= expected_total_ops * 0.9, \
|
||||
f"Not enough operations completed: {results['total_operations']}"
|
||||
|
||||
assert results['success_rate'] >= 95.0, \
|
||||
f"Success rate too low under concurrent load: {results['success_rate']:.2f}%"
|
||||
|
||||
logger.info(f"Concurrent processing results: {results}")
|
||||
|
||||
async def _simulate_exchange_load(self, processor, data_generator, exchange, monitor):
|
||||
"""Simulate load from a single exchange"""
|
||||
for i in range(100):
|
||||
try:
|
||||
symbol = LoadTestConfig.SYMBOLS[i % len(LoadTestConfig.SYMBOLS)]
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Alternate between order books and trades
|
||||
if i % 2 == 0:
|
||||
data = data_generator.generate_orderbook(symbol, exchange)
|
||||
processed = processor.normalize_orderbook(data.__dict__, exchange)
|
||||
else:
|
||||
data = data_generator.generate_trade(symbol, exchange)
|
||||
processed = processor.normalize_trade(data.__dict__, exchange)
|
||||
|
||||
end_time = time.time()
|
||||
latency_ms = (end_time - start_time) * 1000
|
||||
|
||||
monitor.record_latency(latency_ms)
|
||||
|
||||
# Small delay to simulate realistic timing
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
except Exception as e:
|
||||
monitor.record_error(e)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_usage_under_load(self, data_generator):
|
||||
"""Test memory usage patterns under sustained load"""
|
||||
processor = DataProcessor()
|
||||
process = psutil.Process()
|
||||
|
||||
# Get baseline memory
|
||||
gc.collect() # Force garbage collection
|
||||
baseline_memory = process.memory_info().rss / 1024 / 1024 # MB
|
||||
|
||||
memory_samples = []
|
||||
|
||||
# Generate sustained load
|
||||
for i in range(1000):
|
||||
symbol = LoadTestConfig.SYMBOLS[i % len(LoadTestConfig.SYMBOLS)]
|
||||
exchange = LoadTestConfig.EXCHANGES[i % len(LoadTestConfig.EXCHANGES)]
|
||||
|
||||
# Generate data
|
||||
orderbook = data_generator.generate_orderbook(symbol, exchange)
|
||||
trade = data_generator.generate_trade(symbol, exchange)
|
||||
|
||||
# Process data
|
||||
processor.normalize_orderbook(orderbook.__dict__, exchange)
|
||||
processor.normalize_trade(trade.__dict__, exchange)
|
||||
|
||||
# Sample memory every 100 operations
|
||||
if i % 100 == 0:
|
||||
current_memory = process.memory_info().rss / 1024 / 1024
|
||||
memory_samples.append(current_memory)
|
||||
|
||||
# Force garbage collection periodically
|
||||
if i % 500 == 0:
|
||||
gc.collect()
|
||||
|
||||
# Final memory check
|
||||
gc.collect()
|
||||
final_memory = process.memory_info().rss / 1024 / 1024
|
||||
|
||||
# Calculate memory statistics
|
||||
max_memory = max(memory_samples) if memory_samples else final_memory
|
||||
memory_growth = final_memory - baseline_memory
|
||||
|
||||
# Verify memory usage is reasonable
|
||||
assert memory_growth < 100, \
|
||||
f"Memory growth too high: {memory_growth:.2f}MB"
|
||||
|
||||
assert max_memory < baseline_memory + 200, \
|
||||
f"Peak memory usage too high: {max_memory:.2f}MB"
|
||||
|
||||
logger.info(f"Memory usage: baseline={baseline_memory:.2f}MB, "
|
||||
f"final={final_memory:.2f}MB, growth={memory_growth:.2f}MB, "
|
||||
f"peak={max_memory:.2f}MB")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stress_test_extreme_load(self, data_generator, performance_monitor):
|
||||
"""Stress test with extreme load conditions"""
|
||||
processor = DataProcessor()
|
||||
monitor = performance_monitor
|
||||
|
||||
# Extreme load parameters
|
||||
EXTREME_TPS = 5000
|
||||
STRESS_DURATION = 30 # seconds
|
||||
|
||||
monitor.start()
|
||||
|
||||
# Generate extreme load
|
||||
tasks = []
|
||||
operations_per_batch = 100
|
||||
batches = EXTREME_TPS // operations_per_batch
|
||||
|
||||
for batch in range(batches):
|
||||
batch_tasks = []
|
||||
for i in range(operations_per_batch):
|
||||
symbol = LoadTestConfig.SYMBOLS[i % len(LoadTestConfig.SYMBOLS)]
|
||||
exchange = LoadTestConfig.EXCHANGES[i % len(LoadTestConfig.EXCHANGES)]
|
||||
|
||||
task = asyncio.create_task(
|
||||
self._process_orderbook_with_timing(
|
||||
processor, data_generator, symbol, exchange, monitor
|
||||
)
|
||||
)
|
||||
batch_tasks.append(task)
|
||||
|
||||
# Process batch
|
||||
await asyncio.gather(*batch_tasks, return_exceptions=True)
|
||||
|
||||
# Small delay between batches
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
monitor.stop()
|
||||
results = monitor.get_results()
|
||||
|
||||
# Under extreme load, we accept lower performance but system should remain stable
|
||||
assert results['success_rate'] >= 80.0, \
|
||||
f"System failed under stress: {results['success_rate']:.2f}% success rate"
|
||||
|
||||
assert results['latency']['p99_ms'] <= 500, \
|
||||
f"P99 latency too high under stress: {results['latency']['p99_ms']:.2f}ms"
|
||||
|
||||
logger.info(f"Stress test results: {results}")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
class TestPerformanceBenchmarks:
|
||||
"""Performance benchmarks for regression testing"""
|
||||
|
||||
def test_orderbook_processing_benchmark(self, benchmark):
|
||||
"""Benchmark order book processing speed"""
|
||||
processor = DataProcessor()
|
||||
generator = DataGenerator()
|
||||
|
||||
def process_orderbook():
|
||||
orderbook = generator.generate_orderbook("BTCUSDT", "binance")
|
||||
return processor.normalize_orderbook(orderbook.__dict__, "binance")
|
||||
|
||||
result = benchmark(process_orderbook)
|
||||
assert result is not None
|
||||
|
||||
def test_trade_processing_benchmark(self, benchmark):
|
||||
"""Benchmark trade processing speed"""
|
||||
processor = DataProcessor()
|
||||
generator = DataGenerator()
|
||||
|
||||
def process_trade():
|
||||
trade = generator.generate_trade("BTCUSDT", "binance")
|
||||
return processor.normalize_trade(trade.__dict__, "binance")
|
||||
|
||||
result = benchmark(process_trade)
|
||||
assert result is not None
|
||||
|
||||
def test_aggregation_benchmark(self, benchmark):
|
||||
"""Benchmark aggregation engine performance"""
|
||||
aggregator = AggregationEngine()
|
||||
generator = DataGenerator()
|
||||
|
||||
def aggregate_data():
|
||||
orderbook = generator.generate_orderbook("BTCUSDT", "binance")
|
||||
buckets = aggregator.create_price_buckets(orderbook)
|
||||
return aggregator.generate_heatmap(buckets)
|
||||
|
||||
result = benchmark(aggregate_data)
|
||||
assert result is not None
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Configure pytest with custom markers"""
|
||||
config.addinivalue_line("markers", "load: mark test as load test")
|
||||
config.addinivalue_line("markers", "benchmark: mark test as benchmark")
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
"""Add custom command line options"""
|
||||
parser.addoption(
|
||||
"--load",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="run load tests"
|
||||
)
|
||||
parser.addoption(
|
||||
"--benchmark",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="run benchmark tests"
|
||||
)
|
||||
108
COBY/tests/test_performance_benchmarks.py
Normal file
108
COBY/tests/test_performance_benchmarks.py
Normal file
@ -0,0 +1,108 @@
|
||||
"""
|
||||
Performance benchmarks and regression tests.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import time
|
||||
import statistics
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Any, Tuple
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..models.core import OrderBookSnapshot, TradeEvent, PriceLevel
|
||||
from ..processing.data_processor import DataProcessor
|
||||
from ..aggregation.aggregation_engine import AggregationEngine
|
||||
from ..connectors.binance_connector import BinanceConnector
|
||||
from ..storage.timescale_manager import TimescaleManager
|
||||
from ..caching.redis_manager import RedisManager
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BenchmarkResult:
|
||||
"""Benchmark result data structure"""
|
||||
name: str
|
||||
duration_ms: float
|
||||
operations_per_second: float
|
||||
memory_usage_mb: float
|
||||
cpu_usage_percent: float
|
||||
timestamp: datetime
|
||||
metadata: Dict[str, Any] = None
|
||||
|
||||
|
||||
class BenchmarkRunner:
|
||||
"""Benchmark execution and result management"""
|
||||
|
||||
def __init__(self, results_file: str = "benchmark_results.json"):
|
||||
self.results_file = Path(results_file)
|
||||
self.results: List[BenchmarkResult] = []
|
||||
self.load_historical_results()
|
||||
|
||||
def load_historical_results(self):
|
||||
"""Load historical benchmark results"""
|
||||
if self.results_file.exists():
|
||||
try:
|
||||
with open(self.results_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
for item in data:
|
||||
result = BenchmarkResult(
|
||||
name=item['name'],
|
||||
duration_ms=item['duration_ms'],
|
||||
operations_per_second=item['operations_per_second'],
|
||||
memory_usage_mb=item['memory_usage_mb'],
|
||||
cpu_usage_percent=item['cpu_usage_percent'],
|
||||
timestamp=datetime.fromisoformat(item['timestamp']),
|
||||
metadata=item.get('metadata', {})
|
||||
)
|
||||
self.results.append(result)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not load historical results: {e}")
|
||||
|
||||
def save_results(self):
|
||||
"""Save benchmark results to file"""
|
||||
try:
|
||||
data = []
|
||||
for result in self.results:
|
||||
data.append({
|
||||
'name': result.name,
|
||||
'duration_ms': result.duration_ms,
|
||||
'operations_per_second': result.operations_per_second,
|
||||
'memory_usage_mb': result.memory_usage_mb,
|
||||
'cpu_usage_percent': result.cpu_usage_percent,
|
||||
'timestamp': result.timestamp.isoformat(),
|
||||
'metadata': result.metadata or {}
|
||||
})
|
||||
|
||||
with open(self.results_file, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Could not save benchmark results: {e}")
|
||||
|
||||
def run_benchmark(self, name: str, func, iterations: int = 1000,
|
||||
warmup: int = 100) -> BenchmarkResult:
|
||||
"""Run a benchmark function"""
|
||||
import psutil
|
||||
|
||||
process = psutil.Process()
|
||||
|
||||
# Warmup
|
||||
for _ in range(warmup):
|
||||
func()
|
||||
|
||||
# Collect baseline metrics
|
||||
initial_memory = process.memory_info().rss / 1024 / 1024
|
||||
initial_cpu = process.cpu_percent()
|
||||
|
||||
# Run benchmark
|
||||
start_time = time.perf_counter()
|
||||
|
||||
for _ in range(iterations):
|
||||
func()
|
||||
|
||||
end_time =
|
||||
Reference in New Issue
Block a user