bucket aggregation

2025-08-04 17:28:55 +03:00
parent 504736c0f7
commit de77b0afa8
10 changed files with 2592 additions and 0 deletions
--- a/COBY/tests/test_binance_connector.py
+++ b/COBY/tests/test_binance_connector.py
@ -0,0 +1,341 @@
+"""
+Tests for Binance exchange connector.
+"""
+
+import pytest
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch
+from datetime import datetime, timezone
+
+from ..connectors.binance_connector import BinanceConnector
+from ..models.core import OrderBookSnapshot, TradeEvent, PriceLevel
+
+
+@pytest.fixture
+def binance_connector():
+    """Create Binance connector for testing"""
+    return BinanceConnector()
+
+
+@pytest.fixture
+def sample_binance_orderbook_data():
+    """Sample Binance order book data"""
+    return {
+        "lastUpdateId": 1027024,
+        "bids": [
+            ["4.00000000", "431.00000000"],
+            ["3.99000000", "9.00000000"]
+        ],
+        "asks": [
+            ["4.00000200", "12.00000000"],
+            ["4.01000000", "18.00000000"]
+        ]
+    }
+
+
+@pytest.fixture
+def sample_binance_depth_update():
+    """Sample Binance depth update message"""
+    return {
+        "e": "depthUpdate",
+        "E": 1672515782136,
+        "s": "BTCUSDT",
+        "U": 157,
+        "u": 160,
+        "b": [
+            ["50000.00", "0.25"],
+            ["49999.00", "0.50"]
+        ],
+        "a": [
+            ["50001.00", "0.30"],
+            ["50002.00", "0.40"]
+        ]
+    }
+
+
+@pytest.fixture
+def sample_binance_trade_update():
+    """Sample Binance trade update message"""
+    return {
+        "e": "trade",
+        "E": 1672515782136,
+        "s": "BTCUSDT",
+        "t": 12345,
+        "p": "50000.50",
+        "q": "0.10",
+        "b": 88,
+        "a": 50,
+        "T": 1672515782134,
+        "m": False,
+        "M": True
+    }
+
+
+class TestBinanceConnector:
+    """Test cases for BinanceConnector"""
+    
+    def test_initialization(self, binance_connector):
+        """Test connector initialization"""
+        assert binance_connector.exchange_name == "binance"
+        assert binance_connector.websocket_url == BinanceConnector.WEBSOCKET_URL
+        assert len(binance_connector.message_handlers) >= 3
+        assert binance_connector.stream_id == 1
+        assert binance_connector.active_streams == []
+    
+    def test_normalize_symbol(self, binance_connector):
+        """Test symbol normalization"""
+        # Test standard format
+        assert binance_connector.normalize_symbol("BTCUSDT") == "BTCUSDT"
+        
+        # Test with separators
+        assert binance_connector.normalize_symbol("BTC-USDT") == "BTCUSDT"
+        assert binance_connector.normalize_symbol("BTC/USDT") == "BTCUSDT"
+        
+        # Test lowercase
+        assert binance_connector.normalize_symbol("btcusdt") == "BTCUSDT"
+        
+        # Test invalid symbol
+        with pytest.raises(Exception):
+            binance_connector.normalize_symbol("")
+    
+    def test_get_message_type(self, binance_connector):
+        """Test message type detection"""
+        # Test depth update
+        depth_msg = {"e": "depthUpdate", "s": "BTCUSDT"}
+        assert binance_connector._get_message_type(depth_msg) == "depthUpdate"
+        
+        # Test trade update
+        trade_msg = {"e": "trade", "s": "BTCUSDT"}
+        assert binance_connector._get_message_type(trade_msg) == "trade"
+        
+        # Test error message
+        error_msg = {"error": {"code": -1121, "msg": "Invalid symbol"}}
+        assert binance_connector._get_message_type(error_msg) == "error"
+        
+        # Test unknown message
+        unknown_msg = {"data": "something"}
+        assert binance_connector._get_message_type(unknown_msg) == "unknown"
+    
+    def test_parse_orderbook_snapshot(self, binance_connector, sample_binance_orderbook_data):
+        """Test order book snapshot parsing"""
+        orderbook = binance_connector._parse_orderbook_snapshot(
+            sample_binance_orderbook_data, 
+            "BTCUSDT"
+        )
+        
+        assert isinstance(orderbook, OrderBookSnapshot)
+        assert orderbook.symbol == "BTCUSDT"
+        assert orderbook.exchange == "binance"
+        assert len(orderbook.bids) == 2
+        assert len(orderbook.asks) == 2
+        assert orderbook.sequence_id == 1027024
+        
+        # Check bid data
+        assert orderbook.bids[0].price == 4.0
+        assert orderbook.bids[0].size == 431.0
+        
+        # Check ask data
+        assert orderbook.asks[0].price == 4.000002
+        assert orderbook.asks[0].size == 12.0
+    
+    @pytest.mark.asyncio
+    async def test_handle_orderbook_update(self, binance_connector, sample_binance_depth_update):
+        """Test order book update handling"""
+        # Mock callback
+        callback_called = False
+        received_data = None
+        
+        def mock_callback(data):
+            nonlocal callback_called, received_data
+            callback_called = True
+            received_data = data
+        
+        binance_connector.add_data_callback(mock_callback)
+        
+        # Handle update
+        await binance_connector._handle_orderbook_update(sample_binance_depth_update)
+        
+        # Verify callback was called
+        assert callback_called
+        assert isinstance(received_data, OrderBookSnapshot)
+        assert received_data.symbol == "BTCUSDT"
+        assert received_data.exchange == "binance"
+        assert len(received_data.bids) == 2
+        assert len(received_data.asks) == 2
+    
+    @pytest.mark.asyncio
+    async def test_handle_trade_update(self, binance_connector, sample_binance_trade_update):
+        """Test trade update handling"""
+        # Mock callback
+        callback_called = False
+        received_data = None
+        
+        def mock_callback(data):
+            nonlocal callback_called, received_data
+            callback_called = True
+            received_data = data
+        
+        binance_connector.add_data_callback(mock_callback)
+        
+        # Handle update
+        await binance_connector._handle_trade_update(sample_binance_trade_update)
+        
+        # Verify callback was called
+        assert callback_called
+        assert isinstance(received_data, TradeEvent)
+        assert received_data.symbol == "BTCUSDT"
+        assert received_data.exchange == "binance"
+        assert received_data.price == 50000.50
+        assert received_data.size == 0.10
+        assert received_data.side == "buy"  # m=False means buyer is not maker
+        assert received_data.trade_id == "12345"
+    
+    @pytest.mark.asyncio
+    async def test_subscribe_orderbook(self, binance_connector):
+        """Test order book subscription"""
+        # Mock WebSocket send
+        binance_connector._send_message = AsyncMock(return_value=True)
+        
+        # Subscribe
+        await binance_connector.subscribe_orderbook("BTCUSDT")
+        
+        # Verify subscription was sent
+        binance_connector._send_message.assert_called_once()
+        call_args = binance_connector._send_message.call_args[0][0]
+        
+        assert call_args["method"] == "SUBSCRIBE"
+        assert "btcusdt@depth@100ms" in call_args["params"]
+        assert call_args["id"] == 1
+        
+        # Verify tracking
+        assert "BTCUSDT" in binance_connector.subscriptions
+        assert "orderbook" in binance_connector.subscriptions["BTCUSDT"]
+        assert "btcusdt@depth@100ms" in binance_connector.active_streams
+        assert binance_connector.stream_id == 2
+    
+    @pytest.mark.asyncio
+    async def test_subscribe_trades(self, binance_connector):
+        """Test trade subscription"""
+        # Mock WebSocket send
+        binance_connector._send_message = AsyncMock(return_value=True)
+        
+        # Subscribe
+        await binance_connector.subscribe_trades("ETHUSDT")
+        
+        # Verify subscription was sent
+        binance_connector._send_message.assert_called_once()
+        call_args = binance_connector._send_message.call_args[0][0]
+        
+        assert call_args["method"] == "SUBSCRIBE"
+        assert "ethusdt@trade" in call_args["params"]
+        assert call_args["id"] == 1
+        
+        # Verify tracking
+        assert "ETHUSDT" in binance_connector.subscriptions
+        assert "trades" in binance_connector.subscriptions["ETHUSDT"]
+        assert "ethusdt@trade" in binance_connector.active_streams
+    
+    @pytest.mark.asyncio
+    async def test_unsubscribe_orderbook(self, binance_connector):
+        """Test order book unsubscription"""
+        # Setup initial subscription
+        binance_connector.subscriptions["BTCUSDT"] = ["orderbook"]
+        binance_connector.active_streams.append("btcusdt@depth@100ms")
+        
+        # Mock WebSocket send
+        binance_connector._send_message = AsyncMock(return_value=True)
+        
+        # Unsubscribe
+        await binance_connector.unsubscribe_orderbook("BTCUSDT")
+        
+        # Verify unsubscription was sent
+        binance_connector._send_message.assert_called_once()
+        call_args = binance_connector._send_message.call_args[0][0]
+        
+        assert call_args["method"] == "UNSUBSCRIBE"
+        assert "btcusdt@depth@100ms" in call_args["params"]
+        
+        # Verify tracking removal
+        assert "BTCUSDT" not in binance_connector.subscriptions
+        assert "btcusdt@depth@100ms" not in binance_connector.active_streams
+    
+    @pytest.mark.asyncio
+    @patch('aiohttp.ClientSession.get')
+    async def test_get_symbols(self, mock_get, binance_connector):
+        """Test getting available symbols"""
+        # Mock API response
+        mock_response = AsyncMock()
+        mock_response.status = 200
+        mock_response.json = AsyncMock(return_value={
+            "symbols": [
+                {"symbol": "BTCUSDT", "status": "TRADING"},
+                {"symbol": "ETHUSDT", "status": "TRADING"},
+                {"symbol": "ADAUSDT", "status": "BREAK"}  # Should be filtered out
+            ]
+        })
+        mock_get.return_value.__aenter__.return_value = mock_response
+        
+        # Get symbols
+        symbols = await binance_connector.get_symbols()
+        
+        # Verify results
+        assert len(symbols) == 2
+        assert "BTCUSDT" in symbols
+        assert "ETHUSDT" in symbols
+        assert "ADAUSDT" not in symbols  # Filtered out due to status
+    
+    @pytest.mark.asyncio
+    @patch('aiohttp.ClientSession.get')
+    async def test_get_orderbook_snapshot(self, mock_get, binance_connector, sample_binance_orderbook_data):
+        """Test getting order book snapshot"""
+        # Mock API response
+        mock_response = AsyncMock()
+        mock_response.status = 200
+        mock_response.json = AsyncMock(return_value=sample_binance_orderbook_data)
+        mock_get.return_value.__aenter__.return_value = mock_response
+        
+        # Get order book snapshot
+        orderbook = await binance_connector.get_orderbook_snapshot("BTCUSDT", depth=20)
+        
+        # Verify results
+        assert isinstance(orderbook, OrderBookSnapshot)
+        assert orderbook.symbol == "BTCUSDT"
+        assert orderbook.exchange == "binance"
+        assert len(orderbook.bids) == 2
+        assert len(orderbook.asks) == 2
+    
+    def test_get_binance_stats(self, binance_connector):
+        """Test getting Binance-specific statistics"""
+        # Add some test data
+        binance_connector.active_streams = ["btcusdt@depth@100ms", "ethusdt@trade"]
+        binance_connector.stream_id = 5
+        
+        stats = binance_connector.get_binance_stats()
+        
+        # Verify Binance-specific stats
+        assert stats['active_streams'] == 2
+        assert len(stats['stream_list']) == 2
+        assert stats['next_stream_id'] == 5
+        
+        # Verify base stats are included
+        assert 'exchange' in stats
+        assert 'connection_status' in stats
+        assert 'message_count' in stats
+
+
+if __name__ == "__main__":
+    # Run a simple test
+    async def simple_test():
+        connector = BinanceConnector()
+        
+        # Test symbol normalization
+        normalized = connector.normalize_symbol("BTC-USDT")
+        print(f"Symbol normalization: BTC-USDT -> {normalized}")
+        
+        # Test message type detection
+        msg_type = connector._get_message_type({"e": "depthUpdate"})
+        print(f"Message type detection: {msg_type}")
+        
+        print("Simple Binance connector test completed")
+    
+    asyncio.run(simple_test())
--- a/COBY/tests/test_data_processor.py
+++ b/COBY/tests/test_data_processor.py
@ -0,0 +1,304 @@
+"""
+Tests for data processing components.
+"""
+
+import pytest
+from datetime import datetime, timezone
+from ..processing.data_processor import StandardDataProcessor
+from ..processing.quality_checker import DataQualityChecker
+from ..processing.anomaly_detector import AnomalyDetector
+from ..processing.metrics_calculator import MetricsCalculator
+from ..models.core import OrderBookSnapshot, TradeEvent, PriceLevel
+
+
+@pytest.fixture
+def data_processor():
+    """Create data processor for testing"""
+    return StandardDataProcessor()
+
+
+@pytest.fixture
+def quality_checker():
+    """Create quality checker for testing"""
+    return DataQualityChecker()
+
+
+@pytest.fixture
+def anomaly_detector():
+    """Create anomaly detector for testing"""
+    return AnomalyDetector()
+
+
+@pytest.fixture
+def metrics_calculator():
+    """Create metrics calculator for testing"""
+    return MetricsCalculator()
+
+
+@pytest.fixture
+def sample_orderbook():
+    """Create sample order book for testing"""
+    return OrderBookSnapshot(
+        symbol="BTCUSDT",
+        exchange="binance",
+        timestamp=datetime.now(timezone.utc),
+        bids=[
+            PriceLevel(price=50000.0, size=1.5),
+            PriceLevel(price=49999.0, size=2.0),
+            PriceLevel(price=49998.0, size=1.0)
+        ],
+        asks=[
+            PriceLevel(price=50001.0, size=1.0),
+            PriceLevel(price=50002.0, size=1.5),
+            PriceLevel(price=50003.0, size=2.0)
+        ]
+    )
+
+
+@pytest.fixture
+def sample_trade():
+    """Create sample trade for testing"""
+    return TradeEvent(
+        symbol="BTCUSDT",
+        exchange="binance",
+        timestamp=datetime.now(timezone.utc),
+        price=50000.5,
+        size=0.1,
+        side="buy",
+        trade_id="test_trade_123"
+    )
+
+
+class TestDataQualityChecker:
+    """Test cases for DataQualityChecker"""
+    
+    def test_orderbook_quality_check(self, quality_checker, sample_orderbook):
+        """Test order book quality checking"""
+        quality_score, issues = quality_checker.check_orderbook_quality(sample_orderbook)
+        
+        assert 0.0 <= quality_score <= 1.0
+        assert isinstance(issues, list)
+        
+        # Good order book should have high quality score
+        assert quality_score > 0.8
+    
+    def test_trade_quality_check(self, quality_checker, sample_trade):
+        """Test trade quality checking"""
+        quality_score, issues = quality_checker.check_trade_quality(sample_trade)
+        
+        assert 0.0 <= quality_score <= 1.0
+        assert isinstance(issues, list)
+        
+        # Good trade should have high quality score
+        assert quality_score > 0.8
+    
+    def test_invalid_orderbook_detection(self, quality_checker):
+        """Test detection of invalid order book"""
+        # Create invalid order book with crossed spread
+        invalid_orderbook = OrderBookSnapshot(
+            symbol="BTCUSDT",
+            exchange="binance",
+            timestamp=datetime.now(timezone.utc),
+            bids=[PriceLevel(price=50002.0, size=1.0)],  # Bid higher than ask
+            asks=[PriceLevel(price=50001.0, size=1.0)]   # Ask lower than bid
+        )
+        
+        quality_score, issues = quality_checker.check_orderbook_quality(invalid_orderbook)
+        
+        assert quality_score < 0.8
+        assert any("crossed book" in issue.lower() for issue in issues)
+
+
+class TestAnomalyDetector:
+    """Test cases for AnomalyDetector"""
+    
+    def test_orderbook_anomaly_detection(self, anomaly_detector, sample_orderbook):
+        """Test order book anomaly detection"""
+        # First few order books should not trigger anomalies
+        for _ in range(5):
+            anomalies = anomaly_detector.detect_orderbook_anomalies(sample_orderbook)
+            assert isinstance(anomalies, list)
+    
+    def test_trade_anomaly_detection(self, anomaly_detector, sample_trade):
+        """Test trade anomaly detection"""
+        # First few trades should not trigger anomalies
+        for _ in range(5):
+            anomalies = anomaly_detector.detect_trade_anomalies(sample_trade)
+            assert isinstance(anomalies, list)
+    
+    def test_price_spike_detection(self, anomaly_detector):
+        """Test price spike detection"""
+        # Create normal order books
+        for i in range(20):
+            normal_orderbook = OrderBookSnapshot(
+                symbol="BTCUSDT",
+                exchange="binance",
+                timestamp=datetime.now(timezone.utc),
+                bids=[PriceLevel(price=50000.0 + i, size=1.0)],
+                asks=[PriceLevel(price=50001.0 + i, size=1.0)]
+            )
+            anomaly_detector.detect_orderbook_anomalies(normal_orderbook)
+        
+        # Create order book with price spike
+        spike_orderbook = OrderBookSnapshot(
+            symbol="BTCUSDT",
+            exchange="binance",
+            timestamp=datetime.now(timezone.utc),
+            bids=[PriceLevel(price=60000.0, size=1.0)],  # 20% spike
+            asks=[PriceLevel(price=60001.0, size=1.0)]
+        )
+        
+        anomalies = anomaly_detector.detect_orderbook_anomalies(spike_orderbook)
+        assert len(anomalies) > 0
+        assert any("spike" in anomaly.lower() for anomaly in anomalies)
+
+
+class TestMetricsCalculator:
+    """Test cases for MetricsCalculator"""
+    
+    def test_orderbook_metrics_calculation(self, metrics_calculator, sample_orderbook):
+        """Test order book metrics calculation"""
+        metrics = metrics_calculator.calculate_orderbook_metrics(sample_orderbook)
+        
+        assert metrics.symbol == "BTCUSDT"
+        assert metrics.exchange == "binance"
+        assert metrics.mid_price == 50000.5  # (50000 + 50001) / 2
+        assert metrics.spread == 1.0  # 50001 - 50000
+        assert metrics.spread_percentage > 0
+        assert metrics.bid_volume == 4.5  # 1.5 + 2.0 + 1.0
+        assert metrics.ask_volume == 4.5  # 1.0 + 1.5 + 2.0
+        assert metrics.volume_imbalance == 0.0  # Equal volumes
+    
+    def test_imbalance_metrics_calculation(self, metrics_calculator, sample_orderbook):
+        """Test imbalance metrics calculation"""
+        imbalance = metrics_calculator.calculate_imbalance_metrics(sample_orderbook)
+        
+        assert imbalance.symbol == "BTCUSDT"
+        assert -1.0 <= imbalance.volume_imbalance <= 1.0
+        assert -1.0 <= imbalance.price_imbalance <= 1.0
+        assert -1.0 <= imbalance.depth_imbalance <= 1.0
+        assert -1.0 <= imbalance.momentum_score <= 1.0
+    
+    def test_liquidity_score_calculation(self, metrics_calculator, sample_orderbook):
+        """Test liquidity score calculation"""
+        liquidity_score = metrics_calculator.calculate_liquidity_score(sample_orderbook)
+        
+        assert 0.0 <= liquidity_score <= 1.0
+        assert liquidity_score > 0.5  # Good order book should have decent liquidity
+
+
+class TestStandardDataProcessor:
+    """Test cases for StandardDataProcessor"""
+    
+    def test_data_validation(self, data_processor, sample_orderbook, sample_trade):
+        """Test data validation"""
+        # Valid data should pass validation
+        assert data_processor.validate_data(sample_orderbook) is True
+        assert data_processor.validate_data(sample_trade) is True
+    
+    def test_metrics_calculation(self, data_processor, sample_orderbook):
+        """Test metrics calculation through processor"""
+        metrics = data_processor.calculate_metrics(sample_orderbook)
+        
+        assert metrics.symbol == "BTCUSDT"
+        assert metrics.mid_price > 0
+        assert metrics.spread > 0
+    
+    def test_anomaly_detection(self, data_processor, sample_orderbook, sample_trade):
+        """Test anomaly detection through processor"""
+        orderbook_anomalies = data_processor.detect_anomalies(sample_orderbook)
+        trade_anomalies = data_processor.detect_anomalies(sample_trade)
+        
+        assert isinstance(orderbook_anomalies, list)
+        assert isinstance(trade_anomalies, list)
+    
+    def test_data_filtering(self, data_processor, sample_orderbook, sample_trade):
+        """Test data filtering"""
+        # Test symbol filter
+        criteria = {'symbols': ['BTCUSDT']}
+        assert data_processor.filter_data(sample_orderbook, criteria) is True
+        assert data_processor.filter_data(sample_trade, criteria) is True
+        
+        criteria = {'symbols': ['ETHUSDT']}
+        assert data_processor.filter_data(sample_orderbook, criteria) is False
+        assert data_processor.filter_data(sample_trade, criteria) is False
+        
+        # Test price range filter
+        criteria = {'price_range': (40000, 60000)}
+        assert data_processor.filter_data(sample_orderbook, criteria) is True
+        assert data_processor.filter_data(sample_trade, criteria) is True
+        
+        criteria = {'price_range': (60000, 70000)}
+        assert data_processor.filter_data(sample_orderbook, criteria) is False
+        assert data_processor.filter_data(sample_trade, criteria) is False
+    
+    def test_data_enrichment(self, data_processor, sample_orderbook, sample_trade):
+        """Test data enrichment"""
+        orderbook_enriched = data_processor.enrich_data(sample_orderbook)
+        trade_enriched = data_processor.enrich_data(sample_trade)
+        
+        # Check enriched data structure
+        assert 'original_data' in orderbook_enriched
+        assert 'quality_score' in orderbook_enriched
+        assert 'anomalies' in orderbook_enriched
+        assert 'processing_timestamp' in orderbook_enriched
+        
+        assert 'original_data' in trade_enriched
+        assert 'quality_score' in trade_enriched
+        assert 'anomalies' in trade_enriched
+        assert 'trade_value' in trade_enriched
+    
+    def test_quality_score_calculation(self, data_processor, sample_orderbook, sample_trade):
+        """Test quality score calculation"""
+        orderbook_score = data_processor.get_data_quality_score(sample_orderbook)
+        trade_score = data_processor.get_data_quality_score(sample_trade)
+        
+        assert 0.0 <= orderbook_score <= 1.0
+        assert 0.0 <= trade_score <= 1.0
+        
+        # Good data should have high quality scores
+        assert orderbook_score > 0.8
+        assert trade_score > 0.8
+    
+    def test_processing_stats(self, data_processor, sample_orderbook, sample_trade):
+        """Test processing statistics"""
+        # Process some data
+        data_processor.validate_data(sample_orderbook)
+        data_processor.validate_data(sample_trade)
+        
+        stats = data_processor.get_processing_stats()
+        
+        assert 'processed_orderbooks' in stats
+        assert 'processed_trades' in stats
+        assert 'quality_failures' in stats
+        assert 'anomalies_detected' in stats
+        assert stats['processed_orderbooks'] >= 1
+        assert stats['processed_trades'] >= 1
+
+
+if __name__ == "__main__":
+    # Run simple tests
+    processor = StandardDataProcessor()
+    
+    # Test with sample data
+    orderbook = OrderBookSnapshot(
+        symbol="BTCUSDT",
+        exchange="test",
+        timestamp=datetime.now(timezone.utc),
+        bids=[PriceLevel(price=50000.0, size=1.0)],
+        asks=[PriceLevel(price=50001.0, size=1.0)]
+    )
+    
+    # Test validation
+    is_valid = processor.validate_data(orderbook)
+    print(f"Order book validation: {'PASSED' if is_valid else 'FAILED'}")
+    
+    # Test metrics
+    metrics = processor.calculate_metrics(orderbook)
+    print(f"Metrics calculation: mid_price={metrics.mid_price}, spread={metrics.spread}")
+    
+    # Test quality score
+    quality_score = processor.get_data_quality_score(orderbook)
+    print(f"Quality score: {quality_score:.2f}")
+    
+    print("Simple data processor test completed")