304 lines
11 KiB
Python
304 lines
11 KiB
Python
"""
|
|
Tests for data processing components.
|
|
"""
|
|
|
|
import pytest
|
|
from datetime import datetime, timezone
|
|
from ..processing.data_processor import StandardDataProcessor
|
|
from ..processing.quality_checker import DataQualityChecker
|
|
from ..processing.anomaly_detector import AnomalyDetector
|
|
from ..processing.metrics_calculator import MetricsCalculator
|
|
from ..models.core import OrderBookSnapshot, TradeEvent, PriceLevel
|
|
|
|
|
|
@pytest.fixture
|
|
def data_processor():
|
|
"""Create data processor for testing"""
|
|
return StandardDataProcessor()
|
|
|
|
|
|
@pytest.fixture
|
|
def quality_checker():
|
|
"""Create quality checker for testing"""
|
|
return DataQualityChecker()
|
|
|
|
|
|
@pytest.fixture
|
|
def anomaly_detector():
|
|
"""Create anomaly detector for testing"""
|
|
return AnomalyDetector()
|
|
|
|
|
|
@pytest.fixture
|
|
def metrics_calculator():
|
|
"""Create metrics calculator for testing"""
|
|
return MetricsCalculator()
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_orderbook():
|
|
"""Create sample order book for testing"""
|
|
return OrderBookSnapshot(
|
|
symbol="BTCUSDT",
|
|
exchange="binance",
|
|
timestamp=datetime.now(timezone.utc),
|
|
bids=[
|
|
PriceLevel(price=50000.0, size=1.5),
|
|
PriceLevel(price=49999.0, size=2.0),
|
|
PriceLevel(price=49998.0, size=1.0)
|
|
],
|
|
asks=[
|
|
PriceLevel(price=50001.0, size=1.0),
|
|
PriceLevel(price=50002.0, size=1.5),
|
|
PriceLevel(price=50003.0, size=2.0)
|
|
]
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def sample_trade():
|
|
"""Create sample trade for testing"""
|
|
return TradeEvent(
|
|
symbol="BTCUSDT",
|
|
exchange="binance",
|
|
timestamp=datetime.now(timezone.utc),
|
|
price=50000.5,
|
|
size=0.1,
|
|
side="buy",
|
|
trade_id="test_trade_123"
|
|
)
|
|
|
|
|
|
class TestDataQualityChecker:
|
|
"""Test cases for DataQualityChecker"""
|
|
|
|
def test_orderbook_quality_check(self, quality_checker, sample_orderbook):
|
|
"""Test order book quality checking"""
|
|
quality_score, issues = quality_checker.check_orderbook_quality(sample_orderbook)
|
|
|
|
assert 0.0 <= quality_score <= 1.0
|
|
assert isinstance(issues, list)
|
|
|
|
# Good order book should have high quality score
|
|
assert quality_score > 0.8
|
|
|
|
def test_trade_quality_check(self, quality_checker, sample_trade):
|
|
"""Test trade quality checking"""
|
|
quality_score, issues = quality_checker.check_trade_quality(sample_trade)
|
|
|
|
assert 0.0 <= quality_score <= 1.0
|
|
assert isinstance(issues, list)
|
|
|
|
# Good trade should have high quality score
|
|
assert quality_score > 0.8
|
|
|
|
def test_invalid_orderbook_detection(self, quality_checker):
|
|
"""Test detection of invalid order book"""
|
|
# Create invalid order book with crossed spread
|
|
invalid_orderbook = OrderBookSnapshot(
|
|
symbol="BTCUSDT",
|
|
exchange="binance",
|
|
timestamp=datetime.now(timezone.utc),
|
|
bids=[PriceLevel(price=50002.0, size=1.0)], # Bid higher than ask
|
|
asks=[PriceLevel(price=50001.0, size=1.0)] # Ask lower than bid
|
|
)
|
|
|
|
quality_score, issues = quality_checker.check_orderbook_quality(invalid_orderbook)
|
|
|
|
assert quality_score < 0.8
|
|
assert any("crossed book" in issue.lower() for issue in issues)
|
|
|
|
|
|
class TestAnomalyDetector:
|
|
"""Test cases for AnomalyDetector"""
|
|
|
|
def test_orderbook_anomaly_detection(self, anomaly_detector, sample_orderbook):
|
|
"""Test order book anomaly detection"""
|
|
# First few order books should not trigger anomalies
|
|
for _ in range(5):
|
|
anomalies = anomaly_detector.detect_orderbook_anomalies(sample_orderbook)
|
|
assert isinstance(anomalies, list)
|
|
|
|
def test_trade_anomaly_detection(self, anomaly_detector, sample_trade):
|
|
"""Test trade anomaly detection"""
|
|
# First few trades should not trigger anomalies
|
|
for _ in range(5):
|
|
anomalies = anomaly_detector.detect_trade_anomalies(sample_trade)
|
|
assert isinstance(anomalies, list)
|
|
|
|
def test_price_spike_detection(self, anomaly_detector):
|
|
"""Test price spike detection"""
|
|
# Create normal order books
|
|
for i in range(20):
|
|
normal_orderbook = OrderBookSnapshot(
|
|
symbol="BTCUSDT",
|
|
exchange="binance",
|
|
timestamp=datetime.now(timezone.utc),
|
|
bids=[PriceLevel(price=50000.0 + i, size=1.0)],
|
|
asks=[PriceLevel(price=50001.0 + i, size=1.0)]
|
|
)
|
|
anomaly_detector.detect_orderbook_anomalies(normal_orderbook)
|
|
|
|
# Create order book with price spike
|
|
spike_orderbook = OrderBookSnapshot(
|
|
symbol="BTCUSDT",
|
|
exchange="binance",
|
|
timestamp=datetime.now(timezone.utc),
|
|
bids=[PriceLevel(price=60000.0, size=1.0)], # 20% spike
|
|
asks=[PriceLevel(price=60001.0, size=1.0)]
|
|
)
|
|
|
|
anomalies = anomaly_detector.detect_orderbook_anomalies(spike_orderbook)
|
|
assert len(anomalies) > 0
|
|
assert any("spike" in anomaly.lower() for anomaly in anomalies)
|
|
|
|
|
|
class TestMetricsCalculator:
|
|
"""Test cases for MetricsCalculator"""
|
|
|
|
def test_orderbook_metrics_calculation(self, metrics_calculator, sample_orderbook):
|
|
"""Test order book metrics calculation"""
|
|
metrics = metrics_calculator.calculate_orderbook_metrics(sample_orderbook)
|
|
|
|
assert metrics.symbol == "BTCUSDT"
|
|
assert metrics.exchange == "binance"
|
|
assert metrics.mid_price == 50000.5 # (50000 + 50001) / 2
|
|
assert metrics.spread == 1.0 # 50001 - 50000
|
|
assert metrics.spread_percentage > 0
|
|
assert metrics.bid_volume == 4.5 # 1.5 + 2.0 + 1.0
|
|
assert metrics.ask_volume == 4.5 # 1.0 + 1.5 + 2.0
|
|
assert metrics.volume_imbalance == 0.0 # Equal volumes
|
|
|
|
def test_imbalance_metrics_calculation(self, metrics_calculator, sample_orderbook):
|
|
"""Test imbalance metrics calculation"""
|
|
imbalance = metrics_calculator.calculate_imbalance_metrics(sample_orderbook)
|
|
|
|
assert imbalance.symbol == "BTCUSDT"
|
|
assert -1.0 <= imbalance.volume_imbalance <= 1.0
|
|
assert -1.0 <= imbalance.price_imbalance <= 1.0
|
|
assert -1.0 <= imbalance.depth_imbalance <= 1.0
|
|
assert -1.0 <= imbalance.momentum_score <= 1.0
|
|
|
|
def test_liquidity_score_calculation(self, metrics_calculator, sample_orderbook):
|
|
"""Test liquidity score calculation"""
|
|
liquidity_score = metrics_calculator.calculate_liquidity_score(sample_orderbook)
|
|
|
|
assert 0.0 <= liquidity_score <= 1.0
|
|
assert liquidity_score > 0.5 # Good order book should have decent liquidity
|
|
|
|
|
|
class TestStandardDataProcessor:
|
|
"""Test cases for StandardDataProcessor"""
|
|
|
|
def test_data_validation(self, data_processor, sample_orderbook, sample_trade):
|
|
"""Test data validation"""
|
|
# Valid data should pass validation
|
|
assert data_processor.validate_data(sample_orderbook) is True
|
|
assert data_processor.validate_data(sample_trade) is True
|
|
|
|
def test_metrics_calculation(self, data_processor, sample_orderbook):
|
|
"""Test metrics calculation through processor"""
|
|
metrics = data_processor.calculate_metrics(sample_orderbook)
|
|
|
|
assert metrics.symbol == "BTCUSDT"
|
|
assert metrics.mid_price > 0
|
|
assert metrics.spread > 0
|
|
|
|
def test_anomaly_detection(self, data_processor, sample_orderbook, sample_trade):
|
|
"""Test anomaly detection through processor"""
|
|
orderbook_anomalies = data_processor.detect_anomalies(sample_orderbook)
|
|
trade_anomalies = data_processor.detect_anomalies(sample_trade)
|
|
|
|
assert isinstance(orderbook_anomalies, list)
|
|
assert isinstance(trade_anomalies, list)
|
|
|
|
def test_data_filtering(self, data_processor, sample_orderbook, sample_trade):
|
|
"""Test data filtering"""
|
|
# Test symbol filter
|
|
criteria = {'symbols': ['BTCUSDT']}
|
|
assert data_processor.filter_data(sample_orderbook, criteria) is True
|
|
assert data_processor.filter_data(sample_trade, criteria) is True
|
|
|
|
criteria = {'symbols': ['ETHUSDT']}
|
|
assert data_processor.filter_data(sample_orderbook, criteria) is False
|
|
assert data_processor.filter_data(sample_trade, criteria) is False
|
|
|
|
# Test price range filter
|
|
criteria = {'price_range': (40000, 60000)}
|
|
assert data_processor.filter_data(sample_orderbook, criteria) is True
|
|
assert data_processor.filter_data(sample_trade, criteria) is True
|
|
|
|
criteria = {'price_range': (60000, 70000)}
|
|
assert data_processor.filter_data(sample_orderbook, criteria) is False
|
|
assert data_processor.filter_data(sample_trade, criteria) is False
|
|
|
|
def test_data_enrichment(self, data_processor, sample_orderbook, sample_trade):
|
|
"""Test data enrichment"""
|
|
orderbook_enriched = data_processor.enrich_data(sample_orderbook)
|
|
trade_enriched = data_processor.enrich_data(sample_trade)
|
|
|
|
# Check enriched data structure
|
|
assert 'original_data' in orderbook_enriched
|
|
assert 'quality_score' in orderbook_enriched
|
|
assert 'anomalies' in orderbook_enriched
|
|
assert 'processing_timestamp' in orderbook_enriched
|
|
|
|
assert 'original_data' in trade_enriched
|
|
assert 'quality_score' in trade_enriched
|
|
assert 'anomalies' in trade_enriched
|
|
assert 'trade_value' in trade_enriched
|
|
|
|
def test_quality_score_calculation(self, data_processor, sample_orderbook, sample_trade):
|
|
"""Test quality score calculation"""
|
|
orderbook_score = data_processor.get_data_quality_score(sample_orderbook)
|
|
trade_score = data_processor.get_data_quality_score(sample_trade)
|
|
|
|
assert 0.0 <= orderbook_score <= 1.0
|
|
assert 0.0 <= trade_score <= 1.0
|
|
|
|
# Good data should have high quality scores
|
|
assert orderbook_score > 0.8
|
|
assert trade_score > 0.8
|
|
|
|
def test_processing_stats(self, data_processor, sample_orderbook, sample_trade):
|
|
"""Test processing statistics"""
|
|
# Process some data
|
|
data_processor.validate_data(sample_orderbook)
|
|
data_processor.validate_data(sample_trade)
|
|
|
|
stats = data_processor.get_processing_stats()
|
|
|
|
assert 'processed_orderbooks' in stats
|
|
assert 'processed_trades' in stats
|
|
assert 'quality_failures' in stats
|
|
assert 'anomalies_detected' in stats
|
|
assert stats['processed_orderbooks'] >= 1
|
|
assert stats['processed_trades'] >= 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Run simple tests
|
|
processor = StandardDataProcessor()
|
|
|
|
# Test with sample data
|
|
orderbook = OrderBookSnapshot(
|
|
symbol="BTCUSDT",
|
|
exchange="test",
|
|
timestamp=datetime.now(timezone.utc),
|
|
bids=[PriceLevel(price=50000.0, size=1.0)],
|
|
asks=[PriceLevel(price=50001.0, size=1.0)]
|
|
)
|
|
|
|
# Test validation
|
|
is_valid = processor.validate_data(orderbook)
|
|
print(f"Order book validation: {'PASSED' if is_valid else 'FAILED'}")
|
|
|
|
# Test metrics
|
|
metrics = processor.calculate_metrics(orderbook)
|
|
print(f"Metrics calculation: mid_price={metrics.mid_price}, spread={metrics.spread}")
|
|
|
|
# Test quality score
|
|
quality_score = processor.get_data_quality_score(orderbook)
|
|
print(f"Quality score: {quality_score:.2f}")
|
|
|
|
print("Simple data processor test completed") |