305 lines
12 KiB
Python
305 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Test Enhanced Dashboard Integration with RL Training Pipeline
|
|
|
|
This script tests the integration between the dashboard and the enhanced RL training pipeline
|
|
to verify that:
|
|
1. Unified data stream is properly initialized
|
|
2. Dashboard receives training data from the enhanced pipeline
|
|
3. Data flows correctly between components
|
|
4. Enhanced RL training receives comprehensive data
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
# Configure logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.FileHandler('test_enhanced_dashboard_integration.log'),
|
|
logging.StreamHandler(sys.stdout)
|
|
]
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Import components
|
|
from core.config import get_config
|
|
from core.data_provider import DataProvider
|
|
from core.enhanced_orchestrator import EnhancedTradingOrchestrator
|
|
from core.unified_data_stream import UnifiedDataStream
|
|
from web.old_archived.scalping_dashboard import RealTimeScalpingDashboard
|
|
|
|
class EnhancedDashboardIntegrationTest:
|
|
"""Test enhanced dashboard integration with RL training pipeline"""
|
|
|
|
def __init__(self):
|
|
"""Initialize test components"""
|
|
self.config = get_config()
|
|
self.data_provider = None
|
|
self.orchestrator = None
|
|
self.unified_stream = None
|
|
self.dashboard = None
|
|
|
|
# Test results
|
|
self.test_results = {
|
|
'data_provider_init': False,
|
|
'orchestrator_init': False,
|
|
'unified_stream_init': False,
|
|
'dashboard_init': False,
|
|
'data_flow_test': False,
|
|
'training_integration_test': False,
|
|
'ui_data_test': False,
|
|
'stream_stats_test': False
|
|
}
|
|
|
|
logger.info("Enhanced Dashboard Integration Test initialized")
|
|
|
|
async def run_tests(self):
|
|
"""Run all integration tests"""
|
|
logger.info("Starting enhanced dashboard integration tests...")
|
|
|
|
try:
|
|
# Test 1: Initialize components
|
|
await self.test_component_initialization()
|
|
|
|
# Test 2: Test data flow
|
|
await self.test_data_flow()
|
|
|
|
# Test 3: Test training integration
|
|
await self.test_training_integration()
|
|
|
|
# Test 4: Test UI data flow
|
|
await self.test_ui_data_flow()
|
|
|
|
# Test 5: Test stream statistics
|
|
await self.test_stream_statistics()
|
|
|
|
# Generate test report
|
|
self.generate_test_report()
|
|
|
|
except Exception as e:
|
|
logger.error(f"Test execution failed: {e}")
|
|
raise
|
|
|
|
async def test_component_initialization(self):
|
|
"""Test component initialization"""
|
|
logger.info("Testing component initialization...")
|
|
|
|
try:
|
|
# Initialize data provider
|
|
self.data_provider = DataProvider(
|
|
symbols=['ETH/USDT', 'BTC/USDT'],
|
|
timeframes=['1s', '1m', '1h', '1d']
|
|
)
|
|
self.test_results['data_provider_init'] = True
|
|
logger.info("✓ Data provider initialized")
|
|
|
|
# Initialize orchestrator
|
|
self.orchestrator = EnhancedTradingOrchestrator(self.data_provider)
|
|
self.test_results['orchestrator_init'] = True
|
|
logger.info("✓ Enhanced orchestrator initialized")
|
|
|
|
# Initialize unified stream
|
|
self.unified_stream = UnifiedDataStream(self.data_provider, self.orchestrator)
|
|
self.test_results['unified_stream_init'] = True
|
|
logger.info("✓ Unified data stream initialized")
|
|
|
|
# Initialize dashboard
|
|
self.dashboard = RealTimeScalpingDashboard(
|
|
data_provider=self.data_provider,
|
|
orchestrator=self.orchestrator
|
|
)
|
|
self.test_results['dashboard_init'] = True
|
|
logger.info("✓ Dashboard initialized with unified stream integration")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Component initialization failed: {e}")
|
|
raise
|
|
|
|
async def test_data_flow(self):
|
|
"""Test data flow through unified stream"""
|
|
logger.info("Testing data flow through unified stream...")
|
|
|
|
try:
|
|
# Start unified streaming
|
|
await self.unified_stream.start_streaming()
|
|
|
|
# Wait for data collection
|
|
logger.info("Waiting for data collection...")
|
|
await asyncio.sleep(10)
|
|
|
|
# Check if data is flowing
|
|
stream_stats = self.unified_stream.get_stream_stats()
|
|
|
|
if stream_stats['tick_cache_size'] > 0:
|
|
logger.info(f"✓ Tick data flowing: {stream_stats['tick_cache_size']} ticks")
|
|
self.test_results['data_flow_test'] = True
|
|
else:
|
|
logger.warning("⚠ No tick data detected")
|
|
|
|
if stream_stats['one_second_bars_count'] > 0:
|
|
logger.info(f"✓ 1s bars generated: {stream_stats['one_second_bars_count']} bars")
|
|
else:
|
|
logger.warning("⚠ No 1s bars generated")
|
|
|
|
logger.info(f"Stream statistics: {stream_stats}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Data flow test failed: {e}")
|
|
raise
|
|
|
|
async def test_training_integration(self):
|
|
"""Test training data integration"""
|
|
logger.info("Testing training data integration...")
|
|
|
|
try:
|
|
# Get latest training data
|
|
training_data = self.unified_stream.get_latest_training_data()
|
|
|
|
if training_data:
|
|
logger.info("✓ Training data packet available")
|
|
logger.info(f" Tick cache: {len(training_data.tick_cache)} ticks")
|
|
logger.info(f" 1s bars: {len(training_data.one_second_bars)} bars")
|
|
logger.info(f" Multi-timeframe data: {len(training_data.multi_timeframe_data)} symbols")
|
|
logger.info(f" CNN features: {'Available' if training_data.cnn_features else 'Not available'}")
|
|
logger.info(f" CNN predictions: {'Available' if training_data.cnn_predictions else 'Not available'}")
|
|
logger.info(f" Market state: {'Available' if training_data.market_state else 'Not available'}")
|
|
logger.info(f" Universal stream: {'Available' if training_data.universal_stream else 'Not available'}")
|
|
|
|
# Check if dashboard can access training data
|
|
if hasattr(self.dashboard, 'latest_training_data') and self.dashboard.latest_training_data:
|
|
logger.info("✓ Dashboard has access to training data")
|
|
self.test_results['training_integration_test'] = True
|
|
else:
|
|
logger.warning("⚠ Dashboard does not have training data access")
|
|
else:
|
|
logger.warning("⚠ No training data available")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Training integration test failed: {e}")
|
|
raise
|
|
|
|
async def test_ui_data_flow(self):
|
|
"""Test UI data flow"""
|
|
logger.info("Testing UI data flow...")
|
|
|
|
try:
|
|
# Get latest UI data
|
|
ui_data = self.unified_stream.get_latest_ui_data()
|
|
|
|
if ui_data:
|
|
logger.info("✓ UI data packet available")
|
|
logger.info(f" Current prices: {ui_data.current_prices}")
|
|
logger.info(f" Tick cache size: {ui_data.tick_cache_size}")
|
|
logger.info(f" 1s bars count: {ui_data.one_second_bars_count}")
|
|
logger.info(f" Streaming status: {ui_data.streaming_status}")
|
|
logger.info(f" Training data available: {ui_data.training_data_available}")
|
|
|
|
# Check if dashboard can access UI data
|
|
if hasattr(self.dashboard, 'latest_ui_data') and self.dashboard.latest_ui_data:
|
|
logger.info("✓ Dashboard has access to UI data")
|
|
self.test_results['ui_data_test'] = True
|
|
else:
|
|
logger.warning("⚠ Dashboard does not have UI data access")
|
|
else:
|
|
logger.warning("⚠ No UI data available")
|
|
|
|
except Exception as e:
|
|
logger.error(f"UI data flow test failed: {e}")
|
|
raise
|
|
|
|
async def test_stream_statistics(self):
|
|
"""Test stream statistics"""
|
|
logger.info("Testing stream statistics...")
|
|
|
|
try:
|
|
# Get comprehensive stream stats
|
|
stream_stats = self.unified_stream.get_stream_stats()
|
|
|
|
logger.info("Stream Statistics:")
|
|
logger.info(f" Total ticks processed: {stream_stats.get('total_ticks_processed', 0)}")
|
|
logger.info(f" Total packets sent: {stream_stats.get('total_packets_sent', 0)}")
|
|
logger.info(f" Consumers served: {stream_stats.get('consumers_served', 0)}")
|
|
logger.info(f" Active consumers: {stream_stats.get('active_consumers', 0)}")
|
|
logger.info(f" Total consumers: {stream_stats.get('total_consumers', 0)}")
|
|
logger.info(f" Processing errors: {stream_stats.get('processing_errors', 0)}")
|
|
logger.info(f" Data quality score: {stream_stats.get('data_quality_score', 0.0)}")
|
|
|
|
if stream_stats.get('active_consumers', 0) > 0:
|
|
logger.info("✓ Stream has active consumers")
|
|
self.test_results['stream_stats_test'] = True
|
|
else:
|
|
logger.warning("⚠ No active consumers detected")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Stream statistics test failed: {e}")
|
|
raise
|
|
|
|
def generate_test_report(self):
|
|
"""Generate comprehensive test report"""
|
|
logger.info("Generating test report...")
|
|
|
|
total_tests = len(self.test_results)
|
|
passed_tests = sum(self.test_results.values())
|
|
|
|
logger.info("=" * 60)
|
|
logger.info("ENHANCED DASHBOARD INTEGRATION TEST REPORT")
|
|
logger.info("=" * 60)
|
|
logger.info(f"Test Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
|
logger.info(f"Total Tests: {total_tests}")
|
|
logger.info(f"Passed Tests: {passed_tests}")
|
|
logger.info(f"Failed Tests: {total_tests - passed_tests}")
|
|
logger.info(f"Success Rate: {(passed_tests / total_tests) * 100:.1f}%")
|
|
logger.info("")
|
|
|
|
logger.info("Test Results:")
|
|
for test_name, result in self.test_results.items():
|
|
status = "✓ PASS" if result else "✗ FAIL"
|
|
logger.info(f" {test_name}: {status}")
|
|
|
|
logger.info("")
|
|
|
|
if passed_tests == total_tests:
|
|
logger.info("🎉 ALL TESTS PASSED! Enhanced dashboard integration is working correctly.")
|
|
logger.info("The dashboard now properly integrates with the enhanced RL training pipeline.")
|
|
else:
|
|
logger.warning("⚠ Some tests failed. Please review the integration.")
|
|
|
|
logger.info("=" * 60)
|
|
|
|
async def cleanup(self):
|
|
"""Cleanup test resources"""
|
|
logger.info("Cleaning up test resources...")
|
|
|
|
try:
|
|
if self.unified_stream:
|
|
await self.unified_stream.stop_streaming()
|
|
|
|
if self.dashboard:
|
|
self.dashboard.stop_streaming()
|
|
|
|
logger.info("✓ Cleanup completed")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Cleanup failed: {e}")
|
|
|
|
async def main():
|
|
"""Main test execution"""
|
|
test = EnhancedDashboardIntegrationTest()
|
|
|
|
try:
|
|
await test.run_tests()
|
|
except Exception as e:
|
|
logger.error(f"Test execution failed: {e}")
|
|
finally:
|
|
await test.cleanup()
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main()) |