cleanup, CNN fixes
This commit is contained in:
Dobromir Popov
@ -1,220 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test Enhanced Dashboard Training Setup
|
||||
|
||||
This script validates that the enhanced dashboard has proper:
|
||||
- Real-time training capabilities
|
||||
- Test case generation
|
||||
- MEXC integration
|
||||
- Model loading and training
|
||||
"""
|
||||
|
||||
import sys
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
# Configure logging for test
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def test_dashboard_training_setup():
|
||||
"""Test the enhanced dashboard training capabilities"""
|
||||
|
||||
print("=" * 60)
|
||||
print("TESTING ENHANCED DASHBOARD TRAINING SETUP")
|
||||
print("=" * 60)
|
||||
|
||||
try:
|
||||
# Test 1: Import all components
|
||||
print("\n1. Testing component imports...")
|
||||
from web.clean_dashboard import CleanTradingDashboard as TradingDashboard, create_clean_dashboard as create_dashboard
|
||||
from core.data_provider import DataProvider
|
||||
from core.orchestrator import TradingOrchestrator
|
||||
from core.trading_executor import TradingExecutor
|
||||
from models import get_model_registry
|
||||
print(" ✓ All components imported successfully")
|
||||
|
||||
# Test 2: Initialize components
|
||||
print("\n2. Testing component initialization...")
|
||||
data_provider = DataProvider()
|
||||
orchestrator = TradingOrchestrator(data_provider)
|
||||
trading_executor = TradingExecutor()
|
||||
model_registry = get_model_registry()
|
||||
print(" ✓ All components initialized")
|
||||
|
||||
# Test 3: Create dashboard with training
|
||||
print("\n3. Testing dashboard creation with training...")
|
||||
dashboard = TradingDashboard(
|
||||
data_provider=data_provider,
|
||||
orchestrator=orchestrator,
|
||||
trading_executor=trading_executor
|
||||
)
|
||||
print(" ✓ Dashboard created successfully")
|
||||
|
||||
# Test 4: Validate training components
|
||||
print("\n4. Testing training components...")
|
||||
|
||||
# Check continuous training
|
||||
has_training = hasattr(dashboard, 'training_active')
|
||||
print(f" ✓ Continuous training: {has_training}")
|
||||
|
||||
# Check training thread
|
||||
has_thread = hasattr(dashboard, 'training_thread')
|
||||
print(f" ✓ Training thread: {has_thread}")
|
||||
|
||||
# Check tick cache
|
||||
cache_capacity = dashboard.tick_cache.maxlen
|
||||
print(f" ✓ Tick cache capacity: {cache_capacity:,} ticks")
|
||||
|
||||
# Check 1-second bars
|
||||
bars_capacity = dashboard.one_second_bars.maxlen
|
||||
print(f" ✓ 1s bars capacity: {bars_capacity} bars")
|
||||
|
||||
# Check WebSocket streaming
|
||||
has_ws = hasattr(dashboard, 'ws_connection')
|
||||
print(f" ✓ WebSocket streaming: {has_ws}")
|
||||
|
||||
# Test 5: Validate training methods
|
||||
print("\n5. Testing training methods...")
|
||||
|
||||
# Check training data methods
|
||||
training_methods = [
|
||||
'send_training_data_to_models',
|
||||
'_prepare_training_data',
|
||||
'_send_data_to_cnn_models',
|
||||
'_send_data_to_rl_models',
|
||||
'_format_data_for_cnn',
|
||||
'_format_data_for_rl',
|
||||
'start_continuous_training',
|
||||
'stop_continuous_training'
|
||||
]
|
||||
|
||||
for method in training_methods:
|
||||
has_method = hasattr(dashboard, method)
|
||||
print(f" ✓ {method}: {has_method}")
|
||||
|
||||
# Test 6: Validate MEXC integration
|
||||
print("\n6. Testing MEXC integration...")
|
||||
mexc_available = dashboard.trading_executor is not None
|
||||
print(f" ✓ MEXC executor available: {mexc_available}")
|
||||
|
||||
if mexc_available:
|
||||
has_trading_enabled = hasattr(dashboard.trading_executor, 'trading_enabled')
|
||||
has_dry_run = hasattr(dashboard.trading_executor, 'dry_run')
|
||||
has_execute_signal = hasattr(dashboard.trading_executor, 'execute_signal')
|
||||
print(f" ✓ Trading enabled flag: {has_trading_enabled}")
|
||||
print(f" ✓ Dry run mode: {has_dry_run}")
|
||||
print(f" ✓ Execute signal method: {has_execute_signal}")
|
||||
|
||||
# Test 7: Test model loading
|
||||
print("\n7. Testing model loading...")
|
||||
dashboard._load_available_models()
|
||||
model_count = len(model_registry.models) if hasattr(model_registry, 'models') else 0
|
||||
print(f" ✓ Models loaded: {model_count}")
|
||||
|
||||
# Test 8: Test training data validation
|
||||
print("\n8. Testing training data validation...")
|
||||
|
||||
# Test with empty cache (should reject)
|
||||
dashboard.tick_cache.clear()
|
||||
result = dashboard.send_training_data_to_models()
|
||||
print(f" ✓ Empty cache rejection: {not result}")
|
||||
|
||||
# Test with simulated tick data
|
||||
from collections import deque
|
||||
import random
|
||||
|
||||
# Add some mock tick data for testing
|
||||
current_time = datetime.now()
|
||||
for i in range(600): # Add 600 ticks (enough for training)
|
||||
tick = {
|
||||
'timestamp': current_time,
|
||||
'price': 3500.0 + random.uniform(-10, 10),
|
||||
'volume': random.uniform(0.1, 10.0),
|
||||
'side': 'buy' if random.random() > 0.5 else 'sell'
|
||||
}
|
||||
dashboard.tick_cache.append(tick)
|
||||
|
||||
print(f" ✓ Added {len(dashboard.tick_cache)} test ticks")
|
||||
|
||||
# Test training with sufficient data
|
||||
result = dashboard.send_training_data_to_models()
|
||||
print(f" ✓ Training with sufficient data: {result}")
|
||||
|
||||
# Test 9: Test continuous training
|
||||
print("\n9. Testing continuous training...")
|
||||
|
||||
# Start training
|
||||
dashboard.start_continuous_training()
|
||||
training_started = getattr(dashboard, 'training_active', False)
|
||||
print(f" ✓ Training started: {training_started}")
|
||||
|
||||
# Wait a moment
|
||||
time.sleep(2)
|
||||
|
||||
# Stop training
|
||||
dashboard.stop_continuous_training()
|
||||
training_stopped = not getattr(dashboard, 'training_active', True)
|
||||
print(f" ✓ Training stopped: {training_stopped}")
|
||||
|
||||
# Test 10: Test dashboard features
|
||||
print("\n10. Testing dashboard features...")
|
||||
|
||||
# Check layout setup
|
||||
has_layout = hasattr(dashboard.app, 'layout')
|
||||
print(f" ✓ Dashboard layout: {has_layout}")
|
||||
|
||||
# Check callbacks
|
||||
has_callbacks = len(dashboard.app.callback_map) > 0
|
||||
print(f" ✓ Dashboard callbacks: {has_callbacks}")
|
||||
|
||||
# Check training metrics display
|
||||
training_metrics = dashboard._create_training_metrics()
|
||||
has_metrics = len(training_metrics) > 0
|
||||
print(f" ✓ Training metrics display: {has_metrics}")
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 60)
|
||||
print("ENHANCED DASHBOARD TRAINING VALIDATION COMPLETE")
|
||||
print("=" * 60)
|
||||
|
||||
features = [
|
||||
"✓ Real-time WebSocket tick streaming",
|
||||
"✓ Continuous model training with real data only",
|
||||
"✓ CNN and RL model integration",
|
||||
"✓ MEXC trading executor integration",
|
||||
"✓ Training metrics visualization",
|
||||
"✓ Test case generation from real market data",
|
||||
"✓ Session-based P&L tracking",
|
||||
"✓ Live trading signal generation"
|
||||
]
|
||||
|
||||
print("\nValidated Features:")
|
||||
for feature in features:
|
||||
print(f" {feature}")
|
||||
|
||||
print(f"\nDashboard Ready For:")
|
||||
print(" • Real market data training (no synthetic data)")
|
||||
print(" • Live MEXC trading execution")
|
||||
print(" • Continuous model improvement")
|
||||
print(" • Test case generation from real trading scenarios")
|
||||
|
||||
print(f"\nTo start the dashboard: python .\\web\\dashboard.py")
|
||||
print(f"Dashboard will be available at: http://127.0.0.1:8050")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ ERROR: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = test_dashboard_training_setup()
|
||||
sys.exit(0 if success else 1)
|
||||
@ -1,151 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Enhanced RL Status Diagnostic Script
|
||||
Quick test to determine why Enhanced RL shows as DISABLED
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def test_enhanced_rl_imports():
|
||||
"""Test Enhanced RL component imports"""
|
||||
logger.info("🔍 Testing Enhanced RL component imports...")
|
||||
|
||||
try:
|
||||
from core.enhanced_orchestrator import EnhancedTradingOrchestrator
|
||||
logger.info("✅ EnhancedTradingOrchestrator import: SUCCESS")
|
||||
except ImportError as e:
|
||||
logger.error(f"❌ EnhancedTradingOrchestrator import: FAILED - {e}")
|
||||
return False
|
||||
|
||||
try:
|
||||
from core.universal_data_adapter import UniversalDataAdapter
|
||||
logger.info("✅ UniversalDataAdapter import: SUCCESS")
|
||||
except ImportError as e:
|
||||
logger.error(f"❌ UniversalDataAdapter import: FAILED - {e}")
|
||||
return False
|
||||
|
||||
try:
|
||||
from core.universal_data_adapter import UniversalDataAdapter, UniversalDataStream
|
||||
logger.info("✅ UnifiedDataStream components import: SUCCESS")
|
||||
except ImportError as e:
|
||||
logger.error(f"❌ UnifiedDataStream components import: FAILED - {e}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def test_dashboard_enhanced_rl_detection():
|
||||
"""Test dashboard Enhanced RL detection logic"""
|
||||
logger.info("🔍 Testing dashboard Enhanced RL detection...")
|
||||
|
||||
try:
|
||||
from core.data_provider import DataProvider
|
||||
from core.enhanced_orchestrator import EnhancedTradingOrchestrator
|
||||
# ENHANCED_RL_AVAILABLE moved to clean_dashboard
|
||||
ENHANCED_RL_AVAILABLE = True
|
||||
|
||||
logger.info(f"ENHANCED_RL_AVAILABLE in dashboard: {ENHANCED_RL_AVAILABLE}")
|
||||
|
||||
# Test orchestrator creation
|
||||
data_provider = DataProvider()
|
||||
orchestrator = EnhancedTradingOrchestrator(data_provider)
|
||||
|
||||
logger.info(f"EnhancedTradingOrchestrator created: {type(orchestrator)}")
|
||||
logger.info(f"isinstance check: {isinstance(orchestrator, EnhancedTradingOrchestrator)}")
|
||||
|
||||
# Test dashboard creation
|
||||
from web.dashboard import TradingDashboard
|
||||
dashboard = TradingDashboard(
|
||||
data_provider=data_provider,
|
||||
orchestrator=orchestrator
|
||||
)
|
||||
|
||||
logger.info(f"Dashboard enhanced_rl_enabled: {dashboard.enhanced_rl_enabled}")
|
||||
logger.info(f"Dashboard enhanced_rl_training_enabled: {dashboard.enhanced_rl_training_enabled}")
|
||||
|
||||
return dashboard.enhanced_rl_training_enabled
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Dashboard Enhanced RL test FAILED: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return False
|
||||
|
||||
def test_main_clean_enhanced_rl():
|
||||
"""Test main_clean.py Enhanced RL setup"""
|
||||
logger.info("🔍 Testing main_clean.py Enhanced RL setup...")
|
||||
|
||||
try:
|
||||
# Import required components
|
||||
from core.data_provider import DataProvider
|
||||
from core.enhanced_orchestrator import EnhancedTradingOrchestrator
|
||||
from config import get_config
|
||||
|
||||
# Simulate main_clean setup
|
||||
config = get_config()
|
||||
data_provider = DataProvider()
|
||||
|
||||
# Create Enhanced Trading Orchestrator
|
||||
model_registry = {} # Simple fallback
|
||||
orchestrator = EnhancedTradingOrchestrator(data_provider)
|
||||
|
||||
logger.info(f"Enhanced orchestrator created: {type(orchestrator)}")
|
||||
|
||||
# Create dashboard
|
||||
from web.dashboard import TradingDashboard
|
||||
dashboard = TradingDashboard(
|
||||
data_provider=data_provider,
|
||||
orchestrator=orchestrator,
|
||||
trading_executor=None
|
||||
)
|
||||
|
||||
logger.info(f"✅ Enhanced RL Status: {'ENABLED' if dashboard.enhanced_rl_training_enabled else 'DISABLED'}")
|
||||
|
||||
if dashboard.enhanced_rl_training_enabled:
|
||||
logger.info("🎉 Enhanced RL is working correctly!")
|
||||
return True
|
||||
else:
|
||||
logger.error("❌ Enhanced RL is DISABLED even with correct setup")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ main_clean Enhanced RL test FAILED: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return False
|
||||
|
||||
def main():
|
||||
"""Run all diagnostic tests"""
|
||||
logger.info("🚀 Enhanced RL Status Diagnostic Starting...")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# Test 1: Component imports
|
||||
imports_ok = test_enhanced_rl_imports()
|
||||
|
||||
# Test 2: Dashboard detection logic
|
||||
dashboard_ok = test_dashboard_enhanced_rl_detection()
|
||||
|
||||
# Test 3: Full main_clean simulation
|
||||
main_clean_ok = test_main_clean_enhanced_rl()
|
||||
|
||||
# Summary
|
||||
logger.info("=" * 60)
|
||||
logger.info("📋 DIAGNOSTIC SUMMARY")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"Enhanced RL Imports: {'✅ PASS' if imports_ok else '❌ FAIL'}")
|
||||
logger.info(f"Dashboard Detection: {'✅ PASS' if dashboard_ok else '❌ FAIL'}")
|
||||
logger.info(f"Main Clean Setup: {'✅ PASS' if main_clean_ok else '❌ FAIL'}")
|
||||
|
||||
if all([imports_ok, dashboard_ok, main_clean_ok]):
|
||||
logger.info("🎉 ALL TESTS PASSED - Enhanced RL should be working!")
|
||||
logger.info("💡 If dashboard still shows DISABLED, restart it with:")
|
||||
logger.info(" python main_clean.py --mode web --port 8050")
|
||||
else:
|
||||
logger.error("❌ TESTS FAILED - Enhanced RL has issues")
|
||||
logger.info("💡 Check the error messages above for specific issues")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,111 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test Enhanced Trading System
|
||||
Verify that both RL and CNN learning pipelines are active
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from core.config import get_config
|
||||
from core.data_provider import DataProvider
|
||||
from core.enhanced_orchestrator import EnhancedTradingOrchestrator
|
||||
from training.enhanced_cnn_trainer import EnhancedCNNTrainer
|
||||
from training.enhanced_rl_trainer import EnhancedRLTrainer
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
async def test_enhanced_system():
|
||||
"""Test the enhanced trading system components"""
|
||||
logger.info("Testing Enhanced Trading System...")
|
||||
|
||||
try:
|
||||
# Initialize components
|
||||
config = get_config()
|
||||
data_provider = DataProvider(config)
|
||||
orchestrator = EnhancedTradingOrchestrator(data_provider)
|
||||
|
||||
# Initialize trainers
|
||||
cnn_trainer = EnhancedCNNTrainer(config, orchestrator)
|
||||
rl_trainer = EnhancedRLTrainer(config, orchestrator)
|
||||
|
||||
logger.info("COMPONENT STATUS:")
|
||||
logger.info(f"✓ Data Provider: {len(config.symbols)} symbols, {len(config.timeframes)} timeframes")
|
||||
logger.info(f"✓ Enhanced Orchestrator: Confidence threshold {orchestrator.confidence_threshold}")
|
||||
logger.info(f"✓ CNN Trainer: Model initialized")
|
||||
logger.info(f"✓ RL Trainer: {len(rl_trainer.agents)} agents initialized")
|
||||
|
||||
# Test decision making
|
||||
logger.info("\nTesting decision making...")
|
||||
decisions_dict = await orchestrator.make_coordinated_decisions()
|
||||
decisions = [decision for decision in decisions_dict.values() if decision is not None]
|
||||
logger.info(f"✓ Generated {len(decisions)} trading decisions")
|
||||
|
||||
for decision in decisions:
|
||||
logger.info(f" - {decision.action} {decision.symbol} @ ${decision.price:.2f} (conf: {decision.confidence:.1%})")
|
||||
|
||||
# Test RL learning capability
|
||||
logger.info("\nTesting RL learning capability...")
|
||||
for symbol, agent in rl_trainer.agents.items():
|
||||
buffer_size = len(agent.replay_buffer)
|
||||
epsilon = agent.epsilon
|
||||
logger.info(f" - {symbol} RL Agent: Buffer={buffer_size}, Epsilon={epsilon:.3f}")
|
||||
|
||||
# Test CNN training capability
|
||||
logger.info("\nTesting CNN training capability...")
|
||||
perfect_moves = orchestrator.get_perfect_moves_for_training()
|
||||
logger.info(f" - Perfect moves available: {len(perfect_moves)}")
|
||||
|
||||
if len(perfect_moves) > 0:
|
||||
logger.info(" - CNN ready for training on perfect moves")
|
||||
else:
|
||||
logger.info(" - CNN waiting for perfect moves to accumulate")
|
||||
|
||||
# Test configuration
|
||||
logger.info("\nTraining Configuration:")
|
||||
logger.info(f" - CNN training interval: {config.training.get('cnn_training_interval', 'N/A')} seconds")
|
||||
logger.info(f" - RL training interval: {config.training.get('rl_training_interval', 'N/A')} seconds")
|
||||
logger.info(f" - Min perfect moves for CNN: {config.training.get('min_perfect_moves', 'N/A')}")
|
||||
logger.info(f" - Min experiences for RL: {config.training.get('min_experiences', 'N/A')}")
|
||||
logger.info(f" - Continuous learning: {config.training.get('continuous_learning', False)}")
|
||||
|
||||
logger.info("\n✅ Enhanced Trading System test completed successfully!")
|
||||
logger.info("LEARNING SYSTEMS STATUS:")
|
||||
logger.info("✓ RL agents ready for continuous learning from trading decisions")
|
||||
logger.info("✓ CNN trainer ready for pattern learning from perfect moves")
|
||||
logger.info("✓ Enhanced orchestrator coordinating multi-modal decisions")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Test failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
async def main():
|
||||
"""Main test function"""
|
||||
logger.info("🚀 Starting Enhanced Trading System Test...")
|
||||
|
||||
success = await test_enhanced_system()
|
||||
|
||||
if success:
|
||||
logger.info("\n🎉 All tests passed! Enhanced trading system is ready.")
|
||||
logger.info("You can now run the enhanced dashboard or main trading system.")
|
||||
else:
|
||||
logger.error("\n💥 Tests failed! Please check the configuration and try again.")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user