gogo2/enhanced_rl_diagnostic.py

#!/usr/bin/env python3
"""
Enhanced RL Diagnostic and Setup Script

This script:
1. Diagnoses why Enhanced RL shows as DISABLED
2. Explains model management and training progression
3. Sets up clean training environment
4. Provides solutions for the reward function issues
"""

import sys
import json
import logging
from datetime import datetime
from pathlib import Path

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

def check_enhanced_rl_availability():
    """Check what's causing Enhanced RL to be disabled"""
    logger.info("🔍 DIAGNOSING ENHANCED RL AVAILABILITY")
    logger.info("=" * 50)
    
    issues = []
    solutions = []
    
    # Test 1: Enhanced components import
    try:
        from core.enhanced_orchestrator import EnhancedTradingOrchestrator
        logger.info("✅ EnhancedTradingOrchestrator imports successfully")
    except ImportError as e:
        issues.append(f"❌ Cannot import EnhancedTradingOrchestrator: {e}")
        solutions.append("Fix: Check core/enhanced_orchestrator.py exists and is valid")
    
    # Test 2: Unified data stream import  
    try:
        from core.unified_data_stream import UnifiedDataStream, TrainingDataPacket, UIDataPacket
        logger.info("✅ Unified data stream components import successfully")
    except ImportError as e:
        issues.append(f"❌ Cannot import unified data stream: {e}")
        solutions.append("Fix: Check core/unified_data_stream.py exists and is valid")
    
    # Test 3: Universal data adapter import
    try:
        from core.universal_data_adapter import UniversalDataAdapter
        logger.info("✅ UniversalDataAdapter imports successfully")
    except ImportError as e:
        issues.append(f"❌ Cannot import UniversalDataAdapter: {e}")
        solutions.append("Fix: Check core/universal_data_adapter.py exists and is valid")
    
    # Test 4: Dashboard initialization logic
    logger.info("🔍 Checking dashboard initialization logic...")
    
    # Simulate dashboard initialization
    try:
        from core.enhanced_orchestrator import EnhancedTradingOrchestrator
        from core.data_provider import DataProvider
        
        data_provider = DataProvider()
        enhanced_orchestrator = EnhancedTradingOrchestrator(
            data_provider=data_provider,
            symbols=['ETH/USDT'],
            enhanced_rl_training=True
        )
        
        # Check the isinstance condition
        if isinstance(enhanced_orchestrator, EnhancedTradingOrchestrator):
            logger.info("✅ EnhancedTradingOrchestrator isinstance check passes")
        else:
            issues.append("❌ isinstance(orchestrator, EnhancedTradingOrchestrator) fails")
            solutions.append("Fix: Ensure dashboard is initialized with EnhancedTradingOrchestrator")
            
    except Exception as e:
        issues.append(f"❌ Cannot create EnhancedTradingOrchestrator: {e}")
        solutions.append("Fix: Check orchestrator initialization parameters")
    
    # Test 5: Main startup script
    logger.info("🔍 Checking main startup configuration...")
    main_file = Path("main_clean.py")
    if main_file.exists():
        content = main_file.read_text()
        if "EnhancedTradingOrchestrator" in content:
            logger.info("✅ main_clean.py uses EnhancedTradingOrchestrator")
        else:
            issues.append("❌ main_clean.py not using EnhancedTradingOrchestrator")
            solutions.append("Fix: Update main_clean.py to use EnhancedTradingOrchestrator")
    
    return issues, solutions

def analyze_model_management():
    """Analyze current model management setup"""
    logger.info("📊 ANALYZING MODEL MANAGEMENT")
    logger.info("=" * 50)
    
    models_dir = Path("models")
    
    # Count different model types
    model_counts = {
        "CNN models": len(list(models_dir.glob("**/cnn*.pt*"))),
        "RL models": len(list(models_dir.glob("**/trading_agent*.pt*"))),
        "Backup models": len(list(models_dir.glob("**/*.backup"))),
        "Total model files": len(list(models_dir.glob("**/*.pt*")))
    }
    
    for model_type, count in model_counts.items():
        logger.info(f"  {model_type}: {count}")
    
    # Check for training progression system
    progress_file = models_dir / "training_progress.json"
    if progress_file.exists():
        logger.info("✅ Training progression file exists")
        try:
            with open(progress_file) as f:
                progress = json.load(f)
            logger.info(f"  Created: {progress.get('created', 'Unknown')}")
            logger.info(f"  Version: {progress.get('version', 'Unknown')}")
        except Exception as e:
            logger.warning(f"⚠️ Cannot read progression file: {e}")
    else:
        logger.info("❌ No training progression tracking found")
    
    # Check for conflicting models
    conflicting_models = [
        "models/cnn_final_20250331_001817.pt.pt",
        "models/cnn_best.pt.pt",
        "models/trading_agent_final.pt",
        "models/trading_agent_best_pnl.pt"
    ]
    
    conflicts = [model for model in conflicting_models if Path(model).exists()]
    if conflicts:
        logger.warning(f"⚠️ Found {len(conflicts)} potentially conflicting model files")
        for conflict in conflicts:
            logger.warning(f"  {conflict}")
    else:
        logger.info("✅ No obvious model conflicts detected")

def analyze_reward_function():
    """Analyze the reward function and training issues"""
    logger.info("🎯 ANALYZING REWARD FUNCTION ISSUES")
    logger.info("=" * 50)
    
    # Read recent dashboard logs to understand the -0.5 reward issue
    log_file = Path("dashboard.log")
    if log_file.exists():
        try:
            with open(log_file, 'r') as f:
                lines = f.readlines()
            
            # Look for reward patterns
            reward_lines = [line for line in lines if "Reward:" in line]
            if reward_lines:
                recent_rewards = reward_lines[-10:]  # Last 10 rewards
                negative_rewards = [line for line in recent_rewards if "-0.5" in line]
                
                logger.info(f"Recent rewards found: {len(recent_rewards)}")
                logger.info(f"Negative -0.5 rewards: {len(negative_rewards)}")
                
                if len(negative_rewards) > 5:
                    logger.warning("⚠️ High number of -0.5 rewards detected")
                    logger.info("This suggests blocked signals are being penalized with fees")
                    logger.info("Solution: Update _queue_signal_for_training to handle blocked signals better")
            
            # Look for blocked signal patterns
            blocked_signals = [line for line in lines if "NOT_EXECUTED" in line]
            if blocked_signals:
                logger.info(f"Blocked signals found: {len(blocked_signals)}")
                recent_blocked = blocked_signals[-5:]
                for line in recent_blocked:
                    logger.info(f"  {line.strip()}")
        
        except Exception as e:
            logger.warning(f"Cannot analyze log file: {e}")
    else:
        logger.info("No dashboard.log found for analysis")

def provide_solutions():
    """Provide comprehensive solutions"""
    logger.info("💡 COMPREHENSIVE SOLUTIONS")
    logger.info("=" * 50)
    
    solutions = {
        "Enhanced RL DISABLED Issue": [
            "1. Update main_clean.py to use EnhancedTradingOrchestrator (already done)",
            "2. Restart the dashboard with: python main_clean.py web",
            "3. Verify Enhanced RL: ENABLED appears in logs"
        ],
        
        "Williams Repeated Initialization": [
            "1. Dashboard reuses Williams instance now (already fixed)",  
            "2. Default strengths changed from [2,3,5,8,13] to [2,3,5] (already done)",
            "3. No more repeated 'Williams Market Structure initialized' logs"
        ],
        
        "Model Management": [
            "1. Run: python cleanup_and_setup_models.py",
            "2. This will backup old models and create clean structure", 
            "3. Set up training progression tracking",
            "4. Initialize fresh training environment"
        ],
        
        "Reward Function (-0.5 Issue)": [
            "1. Blocked signals now get small negative reward (-0.1) instead of fee penalty",
            "2. Synthetic signals handled separately from real trades",
            "3. Reward calculation improved for better learning"
        ],
        
        "CNN Training Sessions": [
            "1. CNN training is disabled by default (no TensorFlow)",
            "2. Williams pivot detection works without CNN",
            "3. Enable CNN when TensorFlow available for enhanced predictions"
        ]
    }
    
    for category, steps in solutions.items():
        logger.info(f"\n{category}:")
        for step in steps:
            logger.info(f"  {step}")

def create_startup_script():
    """Create an optimal startup script"""
    startup_script = """#!/usr/bin/env python3
# Enhanced RL Trading Dashboard Startup Script

import logging
logging.basicConfig(level=logging.INFO)

def main():
    try:
        # Import enhanced components
        from core.data_provider import DataProvider
        from core.enhanced_orchestrator import EnhancedTradingOrchestrator
        from core.trading_executor import TradingExecutor
        from web.clean_dashboard import CleanTradingDashboard as TradingDashboard
        from config import get_config
        
        config = get_config()
        
        # Initialize with enhanced RL support
        data_provider = DataProvider()
        
        enhanced_orchestrator = EnhancedTradingOrchestrator(
            data_provider=data_provider,
            symbols=config.get('symbols', ['ETH/USDT']),
            enhanced_rl_training=True
        )
        
        trading_executor = TradingExecutor()
        
        # Create dashboard with enhanced components
        dashboard = TradingDashboard(
            data_provider=data_provider,
            orchestrator=enhanced_orchestrator,  # Enhanced RL enabled
            trading_executor=trading_executor
        )
        
        print("Enhanced RL Trading Dashboard Starting...")
        print("Enhanced RL: ENABLED")
        print("Williams Pivot Detection: ENABLED") 
        print("Real Market Data: ENABLED")
        print("Access at: http://127.0.0.1:8050")
        
        dashboard.run(host='127.0.0.1', port=8050, debug=False)
        
    except Exception as e:
        print(f"Startup failed: {e}")
        import traceback
        traceback.print_exc()

if __name__ == "__main__":
    main()
"""
    
    with open("start_enhanced_dashboard.py", "w", encoding='utf-8') as f:
        f.write(startup_script)
    
    logger.info("Created start_enhanced_dashboard.py for optimal startup")

def main():
    """Main diagnostic function"""
    print("🔬 ENHANCED RL DIAGNOSTIC AND SETUP")
    print("=" * 60)
    print("Analyzing Enhanced RL issues and providing solutions...")
    print("=" * 60)
    
    # Run diagnostics
    issues, solutions = check_enhanced_rl_availability()
    analyze_model_management()
    analyze_reward_function()
    provide_solutions()
    create_startup_script()
    
    # Summary
    print("\n" + "=" * 60)
    print("📋 SUMMARY")
    print("=" * 60)
    
    if issues:
        print("❌ Issues found:")
        for issue in issues:
            print(f"  {issue}")
        print("\n💡 Solutions:")
        for solution in solutions:
            print(f"  {solution}")
    else:
        print("✅ No critical issues detected!")
    
    print("\n🚀 NEXT STEPS:")
    print("1. Run model cleanup: python cleanup_and_setup_models.py")
    print("2. Start enhanced dashboard: python start_enhanced_dashboard.py") 
    print("3. Verify 'Enhanced RL: ENABLED' in dashboard")
    print("4. Check Williams pivot detection on chart")
    print("5. Monitor training episodes (should not all be -0.5 reward)")

if __name__ == "__main__":
    main()