gogo2/start_overnight_training.py

#!/usr/bin/env python3
"""
Start Overnight Training Session

This script starts a comprehensive overnight training session that:
1. Ensures CNN and COB RL training processes are implemented and running
2. Executes training passes on each signal when predictions change
3. Calculates PnL and records trades in SIM mode
4. Tracks model performance statistics
5. Converts signals to actual trades for performance tracking
"""

import os
import sys
import time
import logging
from datetime import datetime
from pathlib import Path

# Add project root to path
project_root = Path(__file__).parent
sys.path.insert(0, str(project_root))

# Setup logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler(f'overnight_training_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'),
        logging.StreamHandler()
    ]
)

logger = logging.getLogger(__name__)

def main():
    """Start the overnight training session"""
    try:
        logger.info("🌙 STARTING OVERNIGHT TRAINING SESSION")
        logger.info("=" * 80)
        
        # Import required components
        from core.config import get_config, setup_logging
        from core.data_provider import DataProvider
        from core.orchestrator import TradingOrchestrator
        from core.trading_executor import TradingExecutor
        from web.clean_dashboard import CleanTradingDashboard
        
        # Setup logging
        setup_logging()
        
        # Initialize components
        logger.info("Initializing components...")
        
        # Create data provider
        data_provider = DataProvider()
        logger.info("✅ Data Provider initialized")
        
        # Create trading executor in simulation mode
        trading_executor = TradingExecutor()
        trading_executor.simulation_mode = True  # Ensure we're in simulation mode
        logger.info("✅ Trading Executor initialized (SIMULATION MODE)")
        
        # Create orchestrator with enhanced training
        orchestrator = TradingOrchestrator(
            data_provider=data_provider,
            enhanced_rl_training=True
        )
        logger.info("✅ Trading Orchestrator initialized")
        
        # Connect trading executor to orchestrator
        if hasattr(orchestrator, 'set_trading_executor'):
            orchestrator.set_trading_executor(trading_executor)
            logger.info("✅ Trading Executor connected to Orchestrator")
        
        # Create dashboard (this initializes the overnight training coordinator)
        dashboard = CleanTradingDashboard(
            data_provider=data_provider,
            orchestrator=orchestrator,
            trading_executor=trading_executor
        )
        logger.info("✅ Dashboard initialized with Overnight Training Coordinator")
        
        # Start the overnight training session
        logger.info("Starting overnight training session...")
        success = dashboard.start_overnight_training()
        
        if success:
            logger.info("🌙 OVERNIGHT TRAINING SESSION STARTED SUCCESSFULLY")
            logger.info("=" * 80)
            logger.info("Training Features Active:")
            logger.info("✅ CNN training on signal changes")
            logger.info("✅ COB RL training on market microstructure")
            logger.info("✅ DQN training on trading decisions")
            logger.info("✅ Trade execution and recording (SIMULATION)")
            logger.info("✅ Performance tracking and statistics")
            logger.info("✅ Model checkpointing every 50 trades")
            logger.info("✅ Signal-to-trade conversion with PnL calculation")
            logger.info("=" * 80)
            
            # Monitor training progress
            logger.info("Monitoring training progress...")
            logger.info("Press Ctrl+C to stop the training session")
            
            # Keep the session running and periodically report progress
            start_time = datetime.now()
            last_report_time = start_time
            
            while True:
                try:
                    time.sleep(60)  # Check every minute
                    
                    current_time = datetime.now()
                    elapsed_time = current_time - start_time
                    
                    # Get performance summary every 10 minutes
                    if (current_time - last_report_time).total_seconds() >= 600:  # 10 minutes
                        performance = dashboard.get_training_performance_summary()
                        
                        logger.info("=" * 60)
                        logger.info(f"🌙 TRAINING PROGRESS REPORT - {elapsed_time}")
                        logger.info("=" * 60)
                        logger.info(f"Total Signals: {performance.get('total_signals', 0)}")
                        logger.info(f"Total Trades: {performance.get('total_trades', 0)}")
                        logger.info(f"Successful Trades: {performance.get('successful_trades', 0)}")
                        logger.info(f"Success Rate: {performance.get('success_rate', 0):.1%}")
                        logger.info(f"Total P&L: ${performance.get('total_pnl', 0):.2f}")
                        logger.info(f"Models Trained: {', '.join(performance.get('models_trained', []))}")
                        logger.info(f"Training Status: {'ACTIVE' if performance.get('is_running', False) else 'INACTIVE'}")
                        logger.info("=" * 60)
                        
                        last_report_time = current_time
                    
                except KeyboardInterrupt:
                    logger.info("\n🛑 Training session interrupted by user")
                    break
                except Exception as e:
                    logger.error(f"Error during training monitoring: {e}")
                    time.sleep(30)  # Wait 30 seconds before retrying
            
            # Stop the training session
            logger.info("Stopping overnight training session...")
            dashboard.stop_overnight_training()
            
            # Final report
            final_performance = dashboard.get_training_performance_summary()
            total_time = datetime.now() - start_time
            
            logger.info("=" * 80)
            logger.info("🌅 OVERNIGHT TRAINING SESSION COMPLETED")
            logger.info("=" * 80)
            logger.info(f"Total Duration: {total_time}")
            logger.info(f"Final Statistics:")
            logger.info(f"  Total Signals: {final_performance.get('total_signals', 0)}")
            logger.info(f"  Total Trades: {final_performance.get('total_trades', 0)}")
            logger.info(f"  Successful Trades: {final_performance.get('successful_trades', 0)}")
            logger.info(f"  Success Rate: {final_performance.get('success_rate', 0):.1%}")
            logger.info(f"  Total P&L: ${final_performance.get('total_pnl', 0):.2f}")
            logger.info(f"  Models Trained: {', '.join(final_performance.get('models_trained', []))}")
            logger.info("=" * 80)
            
        else:
            logger.error("❌ Failed to start overnight training session")
            return 1
        
    except KeyboardInterrupt:
        logger.info("\n🛑 Training session interrupted by user")
        return 0
    except Exception as e:
        logger.error(f"❌ Error in overnight training session: {e}")
        import traceback
        traceback.print_exc()
        return 1
    
    return 0

if __name__ == "__main__":
    exit_code = main()
    sys.exit(exit_code)