wip training
This commit is contained in:
Dobromir Popov
179
start_overnight_training.py
Normal file
179
start_overnight_training.py
Normal file
@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Start Overnight Training Session
|
||||
|
||||
This script starts a comprehensive overnight training session that:
|
||||
1. Ensures CNN and COB RL training processes are implemented and running
|
||||
2. Executes training passes on each signal when predictions change
|
||||
3. Calculates PnL and records trades in SIM mode
|
||||
4. Tracks model performance statistics
|
||||
5. Converts signals to actual trades for performance tracking
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler(f'overnight_training_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'),
|
||||
logging.StreamHandler()
|
||||
]
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def main():
|
||||
"""Start the overnight training session"""
|
||||
try:
|
||||
logger.info("🌙 STARTING OVERNIGHT TRAINING SESSION")
|
||||
logger.info("=" * 80)
|
||||
|
||||
# Import required components
|
||||
from core.config import get_config, setup_logging
|
||||
from core.data_provider import DataProvider
|
||||
from core.orchestrator import TradingOrchestrator
|
||||
from core.trading_executor import TradingExecutor
|
||||
from web.clean_dashboard import CleanTradingDashboard
|
||||
|
||||
# Setup logging
|
||||
setup_logging()
|
||||
|
||||
# Initialize components
|
||||
logger.info("Initializing components...")
|
||||
|
||||
# Create data provider
|
||||
data_provider = DataProvider()
|
||||
logger.info("✅ Data Provider initialized")
|
||||
|
||||
# Create trading executor in simulation mode
|
||||
trading_executor = TradingExecutor()
|
||||
trading_executor.simulation_mode = True # Ensure we're in simulation mode
|
||||
logger.info("✅ Trading Executor initialized (SIMULATION MODE)")
|
||||
|
||||
# Create orchestrator with enhanced training
|
||||
orchestrator = TradingOrchestrator(
|
||||
data_provider=data_provider,
|
||||
enhanced_rl_training=True
|
||||
)
|
||||
logger.info("✅ Trading Orchestrator initialized")
|
||||
|
||||
# Connect trading executor to orchestrator
|
||||
if hasattr(orchestrator, 'set_trading_executor'):
|
||||
orchestrator.set_trading_executor(trading_executor)
|
||||
logger.info("✅ Trading Executor connected to Orchestrator")
|
||||
|
||||
# Create dashboard (this initializes the overnight training coordinator)
|
||||
dashboard = CleanTradingDashboard(
|
||||
data_provider=data_provider,
|
||||
orchestrator=orchestrator,
|
||||
trading_executor=trading_executor
|
||||
)
|
||||
logger.info("✅ Dashboard initialized with Overnight Training Coordinator")
|
||||
|
||||
# Start the overnight training session
|
||||
logger.info("Starting overnight training session...")
|
||||
success = dashboard.start_overnight_training()
|
||||
|
||||
if success:
|
||||
logger.info("🌙 OVERNIGHT TRAINING SESSION STARTED SUCCESSFULLY")
|
||||
logger.info("=" * 80)
|
||||
logger.info("Training Features Active:")
|
||||
logger.info("✅ CNN training on signal changes")
|
||||
logger.info("✅ COB RL training on market microstructure")
|
||||
logger.info("✅ DQN training on trading decisions")
|
||||
logger.info("✅ Trade execution and recording (SIMULATION)")
|
||||
logger.info("✅ Performance tracking and statistics")
|
||||
logger.info("✅ Model checkpointing every 50 trades")
|
||||
logger.info("✅ Signal-to-trade conversion with PnL calculation")
|
||||
logger.info("=" * 80)
|
||||
|
||||
# Monitor training progress
|
||||
logger.info("Monitoring training progress...")
|
||||
logger.info("Press Ctrl+C to stop the training session")
|
||||
|
||||
# Keep the session running and periodically report progress
|
||||
start_time = datetime.now()
|
||||
last_report_time = start_time
|
||||
|
||||
while True:
|
||||
try:
|
||||
time.sleep(60) # Check every minute
|
||||
|
||||
current_time = datetime.now()
|
||||
elapsed_time = current_time - start_time
|
||||
|
||||
# Get performance summary every 10 minutes
|
||||
if (current_time - last_report_time).total_seconds() >= 600: # 10 minutes
|
||||
performance = dashboard.get_training_performance_summary()
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"🌙 TRAINING PROGRESS REPORT - {elapsed_time}")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"Total Signals: {performance.get('total_signals', 0)}")
|
||||
logger.info(f"Total Trades: {performance.get('total_trades', 0)}")
|
||||
logger.info(f"Successful Trades: {performance.get('successful_trades', 0)}")
|
||||
logger.info(f"Success Rate: {performance.get('success_rate', 0):.1%}")
|
||||
logger.info(f"Total P&L: ${performance.get('total_pnl', 0):.2f}")
|
||||
logger.info(f"Models Trained: {', '.join(performance.get('models_trained', []))}")
|
||||
logger.info(f"Training Status: {'ACTIVE' if performance.get('is_running', False) else 'INACTIVE'}")
|
||||
logger.info("=" * 60)
|
||||
|
||||
last_report_time = current_time
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("\n🛑 Training session interrupted by user")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error during training monitoring: {e}")
|
||||
time.sleep(30) # Wait 30 seconds before retrying
|
||||
|
||||
# Stop the training session
|
||||
logger.info("Stopping overnight training session...")
|
||||
dashboard.stop_overnight_training()
|
||||
|
||||
# Final report
|
||||
final_performance = dashboard.get_training_performance_summary()
|
||||
total_time = datetime.now() - start_time
|
||||
|
||||
logger.info("=" * 80)
|
||||
logger.info("🌅 OVERNIGHT TRAINING SESSION COMPLETED")
|
||||
logger.info("=" * 80)
|
||||
logger.info(f"Total Duration: {total_time}")
|
||||
logger.info(f"Final Statistics:")
|
||||
logger.info(f" Total Signals: {final_performance.get('total_signals', 0)}")
|
||||
logger.info(f" Total Trades: {final_performance.get('total_trades', 0)}")
|
||||
logger.info(f" Successful Trades: {final_performance.get('successful_trades', 0)}")
|
||||
logger.info(f" Success Rate: {final_performance.get('success_rate', 0):.1%}")
|
||||
logger.info(f" Total P&L: ${final_performance.get('total_pnl', 0):.2f}")
|
||||
logger.info(f" Models Trained: {', '.join(final_performance.get('models_trained', []))}")
|
||||
logger.info("=" * 80)
|
||||
|
||||
else:
|
||||
logger.error("❌ Failed to start overnight training session")
|
||||
return 1
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("\n🛑 Training session interrupted by user")
|
||||
return 0
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error in overnight training session: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = main()
|
||||
sys.exit(exit_code)
|
||||
Reference in New Issue
Block a user