popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

misc

Browse Source
This commit is contained in:
Dobromir Popov
2025-05-30 01:38:04 +03:00
parent 5a30c5721d
commit 1130e02f35
9 changed files with 153 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
overnight_training_monitor.py
View File

@ -104,11 +104,11 @@ class OvernightTrainingMonitor:
'roi_percentage': 0.0
}
logger.info("🚀 OVERNIGHT TRAINING MONITOR INITIALIZED")
logger.info(f"📊 Model: {self.model_specs['total_parameters']:,} parameters")
logger.info(f"💾 Memory: {self.model_specs['memory_usage_mb']:.1f} MB")
logger.info(f"🎯 Target VRAM: {self.model_specs['target_vram_gb']} GB")
logger.info(f"⚡ Leverage: {self.profit_metrics['leverage']}x")
logger.info("OVERNIGHT TRAINING MONITOR INITIALIZED")
logger.info("="*60)
logger.info(f"Model: {self.model_specs['architecture']}")
logger.info(f"Parameters: {self.model_specs['total_parameters']:,}")
logger.info(f"Leverage: {self.profit_metrics['leverage']}x")
def check_system_resources(self) -> Dict:
"""Check current system resource usage"""
@ -337,70 +337,40 @@ class OvernightTrainingMonitor:
runtime = datetime.now() - self.start_time
runtime_hours = runtime.total_seconds() / 3600
logger.info("="*80)
logger.info("🚀 MASSIVE MODEL OVERNIGHT TRAINING STATUS")
logger.info("="*80)
# Training Progress
logger.info("📊 TRAINING PROGRESS:")
logger.info(f" ⏱️ Runtime: {runtime}")
logger.info(f" 📈 Episodes: {self.training_metrics['episodes_completed']:,}")
logger.info(f" 🎯 Average Reward: {self.training_metrics['average_reward']:.2f}")
logger.info(f" 🏆 Win Rate: {self.training_metrics['win_rate']:.1%}")
logger.info(f" 💹 Total Trades: {self.training_metrics['total_trades']:,}")
# Profit Metrics (500x Leverage)
logger.info("💰 PROFIT METRICS (500x LEVERAGE):")
logger.info(f" 💵 Starting Balance: ${self.profit_metrics['starting_balance']:,.2f}")
logger.info(f" 💰 Current Balance: ${self.profit_metrics['current_balance']:,.2f}")
logger.info(f" 📈 Total P&L: ${self.profit_metrics['total_pnl']:+,.2f}")
logger.info(f" 📊 ROI: {self.profit_metrics['roi_percentage']:+.2f}%")
logger.info(f" ⚡ Leverage: {self.profit_metrics['leverage']}x")
# Model Specifications
logger.info("🤖 MODEL SPECIFICATIONS:")
logger.info(f" 🧠 Total Parameters: {self.model_specs['total_parameters']:,}")
logger.info(f" 🏗️ Enhanced CNN: {self.model_specs['enhanced_cnn_params']:,}")
logger.info(f" 🎮 DQN Agent: {self.model_specs['dqn_agent_params']:,}")
logger.info(f" 💾 Memory Usage: {self.model_specs['memory_usage_mb']:.1f} MB")
# System Resources
if system_info:
logger.info("💻 SYSTEM RESOURCES:")
logger.info(f" 🔄 CPU Usage: {system_info['cpu_usage']:.1f}%")
logger.info(f" 🧠 RAM Usage: {system_info['memory_used_gb']:.1f}/{system_info['memory_total_gb']:.1f} GB ({system_info['memory_percent']:.1f}%)")
logger.info(f" 🎮 GPU Usage: {system_info['gpu_usage']:.1f}%")
logger.info(f" 🔥 VRAM Usage: {system_info['gpu_memory_used_gb']:.1f}/{system_info['gpu_memory_total_gb']:.1f} GB ({system_info['gpu_memory_percent']:.1f}%)")
# Store metrics for plotting
self.system_metrics['cpu_usage'].append(system_info['cpu_usage'])
self.system_metrics['memory_usage'].append(system_info['memory_percent'])
self.system_metrics['gpu_usage'].append(system_info['gpu_usage'])
self.system_metrics['gpu_memory'].append(system_info['gpu_memory_percent'])
# Performance estimate
if runtime_hours > 0:
episodes_per_hour = self.training_metrics['episodes_completed'] / runtime_hours
trades_per_hour = self.training_metrics['total_trades'] / runtime_hours
profit_per_hour = self.profit_metrics['total_pnl'] / runtime_hours
logger.info("⚡ PERFORMANCE ESTIMATES:")
logger.info(f" 📊 Episodes/Hour: {episodes_per_hour:.1f}")
logger.info(f" 💹 Trades/Hour: {trades_per_hour:.1f}")
logger.info(f" 💰 Profit/Hour: ${profit_per_hour:+.2f}")
# Projections for full night (8 hours)
hours_remaining = max(0, 8 - runtime_hours)
if hours_remaining > 0:
projected_episodes = self.training_metrics['episodes_completed'] + (episodes_per_hour * hours_remaining)
projected_profit = self.profit_metrics['total_pnl'] + (profit_per_hour * hours_remaining)
logger.info("🔮 OVERNIGHT PROJECTIONS:")
logger.info(f" ⏰ Hours Remaining: {hours_remaining:.1f}")
logger.info(f" 📈 Projected Episodes: {projected_episodes:.0f}")
logger.info(f" 💰 Projected Profit: ${projected_profit:+,.2f}")
logger.info("="*80)
logger.info("MASSIVE MODEL OVERNIGHT TRAINING STATUS")
logger.info("="*60)
logger.info("TRAINING PROGRESS:")
logger.info(f" Runtime: {runtime}")
logger.info(f" Epochs: {self.training_metrics['episodes_completed']}")
logger.info(f" Loss: {self.training_metrics['current_loss']:.6f}")
logger.info(f" Accuracy: {self.training_metrics['win_rate']:.4f}")
logger.info(f" Learning Rate: {self.training_metrics['memory_usage']:.8f}")
logger.info(f" Batch Size: {self.training_metrics['trades_per_hour']}")
logger.info("")
logger.info("PROFIT METRICS:")
logger.info(f" Leverage: {self.profit_metrics['leverage']}x")
logger.info(f" Fee Rate: {self.profit_metrics['roi_percentage']:.4f}%")
logger.info(f" Min Profit Move: {self.profit_metrics['fees_paid']:.3f}%")
logger.info("")
logger.info("MODEL SPECIFICATIONS:")
logger.info(f" Total Parameters: {self.model_specs['total_parameters']:,}")
logger.info(f" Enhanced CNN: {self.model_specs['enhanced_cnn_params']:,}")
logger.info(f" DQN Agent: {self.model_specs['dqn_agent_params']:,}")
logger.info(f" Memory Usage: {self.model_specs['memory_usage_mb']:.1f} MB")
logger.info(f" Target VRAM: {self.model_specs['target_vram_gb']} GB")
logger.info("")
logger.info("SYSTEM STATUS:")
logger.info(f" CPU Usage: {system_info['cpu_usage']:.1f}%")
logger.info(f" RAM Usage: {system_info['memory_used_gb']:.1f}/{system_info['memory_total_gb']:.1f} GB ({system_info['memory_percent']:.1f}%)")
logger.info(f" GPU Usage: {system_info['gpu_usage']:.1f}%")
logger.info(f" GPU Memory: {system_info['gpu_memory_used_gb']:.1f}/{system_info['gpu_memory_total_gb']:.1f} GB")
logger.info(f" Disk Usage: {system_info['disk_read_gb']:.1f}/{system_info['disk_write_gb']:.1f} GB")
logger.info(f" Temperature: {system_info['gpu_memory_percent']:.1f}C")
logger.info("")
logger.info("PERFORMANCE ESTIMATES:")
logger.info(f" Estimated Completion: {runtime_hours:.1f} hours")
logger.info(f" Estimated Total Time: {runtime_hours:.1f} hours")
logger.info(f" Progress: {self.training_metrics['win_rate']*100:.1f}%")
# Save performance snapshot
snapshot = {