cash works again!
This commit is contained in:
Dobromir Popov
@ -165,6 +165,104 @@ class OvernightTrainingMonitor:
|
||||
logger.error(f"Error checking system resources: {e}")
|
||||
return {}
|
||||
|
||||
def _parse_training_metrics(self) -> Dict[str, Any]:
|
||||
"""Parse REAL training metrics from log files - NO SYNTHETIC DATA"""
|
||||
try:
|
||||
# Read actual training logs for real metrics
|
||||
training_log_path = Path("logs/trading.log")
|
||||
if not training_log_path.exists():
|
||||
logger.warning("⚠️ No training log found - metrics unavailable")
|
||||
return self._default_metrics()
|
||||
|
||||
# Parse real metrics from training logs
|
||||
with open(training_log_path, 'r') as f:
|
||||
recent_lines = f.readlines()[-100:] # Get last 100 lines
|
||||
|
||||
# Extract real metrics from log lines
|
||||
real_metrics = self._extract_real_metrics(recent_lines)
|
||||
|
||||
if real_metrics:
|
||||
logger.info(f"✅ Parsed {len(real_metrics)} real training metrics")
|
||||
return real_metrics
|
||||
else:
|
||||
logger.warning("⚠️ No real metrics found in logs")
|
||||
return self._default_metrics()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error parsing real training metrics: {e}")
|
||||
return self._default_metrics()
|
||||
|
||||
def _extract_real_metrics(self, log_lines: List[str]) -> Dict[str, Any]:
|
||||
"""Extract real metrics from training log lines"""
|
||||
metrics = {}
|
||||
|
||||
try:
|
||||
# Look for real training indicators
|
||||
loss_values = []
|
||||
trade_counts = []
|
||||
pnl_values = []
|
||||
|
||||
for line in log_lines:
|
||||
# Extract real loss values
|
||||
if "loss:" in line.lower() or "Loss" in line:
|
||||
try:
|
||||
# Extract numeric loss value
|
||||
import re
|
||||
loss_match = re.search(r'loss[:\s]+([\d\.]+)', line, re.IGNORECASE)
|
||||
if loss_match:
|
||||
loss_values.append(float(loss_match.group(1)))
|
||||
except:
|
||||
pass
|
||||
|
||||
# Extract real trade information
|
||||
if "TRADE" in line and "OPENED" in line:
|
||||
trade_counts.append(1)
|
||||
|
||||
# Extract real PnL values
|
||||
if "PnL:" in line:
|
||||
try:
|
||||
pnl_match = re.search(r'PnL[:\s]+\$?([+-]?[\d\.]+)', line)
|
||||
if pnl_match:
|
||||
pnl_values.append(float(pnl_match.group(1)))
|
||||
except:
|
||||
pass
|
||||
|
||||
# Calculate real averages
|
||||
if loss_values:
|
||||
metrics['current_loss'] = sum(loss_values) / len(loss_values)
|
||||
metrics['loss_trend'] = 'decreasing' if len(loss_values) > 1 and loss_values[-1] < loss_values[0] else 'stable'
|
||||
|
||||
if trade_counts:
|
||||
metrics['trades_per_hour'] = len(trade_counts)
|
||||
|
||||
if pnl_values:
|
||||
metrics['total_pnl'] = sum(pnl_values)
|
||||
metrics['avg_pnl'] = sum(pnl_values) / len(pnl_values)
|
||||
metrics['win_rate'] = len([p for p in pnl_values if p > 0]) / len(pnl_values)
|
||||
|
||||
# Add timestamp
|
||||
metrics['timestamp'] = datetime.now()
|
||||
metrics['data_source'] = 'real_training_logs'
|
||||
|
||||
return metrics
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error extracting real metrics: {e}")
|
||||
return {}
|
||||
|
||||
def _default_metrics(self) -> Dict[str, Any]:
|
||||
"""Return default metrics when no real data is available"""
|
||||
return {
|
||||
'current_loss': 0.0,
|
||||
'trades_per_hour': 0,
|
||||
'total_pnl': 0.0,
|
||||
'avg_pnl': 0.0,
|
||||
'win_rate': 0.0,
|
||||
'timestamp': datetime.now(),
|
||||
'data_source': 'no_real_data_available',
|
||||
'loss_trend': 'unknown'
|
||||
}
|
||||
|
||||
def update_training_metrics(self):
|
||||
"""Update training metrics from TensorBoard logs and saved models"""
|
||||
try:
|
||||
@ -186,25 +284,50 @@ class OvernightTrainingMonitor:
|
||||
self.checkpoint_times.append(checkpoint_time)
|
||||
logger.info(f"💾 Latest checkpoint: {latest_checkpoint.name} at {checkpoint_time}")
|
||||
|
||||
# Simulate training progress (replace with actual metrics parsing)
|
||||
runtime_hours = (datetime.now() - self.start_time).total_seconds() / 3600
|
||||
# Parse REAL training metrics from logs - NO SYNTHETIC DATA
|
||||
real_metrics = self._parse_training_metrics()
|
||||
|
||||
# Realistic training progression simulation
|
||||
self.training_metrics['episodes_completed'] = int(runtime_hours * 50) # ~50 episodes per hour
|
||||
self.training_metrics['average_reward'] = min(100, runtime_hours * 10) # Gradual improvement
|
||||
self.training_metrics['win_rate'] = min(0.85, 0.5 + runtime_hours * 0.03) # Win rate improvement
|
||||
self.training_metrics['total_trades'] = int(runtime_hours * 200) # ~200 trades per hour
|
||||
if real_metrics['data_source'] == 'real_training_logs':
|
||||
# Use real metrics from training logs
|
||||
logger.info("✅ Using REAL training metrics")
|
||||
self.training_metrics['total_pnl'] = real_metrics.get('total_pnl', 0.0)
|
||||
self.training_metrics['avg_pnl'] = real_metrics.get('avg_pnl', 0.0)
|
||||
self.training_metrics['win_rate'] = real_metrics.get('win_rate', 0.0)
|
||||
self.training_metrics['current_loss'] = real_metrics.get('current_loss', 0.0)
|
||||
self.training_metrics['trades_per_hour'] = real_metrics.get('trades_per_hour', 0)
|
||||
else:
|
||||
# No real data available - use safe defaults (NO SYNTHETIC)
|
||||
logger.warning("⚠️ No real training metrics available - using zero values")
|
||||
self.training_metrics['total_pnl'] = 0.0
|
||||
self.training_metrics['avg_pnl'] = 0.0
|
||||
self.training_metrics['win_rate'] = 0.0
|
||||
self.training_metrics['current_loss'] = 0.0
|
||||
self.training_metrics['trades_per_hour'] = 0
|
||||
|
||||
# Profit simulation with 500x leverage
|
||||
base_profit_per_hour = np.random.normal(50, 20) # $50/hour average with variance
|
||||
hourly_profit = base_profit_per_hour * self.profit_metrics['leverage'] / 100 # Scale with leverage
|
||||
# Update other real metrics
|
||||
self.training_metrics['memory_usage'] = self.check_system_resources()['memory_percent']
|
||||
self.training_metrics['gpu_usage'] = self.check_system_resources()['gpu_usage']
|
||||
self.training_metrics['training_time'] = (datetime.now() - self.start_time).total_seconds()
|
||||
|
||||
self.profit_metrics['total_pnl'] += hourly_profit
|
||||
self.profit_metrics['current_balance'] = self.profit_metrics['starting_balance'] + self.profit_metrics['total_pnl']
|
||||
self.profit_metrics['roi_percentage'] = (self.profit_metrics['total_pnl'] / self.profit_metrics['starting_balance']) * 100
|
||||
# Log real metrics
|
||||
logger.info(f"🔄 Real Training Metrics Updated:")
|
||||
logger.info(f" 💰 Total PnL: ${self.training_metrics['total_pnl']:.2f}")
|
||||
logger.info(f" 📊 Win Rate: {self.training_metrics['win_rate']:.1%}")
|
||||
logger.info(f" 🔢 Trades: {self.training_metrics['trades_per_hour']}")
|
||||
logger.info(f" 📉 Loss: {self.training_metrics['current_loss']:.4f}")
|
||||
logger.info(f" 💾 Memory: {self.training_metrics['memory_usage']:.1f}%")
|
||||
logger.info(f" 🎮 GPU: {self.training_metrics['gpu_usage']:.1f}%")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating training metrics: {e}")
|
||||
logger.error(f"❌ Error updating real training metrics: {e}")
|
||||
# Set safe defaults on error (NO SYNTHETIC FALLBACK)
|
||||
self.training_metrics.update({
|
||||
'total_pnl': 0.0,
|
||||
'avg_pnl': 0.0,
|
||||
'win_rate': 0.0,
|
||||
'current_loss': 0.0,
|
||||
'trades_per_hour': 0
|
||||
})
|
||||
|
||||
def log_comprehensive_status(self):
|
||||
"""Log comprehensive training status"""
|
||||
|
||||
Reference in New Issue
Block a user