From b3e3a7673fbb8f974d9c8b0f46ee278b8d1fa917 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 29 Jul 2025 15:12:48 +0300 Subject: [PATCH] TZ wip, UI model stats fix --- .vscode/tasks.json | 5 +- core/data_provider.py | 6 +- web/clean_dashboard.py | 303 ++++++++++++++++++++++------------------- 3 files changed, 171 insertions(+), 143 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index e3b0163..b0fdd3d 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,7 +6,8 @@ "type": "shell", "command": "powershell", "args": [ - "-ExecutionPolicy", "Bypass", + "-ExecutionPolicy", + "Bypass", "-File", "scripts/kill_stale_processes.ps1" ], @@ -109,4 +110,4 @@ "problemMatcher": [] } ] -} \ No newline at end of file +} \ No newline at end of file diff --git a/core/data_provider.py b/core/data_provider.py index 54db1ea..ff3c274 100644 --- a/core/data_provider.py +++ b/core/data_provider.py @@ -1515,9 +1515,9 @@ class DataProvider: # Ensure gap_start has same timezone as end_time for comparison if gap_start.tzinfo is None: - gap_start = sofia_tz.localize(gap_start) - elif gap_start.tzinfo != sofia_tz: - gap_start = gap_start.astimezone(sofia_tz) + gap_start = SOFIA_TZ.localize(gap_start) + elif gap_start.tzinfo != SOFIA_TZ: + gap_start = gap_start.astimezone(SOFIA_TZ) if gap_start < end_time: # Need to fill gap from cache_end to now diff --git a/web/clean_dashboard.py b/web/clean_dashboard.py index 21781f7..605f07e 100644 --- a/web/clean_dashboard.py +++ b/web/clean_dashboard.py @@ -73,6 +73,20 @@ from core.standardized_data_provider import StandardizedDataProvider from core.orchestrator import TradingOrchestrator from core.trading_executor import TradingExecutor +# Import timezone utilities for Sofia timezone handling +try: + from utils.timezone_utils import SOFIA_TZ, to_sofia, now_system +except ImportError: + # Fallback if timezone utils not available + import pytz + SOFIA_TZ = pytz.timezone('Europe/Sofia') + def to_sofia(dt): + if dt.tzinfo is None: + dt = pytz.UTC.localize(dt) + return dt.astimezone(SOFIA_TZ) + def now_system(): + return datetime.now(SOFIA_TZ) + # Import standardized models from NN.models.standardized_cnn import StandardizedCNN @@ -668,39 +682,41 @@ class CleanTradingDashboard: return {'loaded_models': {}, 'total_models': 0, 'system_status': 'ERROR'} def _convert_utc_to_local(self, utc_timestamp): - """Convert UTC timestamp to local timezone for display""" + """Convert UTC timestamp to Sofia timezone for display""" try: if utc_timestamp is None: - return datetime.now() + return now_system() # Handle different input types if isinstance(utc_timestamp, str): try: utc_timestamp = pd.to_datetime(utc_timestamp) except: - return datetime.now() + return now_system() # If it's already a datetime object if isinstance(utc_timestamp, datetime): - # If it has timezone info and is UTC, convert to local + # If it has timezone info and is UTC, convert to Sofia timezone if utc_timestamp.tzinfo is not None: if str(utc_timestamp.tzinfo) == 'UTC': - # Convert UTC to local timezone - local_timestamp = utc_timestamp.replace(tzinfo=timezone.utc).astimezone() - return local_timestamp.replace(tzinfo=None) # Remove timezone info for display + # Convert UTC to Sofia timezone + sofia_timestamp = utc_timestamp.replace(tzinfo=timezone.utc).astimezone(SOFIA_TZ) + return sofia_timestamp.replace(tzinfo=None) # Remove timezone info for display else: - # Already has timezone, convert to local - return utc_timestamp.astimezone().replace(tzinfo=None) + # Already has timezone, convert to Sofia + return utc_timestamp.astimezone(SOFIA_TZ).replace(tzinfo=None) else: - # No timezone info, assume it's already local - return utc_timestamp + # No timezone info, assume it's UTC and convert to Sofia + utc_timestamp = utc_timestamp.replace(tzinfo=timezone.utc) + sofia_timestamp = utc_timestamp.astimezone(SOFIA_TZ) + return sofia_timestamp.replace(tzinfo=None) # Fallback - return datetime.now() + return now_system() except Exception as e: - logger.debug(f"Error converting UTC to local time: {e}") - return datetime.now() + logger.debug(f"Error converting UTC to Sofia time: {e}") + return now_system() def _safe_strftime(self, timestamp_val, format_str='%H:%M:%S'): """Safely format timestamp, handling both string and datetime objects""" @@ -3394,11 +3410,14 @@ class CleanTradingDashboard: # Get enhanced training statistics if available enhanced_training_stats = self._get_enhanced_training_stats() - # DEBUG: Check if orchestrator has model statistics + # Get real model statistics from orchestrator + orchestrator_stats = {} if self.orchestrator: try: all_stats = self.orchestrator.get_model_statistics() - logger.debug(f"Orchestrator model statistics: {all_stats}") + if all_stats: + orchestrator_stats = all_stats + logger.debug(f"Retrieved orchestrator model statistics for {len(all_stats)} models") except Exception as e: logger.debug(f"Error getting orchestrator model statistics: {e}") @@ -3486,28 +3505,49 @@ class CleanTradingDashboard: dqn_active = dqn_checkpoint_loaded and dqn_inference_enabled and dqn_model_available dqn_prediction_count = len(self.recent_decisions) if signal_generation_active else 0 - # Get latest DQN prediction - dqn_latest = latest_predictions.get('dqn', {}) - if dqn_latest: - last_action = dqn_latest.get('action', 'NONE') - last_confidence = dqn_latest.get('confidence', 0.72) - timestamp_val = dqn_latest.get('timestamp', datetime.now()) - if isinstance(timestamp_val, str): - last_timestamp = timestamp_val - elif hasattr(timestamp_val, 'strftime'): - last_timestamp = timestamp_val.strftime('%H:%M:%S') - else: - last_timestamp = datetime.now().strftime('%H:%M:%S') + # Get latest DQN prediction from orchestrator statistics + dqn_stats = orchestrator_stats.get('dqn_agent') + if dqn_stats and dqn_stats.predictions_history: + # Get the most recent prediction + latest_pred = list(dqn_stats.predictions_history)[-1] + last_action = latest_pred.get('action', 'NONE') + last_confidence = latest_pred.get('confidence', 0.0) + last_timestamp = latest_pred.get('timestamp', datetime.now()).strftime('%H:%M:%S') if hasattr(latest_pred.get('timestamp'), 'strftime') else datetime.now().strftime('%H:%M:%S') else: - if signal_generation_active and len(self.recent_decisions) > 0: - recent_signal = self.recent_decisions[-1] - last_action = self._get_signal_attribute(recent_signal, 'action', 'SIGNAL_GEN') - last_confidence = self._get_signal_attribute(recent_signal, 'confidence', 0.72) - last_timestamp = datetime.now().strftime('%H:%M:%S') + # Fallback to dashboard predictions + dqn_latest = latest_predictions.get('dqn', {}) + if dqn_latest: + last_action = dqn_latest.get('action', 'NONE') + last_confidence = dqn_latest.get('confidence', 0.72) + timestamp_val = dqn_latest.get('timestamp', datetime.now()) + if isinstance(timestamp_val, str): + last_timestamp = timestamp_val + elif hasattr(timestamp_val, 'strftime'): + last_timestamp = timestamp_val.strftime('%H:%M:%S') + else: + last_timestamp = datetime.now().strftime('%H:%M:%S') else: - last_action = dqn_training_status['status'] - last_confidence = 0.68 - last_timestamp = datetime.now().strftime('%H:%M:%S') + if signal_generation_active and len(self.recent_decisions) > 0: + recent_signal = self.recent_decisions[-1] + last_action = self._get_signal_attribute(recent_signal, 'action', 'SIGNAL_GEN') + last_confidence = self._get_signal_attribute(recent_signal, 'confidence', 0.72) + last_timestamp = datetime.now().strftime('%H:%M:%S') + else: + last_action = dqn_training_status['status'] + last_confidence = 0.68 + last_timestamp = datetime.now().strftime('%H:%M:%S') + + # Get real DQN statistics from orchestrator + dqn_stats = orchestrator_stats.get('dqn_agent') + dqn_current_loss = dqn_stats.current_loss if dqn_stats else None + dqn_best_loss = dqn_stats.best_loss if dqn_stats else None + dqn_accuracy = dqn_stats.accuracy if dqn_stats else None + dqn_total_inferences = dqn_stats.total_inferences if dqn_stats else 0 + dqn_total_trainings = dqn_stats.total_trainings if dqn_stats else 0 + dqn_inference_rate = dqn_stats.inference_rate_per_minute if dqn_stats else 0.0 + dqn_training_rate = dqn_stats.training_rate_per_minute if dqn_stats else 0.0 + dqn_avg_inference_time = dqn_stats.average_inference_time_ms if dqn_stats else 0.0 + dqn_avg_training_time = dqn_stats.average_training_time_ms if dqn_stats else 0.0 dqn_model_info = { 'active': dqn_active, @@ -3516,21 +3556,22 @@ class CleanTradingDashboard: 'timestamp': last_timestamp, 'action': last_action, 'confidence': last_confidence, - 'type': dqn_latest.get('type', 'dqn_signal') if dqn_latest else 'dqn_signal' + 'type': 'dqn_signal' }, - # FIXED: Get REAL loss values from orchestrator model, not placeholders - 'loss_5ma': self._get_real_model_loss('dqn'), + # REAL: Get actual loss values from orchestrator statistics + 'loss_5ma': dqn_current_loss, 'initial_loss': dqn_state.get('initial_loss'), # No fallback - show None if unknown - 'best_loss': self._get_real_best_loss('dqn'), + 'best_loss': dqn_best_loss, + 'accuracy': dqn_accuracy, 'improvement': safe_improvement_calc( dqn_state.get('initial_loss'), - self._get_real_model_loss('dqn'), + dqn_current_loss, 0.0 # No synthetic default improvement ), 'checkpoint_loaded': dqn_checkpoint_loaded, 'model_type': 'DQN', 'description': 'Deep Q-Network Agent (Data Bus Input)', - 'prediction_count': dqn_prediction_count, + 'prediction_count': dqn_total_inferences, 'epsilon': 1.0, 'training_evidence': dqn_training_status['evidence'], 'training_steps': dqn_training_status['training_steps'], @@ -3548,42 +3589,54 @@ class CleanTradingDashboard: 'created_at': dqn_state.get('created_at', 'Unknown'), 'performance_score': dqn_state.get('performance_score', 0.0) }, - # NEW: Timing information + # REAL: Timing information from orchestrator 'timing': { - 'last_inference': dqn_timing['last_inference'].strftime('%H:%M:%S') if dqn_timing['last_inference'] else 'None', - 'last_training': dqn_timing['last_training'].strftime('%H:%M:%S') if dqn_timing['last_training'] else 'None', - 'inferences_per_second': f"{dqn_timing['inferences_per_second']:.2f}", - 'predictions_24h': dqn_timing['prediction_count_24h'], - 'average_inference_time_ms': f"{dqn_timing.get('average_inference_time_ms', 0):.1f}", - 'average_training_time_ms': f"{dqn_timing.get('average_training_time_ms', 0):.1f}" + 'last_inference': dqn_stats.last_inference_time.strftime('%H:%M:%S') if dqn_stats and dqn_stats.last_inference_time else 'None', + 'last_training': dqn_stats.last_training_time.strftime('%H:%M:%S') if dqn_stats and dqn_stats.last_training_time else 'None', + 'inferences_per_second': f"{dqn_inference_rate/60:.2f}", + 'trainings_per_second': f"{dqn_training_rate/60:.2f}", + 'predictions_24h': dqn_total_inferences, + 'trainings_24h': dqn_total_trainings, + 'average_inference_time_ms': f"{dqn_avg_inference_time:.1f}", + 'average_training_time_ms': f"{dqn_avg_training_time:.1f}" }, - # NEW: Performance metrics for split-second decisions - 'performance': self.get_model_performance_metrics().get('dqn', {}), - # ENHANCED: Add toggle state information - 'inference_enabled': dqn_inference_enabled, - 'training_enabled': dqn_training_enabled, - 'status_details': { - 'checkpoint_loaded': dqn_checkpoint_loaded, + # NEW: Performance metrics for split-second decisions + 'performance': self.get_model_performance_metrics().get('dqn', {}), + # ENHANCED: Add toggle state information 'inference_enabled': dqn_inference_enabled, 'training_enabled': dqn_training_enabled, - 'is_training': dqn_training_status['is_training'] - } + 'status_details': { + 'checkpoint_loaded': dqn_checkpoint_loaded, + 'inference_enabled': dqn_inference_enabled, + 'training_enabled': dqn_training_enabled, + 'is_training': dqn_training_status['is_training'] + } } loaded_models['dqn'] = dqn_model_info - # 2. CNN Model Status - using enhanced CNN adapter data + # 2. CNN Model Status - using real orchestrator statistics cnn_state = model_states.get('cnn', {}) cnn_timing = get_model_timing_info('CNN') - # Get enhanced CNN panel data with detailed metrics - cnn_panel_data = self._update_cnn_model_panel() - cnn_active = cnn_panel_data.get('status') not in ['NOT_AVAILABLE', 'ERROR', 'NOT_LOADED'] + # Get real CNN statistics from orchestrator + cnn_stats = orchestrator_stats.get('enhanced_cnn') + cnn_active = cnn_stats is not None - # Use enhanced CNN data for display - cnn_action = cnn_panel_data.get('suggested_action', 'PATTERN_ANALYSIS') - cnn_confidence = cnn_panel_data.get('confidence', 0.0) - cnn_timestamp = cnn_panel_data.get('last_inference_time', 'Never') - cnn_pivot_price = cnn_panel_data.get('pivot_price', 'N/A') + # Get latest CNN prediction from orchestrator statistics + if cnn_stats and cnn_stats.predictions_history: + # Get the most recent prediction + latest_pred = list(cnn_stats.predictions_history)[-1] + cnn_action = latest_pred.get('action', 'PATTERN_ANALYSIS') + cnn_confidence = latest_pred.get('confidence', 0.0) + cnn_timestamp = latest_pred.get('timestamp', datetime.now()).strftime('%H:%M:%S') if hasattr(latest_pred.get('timestamp'), 'strftime') else datetime.now().strftime('%H:%M:%S') + else: + # Fallback to enhanced CNN panel data + cnn_panel_data = self._update_cnn_model_panel() + cnn_action = cnn_panel_data.get('suggested_action', 'PATTERN_ANALYSIS') + cnn_confidence = cnn_panel_data.get('confidence', 0.0) + cnn_timestamp = cnn_panel_data.get('last_inference_time', 'Never') + + cnn_pivot_price = 'N/A' # Will be updated from panel data if needed # Parse pivot price for prediction cnn_predicted_price = 0 @@ -3598,6 +3651,17 @@ class CleanTradingDashboard: cnn_inference_enabled = cnn_toggle_state.get("inference_enabled", True) cnn_training_enabled = cnn_toggle_state.get("training_enabled", True) + # Get real CNN statistics from orchestrator + cnn_current_loss = cnn_stats.current_loss if cnn_stats else None + cnn_best_loss = cnn_stats.best_loss if cnn_stats else None + cnn_accuracy = cnn_stats.accuracy if cnn_stats else None + cnn_total_inferences = cnn_stats.total_inferences if cnn_stats else 0 + cnn_total_trainings = cnn_stats.total_trainings if cnn_stats else 0 + cnn_inference_rate = cnn_stats.inference_rate_per_minute if cnn_stats else 0.0 + cnn_training_rate = cnn_stats.training_rate_per_minute if cnn_stats else 0.0 + cnn_avg_inference_time = cnn_stats.average_inference_time_ms if cnn_stats else 0.0 + cnn_avg_training_time = cnn_stats.average_training_time_ms if cnn_stats else 0.0 + cnn_model_info = { 'active': cnn_active, 'parameters': 50000000, # ~50M params @@ -3609,25 +3673,27 @@ class CleanTradingDashboard: 'pivot_price': cnn_pivot_price, 'type': 'enhanced_cnn_pivot' }, - 'loss_5ma': float(cnn_panel_data.get('last_training_loss', '0.0').replace('f', '')), + # REAL: Get actual loss values from orchestrator statistics + 'loss_5ma': cnn_current_loss, 'initial_loss': cnn_state.get('initial_loss'), - 'best_loss': cnn_state.get('best_loss'), + 'best_loss': cnn_best_loss, + 'accuracy': cnn_accuracy, 'improvement': safe_improvement_calc( cnn_state.get('initial_loss'), - float(cnn_panel_data.get('last_training_loss', '0.0').replace('f', '')), + cnn_current_loss, 0.0 ), # Enhanced timing metrics 'enhanced_timing': { - 'last_inference_time': cnn_panel_data.get('last_inference_time', 'Never'), - 'last_inference_duration': cnn_panel_data.get('last_inference_duration', '0.0ms'), - 'inference_count': cnn_panel_data.get('inference_count', 0), - 'inference_rate': cnn_panel_data.get('inference_rate', '0.00/s'), - 'last_training_time': cnn_panel_data.get('last_training_time', 'Never'), - 'last_training_duration': cnn_panel_data.get('last_training_duration', '0.0ms'), - 'training_count': cnn_panel_data.get('training_count', 0), - 'training_samples': cnn_panel_data.get('training_samples', 0) + 'last_inference_time': cnn_stats.last_inference_time.strftime('%H:%M:%S') if cnn_stats and cnn_stats.last_inference_time else 'Never', + 'last_inference_duration': f"{cnn_avg_inference_time:.1f}ms", + 'inference_count': cnn_total_inferences, + 'inference_rate': f"{cnn_inference_rate/60:.2f}/s", + 'last_training_time': cnn_stats.last_training_time.strftime('%H:%M:%S') if cnn_stats and cnn_stats.last_training_time else 'Never', + 'last_training_duration': f"{cnn_avg_training_time:.1f}ms", + 'training_count': cnn_total_trainings, + 'training_samples': cnn_total_trainings }, 'checkpoint_loaded': cnn_state.get('checkpoint_loaded', False), 'model_type': 'CNN', @@ -3639,14 +3705,16 @@ class CleanTradingDashboard: 'created_at': cnn_state.get('created_at', 'Unknown'), 'performance_score': cnn_state.get('performance_score', 0.0) }, - # NEW: Timing information + # REAL: Timing information from orchestrator 'timing': { - 'last_inference': cnn_timing['last_inference'].strftime('%H:%M:%S') if cnn_timing['last_inference'] else 'None', - 'last_training': cnn_timing['last_training'].strftime('%H:%M:%S') if cnn_timing['last_training'] else 'None', - 'inferences_per_second': f"{cnn_timing['inferences_per_second']:.2f}", - 'predictions_24h': cnn_timing['prediction_count_24h'], - 'average_inference_time_ms': f"{cnn_timing.get('average_inference_time_ms', 0):.1f}", - 'average_training_time_ms': f"{cnn_timing.get('average_training_time_ms', 0):.1f}" + 'last_inference': cnn_stats.last_inference_time.strftime('%H:%M:%S') if cnn_stats and cnn_stats.last_inference_time else 'None', + 'last_training': cnn_stats.last_training_time.strftime('%H:%M:%S') if cnn_stats and cnn_stats.last_training_time else 'None', + 'inferences_per_second': f"{cnn_inference_rate/60:.2f}", + 'trainings_per_second': f"{cnn_training_rate/60:.2f}", + 'predictions_24h': cnn_total_inferences, + 'trainings_24h': cnn_total_trainings, + 'average_inference_time_ms': f"{cnn_avg_inference_time:.1f}", + 'average_training_time_ms': f"{cnn_avg_training_time:.1f}" }, # NEW: Performance metrics for split-second decisions 'performance': self.get_model_performance_metrics().get('cnn', {}), @@ -6362,75 +6430,34 @@ class CleanTradingDashboard: return default def _get_real_model_loss(self, model_name: str) -> Optional[float]: - """Get REAL current loss from the actual model, not placeholders""" + """Get REAL current loss from the orchestrator's model statistics""" try: if not self.orchestrator: return None # No orchestrator = no real data - if model_name == 'dqn' and hasattr(self.orchestrator, 'rl_agent') and self.orchestrator.rl_agent: - # Get real loss from DQN agent - agent = self.orchestrator.rl_agent - if hasattr(agent, 'losses') and len(agent.losses) > 0: - # Average of last 50 losses for current loss - recent_losses = agent.losses[-50:] - return sum(recent_losses) / len(recent_losses) - elif hasattr(agent, 'current_loss') and agent.current_loss is not None: - return float(agent.current_loss) - - elif model_name == 'cnn' and hasattr(self.orchestrator, 'cnn_model') and self.orchestrator.cnn_model: - # Get real loss from CNN model - model = self.orchestrator.cnn_model - if hasattr(model, 'training_losses') and len(getattr(model, 'training_losses',[])) > 0: - recent_losses = getattr(model, 'training_losses',[])[-50:] - return sum(recent_losses) / len(recent_losses) - elif hasattr(model, 'current_loss') and model.current_loss is not None: - return float(model.current_loss) + # Use the orchestrator's model statistics system + model_stats = self.orchestrator.get_model_statistics(model_name) + if model_stats: + return model_stats.current_loss - elif model_name == 'decision' and hasattr(self.orchestrator, 'decision_fusion_network'): - # Get real loss from decision fusion - if hasattr(self.orchestrator, 'fusion_training_data') and len(self.orchestrator.fusion_training_data) > 0: - recent_losses = [entry['loss'] for entry in self.orchestrator.fusion_training_data[-50:]] - if recent_losses: - return sum(recent_losses) / len(recent_losses) - - # Fallback to model states - model_states = self.orchestrator.get_model_states() if hasattr(self.orchestrator, 'get_model_states') else {} - state = model_states.get(model_name, {}) - return state.get('current_loss') # Return None if no real data + return None # Return None if no real data except Exception as e: logger.debug(f"Error getting real loss for {model_name}: {e}") return None # Return None instead of synthetic data def _get_real_best_loss(self, model_name: str) -> Optional[float]: - """Get REAL best loss from the actual model""" + """Get REAL best loss from the orchestrator's model statistics""" try: if not self.orchestrator: return None # No orchestrator = no real data - if model_name == 'dqn' and hasattr(self.orchestrator, 'rl_agent') and self.orchestrator.rl_agent: - agent = self.orchestrator.rl_agent - if hasattr(agent, 'best_loss') and agent.best_loss is not None: - return float(agent.best_loss) - elif hasattr(agent, 'losses') and len(agent.losses) > 0: - return min(agent.losses) - - elif model_name == 'cnn' and hasattr(self.orchestrator, 'cnn_model') and self.orchestrator.cnn_model: - model = self.orchestrator.cnn_model - if hasattr(model, 'best_loss') and model.best_loss is not None: - return float(model.best_loss) - elif hasattr(model, 'training_losses') and len(getattr(model, 'training_losses', [])) > 0: - return min(getattr(model, 'training_losses', [])) + # Use the orchestrator's model statistics system + model_stats = self.orchestrator.get_model_statistics(model_name) + if model_stats: + return model_stats.best_loss - elif model_name == 'decision' and hasattr(self.orchestrator, 'fusion_training_data'): - if len(self.orchestrator.fusion_training_data) > 0: - all_losses = [entry['loss'] for entry in self.orchestrator.fusion_training_data] - return min(all_losses) if all_losses else None - - # Fallback to model states - model_states = self.orchestrator.get_model_states() if hasattr(self.orchestrator, 'get_model_states') else {} - state = model_states.get(model_name, {}) - return state.get('best_loss') # Return None if no real data + return None # Return None if no real data except Exception as e: logger.debug(f"Error getting best loss for {model_name}: {e}")