diff --git a/core/data_provider.py b/core/data_provider.py index 0c96e11..3b41007 100644 --- a/core/data_provider.py +++ b/core/data_provider.py @@ -1267,7 +1267,23 @@ class DataProvider: return df except Exception as e: - logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}") + # Handle Windows-specific invalid argument (e.g., bad timestamps) gracefully + try: + import errno + if hasattr(e, 'errno') and e.errno == errno.EINVAL: + logger.warning(f"Invalid argument while generating 1s candles for {symbol}; trimming tick buffer and falling back") + try: + if hasattr(self, 'cob_raw_ticks') and symbol in getattr(self, 'cob_raw_ticks', {}): + buf = self.cob_raw_ticks[symbol] + drop = max(1, len(buf)//2) + for _ in range(drop): + buf.popleft() + except Exception: + pass + else: + logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}") + except Exception: + logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}") return None def _fetch_from_binance(self, symbol: str, timeframe: str, limit: int) -> Optional[pd.DataFrame]: diff --git a/core/orchestrator.py b/core/orchestrator.py index 620948a..e136d78 100644 --- a/core/orchestrator.py +++ b/core/orchestrator.py @@ -81,11 +81,21 @@ except Exception: from enhanced_realtime_training import EnhancedRealtimeTrainingSystem # type: ignore ENHANCED_TRAINING_AVAILABLE = True except Exception: - EnhancedRealtimeTrainingSystem = None # type: ignore - ENHANCED_TRAINING_AVAILABLE = False - logging.warning( - "EnhancedRealtimeTrainingSystem not found. Real-time training features will be disabled." - ) + # Dynamic sys.path injection as last resort + try: + import sys, os + current_dir = os.path.dirname(os.path.abspath(__file__)) + nn_training_dir = os.path.normpath(os.path.join(current_dir, "..", "NN", "training")) + if nn_training_dir not in sys.path: + sys.path.insert(0, nn_training_dir) + from enhanced_realtime_training import EnhancedRealtimeTrainingSystem # type: ignore + ENHANCED_TRAINING_AVAILABLE = True + except Exception: + EnhancedRealtimeTrainingSystem = None # type: ignore + ENHANCED_TRAINING_AVAILABLE = False + logging.warning( + "EnhancedRealtimeTrainingSystem not found. Real-time training features will be disabled." + ) logger = logging.getLogger(__name__) diff --git a/web/models_training_panel.py b/web/models_training_panel.py index 80d004e..f3691ad 100644 --- a/web/models_training_panel.py +++ b/web/models_training_panel.py @@ -90,6 +90,13 @@ class ModelsTrainingPanel: if model_states and model_name in model_states: ckpt_loaded = bool(model_states[model_name].get("checkpoint_loaded", False)) ckpt_failed = bool(model_states[model_name].get("checkpoint_failed", False)) + # If model started fresh, mark as stored after first successful autosave/best_loss update + if not ckpt_loaded: + cur = model_states[model_name].get("current_loss") + best = model_states[model_name].get("best_loss") + if isinstance(cur, (int, float)) and isinstance(best, (int, float)) and best is not None and cur is not None and cur <= best: + ckpt_loaded = True + checkpoint_filename = model_states[model_name].get("checkpoint_filename") model_info = { "name": model_name, @@ -105,6 +112,7 @@ class ModelsTrainingPanel: "routing_enabled": True, "checkpoint_loaded": ckpt_loaded, "checkpoint_failed": ckpt_failed, + "checkpoint_filename": checkpoint_filename if model_states and model_name in model_states else None, "loss_metrics": loss_metrics, "timing_metrics": timing_metrics, "signal_stats": {}, @@ -178,11 +186,18 @@ class ModelsTrainingPanel: timing = model_data.get("timing_metrics", {}) rate = timing.get("inferences_per_second") + # Tooltip title showing checkpoint info (if any) + title_parts: List[str] = [f"{status_text}"] + ckpt_name = model_data.get('checkpoint_filename') + if ckpt_name: + title_parts.append(f"CKPT: {ckpt_name}") + header_title = " | ".join(title_parts) + return html.Div([ html.Div([ html.Div([ html.I(className=f"{status_icon} me-2 {status_class}"), - html.Strong(f"{model_name.upper()}", className=status_class), + html.Strong(f"{model_name.upper()}", className=status_class, title=header_title), html.Span(f" - {status_text}", className=f"{status_class} small ms-1"), html.Span(" [CKPT]" if model_data.get("checkpoint_loaded") else (" [FAILED]" if model_data.get("checkpoint_failed") else " [FRESH]"), className=f"small {'text-success' if model_data.get('checkpoint_loaded') else 'text-danger' if model_data.get('checkpoint_failed') else 'text-warning'} ms-1") ], style={"flex": "1"}),