modedl ckpt tooltips

core/data_provider.py

@@ -1267,7 +1267,23 @@ class DataProvider:
             return df
             
         except Exception as e:
-            logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}")
+            # Handle Windows-specific invalid argument (e.g., bad timestamps) gracefully
+            try:
+                import errno
+                if hasattr(e, 'errno') and e.errno == errno.EINVAL:
+                    logger.warning(f"Invalid argument while generating 1s candles for {symbol}; trimming tick buffer and falling back")
+                    try:
+                        if hasattr(self, 'cob_raw_ticks') and symbol in getattr(self, 'cob_raw_ticks', {}):
+                            buf = self.cob_raw_ticks[symbol]
+                            drop = max(1, len(buf)//2)
+                            for _ in range(drop):
+                                buf.popleft()
+                    except Exception:
+                        pass
+                else:
+                    logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}")
+            except Exception:
+                logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}")
             return None
     
     def _fetch_from_binance(self, symbol: str, timeframe: str, limit: int) -> Optional[pd.DataFrame]:

core/orchestrator.py

@@ -81,11 +81,21 @@ except Exception:
         from enhanced_realtime_training import EnhancedRealtimeTrainingSystem  # type: ignore
         ENHANCED_TRAINING_AVAILABLE = True
     except Exception:
-        EnhancedRealtimeTrainingSystem = None  # type: ignore
-        ENHANCED_TRAINING_AVAILABLE = False
-        logging.warning(
-            "EnhancedRealtimeTrainingSystem not found. Real-time training features will be disabled."
-        )
+        # Dynamic sys.path injection as last resort
+        try:
+            import sys, os
+            current_dir = os.path.dirname(os.path.abspath(__file__))
+            nn_training_dir = os.path.normpath(os.path.join(current_dir, "..", "NN", "training"))
+            if nn_training_dir not in sys.path:
+                sys.path.insert(0, nn_training_dir)
+            from enhanced_realtime_training import EnhancedRealtimeTrainingSystem  # type: ignore
+            ENHANCED_TRAINING_AVAILABLE = True
+        except Exception:
+            EnhancedRealtimeTrainingSystem = None  # type: ignore
+            ENHANCED_TRAINING_AVAILABLE = False
+            logging.warning(
+                "EnhancedRealtimeTrainingSystem not found. Real-time training features will be disabled."
+            )
 
 logger = logging.getLogger(__name__)
 

web/models_training_panel.py

@@ -90,6 +90,13 @@ class ModelsTrainingPanel:
                     if model_states and model_name in model_states:
                         ckpt_loaded = bool(model_states[model_name].get("checkpoint_loaded", False))
                         ckpt_failed = bool(model_states[model_name].get("checkpoint_failed", False))
+                        # If model started fresh, mark as stored after first successful autosave/best_loss update
+                        if not ckpt_loaded:
+                            cur = model_states[model_name].get("current_loss")
+                            best = model_states[model_name].get("best_loss")
+                            if isinstance(cur, (int, float)) and isinstance(best, (int, float)) and best is not None and cur is not None and cur <= best:
+                                ckpt_loaded = True
+                        checkpoint_filename = model_states[model_name].get("checkpoint_filename")
 
                     model_info = {
                         "name": model_name,
@@ -105,6 +112,7 @@ class ModelsTrainingPanel:
                         "routing_enabled": True,
                         "checkpoint_loaded": ckpt_loaded,
                         "checkpoint_failed": ckpt_failed,
+                        "checkpoint_filename": checkpoint_filename if model_states and model_name in model_states else None,
                         "loss_metrics": loss_metrics,
                         "timing_metrics": timing_metrics,
                         "signal_stats": {},
@@ -178,11 +186,18 @@ class ModelsTrainingPanel:
         timing = model_data.get("timing_metrics", {})
         rate = timing.get("inferences_per_second")
 
+        # Tooltip title showing checkpoint info (if any)
+        title_parts: List[str] = [f"{status_text}"]
+        ckpt_name = model_data.get('checkpoint_filename')
+        if ckpt_name:
+            title_parts.append(f"CKPT: {ckpt_name}")
+        header_title = " | ".join(title_parts)
+
         return html.Div([
             html.Div([
                 html.Div([
                     html.I(className=f"{status_icon} me-2 {status_class}"),
-                    html.Strong(f"{model_name.upper()}", className=status_class),
+                    html.Strong(f"{model_name.upper()}", className=status_class, title=header_title),
                     html.Span(f" - {status_text}", className=f"{status_class} small ms-1"),
                     html.Span(" [CKPT]" if model_data.get("checkpoint_loaded") else (" [FAILED]" if model_data.get("checkpoint_failed") else " [FRESH]"), className=f"small {'text-success' if model_data.get('checkpoint_loaded') else 'text-danger' if model_data.get('checkpoint_failed') else 'text-warning'} ms-1")
                 ], style={"flex": "1"}),