wip training

ANNOTATE/core/real_training_adapter.py

@@ -570,6 +570,62 @@ class RealTrainingAdapter:
                     except Exception as e:
                         logger.debug(f"       {timeframe}: Replay failed: {e}")
                 
+                # CRITICAL FIX: For 1s timeframe, if we still don't have enough data, generate from 1m candles
+                if timeframe == '1s' and (df is None or df.empty or len(df) < min_required_candles):
+                    try:
+                        df_1m = None
+                        
+                        # First, try to get 1m data from already fetched timeframes
+                        if '1m' in fetched_timeframes and '1m' in market_state.get('timeframes', {}):
+                            # Convert dict format to DataFrame
+                            tf_1m = market_state['timeframes']['1m']
+                            try:
+                                df_1m = pd.DataFrame({
+                                    'open': tf_1m['open'],
+                                    'high': tf_1m['high'],
+                                    'low': tf_1m['low'],
+                                    'close': tf_1m['close'],
+                                    'volume': tf_1m['volume']
+                                }, index=pd.to_datetime(tf_1m['timestamps'], utc=True))
+                            except Exception as e:
+                                logger.debug(f"       {timeframe}: Could not convert 1m dict to DataFrame: {e}")
+                        
+                        # If we don't have 1m data yet, fetch it
+                        if df_1m is None or df_1m.empty:
+                            logger.debug(f"       {timeframe}: Fetching 1m data to generate 1s candles...")
+                            if duckdb_storage:
+                                try:
+                                    df_1m = duckdb_storage.get_ohlcv_data(
+                                        symbol=symbol,
+                                        timeframe='1m',
+                                        start_time=tf_start_time,
+                                        end_time=tf_end_time,
+                                        limit=limit,
+                                        direction='before'
+                                    )
+                                except Exception as e:
+                                    logger.debug(f"       {timeframe}: Could not get 1m from DuckDB: {e}")
+                            
+                            if df_1m is None or df_1m.empty:
+                                # Try API for 1m
+                                df_1m = self._fetch_historical_from_api(symbol, '1m', tf_start_time, tf_end_time, limit)
+                        
+                        # Generate 1s candles from 1m if we have 1m data
+                        if df_1m is not None and not df_1m.empty:
+                            # Generate 1s candles from 1m
+                            generated_1s = self._generate_1s_from_1m(df_1m, min_required_candles)
+                            if generated_1s is not None and len(generated_1s) >= min_required_candles:
+                                df = generated_1s
+                                logger.info(f"       {timeframe}: Generated {len(df)} candles from {len(df_1m)} 1m candles")
+                            else:
+                                logger.warning(f"       {timeframe}: Generated only {len(generated_1s) if generated_1s is not None else 0} candles from 1m (need {min_required_candles})")
+                        else:
+                            logger.debug(f"       {timeframe}: No 1m data available to generate 1s candles")
+                    except Exception as e:
+                        logger.warning(f"       {timeframe}: Failed to generate from 1m: {e}")
+                        import traceback
+                        logger.debug(traceback.format_exc())
+                
                 # Validate data quality before storing
                 if df is not None and not df.empty:
                     # Check minimum candle count
@@ -1390,6 +1446,122 @@ class RealTrainingAdapter:
             state_size = agent.state_size if hasattr(agent, 'state_size') else 100
             return [0.0] * state_size
     
+    def _generate_1s_from_1m(self, df_1m: pd.DataFrame, min_candles: int) -> Optional[pd.DataFrame]:
+        """
+        Generate 1s candles from 1m candles by splitting each 1m candle into 60 1s candles.
+        
+        Args:
+            df_1m: DataFrame with 1m OHLCV data
+            min_candles: Minimum number of 1s candles to generate
+            
+        Returns:
+            DataFrame with 1s OHLCV data or None if generation fails
+        """
+        import pandas as pd
+        from datetime import timedelta
+        
+        try:
+            if df_1m is None or df_1m.empty:
+                return None
+            
+            # Ensure we have required columns
+            required_cols = ['open', 'high', 'low', 'close', 'volume']
+            if not all(col in df_1m.columns for col in required_cols):
+                logger.warning("1m DataFrame missing required columns for 1s generation")
+                return None
+            
+            # Generate 1s candles from each 1m candle
+            # Each 1m candle becomes 60 1s candles
+            candles_1s = []
+            
+            for idx, row in df_1m.iterrows():
+                # Get timestamp (handle both index and column)
+                if isinstance(df_1m.index, pd.DatetimeIndex):
+                    timestamp = idx
+                elif 'timestamp' in df_1m.columns:
+                    timestamp = pd.to_datetime(row['timestamp'])
+                else:
+                    logger.warning("Cannot determine timestamp for 1m candle")
+                    continue
+                
+                # Extract OHLCV values
+                open_price = float(row['open'])
+                high_price = float(row['high'])
+                low_price = float(row['low'])
+                close_price = float(row['close'])
+                volume = float(row['volume'])
+                
+                # Calculate price change per second (linear interpolation)
+                price_change = close_price - open_price
+                price_per_second = price_change / 60.0
+                
+                # Volume per second (distributed evenly)
+                volume_per_second = volume / 60.0
+                
+                # Generate 60 1s candles from this 1m candle
+                for second in range(60):
+                    # Calculate timestamp for this second
+                    candle_time = timestamp + timedelta(seconds=second)
+                    
+                    # Interpolate price (linear from open to close)
+                    if second == 0:
+                        candle_open = open_price
+                        candle_close = open_price + price_per_second
+                    elif second == 59:
+                        candle_open = open_price + (price_per_second * 59)
+                        candle_close = close_price
+                    else:
+                        candle_open = open_price + (price_per_second * second)
+                        candle_close = open_price + (price_per_second * (second + 1))
+                    
+                    # High and low: use the interpolated prices, but ensure they stay within the 1m candle's range
+                    # Add small random variation to make it more realistic (but keep within bounds)
+                    candle_high = max(candle_open, candle_close)
+                    candle_low = min(candle_open, candle_close)
+                    
+                    # Ensure high/low don't exceed the 1m candle's range
+                    candle_high = min(candle_high, high_price)
+                    candle_low = max(candle_low, low_price)
+                    
+                    # If high == low, add small spread
+                    if candle_high == candle_low:
+                        spread = (high_price - low_price) / 60.0
+                        candle_high = candle_high + (spread * 0.5)
+                        candle_low = candle_low - (spread * 0.5)
+                    
+                    candles_1s.append({
+                        'timestamp': candle_time,
+                        'open': candle_open,
+                        'high': candle_high,
+                        'low': candle_low,
+                        'close': candle_close,
+                        'volume': volume_per_second
+                    })
+            
+            if not candles_1s:
+                return None
+            
+            # Convert to DataFrame
+            df_1s = pd.DataFrame(candles_1s)
+            df_1s['timestamp'] = pd.to_datetime(df_1s['timestamp'])
+            df_1s = df_1s.set_index('timestamp')
+            
+            # Sort by timestamp
+            df_1s = df_1s.sort_index()
+            
+            # Take the most recent candles up to the limit
+            if len(df_1s) > min_candles:
+                df_1s = df_1s.tail(min_candles)
+            
+            logger.debug(f"Generated {len(df_1s)} 1s candles from {len(df_1m)} 1m candles")
+            return df_1s
+            
+        except Exception as e:
+            logger.warning(f"Error generating 1s candles from 1m: {e}")
+            import traceback
+            logger.debug(traceback.format_exc())
+            return None
+    
     def _fetch_historical_from_api(self, symbol: str, timeframe: str, start_time: datetime, end_time: datetime, limit: int) -> Optional[pd.DataFrame]:
         """
         Fetch historical OHLCV data from exchange APIs for a specific time range
@@ -3116,13 +3288,34 @@ class RealTrainingAdapter:
             logger.info(f"Stopped real-time inference: {inference_id}")
     
     def get_latest_signals(self, limit: int = 50) -> List[Dict]:
-        """Get latest inference signals from all active sessions"""
+        """Get latest inference signals from orchestrator predictions and active sessions"""
-        if not hasattr(self, 'inference_sessions'):
-            return []
-        
         all_signals = []
-        for session in self.inference_sessions.values():
+        
-            all_signals.extend(session.get('signals', []))
+        # CRITICAL FIX: Get signals from orchestrator's stored predictions (primary source)
+        if self.orchestrator and hasattr(self.orchestrator, 'recent_transformer_predictions'):
+            # Get predictions for all symbols
+            for symbol, predictions in self.orchestrator.recent_transformer_predictions.items():
+                if predictions:
+                    # Convert predictions to signal format
+                    for pred in list(predictions)[-limit:]:
+                        signal = {
+                            'timestamp': pred.get('timestamp', datetime.now(timezone.utc).isoformat()),
+                            'action': pred.get('action', 'HOLD'),
+                            'confidence': pred.get('confidence', 0.0),
+                            'current_price': pred.get('current_price', 0.0),
+                            'price': pred.get('current_price', 0.0),  # Alias for compatibility
+                            'predicted_price': pred.get('predicted_price', pred.get('current_price', 0.0)),
+                            'price_change': pred.get('price_change', 0.0),
+                            'model': 'transformer',
+                            'predicted_candle': pred.get('predicted_candle', {}),
+                            'source': pred.get('source', 'live_inference')
+                        }
+                        all_signals.append(signal)
+        
+        # Also get signals from active inference sessions (secondary source)
+        if hasattr(self, 'inference_sessions'):
+            for session in self.inference_sessions.values():
+                all_signals.extend(session.get('signals', []))
         
         # Sort by timestamp and return latest
         all_signals.sort(key=lambda x: x.get('timestamp', ''), reverse=True)

ANNOTATE/data/annotations/annotations_db.json

@@ -1,161 +1,69 @@
 {
   "annotations": [
     {
-      "annotation_id": "5d5c4354-12dd-4e0c-92a8-eff631a5dfab",
+      "annotation_id": "c6a996e4-b475-4419-b51c-93090f555744",
-      "symbol": "ETH/USDT",
-      "timeframe": "1h",
-      "entry": {
-        "timestamp": "2025-10-23 20:00",
-        "price": 3818.72,
-        "index": 5
-      },
-      "exit": {
-        "timestamp": "2025-10-24 05:00",
-        "price": 3989.2,
-        "index": 6
-      },
-      "direction": "LONG",
-      "profit_loss_pct": 4.4643231239787164,
-      "notes": "",
-      "created_at": "2025-10-24T23:35:14.215744",
-      "market_context": {
-        "entry_state": {},
-        "exit_state": {}
-      }
-    },
-    {
-      "annotation_id": "479eb310-c963-4837-b712-70e5a42afb53",
-      "symbol": "ETH/USDT",
-      "timeframe": "1h",
-      "entry": {
-        "timestamp": "2025-10-27 14:00",
-        "price": 4124.52,
-        "index": 329
-      },
-      "exit": {
-        "timestamp": "2025-10-30 20:00",
-        "price": 3680,
-        "index": 352
-      },
-      "direction": "SHORT",
-      "profit_loss_pct": 10.777496532929902,
-      "notes": "",
-      "created_at": "2025-10-31T00:35:00.543886",
-      "market_context": {
-        "entry_state": {},
-        "exit_state": {}
-      }
-    },
-    {
-      "annotation_id": "6b529132-8a3e-488d-b354-db8785ddaa71",
       "symbol": "ETH/USDT",
       "timeframe": "1m",
       "entry": {
-        "timestamp": "2025-11-11 12:07",
+        "timestamp": "2025-12-08 14:53",
-        "price": 3594.33,
+        "price": 3165.73,
-        "index": 144
+        "index": 237
       },
       "exit": {
-        "timestamp": "2025-11-11 20:46",
+        "timestamp": "2025-12-08 18:58",
-        "price": 3429.24,
+        "price": 3075.38,
-        "index": 329
+        "index": 317
       },
       "direction": "SHORT",
-      "profit_loss_pct": 4.593067414511193,
+      "profit_loss_pct": 2.8540020785095352,
       "notes": "",
-      "created_at": "2025-11-11T23:23:00.643510",
+      "created_at": "2025-12-08T20:11:31.371840+00:00",
       "market_context": {
         "entry_state": {},
         "exit_state": {}
       }
     },
     {
-      "annotation_id": "cdf32bb9-8f0f-467d-abc2-d409a7c22bcc",
+      "annotation_id": "908534ca-a803-438b-9130-47219bb8d311",
-      "symbol": "ETH/USDT",
-      "timeframe": "1m",
-      "entry": {
-        "timestamp": "2025-11-12 07:58",
-        "price": 3424.58,
-        "index": 284
-      },
-      "exit": {
-        "timestamp": "2025-11-12 11:08",
-        "price": 3546.35,
-        "index": 329
-      },
-      "direction": "LONG",
-      "profit_loss_pct": 3.5557645025083366,
-      "notes": "",
-      "created_at": "2025-11-12T13:11:31.267142",
-      "market_context": {
-        "entry_state": {},
-        "exit_state": {}
-      }
-    },
-    {
-      "annotation_id": "b01fe6b2-7724-495e-ab01-3f3d3aa0da5d",
       "symbol": "ETH/USDT",
       "timeframe": "1s",
       "entry": {
-        "timestamp": "2025-11-22 13:22:23",
+        "timestamp": "2025-12-08 20:02:03",
-        "price": 2727.52,
+        "price": 3123.03,
-        "index": 53
+        "index": 52
       },
       "exit": {
-        "timestamp": "2025-11-22 13:31:18",
+        "timestamp": "2025-12-08 20:09:30",
-        "price": 2717.9,
+        "price": 3134.03,
-        "index": 104
+        "index": 102
       },
-      "direction": "SHORT",
+      "direction": "LONG",
-      "profit_loss_pct": 0.3527013550771357,
+      "profit_loss_pct": 0.3522220407745042,
       "notes": "",
-      "created_at": "2025-11-22T15:31:43.939943",
+      "created_at": "2025-12-08T20:11:52.510067+00:00",
       "market_context": {
         "entry_state": {},
         "exit_state": {}
       }
     },
     {
-      "annotation_id": "19a566fa-63bb-4ce3-9f30-116127c9fe95",
+      "annotation_id": "7be60d4b-f7cf-4563-aed2-98d869ff361c",
-      "symbol": "ETH/USDT",
-      "timeframe": "1s",
-      "entry": {
-        "timestamp": "2025-11-22 22:25:17",
-        "price": 2761.97,
-        "index": 35
-      },
-      "exit": {
-        "timestamp": "2025-11-22 22:30:40",
-        "price": 2760.15,
-        "index": 49
-      },
-      "direction": "SHORT",
-      "profit_loss_pct": 0.06589499523889503,
-      "notes": "",
-      "created_at": "2025-11-22T22:35:55.606071+00:00",
-      "market_context": {
-        "entry_state": {},
-        "exit_state": {}
-      }
-    },
-    {
-      "annotation_id": "d8fdf474-d122-4474-b4ad-1f3829b1e46d",
       "symbol": "ETH/USDT",
       "timeframe": "1m",
       "entry": {
-        "timestamp": "2025-12-08 14:33",
+        "timestamp": "2025-12-08 14:53",
-        "price": 3178.42,
+        "price": 3165.73,
-        "index": 309
+        "index": 235
       },
       "exit": {
-        "timestamp": "2025-12-08 15:44",
+        "timestamp": "2025-12-08 18:58",
-        "price": 3088.83,
+        "price": 3075.38,
-        "index": 331
+        "index": 315
       },
       "direction": "SHORT",
-      "profit_loss_pct": 2.8186960817009754,
+      "profit_loss_pct": 2.8540020785095352,
       "notes": "",
-      "created_at": "2025-12-08T16:34:38.144316+00:00",
+      "created_at": "2025-12-08T20:27:50.266059+00:00",
       "market_context": {
         "entry_state": {},
         "exit_state": {}
@@ -163,7 +71,7 @@
     }
   ],
   "metadata": {
-    "total_annotations": 7,
+    "total_annotations": 3,
-    "last_updated": "2025-12-08T16:34:38.145818+00:00"
+    "last_updated": "2025-12-08T20:27:50.267314+00:00"
   }
 }

ANNOTATE/web/app.py

@@ -2559,13 +2559,26 @@ class AnnotationDashboard:
                 
                 signals = self.training_adapter.get_latest_signals()
                 
-                # Get metrics from active inference sessions
+                # Get metrics from active inference sessions or orchestrator
                 metrics = {'accuracy': 0.0, 'loss': 0.0}
-                if hasattr(self.training_adapter, 'inference_sessions'):
+                
-                    for session in self.training_adapter.inference_sessions.values():
+                # Try to get metrics from orchestrator first (most recent)
-                        if 'metrics' in session:
+                if self.orchestrator and hasattr(self.orchestrator, 'primary_transformer_trainer'):
-                            metrics = session['metrics']
+                    trainer = self.orchestrator.primary_transformer_trainer
-                            break
+                    if trainer and hasattr(trainer, 'training_history'):
+                        history = trainer.training_history
+                        if history.get('train_accuracy'):
+                            metrics['accuracy'] = history['train_accuracy'][-1] if history['train_accuracy'] else 0.0
+                        if history.get('train_loss'):
+                            metrics['loss'] = history['train_loss'][-1] if history['train_loss'] else 0.0
+                
+                # Fallback to inference session metrics
+                if metrics['accuracy'] == 0.0 and metrics['loss'] == 0.0:
+                    if hasattr(self.training_adapter, 'inference_sessions'):
+                        for session in self.training_adapter.inference_sessions.values():
+                            if 'metrics' in session and session['metrics']:
+                                metrics = session['metrics'].copy()
+                                break
                 
                 return jsonify({
                     'success': True,

ANNOTATE/web/templates/components/training_panel.html

@@ -1236,7 +1236,21 @@
                             const accuracyPct = (data.metrics.accuracy * 100).toFixed(1);
                             const lossVal = data.metrics.loss ? data.metrics.loss.toFixed(4) : '--';
                             
-                            document.getElementById('metric-accuracy').textContent = accuracyPct + '%';
+                            // CRITICAL FIX: Update live-accuracy and live-loss elements
+                            const liveAccuracyEl = document.getElementById('live-accuracy');
+                            const liveLossEl = document.getElementById('live-loss');
+                            if (liveAccuracyEl) {
+                                liveAccuracyEl.textContent = accuracyPct + '%';
+                            }
+                            if (liveLossEl) {
+                                liveLossEl.textContent = lossVal;
+                            }
+                            
+                            // Also update metric-accuracy if it exists
+                            const metricAccuracyEl = document.getElementById('metric-accuracy');
+                            if (metricAccuracyEl) {
+                                metricAccuracyEl.textContent = accuracyPct + '%';
+                            }
                             
                             // Update live banner with metrics
                             const banner = document.getElementById('inference-status');