fix pivot cache db

core/duckdb_storage.py

@@ -245,6 +245,84 @@ class DuckDBStorage:
             traceback.print_exc()
             return None
     
+    def get_last_timestamp(self, symbol: str, timeframe: str) -> Optional[datetime]:
+        """
+        Get the last timestamp for a symbol/timeframe from DuckDB
+        
+        Args:
+            symbol: Trading symbol
+            timeframe: Timeframe
+            
+        Returns:
+            Last timestamp or None if no data exists
+        """
+        try:
+            query = """
+                SELECT MAX(timestamp) as last_timestamp
+                FROM ohlcv_data
+                WHERE symbol = ? AND timeframe = ?
+            """
+            
+            result = self.conn.execute(query, [symbol, timeframe]).fetchone()
+            
+            if result and result[0] is not None:
+                last_timestamp = pd.to_datetime(result[0], unit='ms', utc=True)
+                logger.debug(f"Last timestamp for {symbol} {timeframe}: {last_timestamp}")
+                return last_timestamp
+            
+            return None
+            
+        except Exception as e:
+            logger.error(f"Error getting last timestamp for {symbol} {timeframe}: {e}")
+            return None
+    
+    def get_ohlcv_data_since_timestamp(self, symbol: str, timeframe: str, 
+                                     since_timestamp: datetime, 
+                                     limit: int = 1500) -> Optional[pd.DataFrame]:
+        """
+        Get OHLCV data since a specific timestamp, capped at limit
+        
+        Args:
+            symbol: Trading symbol
+            timeframe: Timeframe
+            since_timestamp: Get data since this timestamp
+            limit: Maximum number of candles (default 1500)
+            
+        Returns:
+            DataFrame with OHLCV data since timestamp
+        """
+        try:
+            query = """
+                SELECT timestamp, open, high, low, close, volume
+                FROM ohlcv_data
+                WHERE symbol = ? AND timeframe = ? AND timestamp > ?
+                ORDER BY timestamp ASC
+                LIMIT ?
+            """
+            
+            params = [
+                symbol, 
+                timeframe, 
+                int(since_timestamp.timestamp() * 1000),
+                limit
+            ]
+            
+            df = self.conn.execute(query, params).df()
+            
+            if df.empty:
+                return None
+            
+            # Convert timestamp to datetime
+            df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
+            df = df.set_index('timestamp')
+            
+            logger.debug(f"Retrieved {len(df)} candles for {symbol} {timeframe} since {since_timestamp}")
+            return df
+            
+        except Exception as e:
+            logger.error(f"Error retrieving OHLCV data since timestamp: {e}")
+            return None
+    
     def store_annotation(self, annotation_id: str, annotation_data: Dict[str, Any],
                         market_snapshots: Dict[str, pd.DataFrame],
                         model_predictions: Optional[List[Dict]] = None) -> bool: