WIP oclcv in storage. migrate do duckdb

ANNOTATE/web/app.py

@@ -163,6 +163,10 @@ class AnnotationDashboard:
         # Setup routes
         self._setup_routes()
         
+        # Start background data refresh after startup
+        if self.data_loader:
+            self._start_background_data_refresh()
+        
         logger.info("Annotation Dashboard initialized")
     
     def _enable_unified_storage_async(self):
@@ -201,6 +205,58 @@ class AnnotationDashboard:
         storage_thread = threading.Thread(target=enable_storage, daemon=True)
         storage_thread.start()
     
+    def _start_background_data_refresh(self):
+        """Start background task to refresh recent data after startup"""
+        def refresh_recent_data():
+            try:
+                import time
+                # Wait for app to fully start
+                time.sleep(5)
+                
+                logger.info("🔄 Starting background data refresh (fetching only recent missing data)")
+                
+                # Disable startup mode to fetch fresh data
+                self.data_loader.disable_startup_mode()
+                
+                # Fetch only last 5 minutes of 1m data and 300 seconds of 1s data
+                symbols = self.config.get('symbols', ['ETH/USDT', 'BTC/USDT'])
+                
+                for symbol in symbols:
+                    try:
+                        # Fetch last 5 candles of 1m data (5 minutes)
+                        logger.info(f"Refreshing recent 1m data for {symbol}")
+                        self.data_provider.get_historical_data(
+                            symbol=symbol,
+                            timeframe='1m',
+                            limit=5,
+                            refresh=True
+                        )
+                        
+                        # Fetch last 300 candles of 1s data (5 minutes)
+                        logger.info(f"Refreshing recent 1s data for {symbol}")
+                        self.data_provider.get_historical_data(
+                            symbol=symbol,
+                            timeframe='1s',
+                            limit=300,
+                            refresh=True
+                        )
+                        
+                        logger.info(f"✅ Refreshed recent data for {symbol}")
+                        
+                    except Exception as e:
+                        logger.warning(f"Could not refresh recent data for {symbol}: {e}")
+                
+                logger.info("✅ Background data refresh completed")
+                
+            except Exception as e:
+                logger.error(f"Error in background data refresh: {e}")
+        
+        # Start in background thread
+        import threading
+        refresh_thread = threading.Thread(target=refresh_recent_data, daemon=True)
+        refresh_thread.start()
+        logger.info("📊 Background data refresh scheduled")
+    
     def _get_pivot_markers_for_timeframe(self, symbol: str, timeframe: str, df: pd.DataFrame) -> dict:
         """
         Get pivot markers for a specific timeframe using WilliamsMarketStructure directly
@@ -526,8 +582,38 @@ class AnnotationDashboard:
                     exit_market_state=exit_market_state
                 )
                 
-                # Save annotation
-                self.annotation_manager.save_annotation(annotation)
+                # Collect market snapshots for SQLite storage
+                market_snapshots = {}
+                if self.data_loader:
+                    try:
+                        # Get OHLCV data for all timeframes around the annotation time
+                        entry_time = datetime.fromisoformat(data['entry']['timestamp'].replace('Z', '+00:00'))
+                        exit_time = datetime.fromisoformat(data['exit']['timestamp'].replace('Z', '+00:00'))
+                        
+                        # Get data from 5 minutes before entry to 5 minutes after exit
+                        start_time = entry_time - timedelta(minutes=5)
+                        end_time = exit_time + timedelta(minutes=5)
+                        
+                        for timeframe in ['1s', '1m', '1h', '1d']:
+                            df = self.data_loader.get_data(
+                                symbol=data['symbol'],
+                                timeframe=timeframe,
+                                start_time=start_time,
+                                end_time=end_time,
+                                limit=1500
+                            )
+                            if df is not None and not df.empty:
+                                market_snapshots[timeframe] = df
+                        
+                        logger.info(f"Collected {len(market_snapshots)} timeframes for annotation storage")
+                    except Exception as e:
+                        logger.error(f"Error collecting market snapshots: {e}")
+                
+                # Save annotation with market snapshots
+                self.annotation_manager.save_annotation(
+                    annotation=annotation,
+                    market_snapshots=market_snapshots
+                )
                 
                 # Automatically generate test case with ±5min data
                 try: