infinite scroll fix

ANNOTATE/core/data_loader.py

@@ -44,7 +44,8 @@ class HistoricalDataLoader:
     def get_data(self, symbol: str, timeframe: str, 
                  start_time: Optional[datetime] = None,
                  end_time: Optional[datetime] = None,
-                 limit: int = 500) -> Optional[pd.DataFrame]:
+                 limit: int = 500,
+                 direction: str = 'latest') -> Optional[pd.DataFrame]:
         """
         Get historical data for symbol and timeframe
         
@@ -54,12 +55,13 @@ class HistoricalDataLoader:
             start_time: Start time for data range
             end_time: End time for data range
             limit: Maximum number of candles to return
+            direction: 'latest' (most recent), 'before' (older data), 'after' (newer data)
             
         Returns:
             DataFrame with OHLCV data or None if unavailable
         """
         # Check memory cache first
-        cache_key = f"{symbol}_{timeframe}_{start_time}_{end_time}_{limit}"
+        cache_key = f"{symbol}_{timeframe}_{start_time}_{end_time}_{limit}_{direction}"
         if cache_key in self.memory_cache:
             cached_data, cached_time = self.memory_cache[cache_key]
             if datetime.now() - cached_time < self.cache_ttl:
@@ -152,8 +154,7 @@ class HistoricalDataLoader:
                             logger.info(f"Loaded {len(df)} candles for {symbol} {timeframe}")
                             return df
             
-            # Fallback: fetch from DataProvider's historical data method
+            # Fallback: Try DuckDB first, then fetch from API if needed
-            # During startup, allow stale cache to avoid slow API calls
             if self.startup_mode:
                 logger.info(f"Loading data for {symbol} {timeframe} (startup mode: allow stale cache)")
                 df = self.data_provider.get_historical_data(
@@ -163,11 +164,33 @@ class HistoricalDataLoader:
                     allow_stale_cache=True
                 )
             else:
-                logger.info(f"Fetching fresh data for {symbol} {timeframe}")
+                # Check DuckDB first for historical data
+                if self.data_provider.duckdb_storage and (start_time or end_time):
+                    logger.info(f"Checking DuckDB for {symbol} {timeframe} historical data (direction={direction})")
+                    df = self.data_provider.duckdb_storage.get_ohlcv_data(
+                        symbol=symbol,
+                        timeframe=timeframe,
+                        start_time=start_time,
+                        end_time=end_time,
+                        limit=limit,
+                        direction=direction
+                    )
+                    
+                    if df is not None and not df.empty:
+                        logger.info(f"✅ Loaded {len(df)} candles from DuckDB for {symbol} {timeframe}")
+                        # Cache in memory
+                        self.memory_cache[cache_key] = (df.copy(), datetime.now())
+                        return df
+                    else:
+                        logger.info(f"No data in DuckDB, fetching from API for {symbol} {timeframe}")
+                
+                # Fetch from API and store in DuckDB
+                logger.info(f"Fetching data from API for {symbol} {timeframe}")
                 df = self.data_provider.get_historical_data(
                     symbol=symbol,
                     timeframe=timeframe,
-                    limit=limit
+                    limit=limit,
+                    refresh=True  # Force API fetch
                 )
             
             if df is not None and not df.empty:

ANNOTATE/data/annotations/annotations_db.json

@@ -1,69 +1,23 @@
 {
   "annotations": [
     {
-      "annotation_id": "967f91f4-5f01-4608-86af-4a006d55bd3c",
+      "annotation_id": "dc35c362-6174-4db4-b4db-8cc58a4ba8e5",
       "symbol": "ETH/USDT",
-      "timeframe": "1m",
+      "timeframe": "1h",
       "entry": {
-        "timestamp": "2025-10-23 14:15",
+        "timestamp": "2025-10-07 13:00",
-        "price": 3821.57,
+        "price": 4755,
-        "index": 421
+        "index": 28
       },
       "exit": {
-        "timestamp": "2025-10-23 15:32",
+        "timestamp": "2025-10-11 21:00",
-        "price": 3874.23,
+        "price": 3643.33,
-        "index": 498
+        "index": 63
       },
-      "direction": "LONG",
+      "direction": "SHORT",
-      "profit_loss_pct": 1.377967693905904,
+      "profit_loss_pct": 23.378969505783388,
       "notes": "",
-      "created_at": "2025-10-23T18:36:05.807749",
+      "created_at": "2025-10-24T22:33:26.187249",
-      "market_context": {
-        "entry_state": {},
-        "exit_state": {}
-      }
-    },
-    {
-      "annotation_id": "a16f92c0-7228-4293-a188-baf74ca9b284",
-      "symbol": "ETH/USDT",
-      "timeframe": "1m",
-      "entry": {
-        "timestamp": "2025-10-24 08:31",
-        "price": 3930.54,
-        "index": 101
-      },
-      "exit": {
-        "timestamp": "2025-10-24 08:46",
-        "price": 3965,
-        "index": 104
-      },
-      "direction": "LONG",
-      "profit_loss_pct": 0.8767243177782196,
-      "notes": "",
-      "created_at": "2025-10-24T12:56:59.390345",
-      "market_context": {
-        "entry_state": {},
-        "exit_state": {}
-      }
-    },
-    {
-      "annotation_id": "ee34388a-fcad-4c7e-8a0a-2e8d205d5357",
-      "symbol": "ETH/USDT",
-      "timeframe": "1m",
-      "entry": {
-        "timestamp": "2025-10-24 04:06",
-        "price": 3895.67,
-        "index": 18
-      },
-      "exit": {
-        "timestamp": "2025-10-24 05:24",
-        "price": 3965,
-        "index": 36
-      },
-      "direction": "LONG",
-      "profit_loss_pct": 1.7796681957147276,
-      "notes": "",
-      "created_at": "2025-10-24T13:55:31.843201",
       "market_context": {
         "entry_state": {},
         "exit_state": {}
@@ -71,7 +25,7 @@
     }
   ],
   "metadata": {
-    "total_annotations": 5,
+    "total_annotations": 1,
-    "last_updated": "2025-10-24T13:55:31.843201"
+    "last_updated": "2025-10-24T22:33:26.194492"
   }
 }

ANNOTATE/web/app.py

@@ -423,13 +423,15 @@ class AnnotationDashboard:
         
         @self.server.route('/api/chart-data', methods=['POST'])
         def get_chart_data():
-            """Get chart data for specified symbol and timeframes"""
+            """Get chart data for specified symbol and timeframes with infinite scroll support"""
             try:
                 data = request.get_json()
                 symbol = data.get('symbol', 'ETH/USDT')
                 timeframes = data.get('timeframes', ['1s', '1m', '1h', '1d'])
                 start_time_str = data.get('start_time')
                 end_time_str = data.get('end_time')
+                limit = data.get('limit', 500)  # Allow client to request more data
+                direction = data.get('direction', 'latest')  # 'latest', 'before', or 'after'
                 
                 if not self.data_loader:
                     return jsonify({
@@ -445,6 +447,10 @@ class AnnotationDashboard:
                 end_time = datetime.fromisoformat(end_time_str.replace('Z', '+00:00')) if end_time_str else None
                 
                 # Fetch data for each timeframe using data loader
+                # This will automatically:
+                # 1. Check DuckDB first
+                # 2. Fetch from API if not in cache
+                # 3. Store in DuckDB for future use
                 chart_data = {}
                 for timeframe in timeframes:
                     df = self.data_loader.get_data(
@@ -452,7 +458,8 @@ class AnnotationDashboard:
                         timeframe=timeframe,
                         start_time=start_time,
                         end_time=end_time,
-                        limit=500
+                        limit=limit,
+                        direction=direction
                     )
                     
                     if df is not None and not df.empty:

ANNOTATE/web/static/js/chart_manager.js

@@ -334,6 +334,11 @@ class ChartManager {
             this.updateChartInfo(timeframe, eventData);
         });
 
+        // Add relayout handler for infinite scroll (load more data when zooming/panning)
+        plotElement.on('plotly_relayout', (eventData) => {
+            this.handleChartRelayout(timeframe, eventData);
+        });
+
         console.log(`Chart created for ${timeframe} with ${data.timestamps.length} candles`);
     }
 
@@ -1088,4 +1093,234 @@ class ChartManager {
             infoElement.textContent = `O: ${open.toFixed(2)} H: ${high.toFixed(2)} L: ${low.toFixed(2)} C: ${close.toFixed(2)}`;
         }
     }
+
+    /**
+     * Handle chart relayout for infinite scroll
+     * Detects when user scrolls/zooms to edges and loads more data
+     */
+    handleChartRelayout(timeframe, eventData) {
+        const chart = this.charts[timeframe];
+        if (!chart || !chart.data) return;
+
+        // Check if this is a range change (zoom/pan)
+        if (!eventData['xaxis.range[0]'] && !eventData['xaxis.range']) return;
+
+        // Get current visible range
+        const xRange = eventData['xaxis.range'] || [eventData['xaxis.range[0]'], eventData['xaxis.range[1]']];
+        if (!xRange || xRange.length !== 2) return;
+
+        const visibleStart = new Date(xRange[0]);
+        const visibleEnd = new Date(xRange[1]);
+
+        // Get data boundaries
+        const dataStart = new Date(chart.data.timestamps[0]);
+        const dataEnd = new Date(chart.data.timestamps[chart.data.timestamps.length - 1]);
+
+        // Calculate threshold (10% of visible range from edge)
+        const visibleRange = visibleEnd - visibleStart;
+        const threshold = visibleRange * 0.1;
+
+        // Check if we're near the left edge (need older data)
+        const nearLeftEdge = (visibleStart - dataStart) < threshold;
+        
+        // Check if we're near the right edge (need newer data)
+        const nearRightEdge = (dataEnd - visibleEnd) < threshold;
+
+        console.log(`Relayout ${timeframe}: visible=${visibleStart.toISOString()} to ${visibleEnd.toISOString()}, data=${dataStart.toISOString()} to ${dataEnd.toISOString()}, nearLeft=${nearLeftEdge}, nearRight=${nearRightEdge}`);
+
+        // Load more data if near edges
+        if (nearLeftEdge) {
+            this.loadMoreData(timeframe, 'before', dataStart);
+        } else if (nearRightEdge) {
+            this.loadMoreData(timeframe, 'after', dataEnd);
+        }
+    }
+
+    /**
+     * Load more historical data for a timeframe
+     */
+    async loadMoreData(timeframe, direction, referenceTime) {
+        const chart = this.charts[timeframe];
+        if (!chart) return;
+
+        // Prevent multiple simultaneous loads
+        if (chart.loading) {
+            console.log(`Already loading data for ${timeframe}, skipping...`);
+            return;
+        }
+
+        chart.loading = true;
+        this.showLoadingIndicator(timeframe, direction);
+
+        try {
+            // Calculate time range to fetch
+            const limit = 500; // Fetch 500 more candles
+            let startTime, endTime;
+
+            if (direction === 'before') {
+                // Load older data
+                endTime = referenceTime.toISOString();
+                startTime = null; // Let backend calculate based on limit
+            } else {
+                // Load newer data
+                startTime = referenceTime.toISOString();
+                endTime = null;
+            }
+
+            console.log(`Loading ${limit} more candles ${direction} ${referenceTime.toISOString()} for ${timeframe}`);
+
+            // Fetch more data from backend
+            const response = await fetch('/api/chart-data', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    symbol: window.appState?.currentSymbol || 'ETH/USDT',
+                    timeframes: [timeframe],
+                    start_time: startTime,
+                    end_time: endTime,
+                    limit: limit,
+                    direction: direction
+                })
+            });
+
+            const result = await response.json();
+
+            if (result.success && result.chart_data && result.chart_data[timeframe]) {
+                const newData = result.chart_data[timeframe];
+                
+                // Merge with existing data
+                this.mergeChartData(timeframe, newData, direction);
+                
+                console.log(`Loaded ${newData.timestamps.length} new candles for ${timeframe}`);
+                window.showSuccess(`Loaded ${newData.timestamps.length} more candles`);
+            } else {
+                console.warn(`No more data available for ${timeframe} ${direction}`);
+                window.showWarning('No more historical data available');
+            }
+
+        } catch (error) {
+            console.error(`Error loading more data for ${timeframe}:`, error);
+            window.showError('Failed to load more data');
+        } finally {
+            chart.loading = false;
+            this.hideLoadingIndicator(timeframe);
+        }
+    }
+
+    /**
+     * Merge new data with existing chart data
+     */
+    mergeChartData(timeframe, newData, direction) {
+        const chart = this.charts[timeframe];
+        if (!chart || !chart.data) return;
+
+        const existingData = chart.data;
+        let mergedData;
+
+        if (direction === 'before') {
+            // Prepend older data
+            mergedData = {
+                timestamps: [...newData.timestamps, ...existingData.timestamps],
+                open: [...newData.open, ...existingData.open],
+                high: [...newData.high, ...existingData.high],
+                low: [...newData.low, ...existingData.low],
+                close: [...newData.close, ...existingData.close],
+                volume: [...newData.volume, ...existingData.volume],
+                pivot_markers: { ...newData.pivot_markers, ...existingData.pivot_markers }
+            };
+        } else {
+            // Append newer data
+            mergedData = {
+                timestamps: [...existingData.timestamps, ...newData.timestamps],
+                open: [...existingData.open, ...newData.open],
+                high: [...existingData.high, ...newData.high],
+                low: [...existingData.low, ...newData.low],
+                close: [...existingData.close, ...newData.close],
+                volume: [...existingData.volume, ...newData.volume],
+                pivot_markers: { ...existingData.pivot_markers, ...newData.pivot_markers }
+            };
+        }
+
+        // Update stored data
+        chart.data = mergedData;
+
+        // Update the chart with merged data
+        this.updateSingleChart(timeframe, mergedData);
+    }
+
+    /**
+     * Update a single chart with new data
+     */
+    updateSingleChart(timeframe, data) {
+        const chart = this.charts[timeframe];
+        if (!chart) return;
+
+        const plotId = chart.plotId;
+
+        // Create volume colors
+        const volumeColors = data.close.map((close, i) => {
+            if (i === 0) return '#3b82f6';
+            return close >= data.open[i] ? '#10b981' : '#ef4444';
+        });
+
+        // Update traces
+        const update = {
+            x: [data.timestamps, data.timestamps],
+            open: [data.open],
+            high: [data.high],
+            low: [data.low],
+            close: [data.close],
+            y: [undefined, data.volume],
+            'marker.color': [undefined, volumeColors]
+        };
+
+        Plotly.restyle(plotId, update, [0, 1]);
+
+        console.log(`Updated ${timeframe} chart with ${data.timestamps.length} candles`);
+    }
+
+    /**
+     * Show loading indicator on chart
+     */
+    showLoadingIndicator(timeframe, direction) {
+        const chart = this.charts[timeframe];
+        if (!chart) return;
+
+        const plotElement = chart.element;
+        const loadingDiv = document.createElement('div');
+        loadingDiv.id = `loading-${timeframe}`;
+        loadingDiv.className = 'chart-loading-indicator';
+        loadingDiv.innerHTML = `
+            <div class="spinner-border spinner-border-sm text-primary" role="status">
+                <span class="visually-hidden">Loading...</span>
+            </div>
+            <span class="ms-2">Loading ${direction === 'before' ? 'older' : 'newer'} data...</span>
+        `;
+        loadingDiv.style.cssText = `
+            position: absolute;
+            top: 10px;
+            ${direction === 'before' ? 'left' : 'right'}: 10px;
+            background: rgba(31, 41, 55, 0.9);
+            color: #f8f9fa;
+            padding: 8px 12px;
+            border-radius: 4px;
+            font-size: 12px;
+            z-index: 1000;
+            display: flex;
+            align-items: center;
+        `;
+
+        plotElement.parentElement.style.position = 'relative';
+        plotElement.parentElement.appendChild(loadingDiv);
+    }
+
+    /**
+     * Hide loading indicator
+     */
+    hideLoadingIndicator(timeframe) {
+        const loadingDiv = document.getElementById(`loading-${timeframe}`);
+        if (loadingDiv) {
+            loadingDiv.remove();
+        }
+    }
 }

ANNOTATE/web/templates/annotation_dashboard.html

@@ -62,7 +62,7 @@
     window.appState = {
         currentSymbol: '{{ current_symbol }}',
         currentTimeframes: {{ timeframes | tojson }},
-    annotations: {{ annotations | tojson }},
+    annotations: { { annotations | tojson } },
     pendingAnnotation: null,
         chartManager: null,
             annotationManager: null,
@@ -299,6 +299,26 @@
         toast.addEventListener('hidden.bs.toast', () => toast.remove());
     }
 
+    function showWarning(message) {
+        const toast = document.createElement('div');
+        toast.className = 'toast align-items-center text-white bg-warning border-0';
+        toast.setAttribute('role', 'alert');
+        toast.innerHTML = `
+            <div class="d-flex">
+                <div class="toast-body">
+                    <i class="fas fa-exclamation-triangle"></i>
+                    ${message}
+                </div>
+                <button type="button" class="btn-close btn-close-white me-2 m-auto" data-bs-dismiss="toast"></button>
+            </div>
+        `;
+
+        document.body.appendChild(toast);
+        const bsToast = new bootstrap.Toast(toast);
+        bsToast.show();
+        toast.addEventListener('hidden.bs.toast', () => toast.remove());
+    }
+
     function deleteAnnotation(annotationId) {
         console.log('=== deleteAnnotation called ===');
         console.log('Annotation ID:', annotationId);
@@ -404,10 +424,12 @@
         console.log('renderAnnotationsList function exists:', typeof renderAnnotationsList);
         console.log('showError function exists:', typeof showError);
         console.log('showSuccess function exists:', typeof showSuccess);
+        console.log('showWarning function exists:', typeof showWarning);
 
         // Make functions globally available
         window.showError = showError;
         window.showSuccess = showSuccess;
+        window.showWarning = showWarning;
         window.renderAnnotationsList = renderAnnotationsList;
         window.deleteAnnotation = deleteAnnotation;
         window.highlightAnnotation = highlightAnnotation;
@@ -418,6 +440,7 @@
         console.log('  - window.renderAnnotationsList:', typeof window.renderAnnotationsList);
         console.log('  - window.showError:', typeof window.showError);
         console.log('  - window.showSuccess:', typeof window.showSuccess);
+        console.log('  - window.showWarning:', typeof window.showWarning);
         console.log('  - window.highlightAnnotation:', typeof window.highlightAnnotation);
 
         // Test call
@@ -430,7 +453,7 @@
 
         // Add a test button to verify functionality (temporary)
         console.log('Adding test delete function to window for debugging...');
-        window.testDeleteFunction = function() {
+        window.testDeleteFunction = function () {
             console.log('Test delete function called');
             console.log('window.deleteAnnotation type:', typeof window.deleteAnnotation);
             if (typeof window.deleteAnnotation === 'function') {
@@ -455,7 +478,7 @@
         testButton.style.top = '10px';
         testButton.style.right = '10px';
         testButton.style.zIndex = '9999';
-        testButton.onclick = function() {
+        testButton.onclick = function () {
             console.log('Test button clicked');
             window.testDeleteFunction();
         };
@@ -495,7 +518,7 @@
             `;
 
             // Add event listeners
-            item.querySelector('.highlight-btn').addEventListener('click', function(e) {
+            item.querySelector('.highlight-btn').addEventListener('click', function (e) {
                 e.stopPropagation();
                 console.log('Highlight button clicked for:', annotation.annotation_id);
                 if (typeof window.highlightAnnotation === 'function') {
@@ -503,7 +526,7 @@
                 }
             });
 
-            item.querySelector('.delete-btn').addEventListener('click', function(e) {
+            item.querySelector('.delete-btn').addEventListener('click', function (e) {
                 e.stopPropagation();
                 console.log('=== Delete button clicked ===');
                 console.log('Annotation ID:', annotation.annotation_id);

core/duckdb_storage.py

@@ -189,7 +189,8 @@ class DuckDBStorage:
     def get_ohlcv_data(self, symbol: str, timeframe: str,
                        start_time: Optional[datetime] = None,
                        end_time: Optional[datetime] = None,
-                       limit: Optional[int] = None) -> Optional[pd.DataFrame]:
+                       limit: Optional[int] = None,
+                       direction: str = 'latest') -> Optional[pd.DataFrame]:
         """
         Query OHLCV data directly from DuckDB table
         
@@ -199,6 +200,7 @@ class DuckDBStorage:
             start_time: Start time filter
             end_time: End time filter
             limit: Maximum number of candles
+            direction: 'latest' (most recent), 'before' (older data), 'after' (newer data)
             
         Returns:
             DataFrame with OHLCV data
@@ -212,6 +214,19 @@ class DuckDBStorage:
             """
             params = [symbol, timeframe]
             
+            # Handle different direction modes
+            if direction == 'before' and end_time:
+                # Get older data: candles BEFORE end_time
+                query += " AND timestamp < ?"
+                params.append(int(end_time.timestamp() * 1000))
+                query += " ORDER BY timestamp DESC"
+            elif direction == 'after' and start_time:
+                # Get newer data: candles AFTER start_time
+                query += " AND timestamp > ?"
+                params.append(int(start_time.timestamp() * 1000))
+                query += " ORDER BY timestamp ASC"
+            else:
+                # Default: get most recent data in range
                 if start_time:
                     query += " AND timestamp >= ?"
                     params.append(int(start_time.timestamp() * 1000))
@@ -236,7 +251,7 @@ class DuckDBStorage:
             df = df.set_index('timestamp')
             df = df.sort_index()
             
-            logger.debug(f"Retrieved {len(df)} candles for {symbol} {timeframe} from DuckDB")
+            logger.debug(f"Retrieved {len(df)} candles for {symbol} {timeframe} from DuckDB (direction={direction})")
             return df
             
         except Exception as e: