From 07b82f0a1ff96cfac8ea566c4a9d3ac1d13b2ce3 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Fri, 24 Oct 2025 22:46:44 +0300 Subject: [PATCH] infinite scroll fix --- ANNOTATE/core/data_loader.py | 35 ++- ANNOTATE/data/annotations/annotations_db.json | 72 +----- ANNOTATE/web/app.py | 11 +- ANNOTATE/web/static/js/chart_manager.js | 235 ++++++++++++++++++ .../web/templates/annotation_dashboard.html | 59 +++-- core/duckdb_storage.py | 35 ++- 6 files changed, 352 insertions(+), 95 deletions(-) diff --git a/ANNOTATE/core/data_loader.py b/ANNOTATE/core/data_loader.py index a572533..1b5236a 100644 --- a/ANNOTATE/core/data_loader.py +++ b/ANNOTATE/core/data_loader.py @@ -44,7 +44,8 @@ class HistoricalDataLoader: def get_data(self, symbol: str, timeframe: str, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None, - limit: int = 500) -> Optional[pd.DataFrame]: + limit: int = 500, + direction: str = 'latest') -> Optional[pd.DataFrame]: """ Get historical data for symbol and timeframe @@ -54,12 +55,13 @@ class HistoricalDataLoader: start_time: Start time for data range end_time: End time for data range limit: Maximum number of candles to return + direction: 'latest' (most recent), 'before' (older data), 'after' (newer data) Returns: DataFrame with OHLCV data or None if unavailable """ # Check memory cache first - cache_key = f"{symbol}_{timeframe}_{start_time}_{end_time}_{limit}" + cache_key = f"{symbol}_{timeframe}_{start_time}_{end_time}_{limit}_{direction}" if cache_key in self.memory_cache: cached_data, cached_time = self.memory_cache[cache_key] if datetime.now() - cached_time < self.cache_ttl: @@ -152,8 +154,7 @@ class HistoricalDataLoader: logger.info(f"Loaded {len(df)} candles for {symbol} {timeframe}") return df - # Fallback: fetch from DataProvider's historical data method - # During startup, allow stale cache to avoid slow API calls + # Fallback: Try DuckDB first, then fetch from API if needed if self.startup_mode: logger.info(f"Loading data for {symbol} {timeframe} (startup mode: allow stale cache)") df = self.data_provider.get_historical_data( @@ -163,11 +164,33 @@ class HistoricalDataLoader: allow_stale_cache=True ) else: - logger.info(f"Fetching fresh data for {symbol} {timeframe}") + # Check DuckDB first for historical data + if self.data_provider.duckdb_storage and (start_time or end_time): + logger.info(f"Checking DuckDB for {symbol} {timeframe} historical data (direction={direction})") + df = self.data_provider.duckdb_storage.get_ohlcv_data( + symbol=symbol, + timeframe=timeframe, + start_time=start_time, + end_time=end_time, + limit=limit, + direction=direction + ) + + if df is not None and not df.empty: + logger.info(f"✅ Loaded {len(df)} candles from DuckDB for {symbol} {timeframe}") + # Cache in memory + self.memory_cache[cache_key] = (df.copy(), datetime.now()) + return df + else: + logger.info(f"No data in DuckDB, fetching from API for {symbol} {timeframe}") + + # Fetch from API and store in DuckDB + logger.info(f"Fetching data from API for {symbol} {timeframe}") df = self.data_provider.get_historical_data( symbol=symbol, timeframe=timeframe, - limit=limit + limit=limit, + refresh=True # Force API fetch ) if df is not None and not df.empty: diff --git a/ANNOTATE/data/annotations/annotations_db.json b/ANNOTATE/data/annotations/annotations_db.json index 55d5bea..2ceafcd 100644 --- a/ANNOTATE/data/annotations/annotations_db.json +++ b/ANNOTATE/data/annotations/annotations_db.json @@ -1,69 +1,23 @@ { "annotations": [ { - "annotation_id": "967f91f4-5f01-4608-86af-4a006d55bd3c", + "annotation_id": "dc35c362-6174-4db4-b4db-8cc58a4ba8e5", "symbol": "ETH/USDT", - "timeframe": "1m", + "timeframe": "1h", "entry": { - "timestamp": "2025-10-23 14:15", - "price": 3821.57, - "index": 421 + "timestamp": "2025-10-07 13:00", + "price": 4755, + "index": 28 }, "exit": { - "timestamp": "2025-10-23 15:32", - "price": 3874.23, - "index": 498 + "timestamp": "2025-10-11 21:00", + "price": 3643.33, + "index": 63 }, - "direction": "LONG", - "profit_loss_pct": 1.377967693905904, + "direction": "SHORT", + "profit_loss_pct": 23.378969505783388, "notes": "", - "created_at": "2025-10-23T18:36:05.807749", - "market_context": { - "entry_state": {}, - "exit_state": {} - } - }, - { - "annotation_id": "a16f92c0-7228-4293-a188-baf74ca9b284", - "symbol": "ETH/USDT", - "timeframe": "1m", - "entry": { - "timestamp": "2025-10-24 08:31", - "price": 3930.54, - "index": 101 - }, - "exit": { - "timestamp": "2025-10-24 08:46", - "price": 3965, - "index": 104 - }, - "direction": "LONG", - "profit_loss_pct": 0.8767243177782196, - "notes": "", - "created_at": "2025-10-24T12:56:59.390345", - "market_context": { - "entry_state": {}, - "exit_state": {} - } - }, - { - "annotation_id": "ee34388a-fcad-4c7e-8a0a-2e8d205d5357", - "symbol": "ETH/USDT", - "timeframe": "1m", - "entry": { - "timestamp": "2025-10-24 04:06", - "price": 3895.67, - "index": 18 - }, - "exit": { - "timestamp": "2025-10-24 05:24", - "price": 3965, - "index": 36 - }, - "direction": "LONG", - "profit_loss_pct": 1.7796681957147276, - "notes": "", - "created_at": "2025-10-24T13:55:31.843201", + "created_at": "2025-10-24T22:33:26.187249", "market_context": { "entry_state": {}, "exit_state": {} @@ -71,7 +25,7 @@ } ], "metadata": { - "total_annotations": 5, - "last_updated": "2025-10-24T13:55:31.843201" + "total_annotations": 1, + "last_updated": "2025-10-24T22:33:26.194492" } } \ No newline at end of file diff --git a/ANNOTATE/web/app.py b/ANNOTATE/web/app.py index c5debde..441f8c9 100644 --- a/ANNOTATE/web/app.py +++ b/ANNOTATE/web/app.py @@ -423,13 +423,15 @@ class AnnotationDashboard: @self.server.route('/api/chart-data', methods=['POST']) def get_chart_data(): - """Get chart data for specified symbol and timeframes""" + """Get chart data for specified symbol and timeframes with infinite scroll support""" try: data = request.get_json() symbol = data.get('symbol', 'ETH/USDT') timeframes = data.get('timeframes', ['1s', '1m', '1h', '1d']) start_time_str = data.get('start_time') end_time_str = data.get('end_time') + limit = data.get('limit', 500) # Allow client to request more data + direction = data.get('direction', 'latest') # 'latest', 'before', or 'after' if not self.data_loader: return jsonify({ @@ -445,6 +447,10 @@ class AnnotationDashboard: end_time = datetime.fromisoformat(end_time_str.replace('Z', '+00:00')) if end_time_str else None # Fetch data for each timeframe using data loader + # This will automatically: + # 1. Check DuckDB first + # 2. Fetch from API if not in cache + # 3. Store in DuckDB for future use chart_data = {} for timeframe in timeframes: df = self.data_loader.get_data( @@ -452,7 +458,8 @@ class AnnotationDashboard: timeframe=timeframe, start_time=start_time, end_time=end_time, - limit=500 + limit=limit, + direction=direction ) if df is not None and not df.empty: diff --git a/ANNOTATE/web/static/js/chart_manager.js b/ANNOTATE/web/static/js/chart_manager.js index 363634e..3a4f649 100644 --- a/ANNOTATE/web/static/js/chart_manager.js +++ b/ANNOTATE/web/static/js/chart_manager.js @@ -334,6 +334,11 @@ class ChartManager { this.updateChartInfo(timeframe, eventData); }); + // Add relayout handler for infinite scroll (load more data when zooming/panning) + plotElement.on('plotly_relayout', (eventData) => { + this.handleChartRelayout(timeframe, eventData); + }); + console.log(`Chart created for ${timeframe} with ${data.timestamps.length} candles`); } @@ -1088,4 +1093,234 @@ class ChartManager { infoElement.textContent = `O: ${open.toFixed(2)} H: ${high.toFixed(2)} L: ${low.toFixed(2)} C: ${close.toFixed(2)}`; } } + + /** + * Handle chart relayout for infinite scroll + * Detects when user scrolls/zooms to edges and loads more data + */ + handleChartRelayout(timeframe, eventData) { + const chart = this.charts[timeframe]; + if (!chart || !chart.data) return; + + // Check if this is a range change (zoom/pan) + if (!eventData['xaxis.range[0]'] && !eventData['xaxis.range']) return; + + // Get current visible range + const xRange = eventData['xaxis.range'] || [eventData['xaxis.range[0]'], eventData['xaxis.range[1]']]; + if (!xRange || xRange.length !== 2) return; + + const visibleStart = new Date(xRange[0]); + const visibleEnd = new Date(xRange[1]); + + // Get data boundaries + const dataStart = new Date(chart.data.timestamps[0]); + const dataEnd = new Date(chart.data.timestamps[chart.data.timestamps.length - 1]); + + // Calculate threshold (10% of visible range from edge) + const visibleRange = visibleEnd - visibleStart; + const threshold = visibleRange * 0.1; + + // Check if we're near the left edge (need older data) + const nearLeftEdge = (visibleStart - dataStart) < threshold; + + // Check if we're near the right edge (need newer data) + const nearRightEdge = (dataEnd - visibleEnd) < threshold; + + console.log(`Relayout ${timeframe}: visible=${visibleStart.toISOString()} to ${visibleEnd.toISOString()}, data=${dataStart.toISOString()} to ${dataEnd.toISOString()}, nearLeft=${nearLeftEdge}, nearRight=${nearRightEdge}`); + + // Load more data if near edges + if (nearLeftEdge) { + this.loadMoreData(timeframe, 'before', dataStart); + } else if (nearRightEdge) { + this.loadMoreData(timeframe, 'after', dataEnd); + } + } + + /** + * Load more historical data for a timeframe + */ + async loadMoreData(timeframe, direction, referenceTime) { + const chart = this.charts[timeframe]; + if (!chart) return; + + // Prevent multiple simultaneous loads + if (chart.loading) { + console.log(`Already loading data for ${timeframe}, skipping...`); + return; + } + + chart.loading = true; + this.showLoadingIndicator(timeframe, direction); + + try { + // Calculate time range to fetch + const limit = 500; // Fetch 500 more candles + let startTime, endTime; + + if (direction === 'before') { + // Load older data + endTime = referenceTime.toISOString(); + startTime = null; // Let backend calculate based on limit + } else { + // Load newer data + startTime = referenceTime.toISOString(); + endTime = null; + } + + console.log(`Loading ${limit} more candles ${direction} ${referenceTime.toISOString()} for ${timeframe}`); + + // Fetch more data from backend + const response = await fetch('/api/chart-data', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + symbol: window.appState?.currentSymbol || 'ETH/USDT', + timeframes: [timeframe], + start_time: startTime, + end_time: endTime, + limit: limit, + direction: direction + }) + }); + + const result = await response.json(); + + if (result.success && result.chart_data && result.chart_data[timeframe]) { + const newData = result.chart_data[timeframe]; + + // Merge with existing data + this.mergeChartData(timeframe, newData, direction); + + console.log(`Loaded ${newData.timestamps.length} new candles for ${timeframe}`); + window.showSuccess(`Loaded ${newData.timestamps.length} more candles`); + } else { + console.warn(`No more data available for ${timeframe} ${direction}`); + window.showWarning('No more historical data available'); + } + + } catch (error) { + console.error(`Error loading more data for ${timeframe}:`, error); + window.showError('Failed to load more data'); + } finally { + chart.loading = false; + this.hideLoadingIndicator(timeframe); + } + } + + /** + * Merge new data with existing chart data + */ + mergeChartData(timeframe, newData, direction) { + const chart = this.charts[timeframe]; + if (!chart || !chart.data) return; + + const existingData = chart.data; + let mergedData; + + if (direction === 'before') { + // Prepend older data + mergedData = { + timestamps: [...newData.timestamps, ...existingData.timestamps], + open: [...newData.open, ...existingData.open], + high: [...newData.high, ...existingData.high], + low: [...newData.low, ...existingData.low], + close: [...newData.close, ...existingData.close], + volume: [...newData.volume, ...existingData.volume], + pivot_markers: { ...newData.pivot_markers, ...existingData.pivot_markers } + }; + } else { + // Append newer data + mergedData = { + timestamps: [...existingData.timestamps, ...newData.timestamps], + open: [...existingData.open, ...newData.open], + high: [...existingData.high, ...newData.high], + low: [...existingData.low, ...newData.low], + close: [...existingData.close, ...newData.close], + volume: [...existingData.volume, ...newData.volume], + pivot_markers: { ...existingData.pivot_markers, ...newData.pivot_markers } + }; + } + + // Update stored data + chart.data = mergedData; + + // Update the chart with merged data + this.updateSingleChart(timeframe, mergedData); + } + + /** + * Update a single chart with new data + */ + updateSingleChart(timeframe, data) { + const chart = this.charts[timeframe]; + if (!chart) return; + + const plotId = chart.plotId; + + // Create volume colors + const volumeColors = data.close.map((close, i) => { + if (i === 0) return '#3b82f6'; + return close >= data.open[i] ? '#10b981' : '#ef4444'; + }); + + // Update traces + const update = { + x: [data.timestamps, data.timestamps], + open: [data.open], + high: [data.high], + low: [data.low], + close: [data.close], + y: [undefined, data.volume], + 'marker.color': [undefined, volumeColors] + }; + + Plotly.restyle(plotId, update, [0, 1]); + + console.log(`Updated ${timeframe} chart with ${data.timestamps.length} candles`); + } + + /** + * Show loading indicator on chart + */ + showLoadingIndicator(timeframe, direction) { + const chart = this.charts[timeframe]; + if (!chart) return; + + const plotElement = chart.element; + const loadingDiv = document.createElement('div'); + loadingDiv.id = `loading-${timeframe}`; + loadingDiv.className = 'chart-loading-indicator'; + loadingDiv.innerHTML = ` + + Loading ${direction === 'before' ? 'older' : 'newer'} data... + `; + loadingDiv.style.cssText = ` + position: absolute; + top: 10px; + ${direction === 'before' ? 'left' : 'right'}: 10px; + background: rgba(31, 41, 55, 0.9); + color: #f8f9fa; + padding: 8px 12px; + border-radius: 4px; + font-size: 12px; + z-index: 1000; + display: flex; + align-items: center; + `; + + plotElement.parentElement.style.position = 'relative'; + plotElement.parentElement.appendChild(loadingDiv); + } + + /** + * Hide loading indicator + */ + hideLoadingIndicator(timeframe) { + const loadingDiv = document.getElementById(`loading-${timeframe}`); + if (loadingDiv) { + loadingDiv.remove(); + } + } } diff --git a/ANNOTATE/web/templates/annotation_dashboard.html b/ANNOTATE/web/templates/annotation_dashboard.html index 9c1689c..89bbbdb 100644 --- a/ANNOTATE/web/templates/annotation_dashboard.html +++ b/ANNOTATE/web/templates/annotation_dashboard.html @@ -62,7 +62,7 @@ window.appState = { currentSymbol: '{{ current_symbol }}', currentTimeframes: {{ timeframes | tojson }}, - annotations: {{ annotations | tojson }}, + annotations: { { annotations | tojson } }, pendingAnnotation: null, chartManager: null, annotationManager: null, @@ -299,24 +299,44 @@ toast.addEventListener('hidden.bs.toast', () => toast.remove()); } + function showWarning(message) { + const toast = document.createElement('div'); + toast.className = 'toast align-items-center text-white bg-warning border-0'; + toast.setAttribute('role', 'alert'); + toast.innerHTML = ` +
+
+ + ${message} +
+ +
+ `; + + document.body.appendChild(toast); + const bsToast = new bootstrap.Toast(toast); + bsToast.show(); + toast.addEventListener('hidden.bs.toast', () => toast.remove()); + } + function deleteAnnotation(annotationId) { console.log('=== deleteAnnotation called ==='); console.log('Annotation ID:', annotationId); console.log('window.appState:', window.appState); console.log('window.appState.annotations:', window.appState?.annotations); - + if (!annotationId) { console.error('No annotation ID provided'); showError('No annotation ID provided'); return; } - + if (!window.appState || !window.appState.annotations) { console.error('appState not initialized'); showError('Application state not initialized. Please refresh the page.'); return; } - + // Check if annotation exists const annotation = window.appState.annotations.find(a => a.annotation_id === annotationId); if (!annotation) { @@ -324,7 +344,7 @@ showError('Annotation not found'); return; } - + console.log('Found annotation:', annotation); console.log('Current annotations count:', window.appState.annotations.length); @@ -351,7 +371,7 @@ if (data.success) { console.log('Delete successful, updating UI...'); - + // Remove from app state const originalCount = window.appState.annotations.length; window.appState.annotations = window.appState.annotations.filter( @@ -404,10 +424,12 @@ console.log('renderAnnotationsList function exists:', typeof renderAnnotationsList); console.log('showError function exists:', typeof showError); console.log('showSuccess function exists:', typeof showSuccess); - + console.log('showWarning function exists:', typeof showWarning); + // Make functions globally available window.showError = showError; window.showSuccess = showSuccess; + window.showWarning = showWarning; window.renderAnnotationsList = renderAnnotationsList; window.deleteAnnotation = deleteAnnotation; window.highlightAnnotation = highlightAnnotation; @@ -418,8 +440,9 @@ console.log(' - window.renderAnnotationsList:', typeof window.renderAnnotationsList); console.log(' - window.showError:', typeof window.showError); console.log(' - window.showSuccess:', typeof window.showSuccess); + console.log(' - window.showWarning:', typeof window.showWarning); console.log(' - window.highlightAnnotation:', typeof window.highlightAnnotation); - + // Test call console.log('Testing window.deleteAnnotation availability...'); if (typeof window.deleteAnnotation === 'function') { @@ -427,10 +450,10 @@ } else { console.error('✗ window.deleteAnnotation is NOT a function!'); } - + // Add a test button to verify functionality (temporary) console.log('Adding test delete function to window for debugging...'); - window.testDeleteFunction = function() { + window.testDeleteFunction = function () { console.log('Test delete function called'); console.log('window.deleteAnnotation type:', typeof window.deleteAnnotation); if (typeof window.deleteAnnotation === 'function') { @@ -446,7 +469,7 @@ console.error('window.deleteAnnotation is NOT available'); } }; - + // Add test button to page (temporary debugging) const testButton = document.createElement('button'); testButton.textContent = 'Test Delete Function'; @@ -455,7 +478,7 @@ testButton.style.top = '10px'; testButton.style.right = '10px'; testButton.style.zIndex = '9999'; - testButton.onclick = function() { + testButton.onclick = function () { console.log('Test button clicked'); window.testDeleteFunction(); }; @@ -493,23 +516,23 @@ `; - + // Add event listeners - item.querySelector('.highlight-btn').addEventListener('click', function(e) { + item.querySelector('.highlight-btn').addEventListener('click', function (e) { e.stopPropagation(); console.log('Highlight button clicked for:', annotation.annotation_id); if (typeof window.highlightAnnotation === 'function') { window.highlightAnnotation(annotation.annotation_id); } }); - - item.querySelector('.delete-btn').addEventListener('click', function(e) { + + item.querySelector('.delete-btn').addEventListener('click', function (e) { e.stopPropagation(); console.log('=== Delete button clicked ==='); console.log('Annotation ID:', annotation.annotation_id); console.log('window.deleteAnnotation type:', typeof window.deleteAnnotation); console.log('window object keys containing delete:', Object.keys(window).filter(k => k.includes('delete'))); - + if (typeof window.deleteAnnotation === 'function') { console.log('Calling window.deleteAnnotation...'); try { @@ -524,7 +547,7 @@ showError('Delete function not available. Please refresh the page.'); } }); - + listElement.appendChild(item); }); } diff --git a/core/duckdb_storage.py b/core/duckdb_storage.py index 14cafa2..fd3ab49 100644 --- a/core/duckdb_storage.py +++ b/core/duckdb_storage.py @@ -189,7 +189,8 @@ class DuckDBStorage: def get_ohlcv_data(self, symbol: str, timeframe: str, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None, - limit: Optional[int] = None) -> Optional[pd.DataFrame]: + limit: Optional[int] = None, + direction: str = 'latest') -> Optional[pd.DataFrame]: """ Query OHLCV data directly from DuckDB table @@ -199,6 +200,7 @@ class DuckDBStorage: start_time: Start time filter end_time: End time filter limit: Maximum number of candles + direction: 'latest' (most recent), 'before' (older data), 'after' (newer data) Returns: DataFrame with OHLCV data @@ -212,15 +214,28 @@ class DuckDBStorage: """ params = [symbol, timeframe] - if start_time: - query += " AND timestamp >= ?" - params.append(int(start_time.timestamp() * 1000)) - - if end_time: - query += " AND timestamp <= ?" + # Handle different direction modes + if direction == 'before' and end_time: + # Get older data: candles BEFORE end_time + query += " AND timestamp < ?" params.append(int(end_time.timestamp() * 1000)) - - query += " ORDER BY timestamp DESC" + query += " ORDER BY timestamp DESC" + elif direction == 'after' and start_time: + # Get newer data: candles AFTER start_time + query += " AND timestamp > ?" + params.append(int(start_time.timestamp() * 1000)) + query += " ORDER BY timestamp ASC" + else: + # Default: get most recent data in range + if start_time: + query += " AND timestamp >= ?" + params.append(int(start_time.timestamp() * 1000)) + + if end_time: + query += " AND timestamp <= ?" + params.append(int(end_time.timestamp() * 1000)) + + query += " ORDER BY timestamp DESC" if limit: query += f" LIMIT {limit}" @@ -236,7 +251,7 @@ class DuckDBStorage: df = df.set_index('timestamp') df = df.sort_index() - logger.debug(f"Retrieved {len(df)} candles for {symbol} {timeframe} from DuckDB") + logger.debug(f"Retrieved {len(df)} candles for {symbol} {timeframe} from DuckDB (direction={direction})") return df except Exception as e: