From 2233a88d3e7dc43ae49a9eab61da0dd24f12ce72 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Fri, 24 Oct 2025 23:04:29 +0300 Subject: [PATCH] infinite lowad WIP --- ANNOTATE/core/data_loader.py | 226 ++++++++++++++++-- ANNOTATE/web/static/js/chart_manager.js | 139 +++++++---- .../web/templates/annotation_dashboard.html | 2 +- test_direction_filter.py | 89 +++++++ test_infinite_scroll_backend.py | 139 +++++++++++ 5 files changed, 522 insertions(+), 73 deletions(-) create mode 100644 test_direction_filter.py create mode 100644 test_infinite_scroll_backend.py diff --git a/ANNOTATE/core/data_loader.py b/ANNOTATE/core/data_loader.py index 1b5236a..1ba6aff 100644 --- a/ANNOTATE/core/data_loader.py +++ b/ANNOTATE/core/data_loader.py @@ -79,11 +79,14 @@ class HistoricalDataLoader: if len(cached_df) >= min(limit, 100): # Use cached if we have at least 100 candles logger.debug(f"Using DataProvider cached data for {symbol} {timeframe} ({len(cached_df)} candles)") - # Filter by time range if specified - if start_time or end_time: - filtered_df = self._filter_by_time_range(cached_df.copy(), start_time, end_time) - else: - filtered_df = cached_df.tail(limit).copy() + # Filter by time range with direction support + filtered_df = self._filter_by_time_range( + cached_df.copy(), + start_time, + end_time, + direction, + limit + ) # Cache in memory self.memory_cache[cache_key] = (filtered_df, datetime.now()) @@ -140,13 +143,14 @@ class HistoricalDataLoader: df = self.data_provider.cached_data[symbol][timeframe] if df is not None and not df.empty: - # Filter by time range if specified - if start_time or end_time: - df = self._filter_by_time_range(df, start_time, end_time) - - # Limit number of candles - if len(df) > limit: - df = df.tail(limit) + # Filter by time range with direction support + df = self._filter_by_time_range( + df.copy(), + start_time, + end_time, + direction, + limit + ) # Cache in memory self.memory_cache[cache_key] = (df.copy(), datetime.now()) @@ -182,10 +186,37 @@ class HistoricalDataLoader: self.memory_cache[cache_key] = (df.copy(), datetime.now()) return df else: - logger.info(f"No data in DuckDB, fetching from API for {symbol} {timeframe}") + logger.info(f"📡 No data in DuckDB, fetching from exchange API for {symbol} {timeframe}") + + # Fetch from exchange API with time range + df = self._fetch_from_exchange_api( + symbol=symbol, + timeframe=timeframe, + start_time=start_time, + end_time=end_time, + limit=limit, + direction=direction + ) + + if df is not None and not df.empty: + # Store in DuckDB for future use + if self.data_provider.duckdb_storage: + stored_count = self.data_provider.duckdb_storage.store_ohlcv_data( + symbol=symbol, + timeframe=timeframe, + df=df + ) + logger.info(f"💾 Stored {stored_count} new candles in DuckDB") + + # Cache in memory + self.memory_cache[cache_key] = (df.copy(), datetime.now()) + return df + else: + logger.warning(f"No data available from exchange API for {symbol} {timeframe}") + return None - # Fetch from API and store in DuckDB - logger.info(f"Fetching data from API for {symbol} {timeframe}") + # Fetch from API and store in DuckDB (no time range specified) + logger.info(f"Fetching latest data from API for {symbol} {timeframe}") df = self.data_provider.get_historical_data( symbol=symbol, timeframe=timeframe, @@ -194,9 +225,14 @@ class HistoricalDataLoader: ) if df is not None and not df.empty: - # Filter by time range if specified - if start_time or end_time: - df = self._filter_by_time_range(df, start_time, end_time) + # Filter by time range with direction support + df = self._filter_by_time_range( + df.copy(), + start_time, + end_time, + direction, + limit + ) # Cache in memory self.memory_cache[cache_key] = (df.copy(), datetime.now()) @@ -211,14 +247,156 @@ class HistoricalDataLoader: logger.error(f"Error loading data for {symbol} {timeframe}: {e}") return None + def _fetch_from_exchange_api(self, symbol: str, timeframe: str, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None, + limit: int = 1000, + direction: str = 'latest') -> Optional[pd.DataFrame]: + """ + Fetch historical data from exchange API (Binance/MEXC) with time range support + + Args: + symbol: Trading pair + timeframe: Timeframe + start_time: Start time for data range + end_time: End time for data range + limit: Maximum number of candles + direction: 'latest', 'before', or 'after' + + Returns: + DataFrame with OHLCV data or None + """ + try: + import requests + from core.api_rate_limiter import get_rate_limiter + + # Convert symbol format for Binance + binance_symbol = symbol.replace('/', '').upper() + + # Convert timeframe + timeframe_map = { + '1s': '1s', '1m': '1m', '5m': '5m', '15m': '15m', '30m': '30m', + '1h': '1h', '4h': '4h', '1d': '1d' + } + binance_timeframe = timeframe_map.get(timeframe, '1m') + + # Build API parameters + params = { + 'symbol': binance_symbol, + 'interval': binance_timeframe, + 'limit': min(limit, 1000) # Binance max is 1000 + } + + # Add time range parameters if specified + if direction == 'before' and end_time: + # Get data ending at end_time + params['endTime'] = int(end_time.timestamp() * 1000) + elif direction == 'after' and start_time: + # Get data starting at start_time + params['startTime'] = int(start_time.timestamp() * 1000) + elif start_time: + params['startTime'] = int(start_time.timestamp() * 1000) + if end_time and direction != 'before': + params['endTime'] = int(end_time.timestamp() * 1000) + + # Use rate limiter + rate_limiter = get_rate_limiter() + url = "https://api.binance.com/api/v3/klines" + + logger.info(f"Fetching from Binance: {symbol} {timeframe} (direction={direction}, limit={limit})") + + response = rate_limiter.make_request('binance_api', url, 'GET', params=params) + + if response is None or response.status_code != 200: + logger.warning(f"Binance API failed, trying MEXC...") + # Try MEXC as fallback + return self._fetch_from_mexc_with_time_range( + symbol, timeframe, start_time, end_time, limit, direction + ) + + data = response.json() + + if not data: + logger.warning(f"No data returned from Binance for {symbol} {timeframe}") + return None + + # Convert to DataFrame + df = pd.DataFrame(data, columns=[ + 'timestamp', 'open', 'high', 'low', 'close', 'volume', + 'close_time', 'quote_volume', 'trades', 'taker_buy_base', + 'taker_buy_quote', 'ignore' + ]) + + # Process columns + df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True) + for col in ['open', 'high', 'low', 'close', 'volume']: + df[col] = df[col].astype(float) + + # Keep only OHLCV columns + df = df[['timestamp', 'open', 'high', 'low', 'close', 'volume']] + df = df.set_index('timestamp') + df = df.sort_index() + + logger.info(f"✅ Fetched {len(df)} candles from Binance for {symbol} {timeframe}") + return df + + except Exception as e: + logger.error(f"Error fetching from exchange API: {e}") + return None + + def _fetch_from_mexc_with_time_range(self, symbol: str, timeframe: str, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None, + limit: int = 1000, + direction: str = 'latest') -> Optional[pd.DataFrame]: + """Fetch from MEXC with time range support (fallback)""" + try: + # MEXC implementation would go here + # For now, just return None to indicate unavailable + logger.warning("MEXC time range fetch not implemented yet") + return None + except Exception as e: + logger.error(f"Error fetching from MEXC: {e}") + return None + def _filter_by_time_range(self, df: pd.DataFrame, start_time: Optional[datetime], - end_time: Optional[datetime]) -> pd.DataFrame: - """Filter DataFrame by time range""" - if start_time: - df = df[df.index >= start_time] - if end_time: - df = df[df.index <= end_time] + end_time: Optional[datetime], + direction: str = 'latest', + limit: int = 500) -> pd.DataFrame: + """ + Filter DataFrame by time range with direction support + + Args: + df: DataFrame to filter + start_time: Start time filter + end_time: End time filter + direction: 'latest', 'before', or 'after' + limit: Maximum number of candles + + Returns: + Filtered DataFrame + """ + if direction == 'before' and end_time: + # Get candles BEFORE end_time + df = df[df.index < end_time] + # Return the most recent N candles before end_time + df = df.tail(limit) + elif direction == 'after' and start_time: + # Get candles AFTER start_time + df = df[df.index > start_time] + # Return the oldest N candles after start_time + df = df.head(limit) + else: + # Default: filter by range + if start_time: + df = df[df.index >= start_time] + if end_time: + df = df[df.index <= end_time] + # Return most recent candles + if len(df) > limit: + df = df.tail(limit) + return df def get_multi_timeframe_data(self, symbol: str, diff --git a/ANNOTATE/web/static/js/chart_manager.js b/ANNOTATE/web/static/js/chart_manager.js index 3a4f649..cdd5f1b 100644 --- a/ANNOTATE/web/static/js/chart_manager.js +++ b/ANNOTATE/web/static/js/chart_manager.js @@ -298,7 +298,7 @@ class ChartManager { return; // Don't process as regular chart click } } - + // Regular chart click for annotation marking this.handleChartClick(timeframe, eventData); }); @@ -308,10 +308,10 @@ class ChartManager { console.log('=== plotly_clickannotation event fired ==='); console.log('Event data:', eventData); console.log('Annotation:', eventData.annotation); - + const annotationName = eventData.annotation.name; console.log('Annotation name:', annotationName); - + if (annotationName) { const parts = annotationName.split('_'); const action = parts[0]; // 'entry', 'exit', or 'delete' @@ -675,22 +675,22 @@ class ChartManager { // Get existing pivot annotations (they have specific names like L1H, L2L) const existingLayout = chart.element.layout || {}; const existingAnnotations = existingLayout.annotations || []; - const pivotAnnotations = existingAnnotations.filter(ann => + const pivotAnnotations = existingAnnotations.filter(ann => ann.text && ann.text.match(/^L\d+[HL]$/) ); - + // Merge pivot annotations with trade annotations const allAnnotations = [...pivotAnnotations, ...plotlyAnnotations]; - + // Get existing pivot shapes const existingShapes = existingLayout.shapes || []; - const pivotShapes = existingShapes.filter(shape => + const pivotShapes = existingShapes.filter(shape => shape.layer === 'below' && shape.line && shape.line.dash === 'dash' ); - + // Merge pivot shapes with trade annotation shapes const allShapes = [...pivotShapes, ...plotlyShapes]; - + // Update chart layout with merged annotations and shapes Plotly.relayout(chart.plotId, { annotations: allAnnotations, @@ -921,15 +921,15 @@ class ChartManager { */ clearAllAnnotations() { console.log('Clearing all annotations from charts'); - + // Clear from memory this.annotations = {}; - + // Update all charts Object.keys(this.charts).forEach(timeframe => { this.updateChartAnnotations(timeframe); }); - + console.log('All annotations cleared from charts'); } @@ -940,23 +940,23 @@ class ChartManager { const chartContainer = document.getElementById('chart-container'); const visibleCharts = document.querySelectorAll('.timeframe-chart:not(.minimized)'); const minimizedCharts = document.querySelectorAll('.timeframe-chart.minimized'); - + // Remove scroll if all charts are visible or if some are minimized if (minimizedCharts.length > 0) { chartContainer.classList.add('no-scroll'); - + // Calculate available height for visible charts const containerHeight = chartContainer.clientHeight; const headerHeight = 50; // Approximate header height const availableHeight = containerHeight - (visibleCharts.length * headerHeight); const chartHeight = Math.max(200, availableHeight / visibleCharts.length); - + // Update visible chart heights visibleCharts.forEach(chart => { const plotElement = chart.querySelector('.chart-plot'); if (plotElement) { plotElement.style.height = `${chartHeight}px`; - + // Trigger Plotly resize const plotId = plotElement.id; if (plotId) { @@ -966,13 +966,13 @@ class ChartManager { }); } else { chartContainer.classList.remove('no-scroll'); - + // Reset to default heights visibleCharts.forEach(chart => { const plotElement = chart.querySelector('.chart-plot'); if (plotElement) { plotElement.style.height = '300px'; - + // Trigger Plotly resize const plotId = plotElement.id; if (plotId) { @@ -981,7 +981,7 @@ class ChartManager { } }); } - + console.log(`Updated chart layout: ${visibleCharts.length} visible, ${minimizedCharts.length} minimized`); } @@ -1122,7 +1122,7 @@ class ChartManager { // Check if we're near the left edge (need older data) const nearLeftEdge = (visibleStart - dataStart) < threshold; - + // Check if we're near the right edge (need newer data) const nearRightEdge = (dataEnd - visibleEnd) < threshold; @@ -1153,23 +1153,28 @@ class ChartManager { this.showLoadingIndicator(timeframe, direction); try { - // Calculate time range to fetch - const limit = 500; // Fetch 500 more candles + // Fetch more candles - no limit, get as much as available let startTime, endTime; if (direction === 'before') { - // Load older data - endTime = referenceTime.toISOString(); - startTime = null; // Let backend calculate based on limit + // Load older data: get candles BEFORE the first candle we have + // Use the actual first timestamp from our data + const firstTimestamp = chart.data.timestamps[0]; + endTime = new Date(firstTimestamp).toISOString(); + startTime = null; + + console.log(`Loading older data before ${endTime} for ${timeframe}`); } else { - // Load newer data - startTime = referenceTime.toISOString(); + // Load newer data: get candles AFTER the last candle we have + // Use the actual last timestamp from our data + const lastTimestamp = chart.data.timestamps[chart.data.timestamps.length - 1]; + startTime = new Date(lastTimestamp).toISOString(); endTime = null; + + console.log(`Loading newer data after ${startTime} for ${timeframe}`); } - console.log(`Loading ${limit} more candles ${direction} ${referenceTime.toISOString()} for ${timeframe}`); - - // Fetch more data from backend + // Fetch more data from backend (no limit - get all available) const response = await fetch('/api/chart-data', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -1178,7 +1183,7 @@ class ChartManager { timeframes: [timeframe], start_time: startTime, end_time: endTime, - limit: limit, + limit: 1000, // Request 1000 candles at a time direction: direction }) }); @@ -1187,10 +1192,17 @@ class ChartManager { if (result.success && result.chart_data && result.chart_data[timeframe]) { const newData = result.chart_data[timeframe]; - + + // Check if we got any new data + if (newData.timestamps.length === 0) { + console.warn(`No more data available for ${timeframe} ${direction}`); + window.showWarning('No more historical data available'); + return; + } + // Merge with existing data this.mergeChartData(timeframe, newData, direction); - + console.log(`Loaded ${newData.timestamps.length} new candles for ${timeframe}`); window.showSuccess(`Loaded ${newData.timestamps.length} more candles`); } else { @@ -1208,36 +1220,67 @@ class ChartManager { } /** - * Merge new data with existing chart data + * Merge new data with existing chart data (with deduplication) */ mergeChartData(timeframe, newData, direction) { const chart = this.charts[timeframe]; if (!chart || !chart.data) return; const existingData = chart.data; + + // Create a set of existing timestamps for deduplication + const existingTimestamps = new Set(existingData.timestamps); + + // Filter out duplicate timestamps from new data + const uniqueIndices = []; + newData.timestamps.forEach((ts, idx) => { + if (!existingTimestamps.has(ts)) { + uniqueIndices.push(idx); + } + }); + + // If no unique data, nothing to merge + if (uniqueIndices.length === 0) { + console.log(`No unique data to merge for ${timeframe}`); + return; + } + + // Extract only unique data points + const uniqueNewData = { + timestamps: uniqueIndices.map(i => newData.timestamps[i]), + open: uniqueIndices.map(i => newData.open[i]), + high: uniqueIndices.map(i => newData.high[i]), + low: uniqueIndices.map(i => newData.low[i]), + close: uniqueIndices.map(i => newData.close[i]), + volume: uniqueIndices.map(i => newData.volume[i]), + pivot_markers: newData.pivot_markers || {} + }; + + console.log(`Merging ${uniqueIndices.length} unique candles (filtered ${newData.timestamps.length - uniqueIndices.length} duplicates)`); + let mergedData; if (direction === 'before') { // Prepend older data mergedData = { - timestamps: [...newData.timestamps, ...existingData.timestamps], - open: [...newData.open, ...existingData.open], - high: [...newData.high, ...existingData.high], - low: [...newData.low, ...existingData.low], - close: [...newData.close, ...existingData.close], - volume: [...newData.volume, ...existingData.volume], - pivot_markers: { ...newData.pivot_markers, ...existingData.pivot_markers } + timestamps: [...uniqueNewData.timestamps, ...existingData.timestamps], + open: [...uniqueNewData.open, ...existingData.open], + high: [...uniqueNewData.high, ...existingData.high], + low: [...uniqueNewData.low, ...existingData.low], + close: [...uniqueNewData.close, ...existingData.close], + volume: [...uniqueNewData.volume, ...existingData.volume], + pivot_markers: { ...uniqueNewData.pivot_markers, ...existingData.pivot_markers } }; } else { // Append newer data mergedData = { - timestamps: [...existingData.timestamps, ...newData.timestamps], - open: [...existingData.open, ...newData.open], - high: [...existingData.high, ...newData.high], - low: [...existingData.low, ...newData.low], - close: [...existingData.close, ...newData.close], - volume: [...existingData.volume, ...newData.volume], - pivot_markers: { ...existingData.pivot_markers, ...newData.pivot_markers } + timestamps: [...existingData.timestamps, ...uniqueNewData.timestamps], + open: [...existingData.open, ...uniqueNewData.open], + high: [...existingData.high, ...uniqueNewData.high], + low: [...existingData.low, ...uniqueNewData.low], + close: [...existingData.close, ...uniqueNewData.close], + volume: [...existingData.volume, ...uniqueNewData.volume], + pivot_markers: { ...existingData.pivot_markers, ...uniqueNewData.pivot_markers } }; } diff --git a/ANNOTATE/web/templates/annotation_dashboard.html b/ANNOTATE/web/templates/annotation_dashboard.html index 89bbbdb..e51704b 100644 --- a/ANNOTATE/web/templates/annotation_dashboard.html +++ b/ANNOTATE/web/templates/annotation_dashboard.html @@ -62,7 +62,7 @@ window.appState = { currentSymbol: '{{ current_symbol }}', currentTimeframes: {{ timeframes | tojson }}, - annotations: { { annotations | tojson } }, + annotations: {{ annotations | tojson }}, pendingAnnotation: null, chartManager: null, annotationManager: null, diff --git a/test_direction_filter.py b/test_direction_filter.py new file mode 100644 index 0000000..2d68edf --- /dev/null +++ b/test_direction_filter.py @@ -0,0 +1,89 @@ +""" +Test the direction filtering logic without DuckDB +""" + +import pandas as pd +from datetime import datetime, timedelta + +def test_filter_by_time_range(): + """Test the _filter_by_time_range logic""" + + print("=" * 80) + print("Testing Direction Filter Logic") + print("=" * 80) + + # Create sample data + base_time = datetime(2024, 10, 24, 12, 0, 0) + timestamps = [base_time + timedelta(minutes=i) for i in range(100)] + + df = pd.DataFrame({ + 'open': range(100), + 'high': range(100), + 'low': range(100), + 'close': range(100), + 'volume': range(100) + }, index=timestamps) + + print(f"\nCreated test data with {len(df)} candles") + print(f"First timestamp: {df.index[0]}") + print(f"Last timestamp: {df.index[-1]}") + + # Test 1: Direction 'before' + print("\n" + "-" * 80) + print("Test 1: Direction 'before' - Get 10 candles BEFORE minute 50") + end_time = timestamps[50] + print(f"End time: {end_time}") + + # Simulate the filter logic + filtered = df[df.index < end_time] + result = filtered.tail(10) + + print(f"Result: {len(result)} candles") + print(f"First: {result.index[0]} (should be minute 40)") + print(f"Last: {result.index[-1]} (should be minute 49)") + + if result.index[-1] < end_time: + print("✅ PASS: Last candle is before end_time") + else: + print("❌ FAIL: Last candle is NOT before end_time") + + # Test 2: Direction 'after' + print("\n" + "-" * 80) + print("Test 2: Direction 'after' - Get 10 candles AFTER minute 50") + start_time = timestamps[50] + print(f"Start time: {start_time}") + + # Simulate the filter logic + filtered = df[df.index > start_time] + result = filtered.head(10) + + print(f"Result: {len(result)} candles") + print(f"First: {result.index[0]} (should be minute 51)") + print(f"Last: {result.index[-1]} (should be minute 60)") + + if result.index[0] > start_time: + print("✅ PASS: First candle is after start_time") + else: + print("❌ FAIL: First candle is NOT after start_time") + + # Test 3: Direction 'latest' (default) + print("\n" + "-" * 80) + print("Test 3: Direction 'latest' - Get most recent 10 candles") + + result = df.tail(10) + + print(f"Result: {len(result)} candles") + print(f"First: {result.index[0]} (should be minute 90)") + print(f"Last: {result.index[-1]} (should be minute 99)") + + if result.index[-1] == df.index[-1]: + print("✅ PASS: Got most recent candles") + else: + print("❌ FAIL: Did NOT get most recent candles") + + print("\n" + "=" * 80) + print("All Tests Complete") + print("=" * 80) + +if __name__ == "__main__": + test_filter_by_time_range() diff --git a/test_infinite_scroll_backend.py b/test_infinite_scroll_backend.py new file mode 100644 index 0000000..9c203c9 --- /dev/null +++ b/test_infinite_scroll_backend.py @@ -0,0 +1,139 @@ +""" +Test infinite scroll backend data loading +""" + +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) + +from datetime import datetime, timedelta +from core.data_provider import DataProvider +from ANNOTATE.core.data_loader import HistoricalDataLoader + +def test_backend_data_loading(): + """Test if backend can load historical data with direction parameter""" + + print("=" * 80) + print("Testing Infinite Scroll Backend Data Loading") + print("=" * 80) + + # Initialize data provider + print("\n1. Initializing DataProvider...") + data_provider = DataProvider() + + # Initialize data loader + print("2. Initializing HistoricalDataLoader...") + data_loader = HistoricalDataLoader(data_provider) + data_loader.disable_startup_mode() # Force fresh data + + symbol = "ETH/USDT" + timeframe = "1m" + + # Test 1: Load initial data (latest) + print(f"\n3. Loading initial data for {symbol} {timeframe}...") + initial_df = data_loader.get_data( + symbol=symbol, + timeframe=timeframe, + limit=100, + direction='latest' + ) + + if initial_df is None or initial_df.empty: + print("❌ FAILED: No initial data loaded") + return + + print(f"✅ Loaded {len(initial_df)} initial candles") + print(f" First timestamp: {initial_df.index[0]}") + print(f" Last timestamp: {initial_df.index[-1]}") + + # Test 2: Load older data (before first timestamp) + print(f"\n4. Loading older data BEFORE {initial_df.index[0]}...") + first_timestamp = initial_df.index[0] + + older_df = data_loader.get_data( + symbol=symbol, + timeframe=timeframe, + end_time=first_timestamp, + limit=100, + direction='before' + ) + + if older_df is None or older_df.empty: + print("❌ FAILED: No older data loaded") + print(" This might mean:") + print(" - No data exists before this timestamp in DuckDB") + print(" - The query is not working correctly") + else: + print(f"✅ Loaded {len(older_df)} older candles") + print(f" First timestamp: {older_df.index[0]}") + print(f" Last timestamp: {older_df.index[-1]}") + + # Check if older data is actually older + if older_df.index[-1] < first_timestamp: + print(f"✅ Data is correctly older (last older candle: {older_df.index[-1]} < first initial: {first_timestamp})") + else: + print(f"❌ WARNING: Data is NOT older! Last older: {older_df.index[-1]} >= first initial: {first_timestamp}") + + # Test 3: Load newer data (after last timestamp) + print(f"\n5. Loading newer data AFTER {initial_df.index[-1]}...") + last_timestamp = initial_df.index[-1] + + newer_df = data_loader.get_data( + symbol=symbol, + timeframe=timeframe, + start_time=last_timestamp, + limit=100, + direction='after' + ) + + if newer_df is None or newer_df.empty: + print("❌ No newer data loaded (this is expected if we're at the latest data)") + else: + print(f"✅ Loaded {len(newer_df)} newer candles") + print(f" First timestamp: {newer_df.index[0]}") + print(f" Last timestamp: {newer_df.index[-1]}") + + # Check if newer data is actually newer + if newer_df.index[0] > last_timestamp: + print(f"✅ Data is correctly newer (first newer candle: {newer_df.index[0]} > last initial: {last_timestamp})") + else: + print(f"❌ WARNING: Data is NOT newer! First newer: {newer_df.index[0]} <= last initial: {last_timestamp}") + + # Test 4: Check DuckDB directly + print(f"\n6. Checking DuckDB storage directly...") + if data_provider.duckdb_storage: + # Get total count + query = "SELECT COUNT(*) as count FROM ohlcv_data WHERE symbol = ? AND timeframe = ?" + result = data_provider.duckdb_storage.conn.execute(query, [symbol, timeframe]).fetchone() + total_count = result[0] if result else 0 + + print(f" Total candles in DuckDB: {total_count}") + + if total_count == 0: + print(" ❌ No data in DuckDB! Need to fetch from API first.") + else: + # Get time range + query = """ + SELECT + MIN(timestamp) as min_ts, + MAX(timestamp) as max_ts + FROM ohlcv_data + WHERE symbol = ? AND timeframe = ? + """ + result = data_provider.duckdb_storage.conn.execute(query, [symbol, timeframe]).fetchone() + + if result: + import pandas as pd + min_ts = pd.to_datetime(result[0], unit='ms', utc=True) + max_ts = pd.to_datetime(result[1], unit='ms', utc=True) + print(f" Time range: {min_ts} to {max_ts}") + print(f" Duration: {max_ts - min_ts}") + else: + print(" ❌ DuckDB storage not available") + + print("\n" + "=" * 80) + print("Test Complete") + print("=" * 80) + +if __name__ == "__main__": + test_backend_data_loading()