diff --git a/ANNOTATE/core/data_loader.py b/ANNOTATE/core/data_loader.py index 0711d53..02c125f 100644 --- a/ANNOTATE/core/data_loader.py +++ b/ANNOTATE/core/data_loader.py @@ -310,19 +310,16 @@ class HistoricalDataLoader: } binance_timeframe = timeframe_map.get(timeframe, '1m') - # Build API parameters + # Build initial API parameters params = { 'symbol': binance_symbol, - 'interval': binance_timeframe, - 'limit': min(limit, 1000) # Binance max is 1000 + 'interval': binance_timeframe } # Add time range parameters if specified if direction == 'before' and end_time: - # Get data ending at end_time params['endTime'] = int(end_time.timestamp() * 1000) elif direction == 'after' and start_time: - # Get data starting at start_time params['startTime'] = int(start_time.timestamp() * 1000) elif start_time: params['startTime'] = int(start_time.timestamp() * 1000) @@ -335,40 +332,91 @@ class HistoricalDataLoader: logger.info(f"Fetching from Binance: {symbol} {timeframe} (direction={direction}, limit={limit})") - response = rate_limiter.make_request('binance_api', url, 'GET', params=params) + # Pagination variables + all_dfs = [] + total_fetched = 0 + is_fetching_forward = (direction == 'after') - if response is None or response.status_code != 200: - logger.warning(f"Binance API failed, trying MEXC...") - # Try MEXC as fallback - return self._fetch_from_mexc_with_time_range( - symbol, timeframe, start_time, end_time, limit, direction - ) + # Fetch loop + while total_fetched < limit: + # Calculate batch limit (max 1000 per request) + batch_limit = min(limit - total_fetched, 1000) + params['limit'] = batch_limit + + response = rate_limiter.make_request('binance_api', url, 'GET', params=params) + + if response is None or response.status_code != 200: + if total_fetched == 0: + logger.warning(f"Binance API failed, trying MEXC...") + return self._fetch_from_mexc_with_time_range( + symbol, timeframe, start_time, end_time, limit, direction + ) + else: + logger.warning("Binance API failed during pagination, returning partial data") + break + + data = response.json() + + if not data: + if total_fetched == 0: + logger.warning(f"No data returned from Binance for {symbol} {timeframe}") + return None + else: + break + + # Convert to DataFrame + df = pd.DataFrame(data, columns=[ + 'timestamp', 'open', 'high', 'low', 'close', 'volume', + 'close_time', 'quote_volume', 'trades', 'taker_buy_base', + 'taker_buy_quote', 'ignore' + ]) + + # Process columns + df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True) + for col in ['open', 'high', 'low', 'close', 'volume']: + df[col] = df[col].astype(float) + + # Keep only OHLCV columns + df = df[['timestamp', 'open', 'high', 'low', 'close', 'volume']] + df = df.set_index('timestamp') + df = df.sort_index() + + if df.empty: + break + + all_dfs.append(df) + total_fetched += len(df) + + # Prepare for next batch + if total_fetched >= limit: + break + + # Update params for next iteration + if is_fetching_forward: + # Next batch starts after the last candle + last_ts = df.index[-1] + params['startTime'] = int(last_ts.value / 10**6) + 1 + # Check if we exceeded end_time + if 'endTime' in params and params['startTime'] > params['endTime']: + break + else: + # Next batch ends before the first candle + first_ts = df.index[0] + params['endTime'] = int(first_ts.value / 10**6) - 1 + # Check if we exceeded start_time + if 'startTime' in params and params['endTime'] < params['startTime']: + break - data = response.json() - - if not data: - logger.warning(f"No data returned from Binance for {symbol} {timeframe}") + # Combine all batches + if not all_dfs: return None + + final_df = pd.concat(all_dfs) + final_df = final_df.sort_index() + final_df = final_df[~final_df.index.duplicated(keep='first')] - # Convert to DataFrame - df = pd.DataFrame(data, columns=[ - 'timestamp', 'open', 'high', 'low', 'close', 'volume', - 'close_time', 'quote_volume', 'trades', 'taker_buy_base', - 'taker_buy_quote', 'ignore' - ]) - - # Process columns - df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True) - for col in ['open', 'high', 'low', 'close', 'volume']: - df[col] = df[col].astype(float) - - # Keep only OHLCV columns - df = df[['timestamp', 'open', 'high', 'low', 'close', 'volume']] - df = df.set_index('timestamp') - df = df.sort_index() - - logger.info(f" Fetched {len(df)} candles from Binance for {symbol} {timeframe}") - return df + logger.info(f" Fetched {len(final_df)} candles from Binance for {symbol} {timeframe} (requested {limit})") + return final_df except Exception as e: logger.error(f"Error fetching from exchange API: {e}") diff --git a/ANNOTATE/web/app.py b/ANNOTATE/web/app.py index 6f4d71c..7572801 100644 --- a/ANNOTATE/web/app.py +++ b/ANNOTATE/web/app.py @@ -2200,9 +2200,18 @@ class AnnotationDashboard: signals = self.training_adapter.get_latest_signals() + # Get metrics from active inference sessions + metrics = {'accuracy': 0.0, 'loss': 0.0} + if hasattr(self.training_adapter, 'inference_sessions'): + for session in self.training_adapter.inference_sessions.values(): + if 'metrics' in session: + metrics = session['metrics'] + break + return jsonify({ 'success': True, - 'signals': signals + 'signals': signals, + 'metrics': metrics }) except Exception as e: diff --git a/ANNOTATE/web/static/js/chart_manager.js b/ANNOTATE/web/static/js/chart_manager.js index 3aeae6a..21d68bc 100644 --- a/ANNOTATE/web/static/js/chart_manager.js +++ b/ANNOTATE/web/static/js/chart_manager.js @@ -11,6 +11,7 @@ class ChartManager { this.syncedTime = null; this.updateTimers = {}; // Track auto-update timers this.autoUpdateEnabled = false; // Auto-update state + this.liveMetricsOverlay = null; // Live metrics display overlay console.log('ChartManager initialized with timeframes:', timeframes); } @@ -27,28 +28,19 @@ class ChartManager { this.autoUpdateEnabled = true; console.log('Starting chart auto-update...'); - // Update 1s chart every 2 seconds (was 20s) + // Update 1s chart every 1 second (was 2s) for live updates if (this.timeframes.includes('1s')) { this.updateTimers['1s'] = setInterval(() => { this.updateChartIncremental('1s'); - }, 2000); // 2 seconds + }, 1000); // 1 second } - // Update 1m chart - sync to whole minutes + every 5s (was 20s) + // Update 1m chart - every 1 second for live candle updates if (this.timeframes.includes('1m')) { - // Calculate ms until next whole minute - const now = new Date(); - const msUntilNextMinute = (60 - now.getSeconds()) * 1000 - now.getMilliseconds(); - - // Update on next whole minute - setTimeout(() => { + // We can poll every second for live updates + this.updateTimers['1m'] = setInterval(() => { this.updateChartIncremental('1m'); - - // Then update every 5s - this.updateTimers['1m'] = setInterval(() => { - this.updateChartIncremental('1m'); - }, 5000); // 5 seconds - }, msUntilNextMinute); + }, 1000); } console.log('Auto-update enabled for:', Object.keys(this.updateTimers)); @@ -130,6 +122,7 @@ class ChartManager { // Go back 2 intervals to be safe const lastTimeMs = new Date(lastTimestamp).getTime(); let lookbackMs = 2000; // Default 2s + if (timeframe === '1s') lookbackMs = 5000; // Increased lookback for 1s to prevent misses if (timeframe === '1m') lookbackMs = 120000; if (timeframe === '1h') lookbackMs = 7200000; @@ -157,6 +150,11 @@ class ChartManager { if (result.success && result.chart_data && result.chart_data[timeframe]) { const newData = result.chart_data[timeframe]; + console.log(`[${timeframe}] Received ${newData.timestamps.length} candles from API`); + if (newData.timestamps.length > 0) { + console.log(`[${timeframe}] First: ${newData.timestamps[0]}, Last: ${newData.timestamps[newData.timestamps.length - 1]}`); + } + if (newData.timestamps.length > 0) { // Smart Merge: // We want to update any existing candles that have changed (live candle) @@ -212,6 +210,8 @@ class ChartManager { // 4. Recalculate and Redraw if (updatesCount > 0 || remainingTimestamps.length > 0) { + console.log(`[${timeframe}] Chart update: ${updatesCount} updated, ${remainingTimestamps.length} new candles`); + this.recalculatePivots(timeframe, chart.data); this.updateSingleChart(timeframe, chart.data); @@ -221,7 +221,9 @@ class ChartManager { counterEl.textContent = window.liveUpdateCount + ' updates'; } - console.debug(`Incrementally updated ${timeframe} chart`); + console.log(`[${timeframe}] Chart updated successfully. Total candles: ${chart.data.timestamps.length}`); + } else { + console.log(`[${timeframe}] No updates needed (no changes detected)`); } } } @@ -410,7 +412,8 @@ class ChartManager { gridcolor: '#374151', color: '#9ca3af', showgrid: true, - zeroline: false + zeroline: false, + fixedrange: false }, yaxis: { title: { @@ -421,7 +424,8 @@ class ChartManager { color: '#9ca3af', showgrid: true, zeroline: false, - domain: [0.3, 1] + domain: [0.3, 1], + fixedrange: false // Allow vertical scaling }, yaxis2: { title: { @@ -432,7 +436,8 @@ class ChartManager { color: '#9ca3af', showgrid: false, zeroline: false, - domain: [0, 0.25] + domain: [0, 0.25], + fixedrange: false }, plot_bgcolor: '#1f2937', paper_bgcolor: '#1f2937', @@ -448,17 +453,30 @@ class ChartManager { const config = { responsive: true, displayModeBar: true, - modeBarButtonsToRemove: ['lasso2d', 'select2d', 'autoScale2d'], + modeBarButtonsToRemove: ['lasso2d', 'select2d'], // Allow autoScale2d displaylogo: false, scrollZoom: true, // Performance optimizations - doubleClick: false, - showAxisDragHandles: false, + doubleClick: 'reset', // Enable double-click reset + showAxisDragHandles: true, // Enable axis dragging showAxisRangeEntryBoxes: false }; // Prepare chart data with pivot bounds const chartData = [candlestickTrace, volumeTrace]; + + // Add pivot dots trace (trace index 2) + const pivotDotsTrace = { + x: [], + y: [], + text: [], + marker: { color: [], size: [], symbol: [] }, + mode: 'markers', + hoverinfo: 'text', + showlegend: false, + yaxis: 'y' + }; + chartData.push(pivotDotsTrace); // Add pivot markers from chart data const shapes = []; @@ -566,13 +584,16 @@ class ChartManager { } }); - // Add pivot dots trace if we have any - if (pivotDots.x.length > 0) { - chartData.push(pivotDots); - } - console.log(`Added ${shapes.length} pivot levels to ${timeframe} chart`); } + + // Populate pivot dots trace (trace index 2) with data + if (pivotDots.x.length > 0) { + pivotDotsTrace.x = pivotDots.x; + pivotDotsTrace.y = pivotDots.y; + pivotDotsTrace.text = pivotDots.text; + pivotDotsTrace.marker = pivotDots.marker; + } // Add shapes and annotations to layout if (shapes.length > 0) { @@ -1857,8 +1878,9 @@ class ChartManager { if (!predictions) return; try { - // Update predictions on 1m chart (primary timeframe for predictions) - const timeframe = '1m'; + // Use the currently active timeframe from app state + // This ensures predictions appear on the chart the user is watching (e.g., '1s') + const timeframe = window.appState?.currentTimeframes?.[0] || '1m'; const chart = this.charts[timeframe]; if (!chart) return; @@ -1894,12 +1916,55 @@ class ChartManager { this._addTrendPrediction(predictions.transformer.trend_vector, predictionShapes, predictionAnnotations); } - // Add ghost candle if available + // Handle Predicted Candles if (predictions.transformer.predicted_candle) { - // Check if we have prediction for this timeframe const candleData = predictions.transformer.predicted_candle[timeframe]; if (candleData) { - this._addGhostCandlePrediction(candleData, timeframe, predictionTraces); + // Get the prediction timestamp from the model (when inference was made) + const predictionTimestamp = predictions.transformer.timestamp || new Date().toISOString(); + + // Calculate the target timestamp (when this prediction is for) + // This should be the NEXT candle after the inference time + const inferenceTime = new Date(predictionTimestamp); + let targetTimestamp; + + if (timeframe === '1s') { + targetTimestamp = new Date(inferenceTime.getTime() + 1000); + } else if (timeframe === '1m') { + targetTimestamp = new Date(inferenceTime.getTime() + 60000); + } else if (timeframe === '1h') { + targetTimestamp = new Date(inferenceTime.getTime() + 3600000); + } else { + targetTimestamp = new Date(inferenceTime.getTime() + 60000); + } + + // 1. Next Candle Prediction (Ghost) + // Show the prediction at its proper timestamp + this._addGhostCandlePrediction(candleData, timeframe, predictionTraces, targetTimestamp); + + // 2. Store as "Last Prediction" for this timeframe + // This allows us to visualize the "Shadow" (prediction vs actual) on the next tick + if (!this.lastPredictions) this.lastPredictions = {}; + + this.lastPredictions[timeframe] = { + timestamp: targetTimestamp.toISOString(), + candle: candleData, + inferenceTime: predictionTimestamp + }; + + console.log(`[${timeframe}] Ghost candle prediction placed at ${targetTimestamp.toISOString()} (inference at ${predictionTimestamp})`); + } + } + + // 3. Render "Shadow Candle" (Previous Prediction for Current Candle) + // If we have a stored prediction that matches the CURRENT candle time, show it + if (this.lastPredictions && this.lastPredictions[timeframe]) { + const lastPred = this.lastPredictions[timeframe]; + const currentTimestamp = chart.data.timestamps[chart.data.timestamps.length - 1]; + + // Compare timestamps (allow small diff for jitter) + if (Math.abs(new Date(lastPred.timestamp).getTime() - new Date(currentTimestamp).getTime()) < 1000) { + this._addShadowCandlePrediction(lastPred.candle, currentTimestamp, predictionTraces); } } } @@ -1914,12 +1979,12 @@ class ChartManager { // Add prediction traces (ghost candles) if (predictionTraces.length > 0) { - // Remove existing ghost traces safely - // We iterate backwards to avoid index shifting issues when deleting + // Remove existing ghost/shadow traces safely const currentTraces = plotElement.data.length; const indicesToRemove = []; for (let i = currentTraces - 1; i >= 0; i--) { - if (plotElement.data[i].name === 'Ghost Prediction') { + const name = plotElement.data[i].name; + if (name === 'Ghost Prediction' || name === 'Shadow Prediction') { indicesToRemove.push(i); } } @@ -2003,26 +2068,32 @@ class ChartManager { }); } - _addGhostCandlePrediction(candleData, timeframe, traces) { + _addGhostCandlePrediction(candleData, timeframe, traces, predictionTimestamp = null) { // candleData is [Open, High, Low, Close, Volume] - // We need to determine the timestamp for this ghost candle - // It should be the NEXT candle after the last one on chart + // predictionTimestamp is when the model made this prediction (optional) + // If not provided, we calculate the next candle time const chart = this.charts[timeframe]; if (!chart || !chart.data) return; - const lastTimestamp = new Date(chart.data.timestamps[chart.data.timestamps.length - 1]); let nextTimestamp; - // Calculate next timestamp based on timeframe - if (timeframe === '1s') { - nextTimestamp = new Date(lastTimestamp.getTime() + 1000); - } else if (timeframe === '1m') { - nextTimestamp = new Date(lastTimestamp.getTime() + 60000); - } else if (timeframe === '1h') { - nextTimestamp = new Date(lastTimestamp.getTime() + 3600000); + if (predictionTimestamp) { + // Use the actual prediction timestamp from the model + nextTimestamp = new Date(predictionTimestamp); } else { - nextTimestamp = new Date(lastTimestamp.getTime() + 60000); // Default 1m + // Fallback: Calculate next timestamp based on timeframe + const lastTimestamp = new Date(chart.data.timestamps[chart.data.timestamps.length - 1]); + + if (timeframe === '1s') { + nextTimestamp = new Date(lastTimestamp.getTime() + 1000); + } else if (timeframe === '1m') { + nextTimestamp = new Date(lastTimestamp.getTime() + 60000); + } else if (timeframe === '1h') { + nextTimestamp = new Date(lastTimestamp.getTime() + 3600000); + } else { + nextTimestamp = new Date(lastTimestamp.getTime() + 60000); // Default 1m + } } const open = candleData[0]; @@ -2059,6 +2130,42 @@ class ChartManager { console.log('Added ghost candle prediction:', ghostTrace); } + _addShadowCandlePrediction(candleData, timestamp, traces) { + // candleData is [Open, High, Low, Close, Volume] + // timestamp is the time where this shadow should appear (matches current candle) + + const open = candleData[0]; + const high = candleData[1]; + const low = candleData[2]; + const close = candleData[3]; + + // Shadow color (purple to distinguish from ghost) + const color = '#8b5cf6'; // Violet + + const shadowTrace = { + x: [timestamp], + open: [open], + high: [high], + low: [low], + close: [close], + type: 'candlestick', + name: 'Shadow Prediction', + increasing: { + line: { color: color, width: 1 }, + fillcolor: 'rgba(139, 92, 246, 0.0)' // Hollow + }, + decreasing: { + line: { color: color, width: 1 }, + fillcolor: 'rgba(139, 92, 246, 0.0)' // Hollow + }, + opacity: 0.7, + hoverinfo: 'x+y+text', + text: ['Past Prediction'] + }; + + traces.push(shadowTrace); + } + _addDQNPrediction(prediction, shapes, annotations) { const timestamp = new Date(prediction.timestamp || Date.now()); const price = prediction.current_price || 0; @@ -2174,4 +2281,98 @@ class ChartManager { opacity: 0.6 + confidence * 0.4 }); } + + /** + * Update live metrics overlay on the active chart + */ + updateLiveMetrics(metrics) { + try { + // Get the active timeframe (first in list) + const activeTimeframe = window.appState?.currentTimeframes?.[0] || '1m'; + const chart = this.charts[activeTimeframe]; + + if (!chart) return; + + const plotId = chart.plotId; + const plotElement = document.getElementById(plotId); + + if (!plotElement) return; + + // Create or update metrics overlay + let overlay = document.getElementById(`metrics-overlay-${activeTimeframe}`); + + if (!overlay) { + // Create overlay div + overlay = document.createElement('div'); + overlay.id = `metrics-overlay-${activeTimeframe}`; + overlay.style.cssText = ` + position: absolute; + top: 10px; + left: 10px; + background: rgba(0, 0, 0, 0.85); + color: #fff; + padding: 8px 12px; + border-radius: 6px; + font-family: 'Courier New', monospace; + font-size: 12px; + z-index: 1000; + pointer-events: none; + border: 1px solid rgba(255, 255, 255, 0.2); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3); + `; + + // Append to plot container + plotElement.parentElement.style.position = 'relative'; + plotElement.parentElement.appendChild(overlay); + } + + // Format metrics + const accuracy = metrics.accuracy ? (metrics.accuracy * 100).toFixed(1) : '--'; + const loss = metrics.loss ? metrics.loss.toFixed(4) : '--'; + + // Determine color based on loss (lower is better) + let lossColor = '#10b981'; // Green + if (metrics.loss > 0.5) { + lossColor = '#ef4444'; // Red + } else if (metrics.loss > 0.3) { + lossColor = '#f59e0b'; // Yellow + } + + // Update content + overlay.innerHTML = ` +