From 44379ae2e46fdd5580298d57beb42753f0620049 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sat, 22 Nov 2025 18:46:44 +0200 Subject: [PATCH] candles wip --- ANNOTATE/web/static/js/chart_manager.js | 241 +++++++++++++++--- .../templates/components/training_panel.html | 132 +++++++++- 2 files changed, 337 insertions(+), 36 deletions(-) diff --git a/ANNOTATE/web/static/js/chart_manager.js b/ANNOTATE/web/static/js/chart_manager.js index a3b5fcf..477c2a8 100644 --- a/ANNOTATE/web/static/js/chart_manager.js +++ b/ANNOTATE/web/static/js/chart_manager.js @@ -143,7 +143,10 @@ class ChartManager { const queryTime = new Date(lastTimeMs - lookbackMs).toISOString(); - // Fetch data starting from overlap point + // Fetch data starting from overlap point + // IMPORTANT: Use larger limit to ensure we don't lose historical candles + // For 1s charts, we need to preserve all 2500 candles, so fetch enough overlap + const fetchLimit = timeframe === '1s' ? 100 : 50; // More candles for 1s to prevent data loss const response = await fetch('/api/chart-data', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -151,7 +154,7 @@ class ChartManager { symbol: window.appState?.currentSymbol || 'ETH/USDT', timeframes: [timeframe], start_time: queryTime, - limit: 50, // Small limit for incremental update + limit: fetchLimit, // Increased limit to preserve more candles direction: 'after' }) }); @@ -231,9 +234,23 @@ class ChartManager { }); } + // CRITICAL: Preserve all historical candles - never truncate below 2500 + // Only keep last 2500 candles if we exceed that limit (to prevent memory issues) + const maxCandles = 2500; + if (chart.data.timestamps.length > maxCandles) { + const excess = chart.data.timestamps.length - maxCandles; + console.log(`[${timeframe}] Truncating ${excess} old candles (keeping last ${maxCandles})`); + chart.data.timestamps = chart.data.timestamps.slice(-maxCandles); + chart.data.open = chart.data.open.slice(-maxCandles); + chart.data.high = chart.data.high.slice(-maxCandles); + chart.data.low = chart.data.low.slice(-maxCandles); + chart.data.close = chart.data.close.slice(-maxCandles); + chart.data.volume = chart.data.volume.slice(-maxCandles); + } + // 4. Recalculate and Redraw if (updatesCount > 0 || remainingTimestamps.length > 0) { - console.log(`[${timeframe}] Chart update: ${updatesCount} updated, ${remainingTimestamps.length} new candles`); + console.log(`[${timeframe}] Chart update: ${updatesCount} updated, ${remainingTimestamps.length} new candles, total: ${chart.data.timestamps.length}`); // Only recalculate pivots if we have NEW candles (not just updates to existing ones) // This prevents unnecessary pivot recalculation on every live candle update @@ -241,6 +258,7 @@ class ChartManager { this.recalculatePivots(timeframe, chart.data); } + // CRITICAL: Ensure we're updating with ALL candles, not just the fetched subset this.updateSingleChart(timeframe, chart.data); window.liveUpdateCount = (window.liveUpdateCount || 0) + 1; @@ -313,8 +331,12 @@ class ChartManager { const volumeTrace = chartData[1]; // Check if this is updating the last candle or adding a new one + // Use more lenient comparison to handle timestamp format differences const lastTimestamp = candlestickTrace.x[candlestickTrace.x.length - 1]; - const isNewCandle = !lastTimestamp || new Date(lastTimestamp).getTime() < candleTimestamp.getTime(); + const lastTimeMs = lastTimestamp ? new Date(lastTimestamp).getTime() : 0; + const candleTimeMs = candleTimestamp.getTime(); + // Consider it a new candle if timestamp is at least 500ms newer (to handle jitter) + const isNewCandle = !lastTimestamp || (candleTimeMs - lastTimeMs) >= 500; if (isNewCandle) { // Add new candle - update both Plotly and internal data structure @@ -410,19 +432,13 @@ class ChartManager { } if (validationCandleIdx >= 0 && validationCandleIdx < chart.data.timestamps.length) { - // Create validation data structure for the confirmed candle - const validationData = { - timestamps: [chart.data.timestamps[validationCandleIdx]], - open: [chart.data.open[validationCandleIdx]], - high: [chart.data.high[validationCandleIdx]], - low: [chart.data.low[validationCandleIdx]], - close: [chart.data.close[validationCandleIdx]], - volume: [chart.data.volume[validationCandleIdx]] - }; + // Pass full chart data for validation (not just one candle) + // This allows the validation function to check all recent candles + console.debug(`[${timeframe}] Triggering validation check for candle at index ${validationCandleIdx}`); + this._checkPredictionAccuracy(timeframe, chart.data); - // Trigger validation check - console.log(`[${timeframe}] Checking validation for confirmed candle at index ${validationCandleIdx}`); - this._checkPredictionAccuracy(timeframe, validationData); + // Refresh prediction display to show validation results + this._refreshPredictionDisplay(timeframe); } } @@ -724,8 +740,15 @@ class ChartManager { plotId: plotId, data: data, element: plotElement, - annotations: [] + annotations: [], + signalBanner: null // Will hold signal banner element }; + + // Add signal banner above chart + const chartContainer = document.getElementById(`chart-${timeframe}`); + if (chartContainer) { + this._addSignalBanner(timeframe, chartContainer); + } // Add click handler for chart and annotations plotElement.on('plotly_click', (eventData) => { @@ -837,8 +860,9 @@ class ChartManager { // Handle vertical zoom drag if (isDraggingYAxis && dragStartY !== null && dragStartRange !== null) { - const deltaY = dragStartY - event.clientY; // Negative = zoom in (drag up), Positive = zoom out (drag down) - const zoomFactor = 1 + (deltaY / 200); // Adjust sensitivity (200px = 2x zoom) + // REVERSED: Positive deltaY (drag down) = zoom in (make candles shorter) + const deltaY = event.clientY - dragStartY; // Positive = drag down, negative = drag up + const zoomFactor = 1 + (deltaY / 100); // Increased sensitivity: 100px = 2x zoom (was 200px) // Clamp zoom factor to reasonable limits const clampedZoom = Math.max(0.1, Math.min(10, zoomFactor)); @@ -909,7 +933,7 @@ class ChartManager { }; } - console.log(`[${timeframe}] Y-axis vertical zoom enabled - drag on left side (Y-axis area) to zoom vertically`); + console.log(`[${timeframe}] Y-axis vertical zoom enabled - drag DOWN to zoom in (shorter candles), drag UP to zoom out`); } /** @@ -2049,6 +2073,31 @@ class ChartManager { const plotElement = document.getElementById(plotId); if (!plotElement) return; + // CRITICAL: Validate data integrity - ensure we have enough candles + if (!data.timestamps || data.timestamps.length === 0) { + console.warn(`[${timeframe}] updateSingleChart called with empty data - skipping update`); + return; + } + + // Check if we're losing candles (should have at least 2500 for live training) + const currentCandleCount = data.timestamps.length; + if (currentCandleCount < 100 && chart.data && chart.data.timestamps && chart.data.timestamps.length > 100) { + console.error(`[${timeframe}] WARNING: Data truncation detected! Had ${chart.data.timestamps.length} candles, now only ${currentCandleCount}. Restoring from chart.data.`); + // Restore from chart.data if it has more candles + data = chart.data; + } + + // Store updated data back to chart for future reference + chart.data = { + timestamps: [...data.timestamps], + open: [...data.open], + high: [...data.high], + low: [...data.low], + close: [...data.close], + volume: [...data.volume], + pivot_markers: data.pivot_markers || chart.data?.pivot_markers || {} + }; + // Create volume colors const volumeColors = data.close.map((close, i) => { if (i === 0) return '#3b82f6'; @@ -2084,7 +2133,7 @@ class ChartManager { // Use react instead of restyle - it's smarter about what to update Plotly.react(plotId, updatedTraces, plotElement.layout, plotElement.config); - console.log(`Updated ${timeframe} chart with ${data.timestamps.length} candles`); + console.log(`[${timeframe}] Updated chart with ${data.timestamps.length} candles`); // Check if any ghost predictions match new actual candles and calculate accuracy this._checkPredictionAccuracy(timeframe, data); @@ -2142,18 +2191,30 @@ class ChartManager { }); } - // Debug logging for unmatched predictions + // Debug logging for unmatched predictions older than 30 seconds if (matchIdx < 0) { // Parse both timestamps to compare const predTimeParsed = new Date(prediction.timestamp); const latestActual = new Date(timestamps[timestamps.length - 1]); + const ageMs = latestActual - predTimeParsed; - if (idx < 3) { // Only log first 3 to avoid spam - console.log(`[${timeframe}] No match for prediction:`, { + // If prediction is older than 30 seconds and still not matched, mark as failed + if (ageMs > 30000) { + prediction.accuracy = { + overall: 0, + directionCorrect: false, + validationStatus: 'EXPIRED (no match)', + errors: { message: `Prediction expired after ${(ageMs / 1000).toFixed(0)}s without match` } + }; + validatedCount++; + console.log(`[${timeframe}] Marked prediction as EXPIRED: ${(ageMs / 1000).toFixed(0)}s old`); + } else if (idx < 3) { + // Only log first 3 unmatched recent predictions to avoid spam + console.debug(`[${timeframe}] No match yet for prediction:`, { predTimestamp: prediction.timestamp, predTime: predTimeParsed.toISOString(), latestActual: latestActual.toISOString(), - timeDiff: (latestActual - predTimeParsed) + 'ms', + ageSeconds: (ageMs / 1000).toFixed(1) + 's', tolerance: tolerance + 'ms', availableTimestamps: timestamps.slice(-3) // Last 3 actual timestamps }); @@ -2529,14 +2590,40 @@ class ChartManager { const inferenceTime = new Date(predictionTimestamp); let targetTimestamp; - if (timeframe === '1s') { - targetTimestamp = new Date(inferenceTime.getTime() + 1000); - } else if (timeframe === '1m') { - targetTimestamp = new Date(inferenceTime.getTime() + 60000); - } else if (timeframe === '1h') { - targetTimestamp = new Date(inferenceTime.getTime() + 3600000); + // Get the last real candle timestamp to ensure we predict the NEXT one + const lastRealCandle = chart.data.timestamps[chart.data.timestamps.length - 1]; + if (lastRealCandle) { + const lastCandleTime = new Date(lastRealCandle); + // Predict for the next candle period + if (timeframe === '1s') { + targetTimestamp = new Date(lastCandleTime.getTime() + 1000); + } else if (timeframe === '1m') { + targetTimestamp = new Date(lastCandleTime.getTime() + 60000); + } else if (timeframe === '1h') { + targetTimestamp = new Date(lastCandleTime.getTime() + 3600000); + } else { + targetTimestamp = new Date(lastCandleTime.getTime() + 60000); + } } else { - targetTimestamp = new Date(inferenceTime.getTime() + 60000); + // Fallback to inference time + period if no real candles yet + if (timeframe === '1s') { + targetTimestamp = new Date(inferenceTime.getTime() + 1000); + } else if (timeframe === '1m') { + targetTimestamp = new Date(inferenceTime.getTime() + 60000); + } else if (timeframe === '1h') { + targetTimestamp = new Date(inferenceTime.getTime() + 3600000); + } else { + targetTimestamp = new Date(inferenceTime.getTime() + 60000); + } + } + + // Round to exact candle boundary to prevent bunching + if (timeframe === '1s') { + targetTimestamp = new Date(Math.floor(targetTimestamp.getTime() / 1000) * 1000); + } else if (timeframe === '1m') { + targetTimestamp = new Date(Math.floor(targetTimestamp.getTime() / 60000) * 60000); + } else if (timeframe === '1h') { + targetTimestamp = new Date(Math.floor(targetTimestamp.getTime() / 3600000) * 3600000); } // 1. Initialize ghost candle history for this timeframe if needed @@ -2621,6 +2708,14 @@ class ChartManager { Plotly.deleteTraces(plotId, indicesToRemove); } + // CRITICAL: Ensure real candles are visible first + // Check that candlestick trace exists and has data + const candlestickTrace = plotElement.data.find(t => t.type === 'candlestick'); + if (!candlestickTrace || !candlestickTrace.x || candlestickTrace.x.length === 0) { + console.warn(`[${timeframe}] No real candles found - skipping prediction display`); + return; + } + // Add new traces - these will overlay on top of real candles // Plotly renders traces in order, so predictions added last appear on top Plotly.addTraces(plotId, predictionTraces); @@ -3064,6 +3159,88 @@ class ChartManager { } } + /** + * Add signal banner above chart to show timeframe-specific signals + */ + _addSignalBanner(timeframe, container) { + try { + const bannerId = `signal-banner-${timeframe}`; + let banner = document.getElementById(bannerId); + + if (!banner) { + banner = document.createElement('div'); + banner.id = bannerId; + banner.className = 'signal-banner'; + banner.style.cssText = ` + position: absolute; + top: 5px; + right: 10px; + padding: 4px 8px; + background-color: rgba(0, 0, 0, 0.7); + border-radius: 4px; + font-size: 11px; + font-weight: bold; + z-index: 1000; + display: none; + `; + banner.innerHTML = ` + [${timeframe}] + -- + -- + `; + container.style.position = 'relative'; + container.insertBefore(banner, container.firstChild); + + // Store reference + if (this.charts[timeframe]) { + this.charts[timeframe].signalBanner = banner; + } + } + } catch (error) { + console.error(`Error adding signal banner for ${timeframe}:`, error); + } + } + + /** + * Update signal banner for a specific timeframe + */ + updateSignalBanner(timeframe, signal, confidence) { + try { + const chart = this.charts[timeframe]; + if (!chart || !chart.signalBanner) return; + + const banner = chart.signalBanner; + const signalText = banner.querySelector('.signal-text'); + const signalConf = banner.querySelector('.signal-confidence'); + + if (!signalText || !signalConf) return; + + // Show banner + banner.style.display = 'block'; + + // Update signal text and color + let signalColor; + if (signal === 'BUY') { + signalColor = '#10b981'; // Green + } else if (signal === 'SELL') { + signalColor = '#ef4444'; // Red + } else { + signalColor = '#6b7280'; // Gray for HOLD + } + + signalText.textContent = signal; + signalText.style.color = signalColor; + + // Update confidence + const confPct = (confidence * 100).toFixed(0); + signalConf.textContent = `${confPct}%`; + signalConf.style.color = confidence >= 0.6 ? '#10b981' : '#9ca3af'; + + } catch (error) { + console.error(`Error updating signal banner for ${timeframe}:`, error); + } + } + /** * Add executed trade marker to chart * Shows entry/exit points, PnL, and position lines diff --git a/ANNOTATE/web/templates/components/training_panel.html b/ANNOTATE/web/templates/components/training_panel.html index 4c616cd..d2cc736 100644 --- a/ANNOTATE/web/templates/components/training_panel.html +++ b/ANNOTATE/web/templates/components/training_panel.html @@ -141,12 +141,42 @@