candles wip

ANNOTATE/web/static/js/chart_manager.js

@@ -143,7 +143,10 @@ class ChartManager {
             
             const queryTime = new Date(lastTimeMs - lookbackMs).toISOString();
             
-            // Fetch data starting from overlap point
+                    // Fetch data starting from overlap point
+            // IMPORTANT: Use larger limit to ensure we don't lose historical candles
+            // For 1s charts, we need to preserve all 2500 candles, so fetch enough overlap
+            const fetchLimit = timeframe === '1s' ? 100 : 50; // More candles for 1s to prevent data loss
             const response = await fetch('/api/chart-data', {
                 method: 'POST',
                 headers: { 'Content-Type': 'application/json' },
@@ -151,7 +154,7 @@ class ChartManager {
                     symbol: window.appState?.currentSymbol || 'ETH/USDT',
                     timeframes: [timeframe],
                     start_time: queryTime,
-                    limit: 50, // Small limit for incremental update
+                    limit: fetchLimit, // Increased limit to preserve more candles
                     direction: 'after'
                 })
             });
@@ -231,9 +234,23 @@ class ChartManager {
                         });
                     }
                     
+                    // CRITICAL: Preserve all historical candles - never truncate below 2500
+                    // Only keep last 2500 candles if we exceed that limit (to prevent memory issues)
+                    const maxCandles = 2500;
+                    if (chart.data.timestamps.length > maxCandles) {
+                        const excess = chart.data.timestamps.length - maxCandles;
+                        console.log(`[${timeframe}] Truncating ${excess} old candles (keeping last ${maxCandles})`);
+                        chart.data.timestamps = chart.data.timestamps.slice(-maxCandles);
+                        chart.data.open = chart.data.open.slice(-maxCandles);
+                        chart.data.high = chart.data.high.slice(-maxCandles);
+                        chart.data.low = chart.data.low.slice(-maxCandles);
+                        chart.data.close = chart.data.close.slice(-maxCandles);
+                        chart.data.volume = chart.data.volume.slice(-maxCandles);
+                    }
+                    
                     // 4. Recalculate and Redraw
                     if (updatesCount > 0 || remainingTimestamps.length > 0) {
-                        console.log(`[${timeframe}] Chart update: ${updatesCount} updated, ${remainingTimestamps.length} new candles`);
+                        console.log(`[${timeframe}] Chart update: ${updatesCount} updated, ${remainingTimestamps.length} new candles, total: ${chart.data.timestamps.length}`);
                         
                         // Only recalculate pivots if we have NEW candles (not just updates to existing ones)
                         // This prevents unnecessary pivot recalculation on every live candle update
@@ -241,6 +258,7 @@ class ChartManager {
                             this.recalculatePivots(timeframe, chart.data);
                         }
                         
+                        // CRITICAL: Ensure we're updating with ALL candles, not just the fetched subset
                         this.updateSingleChart(timeframe, chart.data);
                         
                         window.liveUpdateCount = (window.liveUpdateCount || 0) + 1;
@@ -313,8 +331,12 @@ class ChartManager {
             const volumeTrace = chartData[1];
             
             // Check if this is updating the last candle or adding a new one
+            // Use more lenient comparison to handle timestamp format differences
             const lastTimestamp = candlestickTrace.x[candlestickTrace.x.length - 1];
-            const isNewCandle = !lastTimestamp || new Date(lastTimestamp).getTime() < candleTimestamp.getTime();
+            const lastTimeMs = lastTimestamp ? new Date(lastTimestamp).getTime() : 0;
+            const candleTimeMs = candleTimestamp.getTime();
+            // Consider it a new candle if timestamp is at least 500ms newer (to handle jitter)
+            const isNewCandle = !lastTimestamp || (candleTimeMs - lastTimeMs) >= 500;
             
             if (isNewCandle) {
                 // Add new candle - update both Plotly and internal data structure
@@ -410,19 +432,13 @@ class ChartManager {
                 }
                 
                 if (validationCandleIdx >= 0 && validationCandleIdx < chart.data.timestamps.length) {
-                    // Create validation data structure for the confirmed candle
+                    // Pass full chart data for validation (not just one candle)
-                    const validationData = {
+                    // This allows the validation function to check all recent candles
-                        timestamps: [chart.data.timestamps[validationCandleIdx]],
+                    console.debug(`[${timeframe}] Triggering validation check for candle at index ${validationCandleIdx}`);
-                        open: [chart.data.open[validationCandleIdx]],
+                    this._checkPredictionAccuracy(timeframe, chart.data);
-                        high: [chart.data.high[validationCandleIdx]],
-                        low: [chart.data.low[validationCandleIdx]],
-                        close: [chart.data.close[validationCandleIdx]],
-                        volume: [chart.data.volume[validationCandleIdx]]
-                    };
                     
-                    // Trigger validation check
+                    // Refresh prediction display to show validation results
-                    console.log(`[${timeframe}] Checking validation for confirmed candle at index ${validationCandleIdx}`);
+                    this._refreshPredictionDisplay(timeframe);
-                    this._checkPredictionAccuracy(timeframe, validationData);
                 }
             }
             
@@ -724,8 +740,15 @@ class ChartManager {
             plotId: plotId,
             data: data,
             element: plotElement,
-            annotations: []
+            annotations: [],
+            signalBanner: null  // Will hold signal banner element
         };
+        
+        // Add signal banner above chart
+        const chartContainer = document.getElementById(`chart-${timeframe}`);
+        if (chartContainer) {
+            this._addSignalBanner(timeframe, chartContainer);
+        }
 
         // Add click handler for chart and annotations
         plotElement.on('plotly_click', (eventData) => {
@@ -837,8 +860,9 @@ class ChartManager {
             
             // Handle vertical zoom drag
             if (isDraggingYAxis && dragStartY !== null && dragStartRange !== null) {
-                const deltaY = dragStartY - event.clientY;  // Negative = zoom in (drag up), Positive = zoom out (drag down)
+                // REVERSED: Positive deltaY (drag down) = zoom in (make candles shorter)
-                const zoomFactor = 1 + (deltaY / 200);  // Adjust sensitivity (200px = 2x zoom)
+                const deltaY = event.clientY - dragStartY;  // Positive = drag down, negative = drag up
+                const zoomFactor = 1 + (deltaY / 100);  // Increased sensitivity: 100px = 2x zoom (was 200px)
                 
                 // Clamp zoom factor to reasonable limits
                 const clampedZoom = Math.max(0.1, Math.min(10, zoomFactor));
@@ -909,7 +933,7 @@ class ChartManager {
             };
         }
         
-        console.log(`[${timeframe}] Y-axis vertical zoom enabled - drag on left side (Y-axis area) to zoom vertically`);
+        console.log(`[${timeframe}] Y-axis vertical zoom enabled - drag DOWN to zoom in (shorter candles), drag UP to zoom out`);
     }
 
     /**
@@ -2049,6 +2073,31 @@ class ChartManager {
         const plotElement = document.getElementById(plotId);
         if (!plotElement) return;
 
+        // CRITICAL: Validate data integrity - ensure we have enough candles
+        if (!data.timestamps || data.timestamps.length === 0) {
+            console.warn(`[${timeframe}] updateSingleChart called with empty data - skipping update`);
+            return;
+        }
+        
+        // Check if we're losing candles (should have at least 2500 for live training)
+        const currentCandleCount = data.timestamps.length;
+        if (currentCandleCount < 100 && chart.data && chart.data.timestamps && chart.data.timestamps.length > 100) {
+            console.error(`[${timeframe}] WARNING: Data truncation detected! Had ${chart.data.timestamps.length} candles, now only ${currentCandleCount}. Restoring from chart.data.`);
+            // Restore from chart.data if it has more candles
+            data = chart.data;
+        }
+        
+        // Store updated data back to chart for future reference
+        chart.data = {
+            timestamps: [...data.timestamps],
+            open: [...data.open],
+            high: [...data.high],
+            low: [...data.low],
+            close: [...data.close],
+            volume: [...data.volume],
+            pivot_markers: data.pivot_markers || chart.data?.pivot_markers || {}
+        };
+
         // Create volume colors
         const volumeColors = data.close.map((close, i) => {
             if (i === 0) return '#3b82f6';
@@ -2084,7 +2133,7 @@ class ChartManager {
         // Use react instead of restyle - it's smarter about what to update
         Plotly.react(plotId, updatedTraces, plotElement.layout, plotElement.config);
 
-        console.log(`Updated ${timeframe} chart with ${data.timestamps.length} candles`);
+        console.log(`[${timeframe}] Updated chart with ${data.timestamps.length} candles`);
         
         // Check if any ghost predictions match new actual candles and calculate accuracy
         this._checkPredictionAccuracy(timeframe, data);
@@ -2142,18 +2191,30 @@ class ChartManager {
                 });
             }
             
-            // Debug logging for unmatched predictions
+            // Debug logging for unmatched predictions older than 30 seconds
             if (matchIdx < 0) {
                 // Parse both timestamps to compare
                 const predTimeParsed = new Date(prediction.timestamp);
                 const latestActual = new Date(timestamps[timestamps.length - 1]);
+                const ageMs = latestActual - predTimeParsed;
                 
-                if (idx < 3) {  // Only log first 3 to avoid spam
+                // If prediction is older than 30 seconds and still not matched, mark as failed
-                    console.log(`[${timeframe}] No match for prediction:`, {
+                if (ageMs > 30000) {
+                    prediction.accuracy = {
+                        overall: 0,
+                        directionCorrect: false,
+                        validationStatus: 'EXPIRED (no match)',
+                        errors: { message: `Prediction expired after ${(ageMs / 1000).toFixed(0)}s without match` }
+                    };
+                    validatedCount++;
+                    console.log(`[${timeframe}] Marked prediction as EXPIRED: ${(ageMs / 1000).toFixed(0)}s old`);
+                } else if (idx < 3) {
+                    // Only log first 3 unmatched recent predictions to avoid spam
+                    console.debug(`[${timeframe}] No match yet for prediction:`, {
                         predTimestamp: prediction.timestamp,
                         predTime: predTimeParsed.toISOString(),
                         latestActual: latestActual.toISOString(),
-                        timeDiff: (latestActual - predTimeParsed) + 'ms',
+                        ageSeconds: (ageMs / 1000).toFixed(1) + 's',
                         tolerance: tolerance + 'ms',
                         availableTimestamps: timestamps.slice(-3)  // Last 3 actual timestamps
                     });
@@ -2529,14 +2590,40 @@ class ChartManager {
                         const inferenceTime = new Date(predictionTimestamp);
                         let targetTimestamp;
                         
-                        if (timeframe === '1s') {
+                        // Get the last real candle timestamp to ensure we predict the NEXT one
-                            targetTimestamp = new Date(inferenceTime.getTime() + 1000);
+                        const lastRealCandle = chart.data.timestamps[chart.data.timestamps.length - 1];
-                        } else if (timeframe === '1m') {
+                        if (lastRealCandle) {
-                            targetTimestamp = new Date(inferenceTime.getTime() + 60000);
+                            const lastCandleTime = new Date(lastRealCandle);
-                        } else if (timeframe === '1h') {
+                            // Predict for the next candle period
-                            targetTimestamp = new Date(inferenceTime.getTime() + 3600000);
+                            if (timeframe === '1s') {
+                                targetTimestamp = new Date(lastCandleTime.getTime() + 1000);
+                            } else if (timeframe === '1m') {
+                                targetTimestamp = new Date(lastCandleTime.getTime() + 60000);
+                            } else if (timeframe === '1h') {
+                                targetTimestamp = new Date(lastCandleTime.getTime() + 3600000);
+                            } else {
+                                targetTimestamp = new Date(lastCandleTime.getTime() + 60000);
+                            }
                         } else {
-                            targetTimestamp = new Date(inferenceTime.getTime() + 60000);
+                            // Fallback to inference time + period if no real candles yet
+                            if (timeframe === '1s') {
+                                targetTimestamp = new Date(inferenceTime.getTime() + 1000);
+                            } else if (timeframe === '1m') {
+                                targetTimestamp = new Date(inferenceTime.getTime() + 60000);
+                            } else if (timeframe === '1h') {
+                                targetTimestamp = new Date(inferenceTime.getTime() + 3600000);
+                            } else {
+                                targetTimestamp = new Date(inferenceTime.getTime() + 60000);
+                            }
+                        }
+                        
+                        // Round to exact candle boundary to prevent bunching
+                        if (timeframe === '1s') {
+                            targetTimestamp = new Date(Math.floor(targetTimestamp.getTime() / 1000) * 1000);
+                        } else if (timeframe === '1m') {
+                            targetTimestamp = new Date(Math.floor(targetTimestamp.getTime() / 60000) * 60000);
+                        } else if (timeframe === '1h') {
+                            targetTimestamp = new Date(Math.floor(targetTimestamp.getTime() / 3600000) * 3600000);
                         }
                         
                         // 1. Initialize ghost candle history for this timeframe if needed
@@ -2621,6 +2708,14 @@ class ChartManager {
                     Plotly.deleteTraces(plotId, indicesToRemove);
                 }
                 
+                // CRITICAL: Ensure real candles are visible first
+                // Check that candlestick trace exists and has data
+                const candlestickTrace = plotElement.data.find(t => t.type === 'candlestick');
+                if (!candlestickTrace || !candlestickTrace.x || candlestickTrace.x.length === 0) {
+                    console.warn(`[${timeframe}] No real candles found - skipping prediction display`);
+                    return;
+                }
+                
                 // Add new traces - these will overlay on top of real candles
                 // Plotly renders traces in order, so predictions added last appear on top
                 Plotly.addTraces(plotId, predictionTraces);
@@ -3064,6 +3159,88 @@ class ChartManager {
         }
     }
     
+    /**
+     * Add signal banner above chart to show timeframe-specific signals
+     */
+    _addSignalBanner(timeframe, container) {
+        try {
+            const bannerId = `signal-banner-${timeframe}`;
+            let banner = document.getElementById(bannerId);
+            
+            if (!banner) {
+                banner = document.createElement('div');
+                banner.id = bannerId;
+                banner.className = 'signal-banner';
+                banner.style.cssText = `
+                    position: absolute;
+                    top: 5px;
+                    right: 10px;
+                    padding: 4px 8px;
+                    background-color: rgba(0, 0, 0, 0.7);
+                    border-radius: 4px;
+                    font-size: 11px;
+                    font-weight: bold;
+                    z-index: 1000;
+                    display: none;
+                `;
+                banner.innerHTML = `
+                    <span style="color: #9ca3af;">[${timeframe}]</span>
+                    <span class="signal-text" style="margin-left: 4px;">--</span>
+                    <span class="signal-confidence" style="margin-left: 4px; font-size: 9px; color: #9ca3af;">--</span>
+                `;
+                container.style.position = 'relative';
+                container.insertBefore(banner, container.firstChild);
+                
+                // Store reference
+                if (this.charts[timeframe]) {
+                    this.charts[timeframe].signalBanner = banner;
+                }
+            }
+        } catch (error) {
+            console.error(`Error adding signal banner for ${timeframe}:`, error);
+        }
+    }
+    
+    /**
+     * Update signal banner for a specific timeframe
+     */
+    updateSignalBanner(timeframe, signal, confidence) {
+        try {
+            const chart = this.charts[timeframe];
+            if (!chart || !chart.signalBanner) return;
+            
+            const banner = chart.signalBanner;
+            const signalText = banner.querySelector('.signal-text');
+            const signalConf = banner.querySelector('.signal-confidence');
+            
+            if (!signalText || !signalConf) return;
+            
+            // Show banner
+            banner.style.display = 'block';
+            
+            // Update signal text and color
+            let signalColor;
+            if (signal === 'BUY') {
+                signalColor = '#10b981';  // Green
+            } else if (signal === 'SELL') {
+                signalColor = '#ef4444';  // Red
+            } else {
+                signalColor = '#6b7280';  // Gray for HOLD
+            }
+            
+            signalText.textContent = signal;
+            signalText.style.color = signalColor;
+            
+            // Update confidence
+            const confPct = (confidence * 100).toFixed(0);
+            signalConf.textContent = `${confPct}%`;
+            signalConf.style.color = confidence >= 0.6 ? '#10b981' : '#9ca3af';
+            
+        } catch (error) {
+            console.error(`Error updating signal banner for ${timeframe}:`, error);
+        }
+    }
+    
     /**
      * Add executed trade marker to chart
      * Shows entry/exit points, PnL, and position lines

ANNOTATE/web/templates/components/training_panel.html

@@ -141,12 +141,42 @@
         <!-- Inference Status -->
         <div id="inference-status" style="display: none;">
             <div class="alert alert-success py-2 px-2 mb-2">
-                <div class="d-flex align-items-center mb-1">
+                <div class="d-flex align-items-center justify-content-between mb-1">
-                    <div class="spinner-border spinner-border-sm me-2" role="status">
+                    <div class="d-flex align-items-center">
-                        <span class="visually-hidden">Running...</span>
+                        <div class="spinner-border spinner-border-sm me-2" role="status">
+                            <span class="visually-hidden">Running...</span>
+                        </div>
+                        <strong class="small">🔴 LIVE</strong>
+                    </div>
+                    <!-- Model Performance -->
+                    <div class="small text-end">
+                        <div style="font-size: 0.65rem;">Acc: <span id="live-accuracy" class="fw-bold text-success">--</span></div>
+                        <div style="font-size: 0.65rem;">Loss: <span id="live-loss" class="fw-bold text-warning">--</span></div>
                     </div>
-                    <strong class="small">🔴 LIVE</strong>
                 </div>
+                
+                <!-- Position & PnL Status -->
+                <div class="mb-2 p-2" style="background-color: rgba(0,0,0,0.1); border-radius: 4px;">
+                    <div class="small">
+                        <div class="d-flex justify-content-between">
+                            <span>Position:</span>
+                            <span id="position-status" class="fw-bold text-info">NO POSITION</span>
+                        </div>
+                        <div class="d-flex justify-content-between" id="floating-pnl-row" style="display: none !important;">
+                            <span>Floating PnL:</span>
+                            <span id="floating-pnl" class="fw-bold">--</span>
+                        </div>
+                        <div class="d-flex justify-content-between">
+                            <span>Session PnL:</span>
+                            <span id="session-pnl" class="fw-bold text-success">+$0.00</span>
+                        </div>
+                        <div class="d-flex justify-content-between" style="font-size: 0.7rem; color: #9ca3af;">
+                            <span>Win Rate:</span>
+                            <span id="win-rate">0% (0/0)</span>
+                        </div>
+                    </div>
+                </div>
+                
                 <div class="small">
                     <div>Timeframe: <span id="active-timeframe" class="fw-bold text-primary">--</span></div>
                     <div>Signal: <span id="latest-signal" class="fw-bold">--</span></div>
@@ -285,6 +315,36 @@
 
                     console.log(`✓ Models available: ${data.available_count}, loaded: ${data.loaded_count}`);
 
+                    // Auto-select Transformer (or any loaded model) if available
+                    let modelToSelect = null;
+                    // First try to find Transformer
+                    const transformerModel = data.models.find(m => {
+                        const modelName = (m && typeof m === 'object' && m.name) ? m.name : String(m);
+                        const isLoaded = (m && typeof m === 'object' && 'loaded' in m) ? m.loaded : false;
+                        return modelName === 'Transformer' && isLoaded;
+                    });
+                    
+                    if (transformerModel) {
+                        modelToSelect = 'Transformer';
+                    } else {
+                        // If Transformer not loaded, find any loaded model
+                        const loadedModel = data.models.find(m => {
+                            const isLoaded = (m && typeof m === 'object' && 'loaded' in m) ? m.loaded : false;
+                            return isLoaded;
+                        });
+                        if (loadedModel) {
+                            const modelName = (loadedModel && typeof loadedModel === 'object' && loadedModel.name) ? loadedModel.name : String(loadedModel);
+                            modelToSelect = modelName;
+                        }
+                    }
+                    
+                    // Auto-select if found
+                    if (modelToSelect) {
+                        modelSelect.value = modelToSelect;
+                        selectedModel = modelToSelect;
+                        console.log(`✓ Auto-selected loaded model: ${modelToSelect}`);
+                    }
+
                     // Update button state for currently selected model
                     updateButtonState();
                 } else {
@@ -988,6 +1048,70 @@
         }
     }
     
+    function updatePositionStateDisplay(positionState, sessionMetrics) {
+        /**
+         * Update live trading panel with current position and PnL info
+         */
+        try {
+            // Update position status
+            const positionStatusEl = document.getElementById('position-status');
+            const floatingPnlRow = document.getElementById('floating-pnl-row');
+            const floatingPnlEl = document.getElementById('floating-pnl');
+            
+            if (positionState.has_position) {
+                const posType = positionState.position_type.toUpperCase();
+                const entryPrice = positionState.entry_price.toFixed(2);
+                positionStatusEl.textContent = `${posType} @ $${entryPrice}`;
+                positionStatusEl.className = posType === 'LONG' ? 'fw-bold text-success' : 'fw-bold text-danger';
+                
+                // Show floating PnL
+                if (floatingPnlRow) {
+                    floatingPnlRow.style.display = 'flex !important';
+                    floatingPnlRow.classList.remove('d-none');
+                }
+                const unrealizedPnl = positionState.unrealized_pnl || 0;
+                const pnlColor = unrealizedPnl >= 0 ? 'text-success' : 'text-danger';
+                const pnlSign = unrealizedPnl >= 0 ? '+' : '';
+                floatingPnlEl.textContent = `${pnlSign}${unrealizedPnl.toFixed(2)}%`;
+                floatingPnlEl.className = `fw-bold ${pnlColor}`;
+            } else {
+                positionStatusEl.textContent = 'NO POSITION';
+                positionStatusEl.className = 'fw-bold text-secondary';
+                
+                // Hide floating PnL row
+                if (floatingPnlRow) {
+                    floatingPnlRow.style.display = 'none !important';
+                    floatingPnlRow.classList.add('d-none');
+                }
+            }
+            
+            // Update session PnL
+            const sessionPnlEl = document.getElementById('session-pnl');
+            if (sessionPnlEl && sessionMetrics) {
+                const totalPnl = sessionMetrics.total_pnl || 0;
+                const pnlColor = totalPnl >= 0 ? 'text-success' : 'text-danger';
+                const pnlSign = totalPnl >= 0 ? '+' : '';
+                sessionPnlEl.textContent = `${pnlSign}$${totalPnl.toFixed(2)}`;
+                sessionPnlEl.className = `fw-bold ${pnlColor}`;
+                
+                // Update win rate
+                const winRateEl = document.getElementById('win-rate');
+                if (winRateEl) {
+                    const winRate = sessionMetrics.win_rate || 0;
+                    const winCount = sessionMetrics.win_count || 0;
+                    const totalTrades = sessionMetrics.total_trades || 0;
+                    winRateEl.textContent = `${winRate.toFixed(1)}% (${winCount}/${totalTrades})`;
+                }
+            }
+            
+        } catch (error) {
+            console.error('Error updating position state display:', error);
+        }
+    }
+    
+    // Make function globally accessible for WebSocket handler
+    window.updatePositionStateDisplay = updatePositionStateDisplay;
+    
     function updatePredictionHistory() {
         const historyDiv = document.getElementById('prediction-history');
         if (predictionHistory.length === 0) {