IMPLEMENTED: WIP; realtime candle predictions training

This commit is contained in:
Dobromir Popov
2025-11-22 17:57:58 +02:00
parent 423132dc8f
commit 26cbfd771b
4 changed files with 672 additions and 74 deletions

View File

@@ -15,8 +15,8 @@ class ChartManager {
this.lastPredictionUpdate = {}; // Track last prediction update per timeframe
this.predictionUpdateThrottle = 500; // Min ms between prediction updates
this.lastPredictionHash = null; // Track if predictions actually changed
this.ghostCandleHistory = {}; // Store ghost candles per timeframe (max 10 each)
this.maxGhostCandles = 10; // Maximum number of ghost candles to keep
this.ghostCandleHistory = {}; // Store ghost candles per timeframe (max 50 each)
this.maxGhostCandles = 150; // Maximum number of ghost candles to keep
// Helper to ensure all timestamps are in UTC
this.normalizeTimestamp = (timestamp) => {
@@ -264,15 +264,43 @@ class ChartManager {
*/
updateLatestCandle(symbol, timeframe, candle) {
try {
const plotId = `plot-${timeframe}`;
const plotElement = document.getElementById(plotId);
if (!plotElement) {
console.debug(`Chart ${plotId} not found for live update`);
const chart = this.charts[timeframe];
if (!chart) {
console.debug(`Chart ${timeframe} not found for live update`);
return;
}
// Get current chart data
const plotId = chart.plotId;
const plotElement = document.getElementById(plotId);
if (!plotElement) {
console.debug(`Plot element ${plotId} not found`);
return;
}
// Ensure chart.data exists
if (!chart.data) {
chart.data = {
timestamps: [],
open: [],
high: [],
low: [],
close: [],
volume: []
};
}
// Parse timestamp - format to match chart data format
const candleTimestamp = new Date(candle.timestamp);
const year = candleTimestamp.getUTCFullYear();
const month = String(candleTimestamp.getUTCMonth() + 1).padStart(2, '0');
const day = String(candleTimestamp.getUTCDate()).padStart(2, '0');
const hours = String(candleTimestamp.getUTCHours()).padStart(2, '0');
const minutes = String(candleTimestamp.getUTCMinutes()).padStart(2, '0');
const seconds = String(candleTimestamp.getUTCSeconds()).padStart(2, '0');
const formattedTimestamp = `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
// Get current chart data from Plotly
const chartData = Plotly.Plots.data(plotId);
if (!chartData || chartData.length < 2) {
console.debug(`Chart ${plotId} not initialized yet`);
@@ -282,17 +310,14 @@ class ChartManager {
const candlestickTrace = chartData[0];
const volumeTrace = chartData[1];
// Parse timestamp
const candleTimestamp = new Date(candle.timestamp);
// Check if this is updating the last candle or adding a new one
const lastTimestamp = candlestickTrace.x[candlestickTrace.x.length - 1];
const isNewCandle = !lastTimestamp || new Date(lastTimestamp).getTime() < candleTimestamp.getTime();
if (isNewCandle) {
// Add new candle using extendTraces (most efficient)
// Add new candle - update both Plotly and internal data structure
Plotly.extendTraces(plotId, {
x: [[candleTimestamp]],
x: [[formattedTimestamp]],
open: [[candle.open]],
high: [[candle.high]],
low: [[candle.low]],
@@ -302,27 +327,34 @@ class ChartManager {
// Update volume color based on price direction
const volumeColor = candle.close >= candle.open ? '#10b981' : '#ef4444';
Plotly.extendTraces(plotId, {
x: [[candleTimestamp]],
x: [[formattedTimestamp]],
y: [[candle.volume]],
marker: { color: [[volumeColor]] }
}, [1]);
} else {
// Update last candle using restyle - simpler approach for updating single point
// We need to get the full arrays, modify last element, and send back
// This is less efficient but more reliable for updates than complex index logic
const x = candlestickTrace.x;
const open = candlestickTrace.open;
const high = candlestickTrace.high;
const low = candlestickTrace.low;
const close = candlestickTrace.close;
const volume = volumeTrace.y;
const colors = volumeTrace.marker.color;
// Update internal data structure
chart.data.timestamps.push(formattedTimestamp);
chart.data.open.push(candle.open);
chart.data.high.push(candle.high);
chart.data.low.push(candle.low);
chart.data.close.push(candle.close);
chart.data.volume.push(candle.volume);
console.log(`[${timeframe}] Added new candle: ${formattedTimestamp}`);
} else {
// Update last candle - update both Plotly and internal data structure
const x = [...candlestickTrace.x];
const open = [...candlestickTrace.open];
const high = [...candlestickTrace.high];
const low = [...candlestickTrace.low];
const close = [...candlestickTrace.close];
const volume = [...volumeTrace.y];
const colors = Array.isArray(volumeTrace.marker.color) ? [...volumeTrace.marker.color] : [volumeTrace.marker.color];
const lastIdx = x.length - 1;
// Update local arrays
x[lastIdx] = candleTimestamp;
x[lastIdx] = formattedTimestamp;
open[lastIdx] = candle.open;
high[lastIdx] = candle.high;
low[lastIdx] = candle.low;
@@ -344,9 +376,55 @@ class ChartManager {
y: [volume],
'marker.color': [colors]
}, [1]);
// Update internal data structure
if (chart.data.timestamps.length > lastIdx) {
chart.data.timestamps[lastIdx] = formattedTimestamp;
chart.data.open[lastIdx] = candle.open;
chart.data.high[lastIdx] = candle.high;
chart.data.low[lastIdx] = candle.low;
chart.data.close[lastIdx] = candle.close;
chart.data.volume[lastIdx] = candle.volume;
}
console.log(`[${timeframe}] Updated last candle: ${formattedTimestamp}`);
}
console.debug(`Updated ${timeframe} chart with new candle at ${candleTimestamp.toISOString()}`);
// CRITICAL: Check if we have enough candles to validate predictions (2s delay logic)
// For 1s timeframe: validate against candle[-2] (last confirmed), overlay on candle[-1] (currently forming)
// For other timeframes: validate against candle[-1] when it's confirmed
if (chart.data.timestamps.length >= 2) {
// Determine which candle to validate against based on timeframe
let validationCandleIdx = -1;
if (timeframe === '1s') {
// 2s delay: validate against candle[-2] (last confirmed)
// This candle was closed 1-2 seconds ago
validationCandleIdx = chart.data.timestamps.length - 2;
} else {
// For longer timeframes, validate against last candle when it's confirmed
// A candle is confirmed when a new one starts forming
validationCandleIdx = isNewCandle ? chart.data.timestamps.length - 2 : -1;
}
if (validationCandleIdx >= 0 && validationCandleIdx < chart.data.timestamps.length) {
// Create validation data structure for the confirmed candle
const validationData = {
timestamps: [chart.data.timestamps[validationCandleIdx]],
open: [chart.data.open[validationCandleIdx]],
high: [chart.data.high[validationCandleIdx]],
low: [chart.data.low[validationCandleIdx]],
close: [chart.data.close[validationCandleIdx]],
volume: [chart.data.volume[validationCandleIdx]]
};
// Trigger validation check
console.log(`[${timeframe}] Checking validation for confirmed candle at index ${validationCandleIdx}`);
this._checkPredictionAccuracy(timeframe, validationData);
}
}
console.debug(`Updated ${timeframe} chart with candle at ${formattedTimestamp}`);
} catch (error) {
console.error(`Error updating latest candle for ${timeframe}:`, error);
}
@@ -1873,6 +1951,199 @@ class ChartManager {
Plotly.react(plotId, updatedTraces, plotElement.layout, plotElement.config);
console.log(`Updated ${timeframe} chart with ${data.timestamps.length} candles`);
// Check if any ghost predictions match new actual candles and calculate accuracy
this._checkPredictionAccuracy(timeframe, data);
}
/**
* Calculate prediction accuracy by comparing ghost predictions with actual candles
*/
_checkPredictionAccuracy(timeframe, actualData) {
if (!this.ghostCandleHistory || !this.ghostCandleHistory[timeframe]) return;
const predictions = this.ghostCandleHistory[timeframe];
const timestamps = actualData.timestamps;
const opens = actualData.open;
const highs = actualData.high;
const lows = actualData.low;
const closes = actualData.close;
// Determine tolerance based on timeframe
let tolerance;
if (timeframe === '1s') {
tolerance = 2000; // 2 seconds for 1s charts
} else if (timeframe === '1m') {
tolerance = 60000; // 60 seconds for 1m charts
} else if (timeframe === '1h') {
tolerance = 3600000; // 1 hour for hourly charts
} else {
tolerance = 5000; // 5 seconds default
}
// Check each prediction against actual candles
let validatedCount = 0;
predictions.forEach((prediction, idx) => {
// Skip if already validated
if (prediction.accuracy) return;
// Try multiple matching strategies
let matchIdx = -1;
// Use standard Date object if available, otherwise parse timestamp string
// Prioritize targetTime as it's the raw Date object set during prediction creation
const predTime = prediction.targetTime ? prediction.targetTime.getTime() : new Date(prediction.timestamp).getTime();
// Strategy 1: Find exact or very close match
matchIdx = timestamps.findIndex(ts => {
const actualTime = new Date(ts).getTime();
return Math.abs(predTime - actualTime) < tolerance;
});
// Strategy 2: If no match, find the next candle after prediction
if (matchIdx < 0) {
matchIdx = timestamps.findIndex(ts => {
const actualTime = new Date(ts).getTime();
return actualTime >= predTime && actualTime < predTime + tolerance * 2;
});
}
// Debug logging for unmatched predictions
if (matchIdx < 0) {
// Parse both timestamps to compare
const predTimeParsed = new Date(prediction.timestamp);
const latestActual = new Date(timestamps[timestamps.length - 1]);
if (idx < 3) { // Only log first 3 to avoid spam
console.log(`[${timeframe}] No match for prediction:`, {
predTimestamp: prediction.timestamp,
predTime: predTimeParsed.toISOString(),
latestActual: latestActual.toISOString(),
timeDiff: (latestActual - predTimeParsed) + 'ms',
tolerance: tolerance + 'ms',
availableTimestamps: timestamps.slice(-3) // Last 3 actual timestamps
});
}
}
if (matchIdx >= 0) {
// Found matching actual candle - calculate accuracy INCLUDING VOLUME
const predCandle = prediction.candle; // [O, H, L, C, V]
const actualCandle = [
opens[matchIdx],
highs[matchIdx],
lows[matchIdx],
closes[matchIdx],
actualData.volume ? actualData.volume[matchIdx] : predCandle[4] // Get actual volume if available
];
// Calculate absolute errors for O, H, L, C, V
const errors = {
open: Math.abs(predCandle[0] - actualCandle[0]),
high: Math.abs(predCandle[1] - actualCandle[1]),
low: Math.abs(predCandle[2] - actualCandle[2]),
close: Math.abs(predCandle[3] - actualCandle[3]),
volume: Math.abs(predCandle[4] - actualCandle[4])
};
// Calculate percentage errors for O, H, L, C, V
const pctErrors = {
open: (errors.open / actualCandle[0]) * 100,
high: (errors.high / actualCandle[1]) * 100,
low: (errors.low / actualCandle[2]) * 100,
close: (errors.close / actualCandle[3]) * 100,
volume: actualCandle[4] > 0 ? (errors.volume / actualCandle[4]) * 100 : 0
};
// Average error (OHLC only, volume separate due to different scale)
const avgError = (errors.open + errors.high + errors.low + errors.close) / 4;
const avgPctError = (pctErrors.open + pctErrors.high + pctErrors.low + pctErrors.close) / 4;
// Direction accuracy (did we predict up/down correctly?)
const predDirection = predCandle[3] >= predCandle[0] ? 'up' : 'down';
const actualDirection = actualCandle[3] >= actualCandle[0] ? 'up' : 'down';
const directionCorrect = predDirection === actualDirection;
// Price range accuracy
const priceRange = actualCandle[1] - actualCandle[2]; // High - Low
const accuracy = Math.max(0, 1 - (avgError / priceRange)) * 100;
// Store accuracy metrics
prediction.accuracy = {
errors: errors,
pctErrors: pctErrors,
avgError: avgError,
avgPctError: avgPctError,
directionCorrect: directionCorrect,
accuracy: accuracy,
actualCandle: actualCandle,
validatedAt: new Date().toISOString()
};
validatedCount++;
console.log(`[${timeframe}] Prediction validated (#${validatedCount}):`, {
timestamp: prediction.timestamp,
matchedTo: timestamps[matchIdx],
accuracy: accuracy.toFixed(1) + '%',
avgError: avgError.toFixed(4),
avgPctError: avgPctError.toFixed(2) + '%',
volumeError: pctErrors.volume.toFixed(2) + '%',
direction: directionCorrect ? '✓' : '✗',
timeDiff: Math.abs(predTime - new Date(timestamps[matchIdx]).getTime()) + 'ms',
predicted: {
O: predCandle[0].toFixed(2),
H: predCandle[1].toFixed(2),
L: predCandle[2].toFixed(2),
C: predCandle[3].toFixed(2),
V: predCandle[4].toFixed(2)
},
actual: {
O: actualCandle[0].toFixed(2),
H: actualCandle[1].toFixed(2),
L: actualCandle[2].toFixed(2),
C: actualCandle[3].toFixed(2),
V: actualCandle[4].toFixed(2)
}
});
// Send metrics to backend for training feedback
this._sendPredictionMetrics(timeframe, prediction);
}
});
// Summary log
if (validatedCount > 0) {
const totalPending = predictions.filter(p => !p.accuracy).length;
console.log(`[${timeframe}] Validated ${validatedCount} predictions, ${totalPending} still pending`);
}
}
/**
* Send prediction accuracy metrics to backend for training feedback
*/
_sendPredictionMetrics(timeframe, prediction) {
if (!prediction.accuracy) return;
const metrics = {
timeframe: timeframe,
timestamp: prediction.timestamp,
predicted: prediction.candle, // [O, H, L, C, V]
actual: prediction.accuracy.actualCandle, // [O, H, L, C, V]
errors: prediction.accuracy.errors, // {open, high, low, close, volume}
pctErrors: prediction.accuracy.pctErrors, // {open, high, low, close, volume}
directionCorrect: prediction.accuracy.directionCorrect,
accuracy: prediction.accuracy.accuracy
};
console.log('[Prediction Metrics for Training]', metrics);
// Send to backend via WebSocket for incremental training
if (window.socket && window.socket.connected) {
window.socket.emit('prediction_accuracy', metrics);
console.log(`[${timeframe}] Sent prediction accuracy to backend for training`);
} else {
console.warn('[Training] WebSocket not connected - metrics not sent to backend');
}
}
/**
@@ -2043,9 +2314,9 @@ class ChartManager {
this.ghostCandleHistory[timeframe] = this.ghostCandleHistory[timeframe].slice(-this.maxGhostCandles);
}
// 4. Add all ghost candles from history to traces
// 4. Add all ghost candles from history to traces (with accuracy if validated)
for (const ghost of this.ghostCandleHistory[timeframe]) {
this._addGhostCandlePrediction(ghost.candle, timeframe, predictionTraces, ghost.targetTime);
this._addGhostCandlePrediction(ghost.candle, timeframe, predictionTraces, ghost.targetTime, ghost.accuracy);
}
// 5. Store as "Last Prediction" for shadow rendering
@@ -2057,7 +2328,10 @@ class ChartManager {
inferenceTime: predictionTimestamp
};
console.log(`[${timeframe}] Ghost candle added (${this.ghostCandleHistory[timeframe].length}/${this.maxGhostCandles}) at ${targetTimestamp.toISOString()}`);
console.log(`[${timeframe}] Ghost candle added (${this.ghostCandleHistory[timeframe].length}/${this.maxGhostCandles}) at ${targetTimestamp.toISOString()}`, {
predicted: candleData,
timestamp: formattedTimestamp
});
}
}
@@ -2097,8 +2371,16 @@ class ChartManager {
Plotly.deleteTraces(plotId, indicesToRemove);
}
// Add new traces
// Add new traces - these will overlay on top of real candles
// Plotly renders traces in order, so predictions added last appear on top
Plotly.addTraces(plotId, predictionTraces);
// Ensure predictions are visible above real candles by setting z-order
// Update layout to ensure prediction traces are on top
Plotly.relayout(plotId, {
'xaxis.showspikes': false,
'yaxis.showspikes': false
});
}
} catch (error) {
@@ -2173,9 +2455,10 @@ class ChartManager {
});
}
_addGhostCandlePrediction(candleData, timeframe, traces, predictionTimestamp = null) {
_addGhostCandlePrediction(candleData, timeframe, traces, predictionTimestamp = null, accuracy = null) {
// candleData is [Open, High, Low, Close, Volume]
// predictionTimestamp is when the model made this prediction (optional)
// accuracy is the validation metrics (if actual candle has arrived)
// If not provided, we calculate the next candle time
const chart = this.charts[timeframe];
@@ -2215,8 +2498,46 @@ class ChartManager {
const low = candleData[2];
const close = candleData[3];
// Determine color
const color = close >= open ? '#10b981' : '#ef4444';
// Determine color based on validation status
// Ghost candles should be 30% opacity to see real candles underneath
let color, opacity;
if (accuracy) {
// Validated prediction - color by accuracy
if (accuracy.directionCorrect) {
color = close >= open ? '#10b981' : '#ef4444'; // Green/Red
} else {
color = '#fbbf24'; // Yellow for wrong direction
}
opacity = 0.3; // 30% - see real candle underneath
} else {
// Unvalidated prediction
color = close >= open ? '#10b981' : '#ef4444';
opacity = 0.3; // 30% - see real candle underneath
}
// Build rich tooltip text
let tooltipText = `PREDICTED CANDLE<br>`;
tooltipText += `O: ${open.toFixed(2)} H: ${high.toFixed(2)}<br>`;
tooltipText += `L: ${low.toFixed(2)} C: ${close.toFixed(2)}<br>`;
tooltipText += `Direction: ${close >= open ? 'UP' : 'DOWN'}<br>`;
if (accuracy) {
tooltipText += `<br>--- VALIDATION ---<br>`;
tooltipText += `Accuracy: ${accuracy.accuracy.toFixed(1)}%<br>`;
tooltipText += `Direction: ${accuracy.directionCorrect ? 'CORRECT ✓' : 'WRONG ✗'}<br>`;
tooltipText += `Avg Error: ${accuracy.avgPctError.toFixed(2)}%<br>`;
tooltipText += `<br>ACTUAL vs PREDICTED:<br>`;
tooltipText += `Open: ${accuracy.actualCandle[0].toFixed(2)} vs ${open.toFixed(2)} (${accuracy.pctErrors.open.toFixed(2)}%)<br>`;
tooltipText += `High: ${accuracy.actualCandle[1].toFixed(2)} vs ${high.toFixed(2)} (${accuracy.pctErrors.high.toFixed(2)}%)<br>`;
tooltipText += `Low: ${accuracy.actualCandle[2].toFixed(2)} vs ${low.toFixed(2)} (${accuracy.pctErrors.low.toFixed(2)}%)<br>`;
tooltipText += `Close: ${accuracy.actualCandle[3].toFixed(2)} vs ${close.toFixed(2)} (${accuracy.pctErrors.close.toFixed(2)}%)<br>`;
if (accuracy.actualCandle[4] !== undefined && accuracy.pctErrors.volume !== undefined) {
const predVolume = candleData[4];
tooltipText += `Volume: ${accuracy.actualCandle[4].toFixed(2)} vs ${predVolume.toFixed(2)} (${accuracy.pctErrors.volume.toFixed(2)}%)`;
}
} else {
tooltipText += `<br>Status: AWAITING VALIDATION...`;
}
// Create ghost candle trace with formatted timestamp string (same as real candles)
// 150% wider than normal candles
@@ -2236,14 +2557,14 @@ class ChartManager {
line: { color: color, width: 3 }, // 150% wider
fillcolor: color
},
opacity: 0.6, // 60% transparent
hoverinfo: 'x+y+text',
text: ['Predicted Next Candle'],
opacity: opacity,
hoverinfo: 'text',
text: [tooltipText],
width: 1.5 // 150% width multiplier
};
traces.push(ghostTrace);
console.log('Added ghost candle prediction at:', formattedTimestamp, ghostTrace);
console.log('Added ghost candle prediction at:', formattedTimestamp, accuracy ? 'VALIDATED' : 'pending');
}
_addShadowCandlePrediction(candleData, timestamp, traces) {