t perd viz - wip

This commit is contained in:
Dobromir Popov
2025-11-19 18:46:09 +02:00
parent df5f9b47f2
commit feb6cec275
8 changed files with 919 additions and 9 deletions

View File

@@ -107,7 +107,7 @@ class BacktestRunner:
self.lock = threading.Lock()
def start_backtest(self, backtest_id: str, model, data_provider, symbol: str, timeframe: str,
start_time: Optional[str] = None, end_time: Optional[str] = None):
orchestrator=None, start_time: Optional[str] = None, end_time: Optional[str] = None):
"""Start backtest in background thread"""
# Initialize backtest state
@@ -122,22 +122,34 @@ class BacktestRunner:
'new_predictions': [],
'position': None, # {'type': 'long/short', 'entry_price': float, 'entry_time': str}
'error': None,
'stop_requested': False
'stop_requested': False,
'orchestrator': orchestrator,
'symbol': symbol
}
# Clear previous predictions from orchestrator
if orchestrator and hasattr(orchestrator, 'recent_transformer_predictions'):
if symbol in orchestrator.recent_transformer_predictions:
orchestrator.recent_transformer_predictions[symbol].clear()
if symbol in orchestrator.recent_cnn_predictions:
orchestrator.recent_cnn_predictions[symbol].clear()
if symbol in orchestrator.recent_dqn_predictions:
orchestrator.recent_dqn_predictions[symbol].clear()
logger.info(f"Cleared previous predictions for backtest on {symbol}")
with self.lock:
self.active_backtests[backtest_id] = state
# Run backtest in background thread
thread = threading.Thread(
target=self._run_backtest,
args=(backtest_id, model, data_provider, symbol, timeframe, start_time, end_time)
args=(backtest_id, model, data_provider, symbol, timeframe, orchestrator, start_time, end_time)
)
thread.daemon = True
thread.start()
def _run_backtest(self, backtest_id: str, model, data_provider, symbol: str, timeframe: str,
start_time: Optional[str] = None, end_time: Optional[str] = None):
orchestrator=None, start_time: Optional[str] = None, end_time: Optional[str] = None):
"""Execute backtest candle-by-candle"""
try:
state = self.active_backtests[backtest_id]
@@ -203,10 +215,51 @@ class BacktestRunner:
'price': current_price,
'action': prediction['action'],
'confidence': prediction['confidence'],
'timeframe': timeframe
'timeframe': timeframe,
'current_price': current_price
}
state['new_predictions'].append(pred_data)
# Store in orchestrator for visualization
if orchestrator and hasattr(orchestrator, 'store_transformer_prediction'):
# Determine model type from model class name
model_type = model.__class__.__name__.lower()
# Store in appropriate prediction collection
if 'transformer' in model_type:
orchestrator.store_transformer_prediction(symbol, {
'timestamp': current_time,
'current_price': current_price,
'predicted_price': current_price * (1.01 if prediction['action'] == 'BUY' else 0.99),
'price_change': 1.0 if prediction['action'] == 'BUY' else -1.0,
'confidence': prediction['confidence'],
'action': prediction['action'],
'horizon_minutes': 10
})
elif 'cnn' in model_type:
if hasattr(orchestrator, 'recent_cnn_predictions'):
if symbol not in orchestrator.recent_cnn_predictions:
from collections import deque
orchestrator.recent_cnn_predictions[symbol] = deque(maxlen=50)
orchestrator.recent_cnn_predictions[symbol].append({
'timestamp': current_time,
'current_price': current_price,
'predicted_price': current_price * (1.01 if prediction['action'] == 'BUY' else 0.99),
'confidence': prediction['confidence'],
'direction': 2 if prediction['action'] == 'BUY' else 0
})
elif 'dqn' in model_type or 'rl' in model_type:
if hasattr(orchestrator, 'recent_dqn_predictions'):
if symbol not in orchestrator.recent_dqn_predictions:
from collections import deque
orchestrator.recent_dqn_predictions[symbol] = deque(maxlen=100)
orchestrator.recent_dqn_predictions[symbol].append({
'timestamp': current_time,
'current_price': current_price,
'action': prediction['action'],
'confidence': prediction['confidence']
})
# Execute trade logic
self._execute_trade_logic(state, prediction, current_price, current_time)
@@ -1678,6 +1731,7 @@ class AnnotationDashboard:
data_provider=self.data_provider,
symbol=symbol,
timeframe=timeframe,
orchestrator=self.orchestrator,
start_time=start_time,
end_time=end_time
)
@@ -2024,6 +2078,81 @@ class AnnotationDashboard:
}
})
@self.server.route('/api/live-updates', methods=['POST'])
def get_live_updates():
"""Get live chart and prediction updates (polling endpoint)"""
try:
data = request.get_json()
symbol = data.get('symbol', 'ETH/USDT')
timeframe = data.get('timeframe', '1m')
response = {
'success': True,
'chart_update': None,
'prediction': None
}
# Get latest candle for the requested timeframe
if self.orchestrator and self.orchestrator.data_provider:
try:
# Get latest candle
ohlcv_data = self.orchestrator.data_provider.get_ohlcv_data(symbol, timeframe, limit=1)
if ohlcv_data and len(ohlcv_data) > 0:
latest_candle = ohlcv_data[-1]
response['chart_update'] = {
'symbol': symbol,
'timeframe': timeframe,
'candle': {
'timestamp': latest_candle[0],
'open': float(latest_candle[1]),
'high': float(latest_candle[2]),
'low': float(latest_candle[3]),
'close': float(latest_candle[4]),
'volume': float(latest_candle[5])
}
}
except Exception as e:
logger.debug(f"Error getting latest candle: {e}")
# Get latest model predictions
if self.orchestrator:
try:
# Get latest predictions from orchestrator
predictions = {}
# DQN predictions
if hasattr(self.orchestrator, 'recent_dqn_predictions') and symbol in self.orchestrator.recent_dqn_predictions:
dqn_preds = list(self.orchestrator.recent_dqn_predictions[symbol])
if dqn_preds:
predictions['dqn'] = dqn_preds[-1]
# CNN predictions
if hasattr(self.orchestrator, 'recent_cnn_predictions') and symbol in self.orchestrator.recent_cnn_predictions:
cnn_preds = list(self.orchestrator.recent_cnn_predictions[symbol])
if cnn_preds:
predictions['cnn'] = cnn_preds[-1]
# Transformer predictions
if hasattr(self.orchestrator, 'recent_transformer_predictions') and symbol in self.orchestrator.recent_transformer_predictions:
transformer_preds = list(self.orchestrator.recent_transformer_predictions[symbol])
if transformer_preds:
predictions['transformer'] = transformer_preds[-1]
if predictions:
response['prediction'] = predictions
except Exception as e:
logger.debug(f"Error getting predictions: {e}")
return jsonify(response)
except Exception as e:
logger.error(f"Error in live updates: {e}")
return jsonify({
'success': False,
'error': str(e)
})
@self.server.route('/api/realtime-inference/signals', methods=['GET'])
def get_realtime_signals():
"""Get latest real-time inference signals"""

View File

@@ -111,6 +111,80 @@ class ChartManager {
}
}
/**
* Update latest candle on chart (for live updates)
* Efficiently updates only the last candle or adds a new one
*/
updateLatestCandle(symbol, timeframe, candle) {
try {
const plotId = `plot-${timeframe}`;
const plotElement = document.getElementById(plotId);
if (!plotElement) {
console.debug(`Chart ${plotId} not found for live update`);
return;
}
// Get current chart data
const chartData = Plotly.Plots.data(plotId);
if (!chartData || chartData.length < 2) {
console.debug(`Chart ${plotId} not initialized yet`);
return;
}
const candlestickTrace = chartData[0];
const volumeTrace = chartData[1];
// Parse timestamp
const candleTimestamp = new Date(candle.timestamp);
// Check if this is updating the last candle or adding a new one
const lastTimestamp = candlestickTrace.x[candlestickTrace.x.length - 1];
const isNewCandle = !lastTimestamp || new Date(lastTimestamp).getTime() < candleTimestamp.getTime();
if (isNewCandle) {
// Add new candle using extendTraces (most efficient)
Plotly.extendTraces(plotId, {
x: [[candleTimestamp]],
open: [[candle.open]],
high: [[candle.high]],
low: [[candle.low]],
close: [[candle.close]]
}, [0]);
// Update volume color based on price direction
const volumeColor = candle.close >= candle.open ? '#10b981' : '#ef4444';
Plotly.extendTraces(plotId, {
x: [[candleTimestamp]],
y: [[candle.volume]],
marker: { color: [[volumeColor]] }
}, [1]);
} else {
// Update last candle using restyle
const lastIndex = candlestickTrace.x.length - 1;
Plotly.restyle(plotId, {
'x': [[...candlestickTrace.x.slice(0, lastIndex), candleTimestamp]],
'open': [[...candlestickTrace.open.slice(0, lastIndex), candle.open]],
'high': [[...candlestickTrace.high.slice(0, lastIndex), candle.high]],
'low': [[...candlestickTrace.low.slice(0, lastIndex), candle.low]],
'close': [[...candlestickTrace.close.slice(0, lastIndex), candle.close]]
}, [0]);
// Update volume
const volumeColor = candle.close >= candle.open ? '#10b981' : '#ef4444';
Plotly.restyle(plotId, {
'x': [[...volumeTrace.x.slice(0, lastIndex), candleTimestamp]],
'y': [[...volumeTrace.y.slice(0, lastIndex), candle.volume]],
'marker.color': [[...volumeTrace.marker.color.slice(0, lastIndex), volumeColor]]
}, [1]);
}
console.debug(`Updated ${timeframe} chart with new candle at ${candleTimestamp.toISOString()}`);
} catch (error) {
console.error(`Error updating latest candle for ${timeframe}:`, error);
}
}
/**
* Initialize charts for all timeframes with pivot bounds
*/
@@ -1640,4 +1714,172 @@ class ChartManager {
loadingDiv.remove();
}
}
/**
* Update model predictions on charts
*/
updatePredictions(predictions) {
if (!predictions) return;
try {
// Update predictions on 1m chart (primary timeframe for predictions)
const timeframe = '1m';
const chart = this.charts[timeframe];
if (!chart) return;
const plotId = chart.plotId;
const plotElement = document.getElementById(plotId);
if (!plotElement) return;
// Get current chart data
const chartData = plotElement.data;
if (!chartData || chartData.length < 2) return;
// Prepare prediction markers
const predictionShapes = [];
const predictionAnnotations = [];
// Add DQN predictions (arrows)
if (predictions.dqn) {
this._addDQNPrediction(predictions.dqn, predictionShapes, predictionAnnotations);
}
// Add CNN predictions (trend lines)
if (predictions.cnn) {
this._addCNNPrediction(predictions.cnn, predictionShapes, predictionAnnotations);
}
// Add Transformer predictions (star markers with trend lines)
if (predictions.transformer) {
this._addTransformerPrediction(predictions.transformer, predictionShapes, predictionAnnotations);
}
// Update chart layout with predictions
if (predictionShapes.length > 0 || predictionAnnotations.length > 0) {
Plotly.relayout(plotId, {
shapes: [...(chart.layout.shapes || []), ...predictionShapes],
annotations: [...(chart.layout.annotations || []), ...predictionAnnotations]
});
}
} catch (error) {
console.debug('Error updating predictions:', error);
}
}
_addDQNPrediction(prediction, shapes, annotations) {
const timestamp = new Date(prediction.timestamp || Date.now());
const price = prediction.current_price || 0;
const action = prediction.action || 'HOLD';
const confidence = prediction.confidence || 0;
if (action === 'HOLD' || confidence < 0.4) return;
// Add arrow annotation
annotations.push({
x: timestamp,
y: price,
text: action === 'BUY' ? '▲' : '▼',
showarrow: false,
font: {
size: 16,
color: action === 'BUY' ? '#10b981' : '#ef4444'
},
opacity: 0.5 + confidence * 0.5
});
}
_addCNNPrediction(prediction, shapes, annotations) {
const timestamp = new Date(prediction.timestamp || Date.now());
const currentPrice = prediction.current_price || 0;
const predictedPrice = prediction.predicted_price || currentPrice;
const confidence = prediction.confidence || 0;
if (confidence < 0.4 || currentPrice === 0) return;
// Calculate end time (5 minutes ahead)
const endTime = new Date(timestamp.getTime() + 5 * 60 * 1000);
// Determine color based on direction
const isUp = predictedPrice > currentPrice;
const color = isUp ? 'rgba(0, 255, 0, 0.5)' : 'rgba(255, 0, 0, 0.5)';
// Add trend line
shapes.push({
type: 'line',
x0: timestamp,
y0: currentPrice,
x1: endTime,
y1: predictedPrice,
line: {
color: color,
width: 2,
dash: 'dot'
}
});
// Add target marker
annotations.push({
x: endTime,
y: predictedPrice,
text: '◆',
showarrow: false,
font: {
size: 12,
color: isUp ? '#10b981' : '#ef4444'
},
opacity: 0.5 + confidence * 0.5
});
}
_addTransformerPrediction(prediction, shapes, annotations) {
const timestamp = new Date(prediction.timestamp || Date.now());
const currentPrice = prediction.current_price || 0;
const predictedPrice = prediction.predicted_price || currentPrice;
const confidence = prediction.confidence || 0;
const priceChange = prediction.price_change || 0;
const horizonMinutes = prediction.horizon_minutes || 10;
if (confidence < 0.3 || currentPrice === 0) return;
// Calculate end time
const endTime = new Date(timestamp.getTime() + horizonMinutes * 60 * 1000);
// Determine color based on price change
let color;
if (priceChange > 0.5) {
color = 'rgba(0, 200, 255, 0.6)'; // Cyan for UP
} else if (priceChange < -0.5) {
color = 'rgba(255, 100, 0, 0.6)'; // Orange for DOWN
} else {
color = 'rgba(150, 150, 255, 0.5)'; // Light blue for STABLE
}
// Add trend line
shapes.push({
type: 'line',
x0: timestamp,
y0: currentPrice,
x1: endTime,
y1: predictedPrice,
line: {
color: color,
width: 2 + confidence * 2,
dash: 'dashdot'
}
});
// Add star marker at target
annotations.push({
x: endTime,
y: predictedPrice,
text: '★',
showarrow: false,
font: {
size: 14 + confidence * 6,
color: color
},
opacity: 0.6 + confidence * 0.4
});
}
}

View File

@@ -0,0 +1,222 @@
/**
* Polling-based Live Updates for ANNOTATE
* Replaces WebSocket with simple polling (like clean_dashboard)
*/
class LiveUpdatesPolling {
constructor() {
this.pollInterval = null;
this.pollDelay = 2000; // Poll every 2 seconds (like clean_dashboard)
this.subscriptions = new Set();
this.isPolling = false;
// Callbacks
this.onChartUpdate = null;
this.onPredictionUpdate = null;
this.onConnectionChange = null;
console.log('LiveUpdatesPolling initialized');
}
start() {
if (this.isPolling) {
console.log('Already polling');
return;
}
this.isPolling = true;
this._startPolling();
if (this.onConnectionChange) {
this.onConnectionChange(true);
}
console.log('Started polling for live updates');
}
stop() {
if (this.pollInterval) {
clearInterval(this.pollInterval);
this.pollInterval = null;
}
this.isPolling = false;
if (this.onConnectionChange) {
this.onConnectionChange(false);
}
console.log('Stopped polling for live updates');
}
_startPolling() {
// Poll immediately, then set interval
this._poll();
this.pollInterval = setInterval(() => {
this._poll();
}, this.pollDelay);
}
_poll() {
// Poll each subscription
this.subscriptions.forEach(sub => {
fetch('/api/live-updates', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
symbol: sub.symbol,
timeframe: sub.timeframe
})
})
.then(response => response.json())
.then(data => {
if (data.success) {
// Handle chart update
if (data.chart_update && this.onChartUpdate) {
this.onChartUpdate(data.chart_update);
}
// Handle prediction update
if (data.prediction && this.onPredictionUpdate) {
this.onPredictionUpdate(data.prediction);
}
}
})
.catch(error => {
console.debug('Polling error:', error);
});
});
}
subscribe(symbol, timeframe) {
const key = `${symbol}_${timeframe}`;
this.subscriptions.add({ symbol, timeframe, key });
// Auto-start polling if not already started
if (!this.isPolling) {
this.start();
}
console.log(`Subscribed to live updates: ${symbol} ${timeframe}`);
}
unsubscribe(symbol, timeframe) {
const key = `${symbol}_${timeframe}`;
this.subscriptions.forEach(sub => {
if (sub.key === key) {
this.subscriptions.delete(sub);
}
});
// Stop polling if no subscriptions
if (this.subscriptions.size === 0) {
this.stop();
}
console.log(`Unsubscribed from live updates: ${symbol} ${timeframe}`);
}
isConnected() {
return this.isPolling;
}
}
// Global instance
window.liveUpdatesPolling = null;
// Initialize on page load
document.addEventListener('DOMContentLoaded', function() {
// Initialize polling
window.liveUpdatesPolling = new LiveUpdatesPolling();
// Setup callbacks (same interface as WebSocket version)
window.liveUpdatesPolling.onConnectionChange = function(connected) {
const statusElement = document.getElementById('ws-connection-status');
if (statusElement) {
if (connected) {
statusElement.innerHTML = '<span class="badge bg-success">Live</span>';
} else {
statusElement.innerHTML = '<span class="badge bg-secondary">Offline</span>';
}
}
};
window.liveUpdatesPolling.onChartUpdate = function(data) {
// Update chart with new candle
if (window.appState && window.appState.chartManager) {
window.appState.chartManager.updateLatestCandle(data.symbol, data.timeframe, data.candle);
}
};
window.liveUpdatesPolling.onPredictionUpdate = function(data) {
// Update prediction visualization on charts
if (window.appState && window.appState.chartManager) {
window.appState.chartManager.updatePredictions(data);
}
// Update prediction display
if (typeof updatePredictionDisplay === 'function') {
updatePredictionDisplay(data);
}
// Add to prediction history
if (typeof predictionHistory !== 'undefined') {
predictionHistory.unshift(data);
if (predictionHistory.length > 5) {
predictionHistory = predictionHistory.slice(0, 5);
}
if (typeof updatePredictionHistory === 'function') {
updatePredictionHistory();
}
}
};
// Function to subscribe to all active timeframes
function subscribeToActiveTimeframes() {
if (window.appState && window.appState.currentSymbol && window.appState.currentTimeframes) {
const symbol = window.appState.currentSymbol;
window.appState.currentTimeframes.forEach(timeframe => {
window.liveUpdatesPolling.subscribe(symbol, timeframe);
});
console.log(`Subscribed to live updates for ${symbol}: ${window.appState.currentTimeframes.join(', ')}`);
}
}
// Auto-start polling
console.log('Auto-starting polling for live updates...');
window.liveUpdatesPolling.start();
// Wait for DOM and appState to be ready, then subscribe
function initializeSubscriptions() {
// Wait a bit for charts to initialize
setTimeout(() => {
subscribeToActiveTimeframes();
}, 2000);
}
// Subscribe when DOM is ready
if (document.readyState === 'loading') {
document.addEventListener('DOMContentLoaded', initializeSubscriptions);
} else {
initializeSubscriptions();
}
// Also monitor for appState changes (fallback)
let lastTimeframes = null;
setInterval(() => {
if (window.appState && window.appState.currentTimeframes && window.appState.chartManager) {
const currentTimeframes = window.appState.currentTimeframes.join(',');
if (currentTimeframes !== lastTimeframes) {
lastTimeframes = currentTimeframes;
subscribeToActiveTimeframes();
}
}
}, 3000);
});
// Cleanup on page unload
window.addEventListener('beforeunload', function() {
if (window.liveUpdatesPolling) {
window.liveUpdatesPolling.stop();
}
});

View File

@@ -85,7 +85,7 @@
<script src="{{ url_for('static', filename='js/annotation_manager.js') }}?v={{ range(1, 10000) | random }}"></script>
<script src="{{ url_for('static', filename='js/time_navigator.js') }}?v={{ range(1, 10000) | random }}"></script>
<script src="{{ url_for('static', filename='js/training_controller.js') }}?v={{ range(1, 10000) | random }}"></script>
<script src="{{ url_for('static', filename='js/live_updates_ws.js') }}?v={{ range(1, 10000) | random }}"></script>
<script src="{{ url_for('static', filename='js/live_updates_polling.js') }}?v={{ range(1, 10000) | random }}"></script>
{% block extra_js %}{% endblock %}

View File

@@ -42,6 +42,7 @@
<div class="small">
<div>Epoch: <span id="training-epoch">0</span>/<span id="training-total-epochs">0</span></div>
<div>Loss: <span id="training-loss">--</span></div>
<div>GPU: <span id="training-gpu-util">--</span>% | CPU: <span id="training-cpu-util">--</span>%</div>
</div>
</div>
</div>
@@ -446,6 +447,14 @@
document.getElementById('training-epoch').textContent = progress.current_epoch;
document.getElementById('training-total-epochs').textContent = progress.total_epochs;
document.getElementById('training-loss').textContent = progress.current_loss.toFixed(4);
// Update GPU/CPU utilization
const gpuUtil = progress.gpu_utilization !== null && progress.gpu_utilization !== undefined
? progress.gpu_utilization.toFixed(1) : '--';
const cpuUtil = progress.cpu_utilization !== null && progress.cpu_utilization !== undefined
? progress.cpu_utilization.toFixed(1) : '--';
document.getElementById('training-gpu-util').textContent = gpuUtil;
document.getElementById('training-cpu-util').textContent = cpuUtil;
// Check if complete
if (progress.status === 'completed') {