diff --git a/web/dashboard.py b/web/dashboard.py index 253c959..7d2b61b 100644 --- a/web/dashboard.py +++ b/web/dashboard.py @@ -458,6 +458,908 @@ class TradingDashboard: self.pivot_rl_trainer = None logger.warning(f"Enhanced Pivot RL Trainer not available: {e}") + def _setup_layout(self): + """Setup the dashboard layout""" + self.app.layout = html.Div([ + # Compact Header + html.Div([ + html.H3([ + html.I(className="fas fa-chart-line me-2"), + "Live Trading Dashboard" + ], className="text-white mb-1"), + html.P(f"Ultra-Fast Updates • Portfolio: ${self.starting_balance:,.0f} • {'MEXC Live' if (self.trading_executor and self.trading_executor.trading_enabled and not self.trading_executor.simulation_mode) else 'Demo Mode'}", + className="text-light mb-0 opacity-75 small") + ], className="bg-dark p-2 mb-2"), + + # Auto-refresh component - ultra-fast updates for real-time trading + dcc.Interval( + id='interval-component', + interval=1000, # Update every 1 second for maximum responsiveness + n_intervals=0 + ), + + # Main content - Compact layout + html.Div([ + # Top row - Key metrics and Recent Signals (split layout) + html.Div([ + # Left side - Key metrics (compact cards) + html.Div([ + html.Div([ + html.Div([ + html.H5(id="current-price", className="text-success mb-0 small"), + html.P("Live Price", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="session-pnl", className="mb-0 small"), + html.P("Session P&L", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="total-fees", className="text-warning mb-0 small"), + html.P("Total Fees", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="current-position", className="text-info mb-0 small"), + html.P("Position", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="trade-count", className="text-warning mb-0 small"), + html.P("Trades", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="portfolio-value", className="text-secondary mb-0 small"), + html.P("Portfolio", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="mexc-status", className="text-info mb-0 small"), + html.P("MEXC API", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + ], style={"display": "grid", "gridTemplateColumns": "repeat(4, 1fr)", "gap": "8px", "width": "60%"}), + + + # Right side - Merged: Recent Signals & Model Training - 2 columns + html.Div([ + # Recent Trading Signals Column (50%) + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-robot me-2"), + "Recent Trading Signals" + ], className="card-title mb-2"), + html.Div(id="recent-decisions", style={"height": "160px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "48%"}), + + # Model Training + COB Buckets Column (50%) + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-brain me-2"), + "Training Progress & COB $1 Buckets" + ], className="card-title mb-2"), + html.Div(id="training-metrics", style={"height": "160px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "48%", "marginLeft": "4%"}), + ], style={"width": "48%", "marginLeft": "2%", "display": "flex"}) + ], className="d-flex mb-3"), + + # Charts row - Now full width since training moved up + html.Div([ + # Price chart - Full width with manual trading buttons + html.Div([ + html.Div([ + # Chart header with manual trading buttons + html.Div([ + html.H6([ + html.I(className="fas fa-chart-candlestick me-2"), + "Live 1s Price & Volume Chart (WebSocket Stream)" + ], className="card-title mb-0"), + html.Div([ + html.Button([ + html.I(className="fas fa-arrow-up me-1"), + "BUY" + ], id="manual-buy-btn", className="btn btn-success btn-sm me-2", + style={"fontSize": "10px", "padding": "2px 8px"}), + html.Button([ + html.I(className="fas fa-arrow-down me-1"), + "SELL" + ], id="manual-sell-btn", className="btn btn-danger btn-sm", + style={"fontSize": "10px", "padding": "2px 8px"}) + ], className="d-flex") + ], className="d-flex justify-content-between align-items-center mb-2"), + html.Div([ + dcc.Graph(id="price-chart", style={"height": "400px"}), + # JavaScript for client-side chart data management + html.Script(""" + // Initialize chart data cache and real-time management + window.chartDataCache = window.chartDataCache || {}; + window.chartUpdateInterval = window.chartUpdateInterval || null; + + // Chart data merging function + function mergeChartData(symbol, newData) { + if (!window.chartDataCache[symbol]) { + window.chartDataCache[symbol] = { + ohlc: [], + volume: [], + timestamps: [], + trades: [], + lastUpdate: Date.now(), + maxPoints: 2000 + }; + } + + const cache = window.chartDataCache[symbol]; + + // Merge new OHLC data + if (newData.ohlc && newData.ohlc.length > 0) { + const newTimestamps = newData.timestamps.map(ts => new Date(ts).getTime()); + const existingTimestampMap = new Map(); + + cache.timestamps.forEach((ts, idx) => { + existingTimestampMap.set(new Date(ts).getTime(), idx); + }); + + // Process each new data point + newData.ohlc.forEach((ohlc, i) => { + const newTime = newTimestamps[i]; + const existingIndex = existingTimestampMap.get(newTime); + + if (existingIndex !== undefined) { + // Update existing point + cache.ohlc[existingIndex] = ohlc; + cache.volume[existingIndex] = newData.volume[i]; + } else { + // Add new point + cache.ohlc.push(ohlc); + cache.volume.push(newData.volume[i]); + cache.timestamps.push(newData.timestamps[i]); + } + }); + + // Sort by timestamp to maintain chronological order + const combined = cache.ohlc.map((ohlc, i) => ({ + ohlc: ohlc, + volume: cache.volume[i], + timestamp: cache.timestamps[i], + sortTime: new Date(cache.timestamps[i]).getTime() + })); + + combined.sort((a, b) => a.sortTime - b.sortTime); + + // Keep only the most recent points for performance + if (combined.length > cache.maxPoints) { + combined.splice(0, combined.length - cache.maxPoints); + } + + // Update cache arrays + cache.ohlc = combined.map(item => item.ohlc); + cache.volume = combined.map(item => item.volume); + cache.timestamps = combined.map(item => item.timestamp); + } + + // Merge trade data + if (newData.trade_decisions) { + cache.trades = [...(cache.trades || []), ...newData.trade_decisions]; + // Keep only recent trades + if (cache.trades.length > 100) { + cache.trades = cache.trades.slice(-100); + } + } + + cache.lastUpdate = Date.now(); + console.log(`[CHART CACHE] ${symbol}: ${cache.ohlc.length} points, ${cache.trades.length} trades`); + } + + // Real-time chart update function + function updateChartRealtime(symbol) { + const cache = window.chartDataCache[symbol]; + if (!cache || cache.ohlc.length === 0) return; + + try { + const chartDiv = document.getElementById('price-chart'); + if (chartDiv && chartDiv.data && chartDiv.data.length > 0) { + + // Find the main price trace + let priceTraceIndex = -1; + let volumeTraceIndex = -1; + + for (let i = 0; i < chartDiv.data.length; i++) { + const trace = chartDiv.data[i]; + if (trace.type === 'scatter' && trace.name && trace.name.includes('Price')) { + priceTraceIndex = i; + } else if (trace.name && trace.name.includes('Volume')) { + volumeTraceIndex = i; + } + } + + // Update price data + if (priceTraceIndex !== -1 && cache.ohlc.length > 0) { + const newX = cache.timestamps; + const newY = cache.ohlc.map(ohlc => ohlc.close); + + Plotly.restyle(chartDiv, { + 'x': [newX], + 'y': [newY] + }, [priceTraceIndex]); + } + + // Update volume data + if (volumeTraceIndex !== -1 && cache.volume.length > 0) { + Plotly.restyle(chartDiv, { + 'x': [cache.timestamps], + 'y': [cache.volume] + }, [volumeTraceIndex]); + } + + // Update chart title with latest info + if (cache.ohlc.length > 0) { + const latestPrice = cache.ohlc[cache.ohlc.length - 1].close; + const currentTime = new Date().toLocaleTimeString(); + const newTitle = `${symbol} LIVE CHART | $${latestPrice.toFixed(2)} | ${currentTime} | ${cache.ohlc.length} points`; + + Plotly.relayout(chartDiv, { + 'title.text': newTitle + }); + } + } + } catch (error) { + console.warn('[CHART UPDATE] Error:', error); + } + } + + // Set up real-time updates (1-second interval) + function startChartUpdates(symbol) { + if (window.chartUpdateInterval) { + clearInterval(window.chartUpdateInterval); + } + + window.chartUpdateInterval = setInterval(() => { + if (window.chartDataCache[symbol]) { + updateChartRealtime(symbol); + } + }, 1000); // Update every second + + console.log(`[CHART INIT] Real-time updates started for ${symbol}`); + } + + // Start chart management when page loads + document.addEventListener('DOMContentLoaded', function() { + setTimeout(() => startChartUpdates('ETH/USDT'), 1000); + }); + + // Global function to receive data from Python + window.updateChartData = function(symbol, data) { + mergeChartData(symbol, data); + updateChartRealtime(symbol); + }; + """) + ]) + ], className="card-body p-2") + ], className="card", style={"width": "100%"}), + ], className="row g-2 mb-3"), + + # CNN Model Monitoring & COB Integration - MERGED into 1 row with 4 columns + html.Div([ + # CNN Status Column + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-brain me-2"), + "CNN Model Analysis" + ], className="card-title mb-2"), + html.Div(id="cnn-monitoring-content", style={"height": "280px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "23%"}), + + # COB Status Column + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-layer-group me-2"), + "COB → Training Pipeline" + ], className="card-title mb-2"), + html.Div(id="cob-status-content", style={"height": "280px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "23%", "marginLeft": "2%"}), + + # ETH/USDT COB Details Column + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-ethereum me-2", style={"color": "#627EEA"}), + "ETH/USDT - COB" + ], className="card-title mb-2"), + html.Div(id="eth-cob-content", style={"height": "280px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "23%", "marginLeft": "2%"}), + + # BTC/USDT COB Details Column + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-bitcoin me-2", style={"color": "#F7931A"}), + "BTC/USDT - COB" + ], className="card-title mb-2"), + html.Div(id="btc-cob-content", style={"height": "280px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "23%", "marginLeft": "2%"}), + ], className="d-flex mb-3"), + + # Bottom row - Session performance and system status + html.Div([ + + # Session performance - 1/3 width + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-chart-pie me-2"), + "Session Performance" + ], className="card-title mb-2"), + html.Button( + "Clear Session", + id="clear-history-btn", + className="btn btn-sm btn-outline-danger mb-2", + n_clicks=0 + ), + html.Div(id="session-performance") + ], className="card-body p-2") + ], className="card", style={"width": "32%"}), + + # Closed Trades History - 1/3 width + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-history me-2"), + "Closed Trades History" + ], className="card-title mb-2"), + html.Div([ + html.Div( + id="closed-trades-table", + style={"height": "300px", "overflowY": "auto"} + ) + ]) + ], className="card-body p-2") + ], className="card", style={"width": "32%", "marginLeft": "2%"}), + + # System status and leverage controls - 1/3 width with icon tooltip + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-server me-2"), + "System & Leverage" + ], className="card-title mb-2"), + + # System status + html.Div([ + html.I( + id="system-status-icon", + className="fas fa-circle text-success fa-2x", + title="System Status: All systems operational", + style={"cursor": "pointer"} + ), + html.Div(id="system-status-details", className="small mt-2") + ], className="text-center mb-3"), + + # Leverage Controls + html.Div([ + html.Label([ + html.I(className="fas fa-chart-line me-1"), + "Leverage Multiplier" + ], className="form-label small fw-bold"), + html.Div([ + dcc.Slider( + id='leverage-slider', + min=self.min_leverage, + max=self.max_leverage, + step=self.leverage_step, + value=self.leverage_multiplier, + marks={ + 1: '1x', + 10: '10x', + 25: '25x', + 50: '50x', + 75: '75x', + 100: '100x' + }, + tooltip={ + "placement": "bottom", + "always_visible": True + } + ) + ], className="mb-2"), + html.Div([ + html.Span(id="current-leverage", className="badge bg-warning text-dark"), + html.Span(" • ", className="mx-1"), + html.Span(id="leverage-risk", className="badge bg-info") + ], className="text-center"), + html.Div([ + html.Small("Higher leverage = Higher rewards & risks", className="text-muted") + ], className="text-center mt-1") + ]) + ], className="card-body p-2") + ], className="card", style={"width": "32%", "marginLeft": "2%"}) + ], className="d-flex") + ], className="container-fluid") + ]) + +""" +Trading Dashboard - Clean Web Interface + +This module provides a modern, responsive web dashboard for the trading system: +- Real-time price charts with multiple timeframes +- Model performance monitoring +- Trading decisions visualization +- System health monitoring +- Memory usage tracking +""" + +import asyncio +import dash +from dash import Dash, dcc, html, Input, Output +import plotly.graph_objects as go +from plotly.subplots import make_subplots +import plotly.express as px +import pandas as pd +import numpy as np +from datetime import datetime, timedelta, timezone +import pytz +import logging +import json +import time +import threading +from threading import Thread, Lock +from collections import deque +import warnings +from typing import Dict, List, Optional, Any, Union, Tuple +import websocket +import os +import torch + +# Setup logger immediately after logging import +logger = logging.getLogger(__name__) + +# WebSocket availability check +try: + import websocket + WEBSOCKET_AVAILABLE = True + logger.info("WebSocket client available") +except ImportError: + WEBSOCKET_AVAILABLE = False + logger.warning("websocket-client not available. Real-time data will use API fallback.") + +# Import trading system components +from core.config import get_config +from core.data_provider import DataProvider +from core.orchestrator import TradingOrchestrator, TradingDecision +from core.trading_executor import TradingExecutor +from core.trading_action import TradingAction +from models import get_model_registry + +# Import CNN monitoring +try: + from core.cnn_monitor import get_cnn_dashboard_data + CNN_MONITORING_AVAILABLE = True + logger.info("CNN monitoring system available") +except ImportError: + CNN_MONITORING_AVAILABLE = False + logger.warning("CNN monitoring not available") + def get_cnn_dashboard_data(): + return {'statistics': {'total_predictions_logged': 0}} + + +# Import enhanced RL components if available +try: + from core.enhanced_orchestrator import EnhancedTradingOrchestrator + from core.universal_data_adapter import UniversalDataAdapter + from core.unified_data_stream import UnifiedDataStream, TrainingDataPacket, UIDataPacket + ENHANCED_RL_AVAILABLE = True + logger.info("Enhanced RL training components available") +except ImportError as e: + logger.warning(f"Enhanced RL components not available: {e}") + ENHANCED_RL_AVAILABLE = False + # Force enable for learning - bypass import issues + ENHANCED_RL_AVAILABLE = True + logger.info("Enhanced RL FORCED ENABLED - bypassing import issues for learning") + + # Fallback classes + class UnifiedDataStream: + def __init__(self, *args, **kwargs): pass + def register_consumer(self, *args, **kwargs): return "fallback_consumer" + def start_streaming(self): pass + def stop_streaming(self): pass + def get_latest_training_data(self): return None + def get_latest_ui_data(self): return None + + class TrainingDataPacket: + def __init__(self, *args, **kwargs): pass + + class UIDataPacket: + def __init__(self, *args, **kwargs): pass + +# Import COB integration components if available +try: + from core.cob_integration import COBIntegration + from core.multi_exchange_cob_provider import MultiExchangeCOBProvider, COBSnapshot + COB_INTEGRATION_AVAILABLE = True + logger.info("COB integration components available") +except ImportError as e: + logger.warning(f"COB integration components not available: {e}") + COB_INTEGRATION_AVAILABLE = False + # Create fallback classes + class COBSnapshot: + def __init__(self, *args, **kwargs): + self.symbol = "N/A" + self.consolidated_bids = [] + self.consolidated_asks = [] + self.volume_weighted_mid = 0.0 + self.spread_bps = 0.0 + self.total_bid_liquidity = 0.0 + self.total_ask_liquidity = 0.0 + + +class AdaptiveThresholdLearner: + """Learn optimal confidence thresholds based on real trade outcomes""" + + def __init__(self, initial_threshold: float = 0.30): + self.base_threshold = initial_threshold + self.current_threshold = initial_threshold + self.trade_outcomes = deque(maxlen=100) + self.threshold_history = deque(maxlen=50) + self.learning_rate = 0.02 + self.min_threshold = 0.20 + self.max_threshold = 0.70 + + logger.info(f"[ADAPTIVE] Initialized with starting threshold: {initial_threshold:.2%}") + + def record_trade_outcome(self, confidence: float, pnl: float, threshold_used: float): + """Record a trade outcome to learn from""" + try: + outcome = { + 'confidence': confidence, + 'pnl': pnl, + 'profitable': pnl > 0, + 'threshold_used': threshold_used, + 'timestamp': datetime.now() + } + + self.trade_outcomes.append(outcome) + + # Learn from outcomes + if len(self.trade_outcomes) >= 10: + self._update_threshold() + + except Exception as e: + logger.error(f"Error recording trade outcome: {e}") + + def _update_threshold(self): + """Update threshold based on recent trade statistics""" + try: + recent_trades = list(self.trade_outcomes)[-20:] + if len(recent_trades) < 10: + return + + profitable_count = sum(1 for t in recent_trades if t['profitable']) + win_rate = profitable_count / len(recent_trades) + avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades) + + # Adaptive adjustment logic + if win_rate > 0.60 and avg_pnl > 0.20: + adjustment = -self.learning_rate * 1.5 # Lower threshold for more trades + elif win_rate < 0.40 or avg_pnl < -0.30: + adjustment = self.learning_rate * 2.0 # Raise threshold to be more selective + else: + adjustment = 0 # No change + + old_threshold = self.current_threshold + self.current_threshold = max(self.min_threshold, + min(self.max_threshold, + self.current_threshold + adjustment)) + + if abs(self.current_threshold - old_threshold) > 0.005: + logger.info(f"[ADAPTIVE] Threshold: {old_threshold:.2%} -> {self.current_threshold:.2%} (WR: {win_rate:.1%}, PnL: ${avg_pnl:.2f})") + + except Exception as e: + logger.error(f"Error updating adaptive threshold: {e}") + + def get_current_threshold(self) -> float: + return self.current_threshold + + def get_learning_stats(self) -> Dict[str, Any]: + """Get learning statistics""" + try: + if not self.trade_outcomes: + return {'status': 'No trades recorded yet'} + + recent_trades = list(self.trade_outcomes)[-20:] + profitable_count = sum(1 for t in recent_trades if t['profitable']) + win_rate = profitable_count / len(recent_trades) if recent_trades else 0 + avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades) if recent_trades else 0 + + return { + 'current_threshold': self.current_threshold, + 'base_threshold': self.base_threshold, + 'total_trades': len(self.trade_outcomes), + 'recent_win_rate': win_rate, + 'recent_avg_pnl': avg_pnl, + 'threshold_changes': len(self.threshold_history), + 'learning_active': len(self.trade_outcomes) >= 10 + } + except Exception as e: + return {'error': str(e)} + +class TradingDashboard: + """Enhanced Trading Dashboard with Williams pivot points and unified timezone handling""" + + def __init__(self, data_provider: DataProvider = None, orchestrator: TradingOrchestrator = None, trading_executor: TradingExecutor = None): + self.app = Dash(__name__) + + # Initialize config first + from core.config import get_config + self.config = get_config() + + self.data_provider = data_provider or DataProvider() + self.orchestrator = orchestrator + self.trading_executor = trading_executor + + # Enhanced trading state with leverage support + self.leverage_enabled = True + self.leverage_multiplier = 50.0 # 50x leverage (adjustable via slider) + self.base_capital = 10000.0 + self.current_position = 0.0 # -1 to 1 (short to long) + self.position_size = 0.0 + self.entry_price = 0.0 + self.unrealized_pnl = 0.0 + self.realized_pnl = 0.0 + + # Leverage settings for slider + self.min_leverage = 1.0 + self.max_leverage = 100.0 + self.leverage_step = 1.0 + + # Connect to trading server for leverage functionality + self.trading_server_url = "http://127.0.0.1:8052" + self.training_server_url = "http://127.0.0.1:8053" + self.stream_server_url = "http://127.0.0.1:8054" + + # Enhanced performance tracking + self.leverage_metrics = { + 'leverage_efficiency': 0.0, + 'margin_used': 0.0, + 'margin_available': 10000.0, + 'effective_exposure': 0.0, + 'risk_reward_ratio': 0.0 + } + + # Enhanced models will be loaded through model registry later + + # Rest of initialization... + + # Initialize timezone from config + timezone_name = self.config.get('system', {}).get('timezone', 'Europe/Sofia') + self.timezone = pytz.timezone(timezone_name) + logger.info(f"Dashboard timezone set to: {timezone_name}") + + self.data_provider = data_provider or DataProvider() + + # Use enhanced orchestrator for comprehensive RL training + if orchestrator is None: + from core.enhanced_orchestrator import EnhancedTradingOrchestrator + self.orchestrator = EnhancedTradingOrchestrator( + data_provider=self.data_provider, + symbols=['ETH/USDT', 'BTC/USDT'], + enhanced_rl_training=True + ) + logger.info("Using Enhanced Trading Orchestrator for comprehensive RL training") + else: + self.orchestrator = orchestrator + logger.info(f"Using provided orchestrator: {type(orchestrator).__name__}") + self.enhanced_rl_enabled = True # Force enable Enhanced RL + logger.info("Enhanced RL training FORCED ENABLED for learning") + + self.trading_executor = trading_executor or TradingExecutor() + self.model_registry = get_model_registry() + + # Initialize unified data stream for comprehensive training data + if ENHANCED_RL_AVAILABLE: + self.unified_stream = UnifiedDataStream(self.data_provider, self.orchestrator) + self.stream_consumer_id = self.unified_stream.register_consumer( + consumer_name="TradingDashboard", + callback=self._handle_unified_stream_data, + data_types=['ticks', 'ohlcv', 'training_data', 'ui_data'] + ) + logger.info(f"Unified data stream initialized with consumer ID: {self.stream_consumer_id}") + else: + self.unified_stream = UnifiedDataStream() # Fallback + self.stream_consumer_id = "fallback" + logger.warning("Using fallback unified data stream") + + # Dashboard state + self.recent_decisions = [] + self.recent_signals = [] # Track all signals (not just executed trades) + self.performance_data = {} + self.current_prices = {} + self.last_update = datetime.now() + + # Trading session tracking + self.session_start = datetime.now() + self.session_trades = [] + self.session_pnl = 0.0 + self.current_position = None # {'side': 'BUY', 'price': 3456.78, 'size': 0.1, 'timestamp': datetime} + self.total_realized_pnl = 0.0 + self.total_fees = 0.0 + self.starting_balance = self._get_initial_balance() # Get balance from MEXC or default to 100 + + # Closed trades tracking for accounting + self.closed_trades = [] # List of all closed trades with full details + + # Load existing closed trades from file + logger.info("DASHBOARD: Loading closed trades from file...") + self._load_closed_trades_from_file() + logger.info(f"DASHBOARD: Loaded {len(self.closed_trades)} closed trades") + + # Signal execution settings for scalping - REMOVED FREQUENCY LIMITS + self.min_confidence_threshold = 0.30 # Start lower to allow learning + self.signal_cooldown = 0 # REMOVED: Model decides when to act, no artificial delays + self.last_signal_time = 0 + + # Adaptive threshold learning - starts low and learns optimal thresholds + self.adaptive_learner = AdaptiveThresholdLearner(initial_threshold=0.30) + logger.info("[ADAPTIVE] Adaptive threshold learning enabled - will adjust based on trade outcomes") + + # Lightweight WebSocket implementation for real-time scalping data + self.ws_price_cache = {} # Just current prices, no tick history + self.ws_connection = None + self.ws_thread = None + self.is_streaming = False + + # Performance-focused: only track essentials + self.last_ws_update = 0 + self.ws_update_count = 0 + + # Compatibility stubs for removed tick infrastructure + self.tick_cache = [] # Empty list for compatibility + self.one_second_bars = [] # Empty list for compatibility + + # Enhanced RL Training System - Train on closed trades with comprehensive data + self.rl_training_enabled = True + # Force enable Enhanced RL training (bypass import issues) + self.enhanced_rl_training_enabled = True # Force enabled for CNN training + self.enhanced_rl_enabled = True # Force enabled to show proper status + self.rl_training_stats = { + 'total_training_episodes': 0, + 'profitable_trades_trained': 0, + 'unprofitable_trades_trained': 0, + 'last_training_time': None, + 'training_rewards': deque(maxlen=100), # Last 100 training rewards + 'model_accuracy_trend': deque(maxlen=50), # Track accuracy over time + 'enhanced_rl_episodes': 0, + 'comprehensive_data_packets': 0 + } + self.rl_training_queue = deque(maxlen=1000) # Queue of trades to train on + + # Enhanced training data tracking + self.latest_training_data = None + self.latest_ui_data = None + self.training_data_available = False + + # Load available models for real trading + self._load_available_models() + + # Preload essential data to prevent excessive API calls during dashboard updates + logger.info("Preloading essential market data to cache...") + try: + # Preload key timeframes for main symbols to ensure cache is populated + symbols_to_preload = self.config.symbols or ['ETH/USDT', 'BTC/USDT'] + timeframes_to_preload = ['1m', '1h', '1d'] # Skip 1s since we use WebSocket for that + + for symbol in symbols_to_preload[:2]: # Limit to first 2 symbols + for timeframe in timeframes_to_preload: + try: + # Load data into cache (refresh=True for initial load, then cache will be used) + df = self.data_provider.get_historical_data(symbol, timeframe, limit=100, refresh=True) + if df is not None and not df.empty: + logger.info(f"Preloaded {len(df)} {timeframe} bars for {symbol}") + else: + logger.warning(f"Failed to preload data for {symbol} {timeframe}") + except Exception as e: + logger.warning(f"Error preloading {symbol} {timeframe}: {e}") + + logger.info("Preloading completed - cache populated for frequent queries") + + except Exception as e: + logger.warning(f"Error during preloading: {e}") + + # Create Dash app + self.app = dash.Dash(__name__, external_stylesheets=[ + 'https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css', + 'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css' + ]) + + # # Add custom CSS for model data charts + # self.app.index_string = ''' + # + # + # + # {%metas%} + # {%title%} + # {%favicon%} + # {%css%} + # + # + # + # {%app_entry%} + # + # + # + # ''' + + # Setup layout and callbacks + self._setup_layout() + self._setup_callbacks() + + # Start unified data streaming + self._initialize_streaming() + + # Start continuous training with enhanced RL support + self.start_continuous_training() + + logger.info("Trading Dashboard initialized with enhanced RL training integration") + logger.info(f"Enhanced RL enabled: {self.enhanced_rl_training_enabled}") + logger.info(f"Stream consumer ID: {self.stream_consumer_id}") + + # Initialize Williams Market Structure once + try: + from training.williams_market_structure import WilliamsMarketStructure + self.williams_structure = WilliamsMarketStructure( + swing_strengths=[2, 3, 5], # Simplified for better performance + enable_cnn_feature=True, # Enable CNN training and inference + training_data_provider=self.data_provider # Provide data access for training + ) + logger.info("Williams Market Structure initialized for dashboard with CNN training enabled") + except ImportError: + self.williams_structure = None + logger.warning("Williams Market Structure not available") + + # Initialize Enhanced Pivot RL Trainer for better position management + try: + self.pivot_rl_trainer = create_enhanced_pivot_trainer( + data_provider=self.data_provider, + orchestrator=self.orchestrator + ) + logger.info("Enhanced Pivot RL Trainer initialized for better entry/exit decisions") + logger.info(f"Entry threshold: {self.pivot_rl_trainer.get_current_thresholds()['entry_threshold']:.1%}") + logger.info(f"Exit threshold: {self.pivot_rl_trainer.get_current_thresholds()['exit_threshold']:.1%}") + logger.info(f"Uninvested threshold: {self.pivot_rl_trainer.get_current_thresholds()['uninvested_threshold']:.1%}") + except Exception as e: + self.pivot_rl_trainer = None + logger.warning(f"Enhanced Pivot RL Trainer not available: {e}") + def _to_local_timezone(self, dt: datetime) -> datetime: """Convert datetime to configured local timezone""" try: @@ -479,15 +1381,19 @@ class TradingDashboard: return datetime.now(self.timezone) def _ensure_timezone_consistency(self, df: pd.DataFrame) -> pd.DataFrame: - """Ensure DataFrame index is in consistent timezone""" + """Ensure DataFrame index is in consistent timezone - FIXED to prevent double conversion""" try: if hasattr(df.index, 'tz'): if df.index.tz is None: - # Assume UTC if no timezone - df.index = df.index.tz_localize('UTC') - - # Convert to local timezone - df.index = df.index.tz_convert(self.timezone) + # Data is timezone-naive, assume it's already in local time + # Don't localize as UTC and convert again - this causes double conversion + logger.debug("Data is timezone-naive, assuming local time") + return df + else: + # Data has timezone info, convert to local timezone + df.index = df.index.tz_convert(self.timezone) + # Make timezone-naive to prevent browser double-conversion + df.index = df.index.tz_localize(None) return df except Exception as e: @@ -894,7 +1800,1388 @@ class TradingDashboard: style={"fontSize": "10px", "padding": "2px 8px"}) ], className="d-flex") ], className="d-flex justify-content-between align-items-center mb-2"), - dcc.Graph(id="price-chart", style={"height": "400px"}) + html.Div([ + dcc.Graph(id="price-chart", style={"height": "400px"}), + # JavaScript for client-side chart data management + html.Script(""" + // Initialize chart data cache and real-time management + window.chartDataCache = window.chartDataCache || {}; + window.chartUpdateInterval = window.chartUpdateInterval || null; + + // Chart data merging function + function mergeChartData(symbol, newData) { + if (!window.chartDataCache[symbol]) { + window.chartDataCache[symbol] = { + ohlc: [], + volume: [], + timestamps: [], + trades: [], + lastUpdate: Date.now(), + maxPoints: 2000 + }; + } + + const cache = window.chartDataCache[symbol]; + + // Merge new OHLC data + if (newData.ohlc && newData.ohlc.length > 0) { + const newTimestamps = newData.timestamps.map(ts => new Date(ts).getTime()); + const existingTimestampMap = new Map(); + + cache.timestamps.forEach((ts, idx) => { + existingTimestampMap.set(new Date(ts).getTime(), idx); + }); + + // Process each new data point + newData.ohlc.forEach((ohlc, i) => { + const newTime = newTimestamps[i]; + const existingIndex = existingTimestampMap.get(newTime); + + if (existingIndex !== undefined) { + // Update existing point + cache.ohlc[existingIndex] = ohlc; + cache.volume[existingIndex] = newData.volume[i]; + } else { + // Add new point + cache.ohlc.push(ohlc); + cache.volume.push(newData.volume[i]); + cache.timestamps.push(newData.timestamps[i]); + } + }); + + // Sort by timestamp to maintain chronological order + const combined = cache.ohlc.map((ohlc, i) => ({ + ohlc: ohlc, + volume: cache.volume[i], + timestamp: cache.timestamps[i], + sortTime: new Date(cache.timestamps[i]).getTime() + })); + + combined.sort((a, b) => a.sortTime - b.sortTime); + + // Keep only the most recent points for performance + if (combined.length > cache.maxPoints) { + combined.splice(0, combined.length - cache.maxPoints); + } + + // Update cache arrays + cache.ohlc = combined.map(item => item.ohlc); + cache.volume = combined.map(item => item.volume); + cache.timestamps = combined.map(item => item.timestamp); + } + + // Merge trade data + if (newData.trade_decisions) { + cache.trades = [...(cache.trades || []), ...newData.trade_decisions]; + // Keep only recent trades + if (cache.trades.length > 100) { + cache.trades = cache.trades.slice(-100); + } + } + + cache.lastUpdate = Date.now(); + console.log(`[CHART CACHE] ${symbol}: ${cache.ohlc.length} points, ${cache.trades.length} trades`); + } + + // Real-time chart update function + function updateChartRealtime(symbol) { + const cache = window.chartDataCache[symbol]; + if (!cache || cache.ohlc.length === 0) return; + + try { + const chartDiv = document.getElementById('price-chart'); + if (chartDiv && chartDiv.data && chartDiv.data.length > 0) { + + // Find the main price trace + let priceTraceIndex = -1; + let volumeTraceIndex = -1; + + for (let i = 0; i < chartDiv.data.length; i++) { + const trace = chartDiv.data[i]; + if (trace.type === 'scatter' && trace.name && trace.name.includes('Price')) { + priceTraceIndex = i; + } else if (trace.name && trace.name.includes('Volume')) { + volumeTraceIndex = i; + } + } + + // Update price data + if (priceTraceIndex !== -1 && cache.ohlc.length > 0) { + const newX = cache.timestamps; + const newY = cache.ohlc.map(ohlc => ohlc.close); + + Plotly.restyle(chartDiv, { + 'x': [newX], + 'y': [newY] + }, [priceTraceIndex]); + } + + // Update volume data + if (volumeTraceIndex !== -1 && cache.volume.length > 0) { + Plotly.restyle(chartDiv, { + 'x': [cache.timestamps], + 'y': [cache.volume] + }, [volumeTraceIndex]); + } + + // Update chart title with latest info + if (cache.ohlc.length > 0) { + const latestPrice = cache.ohlc[cache.ohlc.length - 1].close; + const currentTime = new Date().toLocaleTimeString(); + const newTitle = `${symbol} LIVE CHART | $${latestPrice.toFixed(2)} | ${currentTime} | ${cache.ohlc.length} points`; + + Plotly.relayout(chartDiv, { + 'title.text': newTitle + }); + } + } + } catch (error) { + console.warn('[CHART UPDATE] Error:', error); + } + } + + // Set up real-time updates (1-second interval) + function startChartUpdates(symbol) { + if (window.chartUpdateInterval) { + clearInterval(window.chartUpdateInterval); + } + + window.chartUpdateInterval = setInterval(() => { + if (window.chartDataCache[symbol]) { + updateChartRealtime(symbol); + } + }, 1000); // Update every second + + console.log(`[CHART INIT] Real-time updates started for ${symbol}`); + } + + // Start chart management when page loads + document.addEventListener('DOMContentLoaded', function() { + setTimeout(() => startChartUpdates('ETH/USDT'), 1000); + }); + + // Global function to receive data from Python + window.updateChartData = function(symbol, data) { + mergeChartData(symbol, data); + updateChartRealtime(symbol); + }; + """) + ]) + ], className="card-body p-2") + ], className="card", style={"width": "100%"}), + ], className="row g-2 mb-3"), + + # CNN Model Monitoring & COB Integration - MERGED into 1 row with 4 columns + html.Div([ + # CNN Status Column + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-brain me-2"), + "CNN Model Analysis" + ], className="card-title mb-2"), + html.Div(id="cnn-monitoring-content", style={"height": "280px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "23%"}), + + # COB Status Column + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-layer-group me-2"), + "COB → Training Pipeline" + ], className="card-title mb-2"), + html.Div(id="cob-status-content", style={"height": "280px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "23%", "marginLeft": "2%"}), + + # ETH/USDT COB Details Column + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-ethereum me-2", style={"color": "#627EEA"}), + "ETH/USDT - COB" + ], className="card-title mb-2"), + html.Div(id="eth-cob-content", style={"height": "280px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "23%", "marginLeft": "2%"}), + + # BTC/USDT COB Details Column + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-bitcoin me-2", style={"color": "#F7931A"}), + "BTC/USDT - COB" + ], className="card-title mb-2"), + html.Div(id="btc-cob-content", style={"height": "280px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "23%", "marginLeft": "2%"}), + ], className="d-flex mb-3"), + + # Bottom row - Session performance and system status + html.Div([ + + # Session performance - 1/3 width + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-chart-pie me-2"), + "Session Performance" + ], className="card-title mb-2"), + html.Button( + "Clear Session", + id="clear-history-btn", + className="btn btn-sm btn-outline-danger mb-2", + n_clicks=0 + ), + html.Div(id="session-performance") + ], className="card-body p-2") + ], className="card", style={"width": "32%"}), + + # Closed Trades History - 1/3 width + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-history me-2"), + "Closed Trades History" + ], className="card-title mb-2"), + html.Div([ + html.Div( + id="closed-trades-table", + style={"height": "300px", "overflowY": "auto"} + ) + ]) + ], className="card-body p-2") + ], className="card", style={"width": "32%", "marginLeft": "2%"}), + + # System status and leverage controls - 1/3 width with icon tooltip + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-server me-2"), + "System & Leverage" + ], className="card-title mb-2"), + + # System status + html.Div([ + html.I( + id="system-status-icon", + className="fas fa-circle text-success fa-2x", + title="System Status: All systems operational", + style={"cursor": "pointer"} + ), + html.Div(id="system-status-details", className="small mt-2") + ], className="text-center mb-3"), + + # Leverage Controls + html.Div([ + html.Label([ + html.I(className="fas fa-chart-line me-1"), + "Leverage Multiplier" + ], className="form-label small fw-bold"), + html.Div([ + dcc.Slider( + id='leverage-slider', + min=self.min_leverage, + max=self.max_leverage, + step=self.leverage_step, + value=self.leverage_multiplier, + marks={ + 1: '1x', + 10: '10x', + 25: '25x', + 50: '50x', + 75: '75x', + 100: '100x' + }, + tooltip={ + "placement": "bottom", + "always_visible": True + } + ) + ], className="mb-2"), + html.Div([ + html.Span(id="current-leverage", className="badge bg-warning text-dark"), + html.Span(" • ", className="mx-1"), + html.Span(id="leverage-risk", className="badge bg-info") + ], className="text-center"), + html.Div([ + html.Small("Higher leverage = Higher rewards & risks", className="text-muted") + ], className="text-center mt-1") + ]) + ], className="card-body p-2") + ], className="card", style={"width": "32%", "marginLeft": "2%"}) + ], className="d-flex") + ], className="container-fluid") + ]) + +""" +Trading Dashboard - Clean Web Interface + +This module provides a modern, responsive web dashboard for the trading system: +- Real-time price charts with multiple timeframes +- Model performance monitoring +- Trading decisions visualization +- System health monitoring +- Memory usage tracking +""" + +import asyncio +import dash +from dash import Dash, dcc, html, Input, Output +import plotly.graph_objects as go +from plotly.subplots import make_subplots +import plotly.express as px +import pandas as pd +import numpy as np +from datetime import datetime, timedelta, timezone +import pytz +import logging +import json +import time +import threading +from threading import Thread, Lock +from collections import deque +import warnings +from typing import Dict, List, Optional, Any, Union, Tuple +import websocket +import os +import torch + +# Setup logger immediately after logging import +logger = logging.getLogger(__name__) + +# WebSocket availability check +try: + import websocket + WEBSOCKET_AVAILABLE = True + logger.info("WebSocket client available") +except ImportError: + WEBSOCKET_AVAILABLE = False + logger.warning("websocket-client not available. Real-time data will use API fallback.") + +# Import trading system components +from core.config import get_config +from core.data_provider import DataProvider +from core.orchestrator import TradingOrchestrator, TradingDecision +from core.trading_executor import TradingExecutor +from core.trading_action import TradingAction +from models import get_model_registry + +# Import CNN monitoring +try: + from core.cnn_monitor import get_cnn_dashboard_data + CNN_MONITORING_AVAILABLE = True + logger.info("CNN monitoring system available") +except ImportError: + CNN_MONITORING_AVAILABLE = False + logger.warning("CNN monitoring not available") + def get_cnn_dashboard_data(): + return {'statistics': {'total_predictions_logged': 0}} + + +# Import enhanced RL components if available +try: + from core.enhanced_orchestrator import EnhancedTradingOrchestrator + from core.universal_data_adapter import UniversalDataAdapter + from core.unified_data_stream import UnifiedDataStream, TrainingDataPacket, UIDataPacket + ENHANCED_RL_AVAILABLE = True + logger.info("Enhanced RL training components available") +except ImportError as e: + logger.warning(f"Enhanced RL components not available: {e}") + ENHANCED_RL_AVAILABLE = False + # Force enable for learning - bypass import issues + ENHANCED_RL_AVAILABLE = True + logger.info("Enhanced RL FORCED ENABLED - bypassing import issues for learning") + + # Fallback classes + class UnifiedDataStream: + def __init__(self, *args, **kwargs): pass + def register_consumer(self, *args, **kwargs): return "fallback_consumer" + def start_streaming(self): pass + def stop_streaming(self): pass + def get_latest_training_data(self): return None + def get_latest_ui_data(self): return None + + class TrainingDataPacket: + def __init__(self, *args, **kwargs): pass + + class UIDataPacket: + def __init__(self, *args, **kwargs): pass + +# Import COB integration components if available +try: + from core.cob_integration import COBIntegration + from core.multi_exchange_cob_provider import MultiExchangeCOBProvider, COBSnapshot + COB_INTEGRATION_AVAILABLE = True + logger.info("COB integration components available") +except ImportError as e: + logger.warning(f"COB integration components not available: {e}") + COB_INTEGRATION_AVAILABLE = False + # Create fallback classes + class COBSnapshot: + def __init__(self, *args, **kwargs): + self.symbol = "N/A" + self.consolidated_bids = [] + self.consolidated_asks = [] + self.volume_weighted_mid = 0.0 + self.spread_bps = 0.0 + self.total_bid_liquidity = 0.0 + self.total_ask_liquidity = 0.0 + + +class AdaptiveThresholdLearner: + """Learn optimal confidence thresholds based on real trade outcomes""" + + def __init__(self, initial_threshold: float = 0.30): + self.base_threshold = initial_threshold + self.current_threshold = initial_threshold + self.trade_outcomes = deque(maxlen=100) + self.threshold_history = deque(maxlen=50) + self.learning_rate = 0.02 + self.min_threshold = 0.20 + self.max_threshold = 0.70 + + logger.info(f"[ADAPTIVE] Initialized with starting threshold: {initial_threshold:.2%}") + + def record_trade_outcome(self, confidence: float, pnl: float, threshold_used: float): + """Record a trade outcome to learn from""" + try: + outcome = { + 'confidence': confidence, + 'pnl': pnl, + 'profitable': pnl > 0, + 'threshold_used': threshold_used, + 'timestamp': datetime.now() + } + + self.trade_outcomes.append(outcome) + + # Learn from outcomes + if len(self.trade_outcomes) >= 10: + self._update_threshold() + + except Exception as e: + logger.error(f"Error recording trade outcome: {e}") + + def _update_threshold(self): + """Update threshold based on recent trade statistics""" + try: + recent_trades = list(self.trade_outcomes)[-20:] + if len(recent_trades) < 10: + return + + profitable_count = sum(1 for t in recent_trades if t['profitable']) + win_rate = profitable_count / len(recent_trades) + avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades) + + # Adaptive adjustment logic + if win_rate > 0.60 and avg_pnl > 0.20: + adjustment = -self.learning_rate * 1.5 # Lower threshold for more trades + elif win_rate < 0.40 or avg_pnl < -0.30: + adjustment = self.learning_rate * 2.0 # Raise threshold to be more selective + else: + adjustment = 0 # No change + + old_threshold = self.current_threshold + self.current_threshold = max(self.min_threshold, + min(self.max_threshold, + self.current_threshold + adjustment)) + + if abs(self.current_threshold - old_threshold) > 0.005: + logger.info(f"[ADAPTIVE] Threshold: {old_threshold:.2%} -> {self.current_threshold:.2%} (WR: {win_rate:.1%}, PnL: ${avg_pnl:.2f})") + + except Exception as e: + logger.error(f"Error updating adaptive threshold: {e}") + + def get_current_threshold(self) -> float: + return self.current_threshold + + def get_learning_stats(self) -> Dict[str, Any]: + """Get learning statistics""" + try: + if not self.trade_outcomes: + return {'status': 'No trades recorded yet'} + + recent_trades = list(self.trade_outcomes)[-20:] + profitable_count = sum(1 for t in recent_trades if t['profitable']) + win_rate = profitable_count / len(recent_trades) if recent_trades else 0 + avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades) if recent_trades else 0 + + return { + 'current_threshold': self.current_threshold, + 'base_threshold': self.base_threshold, + 'total_trades': len(self.trade_outcomes), + 'recent_win_rate': win_rate, + 'recent_avg_pnl': avg_pnl, + 'threshold_changes': len(self.threshold_history), + 'learning_active': len(self.trade_outcomes) >= 10 + } + except Exception as e: + return {'error': str(e)} + +class TradingDashboard: + """Enhanced Trading Dashboard with Williams pivot points and unified timezone handling""" + + def __init__(self, data_provider: DataProvider = None, orchestrator: TradingOrchestrator = None, trading_executor: TradingExecutor = None): + self.app = Dash(__name__) + + # Initialize config first + from core.config import get_config + self.config = get_config() + + self.data_provider = data_provider or DataProvider() + self.orchestrator = orchestrator + self.trading_executor = trading_executor + + # Enhanced trading state with leverage support + self.leverage_enabled = True + self.leverage_multiplier = 50.0 # 50x leverage (adjustable via slider) + self.base_capital = 10000.0 + self.current_position = 0.0 # -1 to 1 (short to long) + self.position_size = 0.0 + self.entry_price = 0.0 + self.unrealized_pnl = 0.0 + self.realized_pnl = 0.0 + + # Leverage settings for slider + self.min_leverage = 1.0 + self.max_leverage = 100.0 + self.leverage_step = 1.0 + + # Connect to trading server for leverage functionality + self.trading_server_url = "http://127.0.0.1:8052" + self.training_server_url = "http://127.0.0.1:8053" + self.stream_server_url = "http://127.0.0.1:8054" + + # Enhanced performance tracking + self.leverage_metrics = { + 'leverage_efficiency': 0.0, + 'margin_used': 0.0, + 'margin_available': 10000.0, + 'effective_exposure': 0.0, + 'risk_reward_ratio': 0.0 + } + + # Enhanced models will be loaded through model registry later + + # Rest of initialization... + + # Initialize timezone from config + timezone_name = self.config.get('system', {}).get('timezone', 'Europe/Sofia') + self.timezone = pytz.timezone(timezone_name) + logger.info(f"Dashboard timezone set to: {timezone_name}") + + self.data_provider = data_provider or DataProvider() + + # Use enhanced orchestrator for comprehensive RL training + if orchestrator is None: + from core.enhanced_orchestrator import EnhancedTradingOrchestrator + self.orchestrator = EnhancedTradingOrchestrator( + data_provider=self.data_provider, + symbols=['ETH/USDT', 'BTC/USDT'], + enhanced_rl_training=True + ) + logger.info("Using Enhanced Trading Orchestrator for comprehensive RL training") + else: + self.orchestrator = orchestrator + logger.info(f"Using provided orchestrator: {type(orchestrator).__name__}") + self.enhanced_rl_enabled = True # Force enable Enhanced RL + logger.info("Enhanced RL training FORCED ENABLED for learning") + + self.trading_executor = trading_executor or TradingExecutor() + self.model_registry = get_model_registry() + + # Initialize unified data stream for comprehensive training data + if ENHANCED_RL_AVAILABLE: + self.unified_stream = UnifiedDataStream(self.data_provider, self.orchestrator) + self.stream_consumer_id = self.unified_stream.register_consumer( + consumer_name="TradingDashboard", + callback=self._handle_unified_stream_data, + data_types=['ticks', 'ohlcv', 'training_data', 'ui_data'] + ) + logger.info(f"Unified data stream initialized with consumer ID: {self.stream_consumer_id}") + else: + self.unified_stream = UnifiedDataStream() # Fallback + self.stream_consumer_id = "fallback" + logger.warning("Using fallback unified data stream") + + # Dashboard state + self.recent_decisions = [] + self.recent_signals = [] # Track all signals (not just executed trades) + self.performance_data = {} + self.current_prices = {} + self.last_update = datetime.now() + + # Trading session tracking + self.session_start = datetime.now() + self.session_trades = [] + self.session_pnl = 0.0 + self.current_position = None # {'side': 'BUY', 'price': 3456.78, 'size': 0.1, 'timestamp': datetime} + self.total_realized_pnl = 0.0 + self.total_fees = 0.0 + self.starting_balance = self._get_initial_balance() # Get balance from MEXC or default to 100 + + # Closed trades tracking for accounting + self.closed_trades = [] # List of all closed trades with full details + + # Load existing closed trades from file + logger.info("DASHBOARD: Loading closed trades from file...") + self._load_closed_trades_from_file() + logger.info(f"DASHBOARD: Loaded {len(self.closed_trades)} closed trades") + + # Signal execution settings for scalping - REMOVED FREQUENCY LIMITS + self.min_confidence_threshold = 0.30 # Start lower to allow learning + self.signal_cooldown = 0 # REMOVED: Model decides when to act, no artificial delays + self.last_signal_time = 0 + + # Adaptive threshold learning - starts low and learns optimal thresholds + self.adaptive_learner = AdaptiveThresholdLearner(initial_threshold=0.30) + logger.info("[ADAPTIVE] Adaptive threshold learning enabled - will adjust based on trade outcomes") + + # Lightweight WebSocket implementation for real-time scalping data + self.ws_price_cache = {} # Just current prices, no tick history + self.ws_connection = None + self.ws_thread = None + self.is_streaming = False + + # Performance-focused: only track essentials + self.last_ws_update = 0 + self.ws_update_count = 0 + + # Compatibility stubs for removed tick infrastructure + self.tick_cache = [] # Empty list for compatibility + self.one_second_bars = [] # Empty list for compatibility + + # Enhanced RL Training System - Train on closed trades with comprehensive data + self.rl_training_enabled = True + # Force enable Enhanced RL training (bypass import issues) + self.enhanced_rl_training_enabled = True # Force enabled for CNN training + self.enhanced_rl_enabled = True # Force enabled to show proper status + self.rl_training_stats = { + 'total_training_episodes': 0, + 'profitable_trades_trained': 0, + 'unprofitable_trades_trained': 0, + 'last_training_time': None, + 'training_rewards': deque(maxlen=100), # Last 100 training rewards + 'model_accuracy_trend': deque(maxlen=50), # Track accuracy over time + 'enhanced_rl_episodes': 0, + 'comprehensive_data_packets': 0 + } + self.rl_training_queue = deque(maxlen=1000) # Queue of trades to train on + + # Enhanced training data tracking + self.latest_training_data = None + self.latest_ui_data = None + self.training_data_available = False + + # Load available models for real trading + self._load_available_models() + + # Preload essential data to prevent excessive API calls during dashboard updates + logger.info("Preloading essential market data to cache...") + try: + # Preload key timeframes for main symbols to ensure cache is populated + symbols_to_preload = self.config.symbols or ['ETH/USDT', 'BTC/USDT'] + timeframes_to_preload = ['1m', '1h', '1d'] # Skip 1s since we use WebSocket for that + + for symbol in symbols_to_preload[:2]: # Limit to first 2 symbols + for timeframe in timeframes_to_preload: + try: + # Load data into cache (refresh=True for initial load, then cache will be used) + df = self.data_provider.get_historical_data(symbol, timeframe, limit=100, refresh=True) + if df is not None and not df.empty: + logger.info(f"Preloaded {len(df)} {timeframe} bars for {symbol}") + else: + logger.warning(f"Failed to preload data for {symbol} {timeframe}") + except Exception as e: + logger.warning(f"Error preloading {symbol} {timeframe}: {e}") + + logger.info("Preloading completed - cache populated for frequent queries") + + except Exception as e: + logger.warning(f"Error during preloading: {e}") + + # Create Dash app + self.app = dash.Dash(__name__, external_stylesheets=[ + 'https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css', + 'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css' + ]) + + # # Add custom CSS for model data charts + # self.app.index_string = ''' + # + # + # + # {%metas%} + # {%title%} + # {%favicon%} + # {%css%} + # + # + # + # {%app_entry%} + # + # + # + # ''' + + # Setup layout and callbacks + self._setup_layout() + self._setup_callbacks() + + # Start unified data streaming + self._initialize_streaming() + + # Start continuous training with enhanced RL support + self.start_continuous_training() + + logger.info("Trading Dashboard initialized with enhanced RL training integration") + logger.info(f"Enhanced RL enabled: {self.enhanced_rl_training_enabled}") + logger.info(f"Stream consumer ID: {self.stream_consumer_id}") + + # Initialize Williams Market Structure once + try: + from training.williams_market_structure import WilliamsMarketStructure + self.williams_structure = WilliamsMarketStructure( + swing_strengths=[2, 3, 5], # Simplified for better performance + enable_cnn_feature=True, # Enable CNN training and inference + training_data_provider=self.data_provider # Provide data access for training + ) + logger.info("Williams Market Structure initialized for dashboard with CNN training enabled") + except ImportError: + self.williams_structure = None + logger.warning("Williams Market Structure not available") + + # Initialize Enhanced Pivot RL Trainer for better position management + try: + self.pivot_rl_trainer = create_enhanced_pivot_trainer( + data_provider=self.data_provider, + orchestrator=self.orchestrator + ) + logger.info("Enhanced Pivot RL Trainer initialized for better entry/exit decisions") + logger.info(f"Entry threshold: {self.pivot_rl_trainer.get_current_thresholds()['entry_threshold']:.1%}") + logger.info(f"Exit threshold: {self.pivot_rl_trainer.get_current_thresholds()['exit_threshold']:.1%}") + logger.info(f"Uninvested threshold: {self.pivot_rl_trainer.get_current_thresholds()['uninvested_threshold']:.1%}") + except Exception as e: + self.pivot_rl_trainer = None + logger.warning(f"Enhanced Pivot RL Trainer not available: {e}") + + def _to_local_timezone(self, dt: datetime) -> datetime: + """Convert datetime to configured local timezone""" + try: + if dt is None: + return None + + # If datetime is naive, assume it's UTC + if dt.tzinfo is None: + dt = pytz.UTC.localize(dt) + + # Convert to local timezone + return dt.astimezone(self.timezone) + except Exception as e: + logger.warning(f"Error converting timezone: {e}") + return dt + + def _now_local(self) -> datetime: + """Get current time in configured local timezone""" + return datetime.now(self.timezone) + + def _ensure_timezone_consistency(self, df: pd.DataFrame) -> pd.DataFrame: + """Ensure DataFrame index is in consistent timezone - FIXED to prevent double conversion""" + try: + if hasattr(df.index, 'tz'): + if df.index.tz is None: + # Data is timezone-naive, assume it's already in local time + # Don't localize as UTC and convert again - this causes double conversion + logger.debug("Data is timezone-naive, assuming local time") + return df + else: + # Data has timezone info, convert to local timezone + df.index = df.index.tz_convert(self.timezone) + # Make timezone-naive to prevent browser double-conversion + df.index = df.index.tz_localize(None) + + return df + except Exception as e: + logger.warning(f"Error ensuring timezone consistency: {e}") + return df + + def _initialize_streaming(self): + """Initialize unified data streaming and WebSocket fallback""" + try: + # Start lightweight WebSocket for real-time price updates + self._start_lightweight_websocket() + logger.info("Lightweight WebSocket streaming initialized") + + if ENHANCED_RL_AVAILABLE: + # Start unified data stream in background + def start_unified_stream(): + try: + asyncio.run(self.unified_stream.start_streaming()) + logger.info("Unified data stream started") + except Exception as e: + logger.error(f"Error starting unified stream: {e}") + + unified_thread = Thread(target=start_unified_stream, daemon=True) + unified_thread.start() + + # Start background data collection + self._start_enhanced_training_data_collection() + + logger.info("All data streaming initialized") + + except Exception as e: + logger.error(f"Error initializing streaming: {e}") + # Ensure lightweight WebSocket is started as fallback + self._start_lightweight_websocket() + + def _start_enhanced_training_data_collection(self): + """Start enhanced training data collection using unified stream""" + def enhanced_training_loop(): + try: + logger.info("Enhanced training data collection started with unified stream") + + while True: + try: + if ENHANCED_RL_AVAILABLE and self.enhanced_rl_training_enabled: + # Get latest comprehensive training data from unified stream + training_data = self.unified_stream.get_latest_training_data() + + if training_data: + # Send comprehensive training data to enhanced RL pipeline + self._send_comprehensive_training_data_to_enhanced_rl(training_data) + + # Update training statistics + self.rl_training_stats['comprehensive_data_packets'] += 1 + self.training_data_available = True + + # Update context data in orchestrator + if hasattr(self.orchestrator, 'update_context_data'): + self.orchestrator.update_context_data() + + # Initialize extrema trainer if not done + if hasattr(self.orchestrator, 'extrema_trainer'): + if not hasattr(self.orchestrator.extrema_trainer, '_initialized'): + self.orchestrator.extrema_trainer.initialize_context_data() + self.orchestrator.extrema_trainer._initialized = True + logger.info("Extrema trainer context data initialized") + + # Run extrema detection with real data + if hasattr(self.orchestrator, 'extrema_trainer'): + for symbol in self.orchestrator.symbols: + detected = self.orchestrator.extrema_trainer.detect_local_extrema(symbol) + if detected: + logger.debug(f"Detected {len(detected)} extrema for {symbol}") + else: + # Fallback to basic training data collection + self._collect_basic_training_data() + + time.sleep(10) # Update every 10 seconds for enhanced training + + except Exception as e: + logger.error(f"Error in enhanced training loop: {e}") + time.sleep(30) # Wait before retrying + + except Exception as e: + logger.error(f"Enhanced training loop failed: {e}") + + # Start enhanced training thread + training_thread = Thread(target=enhanced_training_loop, daemon=True) + training_thread.start() + logger.info("Enhanced training data collection thread started") + + def _handle_unified_stream_data(self, data_packet: Dict[str, Any]): + """Handle data from unified stream for dashboard and training""" + try: + # Extract UI data for dashboard display + if 'ui_data' in data_packet: + self.latest_ui_data = data_packet['ui_data'] + if hasattr(self.latest_ui_data, 'current_prices'): + self.current_prices.update(self.latest_ui_data.current_prices) + if hasattr(self.latest_ui_data, 'streaming_status'): + self.is_streaming = self.latest_ui_data.streaming_status == 'LIVE' + if hasattr(self.latest_ui_data, 'training_data_available'): + self.training_data_available = self.latest_ui_data.training_data_available + + # Extract training data for enhanced RL + if 'training_data' in data_packet: + self.latest_training_data = data_packet['training_data'] + logger.debug("Received comprehensive training data from unified stream") + + # Extract tick data for dashboard charts + if 'ticks' in data_packet: + ticks = data_packet['ticks'] + for tick in ticks[-100:]: # Keep last 100 ticks + self.tick_cache.append(tick) + + # Extract OHLCV data for dashboard charts + if 'one_second_bars' in data_packet: + bars = data_packet['one_second_bars'] + for bar in bars[-100:]: # Keep last 100 bars + self.one_second_bars.append(bar) + + logger.debug(f"Processed unified stream data packet with keys: {list(data_packet.keys())}") + + except Exception as e: + logger.error(f"Error handling unified stream data: {e}") + + def _send_comprehensive_training_data_to_enhanced_rl(self, training_data: TrainingDataPacket): + """Send comprehensive training data to enhanced RL training pipeline""" + try: + if not self.enhanced_rl_training_enabled: + logger.debug("Enhanced RL training not enabled, skipping comprehensive data send") + return + + # Extract comprehensive training data components + market_state = training_data.market_state if hasattr(training_data, 'market_state') else None + universal_stream = training_data.universal_stream if hasattr(training_data, 'universal_stream') else None + cnn_features = training_data.cnn_features if hasattr(training_data, 'cnn_features') else None + cnn_predictions = training_data.cnn_predictions if hasattr(training_data, 'cnn_predictions') else None + + if market_state and universal_stream: + # Send to enhanced RL trainer if available + if hasattr(self.orchestrator, 'enhanced_rl_trainer'): + try: + # Create comprehensive training step with ~13,400 features + asyncio.run(self.orchestrator.enhanced_rl_trainer.training_step(universal_stream)) + self.rl_training_stats['enhanced_rl_episodes'] += 1 + logger.debug("Sent comprehensive data to enhanced RL trainer") + except Exception as e: + logger.warning(f"Error in enhanced RL training step: {e}") + + # Send to extrema trainer for CNN training with perfect moves + if hasattr(self.orchestrator, 'extrema_trainer'): + try: + extrema_data = self.orchestrator.extrema_trainer.get_extrema_training_data(count=50) + perfect_moves = self.orchestrator.extrema_trainer.get_perfect_moves_for_cnn(count=100) + + if extrema_data: + logger.debug(f"Enhanced RL: {len(extrema_data)} extrema training samples available") + + if perfect_moves: + logger.debug(f"Enhanced RL: {len(perfect_moves)} perfect moves for CNN training") + except Exception as e: + logger.warning(f"Error getting extrema training data: {e}") + + # Send to sensitivity learning DQN for outcome-based learning + if hasattr(self.orchestrator, 'sensitivity_learning_queue'): + try: + if len(self.orchestrator.sensitivity_learning_queue) > 0: + logger.debug("Enhanced RL: Sensitivity learning data available for DQN training") + except Exception as e: + logger.warning(f"Error accessing sensitivity learning queue: {e}") + + # Get context features for models with real market data + if hasattr(self.orchestrator, 'extrema_trainer'): + try: + for symbol in self.orchestrator.symbols: + context_features = self.orchestrator.extrema_trainer.get_context_features_for_model(symbol) + if context_features is not None: + logger.debug(f"Enhanced RL: Context features available for {symbol}: {context_features.shape}") + except Exception as e: + logger.warning(f"Error getting context features: {e}") + + # Log comprehensive training data statistics + tick_count = len(training_data.tick_cache) if hasattr(training_data, 'tick_cache') else 0 + bars_count = len(training_data.one_second_bars) if hasattr(training_data, 'one_second_bars') else 0 + timeframe_count = len(training_data.multi_timeframe_data) if hasattr(training_data, 'multi_timeframe_data') else 0 + + logger.info(f"Enhanced RL Comprehensive Training Data:") + logger.info(f" Tick cache: {tick_count} ticks") + logger.info(f" 1s bars: {bars_count} bars") + logger.info(f" Multi-timeframe data: {timeframe_count} symbols") + logger.info(f" CNN features: {'Available' if cnn_features else 'Not available'}") + logger.info(f" CNN predictions: {'Available' if cnn_predictions else 'Not available'}") + logger.info(f" Market state: {'Available (~13,400 features)' if market_state else 'Not available'}") + logger.info(f" Universal stream: {'Available' if universal_stream else 'Not available'}") + + except Exception as e: + logger.error(f"Error sending comprehensive training data to enhanced RL: {e}") + + def _collect_basic_training_data(self): + """Fallback method to collect basic training data when enhanced RL is not available""" + try: + # Get real tick data from data provider subscribers + for symbol in ['ETH/USDT', 'BTC/USDT']: + try: + # Get recent ticks from data provider + if hasattr(self.data_provider, 'get_recent_ticks'): + recent_ticks = self.data_provider.get_recent_ticks(symbol, count=10) + + for tick in recent_ticks: + # Create tick data from real market data + tick_data = { + 'symbol': tick.symbol, + 'price': tick.price, + 'timestamp': tick.timestamp, + 'volume': tick.volume + } + + # Add to tick cache + self.tick_cache.append(tick_data) + + # Create 1s bar data from real tick + bar_data = { + 'symbol': tick.symbol, + 'open': tick.price, + 'high': tick.price, + 'low': tick.price, + 'close': tick.price, + 'volume': tick.volume, + 'timestamp': tick.timestamp + } + + # Add to 1s bars cache + self.one_second_bars.append(bar_data) + + except Exception as e: + logger.debug(f"No recent tick data available for {symbol}: {e}") + + # Set streaming status based on real data availability + self.is_streaming = len(self.tick_cache) > 0 + + except Exception as e: + logger.warning(f"Error in basic training data collection: {e}") + + def _get_initial_balance(self) -> float: + """Get initial USDT balance from MEXC or return default""" + try: + if self.trading_executor and hasattr(self.trading_executor, 'get_account_balance'): + logger.info("Fetching initial balance from MEXC...") + + # Check if trading is enabled and not in dry run mode + if not self.trading_executor.trading_enabled: + logger.warning("MEXC: Trading not enabled - using default balance") + elif self.trading_executor.simulation_mode: + logger.warning(f"MEXC: {self.trading_executor.trading_mode.upper()} mode enabled - using default balance") + else: + # Get USDT balance from MEXC + balance_info = self.trading_executor.get_account_balance() + if balance_info and 'USDT' in balance_info: + usdt_balance = float(balance_info['USDT'].get('free', 0)) + if usdt_balance > 0: + logger.info(f"MEXC: Retrieved USDT balance: ${usdt_balance:.2f}") + return usdt_balance + else: + logger.warning("MEXC: No USDT balance found in account") + else: + logger.error("MEXC: Failed to retrieve balance info from API") + else: + logger.info("MEXC: Trading executor not available for balance retrieval") + + except Exception as e: + logger.error(f"Error getting MEXC balance: {e}") + import traceback + logger.error(traceback.format_exc()) + + # Fallback to default + default_balance = 100.0 + logger.warning(f"Using default starting balance: ${default_balance:.2f}") + return default_balance + + def _setup_layout(self): + """Setup the dashboard layout""" + self.app.layout = html.Div([ + # Compact Header + html.Div([ + html.H3([ + html.I(className="fas fa-chart-line me-2"), + "Live Trading Dashboard" + ], className="text-white mb-1"), + html.P(f"Ultra-Fast Updates • Portfolio: ${self.starting_balance:,.0f} • {'MEXC Live' if (self.trading_executor and self.trading_executor.trading_enabled and not self.trading_executor.simulation_mode) else 'Demo Mode'}", + className="text-light mb-0 opacity-75 small") + ], className="bg-dark p-2 mb-2"), + + # Auto-refresh component - ultra-fast updates for real-time trading + dcc.Interval( + id='interval-component', + interval=1000, # Update every 1 second for maximum responsiveness + n_intervals=0 + ), + + # Main content - Compact layout + html.Div([ + # Top row - Key metrics and Recent Signals (split layout) + html.Div([ + # Left side - Key metrics (compact cards) + html.Div([ + html.Div([ + html.Div([ + html.H5(id="current-price", className="text-success mb-0 small"), + html.P("Live Price", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="session-pnl", className="mb-0 small"), + html.P("Session P&L", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="total-fees", className="text-warning mb-0 small"), + html.P("Total Fees", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="current-position", className="text-info mb-0 small"), + html.P("Position", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="trade-count", className="text-warning mb-0 small"), + html.P("Trades", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="portfolio-value", className="text-secondary mb-0 small"), + html.P("Portfolio", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + + html.Div([ + html.Div([ + html.H5(id="mexc-status", className="text-info mb-0 small"), + html.P("MEXC API", className="text-muted mb-0 tiny") + ], className="card-body text-center p-2") + ], className="card bg-light", style={"height": "60px"}), + ], style={"display": "grid", "gridTemplateColumns": "repeat(4, 1fr)", "gap": "8px", "width": "60%"}), + + + # Right side - Merged: Recent Signals & Model Training - 2 columns + html.Div([ + # Recent Trading Signals Column (50%) + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-robot me-2"), + "Recent Trading Signals" + ], className="card-title mb-2"), + html.Div(id="recent-decisions", style={"height": "160px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "48%"}), + + # Model Training + COB Buckets Column (50%) + html.Div([ + html.Div([ + html.H6([ + html.I(className="fas fa-brain me-2"), + "Training Progress & COB $1 Buckets" + ], className="card-title mb-2"), + html.Div(id="training-metrics", style={"height": "160px", "overflowY": "auto"}) + ], className="card-body p-2") + ], className="card", style={"width": "48%", "marginLeft": "4%"}), + ], style={"width": "48%", "marginLeft": "2%", "display": "flex"}) + ], className="d-flex mb-3"), + + # Charts row - Now full width since training moved up + html.Div([ + # Price chart - Full width with manual trading buttons + html.Div([ + html.Div([ + # Chart header with manual trading buttons + html.Div([ + html.H6([ + html.I(className="fas fa-chart-candlestick me-2"), + "Live 1s Price & Volume Chart (WebSocket Stream)" + ], className="card-title mb-0"), + html.Div([ + html.Button([ + html.I(className="fas fa-arrow-up me-1"), + "BUY" + ], id="manual-buy-btn", className="btn btn-success btn-sm me-2", + style={"fontSize": "10px", "padding": "2px 8px"}), + html.Button([ + html.I(className="fas fa-arrow-down me-1"), + "SELL" + ], id="manual-sell-btn", className="btn btn-danger btn-sm", + style={"fontSize": "10px", "padding": "2px 8px"}) + ], className="d-flex") + ], className="d-flex justify-content-between align-items-center mb-2"), + html.Div([ + dcc.Graph(id="price-chart", style={"height": "400px"}), + # JavaScript for client-side chart data management + html.Script(""" + // Initialize chart data cache and real-time management + window.chartDataCache = window.chartDataCache || {}; + window.chartUpdateInterval = window.chartUpdateInterval || null; + + // Chart data merging function + function mergeChartData(symbol, newData) { + if (!window.chartDataCache[symbol]) { + window.chartDataCache[symbol] = { + ohlc: [], + volume: [], + timestamps: [], + trades: [], + lastUpdate: Date.now(), + maxPoints: 2000 + }; + } + + const cache = window.chartDataCache[symbol]; + + // Merge new OHLC data + if (newData.ohlc && newData.ohlc.length > 0) { + const newTimestamps = newData.timestamps.map(ts => new Date(ts).getTime()); + const existingTimestampMap = new Map(); + + cache.timestamps.forEach((ts, idx) => { + existingTimestampMap.set(new Date(ts).getTime(), idx); + }); + + // Process each new data point + newData.ohlc.forEach((ohlc, i) => { + const newTime = newTimestamps[i]; + const existingIndex = existingTimestampMap.get(newTime); + + if (existingIndex !== undefined) { + // Update existing point + cache.ohlc[existingIndex] = ohlc; + cache.volume[existingIndex] = newData.volume[i]; + } else { + // Add new point + cache.ohlc.push(ohlc); + cache.volume.push(newData.volume[i]); + cache.timestamps.push(newData.timestamps[i]); + } + }); + + // Sort by timestamp to maintain chronological order + const combined = cache.ohlc.map((ohlc, i) => ({ + ohlc: ohlc, + volume: cache.volume[i], + timestamp: cache.timestamps[i], + sortTime: new Date(cache.timestamps[i]).getTime() + })); + + combined.sort((a, b) => a.sortTime - b.sortTime); + + // Keep only the most recent points for performance + if (combined.length > cache.maxPoints) { + combined.splice(0, combined.length - cache.maxPoints); + } + + // Update cache arrays + cache.ohlc = combined.map(item => item.ohlc); + cache.volume = combined.map(item => item.volume); + cache.timestamps = combined.map(item => item.timestamp); + } + + // Merge trade data + if (newData.trade_decisions) { + cache.trades = [...(cache.trades || []), ...newData.trade_decisions]; + // Keep only recent trades + if (cache.trades.length > 100) { + cache.trades = cache.trades.slice(-100); + } + } + + cache.lastUpdate = Date.now(); + console.log(`[CHART CACHE] ${symbol}: ${cache.ohlc.length} points, ${cache.trades.length} trades`); + } + + // Real-time chart update function + function updateChartRealtime(symbol) { + const cache = window.chartDataCache[symbol]; + if (!cache || cache.ohlc.length === 0) return; + + try { + const chartDiv = document.getElementById('price-chart'); + if (chartDiv && chartDiv.data && chartDiv.data.length > 0) { + + // Find the main price trace + let priceTraceIndex = -1; + let volumeTraceIndex = -1; + + for (let i = 0; i < chartDiv.data.length; i++) { + const trace = chartDiv.data[i]; + if (trace.type === 'scatter' && trace.name && trace.name.includes('Price')) { + priceTraceIndex = i; + } else if (trace.name && trace.name.includes('Volume')) { + volumeTraceIndex = i; + } + } + + // Update price data + if (priceTraceIndex !== -1 && cache.ohlc.length > 0) { + const newX = cache.timestamps; + const newY = cache.ohlc.map(ohlc => ohlc.close); + + Plotly.restyle(chartDiv, { + 'x': [newX], + 'y': [newY] + }, [priceTraceIndex]); + } + + // Update volume data + if (volumeTraceIndex !== -1 && cache.volume.length > 0) { + Plotly.restyle(chartDiv, { + 'x': [cache.timestamps], + 'y': [cache.volume] + }, [volumeTraceIndex]); + } + + // Update chart title with latest info + if (cache.ohlc.length > 0) { + const latestPrice = cache.ohlc[cache.ohlc.length - 1].close; + const currentTime = new Date().toLocaleTimeString(); + const newTitle = `${symbol} LIVE CHART | $${latestPrice.toFixed(2)} | ${currentTime} | ${cache.ohlc.length} points`; + + Plotly.relayout(chartDiv, { + 'title.text': newTitle + }); + } + } + } catch (error) { + console.warn('[CHART UPDATE] Error:', error); + } + } + + // Set up real-time updates (1-second interval) + function startChartUpdates(symbol) { + if (window.chartUpdateInterval) { + clearInterval(window.chartUpdateInterval); + } + + window.chartUpdateInterval = setInterval(() => { + if (window.chartDataCache[symbol]) { + updateChartRealtime(symbol); + } + }, 1000); // Update every second + + console.log(`[CHART INIT] Real-time updates started for ${symbol}`); + } + + // Start chart management when page loads + document.addEventListener('DOMContentLoaded', function() { + setTimeout(() => startChartUpdates('ETH/USDT'), 1000); + }); + + // Global function to receive data from Python + window.updateChartData = function(symbol, data) { + mergeChartData(symbol, data); + updateChartRealtime(symbol); + }; + """) + ]) ], className="card-body p-2") ], className="card", style={"width": "100%"}), ], className="row g-2 mb-3"), @@ -2080,6 +4367,38 @@ class TradingDashboard: stream_status = "LIVE STREAM" if self.is_streaming else "CACHED DATA" tick_count = len(self.tick_cache) + # Prepare incremental data for JavaScript merging and caching + incremental_data = { + 'ohlc': df[['open', 'high', 'low', 'close']].to_dict('records') if not df.empty else [], + 'volume': df['volume'].to_list() if not df.empty else [], + 'timestamps': [ts.isoformat() for ts in df.index] if not df.empty else [], + 'symbol': symbol, + 'is_streaming': self.is_streaming, + 'latest_price': float(latest_price), + 'update_time': current_time, + 'trade_decisions': [ + { + 'timestamp': decision.get('timestamp').isoformat() if isinstance(decision.get('timestamp'), datetime) else str(decision.get('timestamp')), + 'action': decision.get('action'), + 'price': float(decision.get('price', 0)), + 'confidence': float(decision.get('confidence', 0)), + 'executed': decision.get('signal_type') == 'EXECUTED' + } + for decision in self.recent_decisions[-20:] if isinstance(decision, dict) + ], + 'closed_trades': [ + { + 'entry_time': trade.get('entry_time'), + 'exit_time': trade.get('exit_time'), + 'entry_price': float(trade.get('entry_price', 0)), + 'exit_price': float(trade.get('exit_price', 0)), + 'side': trade.get('side'), + 'net_pnl': float(trade.get('net_pnl', 0)) + } + for trade in self.closed_trades[-50:] if isinstance(trade, dict) + ] + } + fig.update_layout( title=f"{symbol} {actual_timeframe.upper()} CHART | ${latest_price:.2f} | {stream_status} | {tick_count} ticks | {current_time}", template="plotly_dark", @@ -2092,7 +4411,17 @@ class TradingDashboard: y=1.02, xanchor="right", x=1 - ) + ), + # Add JavaScript for client-side data management via config + updatemenus=[{ + 'type': 'buttons', + 'visible': False, + 'buttons': [{ + 'label': 'realtime_data', + 'method': 'skip', + 'args': [{'data': incremental_data}] + }] + }] ) # Update y-axis labels @@ -2100,6 +4429,30 @@ class TradingDashboard: fig.update_yaxes(title_text="Volume", row=2, col=1) fig.update_xaxes(title_text="Time", row=2, col=1) + # Send incremental data to JavaScript cache for client-side merging + import json + incremental_data_json = json.dumps(incremental_data, default=str) + fig.add_annotation( + text=f"""""", + showarrow=False, + x=0, y=0, + xref="paper", yref="paper", + font=dict(size=1), + opacity=0 + ) + return fig except Exception as e: @@ -3581,10 +5934,13 @@ class TradingDashboard: self.ws_price_cache['ETHUSDT'] = price self.current_prices['ETHUSDT'] = price - # Create tick data point for chart + # Create tick data point for chart with proper timezone handling + # Use current local time directly (time.time() is system time) + local_time = self._now_local() + tick = { 'timestamp': current_time, - 'datetime': datetime.now(self.timezone), + 'datetime': local_time, # Use properly converted local time 'symbol': 'ETHUSDT', 'price': price, 'open': float(data.get('o', price)),