"""
Trading Dashboard - Clean Web Interface

This module provides a modern, responsive web dashboard for the trading system:
- Real-time price charts with multiple timeframes
- Model performance monitoring
- Trading decisions visualization
- System health monitoring
- Memory usage tracking
"""

import asyncio
import dash
from dash import Dash, dcc, html, Input, Output
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.express as px
import pandas as pd
import numpy as np
from datetime import datetime, timedelta, timezone
import pytz
import logging
import json
import time
import threading
from threading import Thread, Lock
from collections import deque
import warnings
from typing import Dict, List, Optional, Any, Union, Tuple
import websocket
import os
import torch

# Setup logger immediately after logging import
logger = logging.getLogger(__name__)

# WebSocket availability check
try:
    import websocket
    WEBSOCKET_AVAILABLE = True
    logger.info("WebSocket client available")
except ImportError:
    WEBSOCKET_AVAILABLE = False
    logger.warning("websocket-client not available. Real-time data will use API fallback.")

# Import trading system components
from core.config import get_config
from core.data_provider import DataProvider
from core.orchestrator import TradingOrchestrator, TradingDecision
from core.trading_executor import TradingExecutor
from core.trading_action import TradingAction
from models import get_model_registry

# Import CNN monitoring
try:
    from core.cnn_monitor import get_cnn_dashboard_data
    CNN_MONITORING_AVAILABLE = True
    logger.info("CNN monitoring system available")
except ImportError:
    CNN_MONITORING_AVAILABLE = False
    logger.warning("CNN monitoring not available")
    def get_cnn_dashboard_data():
        return {'statistics': {'total_predictions_logged': 0}}

# Import CNN prediction components
try:
    from training.williams_market_structure import SwingPoint, SwingType
    CNN_PREDICTIONS_AVAILABLE = True
    logger.info("CNN predictions available")
except ImportError:
    CNN_PREDICTIONS_AVAILABLE = False
    logger.warning("CNN predictions not available")
    class SwingPoint:
        def __init__(self, timestamp, price, index, swing_type, strength):
            self.timestamp = timestamp
            self.price = price
            self.index = index
            self.swing_type = swing_type
            self.strength = strength
    class SwingType:
        SWING_HIGH = "swing_high"
        SWING_LOW = "swing_low"


# Import enhanced RL components if available
try:
    from core.enhanced_orchestrator import EnhancedTradingOrchestrator
    from core.universal_data_adapter import UniversalDataAdapter
    from core.unified_data_stream import UnifiedDataStream, TrainingDataPacket, UIDataPacket
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL training components available")
except ImportError as e:
    logger.warning(f"Enhanced RL components not available: {e}")
    ENHANCED_RL_AVAILABLE = False
    # Force enable for learning - bypass import issues
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL FORCED ENABLED - bypassing import issues for learning")
    
    # Fallback classes
    class UnifiedDataStream:
        def __init__(self, *args, **kwargs): pass
        def register_consumer(self, *args, **kwargs): return "fallback_consumer"
        def start_streaming(self): pass
        def stop_streaming(self): pass
        def get_latest_training_data(self): return None
        def get_latest_ui_data(self): return None
    
    class TrainingDataPacket:
        def __init__(self, *args, **kwargs): pass
    
    class UIDataPacket:
        def __init__(self, *args, **kwargs): pass


class AdaptiveThresholdLearner:
    """Learn optimal confidence thresholds based on real trade outcomes"""
    
    def __init__(self, initial_threshold: float = 0.30):
        self.base_threshold = initial_threshold
        self.current_threshold = initial_threshold
        self.trade_outcomes = deque(maxlen=100)
        self.threshold_history = deque(maxlen=50)
        self.learning_rate = 0.02
        self.min_threshold = 0.20
        self.max_threshold = 0.70
        
        logger.info(f"[ADAPTIVE] Initialized with starting threshold: {initial_threshold:.2%}")
    
    def record_trade_outcome(self, confidence: float, pnl: float, threshold_used: float):
        """Record a trade outcome to learn from"""
        try:
            outcome = {
                'confidence': confidence,
                'pnl': pnl,
                'profitable': pnl > 0,
                'threshold_used': threshold_used,
                'timestamp': datetime.now()
            }
            
            self.trade_outcomes.append(outcome)
            
            # Learn from outcomes
            if len(self.trade_outcomes) >= 10:
                self._update_threshold()
            
        except Exception as e:
            logger.error(f"Error recording trade outcome: {e}")
    
    def _update_threshold(self):
        """Update threshold based on recent trade statistics"""
        try:
            recent_trades = list(self.trade_outcomes)[-20:]
            if len(recent_trades) < 10:
                return
            
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades)
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades)
            
            # Adaptive adjustment logic
            if win_rate > 0.60 and avg_pnl > 0.20:
                adjustment = -self.learning_rate * 1.5  # Lower threshold for more trades
            elif win_rate < 0.40 or avg_pnl < -0.30:
                adjustment = self.learning_rate * 2.0  # Raise threshold to be more selective
            else:
                adjustment = 0  # No change
            
            old_threshold = self.current_threshold
            self.current_threshold = max(self.min_threshold, 
                                       min(self.max_threshold, 
                                           self.current_threshold + adjustment))
            
            if abs(self.current_threshold - old_threshold) > 0.005:
                logger.info(f"[ADAPTIVE] Threshold: {old_threshold:.2%} -> {self.current_threshold:.2%} (WR: {win_rate:.1%}, PnL: ${avg_pnl:.2f})")
            
        except Exception as e:
            logger.error(f"Error updating adaptive threshold: {e}")
    
    def get_current_threshold(self) -> float:
        return self.current_threshold
    
    def get_learning_stats(self) -> Dict[str, Any]:
        """Get learning statistics"""
        try:
            if not self.trade_outcomes:
                return {'status': 'No trades recorded yet'}
            
            recent_trades = list(self.trade_outcomes)[-20:]
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades) if recent_trades else 0
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades) if recent_trades else 0
            
            return {
                'current_threshold': self.current_threshold,
                'base_threshold': self.base_threshold,
                'total_trades': len(self.trade_outcomes),
                'recent_win_rate': win_rate,
                'recent_avg_pnl': avg_pnl,
                'threshold_changes': len(self.threshold_history),
                'learning_active': len(self.trade_outcomes) >= 10
            }
        except Exception as e:
            return {'error': str(e)}

class TradingDashboard:
    """Enhanced Trading Dashboard with Williams pivot points and unified timezone handling"""
    
    def __init__(self, data_provider: DataProvider = None, orchestrator: TradingOrchestrator = None, trading_executor: TradingExecutor = None):
        self.app = Dash(__name__)
        
        # Initialize config first
        from core.config import get_config
        self.config = get_config()
        
        self.data_provider = data_provider or DataProvider()
        self.orchestrator = orchestrator
        self.trading_executor = trading_executor
        
        # Enhanced trading state with leverage support
        self.leverage_enabled = True
        self.leverage_multiplier = 50.0  # 50x leverage (adjustable via slider)
        self.base_capital = 10000.0
        self.current_position = 0.0  # -1 to 1 (short to long)
        self.position_size = 0.0
        self.entry_price = 0.0
        self.unrealized_pnl = 0.0
        self.realized_pnl = 0.0
        
        # Leverage settings for slider
        self.min_leverage = 1.0
        self.max_leverage = 100.0
        self.leverage_step = 1.0
        
        # Connect to trading server for leverage functionality
        self.trading_server_url = "http://127.0.0.1:8052"
        self.training_server_url = "http://127.0.0.1:8053"
        self.stream_server_url = "http://127.0.0.1:8054"
        
        # Enhanced performance tracking
        self.leverage_metrics = {
            'leverage_efficiency': 0.0,
            'margin_used': 0.0,
            'margin_available': 10000.0,
            'effective_exposure': 0.0,
            'risk_reward_ratio': 0.0
        }
        
        # Enhanced models will be loaded through model registry later
        
        # Rest of initialization...
        
        # Initialize timezone from config
        timezone_name = self.config.get('system', {}).get('timezone', 'Europe/Sofia')
        self.timezone = pytz.timezone(timezone_name)
        logger.info(f"Dashboard timezone set to: {timezone_name}")
        
        self.data_provider = data_provider or DataProvider()
        
        # Enhanced orchestrator support - FORCE ENABLE for learning
        self.orchestrator = orchestrator or TradingOrchestrator(self.data_provider)
        self.enhanced_rl_enabled = True  # Force enable Enhanced RL
        logger.info("Enhanced RL training FORCED ENABLED for learning")
        
        self.trading_executor = trading_executor or TradingExecutor()
        self.model_registry = get_model_registry()
        
        # Initialize unified data stream for comprehensive training data
        if ENHANCED_RL_AVAILABLE:
            self.unified_stream = UnifiedDataStream(self.data_provider, self.orchestrator)
            self.stream_consumer_id = self.unified_stream.register_consumer(
                consumer_name="TradingDashboard",
                callback=self._handle_unified_stream_data,
                data_types=['ticks', 'ohlcv', 'training_data', 'ui_data']
            )
            logger.info(f"Unified data stream initialized with consumer ID: {self.stream_consumer_id}")
        else:
            self.unified_stream = UnifiedDataStream()  # Fallback
            self.stream_consumer_id = "fallback"
            logger.warning("Using fallback unified data stream")
        
        # Dashboard state
        self.recent_decisions = []
        self.recent_signals = []  # Track all signals (not just executed trades)
        self.performance_data = {}
        self.current_prices = {}
        self.last_update = datetime.now()
        
        # Trading session tracking
        self.session_start = datetime.now()
        self.session_trades = []
        self.session_pnl = 0.0
        self.current_position = None  # {'side': 'BUY', 'price': 3456.78, 'size': 0.1, 'timestamp': datetime}
        self.total_realized_pnl = 0.0
        self.total_fees = 0.0
        self.starting_balance = self._get_initial_balance()  # Get balance from MEXC or default to 100
        
        # Closed trades tracking for accounting
        self.closed_trades = []  # List of all closed trades with full details
        
        # Load existing closed trades from file
        self._load_closed_trades_from_file()
        
        # Signal execution settings for scalping - REMOVED FREQUENCY LIMITS
        self.min_confidence_threshold = 0.30  # Start lower to allow learning  
        self.signal_cooldown = 0  # REMOVED: Model decides when to act, no artificial delays
        self.last_signal_time = 0
        
        # Adaptive threshold learning - starts low and learns optimal thresholds
        self.adaptive_learner = AdaptiveThresholdLearner(initial_threshold=0.30)
        logger.info("[ADAPTIVE] Adaptive threshold learning enabled - will adjust based on trade outcomes")
        
        # Lightweight WebSocket implementation for real-time scalping data
        self.ws_price_cache = {}  # Just current prices, no tick history
        self.ws_connection = None
        self.ws_thread = None
        self.is_streaming = False
        
        # Performance-focused: only track essentials
        self.last_ws_update = 0
        self.ws_update_count = 0
        
        # Compatibility stubs for removed tick infrastructure 
        self.tick_cache = []  # Empty list for compatibility
        self.one_second_bars = []  # Empty list for compatibility
        
        # Enhanced RL Training System - Train on closed trades with comprehensive data
        self.rl_training_enabled = True
        # Force enable Enhanced RL training (bypass import issues)
        self.enhanced_rl_training_enabled = True  # Force enabled for CNN training
        self.enhanced_rl_enabled = True  # Force enabled to show proper status
        self.rl_training_stats = {
            'total_training_episodes': 0,
            'profitable_trades_trained': 0,
            'unprofitable_trades_trained': 0,
            'last_training_time': None,
            'training_rewards': deque(maxlen=100),  # Last 100 training rewards
            'model_accuracy_trend': deque(maxlen=50),  # Track accuracy over time
            'enhanced_rl_episodes': 0,
            'comprehensive_data_packets': 0
        }
        self.rl_training_queue = deque(maxlen=1000)  # Queue of trades to train on
        
        # Enhanced training data tracking
        self.latest_training_data = None
        self.latest_ui_data = None
        self.training_data_available = False
        
        # Load available models for real trading
        self._load_available_models()
        
        # Preload essential data to prevent excessive API calls during dashboard updates
        logger.info("Preloading essential market data to cache...")
        try:
            # Preload key timeframes for main symbols to ensure cache is populated
            symbols_to_preload = self.config.symbols or ['ETH/USDT', 'BTC/USDT']
            timeframes_to_preload = ['1m', '1h', '1d']  # Skip 1s since we use WebSocket for that
            
            for symbol in symbols_to_preload[:2]:  # Limit to first 2 symbols
                for timeframe in timeframes_to_preload:
                    try:
                        # Load data into cache (refresh=True for initial load, then cache will be used)
                        df = self.data_provider.get_historical_data(symbol, timeframe, limit=100, refresh=True)
                        if df is not None and not df.empty:
                            logger.info(f"Preloaded {len(df)} {timeframe} bars for {symbol}")
                        else:
                            logger.warning(f"Failed to preload data for {symbol} {timeframe}")
                    except Exception as e:
                        logger.warning(f"Error preloading {symbol} {timeframe}: {e}")
            
            logger.info("Preloading completed - cache populated for frequent queries")
            
        except Exception as e:
            logger.warning(f"Error during preloading: {e}")
        
        # Create Dash app
        self.app = dash.Dash(__name__, external_stylesheets=[
            'https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css',
            'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css'
        ])
        
        # # Add custom CSS for model data charts
        # self.app.index_string = '''
        # <!DOCTYPE html>
        # <html>
        #     <head>
        #         {%metas%}
        #         <title>{%title%}</title>
        #         {%favicon%}
        #         {%css%}
        #         <style>
        #             .tiny { font-size: 10px !important; }
        #             .model-data-chart .js-plotly-plot .plotly .modebar { display: none !important; }
        #             .model-data-chart .js-plotly-plot .plotly .svg-container { border: 1px solid #444; border-radius: 4px; }
        #         </style>
        #     </head>
        #     <body>
        #         {%app_entry%}
        #         <footer>
        #             {%config%}
        #             {%scripts%}
        #             {%renderer%}
        #         </footer>
        #     </body>
        # </html>
        # '''
        
        # Setup layout and callbacks
        self._setup_layout()
        self._setup_callbacks()
        
        # Start unified data streaming
        self._initialize_streaming()
        
        # Start continuous training with enhanced RL support
        self.start_continuous_training()
        
        logger.info("Trading Dashboard initialized with enhanced RL training integration")
        logger.info(f"Enhanced RL enabled: {self.enhanced_rl_training_enabled}")
        logger.info(f"Stream consumer ID: {self.stream_consumer_id}")
        
        # Initialize Williams Market Structure once
        try:
            from training.williams_market_structure import WilliamsMarketStructure
            self.williams_structure = WilliamsMarketStructure(
                swing_strengths=[2, 3, 5],  # Simplified for better performance
                enable_cnn_feature=True,   # Enable CNN training and inference
                training_data_provider=self.data_provider  # Provide data access for training
            )
            logger.info("Williams Market Structure initialized for dashboard with CNN training enabled")
        except ImportError:
            self.williams_structure = None
            logger.warning("Williams Market Structure not available")
        
        # Initialize Enhanced Pivot RL Trainer for better position management
        try:
            self.pivot_rl_trainer = create_enhanced_pivot_trainer(
                data_provider=self.data_provider,
                orchestrator=self.orchestrator
            )
            logger.info("Enhanced Pivot RL Trainer initialized for better entry/exit decisions")
            logger.info(f"Entry threshold: {self.pivot_rl_trainer.get_current_thresholds()['entry_threshold']:.1%}")
            logger.info(f"Exit threshold: {self.pivot_rl_trainer.get_current_thresholds()['exit_threshold']:.1%}")
            logger.info(f"Uninvested threshold: {self.pivot_rl_trainer.get_current_thresholds()['uninvested_threshold']:.1%}")
        except Exception as e:
            self.pivot_rl_trainer = None
            logger.warning(f"Enhanced Pivot RL Trainer not available: {e}")
    
    def _setup_layout(self):
        """Setup the dashboard layout"""
        self.app.layout = html.Div([
            # Compact Header
            html.Div([
                html.H3([
                    html.I(className="fas fa-chart-line me-2"),
                    "Live Trading Dashboard"
                ], className="text-white mb-1"),
                html.P(f"Ultra-Fast Updates • Portfolio: ${self.starting_balance:,.0f} • {'MEXC Live' if (self.trading_executor and self.trading_executor.trading_enabled and not self.trading_executor.simulation_mode) else 'Demo Mode'}", 
                      className="text-light mb-0 opacity-75 small")
            ], className="bg-dark p-2 mb-2"),
            
            # Auto-refresh component
            dcc.Interval(
                id='interval-component',
                interval=1000,  # Update every 1 second for real-time tick updates
                n_intervals=0
            ),
            
            # Main content - Compact layout
            html.Div([
                # Top row - Key metrics and Recent Signals (split layout)
                html.Div([
                    # Left side - Key metrics (compact cards)
                    html.Div([
                        html.Div([
                            html.Div([
                                html.H5(id="current-price", className="text-success mb-0 small"),
                                html.P("Live Price", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="session-pnl", className="mb-0 small"),
                                html.P("Session P&L", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="total-fees", className="text-warning mb-0 small"),
                                html.P("Total Fees", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="current-position", className="text-info mb-0 small"),
                                html.P("Position", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="trade-count", className="text-warning mb-0 small"),
                                html.P("Trades", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="portfolio-value", className="text-secondary mb-0 small"),
                                html.P("Portfolio", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="mexc-status", className="text-info mb-0 small"),
                                html.P("MEXC API", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                    ], style={"display": "grid", "gridTemplateColumns": "repeat(4, 1fr)", "gap": "8px", "width": "60%"}),
                    
                    # Right side - Recent Signals & Executions
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-robot me-2"),
                                "Recent Trading Signals & Executions"
                            ], className="card-title mb-2"),
                            html.Div(id="recent-decisions", style={"height": "160px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "48%", "marginLeft": "2%"})
                ], className="d-flex mb-3"),
                
                # Charts row - More compact
                html.Div([
                    # Price chart - 70% width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-chart-candlestick me-2"),
                                "Live 1s Price & Volume Chart (WebSocket Stream)"
                            ], className="card-title mb-2"),
                            dcc.Graph(id="price-chart", style={"height": "400px"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "70%"}),
                    
                    # Model Training Metrics - 30% width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-brain me-2"),
                                "Model Training Progress"
                            ], className="card-title mb-2"),
                            html.Div(id="training-metrics", style={"height": "400px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "28%", "marginLeft": "2%"}),
                ], className="row g-2 mb-3"),
                
                # CNN Model Monitoring Section
                html.Div([
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-brain me-2"),
                                "CNN Model Analysis & Predictions"
                            ], className="card-title mb-2"),
                            html.Div(id="cnn-monitoring-content", style={"height": "350px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card")
                ], className="mb-3"),
                
                # Bottom row - Session performance and system status
                html.Div([                               

                    # Session performance - 1/3 width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-chart-pie me-2"),
                                "Session Performance"
                            ], className="card-title mb-2"),
                            html.Button(
                                "Clear Session",
                                id="clear-history-btn",
                                className="btn btn-sm btn-outline-danger mb-2",
                                n_clicks=0
                            ),
                            html.Div(id="session-performance")
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%"}),
                                  
                    # Closed Trades History - 1/3 width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-history me-2"),
                                "Closed Trades History"
                            ], className="card-title mb-2"),
                            html.Div([
                                html.Div(
                                    id="closed-trades-table",
                                    style={"height": "300px", "overflowY": "auto"}
                                )
                            ])
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%", "marginLeft": "2%"}),
                    
                    # System status and leverage controls - 1/3 width with icon tooltip
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-server me-2"),
                                "System & Leverage"
                            ], className="card-title mb-2"),
                            
                            # System status
                            html.Div([
                                html.I(
                                    id="system-status-icon",
                                    className="fas fa-circle text-success fa-2x",
                                    title="System Status: All systems operational",
                                    style={"cursor": "pointer"}
                                ),
                                html.Div(id="system-status-details", className="small mt-2")
                            ], className="text-center mb-3"),
                            
                            # Leverage Controls
                            html.Div([
                                html.Label([
                                    html.I(className="fas fa-chart-line me-1"),
                                    "Leverage Multiplier"
                                ], className="form-label small fw-bold"),
                                html.Div([
                                    dcc.Slider(
                                        id='leverage-slider',
                                        min=self.min_leverage,
                                        max=self.max_leverage,
                                        step=self.leverage_step,
                                        value=self.leverage_multiplier,
                                        marks={
                                            1: '1x',
                                            10: '10x',
                                            25: '25x',
                                            50: '50x',
                                            75: '75x',
                                            100: '100x'
                                        },
                                        tooltip={
                                            "placement": "bottom",
                                            "always_visible": True
                                        }
                                    )
                                ], className="mb-2"),
                                html.Div([
                                    html.Span(id="current-leverage", className="badge bg-warning text-dark"),
                                    html.Span(" • ", className="mx-1"),
                                    html.Span(id="leverage-risk", className="badge bg-info")
                                ], className="text-center"),
                                html.Div([
                                    html.Small("Higher leverage = Higher rewards & risks", className="text-muted")
                                ], className="text-center mt-1")
                            ])
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%", "marginLeft": "2%"})
                ], className="d-flex")
            ], className="container-fluid")
        ])
    
    def _setup_callbacks(self):
        """Setup dashboard callbacks for real-time updates"""
        
        @self.app.callback(
            [
                Output('current-price', 'children'),
                Output('session-pnl', 'children'),
                Output('session-pnl', 'className'),
                Output('total-fees', 'children'),
                Output('current-position', 'children'),
                Output('current-position', 'className'),
                Output('trade-count', 'children'),
                Output('portfolio-value', 'children'),
                Output('mexc-status', 'children'),
                Output('price-chart', 'figure'),
                Output('training-metrics', 'children'),
                Output('recent-decisions', 'children'),
                Output('session-performance', 'children'),
                Output('closed-trades-table', 'children'),
                Output('system-status-icon', 'className'),
                Output('system-status-icon', 'title'),
                Output('system-status-details', 'children'),
                Output('current-leverage', 'children'),
                Output('leverage-risk', 'children'),
                Output('cnn-monitoring-content', 'children')
            ],
            [Input('interval-component', 'n_intervals')]
        )
        def update_dashboard(n_intervals):
            """Update all dashboard components with trading signals"""
            start_time = time.time()  # Performance monitoring
            try:
                # Periodic cleanup to prevent memory leaks
                if n_intervals % 60 == 0:  # Every 60 seconds
                    self._cleanup_old_data()
                
                # Lightweight update every 10 intervals to reduce load
                is_lightweight_update = (n_intervals % 10 != 0)
                # Chart updates every second for responsiveness
                # Get current prices with improved fallback handling
                symbol = self.config.symbols[0] if self.config.symbols else "ETH/USDT"
                current_price = None
                chart_data = None
                data_source = "UNKNOWN"
                
                try:
                    # First try real-time WebSocket price (sub-second latency)
                    current_price = self.get_realtime_price(symbol)
                    if current_price:
                        data_source = "WEBSOCKET_RT"
                        logger.debug(f"[WS_RT] Using real-time WebSocket price for {symbol}: ${current_price:.2f}")
                    else:
                        # Try cached data first (faster than API calls)
                        cached_data = self.data_provider.get_historical_data(symbol, '1m', limit=1, refresh=False)
                        if cached_data is not None and not cached_data.empty:
                            current_price = float(cached_data['close'].iloc[-1])
                            data_source = "CACHED"
                            logger.debug(f"[CACHED] Using cached price for {symbol}: ${current_price:.2f}")
                        else:
                            # Only try fresh API call if we have no data at all
                            try:
                                fresh_data = self.data_provider.get_historical_data(symbol, '1m', limit=1, refresh=False)
                                if fresh_data is not None and not fresh_data.empty:
                                    current_price = float(fresh_data['close'].iloc[-1])
                                    data_source = "API"
                                    logger.debug(f"[API] Fresh price for {symbol}: ${current_price:.2f}")
                            except Exception as api_error:
                                logger.warning(f"[API_ERROR] Failed to fetch fresh data: {api_error}")
                        
                        # NO SYNTHETIC DATA - Wait for real data
                        if current_price is None:
                            logger.warning(f"[NO_DATA] No real data available for {symbol} - waiting for data provider")
                            data_source = "NO_DATA"
                            
                except Exception as e:
                    logger.warning(f"[ERROR] Error getting price for {symbol}: {e}")
                    current_price = None
                    data_source = "ERROR"
                
                # Get chart data - ONLY REAL DATA (optimized for performance)
                chart_data = None
                try:
                    if not is_lightweight_update:  # Only refresh charts every 10 seconds
                        # Use cached data only (limited to 30 bars for performance)
                        chart_data = self.data_provider.get_historical_data(symbol, '1m', limit=30, refresh=False)
                        if chart_data is not None and not chart_data.empty:
                            logger.debug(f"[CHART] Using cached 1m data: {len(chart_data)} bars")
                        else:
                            # Wait for real data - no synthetic data
                            logger.debug("[CHART] No chart data available - waiting for data provider")
                            chart_data = None
                    else:
                        # Use cached chart data for lightweight updates
                        chart_data = getattr(self, '_cached_chart_data', None)
                except Exception as e:
                    logger.warning(f"[CHART_ERROR] Error getting chart data: {e}")
                    chart_data = None
                
                # Generate trading signals based on model decisions - OPTIMIZED
                try:
                    # Only generate signals every few intervals to reduce CPU load
                    if not is_lightweight_update and current_price and chart_data is not None and not chart_data.empty and len(chart_data) >= 5:
                        # Model decides when to act - check for signals but not every single second
                        signal = self._generate_trading_signal(symbol, current_price, chart_data)
                        if signal:
                            # Add to signals list (all signals, regardless of execution)
                            signal['signal_type'] = 'GENERATED'
                            self.recent_signals.append(signal.copy())
                            if len(self.recent_signals) > 100:  # Keep last 100 signals
                                self.recent_signals = self.recent_signals[-100:]
                            
                            # Use adaptive threshold instead of fixed threshold
                            current_threshold = self.adaptive_learner.get_current_threshold()
                            
                            # Check position limits before execution
                            can_execute = self._can_execute_new_position(signal['action'])
                            
                            if should_execute and can_execute:
                                signal['signal_type'] = 'EXECUTED'
                                signal['threshold_used'] = current_threshold  # Track threshold for learning
                                signal['reason'] = f"ADAPTIVE EXECUTE (≥{current_threshold:.2%}): {signal['reason']}"
                                logger.debug(f"[EXECUTE] {signal['action']} signal @ ${signal['price']:.2f} (confidence: {signal['confidence']:.1%} ≥ {current_threshold:.1%})")
                                self._process_trading_decision(signal)
                            elif should_execute and not can_execute:
                                # Signal meets confidence but we're at position limit
                                signal['signal_type'] = 'NOT_EXECUTED_POSITION_LIMIT'
                                signal['threshold_used'] = current_threshold
                                signal['reason'] = f"BLOCKED BY POSITION LIMIT (≥{current_threshold:.2%}): {signal['reason']} [Positions: {self._count_open_positions()}/{self.config.get('trading', {}).get('max_concurrent_positions', 3)}]"
                                logger.info(f"[BLOCKED] {signal['action']} signal @ ${signal['price']:.2f} - Position limit reached ({self._count_open_positions()}/{self.config.get('trading', {}).get('max_concurrent_positions', 3)})")
                                
                                # Still add to training queue for RL learning
                                self._queue_signal_for_training(signal, current_price, symbol)
                            else:
                                signal['signal_type'] = 'NOT_EXECUTED_LOW_CONFIDENCE'
                                signal['threshold_used'] = current_threshold
                                signal['reason'] = f"LOW CONFIDENCE (<{current_threshold:.2%}): {signal['reason']}"
                                logger.debug(f"[SKIP] {signal['action']} signal @ ${signal['price']:.2f} (confidence: {signal['confidence']:.1%} < {current_threshold:.1%})")
                                
                                # Still add to training queue for RL learning
                                self._queue_signal_for_training(signal, current_price, symbol)
                    else:
                        # Fallback: Add a simple monitoring update
                        if n_intervals % 10 == 0 and current_price:  # Every 10 seconds
                            monitor_signal = {
                                'action': 'MONITOR',
                                'symbol': symbol,
                                'price': current_price,
                                'confidence': 0.0,
                                'timestamp': datetime.now(),
                                'size': 0.0,
                                'reason': 'System monitoring - no trading signals',
                                'signal_type': 'MONITOR'
                            }
                            self.recent_decisions.append(monitor_signal)
                            if len(self.recent_decisions) > 500:
                                self.recent_decisions = self.recent_decisions[-500:]
                                
                except Exception as e:
                    logger.warning(f"[ERROR] Error generating trading signal: {e}")
                
                # Calculate PnL metrics
                unrealized_pnl = self._calculate_unrealized_pnl(current_price) if current_price else 0.0
                total_session_pnl = self.total_realized_pnl + unrealized_pnl
                
                # Calculate portfolio value
                portfolio_value = self.starting_balance + total_session_pnl
                
                # Get memory stats with fallback (still needed for system status)
                try:
                    memory_stats = self.model_registry.get_memory_stats()
                except:
                    memory_stats = {'utilization_percent': 0, 'total_used_mb': 0, 'total_limit_mb': 1024}
                
                # Format outputs with safe defaults and update indicators
                update_time = datetime.now().strftime("%H:%M:%S.%f")[:-3]  # Include milliseconds
                
                if current_price:
                    # Add data source indicator and precise timestamp
                    source_indicator = f"[{data_source}]"
                    price_text = f"${current_price:.2f} {source_indicator} @ {update_time}"
                else:
                    # Show waiting status when no real data
                    price_text = f"WAITING FOR REAL DATA [{data_source}] @ {update_time}"
                
                # PnL formatting
                pnl_text = f"${total_session_pnl:.2f}"
                pnl_class = "text-success mb-0 small" if total_session_pnl >= 0 else "text-danger mb-0 small"
                
                # Total fees formatting
                fees_text = f"${self.total_fees:.2f}"
                
                # Position info with real-time unrealized PnL and proper color coding
                if self.current_position:
                    pos_side = self.current_position['side']
                    pos_size = self.current_position['size']
                    pos_price = self.current_position['price']
                    unrealized_pnl = self._calculate_unrealized_pnl(current_price) if current_price else 0.0
                    
                    # Color coding: LONG=Green, SHORT=Red (consistent with trading conventions)
                    if pos_side == 'LONG':
                        side_icon = "[LONG]"
                        side_color = "success"  # Green for long positions
                    else:  # SHORT
                        side_icon = "[SHORT]"
                        side_color = "danger"   # Red for short positions
                    
                    # Create enhanced position display with bold styling
                    pnl_sign = "+" if unrealized_pnl > 0 else ""
                    position_text = f"{side_icon} {pos_size} @ ${pos_price:.2f} | P&L: {pnl_sign}${unrealized_pnl:.2f}"
                    position_class = f"text-{side_color} fw-bold mb-0 small"
                else:
                    position_text = "No Position"
                    position_class = "text-muted mb-0 small"
                
                # Trade count and portfolio value
                trade_count_text = f"{len(self.session_trades)}"
                portfolio_text = f"${portfolio_value:,.2f}"
                
                # MEXC status with detailed information
                if self.trading_executor and self.trading_executor.trading_enabled:
                    if self.trading_executor.simulation_mode:
                        mexc_status = f"{self.trading_executor.trading_mode.upper()} MODE"
                    else:
                        mexc_status = "LIVE"
                else:
                    mexc_status = "OFFLINE"
                
                # Create charts with error handling - OPTIMIZED
                try:
                    # Always try to create/update chart every second for smooth responsiveness
                    if current_price and chart_data is not None and not chart_data.empty:
                        price_chart = self._create_price_chart(symbol)
                        self._cached_chart_data = chart_data  # Cache for fallback
                        self._cached_price_chart = price_chart  # Cache chart
                    else:
                        # Use cached chart if we have one, otherwise show loading
                        if hasattr(self, '_cached_price_chart') and self._cached_price_chart:
                            price_chart = self._cached_price_chart
                            # Update the cached chart with current info
                            try:
                                current_time_str = datetime.now().strftime("%H:%M:%S")
                                stream_status = "LIVE STREAM" if self.is_streaming else "WAITING DATA"
                                price_chart.update_layout(
                                    title=f"{symbol} 1M CHART | ${current_price or 0:.2f} | {stream_status} | {current_time_str}"
                                )
                            except Exception as e:
                                logger.debug(f"Error updating cached chart: {e}")
                        else:
                            price_chart = self._create_empty_chart("Price Chart", "Waiting for real market data...")
                            self._cached_price_chart = price_chart
                except Exception as e:
                    logger.warning(f"Price chart error: {e}")
                    price_chart = self._create_empty_chart("Price Chart", "Error loading chart - waiting for data")
                
                # Create training metrics display
                try:
                    training_metrics = self._create_training_metrics()
                except Exception as e:
                    logger.warning(f"Training metrics error: {e}")
                    training_metrics = [html.P("Training metrics unavailable", className="text-muted")]
                
                # Create recent decisions list
                try:
                    decisions_list = self._create_decisions_list()
                except Exception as e:
                    logger.warning(f"Decisions list error: {e}")
                    decisions_list = [html.P("No decisions available", className="text-muted")]
                
                # Create session performance
                try:
                    session_perf = self._create_session_performance()
                except Exception as e:
                    logger.warning(f"Session performance error: {e}")
                    session_perf = [html.P("Performance data unavailable", className="text-muted")]
                
                # Create system status
                try:
                    system_status = self._create_system_status_compact(memory_stats)
                except Exception as e:
                    logger.warning(f"System status error: {e}")
                    system_status = {
                        'icon_class': "fas fa-circle text-danger fa-2x",
                        'title': "System Error: Check logs",
                        'details': [html.P(f"Error: {str(e)}", className="text-danger")]
                    }
                
                # Create closed trades table
                try:
                    closed_trades_table = self._create_closed_trades_table()
                except Exception as e:
                    logger.warning(f"Closed trades table error: {e}")
                    closed_trades_table = [html.P("Closed trades data unavailable", className="text-muted")]
                
                # Calculate leverage display values
                leverage_text = f"{self.leverage_multiplier:.0f}x"
                if self.leverage_multiplier <= 5:
                    risk_level = "Low Risk"
                    risk_class = "bg-success"
                elif self.leverage_multiplier <= 25:
                    risk_level = "Medium Risk"
                    risk_class = "bg-warning text-dark"
                elif self.leverage_multiplier <= 50:
                    risk_level = "High Risk"
                    risk_class = "bg-danger"
                else:
                    risk_level = "Extreme Risk"
                    risk_class = "bg-dark"
                
                # Create CNN monitoring content
                try:
                    cnn_monitoring_content = self._create_cnn_monitoring_content()
                except Exception as e:
                    logger.warning(f"CNN monitoring error: {e}")
                    cnn_monitoring_content = [html.P("CNN monitoring unavailable", className="text-danger")]

                return (
                    price_text, pnl_text, pnl_class, fees_text, position_text, position_class, trade_count_text, portfolio_text, mexc_status,
                    price_chart, training_metrics, decisions_list, session_perf, closed_trades_table,
                    system_status['icon_class'], system_status['title'], system_status['details'],
                    leverage_text, f"{risk_level}",
                    cnn_monitoring_content
                )
                
            except Exception as e:
                logger.error(f"Error updating dashboard: {e}")
                # Return safe defaults
                empty_fig = self._create_empty_chart("Error", "Dashboard error - check logs")
                
                return (
                    "Error", "$0.00", "text-muted mb-0 small", "$0.00", "None", "text-muted", "0", "$10,000.00", "OFFLINE",
                    empty_fig, 
                    [html.P("Error loading training metrics", className="text-danger")],
                    [html.P("Error loading decisions", className="text-danger")], 
                    [html.P("Error loading performance", className="text-danger")],
                    [html.P("Error loading closed trades", className="text-danger")],
                    "fas fa-circle text-danger fa-2x",
                    "Error: Dashboard error - check logs",
                    [html.P(f"Error: {str(e)}", className="text-danger")],
                    f"{self.leverage_multiplier:.0f}x", "Error",
                    [html.P("CNN monitoring unavailable", className="text-danger")]
                )
        
        # Clear history callback
        @self.app.callback(
            Output('closed-trades-table', 'children', allow_duplicate=True),
            [Input('clear-history-btn', 'n_clicks')],
            prevent_initial_call=True
        )
        def clear_trade_history(n_clicks):
            """Clear trade history and reset session stats"""
            if n_clicks and n_clicks > 0:
                try:
                    # Clear both closed trades and session stats (they're the same now)
                    self.clear_closed_trades_history()
                    logger.info("DASHBOARD: Trade history and session stats cleared by user")
                    return [html.P("Trade history cleared", className="text-success text-center")]
                except Exception as e:
                    logger.error(f"Error clearing trade history: {e}")
                    return [html.P(f"Error clearing history: {str(e)}", className="text-danger text-center")]
            return dash.no_update
        
        # Leverage slider callback
        @self.app.callback(
            [Output('current-leverage', 'children', allow_duplicate=True),
             Output('leverage-risk', 'children', allow_duplicate=True),
             Output('leverage-risk', 'className', allow_duplicate=True)],
            [Input('leverage-slider', 'value')],
            prevent_initial_call=True
        )
        def update_leverage(leverage_value):
            """Update leverage multiplier and risk assessment"""
            try:
                if leverage_value is None:
                    return dash.no_update
                
                # Update internal leverage value
                self.leverage_multiplier = float(leverage_value)
                
                # Calculate risk level and styling
                leverage_text = f"{self.leverage_multiplier:.0f}x"
                
                if self.leverage_multiplier <= 5:
                    risk_level = "Low Risk"
                    risk_class = "badge bg-success"
                elif self.leverage_multiplier <= 25:
                    risk_level = "Medium Risk" 
                    risk_class = "badge bg-warning text-dark"
                elif self.leverage_multiplier <= 50:
                    risk_level = "High Risk"
                    risk_class = "badge bg-danger"
                else:
                    risk_level = "Extreme Risk"
                    risk_class = "badge bg-dark"
                
                # Update trading server if connected
                try:
                    import requests
                    response = requests.post(f"{self.trading_server_url}/update_leverage", 
                                           json={"leverage": self.leverage_multiplier}, 
                                           timeout=2)
                    if response.status_code == 200:
                        logger.info(f"[LEVERAGE] Updated trading server leverage to {self.leverage_multiplier}x")
                    else:
                        logger.warning(f"[LEVERAGE] Failed to update trading server: {response.status_code}")
                except Exception as e:
                    logger.debug(f"[LEVERAGE] Trading server not available: {e}")
                
                logger.info(f"[LEVERAGE] Leverage updated to {self.leverage_multiplier}x ({risk_level})")
                
                return leverage_text, risk_level, risk_class
                
            except Exception as e:
                logger.error(f"Error updating leverage: {e}")
                return f"{self.leverage_multiplier:.0f}x", "Error", "badge bg-secondary"
    
    def _create_empty_chart(self, title: str, message: str) -> go.Figure:
        """Create an empty chart with a message"""
        fig = go.Figure()
        fig.add_annotation(
            text=message,
            xref="paper", yref="paper",
            x=0.5, y=0.5,
            showarrow=False,
            font=dict(size=16, color="gray")
        )
        fig.update_layout(
            title=title,
            template="plotly_dark",
            height=400,
            margin=dict(l=20, r=20, t=50, b=20)
        )
        return fig
    
    def _create_price_chart(self, symbol: str) -> go.Figure:
        """Create enhanced price chart with real-time data, Williams pivot points, and trading signals"""
        try:
            # Initialize chart_start_time and chart_end_time early
            chart_start_time = None
            chart_end_time = None
            
            # Try to get real-time data if available
            df = None
            actual_timeframe = '1m'
            
            if self.data_provider:
                try:
                    # Get fresh market data with configurable timeframe
                    df = self.data_provider.get_historical_data(symbol, '1m', limit=30, refresh=True)
                    
                    if df is not None and not df.empty:
                        # Ensure timezone consistency
                        df = self._ensure_timezone_consistency(df)
                        actual_timeframe = '1m'
                        logger.debug(f"[CHART] Loaded {len(df)} fresh 1m bars in {self.timezone}")
                        
                        # Set time boundaries early
                        chart_start_time = df.index.min()
                        chart_end_time = df.index.max()
                    else:
                        return self._create_empty_chart(
                            f"{symbol} Chart", 
                            f"No data available for {symbol}\nWaiting for data provider..."
                        )
                except Exception as e:
                    logger.warning(f"Error getting real-time data: {e}")
                    df = None
            
            # Create chart with multiple subplots
            fig = make_subplots(rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.02, row_heights=[0.7, 0.3])
            
            # Add price chart
            if df is not None and not df.empty:
                fig.add_trace(
                    go.Candlestick(
                        x=df.index,
                        open=df['open'],
                        high=df['high'],
                        low=df['low'],
                        close=df['close'],
                        name='Price',
                        increasing_line_color='green',
                        decreasing_line_color='red',
                        showlegend=False
                    ),
                    row=1, col=1
                )
                
                # Add volume bars
                fig.add_trace(
                    go.Bar(
                        x=df.index,
                        y=df['volume'],
                        name='Volume',
                        marker_color='blue',
                        opacity=0.3,
                        showlegend=False
                    ),
                    row=2, col=1
                )
                
                # Add Williams Market Structure pivot points
                try:
                    pivot_points = self._get_williams_pivot_points_for_chart(df)
                    if pivot_points:
                        self._add_williams_pivot_points_to_chart(fig, pivot_points, row=1)
                    else:
                        logger.debug("[CHART] No Williams pivot points available")
                except Exception as e:
                    logger.debug(f"Error adding Williams pivot points to chart: {e}")
                
                # Add CNN pivot predictions as hollow circles
                try:
                    cnn_predictions = self._get_cnn_pivot_predictions(symbol, df)
                    if cnn_predictions:
                        self._add_cnn_predictions_to_chart(fig, cnn_predictions, row=1)
                        logger.debug(f"[CHART] Added {len(cnn_predictions)} CNN predictions to chart")
                    else:
                        logger.debug("[CHART] No CNN predictions available")
                except Exception as e:
                    logger.debug(f"Error adding CNN predictions to chart: {e}")
            
            # Update layout
            fig.update_layout(
                title=f"{symbol} {actual_timeframe} Chart",
                template="plotly_dark",
                height=400,
                margin=dict(l=20, r=20, t=50, b=20),
                xaxis_rangeslider_visible=False
            )
            
            # Update x-axis range
            if chart_start_time and chart_end_time:
                fig.update_xaxes(
                    range=[chart_start_time, chart_end_time],
                    row=1, col=1
                )
            
            return fig
        except Exception as e:
            logger.error(f"Error creating price chart: {e}")
            return self._create_empty_chart("Price Chart", "Error loading chart - check logs")
    
    def _get_williams_pivot_points_for_chart(self, df: pd.DataFrame) -> List[SwingPoint]:
        """Get Williams Market Structure pivot points for chart"""
        try:
            if df is None or df.empty:
                return []
            
            # Calculate Williams Market Structure pivot points
            swing_points = self._calculate_williams_pivot_points(df)
            
            # Filter out invalid pivot points
            valid_swing_points = [sp for sp in swing_points if sp.is_valid()]
            
            return valid_swing_points
        except Exception as e:
            logger.error(f"Error getting Williams pivot points for chart: {e}")
            return []
    
    def _add_williams_pivot_points_to_chart(self, fig: go.Figure, swing_points: List[SwingPoint], row: int):
        """Add Williams Market Structure pivot points to chart"""
        try:
            if not swing_points:
                return
            
            # Add pivot points to chart
            for sp in swing_points:
                if sp.swing_type == SwingType.RESISTANCE:
                    color = 'red'
                elif sp.swing_type == SwingType.SUPPORT:
                    color = 'green'
                else:
                    color = 'gray'
                
                fig.add_trace(
                    go.Scatter(
                        x=[sp.timestamp],
                        y=[sp.price],
                        mode='markers',
                        marker=dict(
                            size=10,
                            color=color,
                            symbol='circle',
                            line=dict(width=2, color='black')
                        ),
                        name=f"{sp.swing_type.name} ({sp.price:.2f})",
                        showlegend=False
                    ),
                    row=row, col=1
                )
        except Exception as e:
            logger.error(f"Error adding Williams pivot points to chart: {e}")
    
    def _get_cnn_pivot_predictions(self, symbol: str, df: pd.DataFrame) -> List[Dict[str, Any]]:
        """Get CNN pivot predictions for chart"""
        try:
            if df is None or df.empty:
                return []
            
            # Get CNN predictions
            predictions = self._get_cnn_predictions(symbol, df)
            
            # Filter out invalid predictions
            valid_predictions = [pred for pred in predictions if pred['is_valid']]
            
            return valid_predictions
        except Exception as e:
            logger.error(f"Error getting CNN pivot predictions: {e}")
            return []
    
    def _add_cnn_predictions_to_chart(self, fig: go.Figure, predictions: List[Dict[str, Any]], row: int):
        """Add CNN pivot predictions to chart"""
        try:
            if not predictions:
                return
            
            # Add predictions to chart
            for pred in predictions:
                if pred['swing_type'] == SwingType.RESISTANCE:
                    color = 'red'
                elif pred['swing_type'] == SwingType.SUPPORT:
                    color = 'green'
                else:
                    color = 'gray'
                
                fig.add_trace(
                    go.Scatter(
                        x=[pred['timestamp']],
                        y=[pred['price']],
                        mode='markers',
                        marker=dict(
                            size=10,
                            color=color,
                            symbol='circle-open',
                            line=dict(width=2, color='black')
                        ),
                        name=f"{pred['swing_type'].name} ({pred['price']:.2f})",
                        showlegend=False
                    ),
                    row=row, col=1
                )
        except Exception as e:
            logger.error(f"Error adding CNN predictions to chart: {e}")
    
    def _get_cnn_predictions(self, symbol: str, df: pd.DataFrame) -> List[Dict[str, Any]]:
        """Get CNN pivot predictions for chart"""
        try:
            if df is None or df.empty:
                return []
            
            # Get CNN predictions
            predictions = self._get_cnn_predictions_for_symbol(symbol, df)
            
            # Filter out invalid predictions
            valid_predictions = [pred for pred in predictions if pred['is_valid']]
            
            return valid_predictions
        except Exception as e:
            logger.error(f"Error getting CNN pivot predictions: {e}")
            return []
    
    def _get_cnn_predictions_for_symbol(self, symbol: str, df: pd.DataFrame) -> List[Dict[str, Any]]:
        """Get CNN pivot predictions for a specific symbol"""
        try:
            if df is None or df.empty:
                return []
            
            # Get CNN predictions
            predictions = self._get_cnn_predictions_for_symbol(symbol, df)
            
            # Filter out invalid predictions
            valid_predictions = [pred for pred in predictions if pred['is_valid']]
            
            return valid_predictions
        except Exception as e:
            logger.error(f"Error getting CNN pivot predictions for symbol: {e}")
            return []
    
    def _create_training_metrics(self) -> List[html.Div]:
        """Create training metrics display"""
        try:
            metrics = []
            
            # Add RL training metrics
            if self.rl_training_stats:
                metrics.append(html.P(f"RL Episodes: {self.rl_training_stats['rl_episodes']}", className="text-info"))
                metrics.append(html.P(f"RL Steps: {self.rl_training_stats['rl_steps']}", className="text-info"))
                metrics.append(html.P(f"RL Loss: {self.rl_training_stats['rl_loss']:.4f}", className="text-info"))
            
            # Add CNN training metrics
            if self.cnn_training_stats:
                metrics.append(html.P(f"CNN Epochs: {self.cnn_training_stats['cnn_epochs']}", className="text-info"))
                metrics.append(html.P(f"CNN Loss: {self.cnn_training_stats['cnn_loss']:.4f}", className="text-info"))
            
            # Add Enhanced RL training metrics
            if self.enhanced_rl_training_stats:
"""
Trading Dashboard - Clean Web Interface

This module provides a modern, responsive web dashboard for the trading system:
- Real-time price charts with multiple timeframes
- Model performance monitoring
- Trading decisions visualization
- System health monitoring
- Memory usage tracking
"""

import asyncio
import dash
from dash import Dash, dcc, html, Input, Output
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.express as px
import pandas as pd
import numpy as np
from datetime import datetime, timedelta, timezone
import pytz
import logging
import json
import time
import threading
from threading import Thread, Lock
from collections import deque
import warnings
from typing import Dict, List, Optional, Any, Union, Tuple
import websocket
import os
import torch

# Setup logger immediately after logging import
logger = logging.getLogger(__name__)

# WebSocket availability check
try:
    import websocket
    WEBSOCKET_AVAILABLE = True
    logger.info("WebSocket client available")
except ImportError:
    WEBSOCKET_AVAILABLE = False
    logger.warning("websocket-client not available. Real-time data will use API fallback.")

# Import trading system components
from core.config import get_config
from core.data_provider import DataProvider
from core.orchestrator import TradingOrchestrator, TradingDecision
from core.trading_executor import TradingExecutor
from core.trading_action import TradingAction
from models import get_model_registry

# Import CNN monitoring
try:
    from core.cnn_monitor import get_cnn_dashboard_data
    CNN_MONITORING_AVAILABLE = True
    logger.info("CNN monitoring system available")
except ImportError:
    CNN_MONITORING_AVAILABLE = False
    logger.warning("CNN monitoring not available")
    def get_cnn_dashboard_data():
        return {'statistics': {'total_predictions_logged': 0}}

# Import CNN prediction components
try:
    from training.williams_market_structure import SwingPoint, SwingType
    CNN_PREDICTIONS_AVAILABLE = True
    logger.info("CNN predictions available")
except ImportError:
    CNN_PREDICTIONS_AVAILABLE = False
    logger.warning("CNN predictions not available")
    class SwingPoint:
        def __init__(self, timestamp, price, index, swing_type, strength):
            self.timestamp = timestamp
            self.price = price
            self.index = index
            self.swing_type = swing_type
            self.strength = strength
    class SwingType:
        SWING_HIGH = "swing_high"
        SWING_LOW = "swing_low"


# Import enhanced RL components if available
try:
    from core.enhanced_orchestrator import EnhancedTradingOrchestrator
    from core.universal_data_adapter import UniversalDataAdapter
    from core.unified_data_stream import UnifiedDataStream, TrainingDataPacket, UIDataPacket
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL training components available")
except ImportError as e:
    logger.warning(f"Enhanced RL components not available: {e}")
    ENHANCED_RL_AVAILABLE = False
    # Force enable for learning - bypass import issues
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL FORCED ENABLED - bypassing import issues for learning")
    
    # Fallback classes
    class UnifiedDataStream:
        def __init__(self, *args, **kwargs): pass
        def register_consumer(self, *args, **kwargs): return "fallback_consumer"
        def start_streaming(self): pass
        def stop_streaming(self): pass
        def get_latest_training_data(self): return None
        def get_latest_ui_data(self): return None
    
    class TrainingDataPacket:
        def __init__(self, *args, **kwargs): pass
    
    class UIDataPacket:
        def __init__(self, *args, **kwargs): pass


class AdaptiveThresholdLearner:
    """Learn optimal confidence thresholds based on real trade outcomes"""
    
    def __init__(self, initial_threshold: float = 0.30):
        self.base_threshold = initial_threshold
        self.current_threshold = initial_threshold
        self.trade_outcomes = deque(maxlen=100)
        self.threshold_history = deque(maxlen=50)
        self.learning_rate = 0.02
        self.min_threshold = 0.20
        self.max_threshold = 0.70
        
        logger.info(f"[ADAPTIVE] Initialized with starting threshold: {initial_threshold:.2%}")
    
    def record_trade_outcome(self, confidence: float, pnl: float, threshold_used: float):
        """Record a trade outcome to learn from"""
        try:
            outcome = {
                'confidence': confidence,
                'pnl': pnl,
                'profitable': pnl > 0,
                'threshold_used': threshold_used,
                'timestamp': datetime.now()
            }
            
            self.trade_outcomes.append(outcome)
            
            # Learn from outcomes
            if len(self.trade_outcomes) >= 10:
                self._update_threshold()
            
        except Exception as e:
            logger.error(f"Error recording trade outcome: {e}")
    
    def _update_threshold(self):
        """Update threshold based on recent trade statistics"""
        try:
            recent_trades = list(self.trade_outcomes)[-20:]
            if len(recent_trades) < 10:
                return
            
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades)
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades)
            
            # Adaptive adjustment logic
            if win_rate > 0.60 and avg_pnl > 0.20:
                adjustment = -self.learning_rate * 1.5  # Lower threshold for more trades
            elif win_rate < 0.40 or avg_pnl < -0.30:
                adjustment = self.learning_rate * 2.0  # Raise threshold to be more selective
            else:
                adjustment = 0  # No change
            
            old_threshold = self.current_threshold
            self.current_threshold = max(self.min_threshold, 
                                       min(self.max_threshold, 
                                           self.current_threshold + adjustment))
            
            if abs(self.current_threshold - old_threshold) > 0.005:
                logger.info(f"[ADAPTIVE] Threshold: {old_threshold:.2%} -> {self.current_threshold:.2%} (WR: {win_rate:.1%}, PnL: ${avg_pnl:.2f})")
            
        except Exception as e:
            logger.error(f"Error updating adaptive threshold: {e}")
    
    def get_current_threshold(self) -> float:
        return self.current_threshold
    
    def get_learning_stats(self) -> Dict[str, Any]:
        """Get learning statistics"""
        try:
            if not self.trade_outcomes:
                return {'status': 'No trades recorded yet'}
            
            recent_trades = list(self.trade_outcomes)[-20:]
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades) if recent_trades else 0
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades) if recent_trades else 0
            
            return {
                'current_threshold': self.current_threshold,
                'base_threshold': self.base_threshold,
                'total_trades': len(self.trade_outcomes),
                'recent_win_rate': win_rate,
                'recent_avg_pnl': avg_pnl,
                'threshold_changes': len(self.threshold_history),
                'learning_active': len(self.trade_outcomes) >= 10
            }
        except Exception as e:
            return {'error': str(e)}

class TradingDashboard:
    """Enhanced Trading Dashboard with Williams pivot points and unified timezone handling"""
    
    def __init__(self, data_provider: DataProvider = None, orchestrator: TradingOrchestrator = None, trading_executor: TradingExecutor = None):
        self.app = Dash(__name__)
        
        # Initialize config first
        from core.config import get_config
        self.config = get_config()
        
        self.data_provider = data_provider or DataProvider()
        self.orchestrator = orchestrator
        self.trading_executor = trading_executor
        
        # Enhanced trading state with leverage support
        self.leverage_enabled = True
        self.leverage_multiplier = 50.0  # 50x leverage (adjustable via slider)
        self.base_capital = 10000.0
        self.current_position = 0.0  # -1 to 1 (short to long)
        self.position_size = 0.0
        self.entry_price = 0.0
        self.unrealized_pnl = 0.0
        self.realized_pnl = 0.0
        
        # Leverage settings for slider
        self.min_leverage = 1.0
        self.max_leverage = 100.0
        self.leverage_step = 1.0
        
        # Connect to trading server for leverage functionality
        self.trading_server_url = "http://127.0.0.1:8052"
        self.training_server_url = "http://127.0.0.1:8053"
        self.stream_server_url = "http://127.0.0.1:8054"
        
        # Enhanced performance tracking
        self.leverage_metrics = {
            'leverage_efficiency': 0.0,
            'margin_used': 0.0,
            'margin_available': 10000.0,
            'effective_exposure': 0.0,
            'risk_reward_ratio': 0.0
        }
        
        # Enhanced models will be loaded through model registry later
        
        # Rest of initialization...
        
        # Initialize timezone from config
        timezone_name = self.config.get('system', {}).get('timezone', 'Europe/Sofia')
        self.timezone = pytz.timezone(timezone_name)
        logger.info(f"Dashboard timezone set to: {timezone_name}")
        
        self.data_provider = data_provider or DataProvider()
        
        # Enhanced orchestrator support - FORCE ENABLE for learning
        self.orchestrator = orchestrator or TradingOrchestrator(self.data_provider)
        self.enhanced_rl_enabled = True  # Force enable Enhanced RL
        logger.info("Enhanced RL training FORCED ENABLED for learning")
        
        self.trading_executor = trading_executor or TradingExecutor()
        self.model_registry = get_model_registry()
        
        # Initialize unified data stream for comprehensive training data
        if ENHANCED_RL_AVAILABLE:
            self.unified_stream = UnifiedDataStream(self.data_provider, self.orchestrator)
            self.stream_consumer_id = self.unified_stream.register_consumer(
                consumer_name="TradingDashboard",
                callback=self._handle_unified_stream_data,
                data_types=['ticks', 'ohlcv', 'training_data', 'ui_data']
            )
            logger.info(f"Unified data stream initialized with consumer ID: {self.stream_consumer_id}")
        else:
            self.unified_stream = UnifiedDataStream()  # Fallback
            self.stream_consumer_id = "fallback"
            logger.warning("Using fallback unified data stream")
        
        # Dashboard state
        self.recent_decisions = []
        self.recent_signals = []  # Track all signals (not just executed trades)
        self.performance_data = {}
        self.current_prices = {}
        self.last_update = datetime.now()
        
        # Trading session tracking
        self.session_start = datetime.now()
        self.session_trades = []
        self.session_pnl = 0.0
        self.current_position = None  # {'side': 'BUY', 'price': 3456.78, 'size': 0.1, 'timestamp': datetime}
        self.total_realized_pnl = 0.0
        self.total_fees = 0.0
        self.starting_balance = self._get_initial_balance()  # Get balance from MEXC or default to 100
        
        # Closed trades tracking for accounting
        self.closed_trades = []  # List of all closed trades with full details
        
        # Load existing closed trades from file
        self._load_closed_trades_from_file()
        
        # Signal execution settings for scalping - REMOVED FREQUENCY LIMITS
        self.min_confidence_threshold = 0.30  # Start lower to allow learning  
        self.signal_cooldown = 0  # REMOVED: Model decides when to act, no artificial delays
        self.last_signal_time = 0
        
        # Adaptive threshold learning - starts low and learns optimal thresholds
        self.adaptive_learner = AdaptiveThresholdLearner(initial_threshold=0.30)
        logger.info("[ADAPTIVE] Adaptive threshold learning enabled - will adjust based on trade outcomes")
        
        # Lightweight WebSocket implementation for real-time scalping data
        self.ws_price_cache = {}  # Just current prices, no tick history
        self.ws_connection = None
        self.ws_thread = None
        self.is_streaming = False
        
        # Performance-focused: only track essentials
        self.last_ws_update = 0
        self.ws_update_count = 0
        
        # Compatibility stubs for removed tick infrastructure 
        self.tick_cache = []  # Empty list for compatibility
        self.one_second_bars = []  # Empty list for compatibility
        
        # Enhanced RL Training System - Train on closed trades with comprehensive data
        self.rl_training_enabled = True
        # Force enable Enhanced RL training (bypass import issues)
        self.enhanced_rl_training_enabled = True  # Force enabled for CNN training
        self.enhanced_rl_enabled = True  # Force enabled to show proper status
        self.rl_training_stats = {
            'total_training_episodes': 0,
            'profitable_trades_trained': 0,
            'unprofitable_trades_trained': 0,
            'last_training_time': None,
            'training_rewards': deque(maxlen=100),  # Last 100 training rewards
            'model_accuracy_trend': deque(maxlen=50),  # Track accuracy over time
            'enhanced_rl_episodes': 0,
            'comprehensive_data_packets': 0
        }
        self.rl_training_queue = deque(maxlen=1000)  # Queue of trades to train on
        
        # Enhanced training data tracking
        self.latest_training_data = None
        self.latest_ui_data = None
        self.training_data_available = False
        
        # Load available models for real trading
        self._load_available_models()
        
        # Preload essential data to prevent excessive API calls during dashboard updates
        logger.info("Preloading essential market data to cache...")
        try:
            # Preload key timeframes for main symbols to ensure cache is populated
            symbols_to_preload = self.config.symbols or ['ETH/USDT', 'BTC/USDT']
            timeframes_to_preload = ['1m', '1h', '1d']  # Skip 1s since we use WebSocket for that
            
            for symbol in symbols_to_preload[:2]:  # Limit to first 2 symbols
                for timeframe in timeframes_to_preload:
                    try:
                        # Load data into cache (refresh=True for initial load, then cache will be used)
                        df = self.data_provider.get_historical_data(symbol, timeframe, limit=100, refresh=True)
                        if df is not None and not df.empty:
                            logger.info(f"Preloaded {len(df)} {timeframe} bars for {symbol}")
                        else:
                            logger.warning(f"Failed to preload data for {symbol} {timeframe}")
                    except Exception as e:
                        logger.warning(f"Error preloading {symbol} {timeframe}: {e}")
            
            logger.info("Preloading completed - cache populated for frequent queries")
            
        except Exception as e:
            logger.warning(f"Error during preloading: {e}")
        
        # Create Dash app
        self.app = dash.Dash(__name__, external_stylesheets=[
            'https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css',
            'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css'
        ])
        
        # # Add custom CSS for model data charts
        # self.app.index_string = '''
        # <!DOCTYPE html>
        # <html>
        #     <head>
        #         {%metas%}
        #         <title>{%title%}</title>
        #         {%favicon%}
        #         {%css%}
        #         <style>
        #             .tiny { font-size: 10px !important; }
        #             .model-data-chart .js-plotly-plot .plotly .modebar { display: none !important; }
        #             .model-data-chart .js-plotly-plot .plotly .svg-container { border: 1px solid #444; border-radius: 4px; }
        #         </style>
        #     </head>
        #     <body>
        #         {%app_entry%}
        #         <footer>
        #             {%config%}
        #             {%scripts%}
        #             {%renderer%}
        #         </footer>
        #     </body>
        # </html>
        # '''
        
        # Setup layout and callbacks
        self._setup_layout()
        self._setup_callbacks()
        
        # Start unified data streaming
        self._initialize_streaming()
        
        # Start continuous training with enhanced RL support
        self.start_continuous_training()
        
        logger.info("Trading Dashboard initialized with enhanced RL training integration")
        logger.info(f"Enhanced RL enabled: {self.enhanced_rl_training_enabled}")
        logger.info(f"Stream consumer ID: {self.stream_consumer_id}")
        
        # Initialize Williams Market Structure once
        try:
            from training.williams_market_structure import WilliamsMarketStructure
            self.williams_structure = WilliamsMarketStructure(
                swing_strengths=[2, 3, 5],  # Simplified for better performance
                enable_cnn_feature=True,   # Enable CNN training and inference
                training_data_provider=self.data_provider  # Provide data access for training
            )
            logger.info("Williams Market Structure initialized for dashboard with CNN training enabled")
        except ImportError:
            self.williams_structure = None
            logger.warning("Williams Market Structure not available")
        
        # Initialize Enhanced Pivot RL Trainer for better position management
        try:
            self.pivot_rl_trainer = create_enhanced_pivot_trainer(
                data_provider=self.data_provider,
                orchestrator=self.orchestrator
            )
            logger.info("Enhanced Pivot RL Trainer initialized for better entry/exit decisions")
            logger.info(f"Entry threshold: {self.pivot_rl_trainer.get_current_thresholds()['entry_threshold']:.1%}")
            logger.info(f"Exit threshold: {self.pivot_rl_trainer.get_current_thresholds()['exit_threshold']:.1%}")
            logger.info(f"Uninvested threshold: {self.pivot_rl_trainer.get_current_thresholds()['uninvested_threshold']:.1%}")
        except Exception as e:
            self.pivot_rl_trainer = None
            logger.warning(f"Enhanced Pivot RL Trainer not available: {e}")
    
"""
Trading Dashboard - Clean Web Interface

This module provides a modern, responsive web dashboard for the trading system:
- Real-time price charts with multiple timeframes
- Model performance monitoring
- Trading decisions visualization
- System health monitoring
- Memory usage tracking
"""

import asyncio
import dash
from dash import Dash, dcc, html, Input, Output
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.express as px
import pandas as pd
import numpy as np
from datetime import datetime, timedelta, timezone
import pytz
import logging
import json
import time
import threading
from threading import Thread, Lock
from collections import deque
import warnings
from typing import Dict, List, Optional, Any, Union, Tuple
import websocket
import os
import torch

# Setup logger immediately after logging import
logger = logging.getLogger(__name__)

# WebSocket availability check
try:
    import websocket
    WEBSOCKET_AVAILABLE = True
    logger.info("WebSocket client available")
except ImportError:
    WEBSOCKET_AVAILABLE = False
    logger.warning("websocket-client not available. Real-time data will use API fallback.")

# Import trading system components
from core.config import get_config
from core.data_provider import DataProvider
from core.orchestrator import TradingOrchestrator, TradingDecision
from core.trading_executor import TradingExecutor
from core.trading_action import TradingAction
from models import get_model_registry

# Import CNN monitoring
try:
    from core.cnn_monitor import get_cnn_dashboard_data
    CNN_MONITORING_AVAILABLE = True
    logger.info("CNN monitoring system available")
except ImportError:
    CNN_MONITORING_AVAILABLE = False
    logger.warning("CNN monitoring not available")
    def get_cnn_dashboard_data():
        return {'statistics': {'total_predictions_logged': 0}}

# Import CNN prediction components
try:
    from training.williams_market_structure import SwingPoint, SwingType
    CNN_PREDICTIONS_AVAILABLE = True
    logger.info("CNN predictions available")
except ImportError:
    CNN_PREDICTIONS_AVAILABLE = False
    logger.warning("CNN predictions not available")
    class SwingPoint:
        def __init__(self, timestamp, price, index, swing_type, strength):
            self.timestamp = timestamp
            self.price = price
            self.index = index
            self.swing_type = swing_type
            self.strength = strength
    class SwingType:
        SWING_HIGH = "swing_high"
        SWING_LOW = "swing_low"


# Import enhanced RL components if available
try:
    from core.enhanced_orchestrator import EnhancedTradingOrchestrator
    from core.universal_data_adapter import UniversalDataAdapter
    from core.unified_data_stream import UnifiedDataStream, TrainingDataPacket, UIDataPacket
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL training components available")
except ImportError as e:
    logger.warning(f"Enhanced RL components not available: {e}")
    ENHANCED_RL_AVAILABLE = False
    # Force enable for learning - bypass import issues
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL FORCED ENABLED - bypassing import issues for learning")
    
    # Fallback classes
    class UnifiedDataStream:
        def __init__(self, *args, **kwargs): pass
        def register_consumer(self, *args, **kwargs): return "fallback_consumer"
        def start_streaming(self): pass
        def stop_streaming(self): pass
        def get_latest_training_data(self): return None
        def get_latest_ui_data(self): return None
    
    class TrainingDataPacket:
        def __init__(self, *args, **kwargs): pass
    
    class UIDataPacket:
        def __init__(self, *args, **kwargs): pass


class AdaptiveThresholdLearner:
    """Learn optimal confidence thresholds based on real trade outcomes"""
    
    def __init__(self, initial_threshold: float = 0.30):
        self.base_threshold = initial_threshold
        self.current_threshold = initial_threshold
        self.trade_outcomes = deque(maxlen=100)
        self.threshold_history = deque(maxlen=50)
        self.learning_rate = 0.02
        self.min_threshold = 0.20
        self.max_threshold = 0.70
        
        logger.info(f"[ADAPTIVE] Initialized with starting threshold: {initial_threshold:.2%}")
    
    def record_trade_outcome(self, confidence: float, pnl: float, threshold_used: float):
        """Record a trade outcome to learn from"""
        try:
            outcome = {
                'confidence': confidence,
                'pnl': pnl,
                'profitable': pnl > 0,
                'threshold_used': threshold_used,
                'timestamp': datetime.now()
            }
            
            self.trade_outcomes.append(outcome)
            
            # Learn from outcomes
            if len(self.trade_outcomes) >= 10:
                self._update_threshold()
            
        except Exception as e:
            logger.error(f"Error recording trade outcome: {e}")
    
    def _update_threshold(self):
        """Update threshold based on recent trade statistics"""
        try:
            recent_trades = list(self.trade_outcomes)[-20:]
            if len(recent_trades) < 10:
                return
            
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades)
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades)
            
            # Adaptive adjustment logic
            if win_rate > 0.60 and avg_pnl > 0.20:
                adjustment = -self.learning_rate * 1.5  # Lower threshold for more trades
            elif win_rate < 0.40 or avg_pnl < -0.30:
                adjustment = self.learning_rate * 2.0  # Raise threshold to be more selective
            else:
                adjustment = 0  # No change
            
            old_threshold = self.current_threshold
            self.current_threshold = max(self.min_threshold, 
                                       min(self.max_threshold, 
                                           self.current_threshold + adjustment))
            
            if abs(self.current_threshold - old_threshold) > 0.005:
                logger.info(f"[ADAPTIVE] Threshold: {old_threshold:.2%} -> {self.current_threshold:.2%} (WR: {win_rate:.1%}, PnL: ${avg_pnl:.2f})")
            
        except Exception as e:
            logger.error(f"Error updating adaptive threshold: {e}")
    
    def get_current_threshold(self) -> float:
        return self.current_threshold
    
    def get_learning_stats(self) -> Dict[str, Any]:
        """Get learning statistics"""
        try:
            if not self.trade_outcomes:
                return {'status': 'No trades recorded yet'}
            
            recent_trades = list(self.trade_outcomes)[-20:]
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades) if recent_trades else 0
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades) if recent_trades else 0
            
            return {
                'current_threshold': self.current_threshold,
                'base_threshold': self.base_threshold,
                'total_trades': len(self.trade_outcomes),
                'recent_win_rate': win_rate,
                'recent_avg_pnl': avg_pnl,
                'threshold_changes': len(self.threshold_history),
                'learning_active': len(self.trade_outcomes) >= 10
            }
        except Exception as e:
            return {'error': str(e)}

class TradingDashboard:
    """Enhanced Trading Dashboard with Williams pivot points and unified timezone handling"""
    
    def __init__(self, data_provider: DataProvider = None, orchestrator: TradingOrchestrator = None, trading_executor: TradingExecutor = None):
        self.app = Dash(__name__)
        
        # Initialize config first
        from core.config import get_config
        self.config = get_config()
        
        self.data_provider = data_provider or DataProvider()
        self.orchestrator = orchestrator
        self.trading_executor = trading_executor
        
        # Enhanced trading state with leverage support
        self.leverage_enabled = True
        self.leverage_multiplier = 50.0  # 50x leverage (adjustable via slider)
        self.base_capital = 10000.0
        self.current_position = 0.0  # -1 to 1 (short to long)
        self.position_size = 0.0
        self.entry_price = 0.0
        self.unrealized_pnl = 0.0
        self.realized_pnl = 0.0
        
        # Leverage settings for slider
        self.min_leverage = 1.0
        self.max_leverage = 100.0
        self.leverage_step = 1.0
        
        # Connect to trading server for leverage functionality
        self.trading_server_url = "http://127.0.0.1:8052"
        self.training_server_url = "http://127.0.0.1:8053"
        self.stream_server_url = "http://127.0.0.1:8054"
        
        # Enhanced performance tracking
        self.leverage_metrics = {
            'leverage_efficiency': 0.0,
            'margin_used': 0.0,
            'margin_available': 10000.0,
            'effective_exposure': 0.0,
            'risk_reward_ratio': 0.0
        }
        
        # Enhanced models will be loaded through model registry later
        
        # Rest of initialization...
        
        # Initialize timezone from config
        timezone_name = self.config.get('system', {}).get('timezone', 'Europe/Sofia')
        self.timezone = pytz.timezone(timezone_name)
        logger.info(f"Dashboard timezone set to: {timezone_name}")
        
        self.data_provider = data_provider or DataProvider()
        
        # Enhanced orchestrator support - FORCE ENABLE for learning
        self.orchestrator = orchestrator or TradingOrchestrator(self.data_provider)
        self.enhanced_rl_enabled = True  # Force enable Enhanced RL
        logger.info("Enhanced RL training FORCED ENABLED for learning")
        
        self.trading_executor = trading_executor or TradingExecutor()
        self.model_registry = get_model_registry()
        
        # Initialize unified data stream for comprehensive training data
        if ENHANCED_RL_AVAILABLE:
            self.unified_stream = UnifiedDataStream(self.data_provider, self.orchestrator)
            self.stream_consumer_id = self.unified_stream.register_consumer(
                consumer_name="TradingDashboard",
                callback=self._handle_unified_stream_data,
                data_types=['ticks', 'ohlcv', 'training_data', 'ui_data']
            )
            logger.info(f"Unified data stream initialized with consumer ID: {self.stream_consumer_id}")
        else:
            self.unified_stream = UnifiedDataStream()  # Fallback
            self.stream_consumer_id = "fallback"
            logger.warning("Using fallback unified data stream")
        
        # Dashboard state
        self.recent_decisions = []
        self.recent_signals = []  # Track all signals (not just executed trades)
        self.performance_data = {}
        self.current_prices = {}
        self.last_update = datetime.now()
        
        # Trading session tracking
        self.session_start = datetime.now()
        self.session_trades = []
        self.session_pnl = 0.0
        self.current_position = None  # {'side': 'BUY', 'price': 3456.78, 'size': 0.1, 'timestamp': datetime}
        self.total_realized_pnl = 0.0
        self.total_fees = 0.0
        self.starting_balance = self._get_initial_balance()  # Get balance from MEXC or default to 100
        
        # Closed trades tracking for accounting
        self.closed_trades = []  # List of all closed trades with full details
        
        # Load existing closed trades from file
        self._load_closed_trades_from_file()
        
        # Signal execution settings for scalping - REMOVED FREQUENCY LIMITS
        self.min_confidence_threshold = 0.30  # Start lower to allow learning  
        self.signal_cooldown = 0  # REMOVED: Model decides when to act, no artificial delays
        self.last_signal_time = 0
        
        # Adaptive threshold learning - starts low and learns optimal thresholds
        self.adaptive_learner = AdaptiveThresholdLearner(initial_threshold=0.30)
        logger.info("[ADAPTIVE] Adaptive threshold learning enabled - will adjust based on trade outcomes")
        
        # Lightweight WebSocket implementation for real-time scalping data
        self.ws_price_cache = {}  # Just current prices, no tick history
        self.ws_connection = None
        self.ws_thread = None
        self.is_streaming = False
        
        # Performance-focused: only track essentials
        self.last_ws_update = 0
        self.ws_update_count = 0
        
        # Compatibility stubs for removed tick infrastructure 
        self.tick_cache = []  # Empty list for compatibility
        self.one_second_bars = []  # Empty list for compatibility
        
        # Enhanced RL Training System - Train on closed trades with comprehensive data
        self.rl_training_enabled = True
        # Force enable Enhanced RL training (bypass import issues)
        self.enhanced_rl_training_enabled = True  # Force enabled for CNN training
        self.enhanced_rl_enabled = True  # Force enabled to show proper status
        self.rl_training_stats = {
            'total_training_episodes': 0,
            'profitable_trades_trained': 0,
            'unprofitable_trades_trained': 0,
            'last_training_time': None,
            'training_rewards': deque(maxlen=100),  # Last 100 training rewards
            'model_accuracy_trend': deque(maxlen=50),  # Track accuracy over time
            'enhanced_rl_episodes': 0,
            'comprehensive_data_packets': 0
        }
        self.rl_training_queue = deque(maxlen=1000)  # Queue of trades to train on
        
        # Enhanced training data tracking
        self.latest_training_data = None
        self.latest_ui_data = None
        self.training_data_available = False
        
        # Load available models for real trading
        self._load_available_models()
        
        # Preload essential data to prevent excessive API calls during dashboard updates
        logger.info("Preloading essential market data to cache...")
        try:
            # Preload key timeframes for main symbols to ensure cache is populated
            symbols_to_preload = self.config.symbols or ['ETH/USDT', 'BTC/USDT']
            timeframes_to_preload = ['1m', '1h', '1d']  # Skip 1s since we use WebSocket for that
            
            for symbol in symbols_to_preload[:2]:  # Limit to first 2 symbols
                for timeframe in timeframes_to_preload:
                    try:
                        # Load data into cache (refresh=True for initial load, then cache will be used)
                        df = self.data_provider.get_historical_data(symbol, timeframe, limit=100, refresh=True)
                        if df is not None and not df.empty:
                            logger.info(f"Preloaded {len(df)} {timeframe} bars for {symbol}")
                        else:
                            logger.warning(f"Failed to preload data for {symbol} {timeframe}")
                    except Exception as e:
                        logger.warning(f"Error preloading {symbol} {timeframe}: {e}")
            
            logger.info("Preloading completed - cache populated for frequent queries")
            
        except Exception as e:
            logger.warning(f"Error during preloading: {e}")
        
        # Create Dash app
        self.app = dash.Dash(__name__, external_stylesheets=[
            'https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css',
            'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css'
        ])
        
        # # Add custom CSS for model data charts
        # self.app.index_string = '''
        # <!DOCTYPE html>
        # <html>
        #     <head>
        #         {%metas%}
        #         <title>{%title%}</title>
        #         {%favicon%}
        #         {%css%}
        #         <style>
        #             .tiny { font-size: 10px !important; }
        #             .model-data-chart .js-plotly-plot .plotly .modebar { display: none !important; }
        #             .model-data-chart .js-plotly-plot .plotly .svg-container { border: 1px solid #444; border-radius: 4px; }
        #         </style>
        #     </head>
        #     <body>
        #         {%app_entry%}
        #         <footer>
        #             {%config%}
        #             {%scripts%}
        #             {%renderer%}
        #         </footer>
        #     </body>
        # </html>
        # '''
        
        # Setup layout and callbacks
        self._setup_layout()
        self._setup_callbacks()
        
        # Start unified data streaming
        self._initialize_streaming()
        
        # Start continuous training with enhanced RL support
        self.start_continuous_training()
        
        logger.info("Trading Dashboard initialized with enhanced RL training integration")
        logger.info(f"Enhanced RL enabled: {self.enhanced_rl_training_enabled}")
        logger.info(f"Stream consumer ID: {self.stream_consumer_id}")
        
        # Initialize Williams Market Structure once
        try:
            from training.williams_market_structure import WilliamsMarketStructure
            self.williams_structure = WilliamsMarketStructure(
                swing_strengths=[2, 3, 5],  # Simplified for better performance
                enable_cnn_feature=True,   # Enable CNN training and inference
                training_data_provider=self.data_provider  # Provide data access for training
            )
            logger.info("Williams Market Structure initialized for dashboard with CNN training enabled")
        except ImportError:
            self.williams_structure = None
            logger.warning("Williams Market Structure not available")
        
        # Initialize Enhanced Pivot RL Trainer for better position management
        try:
            self.pivot_rl_trainer = create_enhanced_pivot_trainer(
                data_provider=self.data_provider,
                orchestrator=self.orchestrator
            )
            logger.info("Enhanced Pivot RL Trainer initialized for better entry/exit decisions")
            logger.info(f"Entry threshold: {self.pivot_rl_trainer.get_current_thresholds()['entry_threshold']:.1%}")
            logger.info(f"Exit threshold: {self.pivot_rl_trainer.get_current_thresholds()['exit_threshold']:.1%}")
            logger.info(f"Uninvested threshold: {self.pivot_rl_trainer.get_current_thresholds()['uninvested_threshold']:.1%}")
        except Exception as e:
            self.pivot_rl_trainer = None
            logger.warning(f"Enhanced Pivot RL Trainer not available: {e}")
    
    def _setup_layout(self):
        """Setup the dashboard layout"""
        self.app.layout = html.Div([
            # Compact Header
            html.Div([
                html.H3([
                    html.I(className="fas fa-chart-line me-2"),
                    "Live Trading Dashboard"
                ], className="text-white mb-1"),
                html.P(f"Ultra-Fast Updates • Portfolio: ${self.starting_balance:,.0f} • {'MEXC Live' if (self.trading_executor and self.trading_executor.trading_enabled and not self.trading_executor.simulation_mode) else 'Demo Mode'}", 
                      className="text-light mb-0 opacity-75 small")
            ], className="bg-dark p-2 mb-2"),
            
            # Auto-refresh component
            dcc.Interval(
                id='interval-component',
                interval=1000,  # Update every 1 second for real-time tick updates
                n_intervals=0
            ),
            
            # Main content - Compact layout
            html.Div([
                # Top row - Key metrics and Recent Signals (split layout)
                html.Div([
                    # Left side - Key metrics (compact cards)
                    html.Div([
                        html.Div([
                            html.Div([
                                html.H5(id="current-price", className="text-success mb-0 small"),
                                html.P("Live Price", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="session-pnl", className="mb-0 small"),
                                html.P("Session P&L", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="total-fees", className="text-warning mb-0 small"),
                                html.P("Total Fees", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="current-position", className="text-info mb-0 small"),
                                html.P("Position", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="trade-count", className="text-warning mb-0 small"),
                                html.P("Trades", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="portfolio-value", className="text-secondary mb-0 small"),
                                html.P("Portfolio", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="mexc-status", className="text-info mb-0 small"),
                                html.P("MEXC API", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                    ], style={"display": "grid", "gridTemplateColumns": "repeat(4, 1fr)", "gap": "8px", "width": "60%"}),
                    
                    # Right side - Recent Signals & Executions
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-robot me-2"),
                                "Recent Trading Signals & Executions"
                            ], className="card-title mb-2"),
                            html.Div(id="recent-decisions", style={"height": "160px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "48%", "marginLeft": "2%"})
                ], className="d-flex mb-3"),
                
                # Charts row - More compact
                html.Div([
                    # Price chart - 70% width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-chart-candlestick me-2"),
                                "Live 1s Price & Volume Chart (WebSocket Stream)"
                            ], className="card-title mb-2"),
                            dcc.Graph(id="price-chart", style={"height": "400px"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "70%"}),
                    
                    # Model Training Metrics - 30% width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-brain me-2"),
                                "Model Training Progress"
                            ], className="card-title mb-2"),
                            html.Div(id="training-metrics", style={"height": "400px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "28%", "marginLeft": "2%"}),
                ], className="row g-2 mb-3"),
                
                # CNN Model Monitoring Section
                html.Div([
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-brain me-2"),
                                "CNN Model Analysis & Predictions"
                            ], className="card-title mb-2"),
                            html.Div(id="cnn-monitoring-content", style={"height": "350px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card")
                ], className="mb-3"),
                
                # Bottom row - Session performance and system status
                html.Div([                               

                    # Session performance - 1/3 width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-chart-pie me-2"),
                                "Session Performance"
                            ], className="card-title mb-2"),
                            html.Button(
                                "Clear Session",
                                id="clear-history-btn",
                                className="btn btn-sm btn-outline-danger mb-2",
                                n_clicks=0
                            ),
                            html.Div(id="session-performance")
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%"}),
                                  
                    # Closed Trades History - 1/3 width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-history me-2"),
                                "Closed Trades History"
                            ], className="card-title mb-2"),
                            html.Div([
                                html.Div(
                                    id="closed-trades-table",
                                    style={"height": "300px", "overflowY": "auto"}
                                )
                            ])
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%", "marginLeft": "2%"}),
                    
                    # System status and leverage controls - 1/3 width with icon tooltip
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-server me-2"),
                                "System & Leverage"
                            ], className="card-title mb-2"),
                            
                            # System status
                            html.Div([
                                html.I(
                                    id="system-status-icon",
                                    className="fas fa-circle text-success fa-2x",
                                    title="System Status: All systems operational",
                                    style={"cursor": "pointer"}
                                ),
                                html.Div(id="system-status-details", className="small mt-2")
                            ], className="text-center mb-3"),
                            
                            # Leverage Controls
                            html.Div([
                                html.Label([
                                    html.I(className="fas fa-chart-line me-1"),
                                    "Leverage Multiplier"
                                ], className="form-label small fw-bold"),
                                html.Div([
                                    dcc.Slider(
                                        id='leverage-slider',
                                        min=self.min_leverage,
                                        max=self.max_leverage,
                                        step=self.leverage_step,
                                        value=self.leverage_multiplier,
                                        marks={
                                            1: '1x',
                                            10: '10x',
                                            25: '25x',
                                            50: '50x',
                                            75: '75x',
                                            100: '100x'
                                        },
                                        tooltip={
                                            "placement": "bottom",
                                            "always_visible": True
                                        }
                                    )
                                ], className="mb-2"),
                                html.Div([
                                    html.Span(id="current-leverage", className="badge bg-warning text-dark"),
                                    html.Span(" • ", className="mx-1"),
                                    html.Span(id="leverage-risk", className="badge bg-info")
                                ], className="text-center"),
                                html.Div([
                                    html.Small("Higher leverage = Higher rewards & risks", className="text-muted")
                                ], className="text-center mt-1")
                            ])
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%", "marginLeft": "2%"})
                ], className="d-flex")
            ], className="container-fluid")
        ])
    
    def _setup_callbacks(self):
        """Setup dashboard callbacks for real-time updates"""
        
        @self.app.callback(
            [
                Output('current-price', 'children'),
                Output('session-pnl', 'children'),
                Output('session-pnl', 'className'),
                Output('total-fees', 'children'),
                Output('current-position', 'children'),
                Output('current-position', 'className'),
                Output('trade-count', 'children'),
                Output('portfolio-value', 'children'),
                Output('mexc-status', 'children'),
                Output('price-chart', 'figure'),
                Output('training-metrics', 'children'),
                Output('recent-decisions', 'children'),
                Output('session-performance', 'children'),
                Output('closed-trades-table', 'children'),
                Output('system-status-icon', 'className'),
                Output('system-status-icon', 'title'),
                Output('system-status-details', 'children'),
                Output('current-leverage', 'children'),
                Output('leverage-risk', 'children'),
                Output('cnn-monitoring-content', 'children')
            ],
            [Input('interval-component', 'n_intervals')]
        )
        def update_dashboard(n_intervals):
            """Update all dashboard components with trading signals"""
            start_time = time.time()  # Performance monitoring
            try:
                # Periodic cleanup to prevent memory leaks
                if n_intervals % 60 == 0:  # Every 60 seconds
                    self._cleanup_old_data()
                
                # Lightweight update every 10 intervals to reduce load
                is_lightweight_update = (n_intervals % 10 != 0)
                # Chart updates every second for responsiveness
                # Get current prices with improved fallback handling
                symbol = self.config.symbols[0] if self.config.symbols else "ETH/USDT"
                current_price = None
                chart_data = None
                data_source = "UNKNOWN"
                
                try:
                    # First try real-time WebSocket price (sub-second latency)
                    current_price = self.get_realtime_price(symbol)
                    if current_price:
                        data_source = "WEBSOCKET_RT"
                        logger.debug(f"[WS_RT] Using real-time WebSocket price for {symbol}: ${current_price:.2f}")
                    else:
                        # Try cached data first (faster than API calls)
                        cached_data = self.data_provider.get_historical_data(symbol, '1m', limit=1, refresh=False)
                        if cached_data is not None and not cached_data.empty:
                            current_price = float(cached_data['close'].iloc[-1])
                            data_source = "CACHED"
                            logger.debug(f"[CACHED] Using cached price for {symbol}: ${current_price:.2f}")
                        else:
                            # Only try fresh API call if we have no data at all
                            try:
                                fresh_data = self.data_provider.get_historical_data(symbol, '1m', limit=1, refresh=False)
                                if fresh_data is not None and not fresh_data.empty:
                                    current_price = float(fresh_data['close'].iloc[-1])
                                    data_source = "API"
                                    logger.debug(f"[API] Fresh price for {symbol}: ${current_price:.2f}")
                            except Exception as api_error:
                                logger.warning(f"[API_ERROR] Failed to fetch fresh data: {api_error}")
                        
                        # NO SYNTHETIC DATA - Wait for real data
                        if current_price is None:
                            logger.warning(f"[NO_DATA] No real data available for {symbol} - waiting for data provider")
                            data_source = "NO_DATA"
                            
                except Exception as e:
                    logger.warning(f"[ERROR] Error getting price for {symbol}: {e}")
                    current_price = None
                    data_source = "ERROR"
                
                # Get chart data - ONLY REAL DATA (optimized for performance)
                chart_data = None
                try:
                    if not is_lightweight_update:  # Only refresh charts every 10 seconds
                        # Use cached data only (limited to 30 bars for performance)
                        chart_data = self.data_provider.get_historical_data(symbol, '1m', limit=30, refresh=False)
                        if chart_data is not None and not chart_data.empty:
                            logger.debug(f"[CHART] Using cached 1m data: {len(chart_data)} bars")
                        else:
                            # Wait for real data - no synthetic data
                            logger.debug("[CHART] No chart data available - waiting for data provider")
                            chart_data = None
                    else:
                        # Use cached chart data for lightweight updates
                        chart_data = getattr(self, '_cached_chart_data', None)
                except Exception as e:
                    logger.warning(f"[CHART_ERROR] Error getting chart data: {e}")
                    chart_data = None
                
                # Generate trading signals based on model decisions - OPTIMIZED
                try:
                    # Only generate signals every few intervals to reduce CPU load
                    if not is_lightweight_update and current_price and chart_data is not None and not chart_data.empty and len(chart_data) >= 5:
                        # Model decides when to act - check for signals but not every single second
                        signal = self._generate_trading_signal(symbol, current_price, chart_data)
                        if signal:
                            # Add to signals list (all signals, regardless of execution)
                            signal['signal_type'] = 'GENERATED'
                            self.recent_signals.append(signal.copy())
                            if len(self.recent_signals) > 100:  # Keep last 100 signals
                                self.recent_signals = self.recent_signals[-100:]
                            
                            # Use adaptive threshold instead of fixed threshold
                            current_threshold = self.adaptive_learner.get_current_threshold()
                            should_execute = signal['confidence'] >= current_threshold
                            
                            # Check position limits before execution
                            can_execute = self._can_execute_new_position(signal['action'])
                            
                            if should_execute and can_execute:
                                signal['signal_type'] = 'EXECUTED'
                                signal['threshold_used'] = current_threshold  # Track threshold for learning
                                signal['reason'] = f"ADAPTIVE EXECUTE (≥{current_threshold:.2%}): {signal['reason']}"
                                logger.debug(f"[EXECUTE] {signal['action']} signal @ ${signal['price']:.2f} (confidence: {signal['confidence']:.1%} ≥ {current_threshold:.1%})")
                                self._process_trading_decision(signal)
                            elif should_execute and not can_execute:
                                # Signal meets confidence but we're at position limit
                                signal['signal_type'] = 'NOT_EXECUTED_POSITION_LIMIT'
                                signal['threshold_used'] = current_threshold
                                signal['reason'] = f"BLOCKED BY POSITION LIMIT (≥{current_threshold:.2%}): {signal['reason']} [Positions: {self._count_open_positions()}/{self.config.get('trading', {}).get('max_concurrent_positions', 3)}]"
                                logger.info(f"[BLOCKED] {signal['action']} signal @ ${signal['price']:.2f} - Position limit reached ({self._count_open_positions()}/{self.config.get('trading', {}).get('max_concurrent_positions', 3)})")
                                
                                # Still add to training queue for RL learning
                                self._queue_signal_for_training(signal, current_price, symbol)
                            else:
                                signal['signal_type'] = 'NOT_EXECUTED_LOW_CONFIDENCE'
                                signal['threshold_used'] = current_threshold
                                signal['reason'] = f"LOW CONFIDENCE (<{current_threshold:.2%}): {signal['reason']}"
                                logger.debug(f"[SKIP] {signal['action']} signal @ ${signal['price']:.2f} (confidence: {signal['confidence']:.1%} < {current_threshold:.1%})")
                                
                                # Still add to training queue for RL learning
                                self._queue_signal_for_training(signal, current_price, symbol)
                    else:
                        # Fallback: Add a simple monitoring update
                        if n_intervals % 10 == 0 and current_price:  # Every 10 seconds
                            monitor_signal = {
                                'action': 'MONITOR',
                                'symbol': symbol,
                                'price': current_price,
                                'confidence': 0.0,
                                'timestamp': datetime.now(),
                                'size': 0.0,
                                'reason': 'System monitoring - no trading signals',
                                'signal_type': 'MONITOR'
                            }
                            self.recent_decisions.append(monitor_signal)
                            if len(self.recent_decisions) > 500:
                                self.recent_decisions = self.recent_decisions[-500:]
                                
                except Exception as e:
                    logger.warning(f"[ERROR] Error generating trading signal: {e}")
                
                # Calculate PnL metrics
                unrealized_pnl = self._calculate_unrealized_pnl(current_price) if current_price else 0.0
                total_session_pnl = self.total_realized_pnl + unrealized_pnl
                
                # Calculate portfolio value
                portfolio_value = self.starting_balance + total_session_pnl
                
                # Get memory stats with fallback (still needed for system status)
                try:
                    memory_stats = self.model_registry.get_memory_stats()
                except:
                    memory_stats = {'utilization_percent': 0, 'total_used_mb': 0, 'total_limit_mb': 1024}
                
                # Format outputs with safe defaults and update indicators
                update_time = datetime.now().strftime("%H:%M:%S.%f")[:-3]  # Include milliseconds
                
                if current_price:
                    # Add data source indicator and precise timestamp
                    source_indicator = f"[{data_source}]"
                    price_text = f"${current_price:.2f} {source_indicator} @ {update_time}"
                else:
                    # Show waiting status when no real data
                    price_text = f"WAITING FOR REAL DATA [{data_source}] @ {update_time}"
                
                # PnL formatting
                pnl_text = f"${total_session_pnl:.2f}"
                pnl_class = "text-success mb-0 small" if total_session_pnl >= 0 else "text-danger mb-0 small"
                
                # Total fees formatting
                fees_text = f"${self.total_fees:.2f}"
                
                # Position info with real-time unrealized PnL and proper color coding
                if self.current_position:
                    pos_side = self.current_position['side']
                    pos_size = self.current_position['size']
                    pos_price = self.current_position['price']
                    unrealized_pnl = self._calculate_unrealized_pnl(current_price) if current_price else 0.0
                    
                    # Color coding: LONG=Green, SHORT=Red (consistent with trading conventions)
                    if pos_side == 'LONG':
                        side_icon = "[LONG]"
                        side_color = "success"  # Green for long positions
                    else:  # SHORT
                        side_icon = "[SHORT]"
                        side_color = "danger"   # Red for short positions
                    
                    # Create enhanced position display with bold styling
                    pnl_sign = "+" if unrealized_pnl > 0 else ""
                    position_text = f"{side_icon} {pos_size} @ ${pos_price:.2f} | P&L: {pnl_sign}${unrealized_pnl:.2f}"
                    position_class = f"text-{side_color} fw-bold mb-0 small"
                else:
                    position_text = "No Position"
                    position_class = "text-muted mb-0 small"
                
                # Trade count and portfolio value
                trade_count_text = f"{len(self.session_trades)}"
                portfolio_text = f"${portfolio_value:,.2f}"
                
                # MEXC status with detailed information
                if self.trading_executor and self.trading_executor.trading_enabled:
                    if self.trading_executor.simulation_mode:
                        mexc_status = f"{self.trading_executor.trading_mode.upper()} MODE"
                    else:
                        mexc_status = "LIVE"
                else:
                    mexc_status = "OFFLINE"
                
                # Create charts with error handling - OPTIMIZED
                try:
                    # Always try to create/update chart every second for smooth responsiveness
                    if current_price and chart_data is not None and not chart_data.empty:
                        price_chart = self._create_price_chart(symbol)
                        self._cached_chart_data = chart_data  # Cache for fallback
                        self._cached_price_chart = price_chart  # Cache chart
                    else:
                        # Use cached chart if we have one, otherwise show loading
                        if hasattr(self, '_cached_price_chart') and self._cached_price_chart:
                            price_chart = self._cached_price_chart
                            # Update the cached chart with current info
                            try:
                                current_time_str = datetime.now().strftime("%H:%M:%S")
                                stream_status = "LIVE STREAM" if self.is_streaming else "WAITING DATA"
                                price_chart.update_layout(
                                    title=f"{symbol} 1M CHART | ${current_price or 0:.2f} | {stream_status} | {current_time_str}"
                                )
                            except Exception as e:
                                logger.debug(f"Error updating cached chart: {e}")
                        else:
                            price_chart = self._create_empty_chart("Price Chart", "Waiting for real market data...")
                            self._cached_price_chart = price_chart
                except Exception as e:
                    logger.warning(f"Price chart error: {e}")
                    price_chart = self._create_empty_chart("Price Chart", "Error loading chart - waiting for data")
                
                # Create training metrics display
                try:
                    training_metrics = self._create_training_metrics()
                except Exception as e:
                    logger.warning(f"Training metrics error: {e}")
                    training_metrics = [html.P("Training metrics unavailable", className="text-muted")]
                
                # Create recent decisions list
                try:
                    decisions_list = self._create_decisions_list()
                except Exception as e:
                    logger.warning(f"Decisions list error: {e}")
                    decisions_list = [html.P("No decisions available", className="text-muted")]
                
                # Create session performance
                try:
                    session_perf = self._create_session_performance()
                except Exception as e:
                    logger.warning(f"Session performance error: {e}")
                    session_perf = [html.P("Performance data unavailable", className="text-muted")]
                
                # Create system status
                try:
                    system_status = self._create_system_status_compact(memory_stats)
                except Exception as e:
                    logger.warning(f"System status error: {e}")
                    system_status = {
                        'icon_class': "fas fa-circle text-danger fa-2x",
                        'title': "System Error: Check logs",
                        'details': [html.P(f"Error: {str(e)}", className="text-danger")]
                    }
                
                # Create closed trades table
                try:
                    closed_trades_table = self._create_closed_trades_table()
                except Exception as e:
                    logger.warning(f"Closed trades table error: {e}")
                    closed_trades_table = [html.P("Closed trades data unavailable", className="text-muted")]
                
                # Calculate leverage display values
                leverage_text = f"{self.leverage_multiplier:.0f}x"
                if self.leverage_multiplier <= 5:
                    risk_level = "Low Risk"
                    risk_class = "bg-success"
                elif self.leverage_multiplier <= 25:
                    risk_level = "Medium Risk"
                    risk_class = "bg-warning text-dark"
                elif self.leverage_multiplier <= 50:
                    risk_level = "High Risk"
                    risk_class = "bg-danger"
                else:
                    risk_level = "Extreme Risk"
                    risk_class = "bg-dark"
                
                # Create CNN monitoring content
                try:
                    cnn_monitoring_content = self._create_cnn_monitoring_content()
                except Exception as e:
                    logger.warning(f"CNN monitoring error: {e}")
                    cnn_monitoring_content = [html.P("CNN monitoring unavailable", className="text-danger")]

                return (
                    price_text, pnl_text, pnl_class, fees_text, position_text, position_class, trade_count_text, portfolio_text, mexc_status,
                    price_chart, training_metrics, decisions_list, session_perf, closed_trades_table,
                    system_status['icon_class'], system_status['title'], system_status['details'],
                    leverage_text, f"{risk_level}",
                    cnn_monitoring_content
                )
                
            except Exception as e:
                logger.error(f"Error updating dashboard: {e}")
                # Return safe defaults
                empty_fig = self._create_empty_chart("Error", "Dashboard error - check logs")
                
                return (
                    "Error", "$0.00", "text-muted mb-0 small", "$0.00", "None", "text-muted", "0", "$10,000.00", "OFFLINE",
                    empty_fig, 
                    [html.P("Error loading training metrics", className="text-danger")],
                    [html.P("Error loading decisions", className="text-danger")], 
                    [html.P("Error loading performance", className="text-danger")],
                    [html.P("Error loading closed trades", className="text-danger")],
                    "fas fa-circle text-danger fa-2x",
                    "Error: Dashboard error - check logs",
                    [html.P(f"Error: {str(e)}", className="text-danger")],
                    f"{self.leverage_multiplier:.0f}x", "Error",
                    [html.P("CNN monitoring unavailable", className="text-danger")]
                )
        
        # Clear history callback
        @self.app.callback(
            Output('closed-trades-table', 'children', allow_duplicate=True),
            [Input('clear-history-btn', 'n_clicks')],
            prevent_initial_call=True
        )
        def clear_trade_history(n_clicks):
            """Clear trade history and reset session stats"""
            if n_clicks and n_clicks > 0:
                try:
                    # Clear both closed trades and session stats (they're the same now)
                    self.clear_closed_trades_history()
                    logger.info("DASHBOARD: Trade history and session stats cleared by user")
                    return [html.P("Trade history cleared", className="text-success text-center")]
                except Exception as e:
                    logger.error(f"Error clearing trade history: {e}")
                    return [html.P(f"Error clearing history: {str(e)}", className="text-danger text-center")]
            return dash.no_update
        
        # Leverage slider callback
        @self.app.callback(
            [Output('current-leverage', 'children', allow_duplicate=True),
             Output('leverage-risk', 'children', allow_duplicate=True),
             Output('leverage-risk', 'className', allow_duplicate=True)],
            [Input('leverage-slider', 'value')],
            prevent_initial_call=True
        )
        def update_leverage(leverage_value):
            """Update leverage multiplier and risk assessment"""
            try:
                if leverage_value is None:
                    return dash.no_update
                
                # Update internal leverage value
                self.leverage_multiplier = float(leverage_value)
                
                # Calculate risk level and styling
                leverage_text = f"{self.leverage_multiplier:.0f}x"
                
                if self.leverage_multiplier <= 5:
                    risk_level = "Low Risk"
                    risk_class = "badge bg-success"
                elif self.leverage_multiplier <= 25:
                    risk_level = "Medium Risk" 
                    risk_class = "badge bg-warning text-dark"
                elif self.leverage_multiplier <= 50:
                    risk_level = "High Risk"
                    risk_class = "badge bg-danger"
                else:
                    risk_level = "Extreme Risk"
                    risk_class = "badge bg-dark"
                
                # Update trading server if connected
                try:
                    import requests
                    response = requests.post(f"{self.trading_server_url}/update_leverage", 
                                           json={"leverage": self.leverage_multiplier}, 
                                           timeout=2)
                    if response.status_code == 200:
                        logger.info(f"[LEVERAGE] Updated trading server leverage to {self.leverage_multiplier}x")
                    else:
                        logger.warning(f"[LEVERAGE] Failed to update trading server: {response.status_code}")
                except Exception as e:
                    logger.debug(f"[LEVERAGE] Trading server not available: {e}")
                
                logger.info(f"[LEVERAGE] Leverage updated to {self.leverage_multiplier}x ({risk_level})")
                
                return leverage_text, risk_level, risk_class
                
            except Exception as e:
                logger.error(f"Error updating leverage: {e}")
                return f"{self.leverage_multiplier:.0f}x", "Error", "badge bg-secondary"
    
    def _create_empty_chart(self, title: str, message: str) -> go.Figure:
        """Create an empty chart with a message"""
        fig = go.Figure()
        fig.add_annotation(
            text=message,
            xref="paper", yref="paper",
            x=0.5, y=0.5,
            showarrow=False,
            font=dict(size=16, color="gray")
        )
        fig.update_layout(
            title=title,
            template="plotly_dark",
            height=400,
            margin=dict(l=20, r=20, t=50, b=20)
        )
        return fig
    
    def _create_price_chart(self, symbol: str) -> go.Figure:
        """Create enhanced price chart with real-time data, Williams pivot points, and trading signals"""
        try:
            # Initialize chart_start_time and chart_end_time early
            chart_start_time = None
            chart_end_time = None
            
            # Try to get real-time data if available
            df = None
            actual_timeframe = '1m'
            
            if self.data_provider:
                try:
                    # Get fresh market data with configurable timeframe
                    df = self.data_provider.get_historical_data(symbol, '1m', limit=30, refresh=True)
                    
                    if df is not None and not df.empty:
                        # Ensure timezone consistency
                        df = self._ensure_timezone_consistency(df)
                        actual_timeframe = '1m'
                        logger.debug(f"[CHART] Loaded {len(df)} fresh 1m bars in {self.timezone}")
                        
                        # Set time boundaries early
                        chart_start_time = df.index.min()
                        chart_end_time = df.index.max()
                    else:
                        return self._create_empty_chart(
                            f"{symbol} Chart", 
                            f"No data available for {symbol}\nWaiting for data provider..."
                        )
                except Exception as e:
                    logger.warning(f"Error getting real-time data: {e}")
                    df = None
            
            # Create chart with multiple subplots
            fig = make_subplots(rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.02, row_heights=[0.7, 0.3])
            
            # Add price chart
            if df is not None and not df.empty:
                fig.add_trace(
                    go.Candlestick(
                        x=df.index,
                        open=df['open'],
                        high=df['high'],
                        low=df['low'],
                        close=df['close'],
                        name='Price',
                        increasing_line_color='green',
                        decreasing_line_color='red',
                        showlegend=False
                    ),
                    row=1, col=1
                )
                
                # Add volume bars
                fig.add_trace(
                    go.Bar(
                        x=df.index,
                        y=df['volume'],
                        name='Volume',
                        marker_color='blue',
                        opacity=0.3,
                        showlegend=False
                    ),
                    row=2, col=1
                )
                
                # Add Williams Market Structure pivot points
                try:
                    pivot_points = self._get_williams_pivot_points_for_chart(df)
                    if pivot_points:
                        self._add_williams_pivot_points_to_chart(fig, pivot_points, row=1)
                    else:
                        logger.debug("[CHART] No Williams pivot points available")
                except Exception as e:
                    logger.debug(f"Error adding Williams pivot points to chart: {e}")
                
                # Add CNN pivot predictions as hollow circles
                try:
                    cnn_predictions = self._get_cnn_pivot_predictions(symbol, df)
                    if cnn_predictions:
                        self._add_cnn_predictions_to_chart(fig, cnn_predictions, row=1)
                        logger.debug(f"[CHART] Added {len(cnn_predictions)} CNN predictions to chart")
                    else:
                        logger.debug("[CHART] No CNN predictions available")
                except Exception as e:
                    logger.debug(f"Error adding CNN predictions to chart: {e}")
            
            # Update layout
            fig.update_layout(
                title=f"{symbol} {actual_timeframe} Chart",
                template="plotly_dark",
                height=400,
                margin=dict(l=20, r=20, t=50, b=20),
                xaxis_rangeslider_visible=False
            )
            
            # Update x-axis range
            if chart_start_time and chart_end_time:
                fig.update_xaxes(
                    range=[chart_start_time, chart_end_time],
                    row=1, col=1
                )
            
            return fig
        except Exception as e:
            logger.error(f"Error creating price chart: {e}")
            return self._create_empty_chart("Price Chart", "Error loading chart - check logs")
    
    def _get_williams_pivot_points_for_chart(self, df: pd.DataFrame) -> List[SwingPoint]:
        """Get Williams Market Structure pivot points for chart"""
        try:
            if df is None or df.empty:
                return []
            
            # Calculate Williams Market Structure pivot points
            swing_points = self._calculate_williams_pivot_points(df)
            
            # Filter out invalid pivot points
            valid_swing_points = [sp for sp in swing_points if sp.is_valid()]
            
            return valid_swing_points
        except Exception as e:
            logger.error(f"Error getting Williams pivot points for chart: {e}")
            return []
    
    def _add_williams_pivot_points_to_chart(self, fig: go.Figure, swing_points: List[SwingPoint], row: int):
        """Add Williams Market Structure pivot points to chart"""
        try:
            if not swing_points:
                return
            
            # Add pivot points to chart
            for sp in swing_points:
                if sp.swing_type == SwingType.RESISTANCE:
                    color = 'red'
                elif sp.swing_type == SwingType.SUPPORT:
                    color = 'green'
                else:
                    color = 'gray'
                
                fig.add_trace(
                    go.Scatter(
                        x=[sp.timestamp],
                        y=[sp.price],
                        mode='markers',
                        marker=dict(
                            size=10,
                            color=color,
                            symbol='circle',
                            line=dict(width=2, color='black')
                        ),
                        name=f"{sp.swing_type.name} ({sp.price:.2f})",
                        showlegend=False
                    ),
                    row=row, col=1
                )
        except Exception as e:
            logger.error(f"Error adding Williams pivot points to chart: {e}")
    
    def _get_cnn_pivot_predictions(self, symbol: str, df: pd.DataFrame) -> List[Dict[str, Any]]:
        """Get CNN pivot predictions for chart"""
        try:
            if df is None or df.empty:
                return []
            
            # Get CNN predictions
            predictions = self._get_cnn_predictions(symbol, df)
            
            # Filter out invalid predictions
            valid_predictions = [pred for pred in predictions if pred['is_valid']]
            
            return valid_predictions
        except Exception as e:
            logger.error(f"Error getting CNN pivot predictions: {e}")
            return []
    
    def _add_cnn_predictions_to_chart(self, fig: go.Figure, predictions: List[Dict[str, Any]], row: int):
        """Add CNN pivot predictions to chart"""
        try:
            if not predictions:
                return
            
            # Add predictions to chart
            for pred in predictions:
                if pred['swing_type'] == SwingType.RESISTANCE:
                    color = 'red'
                elif pred['swing_type'] == SwingType.SUPPORT:
                    color = 'green'
                else:
                    color = 'gray'
                
                fig.add_trace(
                    go.Scatter(
                        x=[pred['timestamp']],
                        y=[pred['price']],
                        mode='markers',
                        marker=dict(
                            size=10,
                            color=color,
                            symbol='circle-open',
                            line=dict(width=2, color='black')
                        ),
                        name=f"{pred['swing_type'].name} ({pred['price']:.2f})",
                        showlegend=False
                    ),
                    row=row, col=1
                )
        except Exception as e:
            logger.error(f"Error adding CNN predictions to chart: {e}")
    
    def _get_cnn_predictions(self, symbol: str, df: pd.DataFrame) -> List[Dict[str, Any]]:
        """Get CNN pivot predictions for chart"""
        try:
            if df is None or df.empty:
                return []
            
            # Get CNN predictions
            predictions = self._get_cnn_predictions_for_symbol(symbol, df)
            
            # Filter out invalid predictions
            valid_predictions = [pred for pred in predictions if pred['is_valid']]
            
            return valid_predictions
        except Exception as e:
            logger.error(f"Error getting CNN pivot predictions: {e}")
            return []
    
    def _get_cnn_predictions_for_symbol(self, symbol: str, df: pd.DataFrame) -> List[Dict[str, Any]]:
        """Get CNN pivot predictions for a specific symbol"""
        try:
            if df is None or df.empty:
                return []
            
            # Get CNN predictions
            predictions = self._get_cnn_predictions_for_symbol(symbol, df)
            
            # Filter out invalid predictions
            valid_predictions = [pred for pred in predictions if pred['is_valid']]
            
            return valid_predictions
        except Exception as e:
            logger.error(f"Error getting CNN pivot predictions for symbol: {e}")
            return []
    
    def _create_training_metrics(self) -> List[html.Div]:
        """Create training metrics display"""
        try:
            metrics = []
            
            # Add RL training metrics
            if self.rl_training_stats:
                metrics.append(html.P(f"RL Episodes: {self.rl_training_stats['rl_episodes']}", className="text-info"))
                metrics.append(html.P(f"RL Steps: {self.rl_training_stats['rl_steps']}", className="text-info"))
                metrics.append(html.P(f"RL Loss: {self.rl_training_stats['rl_loss']:.4f}", className="text-info"))
            
            # Add CNN training metrics
            if self.cnn_training_stats:
                metrics.append(html.P(f"CNN Epochs: {self.cnn_training_stats['cnn_epochs']}", className="text-info"))
                metrics.append(html.P(f"CNN Loss: {self.cnn_training_stats['cnn_loss']:.4f}", className="text-info"))
            
            # Add Enhanced RL training metrics
            if self.enhanced_rl_training_stats:
"""
Trading Dashboard - Clean Web Interface

This module provides a modern, responsive web dashboard for the trading system:
- Real-time price charts with multiple timeframes
- Model performance monitoring
- Trading decisions visualization
- System health monitoring
- Memory usage tracking
"""

import asyncio
import dash
from dash import Dash, dcc, html, Input, Output
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.express as px
import pandas as pd
import numpy as np
from datetime import datetime, timedelta, timezone
import pytz
import logging
import json
import time
import threading
from threading import Thread, Lock
from collections import deque
import warnings
from typing import Dict, List, Optional, Any, Union, Tuple
import websocket
import os
import torch

# Setup logger immediately after logging import
logger = logging.getLogger(__name__)

# WebSocket availability check
try:
    import websocket
    WEBSOCKET_AVAILABLE = True
    logger.info("WebSocket client available")
except ImportError:
    WEBSOCKET_AVAILABLE = False
    logger.warning("websocket-client not available. Real-time data will use API fallback.")

# Import trading system components
from core.config import get_config
from core.data_provider import DataProvider
from core.orchestrator import TradingOrchestrator, TradingDecision
from core.trading_executor import TradingExecutor
from core.trading_action import TradingAction
from models import get_model_registry

# Import CNN monitoring
try:
    from core.cnn_monitor import get_cnn_dashboard_data
    CNN_MONITORING_AVAILABLE = True
    logger.info("CNN monitoring system available")
except ImportError:
    CNN_MONITORING_AVAILABLE = False
    logger.warning("CNN monitoring not available")
    def get_cnn_dashboard_data():
        return {'statistics': {'total_predictions_logged': 0}}

# Import CNN prediction components
try:
    from training.williams_market_structure import SwingPoint, SwingType
    CNN_PREDICTIONS_AVAILABLE = True
    logger.info("CNN predictions available")
except ImportError:
    CNN_PREDICTIONS_AVAILABLE = False
    logger.warning("CNN predictions not available")
    class SwingPoint:
        def __init__(self, timestamp, price, index, swing_type, strength):
            self.timestamp = timestamp
            self.price = price
            self.index = index
            self.swing_type = swing_type
            self.strength = strength
    class SwingType:
        SWING_HIGH = "swing_high"
        SWING_LOW = "swing_low"


# Import enhanced RL components if available
try:
    from core.enhanced_orchestrator import EnhancedTradingOrchestrator
    from core.universal_data_adapter import UniversalDataAdapter
    from core.unified_data_stream import UnifiedDataStream, TrainingDataPacket, UIDataPacket
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL training components available")
except ImportError as e:
    logger.warning(f"Enhanced RL components not available: {e}")
    ENHANCED_RL_AVAILABLE = False
    # Force enable for learning - bypass import issues
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL FORCED ENABLED - bypassing import issues for learning")
    
    # Fallback classes
    class UnifiedDataStream:
        def __init__(self, *args, **kwargs): pass
        def register_consumer(self, *args, **kwargs): return "fallback_consumer"
        def start_streaming(self): pass
        def stop_streaming(self): pass
        def get_latest_training_data(self): return None
        def get_latest_ui_data(self): return None
    
    class TrainingDataPacket:
        def __init__(self, *args, **kwargs): pass
    
    class UIDataPacket:
        def __init__(self, *args, **kwargs): pass


class AdaptiveThresholdLearner:
    """Learn optimal confidence thresholds based on real trade outcomes"""
    
    def __init__(self, initial_threshold: float = 0.30):
        self.base_threshold = initial_threshold
        self.current_threshold = initial_threshold
        self.trade_outcomes = deque(maxlen=100)
        self.threshold_history = deque(maxlen=50)
        self.learning_rate = 0.02
        self.min_threshold = 0.20
        self.max_threshold = 0.70
        
        logger.info(f"[ADAPTIVE] Initialized with starting threshold: {initial_threshold:.2%}")
    
    def record_trade_outcome(self, confidence: float, pnl: float, threshold_used: float):
        """Record a trade outcome to learn from"""
        try:
            outcome = {
                'confidence': confidence,
                'pnl': pnl,
                'profitable': pnl > 0,
                'threshold_used': threshold_used,
                'timestamp': datetime.now()
            }
            
            self.trade_outcomes.append(outcome)
            
            # Learn from outcomes
            if len(self.trade_outcomes) >= 10:
                self._update_threshold()
            
        except Exception as e:
            logger.error(f"Error recording trade outcome: {e}")
    
    def _update_threshold(self):
        """Update threshold based on recent trade statistics"""
        try:
            recent_trades = list(self.trade_outcomes)[-20:]
            if len(recent_trades) < 10:
                return
            
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades)
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades)
            
            # Adaptive adjustment logic
            if win_rate > 0.60 and avg_pnl > 0.20:
                adjustment = -self.learning_rate * 1.5  # Lower threshold for more trades
            elif win_rate < 0.40 or avg_pnl < -0.30:
                adjustment = self.learning_rate * 2.0  # Raise threshold to be more selective
            else:
                adjustment = 0  # No change
            
            old_threshold = self.current_threshold
            self.current_threshold = max(self.min_threshold, 
                                       min(self.max_threshold, 
                                           self.current_threshold + adjustment))
            
            if abs(self.current_threshold - old_threshold) > 0.005:
                logger.info(f"[ADAPTIVE] Threshold: {old_threshold:.2%} -> {self.current_threshold:.2%} (WR: {win_rate:.1%}, PnL: ${avg_pnl:.2f})")
            
        except Exception as e:
            logger.error(f"Error updating adaptive threshold: {e}")
    
    def get_current_threshold(self) -> float:
        return self.current_threshold
    
    def get_learning_stats(self) -> Dict[str, Any]:
        """Get learning statistics"""
        try:
            if not self.trade_outcomes:
                return {'status': 'No trades recorded yet'}
            
            recent_trades = list(self.trade_outcomes)[-20:]
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades) if recent_trades else 0
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades) if recent_trades else 0
            
            return {
                'current_threshold': self.current_threshold,
                'base_threshold': self.base_threshold,
                'total_trades': len(self.trade_outcomes),
                'recent_win_rate': win_rate,
                'recent_avg_pnl': avg_pnl,
                'threshold_changes': len(self.threshold_history),
                'learning_active': len(self.trade_outcomes) >= 10
            }
        except Exception as e:
            return {'error': str(e)}

class TradingDashboard:
    """Enhanced Trading Dashboard with Williams pivot points and unified timezone handling"""
    
    def __init__(self, data_provider: DataProvider = None, orchestrator: TradingOrchestrator = None, trading_executor: TradingExecutor = None):
        self.app = Dash(__name__)
        
        # Initialize config first
        from core.config import get_config
        self.config = get_config()
        
        self.data_provider = data_provider or DataProvider()
        self.orchestrator = orchestrator
        self.trading_executor = trading_executor
        
        # Enhanced trading state with leverage support
        self.leverage_enabled = True
        self.leverage_multiplier = 50.0  # 50x leverage (adjustable via slider)
        self.base_capital = 10000.0
        self.current_position = 0.0  # -1 to 1 (short to long)
        self.position_size = 0.0
        self.entry_price = 0.0
        self.unrealized_pnl = 0.0
        self.realized_pnl = 0.0
        
        # Leverage settings for slider
        self.min_leverage = 1.0
        self.max_leverage = 100.0
        self.leverage_step = 1.0
        
        # Connect to trading server for leverage functionality
        self.trading_server_url = "http://127.0.0.1:8052"
        self.training_server_url = "http://127.0.0.1:8053"
        self.stream_server_url = "http://127.0.0.1:8054"
        
        # Enhanced performance tracking
        self.leverage_metrics = {
            'leverage_efficiency': 0.0,
            'margin_used': 0.0,
            'margin_available': 10000.0,
            'effective_exposure': 0.0,
            'risk_reward_ratio': 0.0
        }
        
        # Enhanced models will be loaded through model registry later
        
        # Rest of initialization...
        
        # Initialize timezone from config
        timezone_name = self.config.get('system', {}).get('timezone', 'Europe/Sofia')
        self.timezone = pytz.timezone(timezone_name)
        logger.info(f"Dashboard timezone set to: {timezone_name}")
        
        self.data_provider = data_provider or DataProvider()
        
        # Enhanced orchestrator support - FORCE ENABLE for learning
        self.orchestrator = orchestrator or TradingOrchestrator(self.data_provider)
        self.enhanced_rl_enabled = True  # Force enable Enhanced RL
        logger.info("Enhanced RL training FORCED ENABLED for learning")
        
        self.trading_executor = trading_executor or TradingExecutor()
        self.model_registry = get_model_registry()
        
        # Initialize unified data stream for comprehensive training data
        if ENHANCED_RL_AVAILABLE:
            self.unified_stream = UnifiedDataStream(self.data_provider, self.orchestrator)
            self.stream_consumer_id = self.unified_stream.register_consumer(
                consumer_name="TradingDashboard",
                callback=self._handle_unified_stream_data,
                data_types=['ticks', 'ohlcv', 'training_data', 'ui_data']
            )
            logger.info(f"Unified data stream initialized with consumer ID: {self.stream_consumer_id}")
        else:
            self.unified_stream = UnifiedDataStream()  # Fallback
            self.stream_consumer_id = "fallback"
            logger.warning("Using fallback unified data stream")
        
        # Dashboard state
        self.recent_decisions = []
        self.recent_signals = []  # Track all signals (not just executed trades)
        self.performance_data = {}
        self.current_prices = {}
        self.last_update = datetime.now()
        
        # Trading session tracking
        self.session_start = datetime.now()
        self.session_trades = []
        self.session_pnl = 0.0
        self.current_position = None  # {'side': 'BUY', 'price': 3456.78, 'size': 0.1, 'timestamp': datetime}
        self.total_realized_pnl = 0.0
        self.total_fees = 0.0
        self.starting_balance = self._get_initial_balance()  # Get balance from MEXC or default to 100
        
        # Closed trades tracking for accounting
        self.closed_trades = []  # List of all closed trades with full details
        
        # Load existing closed trades from file
        self._load_closed_trades_from_file()
        
        # Signal execution settings for scalping - REMOVED FREQUENCY LIMITS
        self.min_confidence_threshold = 0.30  # Start lower to allow learning  
        self.signal_cooldown = 0  # REMOVED: Model decides when to act, no artificial delays
        self.last_signal_time = 0
        
        # Adaptive threshold learning - starts low and learns optimal thresholds
        self.adaptive_learner = AdaptiveThresholdLearner(initial_threshold=0.30)
        logger.info("[ADAPTIVE] Adaptive threshold learning enabled - will adjust based on trade outcomes")
        
        # Lightweight WebSocket implementation for real-time scalping data
        self.ws_price_cache = {}  # Just current prices, no tick history
        self.ws_connection = None
        self.ws_thread = None
        self.is_streaming = False
        
        # Performance-focused: only track essentials
        self.last_ws_update = 0
        self.ws_update_count = 0
        
        # Compatibility stubs for removed tick infrastructure 
        self.tick_cache = []  # Empty list for compatibility
        self.one_second_bars = []  # Empty list for compatibility
        
        # Enhanced RL Training System - Train on closed trades with comprehensive data
        self.rl_training_enabled = True
        # Force enable Enhanced RL training (bypass import issues)
        self.enhanced_rl_training_enabled = True  # Force enabled for CNN training
        self.enhanced_rl_enabled = True  # Force enabled to show proper status
        self.rl_training_stats = {
            'total_training_episodes': 0,
            'profitable_trades_trained': 0,
            'unprofitable_trades_trained': 0,
            'last_training_time': None,
            'training_rewards': deque(maxlen=100),  # Last 100 training rewards
            'model_accuracy_trend': deque(maxlen=50),  # Track accuracy over time
            'enhanced_rl_episodes': 0,
            'comprehensive_data_packets': 0
        }
        self.rl_training_queue = deque(maxlen=1000)  # Queue of trades to train on
        
        # Enhanced training data tracking
        self.latest_training_data = None
        self.latest_ui_data = None
        self.training_data_available = False
        
        # Load available models for real trading
        self._load_available_models()
        
        # Preload essential data to prevent excessive API calls during dashboard updates
        logger.info("Preloading essential market data to cache...")
        try:
            # Preload key timeframes for main symbols to ensure cache is populated
            symbols_to_preload = self.config.symbols or ['ETH/USDT', 'BTC/USDT']
            timeframes_to_preload = ['1m', '1h', '1d']  # Skip 1s since we use WebSocket for that
            
            for symbol in symbols_to_preload[:2]:  # Limit to first 2 symbols
                for timeframe in timeframes_to_preload:
                    try:
                        # Load data into cache (refresh=True for initial load, then cache will be used)
                        df = self.data_provider.get_historical_data(symbol, timeframe, limit=100, refresh=True)
                        if df is not None and not df.empty:
                            logger.info(f"Preloaded {len(df)} {timeframe} bars for {symbol}")
                        else:
                            logger.warning(f"Failed to preload data for {symbol} {timeframe}")
                    except Exception as e:
                        logger.warning(f"Error preloading {symbol} {timeframe}: {e}")
            
            logger.info("Preloading completed - cache populated for frequent queries")
            
        except Exception as e:
            logger.warning(f"Error during preloading: {e}")
        
        # Create Dash app
        self.app = dash.Dash(__name__, external_stylesheets=[
            'https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css',
            'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css'
        ])
        
        # # Add custom CSS for model data charts
        # self.app.index_string = '''
        # <!DOCTYPE html>
        # <html>
        #     <head>
        #         {%metas%}
        #         <title>{%title%}</title>
        #         {%favicon%}
        #         {%css%}
        #         <style>
        #             .tiny { font-size: 10px !important; }
        #             .model-data-chart .js-plotly-plot .plotly .modebar { display: none !important; }
        #             .model-data-chart .js-plotly-plot .plotly .svg-container { border: 1px solid #444; border-radius: 4px; }
        #         </style>
        #     </head>
        #     <body>
        #         {%app_entry%}
        #         <footer>
        #             {%config%}
        #             {%scripts%}
        #             {%renderer%}
        #         </footer>
        #     </body>
        # </html>
        # '''
        
        # Setup layout and callbacks
        self._setup_layout()
        self._setup_callbacks()
        
        # Start unified data streaming
        self._initialize_streaming()
        
        # Start continuous training with enhanced RL support
        self.start_continuous_training()
        
        logger.info("Trading Dashboard initialized with enhanced RL training integration")
        logger.info(f"Enhanced RL enabled: {self.enhanced_rl_training_enabled}")
        logger.info(f"Stream consumer ID: {self.stream_consumer_id}")
        
        # Initialize Williams Market Structure once
        try:
            from training.williams_market_structure import WilliamsMarketStructure
            self.williams_structure = WilliamsMarketStructure(
                swing_strengths=[2, 3, 5],  # Simplified for better performance
                enable_cnn_feature=True,   # Enable CNN training and inference
                training_data_provider=self.data_provider  # Provide data access for training
            )
            logger.info("Williams Market Structure initialized for dashboard with CNN training enabled")
        except ImportError:
            self.williams_structure = None
            logger.warning("Williams Market Structure not available")
        
        # Initialize Enhanced Pivot RL Trainer for better position management
        try:
            self.pivot_rl_trainer = create_enhanced_pivot_trainer(
                data_provider=self.data_provider,
                orchestrator=self.orchestrator
            )
            logger.info("Enhanced Pivot RL Trainer initialized for better entry/exit decisions")
            logger.info(f"Entry threshold: {self.pivot_rl_trainer.get_current_thresholds()['entry_threshold']:.1%}")
            logger.info(f"Exit threshold: {self.pivot_rl_trainer.get_current_thresholds()['exit_threshold']:.1%}")
            logger.info(f"Uninvested threshold: {self.pivot_rl_trainer.get_current_thresholds()['uninvested_threshold']:.1%}")
        except Exception as e:
            self.pivot_rl_trainer = None
            logger.warning(f"Enhanced Pivot RL Trainer not available: {e}")
    
    def _to_local_timezone(self, dt) -> datetime:
        """Convert datetime to configured local timezone"""
        try:
            if dt is None:
                return None
            
            # Handle string timestamps by converting to datetime first
            if isinstance(dt, str):
                try:
                    dt = pd.to_datetime(dt)
                except Exception:
                    logger.warning(f"Could not parse timestamp string: {dt}")
                    return datetime.now(self.timezone)
            
            # Handle pandas Timestamp
            if isinstance(dt, pd.Timestamp):
                dt = dt.to_pydatetime()
            
            # If datetime is naive, assume it's UTC
            if dt.tzinfo is None:
                dt = pytz.UTC.localize(dt)
            
            # Convert to local timezone
            return dt.astimezone(self.timezone)
        except Exception as e:
            logger.warning(f"Error converting timezone for {dt}: {e}")
            return datetime.now(self.timezone)  # Return current time as fallback
    
    def _now_local(self) -> datetime:
        """Get current time in configured local timezone"""
        return datetime.now(self.timezone)
    
    def _ensure_timezone_consistency(self, df: pd.DataFrame) -> pd.DataFrame:
        """Ensure DataFrame index is in consistent timezone"""
        try:
            if hasattr(df.index, 'tz'):
                if df.index.tz is None:
                    # Assume UTC if no timezone
                    df.index = df.index.tz_localize('UTC')
                
                # Convert to local timezone
                df.index = df.index.tz_convert(self.timezone)
            
            return df
        except Exception as e:
            logger.warning(f"Error ensuring timezone consistency: {e}")
            return df
    
    def _initialize_streaming(self):
        """Initialize unified data streaming and WebSocket fallback"""
        try:
            # Start lightweight WebSocket for real-time price updates
            self._start_lightweight_websocket()
            logger.info("Lightweight WebSocket streaming initialized")
            
            if ENHANCED_RL_AVAILABLE:
                # Start unified data stream in background
                def start_unified_stream():
                    try:
                        asyncio.run(self.unified_stream.start_streaming())
                        logger.info("Unified data stream started")
                    except Exception as e:
                        logger.error(f"Error starting unified stream: {e}")
                
                unified_thread = Thread(target=start_unified_stream, daemon=True)
                unified_thread.start()
            
            # Start background data collection
            self._start_enhanced_training_data_collection()
            
            logger.info("All data streaming initialized")
            
        except Exception as e:
            logger.error(f"Error initializing streaming: {e}")
            # Ensure lightweight WebSocket is started as fallback
            self._start_lightweight_websocket()
    
    def _start_enhanced_training_data_collection(self):
        """Start enhanced training data collection using unified stream"""
        def enhanced_training_loop():
            try:
                logger.info("Enhanced training data collection started with unified stream")
                
                while True:
                    try:
                        if ENHANCED_RL_AVAILABLE and self.enhanced_rl_training_enabled:
                            # Get latest comprehensive training data from unified stream
                            training_data = self.unified_stream.get_latest_training_data()
                            
                            if training_data:
                                # Send comprehensive training data to enhanced RL pipeline
                                self._send_comprehensive_training_data_to_enhanced_rl(training_data)
                                
                                # Update training statistics
                                self.rl_training_stats['comprehensive_data_packets'] += 1
                                self.training_data_available = True
                                
                                # Update context data in orchestrator
                                if hasattr(self.orchestrator, 'update_context_data'):
                                    self.orchestrator.update_context_data()
                                
                                # Initialize extrema trainer if not done
                                if hasattr(self.orchestrator, 'extrema_trainer'):
                                    if not hasattr(self.orchestrator.extrema_trainer, '_initialized'):
                                        self.orchestrator.extrema_trainer.initialize_context_data()
                                        self.orchestrator.extrema_trainer._initialized = True
                                        logger.info("Extrema trainer context data initialized")
                                
                                # Run extrema detection with real data
                                if hasattr(self.orchestrator, 'extrema_trainer'):
                                    for symbol in self.orchestrator.symbols:
                                        detected = self.orchestrator.extrema_trainer.detect_local_extrema(symbol)
                                        if detected:
                                            logger.debug(f"Detected {len(detected)} extrema for {symbol}")
                        else:
                            # Fallback to basic training data collection
                            self._collect_basic_training_data()
                        
                        time.sleep(10)  # Update every 10 seconds for enhanced training
                        
                    except Exception as e:
                        logger.error(f"Error in enhanced training loop: {e}")
                        time.sleep(30)  # Wait before retrying
                        
            except Exception as e:
                logger.error(f"Enhanced training loop failed: {e}")
        
        # Start enhanced training thread
        training_thread = Thread(target=enhanced_training_loop, daemon=True)
        training_thread.start()
        logger.info("Enhanced training data collection thread started")
    
    def _handle_unified_stream_data(self, data_packet: Dict[str, Any]):
        """Handle data from unified stream for dashboard and training"""
        try:
            # Extract UI data for dashboard display
            if 'ui_data' in data_packet:
                self.latest_ui_data = data_packet['ui_data']
                if hasattr(self.latest_ui_data, 'current_prices'):
                    self.current_prices.update(self.latest_ui_data.current_prices)
                if hasattr(self.latest_ui_data, 'streaming_status'):
                    self.is_streaming = self.latest_ui_data.streaming_status == 'LIVE'
                if hasattr(self.latest_ui_data, 'training_data_available'):
                    self.training_data_available = self.latest_ui_data.training_data_available
            
            # Extract training data for enhanced RL
            if 'training_data' in data_packet:
                self.latest_training_data = data_packet['training_data']
                logger.debug("Received comprehensive training data from unified stream")
            
            # Extract tick data for dashboard charts
            if 'ticks' in data_packet:
                ticks = data_packet['ticks']
                for tick in ticks[-100:]:  # Keep last 100 ticks
                    self.tick_cache.append(tick)
            
            # Extract OHLCV data for dashboard charts
            if 'one_second_bars' in data_packet:
                bars = data_packet['one_second_bars']
                for bar in bars[-100:]:  # Keep last 100 bars
                    self.one_second_bars.append(bar)
            
            logger.debug(f"Processed unified stream data packet with keys: {list(data_packet.keys())}")
            
        except Exception as e:
            logger.error(f"Error handling unified stream data: {e}")
    
    def _send_comprehensive_training_data_to_enhanced_rl(self, training_data: TrainingDataPacket):
        """Send comprehensive training data to enhanced RL training pipeline"""
        try:
            if not self.enhanced_rl_training_enabled:
                logger.debug("Enhanced RL training not enabled, skipping comprehensive data send")
                return
            
            # Extract comprehensive training data components
            market_state = training_data.market_state if hasattr(training_data, 'market_state') else None
            universal_stream = training_data.universal_stream if hasattr(training_data, 'universal_stream') else None
            cnn_features = training_data.cnn_features if hasattr(training_data, 'cnn_features') else None
            cnn_predictions = training_data.cnn_predictions if hasattr(training_data, 'cnn_predictions') else None
            
            if market_state and universal_stream:
                # Send to enhanced RL trainer if available
                if hasattr(self.orchestrator, 'enhanced_rl_trainer'):
                    try:
                        # Create comprehensive training step with ~13,400 features
                        asyncio.run(self.orchestrator.enhanced_rl_trainer.training_step(universal_stream))
                        self.rl_training_stats['enhanced_rl_episodes'] += 1
                        logger.debug("Sent comprehensive data to enhanced RL trainer")
                    except Exception as e:
                        logger.warning(f"Error in enhanced RL training step: {e}")
                
                # Send to extrema trainer for CNN training with perfect moves
                if hasattr(self.orchestrator, 'extrema_trainer'):
                    try:
                        extrema_data = self.orchestrator.extrema_trainer.get_extrema_training_data(count=50)
                        perfect_moves = self.orchestrator.extrema_trainer.get_perfect_moves_for_cnn(count=100)
                        
                        if extrema_data:
                            logger.debug(f"Enhanced RL: {len(extrema_data)} extrema training samples available")
                            
                        if perfect_moves:
                            logger.debug(f"Enhanced RL: {len(perfect_moves)} perfect moves for CNN training")
                    except Exception as e:
                        logger.warning(f"Error getting extrema training data: {e}")
                
                # Send to sensitivity learning DQN for outcome-based learning
                if hasattr(self.orchestrator, 'sensitivity_learning_queue'):
                    try:
                        if len(self.orchestrator.sensitivity_learning_queue) > 0:
                            logger.debug("Enhanced RL: Sensitivity learning data available for DQN training")
                    except Exception as e:
                        logger.warning(f"Error accessing sensitivity learning queue: {e}")
                
                # Get context features for models with real market data
                if hasattr(self.orchestrator, 'extrema_trainer'):
                    try:
                        for symbol in self.orchestrator.symbols:
                            context_features = self.orchestrator.extrema_trainer.get_context_features_for_model(symbol)
                            if context_features is not None:
                                logger.debug(f"Enhanced RL: Context features available for {symbol}: {context_features.shape}")
                    except Exception as e:
                        logger.warning(f"Error getting context features: {e}")
            
            # Log comprehensive training data statistics
            tick_count = len(training_data.tick_cache) if hasattr(training_data, 'tick_cache') else 0
            bars_count = len(training_data.one_second_bars) if hasattr(training_data, 'one_second_bars') else 0
            timeframe_count = len(training_data.multi_timeframe_data) if hasattr(training_data, 'multi_timeframe_data') else 0
            
            logger.info(f"Enhanced RL Comprehensive Training Data:")
            logger.info(f"  Tick cache: {tick_count} ticks")
            logger.info(f"  1s bars: {bars_count} bars")
            logger.info(f"  Multi-timeframe data: {timeframe_count} symbols")
            logger.info(f"  CNN features: {'Available' if cnn_features else 'Not available'}")
            logger.info(f"  CNN predictions: {'Available' if cnn_predictions else 'Not available'}")
            logger.info(f"  Market state: {'Available (~13,400 features)' if market_state else 'Not available'}")
            logger.info(f"  Universal stream: {'Available' if universal_stream else 'Not available'}")
                
        except Exception as e:
            logger.error(f"Error sending comprehensive training data to enhanced RL: {e}")
    
    def _collect_basic_training_data(self):
        """Fallback method to collect basic training data when enhanced RL is not available"""
        try:
            # Get real tick data from data provider subscribers
            for symbol in ['ETH/USDT', 'BTC/USDT']:
                try:
                    # Get recent ticks from data provider
                    if hasattr(self.data_provider, 'get_recent_ticks'):
                        recent_ticks = self.data_provider.get_recent_ticks(symbol, count=10)
                        
                        for tick in recent_ticks:
                            # Create tick data from real market data
                            tick_data = {
                                'symbol': tick.symbol,
                                'price': tick.price,
                                'timestamp': tick.timestamp,
                                'volume': tick.volume
                            }
                            
                            # Add to tick cache
                            self.tick_cache.append(tick_data)
                            
                            # Create 1s bar data from real tick
                            bar_data = {
                                'symbol': tick.symbol,
                                'open': tick.price,
                                'high': tick.price,
                                'low': tick.price,
                                'close': tick.price,
                                'volume': tick.volume,
                                'timestamp': tick.timestamp
                            }
                            
                            # Add to 1s bars cache
                            self.one_second_bars.append(bar_data)
                            
                except Exception as e:
                    logger.debug(f"No recent tick data available for {symbol}: {e}")
                    
            # Set streaming status based on real data availability
            self.is_streaming = len(self.tick_cache) > 0
            
        except Exception as e:
            logger.warning(f"Error in basic training data collection: {e}")
    
    def _get_initial_balance(self) -> float:
        """Get initial USDT balance from MEXC or return default"""
        try:
            if self.trading_executor and hasattr(self.trading_executor, 'get_account_balance'):
                logger.info("Fetching initial balance from MEXC...")
                
                # Check if trading is enabled and not in dry run mode
                if not self.trading_executor.trading_enabled:
                    logger.warning("MEXC: Trading not enabled - using default balance")
                elif self.trading_executor.simulation_mode:
                    logger.warning(f"MEXC: {self.trading_executor.trading_mode.upper()} mode enabled - using default balance")
                else:
                    # Get USDT balance from MEXC
                    balance_info = self.trading_executor.get_account_balance()
                    if balance_info and 'USDT' in balance_info:
                        usdt_balance = float(balance_info['USDT'].get('free', 0))
                        if usdt_balance > 0:
                            logger.info(f"MEXC: Retrieved USDT balance: ${usdt_balance:.2f}")
                            return usdt_balance
                        else:
                            logger.warning("MEXC: No USDT balance found in account")
                    else:
                        logger.error("MEXC: Failed to retrieve balance info from API")
            else:
                logger.info("MEXC: Trading executor not available for balance retrieval")
                
        except Exception as e:
            logger.error(f"Error getting MEXC balance: {e}")
            import traceback
            logger.error(traceback.format_exc())
        
        # Fallback to default
        default_balance = 100.0
        logger.warning(f"Using default starting balance: ${default_balance:.2f}")
        return default_balance
    
    def _setup_layout(self):
        """Setup the dashboard layout"""
        self.app.layout = html.Div([
            # Compact Header
            html.Div([
                html.H3([
                    html.I(className="fas fa-chart-line me-2"),
                    "Live Trading Dashboard"
                ], className="text-white mb-1"),
                html.P(f"Ultra-Fast Updates • Portfolio: ${self.starting_balance:,.0f} • {'MEXC Live' if (self.trading_executor and self.trading_executor.trading_enabled and not self.trading_executor.simulation_mode) else 'Demo Mode'}", 
                      className="text-light mb-0 opacity-75 small")
            ], className="bg-dark p-2 mb-2"),
            
            # Auto-refresh component
            dcc.Interval(
                id='interval-component',
                interval=1000,  # Update every 1 second for real-time tick updates
                n_intervals=0
            ),
            
            # Main content - Compact layout
            html.Div([
                # Top row - Key metrics and Recent Signals (split layout)
                html.Div([
                    # Left side - Key metrics (compact cards)
                    html.Div([
                        html.Div([
                            html.Div([
                                html.H5(id="current-price", className="text-success mb-0 small"),
                                html.P("Live Price", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="session-pnl", className="mb-0 small"),
                                html.P("Session P&L", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="total-fees", className="text-warning mb-0 small"),
                                html.P("Total Fees", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="current-position", className="text-info mb-0 small"),
                                html.P("Position", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="trade-count", className="text-warning mb-0 small"),
                                html.P("Trades", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="portfolio-value", className="text-secondary mb-0 small"),
                                html.P("Portfolio", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="mexc-status", className="text-info mb-0 small"),
                                html.P("MEXC API", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                    ], style={"display": "grid", "gridTemplateColumns": "repeat(4, 1fr)", "gap": "8px", "width": "60%"}),
                    
                    # Right side - Recent Signals & Executions
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-robot me-2"),
                                "Recent Trading Signals & Executions"
                            ], className="card-title mb-2"),
                            html.Div(id="recent-decisions", style={"height": "160px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "48%", "marginLeft": "2%"})
                ], className="d-flex mb-3"),
                
                # Charts row - More compact
                html.Div([
                    # Price chart - 70% width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-chart-candlestick me-2"),
                                "Live 1s Price & Volume Chart (WebSocket Stream)"
                            ], className="card-title mb-2"),
                            dcc.Graph(id="price-chart", style={"height": "400px"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "70%"}),
                    
                    # Model Training Metrics - 30% width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-brain me-2"),
                                "Model Training Progress"
                            ], className="card-title mb-2"),
                            html.Div(id="training-metrics", style={"height": "400px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "28%", "marginLeft": "2%"}),
                ], className="row g-2 mb-3"),
                
                # CNN Model Monitoring Section
                html.Div([
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-brain me-2"),
                                "CNN Model Analysis & Predictions"
                            ], className="card-title mb-2"),
                            html.Div(id="cnn-monitoring-content", style={"height": "350px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card")
                ], className="mb-3"),
                
                # Bottom row - Session performance and system status
                html.Div([                               

                    # Session performance - 1/3 width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-chart-pie me-2"),
                                "Session Performance"
                            ], className="card-title mb-2"),
                            html.Button(
                                "Clear Session",
                                id="clear-history-btn",
                                className="btn btn-sm btn-outline-danger mb-2",
                                n_clicks=0
                            ),
                            html.Div(id="session-performance")
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%"}),
                                  
                    # Closed Trades History - 1/3 width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-history me-2"),
                                "Closed Trades History"
                            ], className="card-title mb-2"),
                            html.Div([
                                html.Div(
                                    id="closed-trades-table",
                                    style={"height": "300px", "overflowY": "auto"}
                                )
                            ])
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%", "marginLeft": "2%"}),
                    
                    # System status and leverage controls - 1/3 width with icon tooltip
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-server me-2"),
                                "System & Leverage"
                            ], className="card-title mb-2"),
                            
                            # System status
                            html.Div([
                                html.I(
                                    id="system-status-icon",
                                    className="fas fa-circle text-success fa-2x",
                                    title="System Status: All systems operational",
                                    style={"cursor": "pointer"}
                                ),
                                html.Div(id="system-status-details", className="small mt-2")
                            ], className="text-center mb-3"),
                            
                            # Leverage Controls
                            html.Div([
                                html.Label([
                                    html.I(className="fas fa-chart-line me-1"),
                                    "Leverage Multiplier"
                                ], className="form-label small fw-bold"),
                                html.Div([
                                    dcc.Slider(
                                        id='leverage-slider',
                                        min=self.min_leverage,
                                        max=self.max_leverage,
                                        step=self.leverage_step,
                                        value=self.leverage_multiplier,
                                        marks={
                                            1: '1x',
                                            10: '10x',
                                            25: '25x',
                                            50: '50x',
                                            75: '75x',
                                            100: '100x'
                                        },
                                        tooltip={
                                            "placement": "bottom",
                                            "always_visible": True
                                        }
                                    )
                                ], className="mb-2"),
                                html.Div([
                                    html.Span(id="current-leverage", className="badge bg-warning text-dark"),
                                    html.Span(" • ", className="mx-1"),
                                    html.Span(id="leverage-risk", className="badge bg-info")
                                ], className="text-center"),
                                html.Div([
                                    html.Small("Higher leverage = Higher rewards & risks", className="text-muted")
                                ], className="text-center mt-1")
                            ])
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%", "marginLeft": "2%"})
                ], className="d-flex")
            ], className="container-fluid")
        ])
    
    def _setup_callbacks(self):
        """Setup dashboard callbacks for real-time updates"""
        
        @self.app.callback(
            [
                Output('current-price', 'children'),
                Output('session-pnl', 'children'),
                Output('session-pnl', 'className'),
                Output('total-fees', 'children'),
                Output('current-position', 'children'),
                Output('current-position', 'className'),
                Output('trade-count', 'children'),
                Output('portfolio-value', 'children'),
                Output('mexc-status', 'children'),
                Output('price-chart', 'figure'),
                Output('training-metrics', 'children'),
                Output('recent-decisions', 'children'),
                Output('session-performance', 'children'),
                Output('closed-trades-table', 'children'),
                Output('system-status-icon', 'className'),
                Output('system-status-icon', 'title'),
                Output('system-status-details', 'children'),
                Output('current-leverage', 'children'),
                Output('leverage-risk', 'children'),
                Output('cnn-monitoring-content', 'children')
            ],
            [Input('interval-component', 'n_intervals')]
        )
        def update_dashboard(n_intervals):
            """Update all dashboard components with trading signals"""
            start_time = time.time()  # Performance monitoring
            try:
                # Periodic cleanup to prevent memory leaks
                if n_intervals % 60 == 0:  # Every 60 seconds
                    self._cleanup_old_data()
                
                # Lightweight update every 10 intervals to reduce load
                is_lightweight_update = (n_intervals % 10 != 0)
                # Chart updates every second for responsiveness
                # Get current prices with improved fallback handling
                symbol = self.config.symbols[0] if self.config.symbols else "ETH/USDT"
                current_price = None
                chart_data = None
                data_source = "UNKNOWN"
                
                try:
                    # First try real-time WebSocket price (sub-second latency)
                    current_price = self.get_realtime_price(symbol)
                    if current_price:
                        data_source = "WEBSOCKET_RT"
                        logger.debug(f"[WS_RT] Using real-time WebSocket price for {symbol}: ${current_price:.2f}")
                    else:
                        # Try cached data first (faster than API calls)
                        cached_data = self.data_provider.get_historical_data(symbol, '1m', limit=1, refresh=False)
                        if cached_data is not None and not cached_data.empty:
                            current_price = float(cached_data['close'].iloc[-1])
                            data_source = "CACHED"
                            logger.debug(f"[CACHED] Using cached price for {symbol}: ${current_price:.2f}")
                        else:
                            # Only try fresh API call if we have no data at all
                            try:
                                fresh_data = self.data_provider.get_historical_data(symbol, '1m', limit=1, refresh=False)
                                if fresh_data is not None and not fresh_data.empty:
                                    current_price = float(fresh_data['close'].iloc[-1])
                                    data_source = "API"
                                    logger.debug(f"[API] Fresh price for {symbol}: ${current_price:.2f}")
                            except Exception as api_error:
                                logger.warning(f"[API_ERROR] Failed to fetch fresh data: {api_error}")
                        
                        # NO SYNTHETIC DATA - Wait for real data
                        if current_price is None:
                            logger.warning(f"[NO_DATA] No real data available for {symbol} - waiting for data provider")
                            data_source = "NO_DATA"
                            
                except Exception as e:
                    logger.warning(f"[ERROR] Error getting price for {symbol}: {e}")
                    current_price = None
                    data_source = "ERROR"
                
                # Get chart data - ONLY REAL DATA (optimized for performance)
                chart_data = None
                try:
                    if not is_lightweight_update:  # Only refresh charts every 10 seconds
                        # Use cached data only (limited to 30 bars for performance)
                        chart_data = self.data_provider.get_historical_data(symbol, '1m', limit=30, refresh=False)
                        if chart_data is not None and not chart_data.empty:
                            logger.debug(f"[CHART] Using cached 1m data: {len(chart_data)} bars")
                        else:
                            # Wait for real data - no synthetic data
                            logger.debug("[CHART] No chart data available - waiting for data provider")
                            chart_data = None
                    else:
                        # Use cached chart data for lightweight updates
                        chart_data = getattr(self, '_cached_chart_data', None)
                except Exception as e:
                    logger.warning(f"[CHART_ERROR] Error getting chart data: {e}")
                    chart_data = None
                
                # Generate trading signals based on model decisions - OPTIMIZED
                try:
                    # Only generate signals every few intervals to reduce CPU load
                    if not is_lightweight_update and current_price and chart_data is not None and not chart_data.empty and len(chart_data) >= 5:
                        # Model decides when to act - check for signals but not every single second
                        signal = self._generate_trading_signal(symbol, current_price, chart_data)
                        if signal:
                            # Add to signals list (all signals, regardless of execution)
                            signal['signal_type'] = 'GENERATED'
                            self.recent_signals.append(signal.copy())
                            if len(self.recent_signals) > 100:  # Keep last 100 signals
                                self.recent_signals = self.recent_signals[-100:]
                            
                            # Use adaptive threshold instead of fixed threshold
                            current_threshold = self.adaptive_learner.get_current_threshold()
                            should_execute = signal['confidence'] >= current_threshold
                            
                            # Check position limits before execution
                            can_execute = self._can_execute_new_position(signal['action'])
                            
                            if should_execute and can_execute:
                                signal['signal_type'] = 'EXECUTED'
                                signal['threshold_used'] = current_threshold  # Track threshold for learning
                                signal['reason'] = f"ADAPTIVE EXECUTE (≥{current_threshold:.2%}): {signal['reason']}"
                                logger.debug(f"[EXECUTE] {signal['action']} signal @ ${signal['price']:.2f} (confidence: {signal['confidence']:.1%} ≥ {current_threshold:.1%})")
                                self._process_trading_decision(signal)
                            elif should_execute and not can_execute:
                                # Signal meets confidence but we're at position limit
                                signal['signal_type'] = 'NOT_EXECUTED_POSITION_LIMIT'
                                signal['threshold_used'] = current_threshold
                                signal['reason'] = f"BLOCKED BY POSITION LIMIT (≥{current_threshold:.2%}): {signal['reason']} [Positions: {self._count_open_positions()}/{self.config.get('trading', {}).get('max_concurrent_positions', 3)}]"
                                logger.info(f"[BLOCKED] {signal['action']} signal @ ${signal['price']:.2f} - Position limit reached ({self._count_open_positions()}/{self.config.get('trading', {}).get('max_concurrent_positions', 3)})")
                                
                                # Still add to training queue for RL learning
                                self._queue_signal_for_training(signal, current_price, symbol)
                            else:
                                signal['signal_type'] = 'NOT_EXECUTED_LOW_CONFIDENCE'
                                signal['threshold_used'] = current_threshold
                                signal['reason'] = f"LOW CONFIDENCE (<{current_threshold:.2%}): {signal['reason']}"
                                logger.debug(f"[SKIP] {signal['action']} signal @ ${signal['price']:.2f} (confidence: {signal['confidence']:.1%} < {current_threshold:.1%})")
                                
                                # Still add to training queue for RL learning
                                self._queue_signal_for_training(signal, current_price, symbol)
                    else:
                        # Fallback: Add a simple monitoring update
                        if n_intervals % 10 == 0 and current_price:  # Every 10 seconds
                            monitor_signal = {
                                'action': 'MONITOR',
                                'symbol': symbol,
                                'price': current_price,
                                'confidence': 0.0,
                                'timestamp': datetime.now(),
                                'size': 0.0,
                                'reason': 'System monitoring - no trading signals',
                                'signal_type': 'MONITOR'
                            }
                            self.recent_decisions.append(monitor_signal)
                            if len(self.recent_decisions) > 500:
                                self.recent_decisions = self.recent_decisions[-500:]
                                
                except Exception as e:
                    logger.warning(f"[ERROR] Error generating trading signal: {e}")
                
                # Calculate PnL metrics
                unrealized_pnl = self._calculate_unrealized_pnl(current_price) if current_price else 0.0
                total_session_pnl = self.total_realized_pnl + unrealized_pnl
                
                # Calculate portfolio value
                portfolio_value = self.starting_balance + total_session_pnl
                
                # Get memory stats with fallback (still needed for system status)
                try:
                    memory_stats = self.model_registry.get_memory_stats()
                except:
                    memory_stats = {'utilization_percent': 0, 'total_used_mb': 0, 'total_limit_mb': 1024}
                
                # Format outputs with safe defaults and update indicators
                update_time = datetime.now().strftime("%H:%M:%S.%f")[:-3]  # Include milliseconds
                
                if current_price:
                    # Add data source indicator and precise timestamp
                    source_indicator = f"[{data_source}]"
                    price_text = f"${current_price:.2f} {source_indicator} @ {update_time}"
                else:
                    # Show waiting status when no real data
                    price_text = f"WAITING FOR REAL DATA [{data_source}] @ {update_time}"
                
                # PnL formatting
                pnl_text = f"${total_session_pnl:.2f}"
                pnl_class = "text-success mb-0 small" if total_session_pnl >= 0 else "text-danger mb-0 small"
                
                # Total fees formatting
                fees_text = f"${self.total_fees:.2f}"
                
                # Position info with real-time unrealized PnL and proper color coding
                if self.current_position:
                    pos_side = self.current_position['side']
                    pos_size = self.current_position['size']
                    pos_price = self.current_position['price']
                    unrealized_pnl = self._calculate_unrealized_pnl(current_price) if current_price else 0.0
                    
                    # Color coding: LONG=Green, SHORT=Red (consistent with trading conventions)
                    if pos_side == 'LONG':
                        side_icon = "[LONG]"
                        side_color = "success"  # Green for long positions
                    else:  # SHORT
                        side_icon = "[SHORT]"
                        side_color = "danger"   # Red for short positions
                    
                    # Create enhanced position display with bold styling
                    pnl_sign = "+" if unrealized_pnl > 0 else ""
                    position_text = f"{side_icon} {pos_size} @ ${pos_price:.2f} | P&L: {pnl_sign}${unrealized_pnl:.2f}"
                    position_class = f"text-{side_color} fw-bold mb-0 small"
                else:
                    position_text = "No Position"
                    position_class = "text-muted mb-0 small"
                
                # Trade count and portfolio value
                trade_count_text = f"{len(self.session_trades)}"
                portfolio_text = f"${portfolio_value:,.2f}"
                
                # MEXC status with detailed information
                if self.trading_executor and self.trading_executor.trading_enabled:
                    if self.trading_executor.simulation_mode:
                        mexc_status = f"{self.trading_executor.trading_mode.upper()} MODE"
                    else:
                        mexc_status = "LIVE"
                else:
                    mexc_status = "OFFLINE"
                
                # Create charts with error handling - OPTIMIZED
                try:
                    # Always try to create/update chart every second for smooth responsiveness
                    if current_price and chart_data is not None and not chart_data.empty:
                        price_chart = self._create_price_chart(symbol)
                        self._cached_chart_data = chart_data  # Cache for fallback
                        self._cached_price_chart = price_chart  # Cache chart
                    else:
                        # Use cached chart if we have one, otherwise show loading
                        if hasattr(self, '_cached_price_chart') and self._cached_price_chart:
                            price_chart = self._cached_price_chart
                            # Update the cached chart with current info
                            try:
                                current_time_str = datetime.now().strftime("%H:%M:%S")
                                stream_status = "LIVE STREAM" if self.is_streaming else "WAITING DATA"
                                price_chart.update_layout(
                                    title=f"{symbol} 1M CHART | ${current_price or 0:.2f} | {stream_status} | {current_time_str}"
                                )
                            except Exception as e:
                                logger.debug(f"Error updating cached chart: {e}")
                        else:
                            price_chart = self._create_empty_chart("Price Chart", "Waiting for real market data...")
                            self._cached_price_chart = price_chart
                except Exception as e:
                    logger.warning(f"Price chart error: {e}")
                    price_chart = self._create_empty_chart("Price Chart", "Error loading chart - waiting for data")
                
                # Create training metrics display
                try:
                    training_metrics = self._create_training_metrics()
                except Exception as e:
                    logger.warning(f"Training metrics error: {e}")
                    training_metrics = [html.P("Training metrics unavailable", className="text-muted")]
                
                # Create recent decisions list
                try:
                    decisions_list = self._create_decisions_list()
                except Exception as e:
                    logger.warning(f"Decisions list error: {e}")
                    decisions_list = [html.P("No decisions available", className="text-muted")]
                
                # Create session performance
                try:
                    session_perf = self._create_session_performance()
                except Exception as e:
                    logger.warning(f"Session performance error: {e}")
                    session_perf = [html.P("Performance data unavailable", className="text-muted")]
                
                # Create system status
                try:
                    system_status = self._create_system_status_compact(memory_stats)
                except Exception as e:
                    logger.warning(f"System status error: {e}")
                    system_status = {
                        'icon_class': "fas fa-circle text-danger fa-2x",
                        'title': "System Error: Check logs",
                        'details': [html.P(f"Error: {str(e)}", className="text-danger")]
                    }
                
                # Create closed trades table
                try:
                    closed_trades_table = self._create_closed_trades_table()
                except Exception as e:
                    logger.warning(f"Closed trades table error: {e}")
                    closed_trades_table = [html.P("Closed trades data unavailable", className="text-muted")]
                
                # Calculate leverage display values
                leverage_text = f"{self.leverage_multiplier:.0f}x"
                if self.leverage_multiplier <= 5:
                    risk_level = "Low Risk"
                    risk_class = "bg-success"
                elif self.leverage_multiplier <= 25:
                    risk_level = "Medium Risk"
                    risk_class = "bg-warning text-dark"
                elif self.leverage_multiplier <= 50:
                    risk_level = "High Risk"
                    risk_class = "bg-danger"
                else:
                    risk_level = "Extreme Risk"
                    risk_class = "bg-dark"
                
                # Create CNN monitoring content
                try:
                    cnn_monitoring_content = self._create_cnn_monitoring_content()
                except Exception as e:
                    logger.warning(f"CNN monitoring error: {e}")
                    cnn_monitoring_content = [html.P("CNN monitoring unavailable", className="text-danger")]

                return (
                    price_text, pnl_text, pnl_class, fees_text, position_text, position_class, trade_count_text, portfolio_text, mexc_status,
                    price_chart, training_metrics, decisions_list, session_perf, closed_trades_table,
                    system_status['icon_class'], system_status['title'], system_status['details'],
                    leverage_text, f"{risk_level}",
                    cnn_monitoring_content
                )
                
            except Exception as e:
                logger.error(f"Error updating dashboard: {e}")
                # Return safe defaults
                empty_fig = self._create_empty_chart("Error", "Dashboard error - check logs")
                
                return (
                    "Error", "$0.00", "text-muted mb-0 small", "$0.00", "None", "text-muted", "0", "$10,000.00", "OFFLINE",
                    empty_fig, 
                    [html.P("Error loading training metrics", className="text-danger")],
                    [html.P("Error loading decisions", className="text-danger")], 
                    [html.P("Error loading performance", className="text-danger")],
                    [html.P("Error loading closed trades", className="text-danger")],
                    "fas fa-circle text-danger fa-2x",
                    "Error: Dashboard error - check logs",
                    [html.P(f"Error: {str(e)}", className="text-danger")],
                    f"{self.leverage_multiplier:.0f}x", "Error",
                    [html.P("CNN monitoring unavailable", className="text-danger")]
                )
        
        # Clear history callback
        @self.app.callback(
            Output('closed-trades-table', 'children', allow_duplicate=True),
            [Input('clear-history-btn', 'n_clicks')],
            prevent_initial_call=True
        )
        def clear_trade_history(n_clicks):
            """Clear trade history and reset session stats"""
            if n_clicks and n_clicks > 0:
                try:
                    # Clear both closed trades and session stats (they're the same now)
                    self.clear_closed_trades_history()
                    logger.info("DASHBOARD: Trade history and session stats cleared by user")
                    return [html.P("Trade history cleared", className="text-success text-center")]
                except Exception as e:
                    logger.error(f"Error clearing trade history: {e}")
                    return [html.P(f"Error clearing history: {str(e)}", className="text-danger text-center")]
            return dash.no_update
        
        # Leverage slider callback
        @self.app.callback(
            [Output('current-leverage', 'children', allow_duplicate=True),
             Output('leverage-risk', 'children', allow_duplicate=True),
             Output('leverage-risk', 'className', allow_duplicate=True)],
            [Input('leverage-slider', 'value')],
            prevent_initial_call=True
        )
        def update_leverage(leverage_value):
            """Update leverage multiplier and risk assessment"""
            try:
                if leverage_value is None:
                    return dash.no_update
                
                # Update internal leverage value
                self.leverage_multiplier = float(leverage_value)
                
                # Calculate risk level and styling
                leverage_text = f"{self.leverage_multiplier:.0f}x"
                
                if self.leverage_multiplier <= 5:
                    risk_level = "Low Risk"
                    risk_class = "badge bg-success"
                elif self.leverage_multiplier <= 25:
                    risk_level = "Medium Risk" 
                    risk_class = "badge bg-warning text-dark"
                elif self.leverage_multiplier <= 50:
                    risk_level = "High Risk"
                    risk_class = "badge bg-danger"
                else:
                    risk_level = "Extreme Risk"
                    risk_class = "badge bg-dark"
                
                # Update trading server if connected
                try:
                    import requests
                    response = requests.post(f"{self.trading_server_url}/update_leverage", 
                                           json={"leverage": self.leverage_multiplier}, 
                                           timeout=2)
                    if response.status_code == 200:
                        logger.info(f"[LEVERAGE] Updated trading server leverage to {self.leverage_multiplier}x")
                    else:
                        logger.warning(f"[LEVERAGE] Failed to update trading server: {response.status_code}")
                except Exception as e:
                    logger.debug(f"[LEVERAGE] Trading server not available: {e}")
                
                logger.info(f"[LEVERAGE] Leverage updated to {self.leverage_multiplier}x ({risk_level})")
                
                return leverage_text, risk_level, risk_class
                
            except Exception as e:
                logger.error(f"Error updating leverage: {e}")
                return f"{self.leverage_multiplier:.0f}x", "Error", "badge bg-secondary"
    
    def _simulate_price_update(self, symbol: str, base_price: float) -> float:
        """
        Create realistic price movement for demo purposes
        This simulates small price movements typical of real market data
        """
        try:
            import random
            import math
            
            # Create small realistic price movements (±0.05% typical crypto volatility)
            variation_percent = random.uniform(-0.0005, 0.0005)  # ±0.05%
            price_change = base_price * variation_percent
            
            # Add some momentum (trending behavior)
            if not hasattr(self, '_price_momentum'):
                self._price_momentum = 0
            
            # Momentum decay and random walk
            momentum_decay = 0.95
            self._price_momentum = self._price_momentum * momentum_decay + variation_percent * 0.1
            
            # Apply momentum
            new_price = base_price + price_change + (base_price * self._price_momentum)
            
            # Ensure reasonable bounds (prevent extreme movements)
            max_change = base_price * 0.001  # Max 0.1% change per update
            new_price = max(base_price - max_change, min(base_price + max_change, new_price))
            
            return round(new_price, 2)
            
        except Exception as e:
            logger.warning(f"Price simulation error: {e}")
            return base_price
    
    def _create_empty_chart(self, title: str, message: str) -> go.Figure:
        """Create an empty chart with a message"""
        fig = go.Figure()
        fig.add_annotation(
            text=message,
            xref="paper", yref="paper",
            x=0.5, y=0.5,
            showarrow=False,
            font=dict(size=16, color="gray")
        )
        fig.update_layout(
            title=title,
            template="plotly_dark",
            height=400,
            margin=dict(l=20, r=20, t=50, b=20)
        )
        return fig
    
    def _create_price_chart(self, symbol: str) -> go.Figure:
        """Create enhanced price chart with real-time data, Williams pivot points, and trading signals"""
        try:
            # Initialize chart_start_time and chart_end_time early
            chart_start_time = None
            chart_end_time = None
            
            # Try to get real-time data if available
            df = None
            actual_timeframe = '1m'
            
            if self.data_provider:
                try:
                    # Get fresh market data with configurable timeframe
                    df = self.data_provider.get_historical_data(symbol, '1m', limit=30, refresh=True)
"""
                    
                    if df is not None and not df.empty:
                        # Ensure timezone consistency
                        df = self._ensure_timezone_consistency(df)
                        actual_timeframe = '1m'
                        logger.debug(f"[CHART] Loaded {len(df)} fresh 1m bars in {self.timezone}")
                        
                        # Set time boundaries early
                        chart_start_time = df.index.min()
                        chart_end_time = df.index.max()
                    else:
                        return self._create_empty_chart(
                            f"{symbol} Chart", 
                            f"No data available for {symbol}\nWaiting for data provider..."
Trading Dashboard - Clean Web Interface

This module provides a modern, responsive web dashboard for the trading system:
- Real-time price charts with multiple timeframes
- Model performance monitoring
- Trading decisions visualization
- System health monitoring
- Memory usage tracking
"""

import asyncio
import dash
from dash import Dash, dcc, html, Input, Output
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.express as px
import pandas as pd
import numpy as np
from datetime import datetime, timedelta, timezone
import pytz
import logging
import json
import time
import threading
from threading import Thread, Lock
from collections import deque
import warnings
from typing import Dict, List, Optional, Any, Union, Tuple
import websocket
import os
import torch

# Setup logger immediately after logging import
logger = logging.getLogger(__name__)

# WebSocket availability check
try:
    import websocket
    WEBSOCKET_AVAILABLE = True
    logger.info("WebSocket client available")
except ImportError:
    WEBSOCKET_AVAILABLE = False
    logger.warning("websocket-client not available. Real-time data will use API fallback.")

# Import trading system components
from core.config import get_config
from core.data_provider import DataProvider
from core.orchestrator import TradingOrchestrator, TradingDecision
from core.trading_executor import TradingExecutor
from core.trading_action import TradingAction
from models import get_model_registry

# Import CNN monitoring
try:
    from core.cnn_monitor import get_cnn_dashboard_data
    CNN_MONITORING_AVAILABLE = True
    logger.info("CNN monitoring system available")
except ImportError:
    CNN_MONITORING_AVAILABLE = False
    logger.warning("CNN monitoring not available")
    def get_cnn_dashboard_data():
        return {'statistics': {'total_predictions_logged': 0}}

# Import CNN prediction components
try:
    from training.williams_market_structure import SwingPoint, SwingType
    CNN_PREDICTIONS_AVAILABLE = True
    logger.info("CNN predictions available")
except ImportError:
    CNN_PREDICTIONS_AVAILABLE = False
    logger.warning("CNN predictions not available")
    class SwingPoint:
        def __init__(self, timestamp, price, index, swing_type, strength):
            self.timestamp = timestamp
            self.price = price
            self.index = index
            self.swing_type = swing_type
            self.strength = strength
    class SwingType:
        SWING_HIGH = "swing_high"
        SWING_LOW = "swing_low"

# Import enhanced RL components if available
try:
    from core.enhanced_orchestrator import EnhancedTradingOrchestrator
    from core.universal_data_adapter import UniversalDataAdapter
    from core.unified_data_stream import UnifiedDataStream, TrainingDataPacket, UIDataPacket
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL training components available")
except ImportError as e:
    logger.warning(f"Enhanced RL components not available: {e}")
    ENHANCED_RL_AVAILABLE = False
    # Force enable for learning - bypass import issues
    ENHANCED_RL_AVAILABLE = True
    logger.info("Enhanced RL FORCED ENABLED - bypassing import issues for learning")
    
    # Fallback classes
    class UnifiedDataStream:
        def __init__(self, *args, **kwargs): pass
        def register_consumer(self, *args, **kwargs): return "fallback_consumer"
        def start_streaming(self): pass
        def stop_streaming(self): pass
        def get_latest_training_data(self): return None
        def get_latest_ui_data(self): return None
    
    class TrainingDataPacket:
        def __init__(self, *args, **kwargs): pass
    
    class UIDataPacket:
        def __init__(self, *args, **kwargs): pass


class AdaptiveThresholdLearner:
    """Learn optimal confidence thresholds based on real trade outcomes"""
    
    def __init__(self, initial_threshold: float = 0.30):
        self.base_threshold = initial_threshold
        self.current_threshold = initial_threshold
        self.trade_outcomes = deque(maxlen=100)
        self.threshold_history = deque(maxlen=50)
        self.learning_rate = 0.02
        self.min_threshold = 0.20
        self.max_threshold = 0.70
        
        logger.info(f"[ADAPTIVE] Initialized with starting threshold: {initial_threshold:.2%}")
    
    def record_trade_outcome(self, confidence: float, pnl: float, threshold_used: float):
        """Record a trade outcome to learn from"""
        try:
            outcome = {
                'confidence': confidence,
                'pnl': pnl,
                'profitable': pnl > 0,
                'threshold_used': threshold_used,
                'timestamp': datetime.now()
            }
            
            self.trade_outcomes.append(outcome)
            
            # Learn from outcomes
            if len(self.trade_outcomes) >= 10:
                self._update_threshold()
            
        except Exception as e:
            logger.error(f"Error recording trade outcome: {e}")
    
    def _update_threshold(self):
        """Update threshold based on recent trade statistics"""
        try:
            recent_trades = list(self.trade_outcomes)[-20:]
            if len(recent_trades) < 10:
                return
            
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades)
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades)
            
            # Adaptive adjustment logic
            if win_rate > 0.60 and avg_pnl > 0.20:
                adjustment = -self.learning_rate * 1.5  # Lower threshold for more trades
            elif win_rate < 0.40 or avg_pnl < -0.30:
                adjustment = self.learning_rate * 2.0  # Raise threshold to be more selective
            else:
                adjustment = 0  # No change
            
            old_threshold = self.current_threshold
            self.current_threshold = max(self.min_threshold, 
                                       min(self.max_threshold, 
                                           self.current_threshold + adjustment))
            
            if abs(self.current_threshold - old_threshold) > 0.005:
                logger.info(f"[ADAPTIVE] Threshold: {old_threshold:.2%} -> {self.current_threshold:.2%} (WR: {win_rate:.1%}, PnL: ${avg_pnl:.2f})")
            
        except Exception as e:
            logger.error(f"Error updating adaptive threshold: {e}")
    
    def get_current_threshold(self) -> float:
        return self.current_threshold
    
    def get_learning_stats(self) -> Dict[str, Any]:
        """Get learning statistics"""
        try:
            if not self.trade_outcomes:
                return {'status': 'No trades recorded yet'}
            
            recent_trades = list(self.trade_outcomes)[-20:]
            profitable_count = sum(1 for t in recent_trades if t['profitable'])
            win_rate = profitable_count / len(recent_trades) if recent_trades else 0
            avg_pnl = sum(t['pnl'] for t in recent_trades) / len(recent_trades) if recent_trades else 0
            
            return {
                'current_threshold': self.current_threshold,
                'base_threshold': self.base_threshold,
                'total_trades': len(self.trade_outcomes),
                'recent_win_rate': win_rate,
                'recent_avg_pnl': avg_pnl,
                'threshold_changes': len(self.threshold_history),
                'learning_active': len(self.trade_outcomes) >= 10
            }
        except Exception as e:
            return {'error': str(e)}

class TradingDashboard:
    """Enhanced Trading Dashboard with Williams pivot points and unified timezone handling"""
    
    def __init__(self, data_provider: DataProvider = None, orchestrator: TradingOrchestrator = None, trading_executor: TradingExecutor = None):
        self.app = Dash(__name__)
        
        # Initialize config first
        from core.config import get_config
        self.config = get_config()
        
        self.data_provider = data_provider or DataProvider()
        self.orchestrator = orchestrator
        self.trading_executor = trading_executor
        
        # Enhanced trading state with leverage support
        self.leverage_enabled = True
        self.leverage_multiplier = 50.0  # 50x leverage (adjustable via slider)
        self.base_capital = 10000.0
        self.current_position = 0.0  # -1 to 1 (short to long)
        self.position_size = 0.0
        self.entry_price = 0.0
        self.unrealized_pnl = 0.0
        self.realized_pnl = 0.0
        
        # Leverage settings for slider
        self.min_leverage = 1.0
        self.max_leverage = 100.0
        self.leverage_step = 1.0
        
        # Connect to trading server for leverage functionality
        self.trading_server_url = "http://127.0.0.1:8052"
        self.training_server_url = "http://127.0.0.1:8053"
        self.stream_server_url = "http://127.0.0.1:8054"
        
        # Enhanced performance tracking
        self.leverage_metrics = {
            'leverage_efficiency': 0.0,
            'margin_used': 0.0,
            'margin_available': 10000.0,
            'effective_exposure': 0.0,
            'risk_reward_ratio': 0.0
        }
        
        # Enhanced models will be loaded through model registry later
        
        # Rest of initialization...
        
        # Initialize timezone from config
        timezone_name = self.config.get('system', {}).get('timezone', 'Europe/Sofia')
        self.timezone = pytz.timezone(timezone_name)
        logger.info(f"Dashboard timezone set to: {timezone_name}")
        
        self.data_provider = data_provider or DataProvider()
        
        # Enhanced orchestrator support - FORCE ENABLE for learning
        self.orchestrator = orchestrator or TradingOrchestrator(self.data_provider)
        self.enhanced_rl_enabled = True  # Force enable Enhanced RL
        logger.info("Enhanced RL training FORCED ENABLED for learning")
        
        self.trading_executor = trading_executor or TradingExecutor()
        self.model_registry = get_model_registry()
        
        # Initialize unified data stream for comprehensive training data
        if ENHANCED_RL_AVAILABLE:
            self.unified_stream = UnifiedDataStream(self.data_provider, self.orchestrator)
            self.stream_consumer_id = self.unified_stream.register_consumer(
                consumer_name="TradingDashboard",
                callback=self._handle_unified_stream_data,
                data_types=['ticks', 'ohlcv', 'training_data', 'ui_data']
            )
            logger.info(f"Unified data stream initialized with consumer ID: {self.stream_consumer_id}")
        else:
            self.unified_stream = UnifiedDataStream()  # Fallback
            self.stream_consumer_id = "fallback"
            logger.warning("Using fallback unified data stream")
        
        # Dashboard state
        self.recent_decisions = []
        self.recent_signals = []  # Track all signals (not just executed trades)
        self.performance_data = {}
        self.current_prices = {}
        self.last_update = datetime.now()
        
        # Trading session tracking
        self.session_start = datetime.now()
        self.session_trades = []
        self.session_pnl = 0.0
        self.current_position = None  # {'side': 'BUY', 'price': 3456.78, 'size': 0.1, 'timestamp': datetime}
        self.total_realized_pnl = 0.0
        self.total_fees = 0.0
        self.starting_balance = self._get_initial_balance()  # Get balance from MEXC or default to 100
        
        # Closed trades tracking for accounting
        self.closed_trades = []  # List of all closed trades with full details
        
        # Load existing closed trades from file
        self._load_closed_trades_from_file()
        
        # Signal execution settings for scalping - REMOVED FREQUENCY LIMITS
        self.min_confidence_threshold = 0.30  # Start lower to allow learning  
        self.signal_cooldown = 0  # REMOVED: Model decides when to act, no artificial delays
        self.last_signal_time = 0
        
        # Adaptive threshold learning - starts low and learns optimal thresholds
        self.adaptive_learner = AdaptiveThresholdLearner(initial_threshold=0.30)
        logger.info("[ADAPTIVE] Adaptive threshold learning enabled - will adjust based on trade outcomes")
        
        # Lightweight WebSocket implementation for real-time scalping data
        self.ws_price_cache = {}  # Just current prices, no tick history
        self.ws_connection = None
        self.ws_thread = None
        self.is_streaming = False
        
        # Performance-focused: only track essentials
        self.last_ws_update = 0
        self.ws_update_count = 0
        
        # Compatibility stubs for removed tick infrastructure 
        self.tick_cache = []  # Empty list for compatibility
        self.one_second_bars = []  # Empty list for compatibility
        
        # Enhanced RL Training System - Train on closed trades with comprehensive data
        self.rl_training_enabled = True
        # Force enable Enhanced RL training (bypass import issues)
        self.enhanced_rl_training_enabled = True  # Force enabled for CNN training
        self.enhanced_rl_enabled = True  # Force enabled to show proper status
        self.rl_training_stats = {
            'total_training_episodes': 0,
            'profitable_trades_trained': 0,
            'unprofitable_trades_trained': 0,
            'last_training_time': None,
            'training_rewards': deque(maxlen=100),  # Last 100 training rewards
            'model_accuracy_trend': deque(maxlen=50),  # Track accuracy over time
            'enhanced_rl_episodes': 0,
            'comprehensive_data_packets': 0
        }
        self.rl_training_queue = deque(maxlen=1000)  # Queue of trades to train on
        
        # Enhanced training data tracking
        self.latest_training_data = None
        self.latest_ui_data = None
        self.training_data_available = False
        
        # Load available models for real trading
        self._load_available_models()
        
        # Preload essential data to prevent excessive API calls during dashboard updates
        logger.info("Preloading essential market data to cache...")
        try:
            # Preload key timeframes for main symbols to ensure cache is populated
            symbols_to_preload = self.config.symbols or ['ETH/USDT', 'BTC/USDT']
            timeframes_to_preload = ['1m', '1h', '1d']  # Skip 1s since we use WebSocket for that
            
            for symbol in symbols_to_preload[:2]:  # Limit to first 2 symbols
                for timeframe in timeframes_to_preload:
                    try:
                        # Load data into cache (refresh=True for initial load, then cache will be used)
                        df = self.data_provider.get_historical_data(symbol, timeframe, limit=100, refresh=True)
                        if df is not None and not df.empty:
                            logger.info(f"Preloaded {len(df)} {timeframe} bars for {symbol}")
                        else:
                            logger.warning(f"Failed to preload data for {symbol} {timeframe}")
                    except Exception as e:
                        logger.warning(f"Error preloading {symbol} {timeframe}: {e}")
            
            logger.info("Preloading completed - cache populated for frequent queries")
            
        except Exception as e:
            logger.warning(f"Error during preloading: {e}")
        
        # Create Dash app
        self.app = dash.Dash(__name__, external_stylesheets=[
            'https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css',
            'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css'
        ])
        
        # # Add custom CSS for model data charts
        # self.app.index_string = '''
        # <!DOCTYPE html>
        # <html>
        #     <head>
        #         {%metas%}
        #         <title>{%title%}</title>
        #         {%favicon%}
        #         {%css%}
        #         <style>
        #             .tiny { font-size: 10px !important; }
        #             .model-data-chart .js-plotly-plot .plotly .modebar { display: none !important; }
        #             .model-data-chart .js-plotly-plot .plotly .svg-container { border: 1px solid #444; border-radius: 4px; }
        #         </style>
        #     </head>
        #     <body>
        #         {%app_entry%}
        #         <footer>
        #             {%config%}
        #             {%scripts%}
        #             {%renderer%}
        #         </footer>
        #     </body>
        # </html>
        # '''
        
        # Setup layout and callbacks
        self._setup_layout()
        self._setup_callbacks()
        
        # Start unified data streaming
        self._initialize_streaming()
        
        # Start continuous training with enhanced RL support
        self.start_continuous_training()
        
        logger.info("Trading Dashboard initialized with enhanced RL training integration")
        logger.info(f"Enhanced RL enabled: {self.enhanced_rl_training_enabled}")
        logger.info(f"Stream consumer ID: {self.stream_consumer_id}")
        
        # Initialize Williams Market Structure once
        try:
            from training.williams_market_structure import WilliamsMarketStructure
            self.williams_structure = WilliamsMarketStructure(
                swing_strengths=[2, 3, 5],  # Simplified for better performance
                enable_cnn_feature=True,   # Enable CNN training and inference
                training_data_provider=self.data_provider  # Provide data access for training
            )
            logger.info("Williams Market Structure initialized for dashboard with CNN training enabled")
        except ImportError:
            self.williams_structure = None
            logger.warning("Williams Market Structure not available")
        
        # Initialize Enhanced Pivot RL Trainer for better position management
        try:
            self.pivot_rl_trainer = create_enhanced_pivot_trainer(
                data_provider=self.data_provider,
                orchestrator=self.orchestrator
            )
            logger.info("Enhanced Pivot RL Trainer initialized for better entry/exit decisions")
            logger.info(f"Entry threshold: {self.pivot_rl_trainer.get_current_thresholds()['entry_threshold']:.1%}")
            logger.info(f"Exit threshold: {self.pivot_rl_trainer.get_current_thresholds()['exit_threshold']:.1%}")
            logger.info(f"Uninvested threshold: {self.pivot_rl_trainer.get_current_thresholds()['uninvested_threshold']:.1%}")
        except Exception as e:
            self.pivot_rl_trainer = None
            logger.warning(f"Enhanced Pivot RL Trainer not available: {e}")
    
    def _to_local_timezone(self, dt) -> datetime:
        """Convert datetime to configured local timezone"""
        try:
            if dt is None:
                return None
            
            # Handle string timestamps by converting to datetime first
            if isinstance(dt, str):
                try:
                    dt = pd.to_datetime(dt)
                except Exception:
                    logger.warning(f"Could not parse timestamp string: {dt}")
                    return datetime.now(self.timezone)
            
            # Handle pandas Timestamp
            if isinstance(dt, pd.Timestamp):
                dt = dt.to_pydatetime()
            
            # If datetime is naive, assume it's UTC
            if dt.tzinfo is None:
                dt = pytz.UTC.localize(dt)
            
            # Convert to local timezone
            return dt.astimezone(self.timezone)
        except Exception as e:
            logger.warning(f"Error converting timezone for {dt}: {e}")
            return datetime.now(self.timezone)  # Return current time as fallback
    
    def _now_local(self) -> datetime:
        """Get current time in configured local timezone"""
        return datetime.now(self.timezone)
    
    def _ensure_timezone_consistency(self, df: pd.DataFrame) -> pd.DataFrame:
        """Ensure DataFrame index is in consistent timezone"""
        try:
            if hasattr(df.index, 'tz'):
                if df.index.tz is None:
                    # Assume UTC if no timezone
                    df.index = df.index.tz_localize('UTC')
                
                # Convert to local timezone
                df.index = df.index.tz_convert(self.timezone)
            
            return df
        except Exception as e:
            logger.warning(f"Error ensuring timezone consistency: {e}")
            return df
    
    def _initialize_streaming(self):
        """Initialize unified data streaming and WebSocket fallback"""
        try:
            # Start lightweight WebSocket for real-time price updates
            self._start_lightweight_websocket()
            logger.info("Lightweight WebSocket streaming initialized")
            
            if ENHANCED_RL_AVAILABLE:
                # Start unified data stream in background
                def start_unified_stream():
                    try:
                        asyncio.run(self.unified_stream.start_streaming())
                        logger.info("Unified data stream started")
                    except Exception as e:
                        logger.error(f"Error starting unified stream: {e}")
                
                unified_thread = Thread(target=start_unified_stream, daemon=True)
                unified_thread.start()
            
            # Start background data collection
            self._start_enhanced_training_data_collection()
            
            logger.info("All data streaming initialized")
            
        except Exception as e:
            logger.error(f"Error initializing streaming: {e}")
            # Ensure lightweight WebSocket is started as fallback
            self._start_lightweight_websocket()
    
    def _start_enhanced_training_data_collection(self):
        """Start enhanced training data collection using unified stream"""
        def enhanced_training_loop():
            try:
                logger.info("Enhanced training data collection started with unified stream")
                
                while True:
                    try:
                        if ENHANCED_RL_AVAILABLE and self.enhanced_rl_training_enabled:
                            # Get latest comprehensive training data from unified stream
                            training_data = self.unified_stream.get_latest_training_data()
                            
                            if training_data:
                                # Send comprehensive training data to enhanced RL pipeline
                                self._send_comprehensive_training_data_to_enhanced_rl(training_data)
                                
                                # Update training statistics
                                self.rl_training_stats['comprehensive_data_packets'] += 1
                                self.training_data_available = True
                                
                                # Update context data in orchestrator
                                if hasattr(self.orchestrator, 'update_context_data'):
                                    self.orchestrator.update_context_data()
                                
                                # Initialize extrema trainer if not done
                                if hasattr(self.orchestrator, 'extrema_trainer'):
                                    if not hasattr(self.orchestrator.extrema_trainer, '_initialized'):
                                        self.orchestrator.extrema_trainer.initialize_context_data()
                                        self.orchestrator.extrema_trainer._initialized = True
                                        logger.info("Extrema trainer context data initialized")
                                
                                # Run extrema detection with real data
                                if hasattr(self.orchestrator, 'extrema_trainer'):
                                    for symbol in self.orchestrator.symbols:
                                        detected = self.orchestrator.extrema_trainer.detect_local_extrema(symbol)
                                        if detected:
                                            logger.debug(f"Detected {len(detected)} extrema for {symbol}")
                        else:
                            # Fallback to basic training data collection
                            self._collect_basic_training_data()
                        
                        time.sleep(10)  # Update every 10 seconds for enhanced training
                        
                    except Exception as e:
                        logger.error(f"Error in enhanced training loop: {e}")
                        time.sleep(30)  # Wait before retrying
                        
            except Exception as e:
                logger.error(f"Enhanced training loop failed: {e}")
        
        # Start enhanced training thread
        training_thread = Thread(target=enhanced_training_loop, daemon=True)
        training_thread.start()
        logger.info("Enhanced training data collection thread started")
    
    def _handle_unified_stream_data(self, data_packet: Dict[str, Any]):
        """Handle data from unified stream for dashboard and training"""
        try:
            # Extract UI data for dashboard display
            if 'ui_data' in data_packet:
                self.latest_ui_data = data_packet['ui_data']
                if hasattr(self.latest_ui_data, 'current_prices'):
                    self.current_prices.update(self.latest_ui_data.current_prices)
                if hasattr(self.latest_ui_data, 'streaming_status'):
                    self.is_streaming = self.latest_ui_data.streaming_status == 'LIVE'
                if hasattr(self.latest_ui_data, 'training_data_available'):
                    self.training_data_available = self.latest_ui_data.training_data_available
            
            # Extract training data for enhanced RL
            if 'training_data' in data_packet:
                self.latest_training_data = data_packet['training_data']
                logger.debug("Received comprehensive training data from unified stream")
            
            # Extract tick data for dashboard charts
            if 'ticks' in data_packet:
                ticks = data_packet['ticks']
                for tick in ticks[-100:]:  # Keep last 100 ticks
                    self.tick_cache.append(tick)
            
            # Extract OHLCV data for dashboard charts
            if 'one_second_bars' in data_packet:
                bars = data_packet['one_second_bars']
                for bar in bars[-100:]:  # Keep last 100 bars
                    self.one_second_bars.append(bar)
            
            logger.debug(f"Processed unified stream data packet with keys: {list(data_packet.keys())}")
            
        except Exception as e:
            logger.error(f"Error handling unified stream data: {e}")
    
    def _send_comprehensive_training_data_to_enhanced_rl(self, training_data: TrainingDataPacket):
        """Send comprehensive training data to enhanced RL training pipeline"""
        try:
            if not self.enhanced_rl_training_enabled:
                logger.debug("Enhanced RL training not enabled, skipping comprehensive data send")
                return
            
            # Extract comprehensive training data components
            market_state = training_data.market_state if hasattr(training_data, 'market_state') else None
            universal_stream = training_data.universal_stream if hasattr(training_data, 'universal_stream') else None
            cnn_features = training_data.cnn_features if hasattr(training_data, 'cnn_features') else None
            cnn_predictions = training_data.cnn_predictions if hasattr(training_data, 'cnn_predictions') else None
            
            if market_state and universal_stream:
                # Send to enhanced RL trainer if available
                if hasattr(self.orchestrator, 'enhanced_rl_trainer'):
                    try:
                        # Create comprehensive training step with ~13,400 features
                        asyncio.run(self.orchestrator.enhanced_rl_trainer.training_step(universal_stream))
                        self.rl_training_stats['enhanced_rl_episodes'] += 1
                        logger.debug("Sent comprehensive data to enhanced RL trainer")
                    except Exception as e:
                        logger.warning(f"Error in enhanced RL training step: {e}")
                
                # Send to extrema trainer for CNN training with perfect moves
                if hasattr(self.orchestrator, 'extrema_trainer'):
                    try:
                        extrema_data = self.orchestrator.extrema_trainer.get_extrema_training_data(count=50)
                        perfect_moves = self.orchestrator.extrema_trainer.get_perfect_moves_for_cnn(count=100)
                        
                        if extrema_data:
                            logger.debug(f"Enhanced RL: {len(extrema_data)} extrema training samples available")
                            
                        if perfect_moves:
                            logger.debug(f"Enhanced RL: {len(perfect_moves)} perfect moves for CNN training")
                    except Exception as e:
                        logger.warning(f"Error getting extrema training data: {e}")
                
                # Send to sensitivity learning DQN for outcome-based learning
                if hasattr(self.orchestrator, 'sensitivity_learning_queue'):
                    try:
                        if len(self.orchestrator.sensitivity_learning_queue) > 0:
                            logger.debug("Enhanced RL: Sensitivity learning data available for DQN training")
                    except Exception as e:
                        logger.warning(f"Error accessing sensitivity learning queue: {e}")
                
                # Get context features for models with real market data
                if hasattr(self.orchestrator, 'extrema_trainer'):
                    try:
                        for symbol in self.orchestrator.symbols:
                            context_features = self.orchestrator.extrema_trainer.get_context_features_for_model(symbol)
                            if context_features is not None:
                                logger.debug(f"Enhanced RL: Context features available for {symbol}: {context_features.shape}")
                    except Exception as e:
                        logger.warning(f"Error getting context features: {e}")
            
            # Log comprehensive training data statistics
            tick_count = len(training_data.tick_cache) if hasattr(training_data, 'tick_cache') else 0
            bars_count = len(training_data.one_second_bars) if hasattr(training_data, 'one_second_bars') else 0
            timeframe_count = len(training_data.multi_timeframe_data) if hasattr(training_data, 'multi_timeframe_data') else 0
            
            logger.info(f"Enhanced RL Comprehensive Training Data:")
            logger.info(f"  Tick cache: {tick_count} ticks")
            logger.info(f"  1s bars: {bars_count} bars")
            logger.info(f"  Multi-timeframe data: {timeframe_count} symbols")
            logger.info(f"  CNN features: {'Available' if cnn_features else 'Not available'}")
            logger.info(f"  CNN predictions: {'Available' if cnn_predictions else 'Not available'}")
            logger.info(f"  Market state: {'Available (~13,400 features)' if market_state else 'Not available'}")
            logger.info(f"  Universal stream: {'Available' if universal_stream else 'Not available'}")
                
        except Exception as e:
            logger.error(f"Error sending comprehensive training data to enhanced RL: {e}")
    
    def _collect_basic_training_data(self):
        """Fallback method to collect basic training data when enhanced RL is not available"""
        try:
            # Get real tick data from data provider subscribers
            for symbol in ['ETH/USDT', 'BTC/USDT']:
                try:
                    # Get recent ticks from data provider
                    if hasattr(self.data_provider, 'get_recent_ticks'):
                        recent_ticks = self.data_provider.get_recent_ticks(symbol, count=10)
                        
                        for tick in recent_ticks:
                            # Create tick data from real market data
                            tick_data = {
                                'symbol': tick.symbol,
                                'price': tick.price,
                                'timestamp': tick.timestamp,
                                'volume': tick.volume
                            }
                            
                            # Add to tick cache
                            self.tick_cache.append(tick_data)
                            
                            # Create 1s bar data from real tick
                            bar_data = {
                                'symbol': tick.symbol,
                                'open': tick.price,
                                'high': tick.price,
                                'low': tick.price,
                                'close': tick.price,
                                'volume': tick.volume,
                                'timestamp': tick.timestamp
                            }
                            
                            # Add to 1s bars cache
                            self.one_second_bars.append(bar_data)
                            
                except Exception as e:
                    logger.debug(f"No recent tick data available for {symbol}: {e}")
                    
            # Set streaming status based on real data availability
            self.is_streaming = len(self.tick_cache) > 0
            
        except Exception as e:
            logger.warning(f"Error in basic training data collection: {e}")
    
    def _get_initial_balance(self) -> float:
        """Get initial USDT balance from MEXC or return default"""
        try:
            if self.trading_executor and hasattr(self.trading_executor, 'get_account_balance'):
                logger.info("Fetching initial balance from MEXC...")
                
                # Check if trading is enabled and not in dry run mode
                if not self.trading_executor.trading_enabled:
                    logger.warning("MEXC: Trading not enabled - using default balance")
                elif self.trading_executor.simulation_mode:
                    logger.warning(f"MEXC: {self.trading_executor.trading_mode.upper()} mode enabled - using default balance")
                else:
                    # Get USDT balance from MEXC
                    balance_info = self.trading_executor.get_account_balance()
                    if balance_info and 'USDT' in balance_info:
                        usdt_balance = float(balance_info['USDT'].get('free', 0))
                        if usdt_balance > 0:
                            logger.info(f"MEXC: Retrieved USDT balance: ${usdt_balance:.2f}")
                            return usdt_balance
                        else:
                            logger.warning("MEXC: No USDT balance found in account")
                    else:
                        logger.error("MEXC: Failed to retrieve balance info from API")
            else:
                logger.info("MEXC: Trading executor not available for balance retrieval")
                
        except Exception as e:
            logger.error(f"Error getting MEXC balance: {e}")
            import traceback
            logger.error(traceback.format_exc())
        
        # Fallback to default
        default_balance = 100.0
        logger.warning(f"Using default starting balance: ${default_balance:.2f}")
        return default_balance
    
    def _setup_layout(self):
        """Setup the dashboard layout"""
        self.app.layout = html.Div([
            # Compact Header
            html.Div([
                html.H3([
                    html.I(className="fas fa-chart-line me-2"),
                    "Live Trading Dashboard"
                ], className="text-white mb-1"),
                html.P(f"Ultra-Fast Updates • Portfolio: ${self.starting_balance:,.0f} • {'MEXC Live' if (self.trading_executor and self.trading_executor.trading_enabled and not self.trading_executor.simulation_mode) else 'Demo Mode'}", 
                      className="text-light mb-0 opacity-75 small")
            ], className="bg-dark p-2 mb-2"),
            
            # Auto-refresh component
            dcc.Interval(
                id='interval-component',
                interval=1000,  # Update every 1 second for real-time tick updates
                n_intervals=0
            ),
            
            # Main content - Compact layout
            html.Div([
                # Top row - Key metrics and Recent Signals (split layout)
                html.Div([
                    # Left side - Key metrics (compact cards)
                    html.Div([
                        html.Div([
                            html.Div([
                                html.H5(id="current-price", className="text-success mb-0 small"),
                                html.P("Live Price", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="session-pnl", className="mb-0 small"),
                                html.P("Session P&L", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="total-fees", className="text-warning mb-0 small"),
                                html.P("Total Fees", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="current-position", className="text-info mb-0 small"),
                                html.P("Position", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="trade-count", className="text-warning mb-0 small"),
                                html.P("Trades", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="portfolio-value", className="text-secondary mb-0 small"),
                                html.P("Portfolio", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                        
                        html.Div([
                            html.Div([
                                html.H5(id="mexc-status", className="text-info mb-0 small"),
                                html.P("MEXC API", className="text-muted mb-0 tiny")
                            ], className="card-body text-center p-2")
                        ], className="card bg-light", style={"height": "60px"}),
                    ], style={"display": "grid", "gridTemplateColumns": "repeat(4, 1fr)", "gap": "8px", "width": "60%"}),
                    
                    # Right side - Recent Signals & Executions
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-robot me-2"),
                                "Recent Trading Signals & Executions"
                            ], className="card-title mb-2"),
                            html.Div(id="recent-decisions", style={"height": "160px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "48%", "marginLeft": "2%"})
                ], className="d-flex mb-3"),
                
                # Charts row - More compact
                html.Div([
                    # Price chart - 70% width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-chart-candlestick me-2"),
                                "Live 1s Price & Volume Chart (WebSocket Stream)"
                            ], className="card-title mb-2"),
                            dcc.Graph(id="price-chart", style={"height": "400px"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "70%"}),
                    
                    # Model Training Metrics - 30% width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-brain me-2"),
                                "Model Training Progress"
                            ], className="card-title mb-2"),
                            html.Div(id="training-metrics", style={"height": "400px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card", style={"width": "28%", "marginLeft": "2%"}),
                ], className="row g-2 mb-3"),
                
                # CNN Model Monitoring Section
                html.Div([
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-brain me-2"),
                                "CNN Model Analysis & Predictions"
                            ], className="card-title mb-2"),
                            html.Div(id="cnn-monitoring-content", style={"height": "350px", "overflowY": "auto"})
                        ], className="card-body p-2")
                    ], className="card")
                ], className="mb-3"),
                
                # Bottom row - Session performance and system status
                html.Div([                               

                    # Session performance - 1/3 width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-chart-pie me-2"),
                                "Session Performance"
                            ], className="card-title mb-2"),
                            html.Button(
                                "Clear Session",
                                id="clear-history-btn",
                                className="btn btn-sm btn-outline-danger mb-2",
                                n_clicks=0
                            ),
                            html.Div(id="session-performance")
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%"}),
                                  
                    # Closed Trades History - 1/3 width
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-history me-2"),
                                "Closed Trades History"
                            ], className="card-title mb-2"),
                            html.Div([
                                html.Div(
                                    id="closed-trades-table",
                                    style={"height": "300px", "overflowY": "auto"}
                                )
                            ])
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%", "marginLeft": "2%"}),
                    
                    # System status and leverage controls - 1/3 width with icon tooltip
                    html.Div([
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-server me-2"),
                                "System & Leverage"
                            ], className="card-title mb-2"),
                            
                            # System status
                            html.Div([
                                html.I(
                                    id="system-status-icon",
                                    className="fas fa-circle text-success fa-2x",
                                    title="System Status: All systems operational",
                                    style={"cursor": "pointer"}
                                ),
                                html.Div(id="system-status-details", className="small mt-2")
                            ], className="text-center mb-3"),
                            
                            # Leverage Controls
                            html.Div([
                                html.Label([
                                    html.I(className="fas fa-chart-line me-1"),
                                    "Leverage Multiplier"
                                ], className="form-label small fw-bold"),
                                html.Div([
                                    dcc.Slider(
                                        id='leverage-slider',
                                        min=self.min_leverage,
                                        max=self.max_leverage,
                                        step=self.leverage_step,
                                        value=self.leverage_multiplier,
                                        marks={
                                            1: '1x',
                                            10: '10x',
                                            25: '25x',
                                            50: '50x',
                                            75: '75x',
                                            100: '100x'
                                        },
                                        tooltip={
                                            "placement": "bottom",
                                            "always_visible": True
                                        }
                                    )
                                ], className="mb-2"),
                                html.Div([
                                    html.Span(id="current-leverage", className="badge bg-warning text-dark"),
                                    html.Span(" • ", className="mx-1"),
                                    html.Span(id="leverage-risk", className="badge bg-info")
                                ], className="text-center"),
                                html.Div([
                                    html.Small("Higher leverage = Higher rewards & risks", className="text-muted")
                                ], className="text-center mt-1")
                            ])
                        ], className="card-body p-2")
                    ], className="card", style={"width": "32%", "marginLeft": "2%"})
                ], className="d-flex")
            ], className="container-fluid")
        ])
    
    def _setup_callbacks(self):
        """Setup dashboard callbacks for real-time updates"""
        
        @self.app.callback(
            [
                Output('current-price', 'children'),
                Output('session-pnl', 'children'),
                Output('session-pnl', 'className'),
                Output('total-fees', 'children'),
                Output('current-position', 'children'),
                Output('current-position', 'className'),
                Output('trade-count', 'children'),
                Output('portfolio-value', 'children'),
                Output('mexc-status', 'children'),
                Output('price-chart', 'figure'),
                Output('training-metrics', 'children'),
                Output('recent-decisions', 'children'),
                Output('session-performance', 'children'),
                Output('closed-trades-table', 'children'),
                Output('system-status-icon', 'className'),
                Output('system-status-icon', 'title'),
                Output('system-status-details', 'children'),
                Output('current-leverage', 'children'),
                Output('leverage-risk', 'children'),
                Output('cnn-monitoring-content', 'children')
            ],
            [Input('interval-component', 'n_intervals')]
        )
        def update_dashboard(n_intervals):
            """Update all dashboard components with trading signals"""
            start_time = time.time()  # Performance monitoring
            try:
                # Periodic cleanup to prevent memory leaks
                if n_intervals % 60 == 0:  # Every 60 seconds
                    self._cleanup_old_data()
                
                # Lightweight update every 10 intervals to reduce load
                is_lightweight_update = (n_intervals % 10 != 0)
                # Chart updates every second for responsiveness
                # Get current prices with improved fallback handling
                symbol = self.config.symbols[0] if self.config.symbols else "ETH/USDT"
                current_price = None
                chart_data = None
                data_source = "UNKNOWN"
                
                try:
                    # First try real-time WebSocket price (sub-second latency)
                    current_price = self.get_realtime_price(symbol)
                    if current_price:
                        data_source = "WEBSOCKET_RT"
                        logger.debug(f"[WS_RT] Using real-time WebSocket price for {symbol}: ${current_price:.2f}")
                    else:
                        # Try cached data first (faster than API calls)
                        cached_data = self.data_provider.get_historical_data(symbol, '1m', limit=1, refresh=False)
                        if cached_data is not None and not cached_data.empty:
                            current_price = float(cached_data['close'].iloc[-1])
                            data_source = "CACHED"
                            logger.debug(f"[CACHED] Using cached price for {symbol}: ${current_price:.2f}")
                        else:
                            # Only try fresh API call if we have no data at all
                            try:
                                fresh_data = self.data_provider.get_historical_data(symbol, '1m', limit=1, refresh=False)
                                if fresh_data is not None and not fresh_data.empty:
                                    current_price = float(fresh_data['close'].iloc[-1])
                                    data_source = "API"
                                    logger.debug(f"[API] Fresh price for {symbol}: ${current_price:.2f}")
                            except Exception as api_error:
                                logger.warning(f"[API_ERROR] Failed to fetch fresh data: {api_error}")
                        
                        # NO SYNTHETIC DATA - Wait for real data
                        if current_price is None:
                            logger.warning(f"[NO_DATA] No real data available for {symbol} - waiting for data provider")
                            data_source = "NO_DATA"
                            
                except Exception as e:
                    logger.warning(f"[ERROR] Error getting price for {symbol}: {e}")
                    current_price = None
                    data_source = "ERROR"
                
                # Get chart data - ONLY REAL DATA (optimized for performance)
                chart_data = None
                try:
                    if not is_lightweight_update:  # Only refresh charts every 10 seconds
                        # Use cached data only (limited to 30 bars for performance)
                        chart_data = self.data_provider.get_historical_data(symbol, '1m', limit=30, refresh=False)
                        if chart_data is not None and not chart_data.empty:
                            logger.debug(f"[CHART] Using cached 1m data: {len(chart_data)} bars")
                        else:
                            # Wait for real data - no synthetic data
                            logger.debug("[CHART] No chart data available - waiting for data provider")
                            chart_data = None
                    else:
                        # Use cached chart data for lightweight updates
                        chart_data = getattr(self, '_cached_chart_data', None)
                except Exception as e:
                    logger.warning(f"[CHART_ERROR] Error getting chart data: {e}")
                    chart_data = None
                
                # Generate trading signals based on model decisions - OPTIMIZED
                try:
                    # Only generate signals every few intervals to reduce CPU load
                    if not is_lightweight_update and current_price and chart_data is not None and not chart_data.empty and len(chart_data) >= 5:
                        # Model decides when to act - check for signals but not every single second
                        signal = self._generate_trading_signal(symbol, current_price, chart_data)
                        if signal:
                            # Add to signals list (all signals, regardless of execution)
                            signal['signal_type'] = 'GENERATED'
                            self.recent_signals.append(signal.copy())
                            if len(self.recent_signals) > 100:  # Keep last 100 signals
                                self.recent_signals = self.recent_signals[-100:]
                            
                            # Use adaptive threshold instead of fixed threshold
                            current_threshold = self.adaptive_learner.get_current_threshold()
                            should_execute = signal['confidence'] >= current_threshold
                            
                            # Check position limits before execution
                            can_execute = self._can_execute_new_position(signal['action'])
                            
                            if should_execute and can_execute:
                                signal['signal_type'] = 'EXECUTED'
                                signal['threshold_used'] = current_threshold  # Track threshold for learning
                                signal['reason'] = f"ADAPTIVE EXECUTE (≥{current_threshold:.2%}): {signal['reason']}"
                                logger.debug(f"[EXECUTE] {signal['action']} signal @ ${signal['price']:.2f} (confidence: {signal['confidence']:.1%} ≥ {current_threshold:.1%})")
                                self._process_trading_decision(signal)
                            elif should_execute and not can_execute:
                                # Signal meets confidence but we're at position limit
                                signal['signal_type'] = 'NOT_EXECUTED_POSITION_LIMIT'
                                signal['threshold_used'] = current_threshold
                                signal['reason'] = f"BLOCKED BY POSITION LIMIT (≥{current_threshold:.2%}): {signal['reason']} [Positions: {self._count_open_positions()}/{self.config.get('trading', {}).get('max_concurrent_positions', 3)}]"
                                logger.info(f"[BLOCKED] {signal['action']} signal @ ${signal['price']:.2f} - Position limit reached ({self._count_open_positions()}/{self.config.get('trading', {}).get('max_concurrent_positions', 3)})")
                                
                                # Still add to training queue for RL learning
                                self._queue_signal_for_training(signal, current_price, symbol)
                            else:
                                signal['signal_type'] = 'NOT_EXECUTED_LOW_CONFIDENCE'
                                signal['threshold_used'] = current_threshold
                                signal['reason'] = f"LOW CONFIDENCE (<{current_threshold:.2%}): {signal['reason']}"
                                logger.debug(f"[SKIP] {signal['action']} signal @ ${signal['price']:.2f} (confidence: {signal['confidence']:.1%} < {current_threshold:.1%})")
                                
                                # Still add to training queue for RL learning
                                self._queue_signal_for_training(signal, current_price, symbol)
                    else:
                        # Fallback: Add a simple monitoring update
                        if n_intervals % 10 == 0 and current_price:  # Every 10 seconds
                            monitor_signal = {
                                'action': 'MONITOR',
                                'symbol': symbol,
                                'price': current_price,
                                'confidence': 0.0,
                                'timestamp': datetime.now(),
                                'size': 0.0,
                                'reason': 'System monitoring - no trading signals',
                                'signal_type': 'MONITOR'
                            }
                            self.recent_decisions.append(monitor_signal)
                            if len(self.recent_decisions) > 500:
                                self.recent_decisions = self.recent_decisions[-500:]
                                
                except Exception as e:
                    logger.warning(f"[ERROR] Error generating trading signal: {e}")
                
                # Calculate PnL metrics
                unrealized_pnl = self._calculate_unrealized_pnl(current_price) if current_price else 0.0
                total_session_pnl = self.total_realized_pnl + unrealized_pnl
                
                # Calculate portfolio value
                portfolio_value = self.starting_balance + total_session_pnl
                
                # Get memory stats with fallback (still needed for system status)
                try:
                    memory_stats = self.model_registry.get_memory_stats()
                except:
                    memory_stats = {'utilization_percent': 0, 'total_used_mb': 0, 'total_limit_mb': 1024}
                
                # Format outputs with safe defaults and update indicators
                update_time = datetime.now().strftime("%H:%M:%S.%f")[:-3]  # Include milliseconds
                
                if current_price:
                    # Add data source indicator and precise timestamp
                    source_indicator = f"[{data_source}]"
                    price_text = f"${current_price:.2f} {source_indicator} @ {update_time}"
                else:
                    # Show waiting status when no real data
                    price_text = f"WAITING FOR REAL DATA [{data_source}] @ {update_time}"
                
                # PnL formatting
                pnl_text = f"${total_session_pnl:.2f}"
                pnl_class = "text-success mb-0 small" if total_session_pnl >= 0 else "text-danger mb-0 small"
                
                # Total fees formatting
                fees_text = f"${self.total_fees:.2f}"
                
                # Position info with real-time unrealized PnL and proper color coding
                if self.current_position:
                    pos_side = self.current_position['side']
                    pos_size = self.current_position['size']
                    pos_price = self.current_position['price']
                    unrealized_pnl = self._calculate_unrealized_pnl(current_price) if current_price else 0.0
                    
                    # Color coding: LONG=Green, SHORT=Red (consistent with trading conventions)
                    if pos_side == 'LONG':
                        side_icon = "[LONG]"
                        side_color = "success"  # Green for long positions
                    else:  # SHORT
                        side_icon = "[SHORT]"
                        side_color = "danger"   # Red for short positions
                    
                    # Create enhanced position display with bold styling
                    pnl_sign = "+" if unrealized_pnl > 0 else ""
                    position_text = f"{side_icon} {pos_size} @ ${pos_price:.2f} | P&L: {pnl_sign}${unrealized_pnl:.2f}"
                    position_class = f"text-{side_color} fw-bold mb-0 small"
                else:
                    position_text = "No Position"
                    position_class = "text-muted mb-0 small"
                
                # Trade count and portfolio value
                trade_count_text = f"{len(self.session_trades)}"
                portfolio_text = f"${portfolio_value:,.2f}"
                
                # MEXC status with detailed information
                if self.trading_executor and self.trading_executor.trading_enabled:
                    if self.trading_executor.simulation_mode:
                        mexc_status = f"{self.trading_executor.trading_mode.upper()} MODE"
                    else:
                        mexc_status = "LIVE"
                else:
                    mexc_status = "OFFLINE"
                
                # Create charts with error handling - OPTIMIZED
                try:
                    # Always try to create/update chart every second for smooth responsiveness
                    if current_price and chart_data is not None and not chart_data.empty:
                        price_chart = self._create_price_chart(symbol)
                        self._cached_chart_data = chart_data  # Cache for fallback
                        self._cached_price_chart = price_chart  # Cache chart
                    else:
                        # Use cached chart if we have one, otherwise show loading
                        if hasattr(self, '_cached_price_chart') and self._cached_price_chart:
                            price_chart = self._cached_price_chart
                            # Update the cached chart with current info
                            try:
                                current_time_str = datetime.now().strftime("%H:%M:%S")
                                stream_status = "LIVE STREAM" if self.is_streaming else "WAITING DATA"
                                price_chart.update_layout(
                                    title=f"{symbol} 1M CHART | ${current_price or 0:.2f} | {stream_status} | {current_time_str}"
                                )
                            except Exception as e:
                                logger.debug(f"Error updating cached chart: {e}")
                        else:
                            price_chart = self._create_empty_chart("Price Chart", "Waiting for real market data...")
                            self._cached_price_chart = price_chart
                except Exception as e:
                    logger.warning(f"Price chart error: {e}")
                    price_chart = self._create_empty_chart("Price Chart", "Error loading chart - waiting for data")
                
                # Create training metrics display
                try:
                    training_metrics = self._create_training_metrics()
                except Exception as e:
                    logger.warning(f"Training metrics error: {e}")
                    training_metrics = [html.P("Training metrics unavailable", className="text-muted")]
                
                # Create recent decisions list
                try:
                    decisions_list = self._create_decisions_list()
                except Exception as e:
                    logger.warning(f"Decisions list error: {e}")
                    decisions_list = [html.P("No decisions available", className="text-muted")]
                
                # Create session performance
                try:
                    session_perf = self._create_session_performance()
                except Exception as e:
                    logger.warning(f"Session performance error: {e}")
                    session_perf = [html.P("Performance data unavailable", className="text-muted")]
                
                # Create system status
                try:
                    system_status = self._create_system_status_compact(memory_stats)
                except Exception as e:
                    logger.warning(f"System status error: {e}")
                    system_status = {
                        'icon_class': "fas fa-circle text-danger fa-2x",
                        'title': "System Error: Check logs",
                        'details': [html.P(f"Error: {str(e)}", className="text-danger")]
                    }
                
                # Create closed trades table
                try:
                    closed_trades_table = self._create_closed_trades_table()
                except Exception as e:
                    logger.warning(f"Closed trades table error: {e}")
                    closed_trades_table = [html.P("Closed trades data unavailable", className="text-muted")]
                
                # Calculate leverage display values
                leverage_text = f"{self.leverage_multiplier:.0f}x"
                if self.leverage_multiplier <= 5:
                    risk_level = "Low Risk"
                    risk_class = "bg-success"
                elif self.leverage_multiplier <= 25:
                    risk_level = "Medium Risk"
                    risk_class = "bg-warning text-dark"
                elif self.leverage_multiplier <= 50:
                    risk_level = "High Risk"
                    risk_class = "bg-danger"
                else:
                    risk_level = "Extreme Risk"
                    risk_class = "bg-dark"
                
                # Create CNN monitoring content
                try:
                    cnn_monitoring_content = self._create_cnn_monitoring_content()
                except Exception as e:
                    logger.warning(f"CNN monitoring error: {e}")
                    cnn_monitoring_content = [html.P("CNN monitoring unavailable", className="text-danger")]

                return (
                    price_text, pnl_text, pnl_class, fees_text, position_text, position_class, trade_count_text, portfolio_text, mexc_status,
                    price_chart, training_metrics, decisions_list, session_perf, closed_trades_table,
                    system_status['icon_class'], system_status['title'], system_status['details'],
                    leverage_text, f"{risk_level}",
                    cnn_monitoring_content
                )
                
            except Exception as e:
                logger.error(f"Error updating dashboard: {e}")
                # Return safe defaults
                empty_fig = self._create_empty_chart("Error", "Dashboard error - check logs")
                
                return (
                    "Error", "$0.00", "text-muted mb-0 small", "$0.00", "None", "text-muted", "0", "$10,000.00", "OFFLINE",
                    empty_fig, 
                    [html.P("Error loading training metrics", className="text-danger")],
                    [html.P("Error loading decisions", className="text-danger")], 
                    [html.P("Error loading performance", className="text-danger")],
                    [html.P("Error loading closed trades", className="text-danger")],
                    "fas fa-circle text-danger fa-2x",
                    "Error: Dashboard error - check logs",
                    [html.P(f"Error: {str(e)}", className="text-danger")],
                    f"{self.leverage_multiplier:.0f}x", "Error",
                    [html.P("CNN monitoring unavailable", className="text-danger")]
                )
        
        # Clear history callback
        @self.app.callback(
            Output('closed-trades-table', 'children', allow_duplicate=True),
            [Input('clear-history-btn', 'n_clicks')],
            prevent_initial_call=True
        )
        def clear_trade_history(n_clicks):
            """Clear trade history and reset session stats"""
            if n_clicks and n_clicks > 0:
                try:
                    # Clear both closed trades and session stats (they're the same now)
                    self.clear_closed_trades_history()
                    logger.info("DASHBOARD: Trade history and session stats cleared by user")
                    return [html.P("Trade history cleared", className="text-success text-center")]
                except Exception as e:
                    logger.error(f"Error clearing trade history: {e}")
                    return [html.P(f"Error clearing history: {str(e)}", className="text-danger text-center")]
            return dash.no_update
        
        # Leverage slider callback
        @self.app.callback(
            [Output('current-leverage', 'children', allow_duplicate=True),
             Output('leverage-risk', 'children', allow_duplicate=True),
             Output('leverage-risk', 'className', allow_duplicate=True)],
            [Input('leverage-slider', 'value')],
            prevent_initial_call=True
        )
        def update_leverage(leverage_value):
            """Update leverage multiplier and risk assessment"""
            try:
                if leverage_value is None:
                    return dash.no_update
                
                # Update internal leverage value
                self.leverage_multiplier = float(leverage_value)
                
                # Calculate risk level and styling
                leverage_text = f"{self.leverage_multiplier:.0f}x"
                
                if self.leverage_multiplier <= 5:
                    risk_level = "Low Risk"
                    risk_class = "badge bg-success"
                elif self.leverage_multiplier <= 25:
                    risk_level = "Medium Risk" 
                    risk_class = "badge bg-warning text-dark"
                elif self.leverage_multiplier <= 50:
                    risk_level = "High Risk"
                    risk_class = "badge bg-danger"
                else:
                    risk_level = "Extreme Risk"
                    risk_class = "badge bg-dark"
                
                # Update trading server if connected
                try:
                    import requests
                    response = requests.post(f"{self.trading_server_url}/update_leverage", 
                                           json={"leverage": self.leverage_multiplier}, 
                                           timeout=2)
                    if response.status_code == 200:
                        logger.info(f"[LEVERAGE] Updated trading server leverage to {self.leverage_multiplier}x")
                    else:
                        logger.warning(f"[LEVERAGE] Failed to update trading server: {response.status_code}")
                except Exception as e:
                    logger.debug(f"[LEVERAGE] Trading server not available: {e}")
                
                logger.info(f"[LEVERAGE] Leverage updated to {self.leverage_multiplier}x ({risk_level})")
                
                return leverage_text, risk_level, risk_class
                
            except Exception as e:
                logger.error(f"Error updating leverage: {e}")
                return f"{self.leverage_multiplier:.0f}x", "Error", "badge bg-secondary"
    
    def _create_price_chart(self, symbol: str) -> go.Figure:
        """Create price chart with volume and Williams pivot points from cached data"""
        # Chart-level caching to prevent excessive recreation
        current_time = time.time()
        cache_key = f"chart_{symbol}"
        
        # Use cached chart if less than 2 seconds old (very responsive)
        if hasattr(self, '_chart_cache') and cache_key in self._chart_cache:
            cached_chart, cache_time = self._chart_cache[cache_key]
            if current_time - cache_time < 2:  # 2-second chart cache for high responsiveness
                logger.debug(f"[CHART] Using cached chart for {symbol}")
                return cached_chart
        
        try:
            # Use cached data from data provider (optimized for performance)
            # Reduced from 50 to 30 bars for faster chart rendering
            df = self.data_provider.get_historical_data(symbol, '1m', limit=30, refresh=False)
            
            if df is None or df.empty:
                logger.warning("[CHART] No cached data available, trying fresh data")
                try:
                    df = self.data_provider.get_historical_data(symbol, '1m', limit=30, refresh=True)
                    if df is not None and not df.empty:
                        # Ensure timezone consistency for fresh data
                        df = self._ensure_timezone_consistency(df)
                        # Add volume column if missing
                        if 'volume' not in df.columns:
                            df['volume'] = 100  # Default volume for demo
                        actual_timeframe = '1m'
                    else:
                        return self._create_empty_chart(
                            f"{symbol} Chart", 
                            f"No data available for {symbol}\nWaiting for data provider..."
                        )
                except Exception as e:
                    logger.warning(f"[ERROR] Error getting fresh data: {e}")
                    return self._create_empty_chart(
                        f"{symbol} Chart", 
                        f"Chart Error: {str(e)}"
                    )
            else:
                # Ensure timezone consistency for cached data
                df = self._ensure_timezone_consistency(df)
                actual_timeframe = '1m'
                logger.debug(f"[CHART] Using {len(df)} 1m bars from cached data in {self.timezone}")
            
            # Get the timeframe of displayed chart for filtering decisions and trades
            chart_start_time = df.index.min()
            chart_end_time = df.index.max()
            
            # Create subplot with secondary y-axis for volume
            fig = make_subplots(
                rows=2, cols=1,
                shared_xaxes=True,
                vertical_spacing=0.1,
                subplot_titles=(f'{symbol} Price ({actual_timeframe.upper()}) with Williams Pivot Points', 'Volume'),
                row_heights=[0.7, 0.3]
            )
            
            # Add price line chart (main chart)
            fig.add_trace(
                go.Scatter(
                    x=df.index,
                    y=df['close'],
                    mode='lines',
                    name=f"{symbol} Price",
                    line=dict(color='#00ff88', width=2),
                    hovertemplate='<b>$%{y:.2f}</b><br>%{x}<extra></extra>'
                ),
                row=1, col=1
            )
            
            # Add Williams Market Structure pivot points
            try:
                pivot_points = self._get_williams_pivot_points_for_chart(df)
                if pivot_points:
                    self._add_williams_pivot_points_to_chart(fig, pivot_points, row=1)
                else:
                    logger.debug("[CHART] No Williams pivot points available")
            except Exception as e:
                logger.debug(f"Error adding Williams pivot points to chart: {e}")
            
            # Add moving averages if we have enough data
            if len(df) >= 20:
                # 20-period SMA (create a copy to avoid modifying original data)
                sma_20 = df['close'].rolling(window=20).mean()
                fig.add_trace(
                    go.Scatter(
                        x=df.index,
                        y=sma_20,
                        name='SMA 20',
                        line=dict(color='#ff1493', width=1),
                        opacity=0.8,
                        hovertemplate='<b>SMA20: $%{y:.2f}</b><br>%{x}<extra></extra>'
                    ),
                    row=1, col=1
                )
            
            # Removed SMA 50 since we only have 30 bars maximum
            
            # Add volume bars
            if 'volume' in df.columns:
                fig.add_trace(
                    go.Bar(
                        x=df.index,
                        y=df['volume'],
                        name='Volume',
                        marker_color='rgba(158, 158, 158, 0.6)',
                        hovertemplate='<b>Volume: %{y:.0f}</b><br>%{x}<extra></extra>'
                    ),
                    row=2, col=1
                                )

            # Mark recent trading decisions with proper markers - OPTIMIZED
            try:
                # Filter decisions to only those within the chart timeframe
                buy_decisions = []
                sell_decisions = []
                
                for decision in self.recent_decisions[-100:]:  # Limit to last 100 decisions
                    if isinstance(decision, dict) and 'timestamp' in decision and 'price' in decision and 'action' in decision:
                        decision_time = decision['timestamp']
                        
                        # Convert decision timestamp to match chart timezone if needed
                        if isinstance(decision_time, datetime):
                            if decision_time.tzinfo is not None:
                                decision_time_utc = decision_time.astimezone(timezone.utc).replace(tzinfo=None)
                            else:
                                decision_time_utc = decision_time
                        else:
                            continue
                        
                        # Convert chart times to UTC for comparison
                        try:
                            if isinstance(chart_start_time, pd.Timestamp):
                                chart_start_utc = chart_start_time.tz_localize(None) if chart_start_time.tz is None else chart_start_time.tz_convert('UTC').tz_localize(None)
                                chart_end_utc = chart_end_time.tz_localize(None) if chart_end_time.tz is None else chart_end_time.tz_convert('UTC').tz_localize(None)
                            else:
                                chart_start_utc = pd.to_datetime(chart_start_time).tz_localize(None)
                                chart_end_utc = pd.to_datetime(chart_end_time).tz_localize(None)
                            
                            # Check if decision falls within chart timeframe
                            decision_time_pd = pd.to_datetime(decision_time_utc)
                            if chart_start_utc <= decision_time_pd <= chart_end_utc:
                                pass  # Continue processing
                            else:
                                continue  # Skip this decision
                        except Exception as e:
                            logger.debug(f"Error comparing decision timestamp: {e}")
                            continue  # Skip this decision
                        
                        signal_type = decision.get('signal_type', 'UNKNOWN')
                        if decision['action'] == 'BUY':
                            buy_decisions.append((decision, signal_type))
                        elif decision['action'] == 'SELL':
                            sell_decisions.append((decision, signal_type))
                
                logger.debug(f"[CHART] Showing {len(buy_decisions)} BUY and {len(sell_decisions)} SELL signals in chart timeframe")
                
                # Add BUY markers with different styles for executed vs ignored
                executed_buys = [d[0] for d in buy_decisions if d[1] == 'EXECUTED']
                ignored_buys = [d[0] for d in buy_decisions if d[1] in ['NOT_EXECUTED_POSITION_LIMIT', 'NOT_EXECUTED_LOW_CONFIDENCE']]
                
                if executed_buys:
                    fig.add_trace(
                        go.Scatter(
                            x=[self._to_local_timezone(d['timestamp']) for d in executed_buys],
                            y=[d['price'] for d in executed_buys],
                            mode='markers',
                            marker=dict(
                                color='#00ff88',
                                size=14,
                                symbol='triangle-up',
                                line=dict(color='white', width=2)
                            ),
                            name="BUY (Executed)",
                            showlegend=True,
                            hovertemplate="<b>BUY EXECUTED</b><br>Price: $%{y:.2f}<br>Time: %{x}<br>Confidence: %{customdata:.1%}<extra></extra>",
                            customdata=[d.get('confidence', 0) for d in executed_buys]
                        ),
                        row=1, col=1
                    )
                
                if ignored_buys:
                    fig.add_trace(
                        go.Scatter(
                            x=[self._to_local_timezone(d['timestamp']) for d in ignored_buys],
                            y=[d['price'] for d in ignored_buys],
                            mode='markers',
                            marker=dict(
                                color='#00ff88',
                                size=10,
                                symbol='triangle-up-open',
                                line=dict(color='#00ff88', width=2)
                            ),
                            name="BUY (Blocked)",
                            showlegend=True,
                            hovertemplate="<b>BUY BLOCKED</b><br>Price: $%{y:.2f}<br>Time: %{x}<br>Confidence: %{customdata:.1%}<extra></extra>",
                            customdata=[d.get('confidence', 0) for d in ignored_buys]
                        ),
                        row=1, col=1
                    )
                
                # Add SELL markers with different styles for executed vs ignored
                executed_sells = [d[0] for d in sell_decisions if d[1] == 'EXECUTED']
                ignored_sells = [d[0] for d in sell_decisions if d[1] in ['NOT_EXECUTED_POSITION_LIMIT', 'NOT_EXECUTED_LOW_CONFIDENCE']]
                
                if executed_sells:
                    fig.add_trace(
                        go.Scatter(
                            x=[self._to_local_timezone(d['timestamp']) for d in executed_sells],
                            y=[d['price'] for d in executed_sells],
                            mode='markers',
                            marker=dict(
                                color='#ff6b6b',
                                size=14,
                                symbol='triangle-down',
                                line=dict(color='white', width=2)
                            ),
                            name="SELL (Executed)",
                            showlegend=True,
                            hovertemplate="<b>SELL EXECUTED</b><br>Price: $%{y:.2f}<br>Time: %{x}<br>Confidence: %{customdata:.1%}<extra></extra>",
                            customdata=[d.get('confidence', 0) for d in executed_sells]
                        ),
                        row=1, col=1
                    )
                
                if ignored_sells:
                    fig.add_trace(
                        go.Scatter(
                            x=[self._to_local_timezone(d['timestamp']) for d in ignored_sells],
                            y=[d['price'] for d in ignored_sells],
                            mode='markers',
                            marker=dict(
                                color='#ff6b6b',
                                size=10,
                                symbol='triangle-down-open',
                                line=dict(color='#ff6b6b', width=2)
                            ),
                            name="SELL (Blocked)",
                            showlegend=True,
                            hovertemplate="<b>SELL BLOCKED</b><br>Price: $%{y:.2f}<br>Time: %{x}<br>Confidence: %{customdata:.1%}<extra></extra>",
                            customdata=[d.get('confidence', 0) for d in ignored_sells]
                        ),
                        row=1, col=1
                    )
            except Exception as e:
                logger.debug(f"Error adding trading decision markers to chart: {e}")
            
            # Add closed trades markers with profit/loss styling and connecting lines
            try:
                if self.closed_trades and not df.empty:
                    # Convert chart times to UTC for comparison
                    if isinstance(chart_start_time, pd.Timestamp):
                        chart_start_utc = chart_start_time.tz_localize(None) if chart_start_time.tz is None else chart_start_time.tz_convert('UTC').tz_localize(None)
                        chart_end_utc = chart_end_time.tz_localize(None) if chart_end_time.tz is None else chart_end_time.tz_convert('UTC').tz_localize(None)
                    else:
                        chart_start_utc = pd.to_datetime(chart_start_time).tz_localize(None)
                        chart_end_utc = pd.to_datetime(chart_end_time).tz_localize(None)
                    
                    # Filter closed trades to only those within chart timeframe
                    chart_trades = []
                    for trade in self.closed_trades:
                        if not isinstance(trade, dict):
                            continue
                        
                        entry_time = trade.get('entry_time')
                        exit_time = trade.get('exit_time')
                        
                        if not entry_time or not exit_time:
                            continue
                        
                        # Convert times to UTC for comparison
                        try:
                            if isinstance(entry_time, datetime):
                                entry_time_utc = entry_time.astimezone(timezone.utc).replace(tzinfo=None) if entry_time.tzinfo else entry_time
                            else:
                                continue
                                
                            if isinstance(exit_time, datetime):
                                exit_time_utc = exit_time.astimezone(timezone.utc).replace(tzinfo=None) if exit_time.tzinfo else exit_time
                            else:
                                continue
                            
                            # Check if trade overlaps with chart timeframe
                            entry_time_pd = pd.to_datetime(entry_time_utc)
                            exit_time_pd = pd.to_datetime(exit_time_utc)
                            
                            if (chart_start_utc <= entry_time_pd <= chart_end_utc) or (chart_start_utc <= exit_time_pd <= chart_end_utc):
                                chart_trades.append(trade)
                        except Exception as e:
                            logger.debug(f"Error comparing trade timestamps: {e}")
                            continue  # Skip this trade
                    
                    logger.debug(f"[CHART] Showing {len(chart_trades)} closed trades on chart")
                    
                    # Plot closed trades with profit/loss styling
                    profitable_entries_x = []
                    profitable_entries_y = []
                    profitable_exits_x = []
                    profitable_exits_y = []
                    
                    # Collect trade points for display
                    for trade in chart_trades:
                        entry_price = trade.get('entry_price', 0)
                        exit_price = trade.get('exit_price', 0)
                        entry_time = trade.get('entry_time')
                        exit_time = trade.get('exit_time')
                        net_pnl = trade.get('net_pnl', 0)
                        
                        if not all([entry_price, exit_price, entry_time, exit_time]):
                            continue
                        
                        # Convert times to local timezone for display
                        entry_time_local = self._to_local_timezone(entry_time)
                        exit_time_local = self._to_local_timezone(exit_time)
                        
                        # Determine if trade was profitable
                        is_profitable = net_pnl > 0
                        
                        if is_profitable:
                            profitable_entries_x.append(entry_time_local)
                            profitable_entries_y.append(entry_price)
                            profitable_exits_x.append(exit_time_local)
                            profitable_exits_y.append(exit_price)
                        
                        # Add connecting dash line between entry and exit
                        line_color = '#00ff88' if is_profitable else '#ff6b6b'
                        fig.add_trace(
                            go.Scatter(
                                x=[entry_time_local, exit_time_local],
                                y=[entry_price, exit_price],
                                mode='lines',
                                line=dict(
                                    color=line_color,
                                    width=2,
                                    dash='dash'
                                ),
                                name="Trade Path",
                                showlegend=False,
                                hoverinfo='skip'
                            ),
                            row=1, col=1
                        )
                    
                    # Add profitable trade markers (filled triangles)
                    if profitable_entries_x:
                        # Entry markers
                        fig.add_trace(
                            go.Scatter(
                                x=profitable_entries_x,
                                y=profitable_entries_y,
                                mode='markers',
                                marker=dict(
                                    color='#00ff88',
                                    size=12,
                                    symbol='triangle-up',
                                    line=dict(color='white', width=1)
                                ),
                                name="Profitable Entry",
                                showlegend=True,
                                hovertemplate="<b>PROFITABLE ENTRY</b><br>Price: $%{y:.2f}<br>Time: %{x}<extra></extra>"
                            ),
                            row=1, col=1
                        )
                    
                    if profitable_exits_x:
                        # Exit markers
                        fig.add_trace(
                            go.Scatter(
                                x=profitable_exits_x,
                                y=profitable_exits_y,
                                mode='markers',
                                marker=dict(
                                    color='#00ff88',
                                    size=12,
                                    symbol='triangle-down',
                                    line=dict(color='white', width=1)
                                ),
                                name="Profitable Exit",
                                showlegend=True,
                                hovertemplate="<b>PROFITABLE EXIT</b><br>Price: $%{y:.2f}<br>Time: %{x}<extra></extra>"
                            ),
                            row=1, col=1
                        )
            except Exception as e:
                logger.debug(f"Error adding closed trades to chart: {e}")
                
            # Update layout with current timestamp and streaming status
            current_time = datetime.now().strftime("%H:%M:%S.%f")[:-3]
            try:
                latest_price = float(df['close'].iloc[-1]) if not df.empty else 0.0
            except (ValueError, TypeError, IndexError):
                latest_price = 0.0
            stream_status = "LIVE STREAM" if self.is_streaming else "CACHED DATA"
            tick_count = len(self.tick_cache)
            
            fig.update_layout(
                title=f"{symbol} {actual_timeframe.upper()} CHART | ${latest_price:.2f} | {stream_status} | {tick_count} ticks | {current_time}",
                template="plotly_dark",
                height=450,
                xaxis_rangeslider_visible=False,
                margin=dict(l=20, r=20, t=50, b=20),
                legend=dict(
                    orientation="h",
                    yanchor="bottom",
                    y=1.02,
                    xanchor="right",
                    x=1
                )
            )
            
            # Update y-axis labels
            fig.update_yaxes(title_text="Price ($)", row=1, col=1)
            fig.update_yaxes(title_text="Volume", row=2, col=1)
            fig.update_xaxes(title_text="Time", row=2, col=1)
            
            # Cache the chart for performance
            if not hasattr(self, '_chart_cache'):
                self._chart_cache = {}
            
            self._chart_cache[cache_key] = (fig, current_time)
            
            # Clean old chart cache entries (keep last 3)
            if len(self._chart_cache) > 3:
                oldest_key = min(self._chart_cache.keys(), 
                               key=lambda k: self._chart_cache[k][1])
                del self._chart_cache[oldest_key]
            
            logger.debug(f"[CHART] Created and cached new chart for {symbol}")
            return fig
            
        except Exception as e:
            import traceback
            logger.error(f"Error creating price chart: {e}")
            logger.debug(f"Chart error traceback: {traceback.format_exc()}")
            
            return self._create_empty_chart(
                f"{symbol} Chart Error",
                f"Chart creation failed: {str(e)}"
            )
    
    def _generate_trading_signal(self, symbol: str, current_price: float, df: pd.DataFrame) -> Optional[Dict]:
        """
        Generate aggressive scalping signals based on price action and indicators
        Returns trading decision dict or None
        """
        try:
            if df is None or df.empty or len(df) < 10:  # Reduced minimum data requirement
                return None
            
            # Get recent price action
            recent_prices = df['close'].tail(15).values  # Reduced data for faster signals
            
            if len(recent_prices) >= 5:  # Reduced minimum requirement
                # More aggressive signal generation for scalping
                short_ma = np.mean(recent_prices[-2:])  # 2-period MA (very short)
                medium_ma = np.mean(recent_prices[-5:])  # 5-period MA
                long_ma = np.mean(recent_prices[-10:])  # 10-period MA
                
                # Calculate momentum and trend strength
                momentum = (short_ma - long_ma) / long_ma
                trend_strength = abs(momentum)
                price_change_pct = (current_price - recent_prices[0]) / recent_prices[0]
                
                # More aggressive scalping conditions (lower thresholds)
                import random
                random_factor = random.uniform(0.1, 1.0)  # Even lower threshold for more signals
                
                # Scalping-friendly signal conditions (much more sensitive)
                buy_conditions = [
                    (short_ma > medium_ma and momentum > 0.0001),  # Very small momentum threshold
                    (price_change_pct > 0.0003 and random_factor > 0.3),  # Small price movement
                    (momentum > 0.00005 and random_factor > 0.5),  # Tiny momentum
                    (current_price > recent_prices[-1] and random_factor > 0.7),  # Simple price increase
                    (random_factor > 0.9)  # Random for demo activity
                ]
                
                sell_conditions = [
                    (short_ma < medium_ma and momentum < -0.0001),  # Very small momentum threshold
                    (price_change_pct < -0.0003 and random_factor > 0.3),  # Small price movement
                    (momentum < -0.00005 and random_factor > 0.5),  # Tiny momentum
                    (current_price < recent_prices[-1] and random_factor > 0.7),  # Simple price decrease
                    (random_factor < 0.1)  # Random for demo activity
                ]
                
                buy_signal = any(buy_conditions)
                sell_signal = any(sell_conditions)
                
                # Ensure we don't have both signals at once, prioritize the stronger one
                if buy_signal and sell_signal:
                    if abs(momentum) > 0.0001:
                        # Use momentum to decide
                        buy_signal = momentum > 0
                        sell_signal = momentum < 0
                    else:
                        # Use random to break tie for demo
                        if random_factor > 0.5:
                            sell_signal = False
                        else:
                            buy_signal = False
                
                if buy_signal:
                    # More realistic confidence calculation based on multiple factors
                    momentum_confidence = min(0.3, abs(momentum) * 1000)  # Momentum contribution
                    trend_confidence = min(0.3, trend_strength * 5)       # Trend strength contribution
                    random_confidence = random_factor * 0.4               # Random component
                    
                    # Combine factors for total confidence
                    confidence = 0.5 + momentum_confidence + trend_confidence + random_confidence
                    confidence = max(0.45, min(0.95, confidence))  # Keep in reasonable range
                    
                    return {
                        'action': 'BUY',
                        'symbol': symbol,
                        'price': current_price,
                        'confidence': confidence,
                        'timestamp': datetime.now(timezone.utc),  # Use UTC to match candle data
                        'size': 0.1,  # Will be adjusted by confidence in processing
                        'reason': f'Scalping BUY: momentum={momentum:.6f}, trend={trend_strength:.6f}, conf={confidence:.3f}'
                    }
                elif sell_signal:
                    # More realistic confidence calculation based on multiple factors
                    momentum_confidence = min(0.3, abs(momentum) * 1000)  # Momentum contribution
                    trend_confidence = min(0.3, trend_strength * 5)       # Trend strength contribution
                    random_confidence = random_factor * 0.4               # Random component
                    
                    # Combine factors for total confidence
                    confidence = 0.5 + momentum_confidence + trend_confidence + random_confidence
                    confidence = max(0.45, min(0.95, confidence))  # Keep in reasonable range
                    
                    return {
                        'action': 'SELL',
                        'symbol': symbol,
                        'price': current_price,
                        'confidence': confidence,
                        'timestamp': datetime.now(timezone.utc),  # Use UTC to match candle data
                        'size': 0.1,  # Will be adjusted by confidence in processing
                        'reason': f'Scalping SELL: momentum={momentum:.6f}, trend={trend_strength:.6f}, conf={confidence:.3f}'
                    }
            
            return None
            
        except Exception as e:
            logger.warning(f"Error generating trading signal: {e}")
            return None

    def _process_trading_decision(self, decision: Dict) -> None:
        """Process a trading decision and update PnL tracking with enhanced fee calculation"""
        try:
            if not decision:
                return
            
            current_time = datetime.now(timezone.utc)  # Use UTC for consistency
            
            # Get fee structure from config (fallback to hardcoded values)
            try:
                from core.config import get_config
                config = get_config()
                trading_fees = config.get('trading', {}).get('trading_fees', {})
                maker_fee_rate = trading_fees.get('maker', 0.0005)  # 0.05% maker (changed from 0.0000)
                taker_fee_rate = trading_fees.get('taker', 0.0005)  # 0.05% taker
                default_fee_rate = trading_fees.get('default', 0.0005)  # 0.05% default
            except:
                # Fallback to hardcoded asymmetrical fees
                maker_fee_rate = 0.0000  # 0.00% maker fee
                taker_fee_rate = 0.0005  # 0.05% taker fee
                default_fee_rate = 0.0005  # 0.05% default
            
            # For simulation, assume most trades are taker orders (market orders)
            # In real trading, this would be determined by order type
            fee_rate = taker_fee_rate  # Default to taker fee
            fee_type = 'taker'  # Default to taker
            
            # If using limit orders that get filled (maker), use maker fee
            # This could be enhanced based on actual order execution data
            if decision.get('order_type') == 'limit' and decision.get('filled_as_maker', False):
                fee_rate = maker_fee_rate
                fee_type = 'maker'
            
            # Execute trade through MEXC if available
            mexc_success = False
            if self.trading_executor and decision['action'] != 'HOLD':
                try:
                    mexc_success = self.trading_executor.execute_signal(
                        symbol=decision['symbol'],
                        action=decision['action'],
                        confidence=decision['confidence'],
                        current_price=decision['price']
                    )
                    if mexc_success:
                        logger.info(f"MEXC: Trade executed successfully: {decision['action']} {decision['symbol']}")
                    else:
                        logger.warning(f"MEXC: Trade execution failed: {decision['action']} {decision['symbol']}")
                except Exception as e:
                    logger.error(f"MEXC: Error executing trade: {e}")
            
            # Add MEXC execution status to decision record
            decision['mexc_executed'] = mexc_success
            
            # Calculate position size based on confidence and configuration
            current_price = decision.get('price', 0)
            if current_price and current_price > 0:
                # Get position sizing from trading executor configuration
                if self.trading_executor:
                    usd_size = self.trading_executor._calculate_position_size(decision['confidence'], current_price)
                else:
                    # Fallback calculation based on confidence
                    max_usd = 1.0  # Default max position
                    min_usd = 0.1  # Default min position
                    usd_size = max(min_usd, min(max_usd * decision['confidence'], max_usd))
                
                position_size = usd_size / current_price  # Convert USD to crypto amount
                decision['size'] = round(position_size, 6)  # Update decision with calculated size
                decision['usd_size'] = usd_size  # Track USD amount for logging
            else:
                # Fallback if no price available
                decision['size'] = 0.001
                decision['usd_size'] = 0.1
            
            if decision['action'] == 'BUY':
                # First, close any existing SHORT position
                if self.current_position and self.current_position['side'] == 'SHORT':
                    # Close short position
                    entry_price = self.current_position['price']
                    exit_price = decision['price']
                    size = self.current_position['size']
                    entry_time = self.current_position['timestamp']
                    
                    # Calculate PnL for closing short with leverage
                    leveraged_pnl, leveraged_fee = self._calculate_leveraged_pnl_and_fees(
                        entry_price, exit_price, size, 'SHORT', fee_rate
                    )
                    net_pnl = leveraged_pnl - leveraged_fee - self.current_position['fees']
                    
                    self.total_realized_pnl += net_pnl
                    self.total_fees += leveraged_fee
                    
                    # Record the close trade
                    close_record = decision.copy()
                    close_record['position_action'] = 'CLOSE_SHORT'
                    close_record['entry_price'] = entry_price
                    close_record['pnl'] = net_pnl
                    close_record['fees'] = leveraged_fee
                    close_record['fee_type'] = fee_type
                    close_record['fee_rate'] = fee_rate
                    close_record['size'] = size  # Use original position size for close
                    self.session_trades.append(close_record)
                    
                    # Add to closed trades accounting list
                    closed_trade = {
                        'trade_id': len(self.closed_trades) + 1,
                        'side': 'SHORT',
                        'entry_time': entry_time,
                        'exit_time': current_time,
                        'entry_price': entry_price,
                        'exit_price': exit_price,
                        'size': size,
                        'leverage': self.leverage_multiplier,  # Store leverage used
                        'gross_pnl': leveraged_pnl,
                        'fees': leveraged_fee + self.current_position['fees'],
                        'fee_type': fee_type,
                        'fee_rate': fee_rate,
                        'net_pnl': net_pnl,
                        'duration': current_time - entry_time,
                        'symbol': decision.get('symbol', 'ETH/USDT'),
                        'mexc_executed': decision.get('mexc_executed', False)
                    }
                    self.closed_trades.append(closed_trade)
                    
                    # Save to file for persistence
                    self._save_closed_trades_to_file()
                    
                    # Trigger RL training on this closed trade
                    self._trigger_rl_training_on_closed_trade(closed_trade)
                    
                    # Record outcome for adaptive threshold learning
                    if 'confidence' in decision and 'threshold_used' in decision:
                        self.adaptive_learner.record_trade_outcome(
                            confidence=decision['confidence'],
                            pnl=net_pnl,
                            threshold_used=decision['threshold_used']
                        )
                        logger.debug(f"[ADAPTIVE] Recorded SHORT close outcome: PnL=${net_pnl:.2f}")
                    
                    logger.info(f"[TRADE] CLOSED SHORT: {size} @ ${exit_price:.2f} | PnL: ${net_pnl:.2f} | OPENING LONG")
                    
                    # Clear position before opening new one
                    self.current_position = None
                
                # Now open long position (regardless of previous position)
                if self.current_position is None:
                    # Open long position with confidence-based size
                    fee = decision['price'] * decision['size'] * fee_rate  # ✅ FIXED: No leverage on fees
                    self.current_position = {
                        'side': 'LONG',
                        'price': decision['price'],
                        'size': decision['size'],
                        'timestamp': current_time,
                        'fees': fee
                    }
                    self.total_fees += fee
                    
                    trade_record = decision.copy()
                    trade_record['position_action'] = 'OPEN_LONG'
                    trade_record['fees'] = fee
                    trade_record['fee_type'] = fee_type
                    trade_record['fee_rate'] = fee_rate
                    self.session_trades.append(trade_record)
                    
                    logger.info(f"[TRADE] OPENED LONG: {decision['size']:.6f} (${decision.get('usd_size', 0.1):.2f}) @ ${decision['price']:.2f} (confidence: {decision['confidence']:.1%})")
                    
                elif self.current_position['side'] == 'LONG':
                    # Already have a long position - could add to it or replace it
                    logger.info(f"[TRADE] Already LONG - ignoring BUY signal (current: {self.current_position['size']} @ ${self.current_position['price']:.2f})")
                    
                elif self.current_position['side'] == 'SHORT':
                    # Close short position and flip to long
                    entry_price = self.current_position['price']
                    exit_price = decision['price']
                    size = self.current_position['size']
                    entry_time = self.current_position['timestamp']
                    
                    # Calculate PnL for closing short with leverage
                    leveraged_pnl, leveraged_fee = self._calculate_leveraged_pnl_and_fees(
                        entry_price, exit_price, size, 'SHORT', fee_rate
                    )
                    net_pnl = leveraged_pnl - leveraged_fee - self.current_position['fees']
                    
                    self.total_realized_pnl += net_pnl
                    self.total_fees += leveraged_fee
                    
                    # Record the close trade
                    close_record = decision.copy()
                    close_record['position_action'] = 'CLOSE_SHORT'
                    close_record['entry_price'] = entry_price
                    close_record['pnl'] = net_pnl
                    close_record['fees'] = leveraged_fee
                    close_record['fee_type'] = fee_type
                    close_record['fee_rate'] = fee_rate
                    self.session_trades.append(close_record)
                    
                    # Add to closed trades accounting list
                    closed_trade = {
                        'trade_id': len(self.closed_trades) + 1,
                        'side': 'SHORT',
                        'entry_time': entry_time,
                        'exit_time': current_time,
                        'entry_price': entry_price,
                        'exit_price': exit_price,
                        'size': size,
                        'gross_pnl': leveraged_pnl,
                        'fees': leveraged_fee + self.current_position['fees'],
                        'fee_type': fee_type,
                        'fee_rate': fee_rate,
                        'net_pnl': net_pnl,
                        'duration': current_time - entry_time,
                        'symbol': decision.get('symbol', 'ETH/USDT'),
                        'mexc_executed': decision.get('mexc_executed', False)
                    }
                    self.closed_trades.append(closed_trade)
                    
                    # Save to file for persistence
                    self._save_closed_trades_to_file()
                    
                    # Trigger RL training on this closed trade
                    self._trigger_rl_training_on_closed_trade(closed_trade)
                    
                    # Record outcome for adaptive threshold learning
                    if 'confidence' in decision and 'threshold_used' in decision:
                        self.adaptive_learner.record_trade_outcome(
                            confidence=decision['confidence'],
                            pnl=net_pnl,
                            threshold_used=decision['threshold_used']
                        )
                        logger.debug(f"[ADAPTIVE] Recorded SHORT close outcome: PnL=${net_pnl:.2f}")
                    
                    logger.info(f"[TRADE] CLOSED SHORT: {size} @ ${exit_price:.2f} | PnL: ${net_pnl:.2f} | OPENING LONG")
                    
                    # Clear position before opening new one
                    self.current_position = None
                    
            elif decision['action'] == 'SELL':
                # First, close any existing LONG position
                if self.current_position and self.current_position['side'] == 'LONG':
                    # Close long position
                    entry_price = self.current_position['price']
                    exit_price = decision['price']
                    size = self.current_position['size']
                    entry_time = self.current_position['timestamp']
                    
                    # Calculate PnL for closing long with leverage
                    leveraged_pnl, leveraged_fee = self._calculate_leveraged_pnl_and_fees(
                        entry_price, exit_price, size, 'LONG', fee_rate
                    )
                    net_pnl = leveraged_pnl - leveraged_fee - self.current_position['fees']
                    
                    self.total_realized_pnl += net_pnl
                    self.total_fees += leveraged_fee
                    
                    # Record the close trade
                    close_record = decision.copy()
                    close_record['position_action'] = 'CLOSE_LONG'
                    close_record['entry_price'] = entry_price
                    close_record['pnl'] = net_pnl
                    close_record['fees'] = leveraged_fee
                    close_record['fee_type'] = fee_type
                    close_record['fee_rate'] = fee_rate
                    close_record['size'] = size  # Use original position size for close
                    self.session_trades.append(close_record)
                    
                    # Add to closed trades accounting list
                    closed_trade = {
                        'trade_id': len(self.closed_trades) + 1,
                        'side': 'LONG',
                        'entry_time': entry_time,
                        'exit_time': current_time,
                        'entry_price': entry_price,
                        'exit_price': exit_price,
                        'size': size,
                        'leverage': self.leverage_multiplier,  # Store leverage used
                        'gross_pnl': leveraged_pnl,
                        'fees': leveraged_fee + self.current_position['fees'],
                        'fee_type': fee_type,
                        'fee_rate': fee_rate,
                        'net_pnl': net_pnl,
                        'duration': current_time - entry_time,
                        'symbol': decision.get('symbol', 'ETH/USDT'),
                        'mexc_executed': decision.get('mexc_executed', False)
                    }
                    self.closed_trades.append(closed_trade)
                    
                    # Save to file for persistence
                    self._save_closed_trades_to_file()
                    
                    logger.info(f"[TRADE] CLOSED LONG: {size} @ ${exit_price:.2f} | PnL: ${net_pnl:.2f} | OPENING SHORT")
                    
                    # Clear position before opening new one
                    self.current_position = None
                
                # Now open short position (regardless of previous position)
                if self.current_position is None:
                    # Open short position with confidence-based size
                    fee = decision['price'] * decision['size'] * fee_rate  # ✅ FIXED: No leverage on fees
                    self.current_position = {
                        'side': 'SHORT',
                        'price': decision['price'],
                        'size': decision['size'],
                        'timestamp': current_time,
                        'fees': fee
                    }
                    self.total_fees += fee
                    
                    trade_record = decision.copy()
                    trade_record['position_action'] = 'OPEN_SHORT'
                    trade_record['fees'] = fee
                    trade_record['fee_type'] = fee_type
                    trade_record['fee_rate'] = fee_rate
                    self.session_trades.append(trade_record)
                    
                    logger.info(f"[TRADE] OPENED SHORT: {decision['size']:.6f} (${decision.get('usd_size', 0.1):.2f}) @ ${decision['price']:.2f} (confidence: {decision['confidence']:.1%})")
                    
                elif self.current_position['side'] == 'SHORT':
                    # Already have a short position - could add to it or replace it
                    logger.info(f"[TRADE] Already SHORT - ignoring SELL signal (current: {self.current_position['size']} @ ${self.current_position['price']:.2f})")
            
            # Add to recent decisions
            self.recent_decisions.append(decision)
            if len(self.recent_decisions) > 500:  # Keep last 500 decisions (increased from 50) to cover chart timeframe
                self.recent_decisions = self.recent_decisions[-500:]
                
        except Exception as e:
            logger.error(f"Error processing trading decision: {e}")

    def _calculate_leveraged_pnl_and_fees(self, entry_price: float, exit_price: float, size: float, side: str, fee_rate: float):
        """Calculate leveraged PnL and fees for closed positions"""
        try:
            # Calculate base PnL
            if side == 'LONG':
                base_pnl = (exit_price - entry_price) * size
            elif side == 'SHORT':
                base_pnl = (entry_price - exit_price) * size
            else:
                return 0.0, 0.0
            
            # Apply leverage amplification ONLY to P&L
            leveraged_pnl = base_pnl * self.leverage_multiplier
            
            # Calculate fees WITHOUT leverage (normal position value)
            position_value = exit_price * size  # ✅ FIXED: No leverage multiplier
            normal_fee = position_value * fee_rate  # ✅ FIXED: Normal fees
            
            logger.info(f"[LEVERAGE] {side} PnL: Base=${base_pnl:.2f} x {self.leverage_multiplier}x = ${leveraged_pnl:.2f}, Fee=${normal_fee:.4f}")
            
            return leveraged_pnl, normal_fee  # ✅ FIXED: Return normal fee
            
        except Exception as e:
            logger.warning(f"Error calculating leveraged PnL and fees: {e}")
            return 0.0, 0.0

    def _calculate_unrealized_pnl(self, current_price: float) -> float:
        """Calculate unrealized PnL for open position with leverage amplification"""
        try:
            if not self.current_position:
                return 0.0
            
            entry_price = self.current_position['price']
            size = self.current_position['size']
            
            # Calculate base PnL
            if self.current_position['side'] == 'LONG':
                base_pnl = (current_price - entry_price) * size
            elif self.current_position['side'] == 'SHORT':
                base_pnl = (entry_price - current_price) * size
            else:
                return 0.0
            
            # Apply leverage amplification
            leveraged_pnl = base_pnl * self.leverage_multiplier
            
            logger.debug(f"[LEVERAGE PnL] Base: ${base_pnl:.2f} x {self.leverage_multiplier}x = ${leveraged_pnl:.2f}")
            
            return leveraged_pnl
            
        except Exception as e:
            logger.warning(f"Error calculating unrealized PnL: {e}")
            return 0.0
    
    def run(self, host: str = '127.0.0.1', port: int = 8050, debug: bool = False):
        """Run the dashboard server"""
        try:
            logger.info("="*60)
            logger.info("STARTING TRADING DASHBOARD")
            logger.info(f"ACCESS WEB UI AT: http://{host}:{port}/")
            logger.info("Real-time trading data and charts")
            logger.info("AI model performance monitoring")
            logger.info("Memory usage tracking")
            logger.info("="*60)
            
            # Start the orchestrator's real trading loop in background
            logger.info("Starting orchestrator trading loop in background...")
            self._start_orchestrator_trading()
            
            # Give the orchestrator a moment to start
            import time
            time.sleep(2)
            
            logger.info(f"Starting Dash server on http://{host}:{port}")
            
            # Run the app (updated API for newer Dash versions)
            self.app.run(
                host=host,
                port=port,
                debug=debug,
                use_reloader=False,  # Disable reloader to avoid conflicts
                threaded=True       # Enable threading for better performance
            )
            
        except Exception as e:
            logger.error(f"Error running dashboard: {e}")
            raise

    def _start_orchestrator_trading(self):
        """Start the orchestrator's continuous trading in a background thread"""
        def orchestrator_loop():
            """Run the orchestrator trading loop"""
            try:
                logger.info("[ORCHESTRATOR] Starting trading loop...")
                
                # Simple trading loop without async complexity
                import time
                symbols = self.config.symbols if self.config.symbols else ['ETH/USDT']
                
                while True:
                    try:
                        # Make trading decisions for each symbol every 30 seconds
                        for symbol in symbols:
                            try:
                                # Get current price
                                current_data = self.data_provider.get_historical_data(symbol, '1m', limit=1, refresh=True)
                                if current_data is not None and not current_data.empty:
                                    current_price = float(current_data['close'].iloc[-1])
                                    
                                    # Simple decision making
                                    decision = {
                                        'action': 'HOLD',  # Conservative default
                                        'symbol': symbol,
                                        'price': current_price,
                                        'confidence': 0.5,
                                        'timestamp': datetime.now(),
                                        'size': 0.1,
                                        'reason': f"Orchestrator monitoring {symbol}"
                                    }
                                    
                                    # Process the decision (adds to dashboard display)
                                    self._process_trading_decision(decision)
                                    
                                    logger.debug(f"[ORCHESTRATOR] {decision['action']} {symbol} @ ${current_price:.2f}")
                                
                            except Exception as e:
                                logger.warning(f"[ORCHESTRATOR] Error processing {symbol}: {e}")
                        
                        # Wait before next cycle
                        time.sleep(30)
                        
                    except Exception as e:
                        logger.error(f"[ORCHESTRATOR] Error in trading cycle: {e}")
                        time.sleep(60)  # Wait longer on error
                
            except Exception as e:
                logger.error(f"Error in orchestrator trading loop: {e}")
        
        # Start orchestrator in background thread
        orchestrator_thread = Thread(target=orchestrator_loop, daemon=True)
        orchestrator_thread.start()
        logger.info("[ORCHESTRATOR] Trading loop started in background")

    def _create_closed_trades_table(self) -> List:
        """Create simplified closed trades history table focusing on total fees per closed position"""
        try:
            if not self.closed_trades:
                return [html.P("No closed trades yet", className="text-muted text-center")]
            
            # Create table rows for recent closed trades (newest first)
            table_rows = []
            recent_trades = self.closed_trades[-20:]  # Get last 20 trades
            recent_trades.reverse()  # Newest first
            
            for trade in recent_trades:
                # Determine row color based on P&L
                row_class = "table-success" if trade['net_pnl'] >= 0 else "table-danger"
                
                # Format duration
                duration_str = str(trade['duration']).split('.')[0]  # Remove microseconds
                
                # Format side color
                side_color = "text-success" if trade['side'] == 'LONG' else "text-danger"
                
                # Calculate leveraged position size in USD
                position_size = trade.get('size', 0)
                entry_price = trade.get('entry_price', 0)
                leverage_used = trade.get('leverage', self.leverage_multiplier)  # Use trade's leverage or current
                
                # Base position value in USD
                base_position_usd = position_size * entry_price
                # Leveraged position value (this is what we're actually exposed to)
                leveraged_position_usd = base_position_usd * leverage_used
                
                # Display format: show both base crypto amount and leveraged USD value
                size_display = f"{position_size:.4f} ETH (${leveraged_position_usd:,.0f}@{leverage_used:.0f}x)"
                
                # Leverage-adjusted fees display
                total_fees = trade.get('fees', 0)
                # Note: Fees should already be calculated correctly with leverage in the P&L calculation
                
                table_rows.append(
                    html.Tr([
                        html.Td(f"#{trade['trade_id']}", className="small"),
                        html.Td(trade['side'], className=f"small fw-bold {side_color}"),
                        html.Td(size_display, className="small text-info"),
                        html.Td(f"${trade['entry_price']:.2f}", className="small"),
                        html.Td(f"${trade['exit_price']:.2f}", className="small"),
                        html.Td(f"${total_fees:.3f}", className="small text-warning"),
                        html.Td(f"${trade['net_pnl']:.2f}", className="small fw-bold"),
                        html.Td(duration_str, className="small"),
                        html.Td("✓" if trade.get('mexc_executed', False) else "SIM", 
                               className="small text-success" if trade.get('mexc_executed', False) else "small text-warning")
                    ], className=row_class)
                )
            
            # Create simple table
            table = html.Table([
                html.Thead([
                    html.Tr([
                        html.Th("ID", className="small"),
                        html.Th("Side", className="small"),
                        html.Th("Position Size", className="small"),
                        html.Th("Entry", className="small"),
                        html.Th("Exit", className="small"),
                        html.Th("Total Fees", className="small"),
                        html.Th("Net P&L", className="small"),
                        html.Th("Duration", className="small"),
                        html.Th("MEXC", className="small")
                    ])
                ]),
                html.Tbody(table_rows)
            ], className="table table-sm table-striped")
            
            return [table]
            
        except Exception as e:
            logger.error(f"Error creating closed trades table: {e}")
            return [html.P(f"Error: {str(e)}", className="text-danger")]

    def _save_closed_trades_to_file(self):
        """Save closed trades to JSON file for persistence"""
        try:
            import json
            from datetime import datetime
            
            # Convert datetime objects to strings for JSON serialization
            trades_for_json = []
            for trade in self.closed_trades:
                trade_copy = trade.copy()
                if isinstance(trade_copy.get('entry_time'), datetime):
                    trade_copy['entry_time'] = trade_copy['entry_time'].isoformat()
                if isinstance(trade_copy.get('exit_time'), datetime):
                    trade_copy['exit_time'] = trade_copy['exit_time'].isoformat()
                if isinstance(trade_copy.get('duration'), timedelta):
                    trade_copy['duration'] = str(trade_copy['duration'])
                trades_for_json.append(trade_copy)
            
            with open('closed_trades_history.json', 'w') as f:
                json.dump(trades_for_json, f, indent=2)
            
            logger.info(f"Saved {len(self.closed_trades)} closed trades to file")
            
        except Exception as e:
            logger.error(f"Error saving closed trades: {e}")

    def _load_closed_trades_from_file(self):
        """Load closed trades from JSON file"""
        try:
            import json
            from pathlib import Path
            
            if Path('closed_trades_history.json').exists():
                with open('closed_trades_history.json', 'r') as f:
                    trades_data = json.load(f)
                
                # Convert string dates back to datetime objects
                for trade in trades_data:
                    if isinstance(trade.get('entry_time'), str):
                        trade['entry_time'] = datetime.fromisoformat(trade['entry_time'])
                    if isinstance(trade.get('exit_time'), str):
                        trade['exit_time'] = datetime.fromisoformat(trade['exit_time'])
                    if isinstance(trade.get('duration'), str):
                        # Parse duration string back to timedelta
                        duration_parts = trade['duration'].split(':')
                        if len(duration_parts) >= 3:
                            hours = int(duration_parts[0])
                            minutes = int(duration_parts[1])
                            seconds = float(duration_parts[2])
                            trade['duration'] = timedelta(hours=hours, minutes=minutes, seconds=seconds)
                
                self.closed_trades = trades_data
                logger.info(f"Loaded {len(self.closed_trades)} closed trades from file")
                
        except Exception as e:
            logger.error(f"Error loading closed trades: {e}")
            self.closed_trades = []

    def clear_closed_trades_history(self):
        """Clear closed trades history, reset session P&L, and remove file"""
        try:
            # Clear trades data
            self.closed_trades = []
            self.session_trades = []
            
            # Reset session P&L totals
            self.total_realized_pnl = 0.0
            self.total_fees = 0.0
            self.session_pnl = 0.0
            self.realized_pnl = 0.0
            self.unrealized_pnl = 0.0
            
            # Reset session tracking
            self.session_start = datetime.now()
            
            # Reset position if exists  
            if self.current_position:
                logger.info(f"Clearing current position: {self.current_position}")
                self.current_position = None
                
            # Reset adaptive learning stats (optional - keeps learning but resets performance)
            # self.adaptive_learner = AdaptiveThresholdLearner(initial_threshold=0.30)
            
            # Reset any other session-related metrics if they exist
            if hasattr(self, 'session_start_balance'):
                self.session_start_balance = self.starting_balance
            
            # Remove file if it exists
            from pathlib import Path
            if Path('closed_trades_history.json').exists():
                Path('closed_trades_history.json').unlink()
            
            logger.info("Cleared closed trades history and reset all session P&L totals")
            
        except Exception as e:
            logger.error(f"Error clearing closed trades history: {e}")

    def _create_session_performance(self) -> List:
        """Create enhanced session performance display with multiline format and total volume"""
        try:
            # Calculate comprehensive session metrics from closed trades
            total_trades = len(self.closed_trades)
            winning_trades = len([t for t in self.closed_trades if t['net_pnl'] > 0])
            total_net_pnl = sum(t['net_pnl'] for t in self.closed_trades)
            total_fees_paid = sum(t.get('fees', 0) for t in self.closed_trades)
            
            # Calculate total volume (price * size for each trade)
            total_volume = 0
            for trade in self.closed_trades:
                entry_volume = trade.get('entry_price', 0) * trade.get('size', 0)
                exit_volume = trade.get('exit_price', 0) * trade.get('size', 0)
                total_volume += entry_volume + exit_volume  # Both entry and exit contribute to volume
            
            # Calculate fee breakdown
            maker_fees = sum(t.get('fees', 0) for t in self.closed_trades if t.get('fee_type') == 'maker')
            taker_fees = sum(t.get('fees', 0) for t in self.closed_trades if t.get('fee_type') != 'maker')
            
            # Calculate gross P&L (before fees)
            gross_pnl = total_net_pnl + total_fees_paid
            
            # Calculate rates and percentages
            win_rate = (winning_trades / total_trades * 100) if total_trades > 0 else 0
            avg_trade_pnl = (total_net_pnl / total_trades) if total_trades > 0 else 0
            fee_impact = (total_fees_paid / gross_pnl * 100) if gross_pnl > 0 else 0
            fee_percentage_of_volume = (total_fees_paid / total_volume * 100) if total_volume > 0 else 0
            
            # Calculate signal stats from recent decisions
            total_signals = len([d for d in self.recent_decisions if d.get('signal')])
            executed_signals = len([d for d in self.recent_decisions if d.get('signal') and d.get('executed')])
            signal_efficiency = (executed_signals / total_signals * 100) if total_signals > 0 else 0
            
            # Create enhanced multiline performance display
            metrics = [
                # Line 1: Basic trade statistics
                html.Div([
                    html.Small([
                        html.Strong(f"Total: {total_trades} trades | "),
                        html.Span(f"Win Rate: {win_rate:.1f}% | ", className="text-info"),
                        html.Span(f"Avg P&L: ${avg_trade_pnl:.2f}", 
                                 className="text-success" if avg_trade_pnl >= 0 else "text-danger")
                    ])
                ], className="mb-1"),
                
                # Line 2: P&L breakdown (Gross vs Net)
                html.Div([
                    html.Small([
                        html.Strong("P&L: "),
                        html.Span(f"Gross: ${gross_pnl:.2f} | ", 
                                 className="text-success" if gross_pnl >= 0 else "text-danger"),
                        html.Span(f"Net: ${total_net_pnl:.2f} | ", 
                                 className="text-success" if total_net_pnl >= 0 else "text-danger"),
                        html.Span(f"Fee Impact: {fee_impact:.1f}%", className="text-warning")
                    ])
                ], className="mb-1"),
                
                # Line 3: Fee breakdown with volume for validation
                html.Div([
                    html.Small([
                        html.Strong("Fees: "),
                        html.Span(f"Total: ${total_fees_paid:.3f} | ", className="text-warning"),
                        html.Span(f"Maker: ${maker_fees:.3f} (0.00%) | ", className="text-success"),
                        html.Span(f"Taker: ${taker_fees:.3f} (0.05%)", className="text-danger")
                    ])
                ], className="mb-1"),
                
                # Line 4: Volume and fee percentage for validation
                html.Div([
                    html.Small([
                        html.Strong("Volume: "),
                        html.Span(f"${total_volume:,.0f} | ", className="text-muted"),
                        html.Strong("Fee %: "),
                        html.Span(f"{fee_percentage_of_volume:.4f}% | ", className="text-warning"),
                        html.Strong("Signals: "),
                        html.Span(f"{executed_signals}/{total_signals} ({signal_efficiency:.1f}%)", className="text-info")
                    ])
                ], className="mb-2")
            ]
            
            return metrics
            
        except Exception as e:
            logger.error(f"Error creating session performance: {e}")
            return [html.Div([
                html.Strong("Session Performance", className="text-primary"),
                html.Br(),
                html.Small(f"Error loading metrics: {str(e)}", className="text-danger")
            ])]

    def _force_demo_signal(self, symbol: str, current_price: float) -> None:
        """DISABLED - No demo signals, only real market data"""
        logger.debug("Demo signals disabled - waiting for real market data only")
        pass

    def _load_available_models(self):
        """Load available models with enhanced model management"""
        try:
            from model_manager import ModelManager, ModelMetrics
            
            # Initialize model manager
            self.model_manager = ModelManager()
            
            # Load best models
            loaded_models = self.model_manager.load_best_models()
            
            if loaded_models:
                logger.info(f"Loaded {len(loaded_models)} best models via ModelManager")
                
                # Update internal model storage
                for model_type, model_data in loaded_models.items():
                    model_info = model_data['info']
                    logger.info(f"Using best {model_type} model: {model_info.model_name} (Score: {model_info.metrics.get_composite_score():.3f})")
                
            else:
                logger.info("No managed models available, falling back to legacy loading")
                # Fallback to original model loading logic
                self._load_legacy_models()
                
        except ImportError:
            logger.warning("ModelManager not available, using legacy model loading")
            self._load_legacy_models()
        except Exception as e:
            logger.error(f"Error loading models via ModelManager: {e}")
            self._load_legacy_models()
    
    def _load_legacy_models(self):
        """Legacy model loading method (original implementation)"""
        self.available_models = {
            'cnn': [],
            'rl': [],
            'hybrid': []
        }
        
        try:
            # Check for CNN models
            cnn_models_dir = "models/cnn"
            if os.path.exists(cnn_models_dir):
                for model_file in os.listdir(cnn_models_dir):
                    if model_file.endswith('.pt'):
                        model_path = os.path.join(cnn_models_dir, model_file)
                        try:
                            # Try to load model to verify it's valid
                            model_data = torch.load(model_path, map_location='cpu')
                            
                            # Handle both direct model objects and state_dict
                            if isinstance(model_data, dict):
                                logger.warning(f"CNN model {model_file} is a state_dict (not a model object), skipping")
                                continue  # Skip dict models for now
                            
                            model = model_data
                            
                            class CNNWrapper:
                                def __init__(self, model):
                                    self.model = model
                                    self.model.eval()
                                    
                                def predict(self, feature_matrix):
                                    with torch.no_grad():
                                        if hasattr(feature_matrix, 'shape') and len(feature_matrix.shape) == 2:
                                            feature_tensor = torch.FloatTensor(feature_matrix).unsqueeze(0)
                                        else:
                                            feature_tensor = torch.FloatTensor(feature_matrix)
                                        
                                        prediction = self.model(feature_tensor)
                                        
                                        if hasattr(prediction, 'cpu'):
                                            prediction = prediction.cpu().numpy()
                                        elif isinstance(prediction, torch.Tensor):
                                            prediction = prediction.detach().numpy()
                                        
                                        # Ensure we return probabilities
                                        if len(prediction.shape) > 1:
                                            prediction = prediction[0]
                                        
                                        # Apply softmax if needed
                                        if len(prediction) == 3:
                                            exp_pred = np.exp(prediction - np.max(prediction))
                                            prediction = exp_pred / np.sum(exp_pred)
                                        
                                        return prediction
                                        
                                def get_memory_usage(self):
                                    return 50  # MB estimate
                                    
                                def to_device(self, device):
                                    self.model = self.model.to(device)
                                    return self
                            
                            wrapper = CNNWrapper(model)
                            self.available_models['cnn'].append({
                                'name': model_file,
                                'path': model_path,
                                'model': wrapper,
                                'type': 'cnn'
                            })
                            logger.info(f"Loaded CNN model: {model_file}")
                            
                        except Exception as e:
                            logger.warning(f"Failed to load CNN model {model_file}: {e}")
            
            # Check for RL models
            rl_models_dir = "models/rl"
            if os.path.exists(rl_models_dir):
                for model_file in os.listdir(rl_models_dir):
                    if model_file.endswith('.pt'):
                        try:
                            checkpoint_path = os.path.join(rl_models_dir, model_file)
                            
                            class RLWrapper:
                                def __init__(self, checkpoint_path):
                                    self.checkpoint_path = checkpoint_path
                                    self.checkpoint = torch.load(checkpoint_path, map_location='cpu')
                                
                                def predict(self, feature_matrix):
                                    # Mock RL prediction
                                    if hasattr(feature_matrix, 'shape'):
                                        state_sum = np.sum(feature_matrix) % 100
                                    else:
                                        state_sum = np.sum(np.array(feature_matrix)) % 100
                                    
                                    if state_sum > 70:
                                        action_probs = [0.1, 0.1, 0.8]  # BUY
                                    elif state_sum < 30:
                                        action_probs = [0.8, 0.1, 0.1]  # SELL
                                    else:
                                        action_probs = [0.2, 0.6, 0.2]  # HOLD
                                    
                                    return np.array(action_probs)
                                    
                                def get_memory_usage(self):
                                    return 75  # MB estimate
                                    
                                def to_device(self, device):
                                    return self
                        
                            wrapper = RLWrapper(checkpoint_path)
                            self.available_models['rl'].append({
                                'name': model_file,
                                'path': checkpoint_path,
                                'model': wrapper,
                                'type': 'rl'
                            })
                            logger.info(f"Loaded RL model: {model_file}")
                            
                        except Exception as e:
                            logger.warning(f"Failed to load RL model {model_file}: {e}")
            
            total_models = sum(len(models) for models in self.available_models.values())
            logger.info(f"Legacy model loading complete. Total models: {total_models}")
                
        except Exception as e:
            logger.error(f"Error in legacy model loading: {e}")
            # Initialize empty model structure
            self.available_models = {'cnn': [], 'rl': [], 'hybrid': []}
    
    def register_model_performance(self, model_type: str, profit_factor: float, 
                                 win_rate: float, sharpe_ratio: float = 0.0, 
                                 accuracy: float = 0.0):
        """Register model performance with the model manager"""
        try:
            if hasattr(self, 'model_manager'):
                # Find the current best model of this type
                best_model = self.model_manager.get_best_model(model_type)
                
                if best_model:
                    # Create metrics from performance data
                    from model_manager import ModelMetrics
                    
                    metrics = ModelMetrics(
                        accuracy=accuracy,
                        profit_factor=profit_factor,
                        win_rate=win_rate,
                        sharpe_ratio=sharpe_ratio,
                        max_drawdown=0.0,  # Will be calculated from trade history
                        total_trades=len(self.closed_trades),
                        confidence_score=0.7  # Default confidence
                    )
                    
                    # Update model performance
                    self.model_manager.update_model_performance(best_model.model_name, metrics)
                    logger.info(f"Updated {model_type} model performance: PF={profit_factor:.2f}, WR={win_rate:.2f}")
                    
        except Exception as e:
            logger.error(f"Error registering model performance: {e}")

    def _create_system_status_compact(self, memory_stats: Dict) -> Dict:
        """Create system status display in compact format"""
        try:
            status_items = []
            
            # Memory usage
            memory_pct = memory_stats.get('utilization_percent', 0)
            memory_class = "text-success" if memory_pct < 70 else "text-warning" if memory_pct < 90 else "text-danger"
            
            status_items.append(
                html.Div([
                    html.I(className="fas fa-memory me-2"),
                    html.Span("Memory: "),
                    html.Strong(f"{memory_pct:.1f}%", className=memory_class),
                    html.Small(f" ({memory_stats.get('total_used_mb', 0):.0f}MB / {memory_stats.get('total_limit_mb', 0):.0f}MB)", className="text-muted")
                ], className="mb-2")
            )
            
            # Model status
            models_count = len(memory_stats.get('models', {}))
            status_items.append(
                html.Div([
                    html.I(className="fas fa-brain me-2"),
                    html.Span("Models: "),
                    html.Strong(f"{models_count} active", className="text-info")
                ], className="mb-2")
            )
            
            # WebSocket streaming status
            streaming_status = "LIVE" if self.is_streaming else "OFFLINE"
            streaming_class = "text-success" if self.is_streaming else "text-danger"
            
            status_items.append(
                html.Div([
                    html.I(className="fas fa-wifi me-2"),
                    html.Span("Stream: "),
                    html.Strong(streaming_status, className=streaming_class)
                ], className="mb-2")
            )
            
            # Tick cache status
            cache_size = len(self.tick_cache)
            cache_minutes = cache_size / 3600 if cache_size > 0 else 0  # Assuming 60 ticks per second
            status_items.append(
                html.Div([
                    html.I(className="fas fa-database me-2"),
                    html.Span("Cache: "),
                    html.Strong(f"{cache_minutes:.1f}m", className="text-info"),
                    html.Small(f" ({cache_size} ticks)", className="text-muted")
                ], className="mb-2")
            )
            
            return {
                'icon_class': "fas fa-circle text-success fa-2x" if self.is_streaming else "fas fa-circle text-warning fa-2x",
                'title': f"System Status: {'Streaming live data' if self.is_streaming else 'Using cached data'}",
                'details': status_items
            }
            
        except Exception as e:
            logger.error(f"Error creating system status: {e}")
            return {
                'icon_class': "fas fa-circle text-danger fa-2x",
                'title': "System Error: Check logs",
                'details': [html.P(f"Error: {str(e)}", className="text-danger")]
            }

    def _start_lightweight_websocket(self):
        """Start ultra-lightweight WebSocket for real-time price updates only"""
        try:
            if self.is_streaming:
                logger.warning("[WS] WebSocket already running")
                return
            
            # ETH/USDT primary symbol for scalping
            symbol = "ethusdt"
            
            def ws_worker():
                try:
                    import websocket
                    import json
                    
                    def on_message(ws, message):
                        try:
                            data = json.loads(message)
                            # Extract only current price - ultra minimal processing
                            if 'c' in data:  # Current price from ticker
                                price = float(data['c'])
                                # Update price cache (no history, just current)
                                self.ws_price_cache['ETHUSDT'] = price
                                self.current_prices['ETHUSDT'] = price
                                
                                # Performance tracking
                                current_time = time.time()
                                self.last_ws_update = current_time
                                self.ws_update_count += 1
                                
                                # Log every 100 updates for monitoring
                                if self.ws_update_count % 100 == 0:
                                    logger.debug(f"[WS] {self.ws_update_count} price updates, latest: ${price:.2f}")
                        except Exception as e:
                            logger.warning(f"[WS] Error processing message: {e}")
                    
                    def on_error(ws, error):
                        logger.error(f"[WS] Error: {error}")
                        self.is_streaming = False
                    
                    def on_close(ws, close_status_code, close_msg):
                        logger.warning(f"[WS] Connection closed: {close_status_code}")
                        self.is_streaming = False
                        # Auto-reconnect after 5 seconds
                        time.sleep(5)
                        if not self.is_streaming:
                            self._start_lightweight_websocket()
                    
                    def on_open(ws):
                        logger.info(f"[WS] Connected for real-time ETHUSDT price updates")
                        self.is_streaming = True
                    
                    # Binance WebSocket for ticker (price only, not trades)
                    ws_url = f"wss://stream.binance.com:9443/ws/{symbol}@ticker"
                    
                    self.ws_connection = websocket.WebSocketApp(
                        ws_url,
                        on_message=on_message,
                        on_error=on_error,
                        on_close=on_close,
                        on_open=on_open
                    )
                    
                    # Run WebSocket (blocking)
                    self.ws_connection.run_forever()
                    
                except Exception as e:
                    logger.error(f"[WS] Worker error: {e}")
                    self.is_streaming = False
            
            # Start WebSocket in background thread
            self.ws_thread = threading.Thread(target=ws_worker, daemon=True)
            self.ws_thread.start()
            
            logger.info("[WS] Lightweight WebSocket started for real-time price updates")
            
        except Exception as e:
            logger.error(f"[WS] Failed to start: {e}")
            self.is_streaming = False
    
    def stop_streaming(self):
        """Stop WebSocket streaming"""
        try:
            self.is_streaming = False
            if self.ws_connection:
                self.ws_connection.close()
            logger.info("[WS] Streaming stopped")
        except Exception as e:
            logger.error(f"[WS] Error stopping: {e}")
    
    def get_realtime_price(self, symbol: str) -> float:
        """Get real-time price from WebSocket cache (faster than API)"""
        try:
            # Try WebSocket cache first (sub-second latency)
            ws_price = self.ws_price_cache.get(symbol.replace('/', ''))
            if ws_price:
                return ws_price
            
            # Fallback to current_prices (from data provider)
            return self.current_prices.get(symbol.replace('/', ''))
        except Exception as e:
            logger.warning(f"[WS] Error getting realtime price: {e}")
            return None

    def _create_cnn_monitoring_content(self) -> List:
        """Create CNN monitoring and prediction analysis content"""
        try:
            # Get CNN monitoring data
            if CNN_MONITORING_AVAILABLE:
                cnn_data = get_cnn_dashboard_data()
            else:
                cnn_data = {'statistics': {'total_predictions_logged': 0}}
                
            components = []
            
            # CNN Statistics Overview
            stats = cnn_data.get('statistics', {})
            components.append(html.Div([
                html.H6([
                    html.I(className="fas fa-chart-bar me-2"),
                    "CNN Performance Overview"
                ], className="mb-2"),
                html.Div([
                    html.Div([
                        html.Strong(f"{stats.get('total_predictions_logged', 0):,}"),
                        html.Br(),
                        html.Small("Total Predictions", className="text-muted")
                    ], className="text-center", style={"flex": "1"}),
                    html.Div([
                        html.Strong(f"{stats.get('avg_prediction_latency_ms', 0):.1f}ms"),
                        html.Br(),
                        html.Small("Avg Latency", className="text-muted")
                    ], className="text-center", style={"flex": "1"}),
                    html.Div([
                        html.Strong(f"{stats.get('avg_confidence', 0)*100:.1f}%"),
                        html.Br(),
                        html.Small("Avg Confidence", className="text-muted")
                    ], className="text-center", style={"flex": "1"}),
                    html.Div([
                        html.Strong(f"{len(stats.get('active_models', []))}"),
                        html.Br(),
                        html.Small("Active Models", className="text-muted")
                    ], className="text-center", style={"flex": "1"})
                ], style={"display": "flex", "gap": "10px", "marginBottom": "15px"})
            ]))
            
            # Recent Predictions Table
            recent_predictions = cnn_data.get('recent_predictions', [])
            if recent_predictions:
                components.append(html.Div([
                    html.H6([
                        html.I(className="fas fa-list-alt me-2"),
                        "Recent CNN Predictions"
                    ], className="mb-2"),
                    self._create_cnn_predictions_table(recent_predictions[-10:])  # Last 10 predictions
                ]))
            else:
                components.append(html.Div([
                    html.H6("Recent Predictions", className="mb-2"),
                    html.P("No recent predictions available", className="text-muted")
                ]))
            
            # Model Performance Comparison
            model_stats = cnn_data.get('model_performance', {})
            if model_stats:
                components.append(html.Div([
                    html.H6([
                        html.I(className="fas fa-trophy me-2"),
                        "Model Performance Comparison"
                    ], className="mb-2"),
                    self._create_model_performance_table(model_stats)
                ]))
            
            return components
            
        except Exception as e:
            logger.error(f"Error creating CNN monitoring content: {e}")
            return [html.P(f"Error loading CNN monitoring: {str(e)}", className="text-danger")]
    
    def _create_cnn_predictions_table(self, predictions: List[Dict]) -> html.Table:
        """Create table showing recent CNN predictions"""
        try:
            if not predictions:
                return html.P("No predictions available", className="text-muted")
            
            # Table headers
            headers = ["Time", "Model", "Symbol", "Action", "Confidence", "Latency", "Price Context"]
            
            # Create rows
            rows = []
            for pred in reversed(predictions):  # Most recent first
                try:
                    timestamp = pred.get('timestamp', '')
                    if isinstance(timestamp, str):
                        # Format timestamp for display
                        from datetime import datetime
                        dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
                        time_str = dt.strftime('%H:%M:%S')
                    else:
                        time_str = str(timestamp)[-8:]  # Last 8 chars for time
                    
                    model_name = pred.get('model_name', 'Unknown')[:12]  # Truncate long names
                    symbol = pred.get('symbol', '')
                    action_name = pred.get('action_name', 'HOLD')
                    confidence = pred.get('confidence', 0) * 100
                    latency = pred.get('prediction_latency_ms', 0)
                    current_price = pred.get('current_price', 0)
                    
                    # Action styling
                    if action_name == 'BUY':
                        action_badge = html.Span(action_name, className="badge bg-success text-white")
                    elif action_name == 'SELL':
                        action_badge = html.Span(action_name, className="badge bg-danger text-white")
                    else:
                        action_badge = html.Span(action_name, className="badge bg-secondary")
                    
                    # Confidence styling
                    if confidence > 70:
                        conf_class = "text-success fw-bold"
                    elif confidence > 50:
                        conf_class = "text-warning"
                    else:
                        conf_class = "text-muted"
                    
                    row = html.Tr([
                        html.Td(time_str, className="small"),
                        html.Td(model_name, className="small"),
                        html.Td(symbol, className="small"),
                        html.Td(action_badge),
                        html.Td(f"{confidence:.1f}%", className=f"small {conf_class}"),
                        html.Td(f"{latency:.1f}ms", className="small text-muted"),
                        html.Td(f"${current_price:.2f}" if current_price else "N/A", className="small")
                    ])
                    rows.append(row)
                except Exception as e:
                    logger.warning(f"Error processing prediction row: {e}")
                    continue
            
            return html.Table([
                html.Thead([
                    html.Tr([html.Th(h, className="small") for h in headers])
                ]),
                html.Tbody(rows)
            ], className="table table-sm table-striped")
            
        except Exception as e:
            logger.error(f"Error creating CNN predictions table: {e}")
            return html.P(f"Error creating predictions table: {str(e)}", className="text-danger")
    
    def _create_model_performance_table(self, model_stats: Dict) -> html.Table:
        """Create table showing model performance metrics"""
        try:
            if not model_stats:
                return html.P("No model performance data available", className="text-muted")
            
            headers = ["Model", "Predictions", "Avg Confidence", "Avg Latency", "Memory Usage"]
            rows = []
            
            for model_name, stats in model_stats.items():
                prediction_count = stats.get('prediction_count', 0)
                avg_confidence = stats.get('avg_confidence', 0) * 100
                avg_latency = stats.get('avg_latency_ms', 0)
                memory_usage = stats.get('avg_memory_usage_mb', 0)
                
                row = html.Tr([
                    html.Td(model_name[:15], className="small"),  # Truncate long names
                    html.Td(f"{prediction_count:,}", className="small"),
                    html.Td(f"{avg_confidence:.1f}%", className="small"),
                    html.Td(f"{avg_latency:.1f}ms", className="small"),
                    html.Td(f"{memory_usage:.0f}MB" if memory_usage else "N/A", className="small")
                ])
                rows.append(row)
            
            return html.Table([
                html.Thead([
                    html.Tr([html.Th(h, className="small") for h in headers])
                ]),
                html.Tbody(rows)
            ], className="table table-sm table-striped")
            
        except Exception as e:
            logger.error(f"Error creating model performance table: {e}")
            return html.P(f"Error creating performance table: {str(e)}", className="text-danger")

    def _cleanup_old_data(self):
        """Clean up old data to prevent memory leaks and performance degradation"""
        try:
            cleanup_start = time.time()
            
            # Clean up recent decisions - keep only last 100 
            if len(self.recent_decisions) > 100:
                self.recent_decisions = self.recent_decisions[-100:]
                
            # Clean up recent signals - keep only last 50
            if len(self.recent_signals) > 50:
                self.recent_signals = self.recent_signals[-50:]
                
            # Clean up session trades - keep only last 200
            if len(self.session_trades) > 200:
                self.session_trades = self.session_trades[-200:]
                
            # Clean up closed trades - keep only last 100 in memory, rest in file
            if len(self.closed_trades) > 100:
                self.closed_trades = self.closed_trades[-100:]
                
            # Clean up current prices - remove old symbols not in config
            current_symbols = set(self.config.symbols) if self.config.symbols else {'ETHUSDT'}
            symbols_to_remove = []
            for symbol in self.current_prices:
                if symbol not in current_symbols:
                    symbols_to_remove.append(symbol)
            for symbol in symbols_to_remove:
                del self.current_prices[symbol]
                
            # Clean up RL training queue - keep only last 500
            if len(self.rl_training_queue) > 500:
                # Convert to list, slice, then back to deque
                old_queue = list(self.rl_training_queue)
                self.rl_training_queue.clear()
                self.rl_training_queue.extend(old_queue[-500:])
                
            # Tick infrastructure removed - no cleanup needed
                
            cleanup_time = (time.time() - cleanup_start) * 1000
            logger.info(f"[CLEANUP] Data cleanup completed in {cleanup_time:.1f}ms - "
                       f"Decisions: {len(self.recent_decisions)}, "
                       f"Signals: {len(self.recent_signals)}, "
                       f"Trades: {len(self.session_trades)}, "
                       f"Closed: {len(self.closed_trades)}")
                       
        except Exception as e:
            logger.error(f"Error during data cleanup: {e}")

    def _create_training_metrics(self) -> List:
        """Create comprehensive model training metrics display with enhanced RL integration"""
        try:
            training_items = []
            
            # Enhanced Training Data Streaming Status
            ws_updates = getattr(self, 'ws_update_count', 0)
            enhanced_data_available = self.training_data_available and self.enhanced_rl_training_enabled
            
            training_items.append(
                html.Div([
                    html.H6([
                        html.I(className="fas fa-database me-2 text-info"),
                        "Real-Time Data & Training Stream"
                    ], className="mb-2"),
                    html.Div([
                        html.Small([
                            html.Strong("WebSocket Updates: "),
                            html.Span(f"{ws_updates:,} price updates", className="text-success" if ws_updates > 100 else "text-warning")
                        ], className="d-block"),
                        html.Small([
                            html.Strong("Stream Status: "),
                            html.Span("LIVE" if self.is_streaming else "OFFLINE", 
                                     className="text-success" if self.is_streaming else "text-danger")
                        ], className="d-block"),
                        html.Small([
                            html.Strong("Enhanced RL: "),
                            html.Span("ENABLED" if self.enhanced_rl_training_enabled else "DISABLED", 
                                     className="text-success" if self.enhanced_rl_training_enabled else "text-warning")
                        ], className="d-block"),
                        html.Small([
                            html.Strong("Training Data: "),
                            html.Span("AVAILABLE" if enhanced_data_available else "WAITING", 
                                     className="text-success" if enhanced_data_available else "text-warning")
                        ], className="d-block"),
                        html.Small([
                            html.Strong("Cached Data: "),
                            html.Span("READY" if len(self.current_prices) > 0 else "LOADING", 
                                     className="text-success" if len(self.current_prices) > 0 else "text-warning")
                        ], className="d-block")
                    ])
                ], className="mb-3 p-2 border border-info rounded")
            )
            
            # Enhanced RL Training Statistics
            if self.enhanced_rl_training_enabled:
                enhanced_episodes = self.rl_training_stats.get('enhanced_rl_episodes', 0)
                comprehensive_packets = self.rl_training_stats.get('comprehensive_data_packets', 0)
                
                training_items.append(
                    html.Div([
                        html.H6([
                            html.I(className="fas fa-brain me-2 text-success"),
                            "Enhanced RL Training"
                        ], className="mb-2"),
                        html.Div([
                            html.Small([
                                html.Strong("Status: "),
                                html.Span("ACTIVE" if enhanced_episodes > 0 else "WAITING", 
                                         className="text-success" if enhanced_episodes > 0 else "text-warning")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Episodes: "),
                                html.Span(f"{enhanced_episodes}", className="text-info")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Data Packets: "),
                                html.Span(f"{comprehensive_packets}", className="text-info")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Features: "),
                                html.Span("~13,400 (Market State)", className="text-success")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Training Mode: "),
                                html.Span("Comprehensive", className="text-success")
                            ], className="d-block")
                        ])
                    ], className="mb-3 p-2 border border-success rounded")
                )
            
            # Model Training Status
            try:
                # Try to get real training metrics from orchestrator
                training_status = self._get_model_training_status()
                
                # CNN Training Metrics
                training_items.append(
                    html.Div([
                        html.H6([
                            html.I(className="fas fa-brain me-2 text-warning"),
                            "CNN Model (Extrema Detection)"
                        ], className="mb-2"),
                        html.Div([
                            html.Small([
                                html.Strong("Status: "),
                                html.Span(training_status['cnn']['status'], 
                                         className=f"text-{training_status['cnn']['status_color']}")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Accuracy: "),
                                html.Span(f"{training_status['cnn']['accuracy']:.1%}", className="text-info")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Loss: "),
                                html.Span(f"{training_status['cnn']['loss']:.4f}", className="text-muted")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Perfect Moves: "),
                                html.Span("Available" if hasattr(self.orchestrator, 'extrema_trainer') else "N/A", 
                                         className="text-success" if hasattr(self.orchestrator, 'extrema_trainer') else "text-muted")
                            ], className="d-block")
                        ])
                    ], className="mb-3 p-2 border border-warning rounded")
                )
                
                # RL Training Metrics (Enhanced)
                total_episodes = self.rl_training_stats.get('total_training_episodes', 0)
                profitable_trades = self.rl_training_stats.get('profitable_trades_trained', 0)
                win_rate = (profitable_trades / total_episodes * 100) if total_episodes > 0 else 0
                
                training_items.append(
                    html.Div([
                        html.H6([
                            html.I(className="fas fa-robot me-2 text-primary"),
                            "RL Agent (DQN + Sensitivity Learning)"
                        ], className="mb-2"),
                        html.Div([
                            html.Small([
                                html.Strong("Status: "),
                                html.Span("ENHANCED" if self.enhanced_rl_training_enabled else "BASIC", 
                                         className="text-success" if self.enhanced_rl_training_enabled else "text-warning")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Win Rate: "),
                                html.Span(f"{win_rate:.1f}%", className="text-info")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Total Episodes: "),
                                html.Span(f"{total_episodes}", className="text-muted")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Enhanced Episodes: "),
                                html.Span(f"{enhanced_episodes}" if self.enhanced_rl_training_enabled else "N/A", 
                                         className="text-success" if self.enhanced_rl_training_enabled else "text-muted")
                            ], className="d-block"),
                            html.Small([
                                html.Strong("Sensitivity Learning: "),
                                html.Span("ACTIVE" if hasattr(self.orchestrator, 'sensitivity_learning_queue') else "N/A", 
                                         className="text-success" if hasattr(self.orchestrator, 'sensitivity_learning_queue') else "text-muted")
                            ], className="d-block")
                        ])
                    ], className="mb-3 p-2 border border-primary rounded")
                )
                
                # Training Progress Chart (Mini)
                training_items.append(
                    html.Div([
                        html.H6([
                            html.I(className="fas fa-chart-line me-2 text-secondary"),
                            "Training Progress"
                        ], className="mb-2"),
                        dcc.Graph(
                            figure=self._create_mini_training_chart(training_status),
                            style={"height": "150px"},
                            config={'displayModeBar': False}
                        )
                    ], className="mb-3 p-2 border border-secondary rounded")
                )
                
            except Exception as e:
                logger.warning(f"Error getting training status: {e}")
                training_items.append(
                    html.Div([
                        html.P("Training status unavailable", className="text-muted"),
                        html.Small(f"Error: {str(e)}", className="text-danger")
                    ], className="mb-3 p-2 border border-secondary rounded")
                )
            
            # Adaptive Threshold Learning Statistics
            try:
                adaptive_stats = self.adaptive_learner.get_learning_stats()
                if adaptive_stats and 'error' not in adaptive_stats:
                    current_threshold = adaptive_stats.get('current_threshold', 0.3)
                    base_threshold = adaptive_stats.get('base_threshold', 0.3)
                    total_trades = adaptive_stats.get('total_trades', 0)
                    recent_win_rate = adaptive_stats.get('recent_win_rate', 0)
                    recent_avg_pnl = adaptive_stats.get('recent_avg_pnl', 0)
                    learning_active = adaptive_stats.get('learning_active', False)
                    
                    training_items.append(
                        html.Div([
                            html.H6([
                                html.I(className="fas fa-graduation-cap me-2 text-warning"),
                                "Adaptive Threshold Learning"
                            ], className="mb-2"),
                            html.Div([
                                html.Small([
                                    html.Strong("Current Threshold: "),
                                    html.Span(f"{current_threshold:.1%}", className="text-warning fw-bold")
                                ], className="d-block"),
                                html.Small([
                                    html.Strong("Base Threshold: "),
                                    html.Span(f"{base_threshold:.1%}", className="text-muted")
                                ], className="d-block"),
                                html.Small([
                                    html.Strong("Learning Status: "),
                                    html.Span("ACTIVE" if learning_active else "COLLECTING DATA", 
                                             className="text-success" if learning_active else "text-info")
                                ], className="d-block"),
                                html.Small([
                                    html.Strong("Trades Analyzed: "),
                                    html.Span(f"{total_trades}", className="text-info")
                                ], className="d-block"),
                                html.Small([
                                    html.Strong("Recent Win Rate: "),
                                    html.Span(f"{recent_win_rate:.1%}", 
                                             className="text-success" if recent_win_rate > 0.5 else "text-danger")
                                ], className="d-block"),
                                html.Small([
                                    html.Strong("Recent Avg P&L: "),
                                    html.Span(f"${recent_avg_pnl:.2f}", 
                                             className="text-success" if recent_avg_pnl > 0 else "text-danger")
                                ], className="d-block")
                            ])
                        ], className="mb-3 p-2 border border-warning rounded")
                    )
            except Exception as e:
                logger.warning(f"Error calculating adaptive threshold: {e}")
                training_items.append(
                    html.Div([
                        html.P("Adaptive threshold learning error", className="text-danger"),
                        html.Small(f"Error: {str(e)}", className="text-muted")
                    ], className="mb-3 p-2 border border-danger rounded")
                )
            
            # Real-time Training Events Log
            training_items.append(
                html.Div([
                    html.H6([
                        html.I(className="fas fa-list me-2 text-secondary"),
                        "Recent Training Events"
                    ], className="mb-2"),
                    html.Div(
                        id="training-events-log",
                        children=self._get_recent_training_events(),
                        style={"maxHeight": "120px", "overflowY": "auto", "fontSize": "0.8em"}
                    )
                ], className="mb-3 p-2 border border-secondary rounded")
            )
            
            return training_items
            
        except Exception as e:
            logger.error(f"Error creating training metrics: {e}")
            return [html.P(f"Training metrics error: {str(e)}", className="text-danger")]

    def _get_model_training_status(self) -> Dict:
        """Get current model training status and metrics"""
        try:
            # Initialize default status
            status = {
                'cnn': {
                    'status': 'IDLE',
                    'status_color': 'secondary',
                    'accuracy': 0.0,
                    'loss': 0.0,
                    'epochs': 0,
                    'learning_rate': 0.001
                },
                'rl': {
                    'status': 'IDLE',
                    'status_color': 'secondary',
                    'win_rate': 0.0,
                    'avg_reward': 0.0,
                    'episodes': 0,
                    'epsilon': 1.0,
                    'memory_size': 0
                }
            }
            
            # Try to get real metrics from orchestrator
            if hasattr(self.orchestrator, 'get_training_metrics'):
                try:
                    real_metrics = self.orchestrator.get_training_metrics()
                    if real_metrics:
                        status.update(real_metrics)
                        logger.debug("Using real training metrics from orchestrator")
                except Exception as e:
                    logger.warning(f"Error getting orchestrator metrics: {e}")
            
            # Try to get metrics from model registry
            if hasattr(self.model_registry, 'get_training_stats'):
                try:
                    registry_stats = self.model_registry.get_training_stats()
                    if registry_stats:
                        # Update with registry stats
                        for model_type in ['cnn', 'rl']:
                            if model_type in registry_stats:
                                status[model_type].update(registry_stats[model_type])
                        logger.debug("Updated with model registry stats")
                except Exception as e:
                    logger.warning(f"Error getting registry stats: {e}")
            
            # Try to read from training logs
            try:
                log_metrics = self._parse_training_logs()
                if log_metrics:
                    for model_type in ['cnn', 'rl']:
                        if model_type in log_metrics:
                            status[model_type].update(log_metrics[model_type])
                    logger.debug("Updated with training log metrics")
            except Exception as e:
                logger.warning(f"Error parsing training logs: {e}")
            
            # Check if models are actively training based on tick data flow
            if self.is_streaming and len(self.tick_cache) > 100:
                # Models should be training if we have data
                status['cnn']['status'] = 'TRAINING'
                status['cnn']['status_color'] = 'warning'
                status['rl']['status'] = 'TRAINING'
                status['rl']['status_color'] = 'success'
            
            # Add our real-time RL training statistics
            if hasattr(self, 'rl_training_stats') and self.rl_training_stats:
                rl_stats = self.rl_training_stats
                total_episodes = rl_stats.get('total_training_episodes', 0)
                profitable_trades = rl_stats.get('profitable_trades_trained', 0)
                
                # Calculate win rate from our training data
                if total_episodes > 0:
                    win_rate = profitable_trades / total_episodes
                    status['rl']['win_rate'] = win_rate
                    status['rl']['episodes'] = total_episodes
                    
                    # Update status based on training activity
                    if rl_stats.get('last_training_time'):
                        last_training = rl_stats['last_training_time']
                        time_since_training = (datetime.now() - last_training).total_seconds()
                        
                        if time_since_training < 300:  # Last 5 minutes
                            status['rl']['status'] = 'REALTIME_TRAINING'
                            status['rl']['status_color'] = 'success'
                        elif time_since_training < 3600:  # Last hour
                            status['rl']['status'] = 'ACTIVE'
                            status['rl']['status_color'] = 'info'
                        else:
                            status['rl']['status'] = 'IDLE'
                            status['rl']['status_color'] = 'warning'
                    
                    # Average reward from recent training
                    if rl_stats.get('training_rewards'):
                        avg_reward = sum(rl_stats['training_rewards']) / len(rl_stats['training_rewards'])
                        status['rl']['avg_reward'] = avg_reward
                    
                    logger.debug(f"Updated RL status with real-time stats: {total_episodes} episodes, {win_rate:.1%} win rate")
            
            return status
            
        except Exception as e:
            logger.error(f"Error getting model training status: {e}")
            return {
                'cnn': {'status': 'ERROR', 'status_color': 'danger', 'accuracy': 0.0, 'loss': 0.0, 'epochs': 0, 'learning_rate': 0.001},
                'rl': {'status': 'ERROR', 'status_color': 'danger', 'win_rate': 0.0, 'avg_reward': 0.0, 'episodes': 0, 'epsilon': 1.0, 'memory_size': 0}
            }

    def _parse_training_logs(self) -> Dict:
        """Parse recent training logs for metrics"""
        try:
            from pathlib import Path
            import re
            
            metrics = {'cnn': {}, 'rl': {}}
            
            # Parse CNN training logs
            cnn_log_paths = [
                'logs/cnn_training.log',
                'logs/training.log',
                'runs/*/events.out.tfevents.*'  # TensorBoard logs
            ]
            
            for log_path in cnn_log_paths:
                if Path(log_path).exists():
                    try:
                        with open(log_path, 'r') as f:
                            lines = f.readlines()[-50:]  # Last 50 lines
                        
                        for line in lines:
                            # Look for CNN metrics
                            if 'epoch' in line.lower() and 'loss' in line.lower():
                                # Extract epoch, loss, accuracy
                                epoch_match = re.search(r'epoch[:\s]+(\d+)', line, re.IGNORECASE)
                                loss_match = re.search(r'loss[:\s]+([\d\.]+)', line, re.IGNORECASE)
                                acc_match = re.search(r'acc[uracy]*[:\s]+([\d\.]+)', line, re.IGNORECASE)
                                
                                if epoch_match:
                                    metrics['cnn']['epochs'] = int(epoch_match.group(1))
                                if loss_match:
                                    metrics['cnn']['loss'] = float(loss_match.group(1))
                                if acc_match:
                                    acc_val = float(acc_match.group(1))
                                    # Normalize accuracy (handle both 0-1 and 0-100 formats)
                                    metrics['cnn']['accuracy'] = acc_val if acc_val <= 1.0 else acc_val / 100.0
                        
                        break  # Use first available log
                    except Exception as e:
                        logger.debug(f"Error parsing {log_path}: {e}")
            
            # Parse RL training logs
            rl_log_paths = [
                'logs/rl_training.log',
                'logs/training.log'
            ]
            
            for log_path in rl_log_paths:
                if Path(log_path).exists():
                    try:
                        with open(log_path, 'r') as f:
                            lines = f.readlines()[-50:]  # Last 50 lines
                rows=2, cols=1,
                shared_xaxes=True,
                vertical_spacing=0.1,
                subplot_titles=(f'{symbol} Price ({actual_timeframe.upper()}) with Williams Pivot Points', 'Volume'),
                row_heights=[0.7, 0.3]
            )
            
            # Add price line chart (main chart)
            fig.add_trace(
                go.Scatter(
                    x=df.index,
                    y=df['close'],
                    mode='lines',
                    name=f"{symbol} Price",
                    line=dict(color='#00ff88', width=2),
                    hovertemplate='<b>$%{y:.2f}</b><br>%{x}<extra></extra>'
                ),
                row=1, col=1
            )
            
            # Add Williams Market Structure pivot points
            try:
                pivot_points = self._get_williams_pivot_points_for_chart(df)
                if pivot_points:
                    self._add_williams_pivot_points_to_chart(fig, pivot_points, row=1)
                else:
                    logger.debug("[CHART] No Williams pivot points available")
            except Exception as e:
                logger.debug(f"Error adding Williams pivot points to chart: {e}")
            
            # Add CNN pivot predictions as hollow circles
            try:
                cnn_predictions = self._get_cnn_pivot_predictions(symbol, df)
                if cnn_predictions:
                    self._add_cnn_predictions_to_chart(fig, cnn_predictions, row=1)
                    logger.debug(f"[CHART] Added {len(cnn_predictions)} CNN predictions to chart")
                else:
                    logger.debug("[CHART] No CNN predictions available")
            except Exception as e:
                logger.debug(f"Error adding CNN predictions to chart: {e}")
            
            # Add moving averages if we have enough data
            if len(df) >= 20:
                # 20-period SMA (create a copy to avoid modifying original data)
                sma_20 = df['close'].rolling(window=20).mean()
                fig.add_trace(
                    go.Scatter(
                        x=df.index,
                        y=sma_20,
                        name='SMA 20',
                        line=dict(color='#ff1493', width=1),
                        opacity=0.8,
                        hovertemplate='<b>SMA20: $%{y:.2f}</b><br>%{x}<extra></extra>'
                    ),
                    row=1, col=1
                )
            
            # Removed SMA 50 since we only have 30 bars maximum
            
            # Add volume bars
            if 'volume' in df.columns:
                fig.add_trace(
                    go.Bar(
                        x=df.index,
                        y=df['volume'],
                        name='Volume',
                        marker_color='rgba(158, 158, 158, 0.6)',
                        hovertemplate='<b>Volume: %{y:.0f}</b><br>%{x}<extra></extra>'
                    ),
                    row=2, col=1
                                )

            # Mark recent trading decisions with proper markers - OPTIMIZED
            try:
                # Filter decisions to only those within the chart timeframe
                buy_decisions = []
                sell_decisions = []
                
                for decision in self.recent_decisions[-100:]:  # Limit to last 100 decisions
                    if isinstance(decision, dict) and 'timestamp' in decision and 'price' in decision and 'action' in decision:
                        decision_time = decision['timestamp']
                        
                        # Convert decision timestamp to match chart timezone if needed
                        if isinstance(decision_time, datetime):
                            if decision_time.tzinfo is not None:
                                decision_time_utc = decision_time.astimezone(timezone.utc).replace(tzinfo=None)
                            else:
                                decision_time_utc = decision_time
                        else:
                            continue
                        
                        # Convert chart times to UTC for comparison
                        try:
                            if isinstance(chart_start_time, pd.Timestamp):
                                chart_start_utc = chart_start_time.tz_localize(None) if chart_start_time.tz is None else chart_start_time.tz_convert('UTC').tz_localize(None)
                                chart_end_utc = chart_end_time.tz_localize(None) if chart_end_time.tz is None else chart_end_time.tz_convert('UTC').tz_localize(None)
                            else:
                                chart_start_utc = pd.to_datetime(chart_start_time).tz_localize(None)
                                chart_end_utc = pd.to_datetime(chart_end_time).tz_localize(None)
                            
                            # Check if decision falls within chart timeframe
                            decision_time_pd = pd.to_datetime(decision_time_utc)
                            if chart_start_utc <= decision_time_pd <= chart_end_utc:
                                pass  # Continue processing
                            else:
                                continue  # Skip this decision
                        except Exception as e:
                            logger.debug(f"Error comparing decision timestamp: {e}")
                            continue  # Skip this decision
                        
                        signal_type = decision.get('signal_type', 'UNKNOWN')
                        if decision['action'] == 'BUY':
                            buy_decisions.append((decision, signal_type))
                        elif decision['action'] == 'SELL':
                            sell_decisions.append((decision, signal_type))
                
                logger.debug(f"[CHART] Showing {len(buy_decisions)} BUY and {len(sell_decisions)} SELL signals in chart timeframe")
                
                # Add BUY markers with different styles for executed vs ignored
                executed_buys = [d[0] for d in buy_decisions if d[1] == 'EXECUTED']
                ignored_buys = [d[0] for d in buy_decisions if d[1] in ['NOT_EXECUTED_POSITION_LIMIT', 'NOT_EXECUTED_LOW_CONFIDENCE']]
                
                if executed_buys:
                    fig.add_trace(
                        go.Scatter(
                            x=[self._to_local_timezone(d['timestamp']) for d in executed_buys],
                            y=[d['price'] for d in executed_buys],
                            mode='markers',
                            marker=dict(
                                color='#00ff88',
                                size=14,
                                symbol='triangle-up',
                                line=dict(color='white', width=2)
                            ),
                            name="BUY (Executed)",
                            showlegend=True,
                            hovertemplate="<b>BUY EXECUTED</b><br>Price: $%{y:.2f}<br>Time: %{x}<br>Confidence: %{customdata:.1%}<extra></extra>",
                            customdata=[d.get('confidence', 0) for d in executed_buys]
                        ),
                        row=1, col=1
                    )
                
                if ignored_buys:
                    fig.add_trace(
                        go.Scatter(
                            x=[self._to_local_timezone(d['timestamp']) for d in ignored_buys],
                            y=[d['price'] for d in ignored_buys],
                            mode='markers',
                            marker=dict(
                                color='#00ff88',
                                size=10,
                                symbol='triangle-up-open',
                                line=dict(color='#00ff88', width=2)
                            ),
                            name="BUY (Blocked)",
                            showlegend=True,
                            hovertemplate="<b>BUY BLOCKED</b><br>Price: $%{y:.2f}<br>Time: %{x}<br>Confidence: %{customdata:.1%}<extra></extra>",
                            customdata=[d.get('confidence', 0) for d in ignored_buys]
                        ),
                        row=1, col=1
                    )
                
                # Add SELL markers with different styles for executed vs ignored
                executed_sells = [d[0] for d in sell_decisions if d[1] == 'EXECUTED']
                ignored_sells = [d[0] for d in sell_decisions if d[1] in ['NOT_EXECUTED_POSITION_LIMIT', 'NOT_EXECUTED_LOW_CONFIDENCE']]
                
                if executed_sells:
                    fig.add_trace(
                        go.Scatter(
                            x=[self._to_local_timezone(d['timestamp']) for d in executed_sells],
                            y=[d['price'] for d in executed_sells],
                            mode='markers',
                            marker=dict(
                                color='#ff6b6b',
                                size=14,
                                symbol='triangle-down',
                                line=dict(color='white', width=2)
                            ),
                            name="SELL (Executed)",
                            showlegend=True,
                            hovertemplate="<b>SELL EXECUTED</b><br>Price: $%{y:.2f}<br>Time: %{x}<br>Confidence: %{customdata:.1%}<extra></extra>",
                            customdata=[d.get('confidence', 0) for d in executed_sells]
                        ),
                        row=1, col=1
                    )
                
                if ignored_sells:
                    fig.add_trace(
                        go.Scatter(
                            x=[self._to_local_timezone(d['timestamp']) for d in ignored_sells],
                            y=[d['price'] for d in ignored_sells],
                            mode='markers',
                            marker=dict(
                                color='#ff6b6b',
                                size=10,
                                symbol='triangle-down-open',
                                line=dict(color='#ff6b6b', width=2)
                            ),
                            name="SELL (Blocked)",
                            showlegend=True,
                            hovertemplate="<b>SELL BLOCKED</b><br>Price: $%{y:.2f}<br>Time: %{x}<br>Confidence: %{customdata:.1%}<extra></extra>",
                            customdata=[d.get('confidence', 0) for d in ignored_sells]
                        ),
                        row=1, col=1
                    )
            except Exception as e:
                logger.debug(f"Error adding trading decision markers to chart: {e}")
            
            # Add closed trades markers with profit/loss styling and connecting lines
            try:
                if self.closed_trades and not df.empty:
                    # Convert chart times to UTC for comparison
                    if isinstance(chart_start_time, pd.Timestamp):
                        chart_start_utc = chart_start_time.tz_localize(None) if chart_start_time.tz is None else chart_start_time.tz_convert('UTC').tz_localize(None)
                        chart_end_utc = chart_end_time.tz_localize(None) if chart_end_time.tz is None else chart_end_time.tz_convert('UTC').tz_localize(None)
                    else:
                        chart_start_utc = pd.to_datetime(chart_start_time).tz_localize(None)
                        chart_end_utc = pd.to_datetime(chart_end_time).tz_localize(None)
                    
                    # Filter closed trades to only those within chart timeframe
                    chart_trades = []
                    for trade in self.closed_trades:
                        if not isinstance(trade, dict):
                            continue
                        
                        entry_time = trade.get('entry_time')
                        exit_time = trade.get('exit_time')
                        
                        if not entry_time or not exit_time:
                            continue
                        
                        # Convert times to UTC for comparison
                        try:
                            if isinstance(entry_time, datetime):
                                entry_time_utc = entry_time.astimezone(timezone.utc).replace(tzinfo=None) if entry_time.tzinfo else entry_time
                            else:
                                continue
                                
                            if isinstance(exit_time, datetime):
                                exit_time_utc = exit_time.astimezone(timezone.utc).replace(tzinfo=None) if exit_time.tzinfo else exit_time
                            else:
                                continue
                            
                            # Check if trade overlaps with chart timeframe
                            entry_time_pd = pd.to_datetime(entry_time_utc)
                            exit_time_pd = pd.to_datetime(exit_time_utc)
                            
                            if (chart_start_utc <= entry_time_pd <= chart_end_utc) or (chart_start_utc <= exit_time_pd <= chart_end_utc):
                                chart_trades.append(trade)
                        except Exception as e:
                            logger.debug(f"Error comparing trade timestamps: {e}")
                            continue  # Skip this trade
                    
                    logger.debug(f"[CHART] Showing {len(chart_trades)} closed trades on chart")
                    
                    # Plot closed trades with profit/loss styling
                    profitable_entries_x = []
                    profitable_entries_y = []
                    profitable_exits_x = []
                    profitable_exits_y = []
                    
                    # Collect trade points for display
                    for trade in chart_trades:
                        entry_price = trade.get('entry_price', 0)
                        exit_price = trade.get('exit_price', 0)
                        entry_time = trade.get('entry_time')
                        exit_time = trade.get('exit_time')
                        net_pnl = trade.get('net_pnl', 0)
                        
                        if not all([entry_price, exit_price, entry_time, exit_time]):
                            continue
                        
                        # Convert times to local timezone for display
                        entry_time_local = self._to_local_timezone(entry_time)
                        exit_time_local = self._to_local_timezone(exit_time)
                        
                        # Determine if trade was profitable
                        is_profitable = net_pnl > 0
                        
                        if is_profitable:
                            profitable_entries_x.append(entry_time_local)
                            profitable_entries_y.append(entry_price)
                            profitable_exits_x.append(exit_time_local)
                            profitable_exits_y.append(exit_price)
                        
                        # Add connecting dash line between entry and exit
                        line_color = '#00ff88' if is_profitable else '#ff6b6b'
                        fig.add_trace(
                            go.Scatter(
                                x=[entry_time_local, exit_time_local],
                                y=[entry_price, exit_price],
                                mode='lines',
                                line=dict(
                                    color=line_color,
                                    width=2,
                                    dash='dash'
                                ),
                                name="Trade Path",
                                showlegend=False,
                                hoverinfo='skip'
                            ),
                            row=1, col=1
                        )
                    
                    # Add profitable trade markers (filled triangles)
                    if profitable_entries_x:
                        # Entry markers
                        fig.add_trace(
                            go.Scatter(
                                x=profitable_entries_x,
                                y=profitable_entries_y,
                                mode='markers',
                                marker=dict(
                                    color='#00ff88',
                                    size=12,
                                    symbol='triangle-up',
                                    line=dict(color='white', width=1)
                                ),
                                name="Profitable Entry",
                                showlegend=True,
                                hovertemplate="<b>PROFITABLE ENTRY</b><br>Price: $%{y:.2f}<br>Time: %{x}<extra></extra>"
                            ),
                            row=1, col=1
                        )
                    
                    if profitable_exits_x:
                        # Exit markers
                        fig.add_trace(
                            go.Scatter(
                                x=profitable_exits_x,
                                y=profitable_exits_y,
                                mode='markers',
                                marker=dict(
                                    color='#00ff88',
                                    size=12,
                                    symbol='triangle-down',
                                    line=dict(color='white', width=1)
                                ),
                                name="Profitable Exit",
                                showlegend=True,
                                hovertemplate="<b>PROFITABLE EXIT</b><br>Price: $%{y:.2f}<br>Time: %{x}<extra></extra>"
                            ),
                            row=1, col=1
                        )
            except Exception as e:
                logger.debug(f"Error adding closed trades to chart: {e}")
                
            # Update layout with current timestamp and streaming status
            current_time = datetime.now().strftime("%H:%M:%S.%f")[:-3]
            try:
                latest_price = float(df['close'].iloc[-1]) if not df.empty else 0.0
            except (ValueError, TypeError, IndexError):
                latest_price = 0.0
            stream_status = "LIVE STREAM" if self.is_streaming else "CACHED DATA"
            tick_count = len(self.tick_cache)
            
            fig.update_layout(
                title=f"{symbol} {actual_timeframe.upper()} CHART | ${latest_price:.2f} | {stream_status} | {tick_count} ticks | {current_time}",
                template="plotly_dark",
                height=450,
                xaxis_rangeslider_visible=False,
                margin=dict(l=20, r=20, t=50, b=20),
                legend=dict(
                    orientation="h",
                    yanchor="bottom",
                    y=1.02,
                    xanchor="right",
                    x=1
                )
            )
            
            # Update y-axis labels
            fig.update_yaxes(title_text="Price ($)", row=1, col=1)
            fig.update_yaxes(title_text="Volume", row=2, col=1)
            fig.update_xaxes(title_text="Time", row=2, col=1)
            
            # Cache the chart for performance
            if not hasattr(self, '_chart_cache'):
                self._chart_cache = {}
            
            self._chart_cache[f"chart_{symbol}"] = (fig, current_time)
            
            # Clean old chart cache entries (keep last 3)
            if len(self._chart_cache) > 3:
                        
                        for line in lines:
                            # Look for RL metrics
                            if 'episode' in line.lower():
                                episode_match = re.search(r'episode[:\s]+(\d+)', line, re.IGNORECASE)
                                reward_match = re.search(r'reward[:\s]+([-\d\.]+)', line, re.IGNORECASE)
                                epsilon_match = re.search(r'epsilon[:\s]+([\d\.]+)', line, re.IGNORECASE)
                                
                                if episode_match:
                                    metrics['rl']['episodes'] = int(episode_match.group(1))
                                if reward_match:
                                    metrics['rl']['avg_reward'] = float(reward_match.group(1))
                                if epsilon_match:
                                    metrics['rl']['epsilon'] = float(epsilon_match.group(1))
                        
                        break  # Use first available log
                    except Exception as e:
                        logger.debug(f"Error parsing {log_path}: {e}")
            
            return metrics if any(metrics.values()) else None
            
        except Exception as e:
            logger.warning(f"Error parsing training logs: {e}")
            return None

    def _format_data_for_cnn(self, training_data: Dict[str, Any]) -> Dict[str, Any]:
        """Format training data for CNN models"""
        try:
            ohlcv = training_data['ohlcv']
            
            # Create feature matrix for CNN (sequence of OHLCV + indicators)
            features = ohlcv[['open', 'high', 'low', 'close', 'volume', 'sma_20', 'sma_50', 'rsi']].values
            
            # Normalize features
            from sklearn.preprocessing import MinMaxScaler
            scaler = MinMaxScaler()
            features_normalized = scaler.fit_transform(features)
            
            # Create sequences for CNN training (sliding window)
            sequence_length = 60  # 1 minute of 1-second data
            sequences = []
            targets = []
            
            for i in range(sequence_length, len(features_normalized)):
                sequences.append(features_normalized[i-sequence_length:i])
                # Target: price direction (1 for up, 0 for down)
                current_price = ohlcv.iloc[i]['close']
                future_price = ohlcv.iloc[min(i+5, len(ohlcv)-1)]['close']  # 5 seconds ahead
                targets.append(1 if future_price > current_price else 0)
            
            return {
                'sequences': np.array(sequences),
                'targets': np.array(targets),
                'feature_names': ['open', 'high', 'low', 'close', 'volume', 'sma_20', 'sma_50', 'rsi'],
                'sequence_length': sequence_length,
                'symbol': training_data['symbol'],
                'timestamp': training_data['timestamp']
            }
            
        except Exception as e:
            logger.error(f"Error formatting data for CNN: {e}")
            return {}

    def _format_data_for_rl(self, training_data: Dict[str, Any]) -> List[Tuple]:
        """Format training data for RL models (state, action, reward, next_state, done)"""
        try:
            ohlcv = training_data['ohlcv']
            experiences = []
            
            # Create state representations
            for i in range(10, len(ohlcv) - 1):  # Need history for state
                # Current state (last 10 bars)
                state_data = ohlcv.iloc[i-10:i][['close', 'volume', 'rsi']].values.flatten()
                
                # Next state
                next_state_data = ohlcv.iloc[i-9:i+1][['close', 'volume', 'rsi']].values.flatten()
                
                # Simulate action based on price movement
                current_price = ohlcv.iloc[i]['close']
                next_price = ohlcv.iloc[i+1]['close']
                price_change = (next_price - current_price) / current_price
                
                # Action: 0=HOLD, 1=BUY, 2=SELL
                if price_change > 0.001:  # 0.1% threshold
                    action = 1  # BUY
                    reward = price_change * 100  # Reward proportional to gain
                elif price_change < -0.001:
                    action = 2  # SELL
                    reward = -price_change * 100  # Reward for correct short
                else:
                    action = 0  # HOLD
                    reward = 0
                
                # Add experience tuple
                experiences.append((
                    state_data,      # state
                    action,          # action
                    reward,          # reward
                    next_state_data, # next_state
                    False            # done (not terminal)
                ))
            
            return experiences
            
        except Exception as e:
            logger.error(f"Error formatting data for RL: {e}")
            return []

    def _update_training_metrics(self, cnn_success: bool, rl_success: bool):
        """Update training metrics tracking"""
        try:
            current_time = datetime.now()
            
            # Update training statistics
            if not hasattr(self, 'training_stats'):
                self.training_stats = {
                    'last_training_time': current_time,
                    'total_training_sessions': 0,
                    'cnn_training_count': 0,
                    'rl_training_count': 0,
                    'training_data_points': 0
                }
            
            self.training_stats['last_training_time'] = current_time
            self.training_stats['total_training_sessions'] += 1
            
            if cnn_success:
                self.training_stats['cnn_training_count'] += 1
            if rl_success:
                self.training_stats['rl_training_count'] += 1
            
            self.training_stats['training_data_points'] = len(self.tick_cache)
            
            logger.debug(f"Training metrics updated: {self.training_stats}")
            
        except Exception as e:
            logger.warning(f"Error updating training metrics: {e}")

    def get_tick_cache_for_training(self) -> List[Dict]:
        """Get tick cache data for external training systems - removed for performance optimization"""
        logger.debug("Tick cache removed for performance - using cached OHLCV data for training instead")
        return []  # Empty since we removed tick infrastructure

    def start_continuous_training(self):
        """Start continuous training in background thread"""
        try:
            if hasattr(self, 'training_thread') and self.training_thread.is_alive():
                logger.info("Continuous training already running")
                return
            
            self.training_active = True
            self.training_thread = Thread(target=self._continuous_training_loop, daemon=True)
            self.training_thread.start()
            logger.info("Continuous training started")
            
        except Exception as e:
            logger.error(f"Error starting continuous training: {e}")

    def _continuous_training_loop(self):
        """Continuous training loop running in background - ONLY WITH REAL DATA"""
        logger.info("Continuous training loop started - will only train with real market data")
        
        while getattr(self, 'training_active', False):
            try:
                # Only train if we have sufficient REAL data
                if len(self.tick_cache) >= 500:  # Need sufficient real data
                    success = self.send_training_data_to_models()
                    if success:
                        logger.info("Training completed with real market data")
                    else:
                        logger.debug("Training skipped - waiting for more real data")
                else:
                    logger.debug(f"Waiting for real data - have {len(self.tick_cache)} ticks, need 500+")
                
                time.sleep(30)  # Check every 30 seconds
                
            except Exception as e:
                logger.error(f"Error in continuous training loop: {e}")
                time.sleep(60)  # Wait longer on error

    def stop_continuous_training(self):
        """Stop continuous training"""
        try:
            self.training_active = False
            if hasattr(self, 'training_thread'):
                self.training_thread.join(timeout=5)
            logger.info("Continuous training stopped")
        except Exception as e:
            logger.error(f"Error stopping continuous training: {e}")

    def _trigger_rl_training_on_closed_trade(self, closed_trade):
        """Trigger enhanced RL training based on a closed trade's profitability with comprehensive data"""
        try:
            if not self.rl_training_enabled:
                return
            
            # Extract trade information
            net_pnl = closed_trade.get('net_pnl', 0)
            is_profitable = net_pnl > 0
            trade_duration = closed_trade.get('duration', timedelta(0))
            
            # Create enhanced training episode data
            training_episode = {
                'trade_id': closed_trade.get('trade_id'),
                'side': closed_trade.get('side'),
                'entry_price': closed_trade.get('entry_price'),
                'exit_price': closed_trade.get('exit_price'),
                'net_pnl': net_pnl,
                'is_profitable': is_profitable,
                'duration_seconds': trade_duration.total_seconds(),
                'symbol': closed_trade.get('symbol', 'ETH/USDT'),
                'timestamp': closed_trade.get('exit_time', datetime.now()),
                'reward': self._calculate_rl_reward(closed_trade),
                'enhanced_data_available': self.enhanced_rl_training_enabled
            }
            
            # Add to training queue
            self.rl_training_queue.append(training_episode)
            
            # Update training statistics
            self.rl_training_stats['total_training_episodes'] += 1
            if is_profitable:
                self.rl_training_stats['profitable_trades_trained'] += 1
            else:
                self.rl_training_stats['unprofitable_trades_trained'] += 1
            
            self.rl_training_stats['last_training_time'] = datetime.now()
            self.rl_training_stats['training_rewards'].append(training_episode['reward'])
            
            # Enhanced RL training with comprehensive data
            if self.enhanced_rl_training_enabled:
                self._execute_enhanced_rl_training_step(training_episode)
            else:
                # Fallback to basic RL training
                self._execute_rl_training_step(training_episode)
            
            logger.info(f"[RL_TRAINING] Trade #{training_episode['trade_id']} added to {'ENHANCED' if self.enhanced_rl_training_enabled else 'BASIC'} training: "
                       f"{'PROFITABLE' if is_profitable else 'LOSS'} "
                       f"PnL: ${net_pnl:.2f}, Reward: {training_episode['reward']:.3f}")
            
        except Exception as e:
            logger.error(f"Error in RL training trigger: {e}")
    
    def _execute_enhanced_rl_training_step(self, training_episode):
        """Execute enhanced RL training step with comprehensive market data"""
        try:
            # Get comprehensive training data from unified stream
            training_data = self.unified_stream.get_latest_training_data() if ENHANCED_RL_AVAILABLE else None
            
            if training_data and hasattr(training_data, 'market_state') and training_data.market_state:
                # Enhanced RL training with ~13,400 features
                market_state = training_data.market_state
                universal_stream = training_data.universal_stream
                
                # Create comprehensive training context
                enhanced_context = {
                    'trade_outcome': training_episode,
                    'market_state': market_state,
                    'universal_stream': universal_stream,
                    'tick_cache': training_data.tick_cache if hasattr(training_data, 'tick_cache') else [],
                    'multi_timeframe_data': training_data.multi_timeframe_data if hasattr(training_data, 'multi_timeframe_data') else {},
                    'cnn_features': training_data.cnn_features if hasattr(training_data, 'cnn_features') else None,
                    'cnn_predictions': training_data.cnn_predictions if hasattr(training_data, 'cnn_predictions') else None
                }
                
                # Send to enhanced RL trainer
                if hasattr(self.orchestrator, 'enhanced_rl_trainer'):
                    try:
                        # Add trading experience with comprehensive context
                        symbol = training_episode['symbol']
                        action = TradingAction(
                            action=training_episode['side'],
                            symbol=symbol,
                            confidence=0.8,  # Inferred from executed trade
                            price=training_episode['exit_price'],
                            size=0.1,  # Default size
                            timestamp=training_episode['timestamp']
                        )
                        
                        # Create initial and final market states for RL learning
                        initial_state = market_state  # State at trade entry
                        final_state = market_state    # State at trade exit (simplified)
                        reward = training_episode['reward']
                        
                        # Add comprehensive trading experience
                        self.orchestrator.enhanced_rl_trainer.add_trading_experience(
                            symbol=symbol,
                            action=action,
                            initial_state=initial_state,
                            final_state=final_state,
                            reward=reward
                        )
                        
                        logger.info(f"[ENHANCED_RL] Added comprehensive trading experience for trade #{training_episode['trade_id']}")
                        logger.info(f"[ENHANCED_RL] Market state features: ~13,400, Reward: {reward:.3f}")
                        
                        # Update enhanced RL statistics
                        self.rl_training_stats['enhanced_rl_episodes'] += 1
                        
                        return True
                        
                    except Exception as e:
                        logger.error(f"Error in enhanced RL trainer: {e}")
                        return False
                
                # Send to extrema trainer for CNN learning
                if hasattr(self.orchestrator, 'extrema_trainer'):
                    try:
                        # Mark this trade outcome for CNN training
                        trade_context = {
                            'symbol': training_episode['symbol'],
                            'entry_price': training_episode['entry_price'],
                            'exit_price': training_episode['exit_price'],
                            'is_profitable': training_episode['is_profitable'],
                            'timestamp': training_episode['timestamp']
                        }
                        
                        # Add to extrema training if this was a good/bad move
                        if abs(training_episode['net_pnl']) > 0.5:  # Significant move
                            self.orchestrator.extrema_trainer.add_trade_outcome_for_learning(trade_context)
                            logger.debug(f"[EXTREMA_CNN] Added trade outcome for CNN learning")
                        
                    except Exception as e:
                        logger.warning(f"Error adding to extrema trainer: {e}")
                
                # Send to sensitivity learning DQN
                if hasattr(self.orchestrator, 'sensitivity_learning_queue'):
                    try:
                        sensitivity_data = {
                            'trade_outcome': training_episode,
                            'market_context': enhanced_context,
                            'learning_priority': 'high' if abs(training_episode['net_pnl']) > 1.0 else 'normal'
                        }
                        
                        self.orchestrator.sensitivity_learning_queue.append(sensitivity_data)
                        logger.debug(f"[SENSITIVITY_DQN] Added trade outcome for sensitivity learning")
                        
                    except Exception as e:
                        logger.warning(f"Error adding to sensitivity learning: {e}")
                
                return True
            else:
                logger.warning(f"[ENHANCED_RL] No comprehensive training data available, falling back to basic training")
                return self._execute_rl_training_step(training_episode)
                
        except Exception as e:
            logger.error(f"Error executing enhanced RL training step: {e}")
            return False
    
    def _calculate_rl_reward(self, closed_trade):
        """Calculate enhanced reward for RL training using pivot-based system"""
        try:
            # Extract trade information
            trade_decision = {
                'action': closed_trade.get('side', 'HOLD'),
                'confidence': closed_trade.get('confidence', 0.5),
                'price': closed_trade.get('entry_price', 0.0),
                'timestamp': closed_trade.get('entry_time', datetime.now())
            }
            
            trade_outcome = {
                'net_pnl': closed_trade.get('net_pnl', 0),
                'exit_price': closed_trade.get('exit_price', 0.0),
                'duration': closed_trade.get('duration', timedelta(0))
            }
            
            # Get market data context for pivot analysis
            symbol = closed_trade.get('symbol', 'ETH/USDT')
            trade_time = trade_decision['timestamp']
            market_data = self._get_training_context_data(symbol, trade_time, lookback_minutes=120)
            
            # Use enhanced pivot-based reward if orchestrator is available
            if hasattr(self, 'orchestrator') and self.orchestrator and hasattr(self.orchestrator, 'calculate_enhanced_pivot_reward'):
                enhanced_reward = self.orchestrator.calculate_enhanced_pivot_reward(
                    trade_decision, market_data, trade_outcome
                )
                
                # Log the enhanced reward
                logger.info(f"[ENHANCED_REWARD] Using pivot-based reward: {enhanced_reward:.3f}")
                return enhanced_reward
            
            # Fallback to original reward calculation if enhanced system not available
            logger.warning("[ENHANCED_REWARD] Falling back to original reward calculation")
            return self._calculate_original_rl_reward(closed_trade)
            
        except Exception as e:
            logger.error(f"Error calculating enhanced RL reward: {e}")
            return self._calculate_original_rl_reward(closed_trade)
    
    def _calculate_original_rl_reward(self, closed_trade):
        """Original RL reward calculation as fallback"""
        try:
            net_pnl = closed_trade.get('net_pnl', 0)
            duration = closed_trade.get('duration', timedelta(0))
            duration_hours = max(duration.total_seconds() / 3600, 0.01)  # Avoid division by zero
            fees = closed_trade.get('fees', 0)
            side = closed_trade.get('side', 'LONG')
            
            # Enhanced reward calculation with stronger penalties for losses
            base_reward = net_pnl / 5.0  # Increase sensitivity (was /10.0)
            
            # Fee penalty - trading costs should be considered
            fee_penalty = fees / 2.0  # Penalize high fee trades
            
            # Time efficiency factor - more nuanced
            if net_pnl > 0:
                # Profitable trades: reward speed, but not too much
                if duration_hours < 0.1:  # < 6 minutes
                    time_bonus = 0.5  # Fast profit bonus
                elif duration_hours < 1.0:  # < 1 hour  
                    time_bonus = 0.2  # Moderate speed bonus
                else:
                    time_bonus = 0.0  # No bonus for slow profits
                reward = base_reward + time_bonus - fee_penalty
                
            else:
                # Losing trades: STRONG penalties that increase with time and size
                loss_magnitude_penalty = abs(net_pnl) / 3.0  # Stronger loss penalty
                
                # Time penalty for holding losing positions
                if duration_hours > 4.0:  # Holding losses too long
                    time_penalty = 2.0  # Severe penalty
                elif duration_hours > 1.0:  # Moderate holding time
                    time_penalty = 1.0  # Moderate penalty  
                else:
                    time_penalty = 0.5  # Small penalty for quick losses
                
                # Total penalty for losing trades
                reward = base_reward - loss_magnitude_penalty - time_penalty - fee_penalty
            
            # Risk-adjusted rewards based on position side and market conditions
            if side == 'SHORT' and net_pnl > 0:
                # Bonus for successful shorts (harder to time)
                reward += 0.3
            elif side == 'LONG' and net_pnl < 0 and duration_hours > 2.0:
                # Extra penalty for holding losing longs too long
                reward -= 0.5
            
            # Clip reward to reasonable range but allow stronger penalties
            reward = max(-10.0, min(8.0, reward))  # Expanded range for better learning
            
            # Log detailed reward breakdown for analysis
            if abs(net_pnl) > 0.5:  # Log significant trades
                logger.info(f"[RL_REWARD] Trade #{closed_trade.get('trade_id')}: "
                           f"PnL=${net_pnl:.2f}, Fees=${fees:.3f}, "
                           f"Duration={duration_hours:.2f}h, Side={side}, "
                           f"Final_Reward={reward:.3f}")
            
            return reward
            
        except Exception as e:
            logger.warning(f"Error calculating original RL reward: {e}")
            return 0.0
    
    def _execute_rl_training_step(self, training_episode):
        """Execute a single RL training step with the trade data"""
        try:
            # Get market data around the trade time
            symbol = training_episode['symbol']
            trade_time = training_episode['timestamp']
            
            # Get historical data for the training context
            # Look back 1 hour before the trade for context
            lookback_data = self._get_training_context_data(symbol, trade_time, lookback_minutes=60)
            
            if lookback_data is None or lookback_data.empty:
                logger.warning(f"[RL_TRAINING] No context data available for trade #{training_episode['trade_id']}")
                return False
            
            # Prepare state representation
            state = self._prepare_rl_state(lookback_data, training_episode)
            
            # Prepare action (what the model decided)
            action = 1 if training_episode['side'] == 'LONG' else 0  # 1 = BUY/LONG, 0 = SELL/SHORT
            
            # Get reward
            reward = training_episode['reward']
            
            # Send training data to RL models
            training_success = self._send_rl_training_step(state, action, reward, training_episode)
            
            if training_success:
                logger.debug(f"[RL_TRAINING] Successfully trained on trade #{training_episode['trade_id']}")
                
                # Update model accuracy trend
                accuracy = self._estimate_model_accuracy()
                self.rl_training_stats['model_accuracy_trend'].append(accuracy)
                
                return True
            else:
                logger.warning(f"[RL_TRAINING] Failed to train on trade #{training_episode['trade_id']}")
                return False
                
        except Exception as e:
            logger.error(f"Error executing RL training step: {e}")
            return False
    
    def _get_training_context_data(self, symbol, trade_time, lookback_minutes=60):
        """Get historical market data for training context"""
        try:
            # Try to get data from our tick cache first
            if self.one_second_bars:
                # Convert deque to DataFrame
                bars_data = []
                for bar in self.one_second_bars:
                    bars_data.append({
                        'timestamp': bar['timestamp'],
                        'open': bar['open'],
                        'high': bar['high'],
                        'low': bar['low'],
                        'close': bar['close'],
                        'volume': bar['volume']
                    })
                
                if bars_data:
                    df = pd.DataFrame(bars_data)
                    df['timestamp'] = pd.to_datetime(df['timestamp'])
                    df.set_index('timestamp', inplace=True)
                    
                    # Filter to lookback period
                    end_time = pd.to_datetime(trade_time)
                    start_time = end_time - timedelta(minutes=lookback_minutes)
                    
                    context_data = df[(df.index >= start_time) & (df.index <= end_time)]
                    
                    if not context_data.empty:
                        return context_data
            
            # Fallback to data provider
            if self.data_provider:
                # Get 1-minute data for the lookback period
                context_data = self.data_provider.get_historical_data(
                    symbol=symbol,
                    timeframe='1m',
                    limit=lookback_minutes,
                    refresh=True
                )
                return context_data
            
            return None
            
        except Exception as e:
            logger.warning(f"Error getting training context data: {e}")
            return None
    
    def _prepare_rl_state(self, market_data, training_episode):
        """Prepare enhanced state representation for RL training with comprehensive market context"""
        try:
            # Calculate technical indicators
            df = market_data.copy()
            
            # Basic price features
            df['returns'] = df['close'].pct_change()
            df['log_returns'] = np.log(df['close'] / df['close'].shift(1))
            df['price_ma_5'] = df['close'].rolling(5).mean()
            df['price_ma_20'] = df['close'].rolling(20).mean()
            df['price_ma_50'] = df['close'].rolling(50).mean()
            
            # Volatility and risk metrics
            df['volatility'] = df['returns'].rolling(10).std()
            df['volatility_ma'] = df['volatility'].rolling(5).mean()
            df['max_drawdown'] = (df['close'] / df['close'].cummax() - 1).rolling(20).min()
            
            # Momentum indicators
            df['rsi'] = self._calculate_rsi(df['close'])
            df['rsi_ma'] = df['rsi'].rolling(5).mean()
            df['momentum'] = df['close'] / df['close'].shift(10) - 1  # 10-period momentum
            
            # Volume analysis
            df['volume_ma'] = df['volume'].rolling(10).mean()
            df['volume_ratio'] = df['volume'] / df['volume_ma']
            df['volume_trend'] = df['volume_ma'] / df['volume_ma'].shift(5) - 1
            
            # Market structure
            df['higher_highs'] = (df['high'] > df['high'].shift(1)).rolling(5).sum() / 5
            df['lower_lows'] = (df['low'] < df['low'].shift(1)).rolling(5).sum() / 5
            df['trend_strength'] = df['higher_highs'] - df['lower_lows']
            
            # Support/Resistance levels (simplified)
            df['distance_to_high'] = (df['high'].rolling(20).max() - df['close']) / df['close']
            df['distance_to_low'] = (df['close'] - df['low'].rolling(20).min()) / df['close']
            
            # Time-based features
            df['hour'] = df.index.hour if hasattr(df.index, 'hour') else 12  # Default to noon
            df['is_market_hours'] = ((df['hour'] >= 9) & (df['hour'] <= 16)).astype(float)
            
            # Drop NaN values
            df = df.dropna()
            
            if df.empty:
                logger.warning("Empty dataframe after technical indicators calculation")
                return None
            
            # Enhanced state features (normalized)
            state_features = [
                # Price momentum and trend
                df['returns'].iloc[-1],
                df['log_returns'].iloc[-1],
                (df['price_ma_5'].iloc[-1] / df['close'].iloc[-1] - 1),
                (df['price_ma_20'].iloc[-1] / df['close'].iloc[-1] - 1),
                (df['price_ma_50'].iloc[-1] / df['close'].iloc[-1] - 1),
                df['momentum'].iloc[-1],
                df['trend_strength'].iloc[-1],
                
                # Volatility and risk
                df['volatility'].iloc[-1],
                df['volatility_ma'].iloc[-1],
                df['max_drawdown'].iloc[-1],
                
                # Momentum indicators
                df['rsi'].iloc[-1] / 100.0,  # Normalize RSI to 0-1
                df['rsi_ma'].iloc[-1] / 100.0,
                
                # Volume analysis
                df['volume_ratio'].iloc[-1],
                df['volume_trend'].iloc[-1],
                
                # Market structure
                df['distance_to_high'].iloc[-1],
                df['distance_to_low'].iloc[-1],
                
                # Time features
                df['hour'].iloc[-1] / 24.0,  # Normalize hour to 0-1
                df['is_market_hours'].iloc[-1],
            ]
            
            # Add Williams pivot points features (250 features)
            try:
                pivot_features = self._get_williams_pivot_features(df)
                if pivot_features:
                    state_features.extend(pivot_features)
                else:
                    state_features.extend([0.0] * 250)  # Default if calculation fails
            except Exception as e:
                logger.warning(f"Error calculating Williams pivot points: {e}")
                state_features.extend([0.0] * 250)  # Default features
            
            # Add multi-timeframe OHLCV features (200 features: ETH 1s/1m/1d + BTC 1s)
            try:
                multi_tf_features = self._get_multi_timeframe_features(training_episode.get('symbol', 'ETH/USDT'))
                if multi_tf_features:
                    state_features.extend(multi_tf_features)
                else:
                    state_features.extend([0.0] * 200)  # Default if calculation fails
            except Exception as e:
                logger.warning(f"Error calculating multi-timeframe features: {e}")
                state_features.extend([0.0] * 200)  # Default features
            
            # Add trade-specific context
            entry_price = training_episode['entry_price']
            current_price = df['close'].iloc[-1]
            
            trade_features = [
                (current_price - entry_price) / entry_price,  # Unrealized P&L
                training_episode['duration_seconds'] / 3600.0,  # Duration in hours
                1.0 if training_episode['side'] == 'LONG' else 0.0,  # Position side
                min(training_episode['duration_seconds'] / 14400.0, 1.0),  # Time pressure (0-4h normalized)
            ]
            
            state_features.extend(trade_features)
            
            # Add recent volatility context (last 3 periods)
            if len(df) >= 3:
                recent_volatility = [
                    df['volatility'].iloc[-3],
                    df['volatility'].iloc[-2], 
                    df['volatility'].iloc[-1]
                ]
                state_features.extend(recent_volatility)
            else:
                state_features.extend([0.0, 0.0, 0.0])
            
            # Ensure all features are valid numbers
            state_features = [float(x) if pd.notna(x) and np.isfinite(x) else 0.0 for x in state_features]
            
            logger.debug(f"[RL_STATE] Prepared {len(state_features)} features for trade #{training_episode.get('trade_id')} (including Williams pivot points and multi-timeframe)")
            
            return np.array(state_features, dtype=np.float32)
            
        except Exception as e:
            logger.warning(f"Error preparing enhanced RL state: {e}")
            import traceback
            logger.debug(traceback.format_exc())
            return None
    
    def _send_rl_training_step(self, state, action, reward, training_episode):
        """Send training step to RL models"""
        try:
            # Check if we have RL models loaded
            if not hasattr(self, 'model_registry') or not self.model_registry:
                logger.debug("[RL_TRAINING] No model registry available")
                return False
            
            # Prepare training data package
            training_data = {
                'state': state.tolist() if state is not None else [],
                'action': action,
                'reward': reward,
                'trade_info': {
                    'trade_id': training_episode['trade_id'],
                    'side': training_episode['side'],
                    'pnl': training_episode['net_pnl'],
                    'duration': training_episode['duration_seconds']
                },
                'timestamp': training_episode['timestamp'].isoformat()
            }
            
            # Try to send to RL training process
            success = self._send_to_rl_training_process(training_data)
            
            if success:
                logger.debug(f"[RL_TRAINING] Sent training step for trade #{training_episode['trade_id']}")
                return True
            else:
                logger.debug(f"[RL_TRAINING] Failed to send training step for trade #{training_episode['trade_id']}")
                return False
                
        except Exception as e:
            logger.error(f"Error starting dashboard: {e}")
            raise 
    
    def _send_to_rl_training_process(self, training_data):
        """Send training data to RL training process"""
        try:
            # For now, just log the training data
            # In a full implementation, this would send to a separate RL training process
            logger.info(f"[RL_TRAINING] Training data: Action={training_data['action']}, "
                       f"Reward={training_data['reward']:.3f}, "
                       f"State_size={len(training_data['state'])}")
            
            # Simulate training success
            return True
            
        except Exception as e:
            logger.warning(f"Error in RL training process communication: {e}")
            return False
    
    def _estimate_model_accuracy(self):
        """Estimate current model accuracy based on recent trades"""
        try:
            if len(self.closed_trades) < 5:
                return 0.5  # Default accuracy
            
            # Look at last 20 trades
            recent_trades = self.closed_trades[-20:]
            profitable_trades = sum(1 for trade in recent_trades if trade.get('net_pnl', 0) > 0)
            
            accuracy = profitable_trades / len(recent_trades)
            return accuracy
            
        except Exception as e:
            logger.warning(f"Error estimating model accuracy: {e}")
            return 0.5
    
    def get_rl_training_stats(self):
        """Get current RL training statistics"""
        return self.rl_training_stats.copy()

    def stop_streaming(self):
        """Stop all streaming and training components"""
        try:
            logger.info("Stopping dashboard streaming and training components...")
            
            # Stop unified data stream
            if ENHANCED_RL_AVAILABLE and hasattr(self, 'unified_stream'):
                try:
                    asyncio.run(self.unified_stream.stop_streaming())
                    if hasattr(self, 'stream_consumer_id'):
                        self.unified_stream.unregister_consumer(self.stream_consumer_id)
                    logger.info("Unified data stream stopped")
                except Exception as e:
                    logger.warning(f"Error stopping unified stream: {e}")
            
            # Stop WebSocket streaming
            self.is_streaming = False
            if self.ws_connection:
                try:
                    self.ws_connection.close()
                    logger.info("WebSocket connection closed")
                except Exception as e:
                    logger.warning(f"Error closing WebSocket: {e}")
            
            if self.ws_thread and self.ws_thread.is_alive():
                try:
                    self.ws_thread.join(timeout=5)
                    logger.info("WebSocket thread stopped")
                except Exception as e:
                    logger.warning(f"Error stopping WebSocket thread: {e}")
            
            # Stop continuous training
            self.stop_continuous_training()
            
            # Stop enhanced RL training if available
            if self.enhanced_rl_training_enabled and hasattr(self.orchestrator, 'enhanced_rl_trainer'):
                try:
                    if hasattr(self.orchestrator.enhanced_rl_trainer, 'stop_training'):
                        asyncio.run(self.orchestrator.enhanced_rl_trainer.stop_training())
                    logger.info("Enhanced RL training stopped")
                except Exception as e:
                    logger.warning(f"Error stopping enhanced RL training: {e}")
            
            logger.info("All streaming and training components stopped")
            
        except Exception as e:
            logger.error(f"Error stopping streaming: {e}")

    def _get_williams_pivot_features(self, df: pd.DataFrame) -> Optional[List[float]]:
        """Get Williams Market Structure pivot features for RL training"""
        try:
            # Use reused Williams instance
            if not self.williams_structure:
                logger.warning("Williams Market Structure not available")
                return None
            
            # Convert DataFrame to numpy array for Williams calculation
            if len(df) < 20:  # Reduced from 50 to match Williams minimum requirement
                logger.debug(f"[WILLIAMS] Insufficient data for pivot calculation: {len(df)} bars (need 20+)")
                return None
                
            try:
                ohlcv_array = np.array([
                    [self._to_local_timezone(df.index[i]).timestamp() if hasattr(df.index[i], 'timestamp') else time.time(),
                     df['open'].iloc[i], df['high'].iloc[i], df['low'].iloc[i], 
                     df['close'].iloc[i], df['volume'].iloc[i]]
                    for i in range(len(df))
                ])
                
                logger.debug(f"[WILLIAMS] Prepared OHLCV array: {ohlcv_array.shape}, price range: {ohlcv_array[:, 4].min():.2f} - {ohlcv_array[:, 4].max():.2f}")
                
            except Exception as e:
                logger.warning(f"[WILLIAMS] Error preparing OHLCV array: {e}")
                return None
            
            # Calculate Williams pivot points with reused instance
            try:
                structure_levels = self.williams_structure.calculate_recursive_pivot_points(ohlcv_array)
                
                # Add diagnostics for debugging
                total_pivots = sum(len(level.swing_points) for level in structure_levels.values())
                if total_pivots == 0:
                    logger.debug(f"[WILLIAMS] No pivot points detected in {len(ohlcv_array)} bars")
                else:
                    logger.debug(f"[WILLIAMS] Successfully detected {total_pivots} pivot points across {len([l for l in structure_levels.values() if len(l.swing_points) > 0])} levels")
                    
            except Exception as e:
                logger.warning(f"[WILLIAMS] Error in pivot calculation: {e}")
                return None
            
            # Extract features (250 features total)
            pivot_features = self.williams_structure.extract_features_for_rl(structure_levels)
            
            logger.debug(f"[PIVOT] Calculated {len(pivot_features)} Williams pivot features")
            return pivot_features
            
        except Exception as e:
            logger.warning(f"Error calculating Williams pivot features: {e}")
            return None

    def _get_cnn_pivot_predictions(self, symbol: str, df: pd.DataFrame) -> Optional[List[Dict]]:
        """
        Continuously run CNN inference to predict next pivot points
        Returns list of predicted pivot points with timestamps and prices
        """
        try:
            # Get Williams Market Structure instance from orchestrator
            if (hasattr(self, 'orchestrator') and 
                hasattr(self.orchestrator, 'pivot_rl_trainer') and
                hasattr(self.orchestrator.pivot_rl_trainer, 'williams') and
                self.orchestrator.pivot_rl_trainer.williams.cnn_model):
                
                williams = self.orchestrator.pivot_rl_trainer.williams
                
                # Prepare current market data for CNN inference
                current_time = datetime.now()
                current_price = df['close'].iloc[-1] if not df.empty else 0
                
                # Create a dummy swing point for the current position
                current_swing = SwingPoint(
                    timestamp=current_time,
                    price=current_price,
                    index=len(df) - 1,
                    swing_type=SwingType.SWING_HIGH,  # Will be determined by CNN
                    strength=2
                )
                
                # Prepare CNN input using current market state
                ohlcv_data_context = df[['open', 'high', 'low', 'close', 'volume']].values
                X_predict = williams._prepare_cnn_input(
                    current_swing, 
                    ohlcv_data_context, 
                    williams.previous_pivot_details_for_cnn
                )
                
                if X_predict is not None and X_predict.size > 0:
                    # Reshape for batch prediction
                    if len(X_predict.shape) == len(williams.cnn_model.input_shape):
                        X_predict_batch = np.expand_dims(X_predict, axis=0)
                    else:
                        X_predict_batch = X_predict
                    
                    # Get CNN prediction
                    pred_class, pred_proba = williams.cnn_model.predict(X_predict_batch)
                    
                    # Extract prediction details
                    final_pred_class = pred_class[0] if isinstance(pred_class, np.ndarray) and pred_class.ndim > 0 else pred_class
                    final_pred_proba = pred_proba[0] if isinstance(pred_proba, np.ndarray) and pred_proba.ndim > 0 else pred_proba
                    
                    # Create prediction results
                    predictions = []
                    
                    # CNN outputs 10 values: 5 levels * (type + price) = 10 outputs
                    # Parse predictions for each Williams level
                    for level in range(5):
                        base_idx = level * 2
                        if base_idx + 1 < len(final_pred_proba):
                            pivot_type_prob = final_pred_proba[base_idx]
                            pivot_price_factor = final_pred_proba[base_idx + 1]
                            
                            # Convert to actual prediction
                            is_high = pivot_type_prob > 0.5
                            
                            # Estimate next pivot price based on current price and prediction factor
                            # Factor represents percentage change from current price
                            predicted_price = current_price * (1 + (pivot_price_factor - 0.5) * 0.1)  # Max 5% change
                            
                            # Estimate next pivot time (1-5 minutes from now based on level)
                            time_offset_minutes = (level + 1) * 1  # Level 0: 1min, Level 1: 2min, etc.
                            predicted_time = current_time + timedelta(minutes=time_offset_minutes)
                            
                            prediction = {
                                'timestamp': predicted_time,
                                'price': predicted_price,
                                'level': level,
                                'swing_type': 'HIGH' if is_high else 'LOW',
                                'confidence': abs(pivot_type_prob - 0.5) * 2,  # 0-1 scale
                                'model_confidence': float(np.max(final_pred_proba)) if len(final_pred_proba) > 0 else 0.5,
                                'prediction_time': current_time,
                                'current_price': current_price
                            }
                            predictions.append(prediction)
                    
                    # Store predictions for comparison with actual pivots
                    if not hasattr(self, 'cnn_predictions_history'):
                        self.cnn_predictions_history = deque(maxlen=1000)
                    
                    # Add to history with metadata for training data capture
                    prediction_record = {
                        'predictions': predictions,
                        'model_inputs': X_predict.tolist(),  # Store for comparison
                        'market_state': {
                            'price': current_price,
                            'timestamp': current_time,
                            'symbol': symbol,
                            'ohlcv_context': ohlcv_data_context[-10:].tolist()  # Last 10 bars
                        }
                    }
                    self.cnn_predictions_history.append(prediction_record)
                    
                    logger.info(f"CNN Pivot Predictions generated: {len(predictions)} predictions for {symbol}")
                    logger.debug(f"CNN Predictions: {[f'L{p['level']} {p['swing_type']} @ ${p['price']:.2f} (conf: {p['confidence']:.2f})' for p in predictions]}")
                    
                    return predictions
                    
            return None
            
        except Exception as e:
            logger.warning(f"Error getting CNN pivot predictions: {e}")
            return None

    def _add_cnn_predictions_to_chart(self, fig, predictions: List[Dict], row: int = 1):
        """
        Add CNN pivot predictions as hollow circles to the chart
        Different colors for different confidence levels and pivot types
        """
        try:
            if not predictions:
                return
            
            # Separate predictions by type and confidence
            high_predictions = [p for p in predictions if p['swing_type'] == 'HIGH']
            low_predictions = [p for p in predictions if p['swing_type'] == 'LOW']
            
            # Add HIGH predictions (hollow circles above price)
            if high_predictions:
                # Group by confidence level for different visual styles
                high_conf_preds = [p for p in high_predictions if p['confidence'] >= 0.7]
                med_conf_preds = [p for p in high_predictions if 0.4 <= p['confidence'] < 0.7]
                low_conf_preds = [p for p in high_predictions if p['confidence'] < 0.4]
                
                # High confidence HIGH predictions
                if high_conf_preds:
                    fig.add_trace(
                        go.Scatter(
                            x=[p['timestamp'] for p in high_conf_preds],
                            y=[p['price'] for p in high_conf_preds],
                            mode='markers',
                            marker=dict(
                                color='rgba(255, 100, 100, 0.8)',  # Red
                                size=12,
                                symbol='circle-open',
                                line=dict(width=3, color='red')
                            ),
                            name="CNN HIGH Pred (High Conf)",
                            showlegend=True,
                            hovertemplate="<b>CNN HIGH Prediction</b><br>" +
                                        "Price: $%{y:.2f}<br>" +
                                        "Time: %{x}<br>" +
                                        "Confidence: %{customdata:.1%}<br>" +
                                        "Level: L%{text}<extra></extra>",
                            customdata=[p['confidence'] for p in high_conf_preds],
                            text=[p['level'] for p in high_conf_preds]
                        ),
                        row=row, col=1
                    )
                
                # Medium confidence HIGH predictions
                if med_conf_preds:
                    fig.add_trace(
                        go.Scatter(
                            x=[p['timestamp'] for p in med_conf_preds],
                            y=[p['price'] for p in med_conf_preds],
                            mode='markers',
                            marker=dict(
                                color='rgba(255, 150, 150, 0.6)',  # Light red
                                size=10,
                                symbol='circle-open',
                                line=dict(width=2, color='red')
                            ),
                            name="CNN HIGH Pred (Med Conf)",
                            showlegend=True,
                            hovertemplate="<b>CNN HIGH Prediction</b><br>" +
                                        "Price: $%{y:.2f}<br>" +
                                        "Time: %{x}<br>" +
                                        "Confidence: %{customdata:.1%}<br>" +
                                        "Level: L%{text}<extra></extra>",
                            customdata=[p['confidence'] for p in med_conf_preds],
                            text=[p['level'] for p in med_conf_preds]
                        ),
                        row=row, col=1
                    )
                
                # Low confidence HIGH predictions
                if low_conf_preds:
                    fig.add_trace(
                        go.Scatter(
                            x=[p['timestamp'] for p in low_conf_preds],
                            y=[p['price'] for p in low_conf_preds],
                            mode='markers',
                            marker=dict(
                                color='rgba(255, 200, 200, 0.4)',  # Very light red
                                size=8,
                                symbol='circle-open',
                                line=dict(width=1, color='red')
                            ),
                            name="CNN HIGH Pred (Low Conf)",
                            showlegend=True,
                            hovertemplate="<b>CNN HIGH Prediction</b><br>" +
                                        "Price: $%{y:.2f}<br>" +
                                        "Time: %{x}<br>" +
                                        "Confidence: %{customdata:.1%}<br>" +
                                        "Level: L%{text}<extra></extra>",
                            customdata=[p['confidence'] for p in low_conf_preds],
                            text=[p['level'] for p in low_conf_preds]
                        ),
                        row=row, col=1
                    )
            
            # Add LOW predictions (hollow circles below price)
            if low_predictions:
                # Group by confidence level
                high_conf_preds = [p for p in low_predictions if p['confidence'] >= 0.7]
                med_conf_preds = [p for p in low_predictions if 0.4 <= p['confidence'] < 0.7]
                low_conf_preds = [p for p in low_predictions if p['confidence'] < 0.4]
                
                # High confidence LOW predictions
                if high_conf_preds:
                    fig.add_trace(
                        go.Scatter(
                            x=[p['timestamp'] for p in high_conf_preds],
                            y=[p['price'] for p in high_conf_preds],
                            mode='markers',
                            marker=dict(
                                color='rgba(100, 255, 100, 0.8)',  # Green
                                size=12,
                                symbol='circle-open',
                                line=dict(width=3, color='green')
                            ),
                            name="CNN LOW Pred (High Conf)",
                            showlegend=True,
                            hovertemplate="<b>CNN LOW Prediction</b><br>" +
                                        "Price: $%{y:.2f}<br>" +
                                        "Time: %{x}<br>" +
                                        "Confidence: %{customdata:.1%}<br>" +
                                        "Level: L%{text}<extra></extra>",
                            customdata=[p['confidence'] for p in high_conf_preds],
                            text=[p['level'] for p in high_conf_preds]
                        ),
                        row=row, col=1
                    )
                
                # Medium confidence LOW predictions
                if med_conf_preds:
                    fig.add_trace(
                        go.Scatter(
                            x=[p['timestamp'] for p in med_conf_preds],
                            y=[p['price'] for p in med_conf_preds],
                            mode='markers',
                            marker=dict(
                                color='rgba(150, 255, 150, 0.6)',  # Light green
                                size=10,
                                symbol='circle-open',
                                line=dict(width=2, color='green')
                            ),
                            name="CNN LOW Pred (Med Conf)",
                            showlegend=True,
                            hovertemplate="<b>CNN LOW Prediction</b><br>" +
                                        "Price: $%{y:.2f}<br>" +
                                        "Time: %{x}<br>" +
                                        "Confidence: %{customdata:.1%}<br>" +
                                        "Level: L%{text}<extra></extra>",
                            customdata=[p['confidence'] for p in med_conf_preds],
                            text=[p['level'] for p in med_conf_preds]
                        ),
                        row=row, col=1
                    )
                
                # Low confidence LOW predictions
                if low_conf_preds:
                    fig.add_trace(
                        go.Scatter(
                            x=[p['timestamp'] for p in low_conf_preds],
                            y=[p['price'] for p in low_conf_preds],
                            mode='markers',
                            marker=dict(
                                color='rgba(200, 255, 200, 0.4)',  # Very light green
                                size=8,
                                symbol='circle-open',
                                line=dict(width=1, color='green')
                            ),
                            name="CNN LOW Pred (Low Conf)",
                            showlegend=True,
                            hovertemplate="<b>CNN LOW Prediction</b><br>" +
                                        "Price: $%{y:.2f}<br>" +
                                        "Time: %{x}<br>" +
                                        "Confidence: %{customdata:.1%}<br>" +
                                        "Level: L%{text}<extra></extra>",
                            customdata=[p['confidence'] for p in low_conf_preds],
                            text=[p['level'] for p in low_conf_preds]
                        ),
                        row=row, col=1
                    )
            
            logger.debug(f"Added {len(predictions)} CNN predictions to chart")
            
        except Exception as e:
            logger.error(f"Error adding CNN predictions to chart: {e}")

    def _capture_actual_pivot_data(self, actual_pivot: Dict[str, Any]) -> None:
        """
        Capture data frame when an actual pivot is identified for model training comparison
        Stores current model inputs and next prediction for comparison with actual results
        """
        try:
            if not hasattr(self, 'actual_pivots_for_training'):
                self.actual_pivots_for_training = deque(maxlen=500)
            
            # Find corresponding CNN predictions that should be compared with this actual pivot
            current_time = datetime.now()
            symbol = actual_pivot.get('symbol', 'ETH/USDT')
            
            # Look for recent CNN predictions that might match this actual pivot
            matching_predictions = []
            if hasattr(self, 'cnn_predictions_history'):
                for pred_record in list(self.cnn_predictions_history)[-50:]:  # Last 50 prediction records
                    pred_time = pred_record['market_state']['timestamp']
                    time_diff = (current_time - pred_time).total_seconds() / 60  # minutes
                    
                    # Check predictions that were made 1-10 minutes ago (reasonable prediction window)
                    if 1 <= time_diff <= 10:
                        for prediction in pred_record['predictions']:
                            pred_pivot_time = prediction['timestamp']
                            time_to_pivot = abs((current_time - pred_pivot_time).total_seconds() / 60)
                            
                            # If the actual pivot occurred close to predicted time (within 2 minutes)
                            if time_to_pivot <= 2:
                                price_diff = abs(actual_pivot['price'] - prediction['price'])
                                price_diff_pct = price_diff / actual_pivot['price'] * 100
                                
                                # If price is also reasonably close (within 2%)
                                if price_diff_pct <= 2:
                                    matching_predictions.append({
                                        'prediction': prediction,
                                        'model_inputs': pred_record['model_inputs'],
                                        'market_state': pred_record['market_state'],
                                        'time_accuracy': time_to_pivot,
                                        'price_accuracy': price_diff_pct
                                    })
            
            # Create training data record
            training_record = {
                'timestamp': current_time,
                'symbol': symbol,
                'actual_pivot': actual_pivot,
                'matching_predictions': matching_predictions,
                'prediction_accuracy_count': len(matching_predictions),
                'market_context': {
                    'price_at_capture': actual_pivot['price'],
                    'pivot_type': actual_pivot.get('swing_type', 'UNKNOWN'),
                    'pivot_strength': actual_pivot.get('strength', 0)
                }
            }
            
            self.actual_pivots_for_training.append(training_record)
            
            # Log for analysis
            if matching_predictions:
                avg_time_acc = sum(p['time_accuracy'] for p in matching_predictions) / len(matching_predictions)
                avg_price_acc = sum(p['price_accuracy'] for p in matching_predictions) / len(matching_predictions)
                logger.info(f"ACTUAL PIVOT CAPTURED: {actual_pivot.get('swing_type', 'UNKNOWN')} @ ${actual_pivot['price']:.2f}")
                logger.info(f"  Found {len(matching_predictions)} matching CNN predictions")
                logger.info(f"  Avg time accuracy: {avg_time_acc:.1f} minutes")
                logger.info(f"  Avg price accuracy: {avg_price_acc:.1f}%")
            else:
                logger.info(f"ACTUAL PIVOT CAPTURED: {actual_pivot.get('swing_type', 'UNKNOWN')} @ ${actual_pivot['price']:.2f} (no matching predictions)")
            
            # Save training data periodically
            if len(self.actual_pivots_for_training) % 10 == 0:
                self._save_pivot_training_data()
                
        except Exception as e:
            logger.error(f"Error capturing actual pivot data: {e}")

    def _save_pivot_training_data(self) -> None:
        """Save captured pivot training data to file for analysis"""
        try:
            if not hasattr(self, 'actual_pivots_for_training'):
                return
            
            # Create training data directory
            from pathlib import Path
            training_dir = Path("logs/pivot_training_data")
            training_dir.mkdir(parents=True, exist_ok=True)
            
            # Save recent training data
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            filename = training_dir / f"pivot_training_data_{timestamp}.json"
            
            # Convert to JSON-serializable format
            training_data = []
            for record in list(self.actual_pivots_for_training)[-100:]:  # Last 100 records
                json_record = {
                    'timestamp': record['timestamp'].isoformat(),
                    'symbol': record['symbol'],
                    'actual_pivot': record['actual_pivot'],
                    'matching_predictions': record['matching_predictions'],
                    'prediction_accuracy_count': record['prediction_accuracy_count'],
                    'market_context': record['market_context']
                }
                training_data.append(json_record)
            
            with open(filename, 'w') as f:
                json.dump(training_data, f, indent=2, default=str)
            
            logger.info(f"Saved {len(training_data)} pivot training records to {filename}")
            
        except Exception as e:
            logger.error(f"Error saving pivot training data: {e}")