import asyncio import json import logging import datetime from typing import Dict, List, Optional import websockets import plotly.graph_objects as go from plotly.subplots import make_subplots import dash from dash import html, dcc from dash.dependencies import Input, Output import pandas as pd import numpy as np from collections import deque import time from threading import Thread import requests import os from datetime import datetime, timedelta import pytz import tzlocal import threading # Configure logging with more detailed format logging.basicConfig( level=logging.INFO, # Changed to DEBUG for more detailed logs format='%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s', handlers=[ logging.StreamHandler(), logging.FileHandler('realtime_chart.log') ] ) logger = logging.getLogger(__name__) # Neural Network integration (conditional import) NN_ENABLED = os.environ.get('ENABLE_NN_MODELS', '0') == '1' nn_orchestrator = None nn_inference_thread = None if NN_ENABLED: try: import sys # Add project root to sys.path if needed project_root = os.path.dirname(os.path.abspath(__file__)) if project_root not in sys.path: sys.path.append(project_root) from NN.main import NeuralNetworkOrchestrator logger.info("Neural Network module enabled") except ImportError as e: logger.warning(f"Failed to import Neural Network module, disabling NN features: {str(e)}") NN_ENABLED = False # NN utility functions def setup_neural_network(): """Initialize the neural network components if enabled""" global nn_orchestrator, NN_ENABLED if not NN_ENABLED: return False try: # Get configuration from environment variables or use defaults symbol = os.environ.get('NN_SYMBOL', 'BTC/USDT') timeframes = os.environ.get('NN_TIMEFRAMES', '1m,5m,1h,4h,1d').split(',') output_size = int(os.environ.get('NN_OUTPUT_SIZE', '3')) # 3 for BUY/HOLD/SELL # Configure the orchestrator config = { 'symbol': symbol, 'timeframes': timeframes, 'window_size': int(os.environ.get('NN_WINDOW_SIZE', '20')), 'n_features': 5, # OHLCV 'output_size': output_size, 'model_dir': 'NN/models/saved', 'data_dir': 'NN/data' } # Initialize the orchestrator logger.info(f"Initializing Neural Network Orchestrator with config: {config}") nn_orchestrator = NeuralNetworkOrchestrator(config) # Start inference thread if enabled inference_interval = int(os.environ.get('NN_INFERENCE_INTERVAL', '60')) if inference_interval > 0: start_nn_inference_thread(inference_interval) return True except Exception as e: logger.error(f"Error setting up neural network: {str(e)}") import traceback logger.error(traceback.format_exc()) NN_ENABLED = False return False def start_nn_inference_thread(interval_seconds): """Start a background thread to periodically run inference with the neural network""" global nn_inference_thread if not NN_ENABLED or nn_orchestrator is None: logger.warning("Cannot start inference thread - Neural Network not enabled or initialized") return False def inference_worker(): """Worker function for the inference thread""" model_type = os.environ.get('NN_MODEL_TYPE', 'cnn') timeframe = os.environ.get('NN_TIMEFRAME', '1h') logger.info(f"Starting neural network inference thread with {interval_seconds}s interval") logger.info(f"Using model type: {model_type}, timeframe: {timeframe}") # Wait a bit for charts to initialize time.sleep(5) # Track active charts active_charts = [] while True: try: # Find active charts if we don't have them yet if not active_charts and 'charts' in globals(): active_charts = globals()['charts'] logger.info(f"Found {len(active_charts)} active charts for NN signals") # Run inference result = nn_orchestrator.run_inference_pipeline( model_type=model_type, timeframe=timeframe ) if result: # Log the result logger.info(f"Neural network inference result: {result}") # Add signal to charts if active_charts: try: if 'action' in result: action = result['action'] timestamp = datetime.fromisoformat(result['timestamp'].replace('Z', '+00:00')) # Get probability if available probability = None if 'probability' in result: probability = result['probability'] elif 'probabilities' in result: probability = result['probabilities'].get(action, None) # Add signal to each chart for chart in active_charts: if hasattr(chart, 'add_nn_signal'): chart.add_nn_signal(action, timestamp, probability) except Exception as e: logger.error(f"Error adding NN signal to chart: {str(e)}") import traceback logger.error(traceback.format_exc()) # Sleep for the interval time.sleep(interval_seconds) except Exception as e: logger.error(f"Error in inference thread: {str(e)}") import traceback logger.error(traceback.format_exc()) time.sleep(5) # Wait a bit before retrying # Create and start the thread nn_inference_thread = threading.Thread(target=inference_worker, daemon=True) nn_inference_thread.start() return True # Try to get local timezone, default to Sofia/EET if not available try: local_timezone = tzlocal.get_localzone() # Get timezone name safely try: tz_name = str(local_timezone) # Handle case where it might be zoneinfo.ZoneInfo object instead of pytz timezone if hasattr(local_timezone, 'zone'): tz_name = local_timezone.zone elif hasattr(local_timezone, 'key'): tz_name = local_timezone.key else: tz_name = str(local_timezone) except: tz_name = "Local" logger.info(f"Detected local timezone: {local_timezone} ({tz_name})") except Exception as e: logger.warning(f"Could not detect local timezone: {str(e)}. Defaulting to Sofia/EET") local_timezone = pytz.timezone('Europe/Sofia') tz_name = "Europe/Sofia" def convert_to_local_time(timestamp): """Convert timestamp to local timezone""" try: if isinstance(timestamp, pd.Timestamp): dt = timestamp.to_pydatetime() elif isinstance(timestamp, np.datetime64): dt = pd.Timestamp(timestamp).to_pydatetime() elif isinstance(timestamp, str): dt = pd.to_datetime(timestamp).to_pydatetime() else: dt = timestamp # If datetime is naive (no timezone), assume it's UTC if dt.tzinfo is None: dt = dt.replace(tzinfo=pytz.UTC) # Convert to local timezone local_dt = dt.astimezone(local_timezone) return local_dt except Exception as e: logger.error(f"Error converting timestamp to local time: {str(e)}") return timestamp class TradeTickStorage: """Store and manage raw trade ticks for display and candle formation""" def __init__(self, symbol: str = None, max_age_seconds: int = 3600): # 1 hour by default, up from 30 min self.ticks = [] self.symbol = symbol or "unknown" # Use symbol name for cache files self.max_age_seconds = max_age_seconds self.last_cleanup_time = time.time() self.last_cache_time = time.time() # Adjust cleanup interval based on max_age_seconds (more time = less frequent cleanup) self.cleanup_interval = min(max(10, max_age_seconds // 60), 60) # Between 10-60 seconds self.cache_interval = 60 # Cache ticks every 60 seconds self.last_tick = None self.cache_dir = "cache" os.makedirs(self.cache_dir, exist_ok=True) logger.info(f"Initialized TradeTickStorage for {self.symbol} with max age: {max_age_seconds} seconds, cleanup interval: {self.cleanup_interval} seconds") # Try to load cached ticks on startup self._load_cached_ticks() def add_tick(self, tick: Dict): """Add a new trade tick to storage""" # Check for duplicate timestamps if self.ticks and tick['timestamp'] == self.ticks[-1]['timestamp']: # For duplicate timestamps, update volume instead of adding new tick self.ticks[-1]['volume'] += tick.get('volume', 0) # Update price to the newest one self.ticks[-1]['price'] = tick['price'] logger.debug(f"Merged tick with duplicate timestamp: {tick['timestamp']}") else: # No duplicate, add as new tick self.ticks.append(tick) self.last_tick = tick logger.debug(f"Added tick: {tick}, total ticks: {len(self.ticks)}") # Only clean up periodically rather than on every tick current_time = time.time() if current_time - self.last_cleanup_time > self.cleanup_interval: self._cleanup() self.last_cleanup_time = current_time # Cache ticks periodically if current_time - self.last_cache_time > self.cache_interval: self._cache_ticks() self.last_cache_time = current_time def _cleanup(self): """Remove ticks older than max_age_seconds""" now = int(time.time() * 1000) # Current time in ms cutoff = now - (self.max_age_seconds * 1000) old_count = len(self.ticks) self.ticks = [tick for tick in self.ticks if tick['timestamp'] >= cutoff] removed = old_count - len(self.ticks) if removed > 0: logger.debug(f"Cleaned up {removed} old ticks, remaining: {len(self.ticks)}") def _cache_ticks(self): """Cache recent ticks to disk""" if not self.ticks: return # Get ticks from last 10 minutes now = int(time.time() * 1000) # Current time in ms cutoff = now - (600 * 1000) # 10 minutes in ms recent_ticks = [tick for tick in self.ticks if tick['timestamp'] >= cutoff] if not recent_ticks: logger.debug("No recent ticks to cache") return # Create symbol-specific filename symbol_safe = self.symbol.replace("/", "_").replace("-", "_").lower() cache_file = os.path.join(self.cache_dir, f"{symbol_safe}_recent_ticks.csv") # Save to disk try: tick_df = pd.DataFrame(recent_ticks) tick_df.to_csv(cache_file, index=False) logger.info(f"Cached {len(recent_ticks)} recent ticks for {self.symbol} to {cache_file}") except Exception as e: logger.error(f"Error caching ticks: {str(e)}") def _load_cached_ticks(self): """Load cached ticks from disk on startup""" # Create symbol-specific filename symbol_safe = self.symbol.replace("/", "_").replace("-", "_").lower() cache_file = os.path.join(self.cache_dir, f"{symbol_safe}_recent_ticks.csv") if not os.path.exists(cache_file): logger.info(f"No cached ticks found for {self.symbol}") return try: # Check if cache is fresh (less than 10 minutes old) file_age = time.time() - os.path.getmtime(cache_file) if file_age > 600: # 10 minutes logger.info(f"Cached ticks for {self.symbol} are too old ({file_age:.1f}s), skipping") return # Load cached ticks tick_df = pd.read_csv(cache_file) if tick_df.empty: logger.info(f"Cached ticks file for {self.symbol} is empty") return # Convert to list of dicts and add to storage cached_ticks = tick_df.to_dict('records') self.ticks.extend(cached_ticks) logger.info(f"Loaded {len(cached_ticks)} cached ticks for {self.symbol} from {cache_file}") except Exception as e: logger.error(f"Error loading cached ticks for {self.symbol}: {str(e)}") import traceback logger.error(traceback.format_exc()) def get_latest_price(self) -> Optional[float]: """Get the latest price from the most recent tick""" if self.last_tick: return self.last_tick.get('price') elif self.ticks: # If last_tick not available but ticks exist, use the last tick self.last_tick = self.ticks[-1] return self.last_tick.get('price') return None def get_price_stats(self) -> Dict: """Get stats about the prices in storage""" if not self.ticks: return { 'min': None, 'max': None, 'latest': None, 'count': 0, 'age_seconds': 0 } prices = [tick['price'] for tick in self.ticks] latest_timestamp = self.ticks[-1]['timestamp'] oldest_timestamp = self.ticks[0]['timestamp'] return { 'min': min(prices), 'max': max(prices), 'latest': prices[-1], 'count': len(prices), 'age_seconds': (latest_timestamp - oldest_timestamp) / 1000 } def get_ticks_as_df(self) -> pd.DataFrame: """Return ticks as a DataFrame""" if not self.ticks: logger.warning("No ticks available for DataFrame conversion") return pd.DataFrame() # Ensure we have fresh data self._cleanup() df = pd.DataFrame(self.ticks) if not df.empty: logger.debug(f"Converting timestamps for {len(df)} ticks") # Ensure timestamp column exists if 'timestamp' not in df.columns: logger.error("Tick data missing timestamp column") return pd.DataFrame() # Check timestamp datatype before conversion sample_ts = df['timestamp'].iloc[0] if len(df) > 0 else None logger.debug(f"Sample timestamp before conversion: {sample_ts}, type: {type(sample_ts)}") # Convert timestamps to datetime try: df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms') logger.debug(f"Timestamps converted to datetime successfully") if len(df) > 0: logger.debug(f"Sample converted timestamp: {df['timestamp'].iloc[0]}") except Exception as e: logger.error(f"Error converting timestamps: {str(e)}") import traceback logger.error(traceback.format_exc()) return pd.DataFrame() return df def get_candles(self, interval_seconds: int = 1, start_time_ms: int = None, end_time_ms: int = None) -> pd.DataFrame: """Convert ticks to OHLCV candles at specified interval with optional time range filtering Args: interval_seconds: The interval in seconds for each candle start_time_ms: Optional start time in milliseconds for filtering end_time_ms: Optional end time in milliseconds for filtering Returns: DataFrame with OHLCV candles """ try: if not self.ticks: logger.warning("No ticks available for candle formation") return pd.DataFrame() # Ensure ticks are up to date try: self._cleanup() except Exception as cleanup_error: logger.error(f"Error cleaning up ticks: {str(cleanup_error)}") # Continue with existing ticks # Get ticks from specified time range if provided if start_time_ms is not None or end_time_ms is not None: logger.debug(f"Filtering ticks for time range from {start_time_ms} to {end_time_ms}") try: filtered_ticks = self.get_ticks_from_time(start_time_ms, end_time_ms) if not filtered_ticks: logger.warning("No ticks in the specified time range") return pd.DataFrame() df = pd.DataFrame(filtered_ticks) except Exception as filter_error: logger.error(f"Error filtering ticks by time: {str(filter_error)}") return pd.DataFrame() else: # Use all available ticks try: df = self.get_ticks_as_df() except Exception as df_error: logger.error(f"Error getting ticks as DataFrame: {str(df_error)}") return pd.DataFrame() if df.empty: logger.warning("Tick DataFrame is empty after filtering/conversion") return pd.DataFrame() logger.info(f"Preparing to create candles from {len(df)} ticks with {interval_seconds}s interval") # First, ensure all required columns exist required_columns = ['timestamp', 'price', 'volume'] for col in required_columns: if col not in df.columns: logger.error(f"Required column '{col}' missing from tick data") return pd.DataFrame() # Make sure DataFrame has no duplicated timestamps before setting index try: if 'timestamp' in df.columns: # Check for duplicate timestamps duplicate_count = df['timestamp'].duplicated().sum() if duplicate_count > 0: logger.warning(f"Found {duplicate_count} duplicate timestamps, keeping the last occurrence") # Keep the last occurrence of each timestamp df = df.drop_duplicates(subset='timestamp', keep='last') # Use timestamp column for resampling df = df.set_index('timestamp') # Make sure index is sorted (required for resampling) if not df.index.is_monotonic_increasing: logger.warning("Timestamp index is not monotonically increasing, sorting...") df = df.sort_index() except Exception as prep_error: logger.error(f"Error preprocessing DataFrame for resampling: {str(prep_error)}") import traceback logger.error(traceback.format_exc()) return pd.DataFrame() # Create interval string for resampling - use 's' instead of deprecated 'S' interval_str = f'{interval_seconds}s' # Resample to create OHLCV candles with multiple fallback options logger.debug(f"Resampling with interval: {interval_str}") candles = None # First attempt - individual column resampling try: # Check that price column exists and has enough data if 'price' not in df.columns: raise ValueError("Price column missing from DataFrame") if len(df) < 2: logger.warning("Not enough data points for resampling, using direct data") # For single data point, create a single candle if len(df) == 1: price_val = df['price'].iloc[0] volume_val = df['volume'].iloc[0] if 'volume' in df.columns else 0 timestamp_val = df.index[0] candles = pd.DataFrame({ 'timestamp': [timestamp_val], 'open': [price_val], 'high': [price_val], 'low': [price_val], 'close': [price_val], 'volume': [volume_val] }) return candles else: # No data return pd.DataFrame() # Resample and aggregate each column separately open_df = df['price'].resample(interval_str).first() high_df = df['price'].resample(interval_str).max() low_df = df['price'].resample(interval_str).min() close_df = df['price'].resample(interval_str).last() volume_df = df['volume'].resample(interval_str).sum() # Check for length mismatches before combining expected_length = len(open_df) if (len(high_df) != expected_length or len(low_df) != expected_length or len(close_df) != expected_length or len(volume_df) != expected_length): logger.warning("Length mismatch in resampled columns, falling back to alternative method") raise ValueError("Length mismatch") # Combine into a single DataFrame candles = pd.DataFrame({ 'open': open_df, 'high': high_df, 'low': low_df, 'close': close_df, 'volume': volume_df }) logger.debug(f"Successfully created {len(candles)} candles with individual column resampling") except Exception as resample_error: logger.error(f"Error in individual column resampling: {str(resample_error)}") # Second attempt - built-in agg method try: logger.debug("Trying fallback resampling method with agg()") candles = df.resample(interval_str).agg({ 'price': ['first', 'max', 'min', 'last'], 'volume': 'sum' }) # Flatten MultiIndex columns candles.columns = ['open', 'high', 'low', 'close', 'volume'] logger.debug(f"Successfully created {len(candles)} candles with agg() method") except Exception as agg_error: logger.error(f"Error in agg() resampling: {str(agg_error)}") # Third attempt - manual candle construction try: logger.debug("Trying manual candle construction method") resampler = df.resample(interval_str) candle_data = [] for name, group in resampler: if not group.empty: candle = { 'timestamp': name, 'open': group['price'].iloc[0], 'high': group['price'].max(), 'low': group['price'].min(), 'close': group['price'].iloc[-1], 'volume': group['volume'].sum() if 'volume' in group.columns else 0 } candle_data.append(candle) if candle_data: candles = pd.DataFrame(candle_data) logger.debug(f"Successfully created {len(candles)} candles with manual method") else: logger.warning("No candles created with manual method") return pd.DataFrame() except Exception as manual_error: logger.error(f"Error in manual candle construction: {str(manual_error)}") import traceback logger.error(traceback.format_exc()) return pd.DataFrame() # Ensure the result isn't empty if candles is None or candles.empty: logger.warning("No candles were created after all resampling attempts") return pd.DataFrame() # Reset index to get timestamp as column try: candles = candles.reset_index() except Exception as reset_error: logger.error(f"Error resetting index: {str(reset_error)}") # Try to create a new DataFrame with the timestamp index as a column try: timestamp_col = candles.index.to_list() candles_dict = candles.to_dict('list') candles_dict['timestamp'] = timestamp_col candles = pd.DataFrame(candles_dict) except Exception as fallback_error: logger.error(f"Error in fallback index reset: {str(fallback_error)}") return pd.DataFrame() # Ensure no NaN values try: nan_count_before = candles.isna().sum().sum() if nan_count_before > 0: logger.warning(f"Found {nan_count_before} NaN values in candles, dropping them") candles = candles.dropna() except Exception as nan_error: logger.error(f"Error handling NaN values: {str(nan_error)}") # Try to fill NaN values instead of dropping try: candles = candles.fillna(method='ffill').fillna(method='bfill') except: pass logger.debug(f"Generated {len(candles)} candles from {len(df)} ticks") return candles except Exception as e: logger.error(f"Unhandled error in candle formation: {str(e)}") import traceback logger.error(traceback.format_exc()) return pd.DataFrame() def get_candle_stats(self) -> Dict: """Get statistics about cached candles for different intervals""" stats = {} # Define intervals to check intervals = [1, 5, 15, 60, 300, 900, 3600] for interval in intervals: candles = self.get_candles(interval_seconds=interval) count = len(candles) if not candles.empty else 0 # Get time range if we have candles time_range = None if count > 0: try: start_time = candles['timestamp'].min() end_time = candles['timestamp'].max() if isinstance(start_time, pd.Timestamp): start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') if isinstance(end_time, pd.Timestamp): end_time = end_time.strftime('%Y-%m-%d %H:%M:%S') time_range = f"{start_time} to {end_time}" except: time_range = "Unknown" stats[f"{interval}s"] = { 'count': count, 'time_range': time_range } return stats def get_ticks_from_time(self, start_time_ms: int = None, end_time_ms: int = None) -> List[Dict]: """Get ticks within a specific time range Args: start_time_ms: Start time in milliseconds (None for no lower bound) end_time_ms: End time in milliseconds (None for no upper bound) Returns: List of ticks within the time range """ if not self.ticks: return [] # Ensure ticks are updated self._cleanup() # Apply time filters if specified filtered_ticks = self.ticks if start_time_ms is not None: filtered_ticks = [tick for tick in filtered_ticks if tick['timestamp'] >= start_time_ms] if end_time_ms is not None: filtered_ticks = [tick for tick in filtered_ticks if tick['timestamp'] <= end_time_ms] logger.debug(f"Retrieved {len(filtered_ticks)} ticks from time range {start_time_ms} to {end_time_ms}") return filtered_ticks def get_time_based_stats(self) -> Dict: """Get statistics about the ticks organized by time periods Returns: Dictionary with statistics for different time periods """ if not self.ticks: return { 'total_ticks': 0, 'periods': {} } # Ensure ticks are updated self._cleanup() now = int(time.time() * 1000) # Current time in ms # Define time periods to analyze periods = { '1min': now - (60 * 1000), '5min': now - (5 * 60 * 1000), '15min': now - (15 * 60 * 1000), '30min': now - (30 * 60 * 1000) } stats = { 'total_ticks': len(self.ticks), 'oldest_tick': self.ticks[0]['timestamp'] if self.ticks else None, 'newest_tick': self.ticks[-1]['timestamp'] if self.ticks else None, 'time_span_seconds': (self.ticks[-1]['timestamp'] - self.ticks[0]['timestamp']) / 1000 if self.ticks else 0, 'periods': {} } # Calculate stats for each period for period_name, cutoff_time in periods.items(): period_ticks = [tick for tick in self.ticks if tick['timestamp'] >= cutoff_time] if period_ticks: prices = [tick['price'] for tick in period_ticks] volumes = [tick.get('volume', 0) for tick in period_ticks] period_stats = { 'tick_count': len(period_ticks), 'min_price': min(prices) if prices else None, 'max_price': max(prices) if prices else None, 'avg_price': sum(prices) / len(prices) if prices else None, 'last_price': period_ticks[-1]['price'] if period_ticks else None, 'total_volume': sum(volumes), 'ticks_per_second': len(period_ticks) / (int(period_name[:-3]) * 60) if period_ticks else 0 } stats['periods'][period_name] = period_stats logger.debug(f"Generated time-based stats: {len(stats['periods'])} periods") return stats class CandlestickData: def __init__(self, max_length: int = 300): self.timestamps = deque(maxlen=max_length) self.opens = deque(maxlen=max_length) self.highs = deque(maxlen=max_length) self.lows = deque(maxlen=max_length) self.closes = deque(maxlen=max_length) self.volumes = deque(maxlen=max_length) self.current_candle = { 'timestamp': None, 'open': None, 'high': None, 'low': None, 'close': None, 'volume': 0 } self.candle_interval = 1 # 1 second by default def update_from_trade(self, trade: Dict): timestamp = trade['timestamp'] price = trade['price'] volume = trade.get('volume', 0) # Round timestamp to nearest candle interval candle_timestamp = int(timestamp / (self.candle_interval * 1000)) * (self.candle_interval * 1000) if self.current_candle['timestamp'] != candle_timestamp: # Save current candle if it exists if self.current_candle['timestamp'] is not None: self.timestamps.append(self.current_candle['timestamp']) self.opens.append(self.current_candle['open']) self.highs.append(self.current_candle['high']) self.lows.append(self.current_candle['low']) self.closes.append(self.current_candle['close']) self.volumes.append(self.current_candle['volume']) logger.debug(f"New candle saved: {self.current_candle}") # Start new candle self.current_candle = { 'timestamp': candle_timestamp, 'open': price, 'high': price, 'low': price, 'close': price, 'volume': volume } logger.debug(f"New candle started: {self.current_candle}") else: # Update current candle if self.current_candle['high'] is None or price > self.current_candle['high']: self.current_candle['high'] = price if self.current_candle['low'] is None or price < self.current_candle['low']: self.current_candle['low'] = price self.current_candle['close'] = price self.current_candle['volume'] += volume logger.debug(f"Updated current candle: {self.current_candle}") def get_dataframe(self) -> pd.DataFrame: # Include current candle in the dataframe if it exists timestamps = list(self.timestamps) opens = list(self.opens) highs = list(self.highs) lows = list(self.lows) closes = list(self.closes) volumes = list(self.volumes) if self.current_candle['timestamp'] is not None: timestamps.append(self.current_candle['timestamp']) opens.append(self.current_candle['open']) highs.append(self.current_candle['high']) lows.append(self.current_candle['low']) closes.append(self.current_candle['close']) volumes.append(self.current_candle['volume']) df = pd.DataFrame({ 'timestamp': timestamps, 'open': opens, 'high': highs, 'low': lows, 'close': closes, 'volume': volumes }) if not df.empty: df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms') return df class BinanceWebSocket: """Binance WebSocket implementation for real-time tick data""" def __init__(self, symbol: str): self.symbol = symbol.replace('/', '').lower() self.ws = None self.running = False self.reconnect_delay = 1 self.max_reconnect_delay = 60 self.message_count = 0 # Binance WebSocket configuration self.ws_url = f"wss://stream.binance.com:9443/ws/{self.symbol}@trade" logger.info(f"Initialized Binance WebSocket for symbol: {self.symbol}") async def connect(self): while True: try: logger.info(f"Attempting to connect to {self.ws_url}") self.ws = await websockets.connect(self.ws_url) logger.info("WebSocket connection established") self.running = True self.reconnect_delay = 1 logger.info(f"Successfully connected to Binance WebSocket for {self.symbol}") return True except Exception as e: logger.error(f"WebSocket connection error: {str(e)}") await asyncio.sleep(self.reconnect_delay) self.reconnect_delay = min(self.reconnect_delay * 2, self.max_reconnect_delay) continue async def receive(self) -> Optional[Dict]: if not self.ws: return None try: message = await self.ws.recv() self.message_count += 1 if self.message_count % 100 == 0: # Log every 100th message to avoid spam logger.info(f"Received message #{self.message_count}") logger.debug(f"Raw message: {message[:200]}...") data = json.loads(message) # Process trade data if 'e' in data and data['e'] == 'trade': trade_data = { 'timestamp': data['T'], # Trade time 'price': float(data['p']), # Price 'volume': float(data['q']), # Quantity 'type': 'trade' } logger.debug(f"Processed trade data: {trade_data}") return trade_data return None except websockets.exceptions.ConnectionClosed: logger.warning("WebSocket connection closed") self.running = False return None except json.JSONDecodeError as e: logger.error(f"JSON decode error: {str(e)}, message: {message[:200]}...") return None except Exception as e: logger.error(f"Error receiving message: {str(e)}") return None async def close(self): """Close the WebSocket connection""" if self.ws: await self.ws.close() self.running = False logger.info("WebSocket connection closed") class BinanceHistoricalData: """Fetch historical candle data from Binance""" def __init__(self): self.base_url = "https://api.binance.com/api/v3/klines" # Create a cache directory if it doesn't exist self.cache_dir = os.path.join(os.getcwd(), "cache") os.makedirs(self.cache_dir, exist_ok=True) logger.info(f"Initialized BinanceHistoricalData with cache directory: {self.cache_dir}") def _get_interval_string(self, interval_seconds: int) -> str: """Convert interval seconds to Binance interval string""" if interval_seconds == 60: # 1m return "1m" elif interval_seconds == 3600: # 1h return "1h" elif interval_seconds == 86400: # 1d return "1d" else: # Default to 1m if not recognized logger.warning(f"Unrecognized interval {interval_seconds}s, defaulting to 1m") return "1m" def _get_cache_filename(self, symbol: str, interval: str) -> str: """Generate cache filename for the symbol and interval""" # Replace any slashes in symbol with underscore safe_symbol = symbol.replace("/", "_") return os.path.join(self.cache_dir, f"{safe_symbol}_{interval}_candles.csv") def _load_from_cache(self, symbol: str, interval: str) -> Optional[pd.DataFrame]: """Load candle data from cache if available and not expired""" filename = self._get_cache_filename(symbol, interval) if not os.path.exists(filename): logger.debug(f"No cache file found for {symbol} {interval}") return None # Check if cache is fresh (less than 1 hour old for anything but 1d, 1 day for 1d) file_age = time.time() - os.path.getmtime(filename) max_age = 86400 if interval == "1d" else 3600 # 1 day for 1d, 1 hour for others if file_age > max_age: logger.debug(f"Cache for {symbol} {interval} is expired ({file_age:.1f}s old)") return None try: df = pd.read_csv(filename) # Convert timestamp string back to datetime df['timestamp'] = pd.to_datetime(df['timestamp']) logger.info(f"Loaded {len(df)} candles from cache for {symbol} {interval}") return df except Exception as e: logger.error(f"Error loading from cache: {str(e)}") return None def _save_to_cache(self, df: pd.DataFrame, symbol: str, interval: str) -> bool: """Save candle data to cache""" if df.empty: logger.warning(f"No data to cache for {symbol} {interval}") return False filename = self._get_cache_filename(symbol, interval) try: df.to_csv(filename, index=False) logger.info(f"Cached {len(df)} candles for {symbol} {interval} to {filename}") return True except Exception as e: logger.error(f"Error saving to cache: {str(e)}") return False def get_historical_candles(self, symbol: str, interval_seconds: int, limit: int = 500) -> pd.DataFrame: """Get historical candle data for the specified symbol and interval""" # Convert to Binance format clean_symbol = symbol.replace("/", "") interval = self._get_interval_string(interval_seconds) # Try to load from cache first cached_data = self._load_from_cache(symbol, interval) if cached_data is not None and len(cached_data) >= limit: return cached_data.tail(limit) # Fetch from API if not cached or insufficient try: logger.info(f"Fetching {limit} historical candles for {symbol} ({interval}) from Binance API") params = { "symbol": clean_symbol, "interval": interval, "limit": limit } response = requests.get(self.base_url, params=params) response.raise_for_status() # Raise exception for HTTP errors # Process the data candles = response.json() if not candles: logger.warning(f"No candles returned from Binance for {symbol} {interval}") return pd.DataFrame() # Convert to DataFrame - Binance returns data in this format: # [ # [ # 1499040000000, // Open time # "0.01634790", // Open # "0.80000000", // High # "0.01575800", // Low # "0.01577100", // Close # "148976.11427815", // Volume # ... // Ignore the rest # ], # ... # ] df = pd.DataFrame(candles, columns=[ "timestamp", "open", "high", "low", "close", "volume", "close_time", "quote_asset_volume", "number_of_trades", "taker_buy_base_asset_volume", "taker_buy_quote_asset_volume", "ignore" ]) # Convert types df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms') for col in ["open", "high", "low", "close", "volume"]: df[col] = df[col].astype(float) # Keep only needed columns df = df[["timestamp", "open", "high", "low", "close", "volume"]] # Cache the results self._save_to_cache(df, symbol, interval) logger.info(f"Successfully fetched {len(df)} candles for {symbol} {interval}") return df except Exception as e: logger.error(f"Error fetching historical data for {symbol} {interval}: {str(e)}") import traceback logger.error(traceback.format_exc()) return pd.DataFrame() class ExchangeWebSocket: """Generic WebSocket interface for cryptocurrency exchanges""" def __init__(self, symbol: str, exchange: str = "binance"): self.symbol = symbol self.exchange = exchange.lower() self.ws = None # Initialize the appropriate WebSocket implementation if self.exchange == "binance": self.ws = BinanceWebSocket(symbol) elif self.exchange == "mexc": self.ws = MEXCWebSocket(symbol) else: raise ValueError(f"Unsupported exchange: {exchange}") async def connect(self): """Connect to the exchange WebSocket""" return await self.ws.connect() async def receive(self) -> Optional[Dict]: """Receive data from the WebSocket""" return await self.ws.receive() async def close(self): """Close the WebSocket connection""" await self.ws.close() @property def running(self): """Check if the WebSocket is running""" return self.ws.running if self.ws else False class CandleCache: def __init__(self, max_candles: int = 5000): self.candles = { '1s': deque(maxlen=max_candles), '1m': deque(maxlen=max_candles), '1h': deque(maxlen=max_candles), '1d': deque(maxlen=max_candles) } logger.info(f"Initialized CandleCache with max candles: {max_candles}") def add_candles(self, interval: str, new_candles: pd.DataFrame): if interval in self.candles and not new_candles.empty: # Convert DataFrame to list of dicts to avoid pandas issues for _, row in new_candles.iterrows(): candle_dict = row.to_dict() self.candles[interval].append(candle_dict) logger.debug(f"Added {len(new_candles)} candles to {interval} cache") def get_recent_candles(self, interval: str, count: int = 500) -> pd.DataFrame: if interval in self.candles and self.candles[interval]: # Convert deque to list of dicts first all_candles = list(self.candles[interval]) # Check if we're requesting more candles than we have if count > len(all_candles): logger.debug(f"Requested {count} candles, but only have {len(all_candles)} for {interval}") count = len(all_candles) recent_candles = all_candles[-count:] logger.debug(f"Returning {len(recent_candles)} recent candles for {interval} (requested {count})") # Create DataFrame and ensure timestamp is datetime type df = pd.DataFrame(recent_candles) if not df.empty and 'timestamp' in df.columns: try: if not pd.api.types.is_datetime64_any_dtype(df['timestamp']): df['timestamp'] = pd.to_datetime(df['timestamp']) except Exception as e: logger.warning(f"Error converting timestamps in get_recent_candles: {str(e)}") return df logger.debug(f"No candles available for {interval}") return pd.DataFrame() def update_cache(self, interval: str, new_candles: pd.DataFrame): """ Update the candle cache for a specific timeframe with new candles Args: interval: The timeframe interval ('1s', '1m', '1h', '1d') new_candles: DataFrame with new candles to add to the cache """ if interval not in self.candles: logger.warning(f"Invalid interval {interval} for cache update") return if new_candles is None or new_candles.empty: logger.debug(f"No new candles to update {interval} cache") return # Check if timestamp column exists if 'timestamp' not in new_candles.columns: logger.warning(f"No timestamp column in new candles for {interval}") return try: # Ensure timestamp is datetime for proper comparison try: if not pd.api.types.is_datetime64_any_dtype(new_candles['timestamp']): logger.debug(f"Converting timestamps to datetime for {interval}") new_candles['timestamp'] = pd.to_datetime(new_candles['timestamp']) except Exception as e: logger.error(f"Error converting timestamps: {str(e)}") # Try a different approach try: new_candles['timestamp'] = pd.to_datetime(new_candles['timestamp'], errors='coerce') # Drop any rows where conversion failed new_candles = new_candles.dropna(subset=['timestamp']) if new_candles.empty: logger.warning(f"All timestamps conversion failed for {interval}") return except Exception as e2: logger.error(f"Second attempt to convert timestamps failed: {str(e2)}") return # Create a copy to avoid modifying the original new_candles_copy = new_candles.copy() # If we have no candles in cache, add all new candles if not self.candles[interval]: logger.debug(f"No existing candles for {interval}, adding all {len(new_candles_copy)} candles") self.add_candles(interval, new_candles_copy) return # Get the timestamp from the last cached candle last_cached_candle = self.candles[interval][-1] if not isinstance(last_cached_candle, dict): logger.warning(f"Last cached candle is not a dictionary for {interval}") last_cached_candle = {'timestamp': None} if 'timestamp' not in last_cached_candle: logger.warning(f"No timestamp in last cached candle for {interval}") last_cached_candle['timestamp'] = None last_cached_time = last_cached_candle['timestamp'] logger.debug(f"Last cached timestamp for {interval}: {last_cached_time}") # If last_cached_time is None, add all candles if last_cached_time is None: logger.debug(f"No valid last cached timestamp, adding all {len(new_candles_copy)} candles for {interval}") self.add_candles(interval, new_candles_copy) return # Convert last_cached_time to datetime if needed if not isinstance(last_cached_time, (pd.Timestamp, datetime)): try: last_cached_time = pd.to_datetime(last_cached_time) except Exception as e: logger.error(f"Cannot convert last cached time to datetime: {str(e)}") # Add all candles as fallback self.add_candles(interval, new_candles_copy) return # Make a backup of current cache before filtering cache_backup = list(self.candles[interval]) # Filter new candles that are after the last cached candle try: filtered_candles = new_candles_copy[new_candles_copy['timestamp'] > last_cached_time] if not filtered_candles.empty: logger.debug(f"Adding {len(filtered_candles)} new candles for {interval}") self.add_candles(interval, filtered_candles) else: # No new candles after last cached time, check for missing candles try: # Get unique timestamps in cache cached_timestamps = set() for candle in self.candles[interval]: if isinstance(candle, dict) and 'timestamp' in candle: ts = candle['timestamp'] if isinstance(ts, (pd.Timestamp, datetime)): cached_timestamps.add(ts) else: try: cached_timestamps.add(pd.to_datetime(ts)) except: pass # Find candles in new_candles that aren't in the cache missing_candles = new_candles_copy[~new_candles_copy['timestamp'].isin(cached_timestamps)] if not missing_candles.empty: logger.info(f"Found {len(missing_candles)} missing candles for {interval}") self.add_candles(interval, missing_candles) else: logger.debug(f"No new or missing candles to add for {interval}") except Exception as missing_error: logger.error(f"Error checking for missing candles: {str(missing_error)}") except Exception as filter_error: logger.error(f"Error filtering candles by timestamp: {str(filter_error)}") # Restore from backup self.candles[interval] = deque(cache_backup, maxlen=self.candles[interval].maxlen) # Try adding all candles as fallback self.add_candles(interval, new_candles_copy) except Exception as e: logger.error(f"Unhandled error updating cache for {interval}: {str(e)}") import traceback logger.error(traceback.format_exc()) def get_candles(self, timeframe: str, count: int = 500) -> pd.DataFrame: """ Get candles for a specific timeframe. This is an alias for get_recent_candles to maintain compatibility with code that expects this method name. Args: timeframe: The timeframe interval ('1s', '1m', '1h', '1d') count: Maximum number of candles to return Returns: DataFrame containing the candles """ try: logger.debug(f"Getting {count} candles for {timeframe} via get_candles()") return self.get_recent_candles(timeframe, count) except Exception as e: logger.error(f"Error in get_candles for {timeframe}: {str(e)}") import traceback logger.error(traceback.format_exc()) return pd.DataFrame() class RealTimeChart: """Real-time chart using Dash and Plotly with WebSocket data feed""" def __init__(self, symbol: str): """Initialize the chart with necessary components""" self.symbol = symbol # Create a multi-page Dash app instead of a simple Dash app self.app = dash.Dash(__name__, suppress_callback_exceptions=True, meta_tags=[{"name": "viewport", "content": "width=device-width, initial-scale=1"}]) self.candlestick_data = CandlestickData() self.tick_storage = TradeTickStorage(symbol=symbol, max_age_seconds=3600) # Store 1 hour of ticks self.ohlcv_cache = { # Cache for different intervals '1s': None, '1m': None, '1h': None, '1d': None } self.candle_cache = CandleCache(max_candles=5000) # Increase max candles to 5000 for better history self.historical_data = BinanceHistoricalData() # For fetching historical data self.last_cache_save_time = time.time() # Track last time we saved cache to disk self.first_render = True # Flag to track first render # Storage for NN signals self.nn_signals = [] logger.info(f"Initializing RealTimeChart for {symbol}") # Load historical data for longer timeframes at startup self._load_historical_data() # Setup the multi-page layout self._setup_app_layout() # Save the loaded cache immediately to ensure we have the data saved self._save_candles_to_disk(force=True) def _setup_app_layout(self): # Button style button_style = { 'background-color': '#4CAF50', 'color': 'white', 'padding': '10px 15px', 'margin': '5px', 'border': 'none', 'border-radius': '5px', 'font-size': '14px', 'cursor': 'pointer', 'transition': 'background-color 0.3s', 'font-weight': 'bold' } active_button_style = { **button_style, 'background-color': '#2E7D32', 'box-shadow': '0 0 5px #2E7D32' } nav_button_style = { 'background-color': '#3f51b5', 'color': 'white', 'padding': '10px 15px', 'margin': '5px', 'border': 'none', 'border-radius': '5px', 'font-size': '14px', 'cursor': 'pointer', 'font-weight': 'bold' } # Content div for the current page content_div = html.Div(id='page-content') # Initialize the layout with navigation self.app.layout = html.Div([ # Header with symbol and title html.Div([ html.H1(f"{self.symbol} Real-Time Data", style={ 'textAlign': 'center', 'color': '#FFFFFF', 'fontFamily': 'Arial, sans-serif', 'margin': '10px', 'textShadow': '2px 2px 4px #000000' }), # Navigation bar html.Div([ dcc.Link( html.Button('Price Chart', style=nav_button_style), href=f'/{self.symbol.replace("/", "-")}/chart', id='chart-link' ), dcc.Link( html.Button('Raw Ticks', style=nav_button_style), href=f'/{self.symbol.replace("/", "-")}/ticks', id='ticks-link' ), ], style={ 'display': 'flex', 'justifyContent': 'center', 'margin': '10px' }), ], style={ 'backgroundColor': '#1E1E1E', 'padding': '10px', 'borderRadius': '5px', 'marginBottom': '10px', 'boxShadow': '0 4px 8px 0 rgba(0,0,0,0.2)' }), # URL Bar dcc.Location(id='url', refresh=False), # Content div will be populated based on the URL content_div, # Interval component for periodic updates dcc.Interval( id='interval-component', interval=500, # Update every 500ms for smoother display n_intervals=0 ) ], style={ 'backgroundColor': '#121212', 'padding': '20px', 'minHeight': '100vh', 'fontFamily': 'Arial, sans-serif' }) # Register URL callback to update page content @self.app.callback( Output('page-content', 'children'), [Input('url', 'pathname')] ) def display_page(pathname): if pathname is None: pathname = f'/{self.symbol.replace("/", "-")}/chart' symbol_path = self.symbol.replace("/", "-") if pathname == f'/{symbol_path}/chart' or pathname == f'/' or pathname == f'/{symbol_path}': return self._get_chart_layout(button_style, active_button_style) elif pathname == f'/{symbol_path}/ticks': return self._get_ticks_layout() else: return html.Div([ html.H1('404 - Page Not Found', style={'textAlign': 'center', 'color': 'white'}) ]) # Register callback to update the interval selection from button clicks self._setup_interval_callback(button_style, active_button_style) # Register callback to update the chart data self._setup_chart_callback() # Register callback to update the ticks data self._setup_ticks_callback() def _get_chart_layout(self, button_style, active_button_style): # Chart page layout return html.Div([ # Interval selection buttons html.Div([ html.Div("Candlestick Interval:", style={ 'color': '#FFFFFF', 'marginRight': '10px', 'fontSize': '16px', 'fontWeight': 'bold' }), html.Button('1s', id='btn-1s', n_clicks=0, style=active_button_style), html.Button('5s', id='btn-5s', n_clicks=0, style=button_style), html.Button('15s', id='btn-15s', n_clicks=0, style=button_style), html.Button('30s', id='btn-30s', n_clicks=0, style=button_style), html.Button('1m', id='btn-1m', n_clicks=0, style=button_style), ], style={ 'display': 'flex', 'alignItems': 'center', 'justifyContent': 'center', 'margin': '15px', 'backgroundColor': '#2C2C2C', 'padding': '10px', 'borderRadius': '5px' }), # Store for current interval dcc.Store(id='interval-store', data={'interval': 1}), # Main chart dcc.Graph( id='live-chart', style={ 'height': '180vh', 'border': '1px solid #444444', 'borderRadius': '5px', 'boxShadow': '0 4px 8px 0 rgba(0,0,0,0.2)' } ), ]) def _get_ticks_layout(self): # Ticks data page layout return html.Div([ # Header and controls html.Div([ html.H2(f"{self.symbol} Raw Tick Data (Last 5 Minutes)", style={ 'textAlign': 'center', 'color': '#FFFFFF', 'margin': '10px 0' }), # Refresh button html.Button('Refresh Data', id='refresh-ticks-btn', n_clicks=0, style={ 'backgroundColor': '#4CAF50', 'color': 'white', 'padding': '10px 20px', 'margin': '10px auto', 'border': 'none', 'borderRadius': '5px', 'fontSize': '14px', 'cursor': 'pointer', 'display': 'block' }), # Time window selector html.Div([ html.Label("Time Window:", style={'color': 'white', 'marginRight': '10px'}), dcc.Dropdown( id='time-window-dropdown', options=[ {'label': 'Last 1 minute', 'value': 60}, {'label': 'Last 5 minutes', 'value': 300}, {'label': 'Last 15 minutes', 'value': 900}, {'label': 'Last 30 minutes', 'value': 1800}, ], value=300, # Default to 5 minutes style={'width': '200px', 'backgroundColor': '#2C2C2C', 'color': 'black'} ) ], style={ 'display': 'flex', 'alignItems': 'center', 'justifyContent': 'center', 'margin': '10px' }), ], style={ 'backgroundColor': '#2C2C2C', 'padding': '10px', 'borderRadius': '5px', 'marginBottom': '15px' }), # Stats cards html.Div(id='tick-stats-cards', style={ 'display': 'flex', 'flexWrap': 'wrap', 'justifyContent': 'space-around', 'marginBottom': '15px' }), # Ticks data table html.Div(id='ticks-table-container', style={ 'backgroundColor': '#232323', 'padding': '10px', 'borderRadius': '5px', 'overflowX': 'auto' }), # Price movement chart html.Div([ html.H3("Price Movement", style={ 'textAlign': 'center', 'color': '#FFFFFF', 'margin': '10px 0' }), dcc.Graph(id='tick-price-chart') ], style={ 'backgroundColor': '#232323', 'padding': '10px', 'borderRadius': '5px', 'marginTop': '15px' }) ]) def _setup_interval_callback(self, button_style, active_button_style): # Callback to update interval based on button clicks and update button styles @self.app.callback( [Output('interval-store', 'data'), Output('btn-1s', 'style'), Output('btn-5s', 'style'), Output('btn-15s', 'style'), Output('btn-30s', 'style'), Output('btn-1m', 'style')], [Input('btn-1s', 'n_clicks'), Input('btn-5s', 'n_clicks'), Input('btn-15s', 'n_clicks'), Input('btn-30s', 'n_clicks'), Input('btn-1m', 'n_clicks')], [dash.dependencies.State('interval-store', 'data')] ) def update_interval(n1, n5, n15, n30, n60, data): ctx = dash.callback_context if not ctx.triggered: # Default state (1s selected) return ({'interval': 1}, active_button_style, button_style, button_style, button_style, button_style) button_id = ctx.triggered[0]['prop_id'].split('.')[0] if button_id == 'btn-1s': return ({'interval': 1}, active_button_style, button_style, button_style, button_style, button_style) elif button_id == 'btn-5s': return ({'interval': 5}, button_style, active_button_style, button_style, button_style, button_style) elif button_id == 'btn-15s': return ({'interval': 15}, button_style, button_style, active_button_style, button_style, button_style) elif button_id == 'btn-30s': return ({'interval': 30}, button_style, button_style, button_style, active_button_style, button_style) elif button_id == 'btn-1m': return ({'interval': 60}, button_style, button_style, button_style, button_style, active_button_style) # Default case - keep current interval and highlight appropriate button current_interval = data.get('interval', 1) styles = [button_style] * 5 # All inactive by default # Set active style based on current interval if current_interval == 1: styles[0] = active_button_style elif current_interval == 5: styles[1] = active_button_style elif current_interval == 15: styles[2] = active_button_style elif current_interval == 30: styles[3] = active_button_style elif current_interval == 60: styles[4] = active_button_style return (data, *styles) def _setup_chart_callback(self): # Callback to update the chart @self.app.callback( Output('live-chart', 'figure'), [Input('interval-component', 'n_intervals'), Input('interval-store', 'data')] ) def update_chart(n, interval_data): try: interval = interval_data.get('interval', 1) logger.debug(f"Updating chart for {self.symbol} with interval {interval}s") # First render flag - used to force cache loading is_first_render = self.first_render if is_first_render: logger.info("First render - ensuring cached data is loaded") self.first_render = False # Check if we have cached data for the requested interval cached_candles = None if interval == 1 and self.ohlcv_cache['1s'] is not None and not self.ohlcv_cache['1s'].empty: cached_candles = self.ohlcv_cache['1s'] logger.info(f"We have {len(cached_candles)} cached 1s candles available") elif interval == 60 and self.ohlcv_cache['1m'] is not None and not self.ohlcv_cache['1m'].empty: cached_candles = self.ohlcv_cache['1m'] logger.info(f"We have {len(cached_candles)} cached 1m candles available") # Get candlesticks from tick storage for real-time data try: df = self.tick_storage.get_candles(interval_seconds=interval) logger.debug(f"Got {len(df) if not df.empty else 0} candles from tick storage") except Exception as candle_error: logger.error(f"Error getting candles from tick storage: {str(candle_error)}") logger.error(f"Using empty DataFrame to avoid chart restart") df = pd.DataFrame() # If we don't have real-time data yet, use cached data if df.empty and cached_candles is not None and not cached_candles.empty: df = cached_candles logger.info(f"Using {len(df)} cached candles for main chart") # Get current price and stats using our enhanced methods try: current_price = self.tick_storage.get_latest_price() price_stats = self.tick_storage.get_price_stats() time_stats = self.tick_storage.get_time_based_stats() candle_stats = self.tick_storage.get_candle_stats() except Exception as stats_error: logger.error(f"Error getting market stats: {str(stats_error)}") # Provide fallback values to avoid chart restart current_price = None price_stats = {'count': 0, 'min': None, 'max': None, 'latest': None, 'age_seconds': 0} time_stats = {'total_ticks': 0, 'periods': {}} candle_stats = {} # Periodically save candles to disk if n % 60 == 0 or is_first_render: # Every 60 chart updates or on first render try: self._save_candles_to_disk() except Exception as save_error: logger.error(f"Error saving candles: {str(save_error)}") logger.debug(f"Current price: {current_price}, Stats: {price_stats}") # Create subplot layout - don't include 1s since that's the main chart try: fig = make_subplots( rows=5, cols=1, vertical_spacing=0.05, subplot_titles=(f'{self.symbol} Price ({interval}s)', 'Volume', '1m OHLCV', '1h OHLCV', '1d OHLCV'), row_heights=[0.2, 0.2, 0.2, 0.2, 0.2] ) except Exception as subplot_error: logger.error(f"Error creating subplot layout: {str(subplot_error)}") # Create a simpler layout if the complex one fails fig = make_subplots( rows=2, cols=1, vertical_spacing=0.05, subplot_titles=(f'{self.symbol} Price ({interval}s)', 'Volume'), row_heights=[0.7, 0.3] ) # Process and add main chart data try: # Main price chart if not df.empty: fig.add_trace( go.Candlestick( x=df.index, open=df['open'], high=df['high'], low=df['low'], close=df['close'], name='OHLC' ), row=1, col=1 ) # Volume chart - use red/green colors based on price movement colors = ['rgba(0,255,0,0.7)' if close >= open else 'rgba(255,0,0,0.7)' for open, close in zip(df['open'], df['close'])] fig.add_trace( go.Bar( x=df.index, y=df['volume'], name='Volume', marker_color=colors ), row=2, col=1 ) except Exception as render_error: logger.error(f"Error rendering main chart data: {str(render_error)}") # Add 1-minute candles in a separate subplot try: candles_1m = self.candle_cache.get_candles(timeframe='1m') if candles_1m is not None and not candles_1m.empty: logger.debug(f"Rendering 1m chart with {len(candles_1m)} candles") # Candlestick chart for 1m fig.add_trace( go.Candlestick( x=candles_1m.index, open=candles_1m['open'], high=candles_1m['high'], low=candles_1m['low'], close=candles_1m['close'], name='1m' ), row=3, col=1 ) # Volume bars for 1m - inline at the bottom of the chart # Determine colors based on price movement colors = ['rgba(0,255,0,0.5)' if close >= open else 'rgba(255,0,0,0.5)' for open, close in zip(candles_1m['open'], candles_1m['close'])] # Add volume as bars at the bottom of the same subplot # Scale down the volume to fit in the same chart if 'volume' in candles_1m.columns: # Get volume to 15% of the chart height volume_data = candles_1m['volume'] if len(volume_data) > 0: y_range = max(candles_1m['high']) - min(candles_1m['low']) if y_range > 0: scale_factor = 0.15 * y_range / max(volume_data) if max(volume_data) > 0 else 1 scaled_volume = volume_data * scale_factor min_price = min(candles_1m['low']) fig.add_trace( go.Bar( x=candles_1m.index, y=scaled_volume, marker_color=colors, name='1m Volume', opacity=0.5, yaxis='y3', base=min(candles_1m['low']) - (y_range * 0.02), # Slightly below price offset=0 ), row=3, col=1 ) # Add a note about the scaled volume fig.add_annotation( x=candles_1m.index[0], y=min_price, text=f"Volume (scaled)", showarrow=False, font=dict(size=10, color="gray"), xanchor="left", yanchor="bottom", row=3, col=1 ) except Exception as render_1m_error: logger.error(f"Error rendering 1m chart: {str(render_1m_error)}") # Add 1-hour candles in a separate subplot try: candles_1h = self.candle_cache.get_candles(timeframe='1h') if candles_1h is not None and not candles_1h.empty: logger.debug(f"Rendering 1h chart with {len(candles_1h)} candles") # Candlestick chart for 1h fig.add_trace( go.Candlestick( x=candles_1h.index, open=candles_1h['open'], high=candles_1h['high'], low=candles_1h['low'], close=candles_1h['close'], name='1h' ), row=4, col=1 ) # Volume bars for 1h - inline at the bottom of the chart # Determine colors based on price movement colors = ['rgba(0,255,0,0.5)' if close >= open else 'rgba(255,0,0,0.5)' for open, close in zip(candles_1h['open'], candles_1h['close'])] # Add volume as bars at the bottom of the same subplot if 'volume' in candles_1h.columns: # Get volume to 15% of the chart height volume_data = candles_1h['volume'] if len(volume_data) > 0: y_range = max(candles_1h['high']) - min(candles_1h['low']) if y_range > 0: scale_factor = 0.15 * y_range / max(volume_data) if max(volume_data) > 0 else 1 scaled_volume = volume_data * scale_factor min_price = min(candles_1h['low']) fig.add_trace( go.Bar( x=candles_1h.index, y=scaled_volume, marker_color=colors, name='1h Volume', opacity=0.5, yaxis='y4', base=min(candles_1h['low']) - (y_range * 0.02), offset=0 ), row=4, col=1 ) # Add a note about the scaled volume fig.add_annotation( x=candles_1h.index[0], y=min_price, text=f"Volume (scaled)", showarrow=False, font=dict(size=10, color="gray"), xanchor="left", yanchor="bottom", row=4, col=1 ) except Exception as render_1h_error: logger.error(f"Error rendering 1h chart: {str(render_1h_error)}") # Add 1-day candles in a separate subplot try: candles_1d = self.candle_cache.get_candles(timeframe='1d') if candles_1d is not None and not candles_1d.empty: logger.debug(f"Rendering 1d chart with {len(candles_1d)} candles") # Candlestick chart for 1d fig.add_trace( go.Candlestick( x=candles_1d.index, open=candles_1d['open'], high=candles_1d['high'], low=candles_1d['low'], close=candles_1d['close'], name='1d' ), row=5, col=1 ) # Volume bars for 1d - inline at the bottom of the chart # Determine colors based on price movement colors = ['rgba(0,255,0,0.5)' if close >= open else 'rgba(255,0,0,0.5)' for open, close in zip(candles_1d['open'], candles_1d['close'])] # Add volume as bars at the bottom of the same subplot if 'volume' in candles_1d.columns: # Get volume to 15% of the chart height volume_data = candles_1d['volume'] if len(volume_data) > 0: y_range = max(candles_1d['high']) - min(candles_1d['low']) if y_range > 0: scale_factor = 0.15 * y_range / max(volume_data) if max(volume_data) > 0 else 1 scaled_volume = volume_data * scale_factor min_price = min(candles_1d['low']) fig.add_trace( go.Bar( x=candles_1d.index, y=scaled_volume, marker_color=colors, name='1d Volume', opacity=0.5, yaxis='y5', base=min(candles_1d['low']) - (y_range * 0.02), offset=0 ), row=5, col=1 ) # Add a note about the scaled volume fig.add_annotation( x=candles_1d.index[0], y=min_price, text=f"Volume (scaled)", showarrow=False, font=dict(size=10, color="gray"), xanchor="left", yanchor="bottom", row=5, col=1 ) except Exception as render_1d_error: logger.error(f"Error rendering 1d chart: {str(render_1d_error)}") # Update layout try: interval_text = { 1: "1 second", 5: "5 seconds", 15: "15 seconds", 30: "30 seconds", 60: "1 minute" }.get(interval, f"{interval}s") fig.update_layout( title_text=f"{self.symbol} Real-Time Data ({interval_text})", title_x=0.5, # Center the title xaxis_rangeslider_visible=False, height=1800, # Increased height for taller display template='plotly_dark', paper_bgcolor='rgba(0,0,0,0)', plot_bgcolor='rgba(25,25,50,1)', font=dict(family="Arial, sans-serif", size=12, color="white"), showlegend=True, legend=dict( yanchor="top", y=0.99, xanchor="left", x=0.01 ) ) # Update axes styling fig.update_xaxes(showgrid=True, gridwidth=1, gridcolor='rgba(128,128,128,0.2)') fig.update_yaxes(showgrid=True, gridwidth=1, gridcolor='rgba(128,128,128,0.2)') # Add neural network signals as annotations if available if hasattr(self, 'nn_signals') and self.nn_signals: for signal in self.nn_signals: try: # Skip HOLD signals for clarity if signal['type'] == 'HOLD': continue # Check if this signal's timestamp is within the visible range signal_ts = signal['timestamp'] # Only add annotations for signals in the visible time range if df is not None and not df.empty: if signal_ts < df.index.min() or signal_ts > df.index.max(): continue # Set color and symbol based on signal type if signal['type'] == 'BUY': color = 'green' symbol = '▲' # Up triangle y_position = df['low'].min() * 0.98 # Below the candles else: # SELL color = 'red' symbol = '▼' # Down triangle y_position = df['high'].max() * 1.02 # Above the candles # Add probability if available text = f"{signal['type']}" if signal['probability'] is not None: text += f" ({signal['probability']:.2f})" # Add annotation fig.add_annotation( x=signal_ts, y=y_position, text=text, showarrow=True, arrowhead=2, arrowsize=1, arrowwidth=2, arrowcolor=color, font=dict(size=12, color=color), bgcolor='rgba(255, 255, 255, 0.8)', bordercolor=color, borderwidth=1, borderpad=4, row=1, col=1 ) except Exception as e: logger.error(f"Error adding NN signal annotation: {str(e)}") return fig except Exception as layout_error: logger.error(f"Error updating layout: {str(layout_error)}") import traceback logger.error(traceback.format_exc()) return go.Figure() except Exception as e: logger.error(f"Error updating chart: {str(e)}") import traceback logger.error(traceback.format_exc()) # Create a minimal figure with error message fig = go.Figure() fig.add_annotation( x=0.5, y=0.5, text=f"Error updating chart: {str(e)}", showarrow=False, font=dict(size=14, color="red"), xref="paper", yref="paper" ) fig.update_layout( height=800, template='plotly_dark', paper_bgcolor='rgba(0,0,0,0)', plot_bgcolor='rgba(25,25,50,1)' ) return fig def _setup_ticks_callback(self): # Callbacks to update the ticks data display # Callback to update stats cards @self.app.callback( Output('tick-stats-cards', 'children'), [Input('interval-component', 'n_intervals'), Input('refresh-ticks-btn', 'n_clicks'), Input('time-window-dropdown', 'value')] ) def update_tick_stats(n_intervals, n_clicks, time_window): # Get time window in seconds window_seconds = time_window if time_window else 300 # Calculate time range for filtering now = int(time.time() * 1000) start_time = now - (window_seconds * 1000) # Get filtered ticks filtered_ticks = self.tick_storage.get_ticks_from_time(start_time_ms=start_time) if not filtered_ticks: return [html.Div("No tick data available in the selected time window.", style={'color': 'white', 'textAlign': 'center', 'margin': '20px'})] # Calculate stats tick_count = len(filtered_ticks) prices = [tick['price'] for tick in filtered_ticks] min_price = min(prices) if prices else 0 max_price = max(prices) if prices else 0 avg_price = sum(prices) / len(prices) if prices else 0 price_range = max_price - min_price volumes = [tick.get('volume', 0) for tick in filtered_ticks] total_volume = sum(volumes) avg_volume = total_volume / len(volumes) if volumes else 0 # Calculate additional volume stats max_volume = max(volumes) if volumes else 0 min_volume = min(volumes) if volumes else 0 median_volume = sorted(volumes)[len(volumes)//2] if volumes else 0 # Calculate trade value in USDT trade_values = [p * v for p, v in zip(prices, volumes)] total_value = sum(trade_values) avg_trade_value = total_value / len(trade_values) if trade_values else 0 first_timestamp = filtered_ticks[0]['timestamp'] last_timestamp = filtered_ticks[-1]['timestamp'] time_span_ms = last_timestamp - first_timestamp time_span_seconds = time_span_ms / 1000 ticks_per_second = tick_count / time_span_seconds if time_span_seconds > 0 else 0 # Create stat cards card_style = { 'backgroundColor': '#1E2130', 'borderRadius': '5px', 'padding': '15px', 'marginBottom': '10px', 'width': '23%', 'boxShadow': '0 2px 4px rgba(0,0,0,0.2)', 'color': 'white', 'display': 'flex', 'flexDirection': 'column', 'alignItems': 'center', 'justifyContent': 'center' } mobile_card_style = { **card_style, 'width': '100%', 'marginBottom': '10px' } value_style = { 'fontSize': '24px', 'fontWeight': 'bold', 'color': '#4CAF50', 'marginTop': '5px' } label_style = { 'fontSize': '14px', 'color': '#AAAAAA' } return [ html.Div([ html.Div("Tick Count", style=label_style), html.Div(f"{tick_count:,}", style=value_style), html.Div(f"{ticks_per_second:.2f} ticks/sec", style=label_style) ], style=card_style), html.Div([ html.Div("Price Range", style=label_style), html.Div(f"{min_price:.2f} - {max_price:.2f}", style=value_style), html.Div(f"Range: {price_range:.2f}", style=label_style) ], style=card_style), html.Div([ html.Div("Average Price", style=label_style), html.Div(f"{avg_price:.2f}", style=value_style), html.Div("USDT", style=label_style) ], style=card_style), html.Div([ html.Div("Total Volume", style=label_style), html.Div(f"{total_volume:.8f}", style=value_style), html.Div(f"Avg: {avg_volume:.8f}", style=label_style) ], style=card_style), html.Div([ html.Div("Max Volume", style=label_style), html.Div(f"{max_volume:.8f}", style=value_style), html.Div("Median: {:.8f}".format(median_volume), style=label_style) ], style=card_style), html.Div([ html.Div("Total Value", style=label_style), html.Div(f"{total_value:.2f}", style=value_style), html.Div(f"Avg: {avg_trade_value:.2f}", style=label_style) ], style=card_style) ] # Callback to update ticks table @self.app.callback( Output('ticks-table-container', 'children'), [Input('interval-component', 'n_intervals'), Input('refresh-ticks-btn', 'n_clicks'), Input('time-window-dropdown', 'value')] ) def update_ticks_table(n_intervals, n_clicks, time_window): # Get time window in seconds window_seconds = time_window if time_window else 300 # Calculate time range for filtering now = int(time.time() * 1000) start_time = now - (window_seconds * 1000) # Get filtered ticks filtered_ticks = self.tick_storage.get_ticks_from_time(start_time_ms=start_time) if not filtered_ticks: return html.Div("No tick data available in the selected time window.", style={'color': 'white', 'textAlign': 'center', 'margin': '20px'}) # Convert to readable format formatted_ticks = [] for tick in filtered_ticks: timestamp_dt = datetime.fromtimestamp(tick['timestamp'] / 1000) formatted_time = timestamp_dt.strftime('%H:%M:%S.%f')[:-3] # Format as HH:MM:SS.mmm formatted_ticks.append({ 'time': formatted_time, 'price': f"{tick['price']:.2f}", 'volume': f"{tick['volume']:.8f}", 'value': f"{tick['price'] * tick['volume']:.2f}" }) # Display only the most recent ticks display_limit = 100 limited_ticks = formatted_ticks[-display_limit:] if len(formatted_ticks) > display_limit else formatted_ticks limited_ticks.reverse() # Show most recent at the top # Create table table_style = { 'width': '100%', 'borderCollapse': 'collapse', 'color': 'white', 'fontFamily': 'monospace' } header_style = { 'backgroundColor': '#1A1A1A', 'fontWeight': 'bold', 'padding': '8px', 'textAlign': 'left', 'borderBottom': '2px solid #444' } cell_style = { 'padding': '6px', 'borderBottom': '1px solid #333', 'textAlign': 'right' } time_cell_style = { **cell_style, 'textAlign': 'left' } # Create table header header = html.Tr([ html.Th("Time", style=header_style), html.Th("Price", style=header_style), html.Th("Volume", style=header_style), html.Th("Value (USDT)", style=header_style) ]) # Create table rows rows = [] for i, tick in enumerate(limited_ticks): row_style = {'backgroundColor': '#1E1E1E'} if i % 2 == 0 else {'backgroundColor': '#252525'} rows.append(html.Tr([ html.Td(tick['time'], style={**time_cell_style, **row_style}), html.Td(tick['price'], style={**cell_style, **row_style}), html.Td(tick['volume'], style={**cell_style, **row_style}), html.Td(tick['value'], style={**cell_style, **row_style}) ])) return [ html.Div([ html.H3(f"Latest {len(limited_ticks)} Ticks (from {len(filtered_ticks)} total)", style={'color': 'white', 'marginBottom': '10px', 'textAlign': 'center'}), html.Table([html.Thead(header), html.Tbody(rows)], style=table_style) ]) ] # Callback to update price chart @self.app.callback( Output('tick-price-chart', 'figure'), [Input('interval-component', 'n_intervals'), Input('refresh-ticks-btn', 'n_clicks'), Input('time-window-dropdown', 'value')] ) def update_price_chart(n_intervals, n_clicks, time_window): # Get time window in seconds window_seconds = time_window if time_window else 300 # Calculate time range for filtering now = int(time.time() * 1000) start_time = now - (window_seconds * 1000) # Get filtered ticks filtered_ticks = self.tick_storage.get_ticks_from_time(start_time_ms=start_time) # Create figure with 2 subplots - price and volume fig = make_subplots( rows=2, cols=1, shared_xaxes=True, vertical_spacing=0.03, row_heights=[0.7, 0.3], subplot_titles=(f"Price Movement (Last {window_seconds // 60} minutes)", "Volume") ) if not filtered_ticks: fig.add_annotation( x=0.5, y=0.5, text="No tick data available in the selected time window.", showarrow=False, font=dict(size=14, color="white"), xref="paper", yref="paper" ) else: # Convert timestamps to datetime for better x-axis display timestamps = [datetime.fromtimestamp(tick['timestamp'] / 1000) for tick in filtered_ticks] prices = [tick['price'] for tick in filtered_ticks] volumes = [tick.get('volume', 0) for tick in filtered_ticks] # Add price scatter plot fig.add_trace( go.Scatter( x=timestamps, y=prices, mode='lines', name='Price', line=dict(color='#4CAF50', width=1.5) ), row=1, col=1 ) # Create a volume profile on the right side of the price chart if len(prices) > 5: # Only create profile if we have enough data # Group prices into bins price_min = min(prices) price_max = max(prices) # Create approximately 20 bins based on price range bin_size = max(0.01, (price_max - price_min) / 20) # Create a dictionary to hold volume by price level volume_by_price = {} # Group volumes by price bins for p, v in zip(prices, volumes): bin_key = round(p / bin_size) * bin_size if bin_key in volume_by_price: volume_by_price[bin_key] += v else: volume_by_price[bin_key] = v # Sort by price level sorted_bins = sorted(volume_by_price.items()) profile_prices = [p for p, _ in sorted_bins] profile_volumes = [v for _, v in sorted_bins] # Add separate volume profile trace fig.add_trace( go.Bar( y=profile_prices, x=profile_volumes, orientation='h', name='Volume Profile', marker=dict( color=['#33CC33' if p <= latest_price else '#FF4136' for p in profile_prices], opacity=0.5 ), showlegend=True, hovertemplate='Price: %{y:.2f}
Volume: %{x:.8f}' ), row=1, col=1 ) # Add a line marking the latest price fig.add_shape( type="line", y0=latest_price, y1=latest_price, x0=0, x1=max(profile_volumes) * 1.1, line=dict(color="yellow", width=1, dash="dash"), row=1, col=1 ) # Add volume bars in separate subplot # Color volume bars green for price increases, red for decreases if len(timestamps) > 1: # Compare each price with the previous to determine color colors = [] for i in range(len(prices)): if i == 0: colors.append('#33CC33') # Default to green for first tick else: if prices[i] >= prices[i-1]: colors.append('#33CC33') # Green for price increase/same else: colors.append('#FF4136') # Red for price decrease else: colors = ['#33CC33'] # Default green if only one tick fig.add_trace( go.Bar( x=timestamps, y=volumes, name='Volume', marker=dict(color=colors) ), row=2, col=1 ) # Compute stats for annotations latest_price = prices[-1] if prices else 0 total_volume = sum(volumes) max_volume = max(volumes) if volumes else 0 avg_volume = total_volume / len(volumes) if volumes else 0 # Add annotations for latest price fig.add_annotation( x=timestamps[-1] if timestamps else 0, y=latest_price, text=f"{latest_price:.2f}", showarrow=True, arrowhead=1, arrowsize=1, arrowwidth=2, arrowcolor="#4CAF50", font=dict(size=12, color="#4CAF50"), xshift=50, row=1, col=1 ) # Add annotations for volume stats fig.add_annotation( x=timestamps[-1] if timestamps else 0, y=max_volume, text=f"Max: {max_volume:.8f}", showarrow=False, font=dict(size=10, color="rgba(128, 128, 255, 1)"), xshift=50, row=2, col=1 ) # Update layout fig.update_layout( title_text=f"{self.symbol} Tick Data", title_x=0.5, template='plotly_dark', paper_bgcolor='rgba(0,0,0,0)', plot_bgcolor='rgba(25,25,50,1)', height=600, # Increased height for better visualization margin=dict(l=40, r=40, t=80, b=40), legend=dict( orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1 ), hovermode="x unified" # Show all data points at the same x-coordinate ) # Update x-axis to be shared fig.update_xaxes( showgrid=True, gridwidth=1, gridcolor='rgba(128,128,128,0.2)', rangeslider_visible=False ) # Update y-axes fig.update_yaxes( title_text="Price (USDT)", showgrid=True, gridwidth=1, gridcolor='rgba(128,128,128,0.2)', row=1, col=1 ) fig.update_yaxes( title_text="Volume", showgrid=True, gridwidth=1, gridcolor='rgba(128,128,128,0.2)', row=2, col=1 ) return fig def _interval_to_seconds(self, interval_key: str) -> int: """Convert interval key to seconds""" mapping = { '1s': 1, '1m': 60, '1h': 3600, '1d': 86400 } return mapping.get(interval_key, 1) async def start_websocket(self): ws = ExchangeWebSocket(self.symbol) connection_attempts = 0 max_attempts = 10 # Maximum connection attempts before longer waiting period while True: # Keep trying to maintain connection connection_attempts += 1 if not await ws.connect(): logger.error(f"Failed to connect to exchange for {self.symbol}") # Gradually increase wait time based on number of connection failures wait_time = min(5 * connection_attempts, 60) # Cap at 60 seconds logger.warning(f"Waiting {wait_time} seconds before retry (attempt {connection_attempts})") if connection_attempts >= max_attempts: logger.warning(f"Reached {max_attempts} connection attempts, taking a longer break") await asyncio.sleep(120) # 2 minutes wait after max attempts connection_attempts = 0 # Reset counter else: await asyncio.sleep(wait_time) continue # Successfully connected connection_attempts = 0 try: logger.info(f"WebSocket connected for {self.symbol}, beginning data collection") tick_count = 0 last_tick_count_log = time.time() last_status_report = time.time() # Track stats for reporting price_min = float('inf') price_max = float('-inf') price_last = None volume_total = 0 start_collection_time = time.time() while True: if not ws.running: logger.warning(f"WebSocket connection lost for {self.symbol}, breaking loop") break data = await ws.receive() if data: if data.get('type') == 'kline': # Use kline data directly for candlestick trade_data = { 'timestamp': data['timestamp'], 'price': data['price'], 'volume': data['volume'], 'open': data['open'], 'high': data['high'], 'low': data['low'] } logger.debug(f"Received kline data: {data}") else: # Use trade data trade_data = { 'timestamp': data['timestamp'], 'price': data['price'], 'volume': data['volume'] } # Update stats price = trade_data['price'] volume = trade_data['volume'] price_min = min(price_min, price) price_max = max(price_max, price) price_last = price volume_total += volume # Store raw tick in the tick storage self.tick_storage.add_tick(trade_data) tick_count += 1 # Also update the old candlestick data for backward compatibility self.candlestick_data.update_from_trade(trade_data) # Log tick counts periodically current_time = time.time() if current_time - last_tick_count_log >= 10: # Log every 10 seconds elapsed = current_time - last_tick_count_log tps = tick_count / elapsed if elapsed > 0 else 0 logger.info(f"{self.symbol}: Collected {tick_count} ticks in last {elapsed:.1f}s ({tps:.2f} ticks/sec), total: {len(self.tick_storage.ticks)}") last_tick_count_log = current_time tick_count = 0 # Check if ticks are being converted to candles if len(self.tick_storage.ticks) > 0: sample_df = self.tick_storage.get_candles(interval_seconds=1) logger.info(f"{self.symbol}: Sample candle count: {len(sample_df)}") # Periodic status report (every 60 seconds) if current_time - last_status_report >= 60: elapsed_total = current_time - start_collection_time logger.info(f"{self.symbol} Status Report:") logger.info(f" Collection time: {elapsed_total:.1f} seconds") logger.info(f" Price range: {price_min:.2f} - {price_max:.2f} (last: {price_last:.2f})") logger.info(f" Total volume: {volume_total:.8f}") logger.info(f" Active ticks in storage: {len(self.tick_storage.ticks)}") # Reset stats for next period last_status_report = current_time price_min = float('inf') if price_last is None else price_last price_max = float('-inf') if price_last is None else price_last volume_total = 0 await asyncio.sleep(0.01) except websockets.exceptions.ConnectionClosed as e: logger.error(f"WebSocket connection closed for {self.symbol}: {str(e)}") except Exception as e: logger.error(f"Error in WebSocket loop for {self.symbol}: {str(e)}") import traceback logger.error(traceback.format_exc()) finally: logger.info(f"Closing WebSocket connection for {self.symbol}") await ws.close() logger.info(f"Waiting 5 seconds before reconnecting {self.symbol} WebSocket...") await asyncio.sleep(5) def run(self, host='localhost', port=8050): try: logger.info(f"Starting Dash server for {self.symbol} on {host}:{port}") self.app.run(debug=False, host=host, port=port) except Exception as e: logger.error(f"Error running Dash server on port {port}: {str(e)}") # Try an alternative port if the primary one is in use if "Address already in use" in str(e): alt_port = port + 100 logger.warning(f"Port {port} is busy, trying alternative port {alt_port}") try: self.app.run(debug=False, host=host, port=alt_port) except Exception as alt_e: logger.error(f"Error running Dash server on alternative port {alt_port}: {str(alt_e)}") else: # Re-raise other exceptions raise def _load_historical_data(self): """Load historical data for all timeframes from Binance API and local cache""" try: logger.info(f"Loading historical data for {self.symbol}...") # Define intervals to fetch intervals = { '1s': 1, '1m': 60, '1h': 3600, '1d': 86400 } # Track load status load_status = {interval: False for interval in intervals.keys()} # First try to load from local cache files logger.info("Step 1: Loading from local cache files...") for interval_key, interval_seconds in intervals.items(): try: cache_file = os.path.join(self.historical_data.cache_dir, f"{self.symbol.replace('/', '_')}_{interval_key}_candles.csv") logger.info(f"Checking for cached {interval_key} data at {cache_file}") if os.path.exists(cache_file): # Check if cache is fresh (less than 1 day old for anything but 1d, 3 days for 1d) file_age = time.time() - os.path.getmtime(cache_file) max_age = 259200 if interval_key == '1d' else 86400 # 3 days for 1d, 1 day for others logger.info(f"Cache file age: {file_age:.1f}s, max allowed: {max_age}s") if file_age <= max_age: logger.info(f"Loading {interval_key} candles from cache") cached_df = pd.read_csv(cache_file) if not cached_df.empty: # Diagnostic info about the loaded data logger.info(f"Loaded {len(cached_df)} candles from {cache_file}") logger.info(f"Columns: {cached_df.columns.tolist()}") logger.info(f"First few rows: {cached_df.head(2).to_dict('records')}") # Convert timestamp string back to datetime if 'timestamp' in cached_df.columns: try: if not pd.api.types.is_datetime64_any_dtype(cached_df['timestamp']): cached_df['timestamp'] = pd.to_datetime(cached_df['timestamp']) logger.info("Successfully converted timestamps to datetime") except Exception as e: logger.warning(f"Could not convert timestamp column for {interval_key}: {str(e)}") # Only keep the last 2000 candles for memory efficiency if len(cached_df) > 2000: cached_df = cached_df.tail(2000) logger.info(f"Truncated to last 2000 candles") # Add to cache for _, row in cached_df.iterrows(): candle_dict = row.to_dict() self.candle_cache.candles[interval_key].append(candle_dict) # Update ohlcv_cache self.ohlcv_cache[interval_key] = self.candle_cache.get_recent_candles(interval_key, count=2000) logger.info(f"Successfully loaded {len(self.ohlcv_cache[interval_key])} cached {interval_key} candles") if len(self.ohlcv_cache[interval_key]) >= 500: load_status[interval_key] = True # Skip fetching from API if we loaded from cache (except for 1d timeframe which we always refresh) if interval_key != '1d': continue else: logger.info(f"Cache file for {interval_key} is too old ({file_age:.1f}s)") else: logger.info(f"No cache file found for {interval_key}") except Exception as e: logger.error(f"Error loading cached {interval_key} candles: {str(e)}") import traceback logger.error(traceback.format_exc()) # For timeframes other than 1s, fetch from API as backup or for fresh data logger.info("Step 2: Fetching data from API for missing timeframes...") for interval_key, interval_seconds in intervals.items(): # Skip 1s for API requests if interval_key == '1s' or load_status[interval_key]: logger.info(f"Skipping API fetch for {interval_key}: already loaded or 1s timeframe") continue # Fetch historical data from API try: logger.info(f"Fetching {interval_key} candles from API for {self.symbol}") historical_df = self.historical_data.get_historical_candles( symbol=self.symbol, interval_seconds=interval_seconds, limit=500 # Get 500 candles ) if not historical_df.empty: logger.info(f"Loaded {len(historical_df)} historical candles for {self.symbol} {interval_key} from API") # If we already have data in cache, merge with new data to avoid duplicates if self.ohlcv_cache[interval_key] is not None and not self.ohlcv_cache[interval_key].empty: existing_df = self.ohlcv_cache[interval_key] # Get the latest timestamp from existing data latest_time = existing_df['timestamp'].max() # Only keep newer records from API new_candles = historical_df[historical_df['timestamp'] > latest_time] if not new_candles.empty: logger.info(f"Adding {len(new_candles)} new candles to existing {interval_key} cache") # Add to cache for _, row in new_candles.iterrows(): candle_dict = row.to_dict() self.candle_cache.candles[interval_key].append(candle_dict) else: # No existing data, add all from API for _, row in historical_df.iterrows(): candle_dict = row.to_dict() self.candle_cache.candles[interval_key].append(candle_dict) # Update ohlcv_cache with combined data self.ohlcv_cache[interval_key] = self.candle_cache.get_recent_candles(interval_key, count=2000) logger.info(f"Total {interval_key} candles in cache: {len(self.ohlcv_cache[interval_key])}") if len(self.ohlcv_cache[interval_key]) >= 500: load_status[interval_key] = True else: logger.warning(f"No historical data available from API for {self.symbol} {interval_key}") except Exception as e: logger.error(f"Error fetching {interval_key} data from API: {str(e)}") import traceback logger.error(traceback.format_exc()) # Log summary of loaded data logger.info("Historical data load summary:") for interval_key in intervals.keys(): count = len(self.ohlcv_cache[interval_key]) if self.ohlcv_cache[interval_key] is not None else 0 status = "Success" if load_status[interval_key] else "Failed" if count > 0 and count < 500: status = "Partial" logger.info(f"{interval_key}: {count} candles - {status}") except Exception as e: logger.error(f"Error in _load_historical_data: {str(e)}") import traceback logger.error(traceback.format_exc()) def _save_candles_to_disk(self, force=False): """Save current candle cache to disk for persistence between runs""" try: # Only save if we have data and sufficient time has passed (every 5 minutes) current_time = time.time() if not force and current_time - self.last_cache_save_time < 300: # 5 minutes return # Save each timeframe's candles to disk for interval_key, candles in self.candle_cache.candles.items(): if candles: # Convert to DataFrame df = pd.DataFrame(list(candles)) if not df.empty: # Ensure timestamp is properly formatted if 'timestamp' in df.columns: try: if not pd.api.types.is_datetime64_any_dtype(df['timestamp']): df['timestamp'] = pd.to_datetime(df['timestamp']) except: logger.warning(f"Could not convert timestamp column for {interval_key}") # Save to disk in the cache directory cache_file = os.path.join(self.historical_data.cache_dir, f"{self.symbol.replace('/', '_')}_{interval_key}_candles.csv") df.to_csv(cache_file, index=False) logger.info(f"Saved {len(df)} {interval_key} candles to {cache_file}") self.last_cache_save_time = current_time logger.info(f"Saved all candle caches to disk at {datetime.now()}") except Exception as e: logger.error(f"Error saving candles to disk: {str(e)}") import traceback logger.error(traceback.format_exc()) def add_nn_signal(self, signal_type, timestamp, probability=None): """Add a neural network signal to be displayed on the chart Args: signal_type: The type of signal (BUY, SELL, HOLD) timestamp: The timestamp for the signal probability: Optional probability/confidence value """ if signal_type not in ['BUY', 'SELL', 'HOLD']: logger.warning(f"Invalid NN signal type: {signal_type}") return # Convert timestamp to datetime if it's not already if not isinstance(timestamp, datetime): try: if isinstance(timestamp, str): timestamp = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) elif isinstance(timestamp, (int, float)): timestamp = datetime.fromtimestamp(timestamp / 1000.0) except Exception as e: logger.error(f"Error converting timestamp for NN signal: {str(e)}") timestamp = datetime.now() # Add the signal to our list self.nn_signals.append({ 'type': signal_type, 'timestamp': timestamp, 'probability': probability, 'added': datetime.now() }) # Only keep the most recent 50 signals if len(self.nn_signals) > 50: self.nn_signals = self.nn_signals[-50:] logger.info(f"Added NN signal: {signal_type} at {timestamp}") async def main(): global charts # Make charts globally accessible for NN integration symbols = ["ETH/USDT", "BTC/USDT"] logger.info(f"Starting application for symbols: {symbols}") # Initialize neural network if enabled if NN_ENABLED: logger.info("Initializing Neural Network integration...") if setup_neural_network(): logger.info("Neural Network integration initialized successfully") else: logger.warning("Neural Network integration failed to initialize") charts = [] websocket_tasks = [] # Create a chart and websocket task for each symbol for symbol in symbols: chart = RealTimeChart(symbol) charts.append(chart) websocket_tasks.append(asyncio.create_task(chart.start_websocket())) # Run Dash in a separate thread to not block the event loop server_threads = [] for i, chart in enumerate(charts): port = 8050 + i # Use different ports for each chart logger.info(f"Starting chart for {chart.symbol} on port {port}") thread = Thread(target=lambda c=chart, p=port: c.run(port=p)) # Ensure correct port is passed thread.daemon = True thread.start() server_threads.append(thread) logger.info(f"Thread started for {chart.symbol} on port {port}") try: # Keep the main task running while True: await asyncio.sleep(1) except KeyboardInterrupt: logger.info("Shutting down...") except Exception as e: logger.error(f"Unexpected error: {str(e)}") finally: for task in websocket_tasks: task.cancel() try: await task except asyncio.CancelledError: pass if __name__ == "__main__": try: asyncio.run(main()) except KeyboardInterrupt: logger.info("Application terminated by user")