""" Smart Data Updater Efficiently manages data updates using: 1. Initial historical data load (once) 2. Live tick data from WebSocket 3. Periodic HTTP updates (1m every minute, 1h every hour) 4. Smart candle construction from ticks """ import threading import time import logging from datetime import datetime, timedelta from typing import Dict, List, Optional, Any import pandas as pd import numpy as np from collections import deque from .data_cache import get_data_cache, DataCache from .data_models import OHLCVBar logger = logging.getLogger(__name__) class SmartDataUpdater: """ Smart data updater that efficiently manages market data with minimal API calls """ def __init__(self, data_provider, symbols: List[str]): self.data_provider = data_provider self.symbols = symbols self.cache = get_data_cache() self.running = False # Tick data for candle construction self.tick_buffers: Dict[str, deque] = {symbol: deque(maxlen=1000) for symbol in symbols} self.tick_locks: Dict[str, threading.Lock] = {symbol: threading.Lock() for symbol in symbols} # Current candle construction self.current_candles: Dict[str, Dict[str, Dict]] = {} # {symbol: {timeframe: candle_data}} self.candle_locks: Dict[str, threading.Lock] = {symbol: threading.Lock() for symbol in symbols} # Update timers self.last_updates: Dict[str, Dict[str, datetime]] = {} # {symbol: {timeframe: last_update}} # Update intervals (in seconds) self.update_intervals = { '1s': 10, # Update 1s candles every 10 seconds from ticks '1m': 60, # Update 1m candles every minute via HTTP '1h': 3600, # Update 1h candles every hour via HTTP '1d': 86400 # Update 1d candles every day via HTTP } logger.info(f"SmartDataUpdater initialized for {len(symbols)} symbols") def start(self): """Start the smart data updater""" if self.running: return self.running = True # Load initial historical data self._load_initial_historical_data() # Start update threads self.update_thread = threading.Thread(target=self._update_worker, daemon=True) self.update_thread.start() # Start tick processing thread self.tick_thread = threading.Thread(target=self._tick_processor, daemon=True) self.tick_thread.start() logger.info("SmartDataUpdater started") def stop(self): """Stop the smart data updater""" self.running = False logger.info("SmartDataUpdater stopped") def add_tick(self, symbol: str, price: float, volume: float, timestamp: datetime = None): """Add tick data for candle construction""" if symbol not in self.tick_buffers: return tick_data = { 'price': price, 'volume': volume, 'timestamp': timestamp or datetime.now() } with self.tick_locks[symbol]: self.tick_buffers[symbol].append(tick_data) def _load_initial_historical_data(self): """Load initial historical data for all symbols and timeframes""" logger.info("Loading initial historical data...") timeframes = ['1s', '1m', '1h', '1d'] limits = {'1s': 300, '1m': 300, '1h': 300, '1d': 300} for symbol in self.symbols: self.last_updates[symbol] = {} self.current_candles[symbol] = {} for timeframe in timeframes: try: limit = limits.get(timeframe, 300) # Get historical data df = None if hasattr(self.data_provider, 'get_historical_data'): df = self.data_provider.get_historical_data(symbol, timeframe, limit=limit) if df is not None and not df.empty: # Store in cache self.cache.store_historical_data(symbol, timeframe, df) # Update current candle data from latest bar latest_bar = df.iloc[-1] self._update_current_candle_from_bar(symbol, timeframe, latest_bar) # Update cache with latest OHLCV ohlcv_bar = self._df_row_to_ohlcv_bar(symbol, timeframe, latest_bar, df.index[-1]) self.cache.update(f'ohlcv_{timeframe}', symbol, ohlcv_bar, 'historical') self.last_updates[symbol][timeframe] = datetime.now() logger.info(f"Loaded {len(df)} {timeframe} bars for {symbol}") else: logger.warning(f"No historical data for {symbol} {timeframe}") except Exception as e: logger.error(f"Error loading historical data for {symbol} {timeframe}: {e}") # Calculate initial technical indicators self._calculate_technical_indicators() logger.info("Initial historical data loading completed") def _update_worker(self): """Background worker for periodic data updates""" while self.running: try: current_time = datetime.now() for symbol in self.symbols: for timeframe in ['1m', '1h', '1d']: # Skip 1s (built from ticks) try: # Check if it's time to update last_update = self.last_updates[symbol].get(timeframe) interval = self.update_intervals[timeframe] if not last_update or (current_time - last_update).total_seconds() >= interval: self._update_timeframe_data(symbol, timeframe) self.last_updates[symbol][timeframe] = current_time except Exception as e: logger.error(f"Error updating {symbol} {timeframe}: {e}") # Update technical indicators every minute if current_time.second < 10: # Update in first 10 seconds of each minute self._calculate_technical_indicators() time.sleep(10) # Check every 10 seconds except Exception as e: logger.error(f"Error in update worker: {e}") time.sleep(30) def _tick_processor(self): """Process ticks to build 1s candles""" while self.running: try: current_time = datetime.now() for symbol in self.symbols: # Check if it's time to update 1s candles last_update = self.last_updates[symbol].get('1s') if not last_update or (current_time - last_update).total_seconds() >= self.update_intervals['1s']: self._build_1s_candle_from_ticks(symbol) self.last_updates[symbol]['1s'] = current_time time.sleep(5) # Process every 5 seconds except Exception as e: logger.error(f"Error in tick processor: {e}") time.sleep(10) def _update_timeframe_data(self, symbol: str, timeframe: str): """Update data for a specific timeframe via HTTP""" try: # Get latest data from API df = None if hasattr(self.data_provider, 'get_latest_candles'): df = self.data_provider.get_latest_candles(symbol, timeframe, limit=1) elif hasattr(self.data_provider, 'get_historical_data'): df = self.data_provider.get_historical_data(symbol, timeframe, limit=1) if df is not None and not df.empty: latest_bar = df.iloc[-1] # Update current candle self._update_current_candle_from_bar(symbol, timeframe, latest_bar) # Update cache ohlcv_bar = self._df_row_to_ohlcv_bar(symbol, timeframe, latest_bar, df.index[-1]) self.cache.update(f'ohlcv_{timeframe}', symbol, ohlcv_bar, 'http_update') logger.debug(f"Updated {symbol} {timeframe} via HTTP") except Exception as e: logger.error(f"Error updating {symbol} {timeframe} via HTTP: {e}") def _build_1s_candle_from_ticks(self, symbol: str): """Build 1s candle from accumulated ticks""" try: with self.tick_locks[symbol]: ticks = list(self.tick_buffers[symbol]) if not ticks: return # Get ticks from last 10 seconds cutoff_time = datetime.now() - timedelta(seconds=10) recent_ticks = [tick for tick in ticks if tick['timestamp'] >= cutoff_time] if not recent_ticks: return # Build OHLCV from ticks prices = [tick['price'] for tick in recent_ticks] volumes = [tick['volume'] for tick in recent_ticks] ohlcv_data = { 'open': prices[0], 'high': max(prices), 'low': min(prices), 'close': prices[-1], 'volume': sum(volumes) } # Update current candle with self.candle_locks[symbol]: self.current_candles[symbol]['1s'] = ohlcv_data # Create OHLCV bar and update cache ohlcv_bar = OHLCVBar( symbol=symbol, timestamp=recent_ticks[-1]['timestamp'], open=ohlcv_data['open'], high=ohlcv_data['high'], low=ohlcv_data['low'], close=ohlcv_data['close'], volume=ohlcv_data['volume'], timeframe='1s' ) self.cache.update('ohlcv_1s', symbol, ohlcv_bar, 'tick_constructed') logger.debug(f"Built 1s candle for {symbol} from {len(recent_ticks)} ticks") except Exception as e: logger.error(f"Error building 1s candle from ticks for {symbol}: {e}") def _update_current_candle_from_bar(self, symbol: str, timeframe: str, bar_data): """Update current candle data from a bar""" try: with self.candle_locks[symbol]: self.current_candles[symbol][timeframe] = { 'open': float(bar_data['open']), 'high': float(bar_data['high']), 'low': float(bar_data['low']), 'close': float(bar_data['close']), 'volume': float(bar_data['volume']) } except Exception as e: logger.error(f"Error updating current candle for {symbol} {timeframe}: {e}") def _df_row_to_ohlcv_bar(self, symbol: str, timeframe: str, row, timestamp) -> OHLCVBar: """Convert DataFrame row to OHLCVBar""" return OHLCVBar( symbol=symbol, timestamp=timestamp if hasattr(timestamp, 'to_pydatetime') else datetime.now(), open=float(row['open']), high=float(row['high']), low=float(row['low']), close=float(row['close']), volume=float(row['volume']), timeframe=timeframe ) def _calculate_technical_indicators(self): """Calculate technical indicators for all symbols""" try: for symbol in self.symbols: # Use 1m historical data for indicators df = self.cache.get_historical_data(symbol, '1m') if df is None or len(df) < 20: continue indicators = {} try: import ta # RSI if len(df) >= 14: indicators['rsi'] = ta.momentum.RSIIndicator(df['close']).rsi().iloc[-1] # Moving averages if len(df) >= 20: indicators['sma_20'] = df['close'].rolling(20).mean().iloc[-1] if len(df) >= 12: indicators['ema_12'] = df['close'].ewm(span=12).mean().iloc[-1] if len(df) >= 26: indicators['ema_26'] = df['close'].ewm(span=26).mean().iloc[-1] if 'ema_12' in indicators: indicators['macd'] = indicators['ema_12'] - indicators['ema_26'] # Bollinger Bands if len(df) >= 20: bb_period = 20 bb_std = 2 sma = df['close'].rolling(bb_period).mean() std = df['close'].rolling(bb_period).std() indicators['bb_upper'] = (sma + (std * bb_std)).iloc[-1] indicators['bb_lower'] = (sma - (std * bb_std)).iloc[-1] indicators['bb_middle'] = sma.iloc[-1] # Remove NaN values indicators = {k: float(v) for k, v in indicators.items() if not pd.isna(v)} if indicators: self.cache.update('technical_indicators', symbol, indicators, 'calculated') logger.debug(f"Calculated {len(indicators)} indicators for {symbol}") except Exception as e: logger.error(f"Error calculating indicators for {symbol}: {e}") except Exception as e: logger.error(f"Error in technical indicators calculation: {e}") def get_current_price(self, symbol: str) -> Optional[float]: """Get current price from latest 1s candle""" ohlcv_1s = self.cache.get('ohlcv_1s', symbol) if ohlcv_1s: return ohlcv_1s.close return None def get_status(self) -> Dict[str, Any]: """Get updater status""" status = { 'running': self.running, 'symbols': self.symbols, 'last_updates': self.last_updates, 'tick_buffer_sizes': {symbol: len(buffer) for symbol, buffer in self.tick_buffers.items()}, 'cache_status': self.cache.get_status() } return status