358 lines
15 KiB
Python
358 lines
15 KiB
Python
"""
|
|
Smart Data Updater
|
|
|
|
Efficiently manages data updates using:
|
|
1. Initial historical data load (once)
|
|
2. Live tick data from WebSocket
|
|
3. Periodic HTTP updates (1m every minute, 1h every hour)
|
|
4. Smart candle construction from ticks
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, List, Optional, Any
|
|
import pandas as pd
|
|
import numpy as np
|
|
from collections import deque
|
|
|
|
from .data_cache import get_data_cache, DataCache
|
|
from .data_models import OHLCVBar
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class SmartDataUpdater:
|
|
"""
|
|
Smart data updater that efficiently manages market data with minimal API calls
|
|
"""
|
|
|
|
def __init__(self, data_provider, symbols: List[str]):
|
|
self.data_provider = data_provider
|
|
self.symbols = symbols
|
|
self.cache = get_data_cache()
|
|
self.running = False
|
|
|
|
# Tick data for candle construction
|
|
self.tick_buffers: Dict[str, deque] = {symbol: deque(maxlen=1000) for symbol in symbols}
|
|
self.tick_locks: Dict[str, threading.Lock] = {symbol: threading.Lock() for symbol in symbols}
|
|
|
|
# Current candle construction
|
|
self.current_candles: Dict[str, Dict[str, Dict]] = {} # {symbol: {timeframe: candle_data}}
|
|
self.candle_locks: Dict[str, threading.Lock] = {symbol: threading.Lock() for symbol in symbols}
|
|
|
|
# Update timers
|
|
self.last_updates: Dict[str, Dict[str, datetime]] = {} # {symbol: {timeframe: last_update}}
|
|
|
|
# Update intervals (in seconds)
|
|
self.update_intervals = {
|
|
'1s': 10, # Update 1s candles every 10 seconds from ticks
|
|
'1m': 60, # Update 1m candles every minute via HTTP
|
|
'1h': 3600, # Update 1h candles every hour via HTTP
|
|
'1d': 86400 # Update 1d candles every day via HTTP
|
|
}
|
|
|
|
logger.info(f"SmartDataUpdater initialized for {len(symbols)} symbols")
|
|
|
|
def start(self):
|
|
"""Start the smart data updater"""
|
|
if self.running:
|
|
return
|
|
|
|
self.running = True
|
|
|
|
# Load initial historical data
|
|
self._load_initial_historical_data()
|
|
|
|
# Start update threads
|
|
self.update_thread = threading.Thread(target=self._update_worker, daemon=True)
|
|
self.update_thread.start()
|
|
|
|
# Start tick processing thread
|
|
self.tick_thread = threading.Thread(target=self._tick_processor, daemon=True)
|
|
self.tick_thread.start()
|
|
|
|
logger.info("SmartDataUpdater started")
|
|
|
|
def stop(self):
|
|
"""Stop the smart data updater"""
|
|
self.running = False
|
|
logger.info("SmartDataUpdater stopped")
|
|
|
|
def add_tick(self, symbol: str, price: float, volume: float, timestamp: datetime = None):
|
|
"""Add tick data for candle construction"""
|
|
if symbol not in self.tick_buffers:
|
|
return
|
|
|
|
tick_data = {
|
|
'price': price,
|
|
'volume': volume,
|
|
'timestamp': timestamp or datetime.now()
|
|
}
|
|
|
|
with self.tick_locks[symbol]:
|
|
self.tick_buffers[symbol].append(tick_data)
|
|
|
|
def _load_initial_historical_data(self):
|
|
"""Load initial historical data for all symbols and timeframes"""
|
|
logger.info("Loading initial historical data...")
|
|
|
|
timeframes = ['1s', '1m', '1h', '1d']
|
|
limits = {'1s': 300, '1m': 300, '1h': 300, '1d': 300}
|
|
|
|
for symbol in self.symbols:
|
|
self.last_updates[symbol] = {}
|
|
self.current_candles[symbol] = {}
|
|
|
|
for timeframe in timeframes:
|
|
try:
|
|
limit = limits.get(timeframe, 300)
|
|
|
|
# Get historical data
|
|
df = None
|
|
if hasattr(self.data_provider, 'get_historical_data'):
|
|
df = self.data_provider.get_historical_data(symbol, timeframe, limit=limit)
|
|
|
|
if df is not None and not df.empty:
|
|
# Store in cache
|
|
self.cache.store_historical_data(symbol, timeframe, df)
|
|
|
|
# Update current candle data from latest bar
|
|
latest_bar = df.iloc[-1]
|
|
self._update_current_candle_from_bar(symbol, timeframe, latest_bar)
|
|
|
|
# Update cache with latest OHLCV
|
|
ohlcv_bar = self._df_row_to_ohlcv_bar(symbol, timeframe, latest_bar, df.index[-1])
|
|
self.cache.update(f'ohlcv_{timeframe}', symbol, ohlcv_bar, 'historical')
|
|
|
|
self.last_updates[symbol][timeframe] = datetime.now()
|
|
logger.info(f"Loaded {len(df)} {timeframe} bars for {symbol}")
|
|
else:
|
|
logger.warning(f"No historical data for {symbol} {timeframe}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error loading historical data for {symbol} {timeframe}: {e}")
|
|
|
|
# Calculate initial technical indicators
|
|
self._calculate_technical_indicators()
|
|
|
|
logger.info("Initial historical data loading completed")
|
|
|
|
def _update_worker(self):
|
|
"""Background worker for periodic data updates"""
|
|
while self.running:
|
|
try:
|
|
current_time = datetime.now()
|
|
|
|
for symbol in self.symbols:
|
|
for timeframe in ['1m', '1h', '1d']: # Skip 1s (built from ticks)
|
|
try:
|
|
# Check if it's time to update
|
|
last_update = self.last_updates[symbol].get(timeframe)
|
|
interval = self.update_intervals[timeframe]
|
|
|
|
if not last_update or (current_time - last_update).total_seconds() >= interval:
|
|
self._update_timeframe_data(symbol, timeframe)
|
|
self.last_updates[symbol][timeframe] = current_time
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error updating {symbol} {timeframe}: {e}")
|
|
|
|
# Update technical indicators every minute
|
|
if current_time.second < 10: # Update in first 10 seconds of each minute
|
|
self._calculate_technical_indicators()
|
|
|
|
time.sleep(10) # Check every 10 seconds
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in update worker: {e}")
|
|
time.sleep(30)
|
|
|
|
def _tick_processor(self):
|
|
"""Process ticks to build 1s candles"""
|
|
while self.running:
|
|
try:
|
|
current_time = datetime.now()
|
|
|
|
for symbol in self.symbols:
|
|
# Check if it's time to update 1s candles
|
|
last_update = self.last_updates[symbol].get('1s')
|
|
if not last_update or (current_time - last_update).total_seconds() >= self.update_intervals['1s']:
|
|
self._build_1s_candle_from_ticks(symbol)
|
|
self.last_updates[symbol]['1s'] = current_time
|
|
|
|
time.sleep(5) # Process every 5 seconds
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in tick processor: {e}")
|
|
time.sleep(10)
|
|
|
|
def _update_timeframe_data(self, symbol: str, timeframe: str):
|
|
"""Update data for a specific timeframe via HTTP"""
|
|
try:
|
|
# Get latest data from API
|
|
df = None
|
|
if hasattr(self.data_provider, 'get_latest_candles'):
|
|
df = self.data_provider.get_latest_candles(symbol, timeframe, limit=1)
|
|
elif hasattr(self.data_provider, 'get_historical_data'):
|
|
df = self.data_provider.get_historical_data(symbol, timeframe, limit=1)
|
|
|
|
if df is not None and not df.empty:
|
|
latest_bar = df.iloc[-1]
|
|
|
|
# Update current candle
|
|
self._update_current_candle_from_bar(symbol, timeframe, latest_bar)
|
|
|
|
# Update cache
|
|
ohlcv_bar = self._df_row_to_ohlcv_bar(symbol, timeframe, latest_bar, df.index[-1])
|
|
self.cache.update(f'ohlcv_{timeframe}', symbol, ohlcv_bar, 'http_update')
|
|
|
|
logger.debug(f"Updated {symbol} {timeframe} via HTTP")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error updating {symbol} {timeframe} via HTTP: {e}")
|
|
|
|
def _build_1s_candle_from_ticks(self, symbol: str):
|
|
"""Build 1s candle from accumulated ticks"""
|
|
try:
|
|
with self.tick_locks[symbol]:
|
|
ticks = list(self.tick_buffers[symbol])
|
|
|
|
if not ticks:
|
|
return
|
|
|
|
# Get ticks from last 10 seconds
|
|
cutoff_time = datetime.now() - timedelta(seconds=10)
|
|
recent_ticks = [tick for tick in ticks if tick['timestamp'] >= cutoff_time]
|
|
|
|
if not recent_ticks:
|
|
return
|
|
|
|
# Build OHLCV from ticks
|
|
prices = [tick['price'] for tick in recent_ticks]
|
|
volumes = [tick['volume'] for tick in recent_ticks]
|
|
|
|
ohlcv_data = {
|
|
'open': prices[0],
|
|
'high': max(prices),
|
|
'low': min(prices),
|
|
'close': prices[-1],
|
|
'volume': sum(volumes)
|
|
}
|
|
|
|
# Update current candle
|
|
with self.candle_locks[symbol]:
|
|
self.current_candles[symbol]['1s'] = ohlcv_data
|
|
|
|
# Create OHLCV bar and update cache
|
|
ohlcv_bar = OHLCVBar(
|
|
symbol=symbol,
|
|
timestamp=recent_ticks[-1]['timestamp'],
|
|
open=ohlcv_data['open'],
|
|
high=ohlcv_data['high'],
|
|
low=ohlcv_data['low'],
|
|
close=ohlcv_data['close'],
|
|
volume=ohlcv_data['volume'],
|
|
timeframe='1s'
|
|
)
|
|
|
|
self.cache.update('ohlcv_1s', symbol, ohlcv_bar, 'tick_constructed')
|
|
logger.debug(f"Built 1s candle for {symbol} from {len(recent_ticks)} ticks")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error building 1s candle from ticks for {symbol}: {e}")
|
|
|
|
def _update_current_candle_from_bar(self, symbol: str, timeframe: str, bar_data):
|
|
"""Update current candle data from a bar"""
|
|
try:
|
|
with self.candle_locks[symbol]:
|
|
self.current_candles[symbol][timeframe] = {
|
|
'open': float(bar_data['open']),
|
|
'high': float(bar_data['high']),
|
|
'low': float(bar_data['low']),
|
|
'close': float(bar_data['close']),
|
|
'volume': float(bar_data['volume'])
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Error updating current candle for {symbol} {timeframe}: {e}")
|
|
|
|
def _df_row_to_ohlcv_bar(self, symbol: str, timeframe: str, row, timestamp) -> OHLCVBar:
|
|
"""Convert DataFrame row to OHLCVBar"""
|
|
return OHLCVBar(
|
|
symbol=symbol,
|
|
timestamp=timestamp if hasattr(timestamp, 'to_pydatetime') else datetime.now(),
|
|
open=float(row['open']),
|
|
high=float(row['high']),
|
|
low=float(row['low']),
|
|
close=float(row['close']),
|
|
volume=float(row['volume']),
|
|
timeframe=timeframe
|
|
)
|
|
|
|
def _calculate_technical_indicators(self):
|
|
"""Calculate technical indicators for all symbols"""
|
|
try:
|
|
for symbol in self.symbols:
|
|
# Use 1m historical data for indicators
|
|
df = self.cache.get_historical_data(symbol, '1m')
|
|
if df is None or len(df) < 20:
|
|
continue
|
|
|
|
indicators = {}
|
|
try:
|
|
import ta
|
|
|
|
# RSI
|
|
if len(df) >= 14:
|
|
indicators['rsi'] = ta.momentum.RSIIndicator(df['close']).rsi().iloc[-1]
|
|
|
|
# Moving averages
|
|
if len(df) >= 20:
|
|
indicators['sma_20'] = df['close'].rolling(20).mean().iloc[-1]
|
|
if len(df) >= 12:
|
|
indicators['ema_12'] = df['close'].ewm(span=12).mean().iloc[-1]
|
|
if len(df) >= 26:
|
|
indicators['ema_26'] = df['close'].ewm(span=26).mean().iloc[-1]
|
|
if 'ema_12' in indicators:
|
|
indicators['macd'] = indicators['ema_12'] - indicators['ema_26']
|
|
|
|
# Bollinger Bands
|
|
if len(df) >= 20:
|
|
bb_period = 20
|
|
bb_std = 2
|
|
sma = df['close'].rolling(bb_period).mean()
|
|
std = df['close'].rolling(bb_period).std()
|
|
indicators['bb_upper'] = (sma + (std * bb_std)).iloc[-1]
|
|
indicators['bb_lower'] = (sma - (std * bb_std)).iloc[-1]
|
|
indicators['bb_middle'] = sma.iloc[-1]
|
|
|
|
# Remove NaN values
|
|
indicators = {k: float(v) for k, v in indicators.items() if not pd.isna(v)}
|
|
|
|
if indicators:
|
|
self.cache.update('technical_indicators', symbol, indicators, 'calculated')
|
|
logger.debug(f"Calculated {len(indicators)} indicators for {symbol}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error calculating indicators for {symbol}: {e}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in technical indicators calculation: {e}")
|
|
|
|
def get_current_price(self, symbol: str) -> Optional[float]:
|
|
"""Get current price from latest 1s candle"""
|
|
ohlcv_1s = self.cache.get('ohlcv_1s', symbol)
|
|
if ohlcv_1s:
|
|
return ohlcv_1s.close
|
|
return None
|
|
|
|
def get_status(self) -> Dict[str, Any]:
|
|
"""Get updater status"""
|
|
status = {
|
|
'running': self.running,
|
|
'symbols': self.symbols,
|
|
'last_updates': self.last_updates,
|
|
'tick_buffer_sizes': {symbol: len(buffer) for symbol, buffer in self.tick_buffers.items()},
|
|
'cache_status': self.cache.get_status()
|
|
}
|
|
return status |