inrefence predictions fix
This commit is contained in:
@ -6,7 +6,8 @@ Integrates with SmartDataUpdater for efficient data management.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Any
|
||||
import pandas as pd
|
||||
|
||||
@ -29,6 +30,11 @@ class SimplifiedDataIntegration:
|
||||
# Initialize smart data updater
|
||||
self.data_updater = SmartDataUpdater(data_provider, symbols)
|
||||
|
||||
# Pre-built OHLCV data cache for instant access
|
||||
self._ohlcv_cache = {} # {symbol: {timeframe: List[OHLCVBar]}}
|
||||
self._ohlcv_cache_lock = threading.RLock()
|
||||
self._last_cache_update = {} # {symbol: {timeframe: datetime}}
|
||||
|
||||
# Register for tick data if available
|
||||
self._setup_tick_integration()
|
||||
|
||||
@ -61,6 +67,8 @@ class SimplifiedDataIntegration:
|
||||
def _on_tick_data(self, symbol: str, price: float, volume: float, timestamp: datetime = None):
|
||||
"""Handle incoming tick data"""
|
||||
self.data_updater.add_tick(symbol, price, volume, timestamp)
|
||||
# Invalidate OHLCV cache for this symbol
|
||||
self._invalidate_ohlcv_cache(symbol)
|
||||
|
||||
def _on_websocket_data(self, symbol: str, data: Dict[str, Any]):
|
||||
"""Handle WebSocket data updates"""
|
||||
@ -68,12 +76,28 @@ class SimplifiedDataIntegration:
|
||||
# Extract price and volume from WebSocket data
|
||||
if 'price' in data and 'volume' in data:
|
||||
self.data_updater.add_tick(symbol, data['price'], data['volume'])
|
||||
# Invalidate OHLCV cache for this symbol
|
||||
self._invalidate_ohlcv_cache(symbol)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing WebSocket data: {e}")
|
||||
|
||||
def _invalidate_ohlcv_cache(self, symbol: str):
|
||||
"""Invalidate OHLCV cache for a symbol when new data arrives"""
|
||||
try:
|
||||
with self._ohlcv_cache_lock:
|
||||
# Remove cached data for all timeframes of this symbol
|
||||
keys_to_remove = [key for key in self._ohlcv_cache.keys() if key.startswith(f"{symbol}_")]
|
||||
for key in keys_to_remove:
|
||||
if key in self._ohlcv_cache:
|
||||
del self._ohlcv_cache[key]
|
||||
if key in self._last_cache_update:
|
||||
del self._last_cache_update[key]
|
||||
except Exception as e:
|
||||
logger.error(f"Error invalidating OHLCV cache for {symbol}: {e}")
|
||||
|
||||
def build_base_data_input(self, symbol: str) -> Optional[BaseDataInput]:
|
||||
"""
|
||||
Build BaseDataInput from cached data (much simpler than FIFO queues)
|
||||
Build BaseDataInput from cached data (optimized for speed)
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol
|
||||
@ -82,22 +106,7 @@ class SimplifiedDataIntegration:
|
||||
BaseDataInput with consistent data structure
|
||||
"""
|
||||
try:
|
||||
# Check if we have minimum required data
|
||||
required_timeframes = ['1s', '1m', '1h', '1d']
|
||||
missing_timeframes = []
|
||||
|
||||
for timeframe in required_timeframes:
|
||||
if not self.cache.has_data(f'ohlcv_{timeframe}', symbol, max_age_seconds=300):
|
||||
missing_timeframes.append(timeframe)
|
||||
|
||||
if missing_timeframes:
|
||||
logger.warning(f"Missing data for {symbol}: {missing_timeframes}")
|
||||
|
||||
# Try to use historical data as fallback
|
||||
if not self._try_historical_fallback(symbol, missing_timeframes):
|
||||
return None
|
||||
|
||||
# Get current OHLCV data
|
||||
# Get OHLCV data directly from optimized cache (no validation checks for speed)
|
||||
ohlcv_1s_list = self._get_ohlcv_data_list(symbol, '1s', 300)
|
||||
ohlcv_1m_list = self._get_ohlcv_data_list(symbol, '1m', 300)
|
||||
ohlcv_1h_list = self._get_ohlcv_data_list(symbol, '1h', 300)
|
||||
@ -109,18 +118,13 @@ class SimplifiedDataIntegration:
|
||||
if not btc_ohlcv_1s_list:
|
||||
# Use ETH data as fallback
|
||||
btc_ohlcv_1s_list = ohlcv_1s_list
|
||||
logger.debug(f"Using {symbol} data as BTC fallback")
|
||||
|
||||
# Get technical indicators
|
||||
# Get cached data (fast lookups)
|
||||
technical_indicators = self.cache.get('technical_indicators', symbol) or {}
|
||||
|
||||
# Get COB data if available
|
||||
cob_data = self.cache.get('cob_data', symbol)
|
||||
|
||||
# Get recent model predictions
|
||||
last_predictions = self._get_recent_predictions(symbol)
|
||||
|
||||
# Build BaseDataInput
|
||||
# Build BaseDataInput (no validation for speed - assume data is good)
|
||||
base_data = BaseDataInput(
|
||||
symbol=symbol,
|
||||
timestamp=datetime.now(),
|
||||
@ -134,11 +138,6 @@ class SimplifiedDataIntegration:
|
||||
last_predictions=last_predictions
|
||||
)
|
||||
|
||||
# Validate the data
|
||||
if not base_data.validate():
|
||||
logger.warning(f"BaseDataInput validation failed for {symbol}")
|
||||
return None
|
||||
|
||||
return base_data
|
||||
|
||||
except Exception as e:
|
||||
@ -146,11 +145,39 @@ class SimplifiedDataIntegration:
|
||||
return None
|
||||
|
||||
def _get_ohlcv_data_list(self, symbol: str, timeframe: str, max_count: int) -> List[OHLCVBar]:
|
||||
"""Get OHLCV data list from cache and historical data"""
|
||||
"""Get OHLCV data list from pre-built cache for instant access"""
|
||||
try:
|
||||
with self._ohlcv_cache_lock:
|
||||
cache_key = f"{symbol}_{timeframe}"
|
||||
|
||||
# Check if we have fresh cached data (updated within last 5 seconds)
|
||||
last_update = self._last_cache_update.get(cache_key)
|
||||
if (last_update and
|
||||
(datetime.now() - last_update).total_seconds() < 5 and
|
||||
cache_key in self._ohlcv_cache):
|
||||
|
||||
cached_data = self._ohlcv_cache[cache_key]
|
||||
return cached_data[-max_count:] if len(cached_data) >= max_count else cached_data
|
||||
|
||||
# Need to rebuild cache for this symbol/timeframe
|
||||
data_list = self._build_ohlcv_cache(symbol, timeframe, max_count)
|
||||
|
||||
# Cache the result
|
||||
self._ohlcv_cache[cache_key] = data_list
|
||||
self._last_cache_update[cache_key] = datetime.now()
|
||||
|
||||
return data_list[-max_count:] if len(data_list) >= max_count else data_list
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting OHLCV data list for {symbol}/{timeframe}: {e}")
|
||||
return self._create_dummy_data_list(symbol, timeframe, max_count)
|
||||
|
||||
def _build_ohlcv_cache(self, symbol: str, timeframe: str, max_count: int) -> List[OHLCVBar]:
|
||||
"""Build OHLCV cache from historical and current data"""
|
||||
try:
|
||||
data_list = []
|
||||
|
||||
# Get historical data first
|
||||
# Get historical data first (this should be fast as it's already cached)
|
||||
historical_df = self.cache.get_historical_data(symbol, timeframe)
|
||||
if historical_df is not None and not historical_df.empty:
|
||||
# Convert historical data to OHLCVBar objects
|
||||
@ -174,34 +201,14 @@ class SimplifiedDataIntegration:
|
||||
|
||||
# Ensure we have the right amount of data (pad if necessary)
|
||||
while len(data_list) < max_count:
|
||||
# Pad with the last available data or create dummy data
|
||||
if data_list:
|
||||
last_bar = data_list[-1]
|
||||
dummy_bar = OHLCVBar(
|
||||
symbol=symbol,
|
||||
timestamp=last_bar.timestamp,
|
||||
open=last_bar.close,
|
||||
high=last_bar.close,
|
||||
low=last_bar.close,
|
||||
close=last_bar.close,
|
||||
volume=0.0,
|
||||
timeframe=timeframe
|
||||
)
|
||||
else:
|
||||
# Create completely dummy data
|
||||
dummy_bar = OHLCVBar(
|
||||
symbol=symbol,
|
||||
timestamp=datetime.now(),
|
||||
open=0.0, high=0.0, low=0.0, close=0.0, volume=0.0,
|
||||
timeframe=timeframe
|
||||
)
|
||||
data_list.append(dummy_bar)
|
||||
data_list.extend(self._create_dummy_data_list(symbol, timeframe, max_count - len(data_list)))
|
||||
|
||||
return data_list[-max_count:] # Return last max_count items
|
||||
return data_list
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting OHLCV data list for {symbol} {timeframe}: {e}")
|
||||
return []
|
||||
logger.error(f"Error building OHLCV cache for {symbol}/{timeframe}: {e}")
|
||||
return self._create_dummy_data_list(symbol, timeframe, max_count)
|
||||
|
||||
|
||||
def _try_historical_fallback(self, symbol: str, missing_timeframes: List[str]) -> bool:
|
||||
"""Try to use historical data for missing timeframes"""
|
||||
|
Reference in New Issue
Block a user