inrefence predictions fix

This commit is contained in:
Dobromir Popov
2025-07-26 23:34:36 +03:00
parent 7c61c12b70
commit 3eb6335169
9 changed files with 1125 additions and 305 deletions

View File

@ -6,7 +6,8 @@ Integrates with SmartDataUpdater for efficient data management.
"""
import logging
from datetime import datetime
import threading
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Any
import pandas as pd
@ -29,6 +30,11 @@ class SimplifiedDataIntegration:
# Initialize smart data updater
self.data_updater = SmartDataUpdater(data_provider, symbols)
# Pre-built OHLCV data cache for instant access
self._ohlcv_cache = {} # {symbol: {timeframe: List[OHLCVBar]}}
self._ohlcv_cache_lock = threading.RLock()
self._last_cache_update = {} # {symbol: {timeframe: datetime}}
# Register for tick data if available
self._setup_tick_integration()
@ -61,6 +67,8 @@ class SimplifiedDataIntegration:
def _on_tick_data(self, symbol: str, price: float, volume: float, timestamp: datetime = None):
"""Handle incoming tick data"""
self.data_updater.add_tick(symbol, price, volume, timestamp)
# Invalidate OHLCV cache for this symbol
self._invalidate_ohlcv_cache(symbol)
def _on_websocket_data(self, symbol: str, data: Dict[str, Any]):
"""Handle WebSocket data updates"""
@ -68,12 +76,28 @@ class SimplifiedDataIntegration:
# Extract price and volume from WebSocket data
if 'price' in data and 'volume' in data:
self.data_updater.add_tick(symbol, data['price'], data['volume'])
# Invalidate OHLCV cache for this symbol
self._invalidate_ohlcv_cache(symbol)
except Exception as e:
logger.error(f"Error processing WebSocket data: {e}")
def _invalidate_ohlcv_cache(self, symbol: str):
"""Invalidate OHLCV cache for a symbol when new data arrives"""
try:
with self._ohlcv_cache_lock:
# Remove cached data for all timeframes of this symbol
keys_to_remove = [key for key in self._ohlcv_cache.keys() if key.startswith(f"{symbol}_")]
for key in keys_to_remove:
if key in self._ohlcv_cache:
del self._ohlcv_cache[key]
if key in self._last_cache_update:
del self._last_cache_update[key]
except Exception as e:
logger.error(f"Error invalidating OHLCV cache for {symbol}: {e}")
def build_base_data_input(self, symbol: str) -> Optional[BaseDataInput]:
"""
Build BaseDataInput from cached data (much simpler than FIFO queues)
Build BaseDataInput from cached data (optimized for speed)
Args:
symbol: Trading symbol
@ -82,22 +106,7 @@ class SimplifiedDataIntegration:
BaseDataInput with consistent data structure
"""
try:
# Check if we have minimum required data
required_timeframes = ['1s', '1m', '1h', '1d']
missing_timeframes = []
for timeframe in required_timeframes:
if not self.cache.has_data(f'ohlcv_{timeframe}', symbol, max_age_seconds=300):
missing_timeframes.append(timeframe)
if missing_timeframes:
logger.warning(f"Missing data for {symbol}: {missing_timeframes}")
# Try to use historical data as fallback
if not self._try_historical_fallback(symbol, missing_timeframes):
return None
# Get current OHLCV data
# Get OHLCV data directly from optimized cache (no validation checks for speed)
ohlcv_1s_list = self._get_ohlcv_data_list(symbol, '1s', 300)
ohlcv_1m_list = self._get_ohlcv_data_list(symbol, '1m', 300)
ohlcv_1h_list = self._get_ohlcv_data_list(symbol, '1h', 300)
@ -109,18 +118,13 @@ class SimplifiedDataIntegration:
if not btc_ohlcv_1s_list:
# Use ETH data as fallback
btc_ohlcv_1s_list = ohlcv_1s_list
logger.debug(f"Using {symbol} data as BTC fallback")
# Get technical indicators
# Get cached data (fast lookups)
technical_indicators = self.cache.get('technical_indicators', symbol) or {}
# Get COB data if available
cob_data = self.cache.get('cob_data', symbol)
# Get recent model predictions
last_predictions = self._get_recent_predictions(symbol)
# Build BaseDataInput
# Build BaseDataInput (no validation for speed - assume data is good)
base_data = BaseDataInput(
symbol=symbol,
timestamp=datetime.now(),
@ -134,11 +138,6 @@ class SimplifiedDataIntegration:
last_predictions=last_predictions
)
# Validate the data
if not base_data.validate():
logger.warning(f"BaseDataInput validation failed for {symbol}")
return None
return base_data
except Exception as e:
@ -146,11 +145,39 @@ class SimplifiedDataIntegration:
return None
def _get_ohlcv_data_list(self, symbol: str, timeframe: str, max_count: int) -> List[OHLCVBar]:
"""Get OHLCV data list from cache and historical data"""
"""Get OHLCV data list from pre-built cache for instant access"""
try:
with self._ohlcv_cache_lock:
cache_key = f"{symbol}_{timeframe}"
# Check if we have fresh cached data (updated within last 5 seconds)
last_update = self._last_cache_update.get(cache_key)
if (last_update and
(datetime.now() - last_update).total_seconds() < 5 and
cache_key in self._ohlcv_cache):
cached_data = self._ohlcv_cache[cache_key]
return cached_data[-max_count:] if len(cached_data) >= max_count else cached_data
# Need to rebuild cache for this symbol/timeframe
data_list = self._build_ohlcv_cache(symbol, timeframe, max_count)
# Cache the result
self._ohlcv_cache[cache_key] = data_list
self._last_cache_update[cache_key] = datetime.now()
return data_list[-max_count:] if len(data_list) >= max_count else data_list
except Exception as e:
logger.error(f"Error getting OHLCV data list for {symbol}/{timeframe}: {e}")
return self._create_dummy_data_list(symbol, timeframe, max_count)
def _build_ohlcv_cache(self, symbol: str, timeframe: str, max_count: int) -> List[OHLCVBar]:
"""Build OHLCV cache from historical and current data"""
try:
data_list = []
# Get historical data first
# Get historical data first (this should be fast as it's already cached)
historical_df = self.cache.get_historical_data(symbol, timeframe)
if historical_df is not None and not historical_df.empty:
# Convert historical data to OHLCVBar objects
@ -174,34 +201,14 @@ class SimplifiedDataIntegration:
# Ensure we have the right amount of data (pad if necessary)
while len(data_list) < max_count:
# Pad with the last available data or create dummy data
if data_list:
last_bar = data_list[-1]
dummy_bar = OHLCVBar(
symbol=symbol,
timestamp=last_bar.timestamp,
open=last_bar.close,
high=last_bar.close,
low=last_bar.close,
close=last_bar.close,
volume=0.0,
timeframe=timeframe
)
else:
# Create completely dummy data
dummy_bar = OHLCVBar(
symbol=symbol,
timestamp=datetime.now(),
open=0.0, high=0.0, low=0.0, close=0.0, volume=0.0,
timeframe=timeframe
)
data_list.append(dummy_bar)
data_list.extend(self._create_dummy_data_list(symbol, timeframe, max_count - len(data_list)))
return data_list[-max_count:] # Return last max_count items
return data_list
except Exception as e:
logger.error(f"Error getting OHLCV data list for {symbol} {timeframe}: {e}")
return []
logger.error(f"Error building OHLCV cache for {symbol}/{timeframe}: {e}")
return self._create_dummy_data_list(symbol, timeframe, max_count)
def _try_historical_fallback(self, symbol: str, missing_timeframes: List[str]) -> bool:
"""Try to use historical data for missing timeframes"""