popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TZ fix again - wip

Browse Source
This commit is contained in:
Dobromir Popov
2025-08-08 01:41:30 +03:00
parent ba532327b6
commit ded7e7f008
5 changed files with 216 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
core/data_provider.py
View File

@ -729,11 +729,19 @@ class DataProvider:
logger.error(f"Error getting current COB imbalance for {symbol}: {e}")
return {'imbalance': 0.0, 'price_range': price_range, 'error': str(e)}
def _calculate_cob_imbalance(self, cob_data: Dict, price_range: float) -> float:
"""Calculate order book imbalance within specified price range around mid price"""
def _calculate_cob_imbalance(self, cob_data: Any, price_range: float) -> float:
"""Calculate order book imbalance within specified price range around mid price.
Accepts dict snapshot or COBData-like objects (with bids/asks as list of [price, size]).
"""
try:
bids = cob_data.get('bids', [])
asks = cob_data.get('asks', [])
# Normalize input
if isinstance(cob_data, dict):
bids = cob_data.get('bids', [])
asks = cob_data.get('asks', [])
else:
# Try attribute access (COBData-like or snapshot objects)
bids = getattr(cob_data, 'bids', []) or []
asks = getattr(cob_data, 'asks', []) or []
if not bids or not asks:
return 0.0
@ -1207,17 +1215,15 @@ class DataProvider:
logger.warning(f"No valid candles generated for {symbol}")
return None
# Convert to DataFrame
# Convert to DataFrame (timestamps remain UTC tz-aware)
df = pd.DataFrame(candles)
# Ensure timestamps are timezone-aware (UTC to match COB WebSocket data)
if not df.empty and 'timestamp' in df.columns:
import pytz
utc = pytz.UTC
# If timestamps are not timezone-aware, make them UTC
# Normalize to UTC tz-aware using pandas idioms
if df['timestamp'].dt.tz is None:
df['timestamp'] = df['timestamp'].dt.tz_localize(utc)
df['timestamp'] = pd.to_datetime(df['timestamp'], utc=True)
else:
df['timestamp'] = df['timestamp'].dt.tz_convert(utc)
df['timestamp'] = df['timestamp'].dt.tz_convert('UTC')
df = df.sort_values('timestamp').reset_index(drop=True)
@ -2153,7 +2159,7 @@ class DataProvider:
# Get cached data (fast lookups)
technical_indicators = self._get_latest_technical_indicators(symbol)
cob_data = self._get_latest_cob_data_object(symbol)
last_predictions = {} # TODO: Implement model prediction caching
last_predictions = {}
# Build BaseDataInput (no validation for speed - assume data is good)
base_data = BaseDataInput(
@ -4655,26 +4661,53 @@ class DataProvider:
return subscriber_id
def get_latest_cob_data(self, symbol: str) -> dict:
"""Get latest COB data for a symbol"""
"""Get the most recent valid COB snapshot.
Falls back to the last valid snapshot in cache if the most recent is invalid.
A snapshot is considered valid if bids and asks are non-empty and stats.mid_price > 0.
"""
with self.subscriber_lock:
# Use the original symbol format for cache lookup (matches how data is stored)
logger.debug(f"Getting COB data for {symbol}")
if not hasattr(self, 'cob_data_cache'):
cache = getattr(self, 'cob_data_cache', None)
if not cache:
logger.debug("COB data cache not initialized")
return {}
if symbol not in self.cob_data_cache:
logger.debug(f"Symbol {symbol} not in COB cache. Available: {list(self.cob_data_cache.keys())}")
if symbol not in cache:
logger.debug(f"Symbol {symbol} not in COB cache. Available: {list(cache.keys())}")
return {}
if not self.cob_data_cache[symbol]:
snapshots = cache.get(symbol) or []
if not snapshots:
logger.debug(f"COB cache for {symbol} is empty")
return {}
latest_data = self.cob_data_cache[symbol][-1]
logger.debug(f"Latest COB data type for {symbol}: {type(latest_data)}")
return latest_data
def is_valid(snap: dict) -> bool:
try:
bids = snap.get('bids') or []
asks = snap.get('asks') or []
stats = snap.get('stats') or {}
mid_price = float(stats.get('mid_price', 0) or 0)
return bool(bids) and bool(asks) and mid_price > 0
except Exception:
return False
# Walk cache backwards to find the most recent valid snapshot
for snap in reversed(snapshots):
if is_valid(snap):
# Annotate staleness info in stats if timestamp present
try:
ts_ms = snap.get('timestamp')
if isinstance(ts_ms, (int, float)):
import time
age_ms = int(time.time() * 1000) - int(ts_ms)
if isinstance(snap.get('stats'), dict):
snap['stats']['age_ms'] = max(age_ms, 0)
except Exception:
pass
return snap
# No valid snapshot found
logger.debug(f"No valid COB snapshot found for {symbol}")
return {}
def get_cob_raw_ticks(self, symbol: str, count: int = 100) -> List[dict]:
"""Get raw COB ticks for a symbol (100+ updates per second)"""

8
core/standardized_data_provider.py
View File

@ -33,9 +33,9 @@ class StandardizedDataProvider(DataProvider):
"""Initialize the standardized data provider"""
super().__init__(symbols, timeframes)
# Standardized data storage
# Standardized data storage (separate COB cache to avoid colliding with parent caches)
self.base_data_cache: Dict[str, BaseDataInput] = {} # {symbol: BaseDataInput}
self.cob_data_cache: Dict[str, COBData] = {} # {symbol: COBData}
self.standardized_cob_data_cache: Dict[str, COBData] = {} # {symbol: COBData}
# Model output management with extensible storage
self.model_output_manager = ModelOutputManager(
@ -50,7 +50,7 @@ class StandardizedDataProvider(DataProvider):
# Initialize caches for each symbol
for symbol in self.symbols:
self.base_data_cache[symbol] = None
self.cob_data_cache[symbol] = None
self.standardized_cob_data_cache[symbol] = None
self.cob_imbalance_history[symbol] = deque(maxlen=300) # 5 minutes of 1s data
# Ensure live price cache exists (in case parent didn't initialize it)
@ -253,7 +253,7 @@ class StandardizedDataProvider(DataProvider):
cob_obj.ma_60s_imbalance = ma_data.get('60s', {})
# Cache and return
self.cob_data_cache[symbol] = cob_obj
self.standardized_cob_data_cache[symbol] = cob_obj
return cob_obj
except Exception as e: