popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactoring. inference real data triggers

Browse Source
This commit is contained in:
Dobromir Popov
2025-12-09 11:59:15 +02:00
parent 1c1ebf6d7e
commit 992d6de25b
9 changed files with 1970 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
core/data_provider.py
View File

@@ -236,6 +236,11 @@ class DataProvider:
self.raw_tick_callbacks = []
self.ohlcv_bar_callbacks = []
# Event subscriptions for inference training system
self.candle_completion_callbacks: Dict[Tuple[str, str], List[Callable]] = {} # {(symbol, timeframe): [callbacks]}
self.pivot_event_callbacks: Dict[Tuple[str, str, str], List[Callable]] = {} # {(symbol, timeframe, pivot_type): [callbacks]}
self.last_completed_candles: Dict[Tuple[str, str], datetime] = {} # Track last completed candle per symbol/timeframe
# Performance tracking for subscribers
self.distribution_stats = {
'total_ticks_received': 0,
@@ -267,6 +272,7 @@ class DataProvider:
self.pivot_points_cache: Dict[str, Dict[int, TrendLevel]] = {} # {symbol: {level: TrendLevel}}
self.last_pivot_calculation: Dict[str, datetime] = {}
self.pivot_calculation_interval = timedelta(minutes=5) # Recalculate every 5 minutes
self.last_emitted_pivots: Dict[Tuple[str, str], List[Tuple[str, datetime]]] = {} # Track emitted pivots to avoid duplicates
# Unified storage system (optional, initialized on demand)
self.unified_storage: Optional['UnifiedDataProviderExtension'] = None
@@ -2833,6 +2839,9 @@ class DataProvider:
williams = self.williams_structure[symbol]
pivot_levels = williams.calculate_recursive_pivot_points(ohlcv_array)
# Emit pivot events for new pivots (L2L, L2H, L3L, L3H, etc.)
self._check_and_emit_pivot_events(symbol, tf, pivot_levels)
logger.debug(f"Retrieved Williams pivot levels for {symbol}: {len(pivot_levels)} levels")
return pivot_levels
@@ -3580,6 +3589,11 @@ class DataProvider:
# Check if we need a new candle
if not candle_queue or candle_queue[-1]['timestamp'] != candle_start:
# Emit candle completion event for previous candle (if exists)
if candle_queue:
completed_candle = candle_queue[-1]
self._emit_candle_completion(symbol, timeframe, completed_candle)
# Create new candle
new_candle = {
'timestamp': candle_start,
@@ -3601,6 +3615,165 @@ class DataProvider:
except Exception as e:
logger.error(f"Error updating candle for {symbol} {timeframe}: {e}")
def subscribe_candle_completion(self, callback: Callable, symbol: str, timeframe: str) -> None:
"""
Subscribe to candle completion events.
Args:
callback: Function to call when candle completes: callback(event: CandleCompletionEvent)
symbol: Trading symbol
timeframe: Timeframe (1m, 5m, etc.)
"""
key = (symbol, timeframe)
if key not in self.candle_completion_callbacks:
self.candle_completion_callbacks[key] = []
self.candle_completion_callbacks[key].append(callback)
logger.debug(f"Subscribed to candle completion: {symbol} {timeframe}")
def subscribe_pivot_events(self, callback: Callable, symbol: str, timeframe: str, pivot_types: List[str]) -> None:
"""
Subscribe to pivot point events (L2L, L2H, etc.).
Args:
callback: Function to call when pivot detected: callback(event: PivotEvent)
symbol: Trading symbol
timeframe: Timeframe
pivot_types: List of pivot types to subscribe to (e.g., ['L2L', 'L2H', 'L3L'])
"""
for pivot_type in pivot_types:
key = (symbol, timeframe, pivot_type)
if key not in self.pivot_event_callbacks:
self.pivot_event_callbacks[key] = []
self.pivot_event_callbacks[key].append(callback)
logger.debug(f"Subscribed to pivot events: {symbol} {timeframe} {pivot_types}")
def _emit_candle_completion(self, symbol: str, timeframe: str, candle: Dict) -> None:
"""Emit candle completion event to subscribers"""
try:
from ANNOTATE.core.inference_training_system import CandleCompletionEvent
key = (symbol, timeframe)
if key not in self.candle_completion_callbacks:
return
# Check if we already emitted for this candle
last_emitted = self.last_completed_candles.get(key)
if last_emitted == candle['timestamp']:
return # Already emitted
# Create event
event = CandleCompletionEvent(
symbol=symbol,
timeframe=timeframe,
timestamp=candle['timestamp'],
ohlcv={
'open': float(candle['open']),
'high': float(candle['high']),
'low': float(candle['low']),
'close': float(candle['close']),
'volume': float(candle.get('volume', 0))
}
)
# Notify subscribers
for callback in self.candle_completion_callbacks[key]:
try:
callback(event)
except Exception as e:
logger.error(f"Error in candle completion callback: {e}", exc_info=True)
# Mark as emitted
self.last_completed_candles[key] = candle['timestamp']
except Exception as e:
logger.error(f"Error emitting candle completion event: {e}", exc_info=True)
def _check_and_emit_pivot_events(self, symbol: str, timeframe: str, pivot_levels: Dict[int, Any]) -> None:
"""
Check for new pivots and emit events to subscribers.
Args:
symbol: Trading symbol
timeframe: Timeframe
pivot_levels: Dict of pivot levels from Williams Market Structure
"""
try:
key = (symbol, timeframe)
last_emitted = self.last_emitted_pivots.get(key, [])
# Check levels 2, 3, 4, 5 for new pivots
for level in [2, 3, 4, 5]:
if level not in pivot_levels:
continue
trend_level = pivot_levels[level]
if not hasattr(trend_level, 'pivot_points') or not trend_level.pivot_points:
continue
# Get latest pivot for this level
latest_pivot = trend_level.pivot_points[-1] if trend_level.pivot_points else None
if not latest_pivot:
continue
# Check if we've already emitted this pivot
pivot_id = (f"L{level}{latest_pivot.pivot_type[0].upper()}", latest_pivot.timestamp)
if pivot_id in last_emitted:
continue
# Emit event
pivot_type = f"L{level}{latest_pivot.pivot_type[0].upper()}" # L2L, L2H, L3L, etc.
self._emit_pivot_event(
symbol=symbol,
timeframe=timeframe,
pivot_type=pivot_type,
timestamp=latest_pivot.timestamp,
price=latest_pivot.price,
level=level,
strength=latest_pivot.strength
)
# Mark as emitted
last_emitted.append(pivot_id)
# Keep only last 100 emitted pivots to prevent memory bloat
if len(last_emitted) > 100:
last_emitted = last_emitted[-100:]
self.last_emitted_pivots[key] = last_emitted
except Exception as e:
logger.error(f"Error checking and emitting pivot events: {e}", exc_info=True)
def _emit_pivot_event(self, symbol: str, timeframe: str, pivot_type: str,
timestamp: datetime, price: float, level: int, strength: float) -> None:
"""Emit pivot event to subscribers"""
try:
from ANNOTATE.core.inference_training_system import PivotEvent
key = (symbol, timeframe, pivot_type)
if key not in self.pivot_event_callbacks:
return
# Create event
event = PivotEvent(
symbol=symbol,
timeframe=timeframe,
timestamp=timestamp,
pivot_type=pivot_type,
price=price,
level=level,
strength=strength
)
# Notify subscribers
for callback in self.pivot_event_callbacks[key]:
try:
callback(event)
except Exception as e:
logger.error(f"Error in pivot event callback: {e}", exc_info=True)
except Exception as e:
logger.error(f"Error emitting pivot event: {e}", exc_info=True)
def get_latest_candles(self, symbol: str, timeframe: str, limit: int = 100) -> pd.DataFrame:
"""Get the latest candles from cached data only"""
try: