different confidence for open/close position

core/data_provider.py

@@ -27,10 +27,9 @@ from dataclasses import dataclass, field
 import ta
 from threading import Thread, Lock
 from collections import deque
-from dataclasses import dataclass, field
-import uuid
 
 from .config import get_config
+from .tick_aggregator import RealTimeTickAggregator, RawTick, OHLCVBar
 
 logger = logging.getLogger(__name__)
 
@@ -88,13 +87,24 @@ class DataProvider:
             binance_symbol = symbol.replace('/', '').upper()
             self.tick_buffers[binance_symbol] = deque(maxlen=self.buffer_size)
         
+        # Initialize tick aggregator for raw tick processing
+        binance_symbols = [symbol.replace('/', '').upper() for symbol in self.symbols]
+        self.tick_aggregator = RealTimeTickAggregator(symbols=binance_symbols)
+        
+        # Raw tick and OHLCV bar callbacks
+        self.raw_tick_callbacks = []
+        self.ohlcv_bar_callbacks = []
+        
         # Performance tracking for subscribers
         self.distribution_stats = {
             'total_ticks_received': 0,
             'total_ticks_distributed': 0,
             'distribution_errors': 0,
             'last_tick_time': {},
-            'ticks_per_symbol': {symbol.replace('/', '').upper(): 0 for symbol in self.symbols}
+            'ticks_per_symbol': {symbol.replace('/', '').upper(): 0 for symbol in self.symbols},
+            'raw_ticks_processed': 0,
+            'ohlcv_bars_created': 0,
+            'patterns_detected': 0
         }
         
         # Data validation
@@ -474,14 +484,52 @@ class DataProvider:
                 logger.warning(f"Invalid tick data for {symbol}: price={price}, volume={volume_usdt}")
                 return
             
-            # Create standardized tick
+            # Process raw tick through aggregator
+            side = 'sell' if is_buyer_maker else 'buy'
+            raw_tick, completed_bar = self.tick_aggregator.process_tick(
+                symbol=symbol,
+                timestamp=timestamp,
+                price=price,
+                volume=volume_usdt,
+                quantity=quantity,
+                side=side,
+                trade_id=str(trade_id)
+            )
+            
+            # Update statistics
+            self.distribution_stats['total_ticks_received'] += 1
+            self.distribution_stats['ticks_per_symbol'][symbol] += 1
+            self.distribution_stats['last_tick_time'][symbol] = timestamp
+            self.last_prices[symbol] = price
+            
+            if raw_tick:
+                self.distribution_stats['raw_ticks_processed'] += 1
+                
+                # Notify raw tick callbacks
+                for callback in self.raw_tick_callbacks:
+                    try:
+                        callback(raw_tick)
+                    except Exception as e:
+                        logger.error(f"Error in raw tick callback: {e}")
+            
+            if completed_bar:
+                self.distribution_stats['ohlcv_bars_created'] += 1
+                
+                # Notify OHLCV bar callbacks
+                for callback in self.ohlcv_bar_callbacks:
+                    try:
+                        callback(completed_bar)
+                    except Exception as e:
+                        logger.error(f"Error in OHLCV bar callback: {e}")
+            
+            # Create standardized tick for legacy compatibility
             tick = MarketTick(
                 symbol=symbol,
                 timestamp=timestamp,
                 price=price,
                 volume=volume_usdt,
                 quantity=quantity,
-                side='sell' if is_buyer_maker else 'buy',
+                side=side,
                 trade_id=str(trade_id),
                 is_buyer_maker=is_buyer_maker,
                 raw_data=trade_data
@@ -490,12 +538,6 @@ class DataProvider:
             # Add to buffer
             self.tick_buffers[symbol].append(tick)
             
-            # Update statistics
-            self.distribution_stats['total_ticks_received'] += 1
-            self.distribution_stats['ticks_per_symbol'][symbol] += 1
-            self.distribution_stats['last_tick_time'][symbol] = timestamp
-            self.last_prices[symbol] = price
-            
             # Update current prices and candles
             await self._process_tick(symbol, tick)
             
@@ -1033,6 +1075,36 @@ class DataProvider:
             return list(self.tick_buffers[binance_symbol])[-count:]
         return []
     
+    def subscribe_to_raw_ticks(self, callback: Callable[[RawTick], None]) -> str:
+        """Subscribe to raw tick data with timing information"""
+        subscriber_id = str(uuid.uuid4())
+        self.raw_tick_callbacks.append(callback)
+        logger.info(f"Raw tick subscriber added: {subscriber_id}")
+        return subscriber_id
+    
+    def subscribe_to_ohlcv_bars(self, callback: Callable[[OHLCVBar], None]) -> str:
+        """Subscribe to 1s OHLCV bars calculated from ticks"""
+        subscriber_id = str(uuid.uuid4())
+        self.ohlcv_bar_callbacks.append(callback)
+        logger.info(f"OHLCV bar subscriber added: {subscriber_id}")
+        return subscriber_id
+    
+    def get_raw_tick_features(self, symbol: str, window_size: int = 50) -> Optional[np.ndarray]:
+        """Get raw tick features for model consumption"""
+        return self.tick_aggregator.get_tick_features_for_model(symbol, window_size)
+    
+    def get_ohlcv_features(self, symbol: str, window_size: int = 60) -> Optional[np.ndarray]:
+        """Get 1s OHLCV features for model consumption"""
+        return self.tick_aggregator.get_ohlcv_features_for_model(symbol, window_size)
+    
+    def get_detected_patterns(self, symbol: str, count: int = 50) -> List:
+        """Get recently detected tick patterns"""
+        return self.tick_aggregator.get_detected_patterns(symbol, count)
+    
+    def get_tick_aggregator_stats(self) -> Dict[str, Any]:
+        """Get tick aggregator statistics"""
+        return self.tick_aggregator.get_statistics()
+    
     def get_subscriber_stats(self) -> Dict[str, Any]:
         """Get subscriber and distribution statistics"""
         with self.subscriber_lock:
@@ -1048,10 +1120,16 @@ class DataProvider:
                 for sid, s in self.subscribers.items()
             }
         
+        # Get tick aggregator stats
+        aggregator_stats = self.get_tick_aggregator_stats()
+        
         return {
             'active_subscribers': active_subscribers,
             'total_subscribers': len(self.subscribers),
+            'raw_tick_callbacks': len(self.raw_tick_callbacks),
+            'ohlcv_bar_callbacks': len(self.ohlcv_bar_callbacks),
             'subscriber_details': subscriber_stats,
             'distribution_stats': self.distribution_stats.copy(),
-            'buffer_sizes': {symbol: len(buffer) for symbol, buffer in self.tick_buffers.items()}
+            'buffer_sizes': {symbol: len(buffer) for symbol, buffer in self.tick_buffers.items()},
+            'tick_aggregator': aggregator_stats
         }