From 3cadae60f70c2c1c9d1fcb193292f5f069a3ad8a Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Wed, 18 Jun 2025 16:23:47 +0300 Subject: [PATCH] COB data and dash --- .vscode/launch.json | 14 + MULTI_EXCHANGE_COB_PROVIDER_SUMMARY.md | 409 +++++ core/bookmap_data_provider.py | 952 ++++++++++++ core/bookmap_integration.py | 1839 +++++++++++++++++++++++ core/cob_integration.py | 597 ++++++++ core/data_provider.py | 76 +- core/enhanced_orchestrator.py | 62 +- core/multi_exchange_cob_provider.py | 1268 ++++++++++++++++ run_cob_dashboard.py | 35 + simple_cob_dashboard.py | 401 +++++ test_enhanced_order_flow_integration.py | 318 ++++ test_free_orderbook_integration.py | Bin 0 -> 15216 bytes test_multi_exchange_cob.py | 327 ++++ test_realtime_cob.py | 92 ++ web/cob_dashboard.html | 689 +++++++++ web/cob_realtime_dashboard.py | 479 ++++++ 16 files changed, 7539 insertions(+), 19 deletions(-) create mode 100644 MULTI_EXCHANGE_COB_PROVIDER_SUMMARY.md create mode 100644 core/bookmap_data_provider.py create mode 100644 core/bookmap_integration.py create mode 100644 core/cob_integration.py create mode 100644 core/multi_exchange_cob_provider.py create mode 100644 run_cob_dashboard.py create mode 100644 simple_cob_dashboard.py create mode 100644 test_enhanced_order_flow_integration.py create mode 100644 test_free_orderbook_integration.py create mode 100644 test_multi_exchange_cob.py create mode 100644 test_realtime_cob.py create mode 100644 web/cob_dashboard.html create mode 100644 web/cob_realtime_dashboard.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 32969e8..91914d3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -64,6 +64,20 @@ "env": { "PYTHONUNBUFFERED": "1" } + }, + { + "name": "šŸ“ˆ COB Data Provider Dashboard", + "type": "python", + "request": "launch", + "program": "web/cob_realtime_dashboard.py", + "console": "integratedTerminal", + "justMyCode": false, + "env": { + "PYTHONUNBUFFERED": "1", + "COB_BTC_BUCKET_SIZE": "10", + "COB_ETH_BUCKET_SIZE": "1" + }, + "preLaunchTask": "Kill Stale Processes" } ], "compounds": [ diff --git a/MULTI_EXCHANGE_COB_PROVIDER_SUMMARY.md b/MULTI_EXCHANGE_COB_PROVIDER_SUMMARY.md new file mode 100644 index 0000000..6dec877 --- /dev/null +++ b/MULTI_EXCHANGE_COB_PROVIDER_SUMMARY.md @@ -0,0 +1,409 @@ +# Multi-Exchange Consolidated Order Book (COB) Data Provider + +## Overview + +This document describes the implementation of a comprehensive multi-exchange Consolidated Order Book (COB) data provider for the gogo2 trading system. The system aggregates real-time order book data from multiple cryptocurrency exchanges to provide enhanced market liquidity analysis and fine-grain volume bucket data. + +## BookMap API Analysis + +### What is BookMap? + +BookMap is a professional trading platform that provides: +- **Multibook**: Consolidated order book data from multiple exchanges +- **Real-time market depth visualization** +- **Order flow analysis tools** +- **Market microstructure analytics** + +### BookMap API Capabilities + +Based on research, BookMap offers three types of APIs: + +1. **L1 (Add-ons API)**: For creating custom indicators and trading strategies within BookMap +2. **L0 (Connect API)**: For creating custom market data connections (requires approval) +3. **Broadcasting API (BrAPI)**: For data sharing between BookMap add-ons + +### BookMap Multibook Features + +BookMap's Multibook provides: +- **Pre-configured synthetic instruments** combining data from major exchanges: + - **USD Spot**: BTC, ETH, ADA, etc. from Bitstamp, Bitfinex, Coinbase Pro, Kraken + - **USDT Spot**: BTC, ETH, DOGE, etc. from Binance, Huobi, Poloniex + - **USDT Perpetual Futures**: From Binance Futures, Bitget, Bybit, OKEx +- **Consolidated order book visualization** +- **Cross-exchange arbitrage detection** +- **Volume-weighted pricing** + +### Limitations for External Use + +**Important Finding**: BookMap's APIs are primarily designed for: +- Creating add-ons **within** the BookMap platform +- Extending BookMap's functionality +- **NOT for external data consumption** + +The APIs do not provide a simple way to consume Multibook data externally for use in other trading systems. + +### Cost and Accessibility + +- BookMap Multibook requires **Global Plus subscription** +- External API access requires approval and specific use cases +- Focus is on professional institutional users + +## Our Implementation Approach + +Given the limitations of accessing BookMap's data externally, we've implemented our own multi-exchange COB provider that replicates and extends BookMap's functionality. + +## Architecture + +### Core Components + +1. **MultiExchangeCOBProvider** (`core/multi_exchange_cob_provider.py`) + - Main aggregation engine + - Real-time WebSocket connections to multiple exchanges + - Order book consolidation logic + - Fine-grain price bucket generation + +2. **COBIntegration** (`core/cob_integration.py`) + - Integration layer with existing gogo2 system + - CNN/DQN feature generation + - Dashboard data formatting + - Trading signal generation + +### Supported Exchanges + +| Exchange | WebSocket URL | Market Share Weight | Symbols Supported | +|----------|---------------|-------------------|-------------------| +| Binance | wss://stream.binance.com:9443/ws/ | 30% | BTC/USDT, ETH/USDT | +| Coinbase Pro | wss://ws-feed.exchange.coinbase.com | 25% | BTC-USD, ETH-USD | +| Kraken | wss://ws.kraken.com | 20% | XBT/USDT, ETH/USDT | +| Huobi | wss://api.huobi.pro/ws | 15% | btcusdt, ethusdt | +| Bitfinex | wss://api-pub.bitfinex.com/ws/2 | 10% | tBTCUST, tETHUST | + +## Key Features + +### 1. Real-Time Order Book Aggregation + +```python +@dataclass +class ConsolidatedOrderBookLevel: + price: float + total_size: float + total_volume_usd: float + total_orders: int + side: str + exchange_breakdown: Dict[str, ExchangeOrderBookLevel] + dominant_exchange: str + liquidity_score: float + timestamp: datetime +``` + +### 2. Fine-Grain Price Buckets + +- **Configurable bucket size** (default: 1 basis point) +- **Volume aggregation** at each price level +- **Exchange attribution** for each bucket +- **Real-time bucket updates** every 100ms + +```python +price_buckets = { + 'bids': { + bucket_key: { + 'price': bucket_price, + 'volume_usd': total_volume, + 'size': total_size, + 'orders': total_orders, + 'exchanges': ['binance', 'coinbase'] + } + }, + 'asks': { ... } +} +``` + +### 3. Market Microstructure Analysis + +- **Volume-weighted mid price** calculation +- **Liquidity imbalance** detection +- **Cross-exchange spread** analysis +- **Exchange dominance** metrics +- **Market depth** distribution + +### 4. CNN/DQN Integration + +#### CNN Features (220 dimensions) +- **Order book levels**: 20 levels Ɨ 5 features Ɨ 2 sides = 200 features +- **Market microstructure**: 20 additional features +- **Normalized and scaled** for neural network consumption + +#### DQN State Features (30 dimensions) +- **Normalized order book state**: 20 features +- **Market state indicators**: 10 features +- **Real-time market regime** detection + +### 5. Trading Signal Generation + +- **Liquidity imbalance signals** +- **Arbitrage opportunity detection** +- **Liquidity anomaly alerts** +- **Market microstructure pattern recognition** + +## Implementation Details + +### Data Structures + +```python +@dataclass +class COBSnapshot: + symbol: str + timestamp: datetime + consolidated_bids: List[ConsolidatedOrderBookLevel] + consolidated_asks: List[ConsolidatedOrderBookLevel] + exchanges_active: List[str] + volume_weighted_mid: float + total_bid_liquidity: float + total_ask_liquidity: float + spread_bps: float + liquidity_imbalance: float + price_buckets: Dict[str, Dict[str, float]] +``` + +### Real-Time Processing + +1. **WebSocket Connections**: Independent connections to each exchange +2. **Order Book Updates**: Process depth updates at 100ms intervals +3. **Consolidation Engine**: Aggregate order books every 100ms +4. **Bucket Generation**: Create fine-grain volume buckets +5. **Feature Generation**: Compute CNN/DQN features in real-time +6. **Signal Detection**: Analyze patterns and generate trading signals + +### Performance Optimizations + +- **Asynchronous processing** for all WebSocket connections +- **Lock-based synchronization** for thread-safe data access +- **Deque-based storage** for efficient historical data management +- **Configurable update frequencies** for different components + +## Integration with Existing System + +### Dashboard Integration + +```python +# Add COB data to dashboard +cob_integration.add_dashboard_callback(dashboard.update_cob_data) + +# Dashboard receives: +{ + 'consolidated_bids': [...], + 'consolidated_asks': [...], + 'price_buckets': {...}, + 'market_quality': {...}, + 'recent_signals': [...] +} +``` + +### AI Model Integration + +```python +# CNN feature generation +cob_integration.add_cnn_callback(cnn_model.process_cob_features) + +# DQN state updates +cob_integration.add_dqn_callback(dqn_agent.update_cob_state) +``` + +### Trading System Integration + +```python +# Signal-based trading +for signal in cob_integration.get_recent_signals(symbol): + if signal['confidence'] > 0.8: + trading_executor.process_cob_signal(signal) +``` + +## Usage Examples + +### Basic Setup + +```python +from core.multi_exchange_cob_provider import MultiExchangeCOBProvider +from core.cob_integration import COBIntegration + +# Initialize COB provider +symbols = ['BTC/USDT', 'ETH/USDT'] +cob_provider = MultiExchangeCOBProvider( + symbols=symbols, + bucket_size_bps=1.0 # 1 basis point granularity +) + +# Integration layer +cob_integration = COBIntegration(symbols=symbols) + +# Start streaming +await cob_integration.start() +``` + +### Accessing Data + +```python +# Get consolidated order book +cob_snapshot = cob_integration.get_cob_snapshot('BTC/USDT') + +# Get fine-grain price buckets +price_buckets = cob_integration.get_price_buckets('BTC/USDT') + +# Get exchange breakdown +exchange_breakdown = cob_integration.get_exchange_breakdown('BTC/USDT') + +# Get CNN features +cnn_features = cob_integration.get_cob_features('BTC/USDT') + +# Get recent trading signals +signals = cob_integration.get_recent_signals('BTC/USDT', count=10) +``` + +### Market Analysis + +```python +# Market depth analysis +depth_analysis = cob_integration.get_market_depth_analysis('BTC/USDT') + +print(f"Active exchanges: {depth_analysis['exchanges_active']}") +print(f"Total liquidity: ${depth_analysis['total_bid_liquidity'] + depth_analysis['total_ask_liquidity']:,.0f}") +print(f"Spread: {depth_analysis['spread_bps']:.2f} bps") +print(f"Liquidity imbalance: {depth_analysis['liquidity_imbalance']:.3f}") +``` + +## Testing + +Use the provided test script to validate functionality: + +```bash +python test_multi_exchange_cob.py +``` + +The test script provides: +- **Basic functionality testing** +- **Feature generation validation** +- **Dashboard integration testing** +- **Signal analysis verification** +- **Performance monitoring** +- **Comprehensive test reporting** + +## Advantages Over BookMap + +### Our Implementation Benefits + +1. **Full Control**: Complete customization of aggregation logic +2. **Cost Effective**: Uses free exchange APIs instead of paid BookMap subscription +3. **Direct Integration**: Seamless integration with existing gogo2 architecture +4. **Extended Features**: Custom signal generation and analysis +5. **Fine-Grain Control**: Configurable bucket sizes and update frequencies +6. **Open Source**: Fully customizable and extensible + +### Comparison with BookMap Multibook + +| Feature | BookMap Multibook | Our Implementation | +|---------|------------------|-------------------| +| **Data Sources** | Pre-configured instruments | Fully configurable exchanges | +| **Cost** | Global Plus subscription | Free (exchange APIs) | +| **Integration** | BookMap platform only | Direct gogo2 integration | +| **Customization** | Limited | Full control | +| **Bucket Granularity** | Fixed by BookMap | Configurable (1 bps default) | +| **Signal Generation** | BookMap's algorithms | Custom trading signals | +| **AI Integration** | Limited | Native CNN/DQN features | +| **Real-time Updates** | BookMap frequency | 100ms configurable | + +## Future Enhancements + +### Planned Improvements + +1. **Additional Exchanges**: OKX, Bybit, KuCoin integration +2. **Options/Futures Support**: Extend beyond spot markets +3. **Advanced Analytics**: Machine learning-based pattern recognition +4. **Risk Management**: Real-time exposure and risk metrics +5. **Cross-Asset Analysis**: Multi-symbol correlation analysis +6. **Historical Analysis**: COB pattern backtesting +7. **API Rate Optimization**: Intelligent request management +8. **Fault Tolerance**: Exchange failover and redundancy + +### Performance Optimizations + +1. **WebSocket Pooling**: Shared connections for multiple symbols +2. **Data Compression**: Optimized data structures +3. **Caching Strategies**: Intelligent feature caching +4. **Parallel Processing**: Multi-threaded consolidation +5. **Memory Management**: Optimized historical data storage + +## Configuration + +### Exchange Configuration + +```python +exchange_configs = { + 'binance': ExchangeConfig( + exchange_type=ExchangeType.BINANCE, + weight=0.3, # 30% weight in aggregation + websocket_url="wss://stream.binance.com:9443/ws/", + symbols_mapping={'BTC/USDT': 'BTCUSDT'}, + rate_limits={'requests_per_minute': 1200} + ) +} +``` + +### Bucket Configuration + +```python +# Configure price bucket granularity +bucket_size_bps = 1.0 # 1 basis point per bucket +bucket_update_frequency = 100 # Update every 100ms +``` + +### Feature Configuration + +```python +# CNN feature dimensions +cnn_feature_config = { + 'order_book_levels': 20, + 'features_per_level': 5, + 'microstructure_features': 20, + 'total_dimensions': 220 +} +``` + +## Monitoring and Diagnostics + +### Performance Metrics + +- **Update rates**: COB updates per second +- **Processing latency**: Time from exchange update to consolidation +- **Feature generation time**: CNN/DQN feature computation time +- **Memory usage**: Data structure memory consumption +- **Connection health**: WebSocket connection status + +### Logging + +Comprehensive logging includes: +- Exchange connection events +- Order book update statistics +- Feature generation metrics +- Signal generation events +- Error handling and recovery + +## Conclusion + +The Multi-Exchange COB Provider successfully replicates and extends BookMap's Multibook functionality while providing: + +1. **Superior Integration** with the gogo2 trading system +2. **Cost Effectiveness** using free exchange APIs +3. **Enhanced Customization** for specific trading requirements +4. **Real-time Performance** optimized for high-frequency trading +5. **Advanced Analytics** with native AI model integration + +This implementation provides a robust foundation for multi-exchange order book analysis and represents a significant enhancement to the gogo2 trading platform's market data capabilities. + +## Files Created + +1. `core/multi_exchange_cob_provider.py` - Main COB aggregation engine +2. `core/cob_integration.py` - Integration layer with gogo2 system +3. `test_multi_exchange_cob.py` - Comprehensive testing framework +4. `MULTI_EXCHANGE_COB_PROVIDER_SUMMARY.md` - This documentation + +The system is ready for integration and testing with the existing gogo2 trading infrastructure. \ No newline at end of file diff --git a/core/bookmap_data_provider.py b/core/bookmap_data_provider.py new file mode 100644 index 0000000..c25bc47 --- /dev/null +++ b/core/bookmap_data_provider.py @@ -0,0 +1,952 @@ +""" +Bookmap Order Book Data Provider + +This module integrates with Bookmap to gather: +- Current Order Book (COB) data +- Session Volume Profile (SVP) data +- Order book sweeps and momentum trades detection +- Real-time order size heatmap matrix (last 10 minutes) +- Level 2 market depth analysis + +The data is processed and fed to CNN and DQN networks for enhanced trading decisions. +""" + +import asyncio +import json +import logging +import time +import websockets +import numpy as np +import pandas as pd +from datetime import datetime, timedelta +from typing import Dict, List, Optional, Tuple, Any, Callable +from collections import deque, defaultdict +from dataclasses import dataclass +from threading import Thread, Lock +import requests + +logger = logging.getLogger(__name__) + +@dataclass +class OrderBookLevel: + """Represents a single order book level""" + price: float + size: float + orders: int + side: str # 'bid' or 'ask' + timestamp: datetime + +@dataclass +class OrderBookSnapshot: + """Complete order book snapshot""" + symbol: str + timestamp: datetime + bids: List[OrderBookLevel] + asks: List[OrderBookLevel] + spread: float + mid_price: float + +@dataclass +class VolumeProfileLevel: + """Volume profile level data""" + price: float + volume: float + buy_volume: float + sell_volume: float + trades_count: int + vwap: float + +@dataclass +class OrderFlowSignal: + """Order flow signal detection""" + timestamp: datetime + signal_type: str # 'sweep', 'absorption', 'iceberg', 'momentum' + price: float + volume: float + confidence: float + description: str + +class BookmapDataProvider: + """ + Real-time order book data provider using Bookmap-style analysis + + Features: + - Level 2 order book monitoring + - Order flow detection (sweeps, absorptions) + - Volume profile analysis + - Order size heatmap generation + - Market microstructure analysis + """ + + def __init__(self, symbols: List[str] = None, depth_levels: int = 20): + """ + Initialize Bookmap data provider + + Args: + symbols: List of symbols to monitor + depth_levels: Number of order book levels to track + """ + self.symbols = symbols or ['ETHUSDT', 'BTCUSDT'] + self.depth_levels = depth_levels + self.is_streaming = False + + # Order book data storage + self.order_books: Dict[str, OrderBookSnapshot] = {} + self.order_book_history: Dict[str, deque] = {} + self.volume_profiles: Dict[str, List[VolumeProfileLevel]] = {} + + # Heatmap data (10-minute rolling window) + self.heatmap_window = timedelta(minutes=10) + self.order_heatmaps: Dict[str, deque] = {} + self.price_levels: Dict[str, List[float]] = {} + + # Order flow detection + self.flow_signals: Dict[str, deque] = {} + self.sweep_threshold = 0.8 # Minimum confidence for sweep detection + self.absorption_threshold = 0.7 # Minimum confidence for absorption + + # Market microstructure metrics + self.bid_ask_spreads: Dict[str, deque] = {} + self.order_book_imbalances: Dict[str, deque] = {} + self.liquidity_metrics: Dict[str, Dict] = {} + + # WebSocket connections + self.websocket_tasks: Dict[str, asyncio.Task] = {} + self.data_lock = Lock() + + # Callbacks for CNN/DQN integration + self.cnn_callbacks: List[Callable] = [] + self.dqn_callbacks: List[Callable] = [] + + # Performance tracking + self.update_counts = defaultdict(int) + self.last_update_times = {} + + # Initialize data structures + for symbol in self.symbols: + self.order_book_history[symbol] = deque(maxlen=1000) + self.order_heatmaps[symbol] = deque(maxlen=600) # 10 min at 1s intervals + self.flow_signals[symbol] = deque(maxlen=500) + self.bid_ask_spreads[symbol] = deque(maxlen=1000) + self.order_book_imbalances[symbol] = deque(maxlen=1000) + self.liquidity_metrics[symbol] = { + 'total_bid_size': 0.0, + 'total_ask_size': 0.0, + 'weighted_mid': 0.0, + 'liquidity_ratio': 1.0 + } + + logger.info(f"BookmapDataProvider initialized for {len(self.symbols)} symbols") + logger.info(f"Tracking {depth_levels} order book levels per side") + + def add_cnn_callback(self, callback: Callable[[str, Dict], None]): + """Add callback for CNN model updates""" + self.cnn_callbacks.append(callback) + logger.info(f"Added CNN callback: {len(self.cnn_callbacks)} total") + + def add_dqn_callback(self, callback: Callable[[str, Dict], None]): + """Add callback for DQN model updates""" + self.dqn_callbacks.append(callback) + logger.info(f"Added DQN callback: {len(self.dqn_callbacks)} total") + + async def start_streaming(self): + """Start real-time order book streaming""" + if self.is_streaming: + logger.warning("Bookmap streaming already active") + return + + self.is_streaming = True + logger.info("Starting Bookmap order book streaming") + + # Start order book streams for each symbol + for symbol in self.symbols: + # Order book depth stream + depth_task = asyncio.create_task(self._stream_order_book_depth(symbol)) + self.websocket_tasks[f"{symbol}_depth"] = depth_task + + # Trade stream for order flow analysis + trade_task = asyncio.create_task(self._stream_trades(symbol)) + self.websocket_tasks[f"{symbol}_trades"] = trade_task + + # Start analysis threads + analysis_task = asyncio.create_task(self._continuous_analysis()) + self.websocket_tasks["analysis"] = analysis_task + + logger.info(f"Started streaming for {len(self.symbols)} symbols") + + async def stop_streaming(self): + """Stop order book streaming""" + if not self.is_streaming: + return + + logger.info("Stopping Bookmap streaming") + self.is_streaming = False + + # Cancel all tasks + for name, task in self.websocket_tasks.items(): + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + self.websocket_tasks.clear() + logger.info("Bookmap streaming stopped") + + async def _stream_order_book_depth(self, symbol: str): + """Stream order book depth data""" + binance_symbol = symbol.lower() + url = f"wss://stream.binance.com:9443/ws/{binance_symbol}@depth20@100ms" + + while self.is_streaming: + try: + async with websockets.connect(url) as websocket: + logger.info(f"Order book depth WebSocket connected for {symbol}") + + async for message in websocket: + if not self.is_streaming: + break + + try: + data = json.loads(message) + await self._process_depth_update(symbol, data) + except Exception as e: + logger.warning(f"Error processing depth for {symbol}: {e}") + + except Exception as e: + logger.error(f"Depth WebSocket error for {symbol}: {e}") + if self.is_streaming: + await asyncio.sleep(2) + + async def _stream_trades(self, symbol: str): + """Stream trade data for order flow analysis""" + binance_symbol = symbol.lower() + url = f"wss://stream.binance.com:9443/ws/{binance_symbol}@trade" + + while self.is_streaming: + try: + async with websockets.connect(url) as websocket: + logger.info(f"Trade WebSocket connected for {symbol}") + + async for message in websocket: + if not self.is_streaming: + break + + try: + data = json.loads(message) + await self._process_trade_update(symbol, data) + except Exception as e: + logger.warning(f"Error processing trade for {symbol}: {e}") + + except Exception as e: + logger.error(f"Trade WebSocket error for {symbol}: {e}") + if self.is_streaming: + await asyncio.sleep(2) + + async def _process_depth_update(self, symbol: str, data: Dict): + """Process order book depth update""" + try: + timestamp = datetime.now() + + # Parse bids and asks + bids = [] + asks = [] + + for bid_data in data.get('bids', []): + price = float(bid_data[0]) + size = float(bid_data[1]) + bids.append(OrderBookLevel( + price=price, + size=size, + orders=1, # Binance doesn't provide order count + side='bid', + timestamp=timestamp + )) + + for ask_data in data.get('asks', []): + price = float(ask_data[0]) + size = float(ask_data[1]) + asks.append(OrderBookLevel( + price=price, + size=size, + orders=1, + side='ask', + timestamp=timestamp + )) + + # Sort order book levels + bids.sort(key=lambda x: x.price, reverse=True) + asks.sort(key=lambda x: x.price) + + # Calculate spread and mid price + if bids and asks: + best_bid = bids[0].price + best_ask = asks[0].price + spread = best_ask - best_bid + mid_price = (best_bid + best_ask) / 2 + else: + spread = 0.0 + mid_price = 0.0 + + # Create order book snapshot + snapshot = OrderBookSnapshot( + symbol=symbol, + timestamp=timestamp, + bids=bids, + asks=asks, + spread=spread, + mid_price=mid_price + ) + + with self.data_lock: + self.order_books[symbol] = snapshot + self.order_book_history[symbol].append(snapshot) + + # Update liquidity metrics + self._update_liquidity_metrics(symbol, snapshot) + + # Update order book imbalance + self._calculate_order_book_imbalance(symbol, snapshot) + + # Update heatmap data + self._update_order_heatmap(symbol, snapshot) + + # Update counters + self.update_counts[f"{symbol}_depth"] += 1 + self.last_update_times[f"{symbol}_depth"] = timestamp + + except Exception as e: + logger.error(f"Error processing depth update for {symbol}: {e}") + + async def _process_trade_update(self, symbol: str, data: Dict): + """Process trade data for order flow analysis""" + try: + timestamp = datetime.fromtimestamp(int(data['T']) / 1000) + price = float(data['p']) + quantity = float(data['q']) + is_buyer_maker = data['m'] + + # Analyze for order flow signals + await self._analyze_order_flow(symbol, timestamp, price, quantity, is_buyer_maker) + + # Update volume profile + self._update_volume_profile(symbol, price, quantity, is_buyer_maker) + + self.update_counts[f"{symbol}_trades"] += 1 + + except Exception as e: + logger.error(f"Error processing trade for {symbol}: {e}") + + def _update_liquidity_metrics(self, symbol: str, snapshot: OrderBookSnapshot): + """Update liquidity metrics from order book snapshot""" + try: + total_bid_size = sum(level.size for level in snapshot.bids) + total_ask_size = sum(level.size for level in snapshot.asks) + + # Calculate weighted mid price + if snapshot.bids and snapshot.asks: + bid_weight = total_bid_size / (total_bid_size + total_ask_size) + ask_weight = total_ask_size / (total_bid_size + total_ask_size) + weighted_mid = (snapshot.bids[0].price * ask_weight + + snapshot.asks[0].price * bid_weight) + else: + weighted_mid = snapshot.mid_price + + # Liquidity ratio (bid/ask balance) + if total_ask_size > 0: + liquidity_ratio = total_bid_size / total_ask_size + else: + liquidity_ratio = 1.0 + + self.liquidity_metrics[symbol] = { + 'total_bid_size': total_bid_size, + 'total_ask_size': total_ask_size, + 'weighted_mid': weighted_mid, + 'liquidity_ratio': liquidity_ratio, + 'spread_bps': (snapshot.spread / snapshot.mid_price) * 10000 if snapshot.mid_price > 0 else 0 + } + + except Exception as e: + logger.error(f"Error updating liquidity metrics for {symbol}: {e}") + + def _calculate_order_book_imbalance(self, symbol: str, snapshot: OrderBookSnapshot): + """Calculate order book imbalance ratio""" + try: + if not snapshot.bids or not snapshot.asks: + return + + # Calculate imbalance for top N levels + n_levels = min(5, len(snapshot.bids), len(snapshot.asks)) + + total_bid_size = sum(snapshot.bids[i].size for i in range(n_levels)) + total_ask_size = sum(snapshot.asks[i].size for i in range(n_levels)) + + if total_bid_size + total_ask_size > 0: + imbalance = (total_bid_size - total_ask_size) / (total_bid_size + total_ask_size) + else: + imbalance = 0.0 + + self.order_book_imbalances[symbol].append({ + 'timestamp': snapshot.timestamp, + 'imbalance': imbalance, + 'bid_size': total_bid_size, + 'ask_size': total_ask_size + }) + + except Exception as e: + logger.error(f"Error calculating imbalance for {symbol}: {e}") + + def _update_order_heatmap(self, symbol: str, snapshot: OrderBookSnapshot): + """Update order size heatmap matrix""" + try: + # Create heatmap entry + heatmap_entry = { + 'timestamp': snapshot.timestamp, + 'mid_price': snapshot.mid_price, + 'levels': {} + } + + # Add bid levels + for level in snapshot.bids: + price_offset = level.price - snapshot.mid_price + heatmap_entry['levels'][price_offset] = { + 'side': 'bid', + 'size': level.size, + 'price': level.price + } + + # Add ask levels + for level in snapshot.asks: + price_offset = level.price - snapshot.mid_price + heatmap_entry['levels'][price_offset] = { + 'side': 'ask', + 'size': level.size, + 'price': level.price + } + + self.order_heatmaps[symbol].append(heatmap_entry) + + # Clean old entries (keep 10 minutes) + cutoff_time = snapshot.timestamp - self.heatmap_window + while (self.order_heatmaps[symbol] and + self.order_heatmaps[symbol][0]['timestamp'] < cutoff_time): + self.order_heatmaps[symbol].popleft() + + except Exception as e: + logger.error(f"Error updating heatmap for {symbol}: {e}") + + def _update_volume_profile(self, symbol: str, price: float, quantity: float, is_buyer_maker: bool): + """Update volume profile with new trade""" + try: + # Initialize if not exists + if symbol not in self.volume_profiles: + self.volume_profiles[symbol] = [] + + # Find or create price level + price_level = None + for level in self.volume_profiles[symbol]: + if abs(level.price - price) < 0.01: # Price tolerance + price_level = level + break + + if not price_level: + price_level = VolumeProfileLevel( + price=price, + volume=0.0, + buy_volume=0.0, + sell_volume=0.0, + trades_count=0, + vwap=price + ) + self.volume_profiles[symbol].append(price_level) + + # Update volume profile + volume = price * quantity + old_total = price_level.volume + + price_level.volume += volume + price_level.trades_count += 1 + + if is_buyer_maker: + price_level.sell_volume += volume + else: + price_level.buy_volume += volume + + # Update VWAP + if price_level.volume > 0: + price_level.vwap = ((price_level.vwap * old_total) + (price * volume)) / price_level.volume + + except Exception as e: + logger.error(f"Error updating volume profile for {symbol}: {e}") + + async def _analyze_order_flow(self, symbol: str, timestamp: datetime, price: float, + quantity: float, is_buyer_maker: bool): + """Analyze order flow for sweep and absorption patterns""" + try: + # Get recent order book data + if symbol not in self.order_book_history or not self.order_book_history[symbol]: + return + + recent_snapshots = list(self.order_book_history[symbol])[-10:] # Last 10 snapshots + + # Check for order book sweeps + sweep_signal = self._detect_order_sweep(symbol, recent_snapshots, price, quantity, is_buyer_maker) + if sweep_signal: + self.flow_signals[symbol].append(sweep_signal) + await self._notify_flow_signal(symbol, sweep_signal) + + # Check for absorption patterns + absorption_signal = self._detect_absorption(symbol, recent_snapshots, price, quantity) + if absorption_signal: + self.flow_signals[symbol].append(absorption_signal) + await self._notify_flow_signal(symbol, absorption_signal) + + # Check for momentum trades + momentum_signal = self._detect_momentum_trade(symbol, price, quantity, is_buyer_maker) + if momentum_signal: + self.flow_signals[symbol].append(momentum_signal) + await self._notify_flow_signal(symbol, momentum_signal) + + except Exception as e: + logger.error(f"Error analyzing order flow for {symbol}: {e}") + + def _detect_order_sweep(self, symbol: str, snapshots: List[OrderBookSnapshot], + price: float, quantity: float, is_buyer_maker: bool) -> Optional[OrderFlowSignal]: + """Detect order book sweep patterns""" + try: + if len(snapshots) < 2: + return None + + before_snapshot = snapshots[-2] + after_snapshot = snapshots[-1] + + # Check if multiple levels were consumed + if is_buyer_maker: # Sell order, check ask side + levels_consumed = 0 + total_consumed_size = 0 + + for level in before_snapshot.asks[:5]: # Check top 5 levels + if level.price <= price: + levels_consumed += 1 + total_consumed_size += level.size + + if levels_consumed >= 2 and total_consumed_size > quantity * 1.5: + confidence = min(0.9, levels_consumed / 5.0 + 0.3) + + return OrderFlowSignal( + timestamp=datetime.now(), + signal_type='sweep', + price=price, + volume=quantity * price, + confidence=confidence, + description=f"Sell sweep: {levels_consumed} levels, {total_consumed_size:.2f} size" + ) + else: # Buy order, check bid side + levels_consumed = 0 + total_consumed_size = 0 + + for level in before_snapshot.bids[:5]: + if level.price >= price: + levels_consumed += 1 + total_consumed_size += level.size + + if levels_consumed >= 2 and total_consumed_size > quantity * 1.5: + confidence = min(0.9, levels_consumed / 5.0 + 0.3) + + return OrderFlowSignal( + timestamp=datetime.now(), + signal_type='sweep', + price=price, + volume=quantity * price, + confidence=confidence, + description=f"Buy sweep: {levels_consumed} levels, {total_consumed_size:.2f} size" + ) + + return None + + except Exception as e: + logger.error(f"Error detecting sweep for {symbol}: {e}") + return None + + def _detect_absorption(self, symbol: str, snapshots: List[OrderBookSnapshot], + price: float, quantity: float) -> Optional[OrderFlowSignal]: + """Detect absorption patterns where large orders are absorbed without price movement""" + try: + if len(snapshots) < 3: + return None + + # Check if large order was absorbed with minimal price impact + volume_threshold = 10000 # $10K minimum for absorption + price_impact_threshold = 0.001 # 0.1% max price impact + + trade_value = price * quantity + if trade_value < volume_threshold: + return None + + # Calculate price impact + price_before = snapshots[-3].mid_price + price_after = snapshots[-1].mid_price + price_impact = abs(price_after - price_before) / price_before + + if price_impact < price_impact_threshold: + confidence = min(0.8, (trade_value / 50000) * 0.5 + 0.3) # Scale with size + + return OrderFlowSignal( + timestamp=datetime.now(), + signal_type='absorption', + price=price, + volume=trade_value, + confidence=confidence, + description=f"Absorption: ${trade_value:.0f} with {price_impact*100:.3f}% impact" + ) + + return None + + except Exception as e: + logger.error(f"Error detecting absorption for {symbol}: {e}") + return None + + def _detect_momentum_trade(self, symbol: str, price: float, quantity: float, + is_buyer_maker: bool) -> Optional[OrderFlowSignal]: + """Detect momentum trades based on size and direction""" + try: + trade_value = price * quantity + momentum_threshold = 25000 # $25K minimum for momentum classification + + if trade_value < momentum_threshold: + return None + + # Calculate confidence based on trade size + confidence = min(0.9, trade_value / 100000 * 0.6 + 0.3) + + direction = "sell" if is_buyer_maker else "buy" + + return OrderFlowSignal( + timestamp=datetime.now(), + signal_type='momentum', + price=price, + volume=trade_value, + confidence=confidence, + description=f"Large {direction}: ${trade_value:.0f}" + ) + + except Exception as e: + logger.error(f"Error detecting momentum for {symbol}: {e}") + return None + + async def _notify_flow_signal(self, symbol: str, signal: OrderFlowSignal): + """Notify CNN and DQN models of order flow signals""" + try: + signal_data = { + 'signal_type': signal.signal_type, + 'price': signal.price, + 'volume': signal.volume, + 'confidence': signal.confidence, + 'timestamp': signal.timestamp, + 'description': signal.description + } + + # Notify CNN callbacks + for callback in self.cnn_callbacks: + try: + callback(symbol, signal_data) + except Exception as e: + logger.warning(f"Error in CNN callback: {e}") + + # Notify DQN callbacks + for callback in self.dqn_callbacks: + try: + callback(symbol, signal_data) + except Exception as e: + logger.warning(f"Error in DQN callback: {e}") + + except Exception as e: + logger.error(f"Error notifying flow signal: {e}") + + async def _continuous_analysis(self): + """Continuous analysis of market microstructure""" + while self.is_streaming: + try: + await asyncio.sleep(1) # Analyze every second + + for symbol in self.symbols: + # Generate CNN features + cnn_features = self.get_cnn_features(symbol) + if cnn_features is not None: + for callback in self.cnn_callbacks: + try: + callback(symbol, {'features': cnn_features, 'type': 'orderbook'}) + except Exception as e: + logger.warning(f"Error in CNN feature callback: {e}") + + # Generate DQN state features + dqn_features = self.get_dqn_state_features(symbol) + if dqn_features is not None: + for callback in self.dqn_callbacks: + try: + callback(symbol, {'state': dqn_features, 'type': 'orderbook'}) + except Exception as e: + logger.warning(f"Error in DQN state callback: {e}") + + except Exception as e: + logger.error(f"Error in continuous analysis: {e}") + await asyncio.sleep(5) + + def get_cnn_features(self, symbol: str) -> Optional[np.ndarray]: + """Generate CNN input features from order book data""" + try: + if symbol not in self.order_books: + return None + + snapshot = self.order_books[symbol] + features = [] + + # Order book features (40 features: 20 levels x 2 sides) + for i in range(min(20, len(snapshot.bids))): + bid = snapshot.bids[i] + features.append(bid.size) + features.append(bid.price - snapshot.mid_price) # Price offset + + # Pad if not enough bid levels + while len(features) < 40: + features.extend([0.0, 0.0]) + + for i in range(min(20, len(snapshot.asks))): + ask = snapshot.asks[i] + features.append(ask.size) + features.append(ask.price - snapshot.mid_price) # Price offset + + # Pad if not enough ask levels + while len(features) < 80: + features.extend([0.0, 0.0]) + + # Liquidity metrics (10 features) + metrics = self.liquidity_metrics.get(symbol, {}) + features.extend([ + metrics.get('total_bid_size', 0.0), + metrics.get('total_ask_size', 0.0), + metrics.get('liquidity_ratio', 1.0), + metrics.get('spread_bps', 0.0), + snapshot.spread, + metrics.get('weighted_mid', snapshot.mid_price) - snapshot.mid_price, + len(snapshot.bids), + len(snapshot.asks), + snapshot.mid_price, + time.time() % 86400 # Time of day + ]) + + # Order book imbalance features (5 features) + if self.order_book_imbalances[symbol]: + latest_imbalance = self.order_book_imbalances[symbol][-1] + features.extend([ + latest_imbalance['imbalance'], + latest_imbalance['bid_size'], + latest_imbalance['ask_size'], + latest_imbalance['bid_size'] + latest_imbalance['ask_size'], + abs(latest_imbalance['imbalance']) + ]) + else: + features.extend([0.0, 0.0, 0.0, 0.0, 0.0]) + + # Flow signal features (5 features) + recent_signals = [s for s in self.flow_signals[symbol] + if (datetime.now() - s.timestamp).seconds < 60] + + sweep_count = sum(1 for s in recent_signals if s.signal_type == 'sweep') + absorption_count = sum(1 for s in recent_signals if s.signal_type == 'absorption') + momentum_count = sum(1 for s in recent_signals if s.signal_type == 'momentum') + + max_confidence = max([s.confidence for s in recent_signals], default=0.0) + total_flow_volume = sum(s.volume for s in recent_signals) + + features.extend([ + sweep_count, + absorption_count, + momentum_count, + max_confidence, + total_flow_volume + ]) + + return np.array(features, dtype=np.float32) + + except Exception as e: + logger.error(f"Error generating CNN features for {symbol}: {e}") + return None + + def get_dqn_state_features(self, symbol: str) -> Optional[np.ndarray]: + """Generate DQN state features from order book data""" + try: + if symbol not in self.order_books: + return None + + snapshot = self.order_books[symbol] + state_features = [] + + # Normalized order book state (20 features) + total_bid_size = sum(level.size for level in snapshot.bids[:10]) + total_ask_size = sum(level.size for level in snapshot.asks[:10]) + total_size = total_bid_size + total_ask_size + + if total_size > 0: + for i in range(min(10, len(snapshot.bids))): + state_features.append(snapshot.bids[i].size / total_size) + + # Pad bids + while len(state_features) < 10: + state_features.append(0.0) + + for i in range(min(10, len(snapshot.asks))): + state_features.append(snapshot.asks[i].size / total_size) + + # Pad asks + while len(state_features) < 20: + state_features.append(0.0) + else: + state_features.extend([0.0] * 20) + + # Market state indicators (10 features) + metrics = self.liquidity_metrics.get(symbol, {}) + + # Normalize spread as percentage + spread_pct = (snapshot.spread / snapshot.mid_price) if snapshot.mid_price > 0 else 0 + + # Liquidity imbalance + liquidity_ratio = metrics.get('liquidity_ratio', 1.0) + liquidity_imbalance = (liquidity_ratio - 1) / (liquidity_ratio + 1) + + # Recent flow signals strength + recent_signals = [s for s in self.flow_signals[symbol] + if (datetime.now() - s.timestamp).seconds < 30] + flow_strength = sum(s.confidence for s in recent_signals) / max(len(recent_signals), 1) + + # Price volatility (from recent snapshots) + if len(self.order_book_history[symbol]) >= 10: + recent_prices = [s.mid_price for s in list(self.order_book_history[symbol])[-10:]] + price_volatility = np.std(recent_prices) / np.mean(recent_prices) if recent_prices else 0 + else: + price_volatility = 0 + + state_features.extend([ + spread_pct * 10000, # Spread in basis points + liquidity_imbalance, + flow_strength, + price_volatility * 100, # Volatility as percentage + min(len(snapshot.bids), 20) / 20, # Book depth ratio + min(len(snapshot.asks), 20) / 20, + sweep_count / 10 if 'sweep_count' in locals() else 0, # From CNN features + absorption_count / 5 if 'absorption_count' in locals() else 0, + momentum_count / 5 if 'momentum_count' in locals() else 0, + (datetime.now().hour * 60 + datetime.now().minute) / 1440 # Time of day normalized + ]) + + return np.array(state_features, dtype=np.float32) + + except Exception as e: + logger.error(f"Error generating DQN features for {symbol}: {e}") + return None + + def get_order_heatmap_matrix(self, symbol: str, levels: int = 40) -> Optional[np.ndarray]: + """Generate order size heatmap matrix for dashboard visualization""" + try: + if symbol not in self.order_heatmaps or not self.order_heatmaps[symbol]: + return None + + # Create price levels around current mid price + current_snapshot = self.order_books.get(symbol) + if not current_snapshot: + return None + + mid_price = current_snapshot.mid_price + price_step = mid_price * 0.0001 # 1 basis point steps + + # Create matrix: time x price levels + time_window = min(600, len(self.order_heatmaps[symbol])) # 10 minutes max + heatmap_matrix = np.zeros((time_window, levels)) + + # Fill matrix with order sizes + for t, entry in enumerate(list(self.order_heatmaps[symbol])[-time_window:]): + for price_offset, level_data in entry['levels'].items(): + # Convert price offset to matrix index + level_idx = int((price_offset + (levels/2) * price_step) / price_step) + + if 0 <= level_idx < levels: + size_weight = 1.0 if level_data['side'] == 'bid' else -1.0 + heatmap_matrix[t, level_idx] = level_data['size'] * size_weight + + return heatmap_matrix + + except Exception as e: + logger.error(f"Error generating heatmap matrix for {symbol}: {e}") + return None + + def get_volume_profile_data(self, symbol: str) -> Optional[List[Dict]]: + """Get session volume profile data""" + try: + if symbol not in self.volume_profiles: + return None + + profile_data = [] + for level in sorted(self.volume_profiles[symbol], key=lambda x: x.price): + profile_data.append({ + 'price': level.price, + 'volume': level.volume, + 'buy_volume': level.buy_volume, + 'sell_volume': level.sell_volume, + 'trades_count': level.trades_count, + 'vwap': level.vwap, + 'net_volume': level.buy_volume - level.sell_volume + }) + + return profile_data + + except Exception as e: + logger.error(f"Error getting volume profile for {symbol}: {e}") + return None + + def get_current_order_book(self, symbol: str) -> Optional[Dict]: + """Get current order book snapshot""" + try: + if symbol not in self.order_books: + return None + + snapshot = self.order_books[symbol] + + return { + 'timestamp': snapshot.timestamp.isoformat(), + 'symbol': symbol, + 'mid_price': snapshot.mid_price, + 'spread': snapshot.spread, + 'bids': [{'price': l.price, 'size': l.size} for l in snapshot.bids[:20]], + 'asks': [{'price': l.price, 'size': l.size} for l in snapshot.asks[:20]], + 'liquidity_metrics': self.liquidity_metrics.get(symbol, {}), + 'recent_signals': [ + { + 'type': s.signal_type, + 'price': s.price, + 'volume': s.volume, + 'confidence': s.confidence, + 'timestamp': s.timestamp.isoformat() + } + for s in list(self.flow_signals[symbol])[-5:] # Last 5 signals + ] + } + + except Exception as e: + logger.error(f"Error getting order book for {symbol}: {e}") + return None + + def get_statistics(self) -> Dict[str, Any]: + """Get provider statistics""" + return { + 'symbols': self.symbols, + 'is_streaming': self.is_streaming, + 'update_counts': dict(self.update_counts), + 'last_update_times': {k: v.isoformat() if isinstance(v, datetime) else v + for k, v in self.last_update_times.items()}, + 'order_books_active': len(self.order_books), + 'flow_signals_total': sum(len(signals) for signals in self.flow_signals.values()), + 'cnn_callbacks': len(self.cnn_callbacks), + 'dqn_callbacks': len(self.dqn_callbacks), + 'websocket_tasks': len(self.websocket_tasks) + } \ No newline at end of file diff --git a/core/bookmap_integration.py b/core/bookmap_integration.py new file mode 100644 index 0000000..904e253 --- /dev/null +++ b/core/bookmap_integration.py @@ -0,0 +1,1839 @@ +""" +Order Book Analysis Integration (Free Data Sources) + +This module provides Bookmap-style functionality using free order book data: +- Current Order Book (COB) analysis using Binance free depth streams +- Session Volume Profile (SVP) calculated from trade and depth data +- Order flow detection (sweeps, absorptions, momentum) +- Real-time order book heatmap generation +- Level 2 market depth streaming (20 levels via Binance free API) + +Data is fed to CNN and DQN networks for enhanced trading decisions. +Uses only free data sources - no paid APIs required. +""" + +import asyncio +import json +import logging +import time +import websockets +import numpy as np +import pandas as pd +from datetime import datetime, timedelta +from typing import Dict, List, Optional, Tuple, Any, Callable +from collections import deque, defaultdict +from dataclasses import dataclass +from threading import Thread, Lock +import requests + +logger = logging.getLogger(__name__) + +@dataclass +class OrderBookLevel: + """Single order book level""" + price: float + size: float + orders: int + side: str # 'bid' or 'ask' + timestamp: datetime + +@dataclass +class OrderBookSnapshot: + """Complete order book snapshot""" + symbol: str + timestamp: datetime + bids: List[OrderBookLevel] + asks: List[OrderBookLevel] + spread: float + mid_price: float + +@dataclass +class VolumeProfileLevel: + """Volume profile level data""" + price: float + volume: float + buy_volume: float + sell_volume: float + trades_count: int + vwap: float + +@dataclass +class OrderFlowSignal: + """Order flow signal detection""" + timestamp: datetime + signal_type: str # 'sweep', 'absorption', 'iceberg', 'momentum' + price: float + volume: float + confidence: float + description: str + +class BookmapIntegration: + """ + Order book analysis using free data sources + + Features: + - Real-time order book monitoring (Binance free depth@20 levels) + - Order flow pattern detection + - Enhanced Session Volume Profile (SVP) analysis + - Market microstructure metrics + - CNN/DQN model integration + - High-frequency order book snapshots for pattern detection + """ + + def __init__(self, symbols: List[str] = None): + self.symbols = symbols or ['ETHUSDT', 'BTCUSDT'] + self.is_streaming = False + + # Data storage + self.order_books: Dict[str, OrderBookSnapshot] = {} + self.order_book_history: Dict[str, deque] = {} + self.volume_profiles: Dict[str, List[VolumeProfileLevel]] = {} + self.flow_signals: Dict[str, deque] = {} + + # Enhanced Session Volume Profile tracking + self.session_start_time = {} # Track session start for each symbol + self.session_volume_profiles: Dict[str, List[VolumeProfileLevel]] = {} + self.price_level_cache: Dict[str, Dict[float, VolumeProfileLevel]] = {} + + # Heatmap data (10-minute rolling window) + self.heatmap_window = timedelta(minutes=10) + self.order_heatmaps: Dict[str, deque] = {} + + # Market metrics + self.liquidity_metrics: Dict[str, Dict] = {} + self.order_book_imbalances: Dict[str, deque] = {} + + # Enhanced Order Flow Analysis + self.aggressive_passive_ratios: Dict[str, deque] = {} + self.trade_size_distributions: Dict[str, deque] = {} + self.market_maker_taker_flows: Dict[str, deque] = {} + self.order_flow_intensity: Dict[str, deque] = {} + self.liquidity_consumption_rates: Dict[str, deque] = {} + self.price_impact_measurements: Dict[str, deque] = {} + + # Advanced metrics for institutional vs retail detection + self.large_order_threshold = 50000 # $50K+ considered institutional + self.block_trade_threshold = 100000 # $100K+ considered block trades + self.iceberg_detection_window = 30 # seconds for iceberg detection + self.trade_clustering_window = 5 # seconds for trade clustering analysis + + # Free data source optimization + self.depth_snapshots_per_second = 10 # 100ms updates = 10 per second + self.trade_aggregation_window = 1.0 # 1 second aggregation + self.last_trade_aggregation = {} + + # WebSocket connections + self.websocket_tasks: Dict[str, asyncio.Task] = {} + self.data_lock = Lock() + + # Model callbacks + self.cnn_callbacks: List[Callable] = [] + self.dqn_callbacks: List[Callable] = [] + + # Initialize data structures + for symbol in self.symbols: + self.order_book_history[symbol] = deque(maxlen=1000) + self.order_heatmaps[symbol] = deque(maxlen=600) # 10 min at 1s intervals + self.flow_signals[symbol] = deque(maxlen=500) + self.order_book_imbalances[symbol] = deque(maxlen=1000) + self.session_volume_profiles[symbol] = [] + self.price_level_cache[symbol] = {} + self.session_start_time[symbol] = datetime.now() + self.last_trade_aggregation[symbol] = datetime.now() + + # Enhanced order flow analysis buffers + self.aggressive_passive_ratios[symbol] = deque(maxlen=300) # 5 minutes at 1s intervals + self.trade_size_distributions[symbol] = deque(maxlen=1000) + self.market_maker_taker_flows[symbol] = deque(maxlen=600) + self.order_flow_intensity[symbol] = deque(maxlen=300) + self.liquidity_consumption_rates[symbol] = deque(maxlen=300) + self.price_impact_measurements[symbol] = deque(maxlen=300) + + self.liquidity_metrics[symbol] = { + 'total_bid_size': 0.0, + 'total_ask_size': 0.0, + 'weighted_mid': 0.0, + 'liquidity_ratio': 1.0, + 'avg_spread_bps': 0.0, + 'volume_weighted_spread': 0.0 + } + + logger.info(f"Order Book Integration initialized for symbols: {self.symbols}") + logger.info("Using FREE data sources: Binance WebSocket depth@20 + trades") + + def add_cnn_callback(self, callback: Callable[[str, Dict], None]): + """Add CNN model callback""" + self.cnn_callbacks.append(callback) + logger.info(f"Added CNN callback: {len(self.cnn_callbacks)} total") + + def add_dqn_callback(self, callback: Callable[[str, Dict], None]): + """Add DQN model callback""" + self.dqn_callbacks.append(callback) + logger.info(f"Added DQN callback: {len(self.dqn_callbacks)} total") + + async def start_streaming(self): + """Start order book data streaming""" + if self.is_streaming: + logger.warning("Bookmap streaming already active") + return + + self.is_streaming = True + logger.info("Starting Bookmap order book streaming") + + # Start streams for each symbol + for symbol in self.symbols: + # Order book depth stream (20 levels, 100ms updates) + depth_task = asyncio.create_task(self._stream_order_book_depth(symbol)) + self.websocket_tasks[f"{symbol}_depth"] = depth_task + + # Trade stream for order flow analysis + trade_task = asyncio.create_task(self._stream_trades(symbol)) + self.websocket_tasks[f"{symbol}_trades"] = trade_task + + # Aggregated trade stream (for larger trades and better order flow analysis) + agg_trade_task = asyncio.create_task(self._stream_aggregate_trades(symbol)) + self.websocket_tasks[f"{symbol}_aggTrade"] = agg_trade_task + + # 24hr ticker stream (for volume and statistical analysis) + ticker_task = asyncio.create_task(self._stream_ticker(symbol)) + self.websocket_tasks[f"{symbol}_ticker"] = ticker_task + + # Start continuous analysis + analysis_task = asyncio.create_task(self._continuous_analysis()) + self.websocket_tasks["analysis"] = analysis_task + + logger.info(f"Started streaming for {len(self.symbols)} symbols") + + async def stop_streaming(self): + """Stop streaming""" + if not self.is_streaming: + return + + logger.info("Stopping Bookmap streaming") + self.is_streaming = False + + # Cancel all tasks + for name, task in self.websocket_tasks.items(): + if not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + self.websocket_tasks.clear() + logger.info("Bookmap streaming stopped") + + async def _stream_order_book_depth(self, symbol: str): + """Stream order book depth data""" + binance_symbol = symbol.lower() + url = f"wss://stream.binance.com:9443/ws/{binance_symbol}@depth20@100ms" + + while self.is_streaming: + try: + async with websockets.connect(url) as websocket: + logger.info(f"Order book depth connected for {symbol}") + + async for message in websocket: + if not self.is_streaming: + break + + try: + data = json.loads(message) + await self._process_depth_update(symbol, data) + except Exception as e: + logger.warning(f"Error processing depth for {symbol}: {e}") + + except Exception as e: + logger.error(f"Depth WebSocket error for {symbol}: {e}") + if self.is_streaming: + await asyncio.sleep(2) + + async def _stream_trades(self, symbol: str): + """Stream individual trade data for order flow analysis""" + binance_symbol = symbol.lower() + url = f"wss://stream.binance.com:9443/ws/{binance_symbol}@trade" + + while self.is_streaming: + try: + async with websockets.connect(url) as websocket: + logger.info(f"Trade stream connected for {symbol}") + + async for message in websocket: + if not self.is_streaming: + break + + try: + data = json.loads(message) + await self._process_trade_update(symbol, data) + except Exception as e: + logger.warning(f"Error processing trade for {symbol}: {e}") + + except Exception as e: + logger.error(f"Trade WebSocket error for {symbol}: {e}") + if self.is_streaming: + await asyncio.sleep(2) + + async def _stream_aggregate_trades(self, symbol: str): + """Stream aggregated trade data for institutional order flow detection""" + binance_symbol = symbol.lower() + url = f"wss://stream.binance.com:9443/ws/{binance_symbol}@aggTrade" + + while self.is_streaming: + try: + async with websockets.connect(url) as websocket: + logger.info(f"Aggregate Trade stream connected for {symbol}") + + async for message in websocket: + if not self.is_streaming: + break + + try: + data = json.loads(message) + await self._process_aggregate_trade_update(symbol, data) + except Exception as e: + logger.warning(f"Error processing aggTrade for {symbol}: {e}") + + except Exception as e: + logger.error(f"Aggregate Trade WebSocket error for {symbol}: {e}") + if self.is_streaming: + await asyncio.sleep(2) + + async def _stream_ticker(self, symbol: str): + """Stream 24hr ticker data for volume analysis""" + binance_symbol = symbol.lower() + url = f"wss://stream.binance.com:9443/ws/{binance_symbol}@ticker" + + while self.is_streaming: + try: + async with websockets.connect(url) as websocket: + logger.info(f"Ticker stream connected for {symbol}") + + async for message in websocket: + if not self.is_streaming: + break + + try: + data = json.loads(message) + await self._process_ticker_update(symbol, data) + except Exception as e: + logger.warning(f"Error processing ticker for {symbol}: {e}") + + except Exception as e: + logger.error(f"Ticker WebSocket error for {symbol}: {e}") + if self.is_streaming: + await asyncio.sleep(2) + + async def _process_depth_update(self, symbol: str, data: Dict): + """Process order book depth update""" + try: + timestamp = datetime.now() + + # Parse bids and asks + bids = [] + asks = [] + + for bid_data in data.get('bids', []): + price = float(bid_data[0]) + size = float(bid_data[1]) + bids.append(OrderBookLevel( + price=price, + size=size, + orders=1, + side='bid', + timestamp=timestamp + )) + + for ask_data in data.get('asks', []): + price = float(ask_data[0]) + size = float(ask_data[1]) + asks.append(OrderBookLevel( + price=price, + size=size, + orders=1, + side='ask', + timestamp=timestamp + )) + + # Sort levels + bids.sort(key=lambda x: x.price, reverse=True) + asks.sort(key=lambda x: x.price) + + # Calculate spread and mid price + if bids and asks: + best_bid = bids[0].price + best_ask = asks[0].price + spread = best_ask - best_bid + mid_price = (best_bid + best_ask) / 2 + else: + spread = 0.0 + mid_price = 0.0 + + # Create snapshot + snapshot = OrderBookSnapshot( + symbol=symbol, + timestamp=timestamp, + bids=bids, + asks=asks, + spread=spread, + mid_price=mid_price + ) + + with self.data_lock: + self.order_books[symbol] = snapshot + self.order_book_history[symbol].append(snapshot) + + # Update metrics + self._update_liquidity_metrics(symbol, snapshot) + self._calculate_order_book_imbalance(symbol, snapshot) + self._update_order_heatmap(symbol, snapshot) + + except Exception as e: + logger.error(f"Error processing depth update for {symbol}: {e}") + + async def _process_trade_update(self, symbol: str, data: Dict): + """Process individual trade data with enhanced order flow analysis""" + try: + timestamp = datetime.fromtimestamp(int(data['T']) / 1000) + price = float(data['p']) + quantity = float(data['q']) + is_buyer_maker = data['m'] + trade_id = data.get('t', '') + + # Calculate trade value + trade_value = price * quantity + + # Enhanced order flow analysis + await self._analyze_enhanced_order_flow(symbol, timestamp, price, quantity, trade_value, is_buyer_maker, 'individual') + + # Traditional order flow analysis + await self._analyze_order_flow(symbol, timestamp, price, quantity, is_buyer_maker) + + # Update volume profile + self._update_volume_profile(symbol, price, quantity, is_buyer_maker) + + except Exception as e: + logger.error(f"Error processing trade for {symbol}: {e}") + + async def _process_aggregate_trade_update(self, symbol: str, data: Dict): + """Process aggregated trade data for institutional flow detection""" + try: + timestamp = datetime.fromtimestamp(int(data['T']) / 1000) + price = float(data['p']) + quantity = float(data['q']) + is_buyer_maker = data['m'] + first_trade_id = data.get('f', '') + last_trade_id = data.get('l', '') + + # Calculate trade value and aggregation size + trade_value = price * quantity + trade_aggregation_size = int(last_trade_id) - int(first_trade_id) + 1 if first_trade_id and last_trade_id else 1 + + # Enhanced analysis for aggregated trades (institutional detection) + await self._analyze_enhanced_order_flow(symbol, timestamp, price, quantity, trade_value, is_buyer_maker, 'aggregated', trade_aggregation_size) + + # Detect large block trades and iceberg orders + await self._detect_institutional_activity(symbol, timestamp, price, quantity, trade_value, trade_aggregation_size, is_buyer_maker) + + except Exception as e: + logger.error(f"Error processing aggregate trade for {symbol}: {e}") + + async def _process_ticker_update(self, symbol: str, data: Dict): + """Process ticker data for volume and statistical analysis""" + try: + # Extract relevant ticker data + volume_24h = float(data.get('v', 0)) # 24hr volume + quote_volume_24h = float(data.get('q', 0)) # 24hr quote volume + price_change_24h = float(data.get('P', 0)) # 24hr price change % + high_24h = float(data.get('h', 0)) + low_24h = float(data.get('l', 0)) + weighted_avg_price = float(data.get('w', 0)) # Weighted average price + + # Update volume statistics for relative analysis + self._update_volume_statistics(symbol, volume_24h, quote_volume_24h, weighted_avg_price) + + except Exception as e: + logger.error(f"Error processing ticker for {symbol}: {e}") + + def _update_liquidity_metrics(self, symbol: str, snapshot: OrderBookSnapshot): + """Update liquidity metrics""" + try: + total_bid_size = sum(level.size for level in snapshot.bids) + total_ask_size = sum(level.size for level in snapshot.asks) + + # Weighted mid price + if snapshot.bids and snapshot.asks: + bid_weight = total_bid_size / (total_bid_size + total_ask_size) + ask_weight = total_ask_size / (total_bid_size + total_ask_size) + weighted_mid = (snapshot.bids[0].price * ask_weight + + snapshot.asks[0].price * bid_weight) + else: + weighted_mid = snapshot.mid_price + + # Liquidity ratio + liquidity_ratio = total_bid_size / total_ask_size if total_ask_size > 0 else 1.0 + + self.liquidity_metrics[symbol] = { + 'total_bid_size': total_bid_size, + 'total_ask_size': total_ask_size, + 'weighted_mid': weighted_mid, + 'liquidity_ratio': liquidity_ratio, + 'spread_bps': (snapshot.spread / snapshot.mid_price) * 10000 if snapshot.mid_price > 0 else 0 + } + + except Exception as e: + logger.error(f"Error updating liquidity metrics for {symbol}: {e}") + + def _calculate_order_book_imbalance(self, symbol: str, snapshot: OrderBookSnapshot): + """Calculate order book imbalance""" + try: + if not snapshot.bids or not snapshot.asks: + return + + # Top 5 levels imbalance + n_levels = min(5, len(snapshot.bids), len(snapshot.asks)) + + total_bid_size = sum(snapshot.bids[i].size for i in range(n_levels)) + total_ask_size = sum(snapshot.asks[i].size for i in range(n_levels)) + + if total_bid_size + total_ask_size > 0: + imbalance = (total_bid_size - total_ask_size) / (total_bid_size + total_ask_size) + else: + imbalance = 0.0 + + self.order_book_imbalances[symbol].append({ + 'timestamp': snapshot.timestamp, + 'imbalance': imbalance, + 'bid_size': total_bid_size, + 'ask_size': total_ask_size + }) + + except Exception as e: + logger.error(f"Error calculating imbalance for {symbol}: {e}") + + def _update_order_heatmap(self, symbol: str, snapshot: OrderBookSnapshot): + """Update order heatmap matrix""" + try: + heatmap_entry = { + 'timestamp': snapshot.timestamp, + 'mid_price': snapshot.mid_price, + 'levels': {} + } + + # Add bid levels + for level in snapshot.bids: + price_offset = level.price - snapshot.mid_price + heatmap_entry['levels'][price_offset] = { + 'side': 'bid', + 'size': level.size, + 'price': level.price + } + + # Add ask levels + for level in snapshot.asks: + price_offset = level.price - snapshot.mid_price + heatmap_entry['levels'][price_offset] = { + 'side': 'ask', + 'size': level.size, + 'price': level.price + } + + self.order_heatmaps[symbol].append(heatmap_entry) + + # Clean old entries + cutoff_time = snapshot.timestamp - self.heatmap_window + while (self.order_heatmaps[symbol] and + self.order_heatmaps[symbol][0]['timestamp'] < cutoff_time): + self.order_heatmaps[symbol].popleft() + + except Exception as e: + logger.error(f"Error updating heatmap for {symbol}: {e}") + + def _update_volume_profile(self, symbol: str, price: float, quantity: float, is_buyer_maker: bool): + """Enhanced Session Volume Profile (SVP) update using free data""" + try: + # Calculate trade volume in USDT + volume = price * quantity + + # Use price level caching for better performance + price_key = round(price, 2) # Round to 2 decimal places for price level grouping + + # Update session volume profile + if price_key not in self.price_level_cache[symbol]: + self.price_level_cache[symbol][price_key] = VolumeProfileLevel( + price=price_key, + volume=0.0, + buy_volume=0.0, + sell_volume=0.0, + trades_count=0, + vwap=price + ) + + level = self.price_level_cache[symbol][price_key] + old_total_volume = level.volume + old_total_quantity = level.trades_count + + # Update volume metrics + level.volume += volume + level.trades_count += 1 + + # Update buy/sell volume breakdown + if is_buyer_maker: + level.sell_volume += volume # Market maker is selling + else: + level.buy_volume += volume # Market maker is buying + + # Calculate Volume Weighted Average Price (VWAP) for this level + if level.volume > 0: + level.vwap = ((level.vwap * old_total_volume) + (price * volume)) / level.volume + + # Also update the rolling volume profile (last 10 minutes) + self._update_rolling_volume_profile(symbol, price_key, volume, is_buyer_maker) + + # Session reset detection (every 24 hours or major price gaps) + current_time = datetime.now() + if self._should_reset_session(symbol, current_time, price): + self._reset_session_volume_profile(symbol, current_time) + + except Exception as e: + logger.error(f"Error updating Session Volume Profile for {symbol}: {e}") + + def _update_rolling_volume_profile(self, symbol: str, price_key: float, volume: float, is_buyer_maker: bool): + """Update rolling 10-minute volume profile for real-time heatmap""" + try: + # Find or create level in regular volume profile + price_level = None + for level in self.volume_profiles.get(symbol, []): + if abs(level.price - price_key) < 0.01: + price_level = level + break + + if not price_level: + if symbol not in self.volume_profiles: + self.volume_profiles[symbol] = [] + + price_level = VolumeProfileLevel( + price=price_key, + volume=0.0, + buy_volume=0.0, + sell_volume=0.0, + trades_count=0, + vwap=price_key + ) + self.volume_profiles[symbol].append(price_level) + + # Update rolling metrics + old_volume = price_level.volume + price_level.volume += volume + price_level.trades_count += 1 + + if is_buyer_maker: + price_level.sell_volume += volume + else: + price_level.buy_volume += volume + + # Update VWAP + if price_level.volume > 0: + price_level.vwap = ((price_level.vwap * old_volume) + (price_key * volume)) / price_level.volume + + except Exception as e: + logger.error(f"Error updating rolling volume profile for {symbol}: {e}") + + def _should_reset_session(self, symbol: str, current_time: datetime, current_price: float) -> bool: + """Determine if session volume profile should be reset""" + try: + session_start = self.session_start_time.get(symbol) + if not session_start: + return False + + # Reset every 24 hours (daily session) + if (current_time - session_start).total_seconds() > 86400: # 24 hours + return True + + # Reset on major price gaps (> 5% from session VWAP) + if self.price_level_cache.get(symbol): + total_volume = sum(level.volume for level in self.price_level_cache[symbol].values()) + if total_volume > 0: + weighted_price = sum(level.vwap * level.volume for level in self.price_level_cache[symbol].values()) / total_volume + price_gap = abs(current_price - weighted_price) / weighted_price + if price_gap > 0.05: # 5% gap + return True + + return False + + except Exception as e: + logger.error(f"Error checking session reset for {symbol}: {e}") + return False + + def _reset_session_volume_profile(self, symbol: str, reset_time: datetime): + """Reset session volume profile""" + try: + logger.info(f"Resetting session volume profile for {symbol}") + self.session_start_time[symbol] = reset_time + self.price_level_cache[symbol] = {} + self.session_volume_profiles[symbol] = [] + + except Exception as e: + logger.error(f"Error resetting session volume profile for {symbol}: {e}") + + async def _analyze_order_flow(self, symbol: str, timestamp: datetime, price: float, + quantity: float, is_buyer_maker: bool): + """Analyze order flow patterns""" + try: + if symbol not in self.order_book_history or not self.order_book_history[symbol]: + return + + recent_snapshots = list(self.order_book_history[symbol])[-10:] + + # Check for sweeps + sweep_signal = self._detect_order_sweep(symbol, recent_snapshots, price, quantity, is_buyer_maker) + if sweep_signal: + self.flow_signals[symbol].append(sweep_signal) + await self._notify_flow_signal(symbol, sweep_signal) + + # Check for absorption + absorption_signal = self._detect_absorption(symbol, recent_snapshots, price, quantity) + if absorption_signal: + self.flow_signals[symbol].append(absorption_signal) + await self._notify_flow_signal(symbol, absorption_signal) + + # Check for momentum + momentum_signal = self._detect_momentum_trade(symbol, price, quantity, is_buyer_maker) + if momentum_signal: + self.flow_signals[symbol].append(momentum_signal) + await self._notify_flow_signal(symbol, momentum_signal) + + except Exception as e: + logger.error(f"Error analyzing order flow for {symbol}: {e}") + + async def _analyze_enhanced_order_flow(self, symbol: str, timestamp: datetime, price: float, + quantity: float, trade_value: float, is_buyer_maker: bool, + trade_type: str, aggregation_size: int = 1): + """Enhanced order flow analysis with aggressive vs passive ratios""" + try: + # Determine if trade is aggressive (taker) or passive (maker) + is_aggressive = not is_buyer_maker # In Binance data, m=false means buyer is taker (aggressive) + + # Calculate aggressive vs passive ratios + self._update_aggressive_passive_ratio(symbol, timestamp, trade_value, is_aggressive) + + # Update trade size distribution + self._update_trade_size_distribution(symbol, timestamp, trade_value, trade_type) + + # Update market maker vs taker flow + self._update_market_maker_taker_flow(symbol, timestamp, trade_value, is_buyer_maker, is_aggressive) + + # Calculate order flow intensity + self._update_order_flow_intensity(symbol, timestamp, trade_value, aggregation_size) + + # Measure liquidity consumption + await self._measure_liquidity_consumption(symbol, timestamp, price, quantity, trade_value, is_aggressive) + + # Measure price impact + await self._measure_price_impact(symbol, timestamp, price, trade_value, is_aggressive) + + except Exception as e: + logger.error(f"Error in enhanced order flow analysis for {symbol}: {e}") + + def _update_aggressive_passive_ratio(self, symbol: str, timestamp: datetime, trade_value: float, is_aggressive: bool): + """Update aggressive vs passive participant ratios""" + try: + current_window = [] + cutoff_time = timestamp - timedelta(seconds=60) # 1-minute window + + # Filter recent trades within window + for entry in self.aggressive_passive_ratios[symbol]: + if entry['timestamp'] > cutoff_time: + current_window.append(entry) + + # Add current trade + current_window.append({ + 'timestamp': timestamp, + 'trade_value': trade_value, + 'is_aggressive': is_aggressive + }) + + # Calculate ratios + aggressive_volume = sum(t['trade_value'] for t in current_window if t['is_aggressive']) + passive_volume = sum(t['trade_value'] for t in current_window if not t['is_aggressive']) + total_volume = aggressive_volume + passive_volume + + if total_volume > 0: + aggressive_ratio = aggressive_volume / total_volume + passive_ratio = passive_volume / total_volume + + ratio_data = { + 'timestamp': timestamp, + 'aggressive_ratio': aggressive_ratio, + 'passive_ratio': passive_ratio, + 'aggressive_volume': aggressive_volume, + 'passive_volume': passive_volume, + 'total_volume': total_volume, + 'trade_count': len(current_window), + 'avg_aggressive_size': aggressive_volume / max(1, sum(1 for t in current_window if t['is_aggressive'])), + 'avg_passive_size': passive_volume / max(1, sum(1 for t in current_window if not t['is_aggressive'])) + } + + # Update buffer + self.aggressive_passive_ratios[symbol].clear() + self.aggressive_passive_ratios[symbol].extend(current_window) + + # Store calculated ratios for use by models + if not hasattr(self, 'current_flow_ratios'): + self.current_flow_ratios = {} + self.current_flow_ratios[symbol] = ratio_data + + except Exception as e: + logger.error(f"Error updating aggressive/passive ratio for {symbol}: {e}") + + def _update_trade_size_distribution(self, symbol: str, timestamp: datetime, trade_value: float, trade_type: str): + """Update trade size distribution for institutional vs retail detection""" + try: + # Classify trade size + if trade_value < 1000: + size_category = 'micro' # < $1K (retail) + elif trade_value < 10000: + size_category = 'small' # $1K-$10K (retail/small institutional) + elif trade_value < 50000: + size_category = 'medium' # $10K-$50K (institutional) + elif trade_value < 100000: + size_category = 'large' # $50K-$100K (large institutional) + else: + size_category = 'block' # > $100K (block trades) + + trade_data = { + 'timestamp': timestamp, + 'trade_value': trade_value, + 'trade_type': trade_type, + 'size_category': size_category, + 'is_institutional': trade_value >= self.large_order_threshold, + 'is_block_trade': trade_value >= self.block_trade_threshold + } + + self.trade_size_distributions[symbol].append(trade_data) + + except Exception as e: + logger.error(f"Error updating trade size distribution for {symbol}: {e}") + + def _update_market_maker_taker_flow(self, symbol: str, timestamp: datetime, trade_value: float, + is_buyer_maker: bool, is_aggressive: bool): + """Update market maker vs taker flow analysis""" + try: + flow_data = { + 'timestamp': timestamp, + 'trade_value': trade_value, + 'is_buyer_maker': is_buyer_maker, + 'is_aggressive': is_aggressive, + 'flow_direction': 'buy_aggressive' if not is_buyer_maker else 'sell_aggressive', + 'market_maker_side': 'sell' if is_buyer_maker else 'buy' + } + + self.market_maker_taker_flows[symbol].append(flow_data) + + except Exception as e: + logger.error(f"Error updating market maker/taker flow for {symbol}: {e}") + + def _update_order_flow_intensity(self, symbol: str, timestamp: datetime, trade_value: float, aggregation_size: int): + """Calculate order flow intensity based on trade frequency and size""" + try: + # Calculate intensity based on trade value and aggregation + base_intensity = trade_value / 10000 # Normalize by $10K + aggregation_intensity = aggregation_size / 10 # Normalize aggregation factor + + # Time-based intensity (trades per second) + recent_trades = [t for t in self.order_flow_intensity[symbol] + if (timestamp - t['timestamp']).total_seconds() < 10] + time_intensity = len(recent_trades) / 10 # Trades per second over 10s window + + intensity_score = base_intensity * (1 + aggregation_intensity) * (1 + time_intensity) + + intensity_data = { + 'timestamp': timestamp, + 'intensity_score': intensity_score, + 'base_intensity': base_intensity, + 'aggregation_intensity': aggregation_intensity, + 'time_intensity': time_intensity, + 'trade_value': trade_value, + 'aggregation_size': aggregation_size + } + + self.order_flow_intensity[symbol].append(intensity_data) + + except Exception as e: + logger.error(f"Error updating order flow intensity for {symbol}: {e}") + + async def _measure_liquidity_consumption(self, symbol: str, timestamp: datetime, price: float, + quantity: float, trade_value: float, is_aggressive: bool): + """Measure liquidity consumption rates""" + try: + if not is_aggressive: + return # Only measure for aggressive trades + + current_snapshot = self.order_books.get(symbol) + if not current_snapshot: + return + + # Calculate how much liquidity was consumed + if price >= current_snapshot.mid_price: # Buy-side consumption + consumed_liquidity = 0 + for ask_level in current_snapshot.asks: + if ask_level.price <= price: + consumed_liquidity += min(ask_level.size, quantity) * ask_level.price + quantity -= ask_level.size + if quantity <= 0: + break + else: # Sell-side consumption + consumed_liquidity = 0 + for bid_level in current_snapshot.bids: + if bid_level.price >= price: + consumed_liquidity += min(bid_level.size, quantity) * bid_level.price + quantity -= bid_level.size + if quantity <= 0: + break + + consumption_rate = consumed_liquidity / trade_value if trade_value > 0 else 0 + + consumption_data = { + 'timestamp': timestamp, + 'price': price, + 'trade_value': trade_value, + 'consumed_liquidity': consumed_liquidity, + 'consumption_rate': consumption_rate, + 'side': 'buy' if price >= current_snapshot.mid_price else 'sell' + } + + self.liquidity_consumption_rates[symbol].append(consumption_data) + + except Exception as e: + logger.error(f"Error measuring liquidity consumption for {symbol}: {e}") + + async def _measure_price_impact(self, symbol: str, timestamp: datetime, price: float, + trade_value: float, is_aggressive: bool): + """Measure price impact of trades""" + try: + if not is_aggressive: + return + + # Get price before and after (approximated by looking at recent snapshots) + recent_snapshots = list(self.order_book_history[symbol])[-5:] + if len(recent_snapshots) < 2: + return + + price_before = recent_snapshots[-2].mid_price + price_after = recent_snapshots[-1].mid_price + + price_impact = abs(price_after - price_before) / price_before if price_before > 0 else 0 + impact_per_dollar = price_impact / (trade_value / 1000000) if trade_value > 0 else 0 # Impact per $1M + + impact_data = { + 'timestamp': timestamp, + 'trade_price': price, + 'trade_value': trade_value, + 'price_before': price_before, + 'price_after': price_after, + 'price_impact': price_impact, + 'impact_per_million': impact_per_dollar, + 'impact_category': self._categorize_impact(price_impact) + } + + self.price_impact_measurements[symbol].append(impact_data) + + except Exception as e: + logger.error(f"Error measuring price impact for {symbol}: {e}") + + def _categorize_impact(self, price_impact: float) -> str: + """Categorize price impact level""" + if price_impact < 0.0001: # < 0.01% + return 'minimal' + elif price_impact < 0.001: # < 0.1% + return 'low' + elif price_impact < 0.005: # < 0.5% + return 'medium' + elif price_impact < 0.01: # < 1% + return 'high' + else: + return 'extreme' + + async def _detect_institutional_activity(self, symbol: str, timestamp: datetime, price: float, + quantity: float, trade_value: float, aggregation_size: int, + is_buyer_maker: bool): + """Detect institutional trading activity patterns""" + try: + # Block trade detection + if trade_value >= self.block_trade_threshold: + signal = OrderFlowSignal( + timestamp=timestamp, + signal_type='block_trade', + price=price, + volume=trade_value, + confidence=min(0.95, trade_value / 500000), # Higher confidence for larger trades + description=f"Block trade: ${trade_value:.0f} ({'Buy' if not is_buyer_maker else 'Sell'})" + ) + self.flow_signals[symbol].append(signal) + await self._notify_flow_signal(symbol, signal) + + # Iceberg order detection (multiple large aggregated trades in sequence) + await self._detect_iceberg_orders(symbol, timestamp, price, trade_value, aggregation_size, is_buyer_maker) + + # High-frequency activity detection + await self._detect_hft_activity(symbol, timestamp, trade_value, aggregation_size) + + except Exception as e: + logger.error(f"Error detecting institutional activity for {symbol}: {e}") + + async def _detect_iceberg_orders(self, symbol: str, timestamp: datetime, price: float, + trade_value: float, aggregation_size: int, is_buyer_maker: bool): + """Detect iceberg order patterns""" + try: + if trade_value < self.large_order_threshold: + return + + # Look for similar-sized trades in recent history + cutoff_time = timestamp - timedelta(seconds=self.iceberg_detection_window) + recent_large_trades = [] + + for trade_data in self.trade_size_distributions[symbol]: + if (trade_data['timestamp'] > cutoff_time and + trade_data['trade_value'] >= self.large_order_threshold): + recent_large_trades.append(trade_data) + + # Iceberg pattern: 3+ large trades with similar sizes + if len(recent_large_trades) >= 3: + avg_size = sum(t['trade_value'] for t in recent_large_trades) / len(recent_large_trades) + size_consistency = all(abs(t['trade_value'] - avg_size) / avg_size < 0.2 for t in recent_large_trades) + + if size_consistency: + total_iceberg_volume = sum(t['trade_value'] for t in recent_large_trades) + confidence = min(0.9, len(recent_large_trades) / 10 + total_iceberg_volume / 1000000) + + signal = OrderFlowSignal( + timestamp=timestamp, + signal_type='iceberg', + price=price, + volume=total_iceberg_volume, + confidence=confidence, + description=f"Iceberg: {len(recent_large_trades)} trades, ${total_iceberg_volume:.0f} total" + ) + self.flow_signals[symbol].append(signal) + await self._notify_flow_signal(symbol, signal) + + except Exception as e: + logger.error(f"Error detecting iceberg orders for {symbol}: {e}") + + async def _detect_hft_activity(self, symbol: str, timestamp: datetime, trade_value: float, aggregation_size: int): + """Detect high-frequency trading activity""" + try: + # Look for high-frequency patterns (many small trades in rapid succession) + cutoff_time = timestamp - timedelta(seconds=5) + recent_trades = [t for t in self.order_flow_intensity[symbol] if t['timestamp'] > cutoff_time] + + if len(recent_trades) >= 20: # 20+ trades in 5 seconds + avg_trade_size = sum(t['trade_value'] for t in recent_trades) / len(recent_trades) + + if avg_trade_size < 5000: # Small average trade size suggests HFT + total_hft_volume = sum(t['trade_value'] for t in recent_trades) + confidence = min(0.8, len(recent_trades) / 50) + + signal = OrderFlowSignal( + timestamp=timestamp, + signal_type='hft_activity', + price=0, # Multiple prices + volume=total_hft_volume, + confidence=confidence, + description=f"HFT: {len(recent_trades)} trades in 5s, avg ${avg_trade_size:.0f}" + ) + self.flow_signals[symbol].append(signal) + await self._notify_flow_signal(symbol, signal) + + except Exception as e: + logger.error(f"Error detecting HFT activity for {symbol}: {e}") + + def _update_volume_statistics(self, symbol: str, volume_24h: float, quote_volume_24h: float, weighted_avg_price: float): + """Update volume statistics for relative analysis""" + try: + # Store 24h volume data for relative comparisons + if not hasattr(self, 'volume_stats'): + self.volume_stats = {} + + self.volume_stats[symbol] = { + 'volume_24h': volume_24h, + 'quote_volume_24h': quote_volume_24h, + 'weighted_avg_price': weighted_avg_price, + 'timestamp': datetime.now() + } + + except Exception as e: + logger.error(f"Error updating volume statistics for {symbol}: {e}") + + def _detect_order_sweep(self, symbol: str, snapshots: List[OrderBookSnapshot], + price: float, quantity: float, is_buyer_maker: bool) -> Optional[OrderFlowSignal]: + """Detect order book sweeps""" + try: + if len(snapshots) < 2: + return None + + before_snapshot = snapshots[-2] + + if is_buyer_maker: # Sell order, check ask side + levels_consumed = 0 + total_consumed_size = 0 + + for level in before_snapshot.asks[:5]: + if level.price <= price: + levels_consumed += 1 + total_consumed_size += level.size + + if levels_consumed >= 2 and total_consumed_size > quantity * 1.5: + confidence = min(0.9, levels_consumed / 5.0 + 0.3) + + return OrderFlowSignal( + timestamp=datetime.now(), + signal_type='sweep', + price=price, + volume=quantity * price, + confidence=confidence, + description=f"Sell sweep: {levels_consumed} levels" + ) + else: # Buy order, check bid side + levels_consumed = 0 + total_consumed_size = 0 + + for level in before_snapshot.bids[:5]: + if level.price >= price: + levels_consumed += 1 + total_consumed_size += level.size + + if levels_consumed >= 2 and total_consumed_size > quantity * 1.5: + confidence = min(0.9, levels_consumed / 5.0 + 0.3) + + return OrderFlowSignal( + timestamp=datetime.now(), + signal_type='sweep', + price=price, + volume=quantity * price, + confidence=confidence, + description=f"Buy sweep: {levels_consumed} levels" + ) + + return None + + except Exception as e: + logger.error(f"Error detecting sweep for {symbol}: {e}") + return None + + def _detect_absorption(self, symbol: str, snapshots: List[OrderBookSnapshot], + price: float, quantity: float) -> Optional[OrderFlowSignal]: + """Detect absorption patterns""" + try: + if len(snapshots) < 3: + return None + + volume_threshold = 10000 # $10K minimum + price_impact_threshold = 0.001 # 0.1% max impact + + trade_value = price * quantity + if trade_value < volume_threshold: + return None + + # Calculate price impact + price_before = snapshots[-3].mid_price + price_after = snapshots[-1].mid_price + price_impact = abs(price_after - price_before) / price_before + + if price_impact < price_impact_threshold: + confidence = min(0.8, (trade_value / 50000) * 0.5 + 0.3) + + return OrderFlowSignal( + timestamp=datetime.now(), + signal_type='absorption', + price=price, + volume=trade_value, + confidence=confidence, + description=f"Absorption: ${trade_value:.0f}" + ) + + return None + + except Exception as e: + logger.error(f"Error detecting absorption for {symbol}: {e}") + return None + + def _detect_momentum_trade(self, symbol: str, price: float, quantity: float, + is_buyer_maker: bool) -> Optional[OrderFlowSignal]: + """Detect momentum trades""" + try: + trade_value = price * quantity + momentum_threshold = 25000 # $25K minimum + + if trade_value < momentum_threshold: + return None + + confidence = min(0.9, trade_value / 100000 * 0.6 + 0.3) + direction = "sell" if is_buyer_maker else "buy" + + return OrderFlowSignal( + timestamp=datetime.now(), + signal_type='momentum', + price=price, + volume=trade_value, + confidence=confidence, + description=f"Large {direction}: ${trade_value:.0f}" + ) + + except Exception as e: + logger.error(f"Error detecting momentum for {symbol}: {e}") + return None + + async def _notify_flow_signal(self, symbol: str, signal: OrderFlowSignal): + """Notify models of flow signals""" + try: + signal_data = { + 'signal_type': signal.signal_type, + 'price': signal.price, + 'volume': signal.volume, + 'confidence': signal.confidence, + 'timestamp': signal.timestamp, + 'description': signal.description + } + + # Notify CNN callbacks + for callback in self.cnn_callbacks: + try: + callback(symbol, signal_data) + except Exception as e: + logger.warning(f"Error in CNN callback: {e}") + + # Notify DQN callbacks + for callback in self.dqn_callbacks: + try: + callback(symbol, signal_data) + except Exception as e: + logger.warning(f"Error in DQN callback: {e}") + + except Exception as e: + logger.error(f"Error notifying flow signal: {e}") + + async def _continuous_analysis(self): + """Continuous analysis and model feeding""" + while self.is_streaming: + try: + await asyncio.sleep(1) # Analyze every second + + for symbol in self.symbols: + # Generate features for models + cnn_features = self.get_cnn_features(symbol) + if cnn_features is not None: + for callback in self.cnn_callbacks: + try: + callback(symbol, {'features': cnn_features, 'type': 'orderbook'}) + except Exception as e: + logger.warning(f"Error in CNN feature callback: {e}") + + dqn_features = self.get_dqn_state_features(symbol) + if dqn_features is not None: + for callback in self.dqn_callbacks: + try: + callback(symbol, {'state': dqn_features, 'type': 'orderbook'}) + except Exception as e: + logger.warning(f"Error in DQN state callback: {e}") + + except Exception as e: + logger.error(f"Error in continuous analysis: {e}") + await asyncio.sleep(5) + + def get_cnn_features(self, symbol: str) -> Optional[np.ndarray]: + """Generate CNN features from order book data""" + try: + if symbol not in self.order_books: + return None + + snapshot = self.order_books[symbol] + features = [] + + # Order book features (80 features: 20 levels x 2 sides x 2 values) + for i in range(min(20, len(snapshot.bids))): + bid = snapshot.bids[i] + features.append(bid.size) + features.append(bid.price - snapshot.mid_price) + + # Pad bids + while len(features) < 40: + features.extend([0.0, 0.0]) + + for i in range(min(20, len(snapshot.asks))): + ask = snapshot.asks[i] + features.append(ask.size) + features.append(ask.price - snapshot.mid_price) + + # Pad asks + while len(features) < 80: + features.extend([0.0, 0.0]) + + # Liquidity metrics (10 features) + metrics = self.liquidity_metrics.get(symbol, {}) + features.extend([ + metrics.get('total_bid_size', 0.0), + metrics.get('total_ask_size', 0.0), + metrics.get('liquidity_ratio', 1.0), + metrics.get('spread_bps', 0.0), + snapshot.spread, + metrics.get('weighted_mid', snapshot.mid_price) - snapshot.mid_price, + len(snapshot.bids), + len(snapshot.asks), + snapshot.mid_price, + time.time() % 86400 # Time of day + ]) + + # Order book imbalance (5 features) + if self.order_book_imbalances[symbol]: + latest_imbalance = self.order_book_imbalances[symbol][-1] + features.extend([ + latest_imbalance['imbalance'], + latest_imbalance['bid_size'], + latest_imbalance['ask_size'], + latest_imbalance['bid_size'] + latest_imbalance['ask_size'], + abs(latest_imbalance['imbalance']) + ]) + else: + features.extend([0.0, 0.0, 0.0, 0.0, 0.0]) + + # Enhanced flow signals (15 features) + recent_signals = [s for s in self.flow_signals[symbol] + if (datetime.now() - s.timestamp).seconds < 60] + + sweep_count = sum(1 for s in recent_signals if s.signal_type == 'sweep') + absorption_count = sum(1 for s in recent_signals if s.signal_type == 'absorption') + momentum_count = sum(1 for s in recent_signals if s.signal_type == 'momentum') + block_count = sum(1 for s in recent_signals if s.signal_type == 'block_trade') + iceberg_count = sum(1 for s in recent_signals if s.signal_type == 'iceberg') + hft_count = sum(1 for s in recent_signals if s.signal_type == 'hft_activity') + max_confidence = max([s.confidence for s in recent_signals], default=0.0) + total_flow_volume = sum(s.volume for s in recent_signals) + + # Enhanced order flow metrics + flow_metrics = self.get_enhanced_order_flow_metrics(symbol) + if flow_metrics: + aggressive_ratio = flow_metrics['aggressive_passive']['aggressive_ratio'] + institutional_ratio = flow_metrics['institutional_retail']['institutional_ratio'] + flow_intensity = flow_metrics['flow_intensity']['current_intensity'] + avg_consumption_rate = flow_metrics['liquidity']['avg_consumption_rate'] + avg_price_impact = flow_metrics['price_impact']['avg_impact'] / 10000 # Normalize from basis points + buy_pressure = flow_metrics['maker_taker_flow']['buy_pressure'] + sell_pressure = flow_metrics['maker_taker_flow']['sell_pressure'] + else: + aggressive_ratio = 0.5 + institutional_ratio = 0.5 + flow_intensity = 0.0 + avg_consumption_rate = 0.0 + avg_price_impact = 0.0 + buy_pressure = 0.5 + sell_pressure = 0.5 + + features.extend([ + sweep_count, + absorption_count, + momentum_count, + block_count, + iceberg_count, + hft_count, + max_confidence, + total_flow_volume, + aggressive_ratio, + institutional_ratio, + flow_intensity, + avg_consumption_rate, + avg_price_impact, + buy_pressure, + sell_pressure + ]) + + return np.array(features, dtype=np.float32) + + except Exception as e: + logger.error(f"Error generating CNN features for {symbol}: {e}") + return None + + def get_dqn_state_features(self, symbol: str) -> Optional[np.ndarray]: + """Generate DQN state features""" + try: + if symbol not in self.order_books: + return None + + snapshot = self.order_books[symbol] + state_features = [] + + # Normalized order book state (20 features) + total_bid_size = sum(level.size for level in snapshot.bids[:10]) + total_ask_size = sum(level.size for level in snapshot.asks[:10]) + total_size = total_bid_size + total_ask_size + + if total_size > 0: + for i in range(min(10, len(snapshot.bids))): + state_features.append(snapshot.bids[i].size / total_size) + + while len(state_features) < 10: + state_features.append(0.0) + + for i in range(min(10, len(snapshot.asks))): + state_features.append(snapshot.asks[i].size / total_size) + + while len(state_features) < 20: + state_features.append(0.0) + else: + state_features.extend([0.0] * 20) + + # Enhanced market state indicators (20 features) + metrics = self.liquidity_metrics.get(symbol, {}) + + spread_pct = (snapshot.spread / snapshot.mid_price) if snapshot.mid_price > 0 else 0 + liquidity_ratio = metrics.get('liquidity_ratio', 1.0) + liquidity_imbalance = (liquidity_ratio - 1) / (liquidity_ratio + 1) + + # Flow strength + recent_signals = [s for s in self.flow_signals[symbol] + if (datetime.now() - s.timestamp).seconds < 30] + flow_strength = sum(s.confidence for s in recent_signals) / max(len(recent_signals), 1) + + # Price volatility + if len(self.order_book_history[symbol]) >= 10: + recent_prices = [s.mid_price for s in list(self.order_book_history[symbol])[-10:]] + price_volatility = np.std(recent_prices) / np.mean(recent_prices) if recent_prices else 0 + else: + price_volatility = 0 + + # Enhanced order flow metrics for DQN + flow_metrics = self.get_enhanced_order_flow_metrics(symbol) + if flow_metrics: + aggressive_ratio = flow_metrics['aggressive_passive']['aggressive_ratio'] + institutional_ratio = flow_metrics['institutional_retail']['institutional_ratio'] + flow_intensity = min(flow_metrics['flow_intensity']['current_intensity'] / 10, 1.0) # Normalize + consumption_rate = flow_metrics['liquidity']['avg_consumption_rate'] + price_impact = min(flow_metrics['price_impact']['avg_impact'] / 100, 1.0) # Normalize basis points + buy_pressure = flow_metrics['maker_taker_flow']['buy_pressure'] + sell_pressure = flow_metrics['maker_taker_flow']['sell_pressure'] + + # Trade size distribution ratios + size_dist = flow_metrics['size_distribution'] + total_trades = sum(size_dist.values()) or 1 + retail_ratio = (size_dist.get('micro', 0) + size_dist.get('small', 0)) / total_trades + institutional_trade_ratio = (size_dist.get('large', 0) + size_dist.get('block', 0)) / total_trades + + # Recent activity indicators + block_activity = min(size_dist.get('block', 0) / 10, 1.0) # Normalize + else: + aggressive_ratio = 0.5 + institutional_ratio = 0.5 + flow_intensity = 0.0 + consumption_rate = 0.0 + price_impact = 0.0 + buy_pressure = 0.5 + sell_pressure = 0.5 + retail_ratio = 0.5 + institutional_trade_ratio = 0.5 + block_activity = 0.0 + + state_features.extend([ + spread_pct * 10000, # Spread in basis points + liquidity_imbalance, + flow_strength, + price_volatility * 100, + min(len(snapshot.bids), 20) / 20, + min(len(snapshot.asks), 20) / 20, + len([s for s in recent_signals if s.signal_type == 'sweep']) / 10, + len([s for s in recent_signals if s.signal_type == 'absorption']) / 5, + len([s for s in recent_signals if s.signal_type == 'momentum']) / 5, + (datetime.now().hour * 60 + datetime.now().minute) / 1440, + # Enhanced order flow state features + aggressive_ratio, + institutional_ratio, + flow_intensity, + consumption_rate, + price_impact, + buy_pressure, + sell_pressure, + retail_ratio, + institutional_trade_ratio, + block_activity + ]) + + return np.array(state_features, dtype=np.float32) + + except Exception as e: + logger.error(f"Error generating DQN features for {symbol}: {e}") + return None + + def get_order_heatmap_matrix(self, symbol: str, levels: int = 40) -> Optional[np.ndarray]: + """Generate heatmap matrix for visualization""" + try: + if symbol not in self.order_heatmaps or not self.order_heatmaps[symbol]: + return None + + current_snapshot = self.order_books.get(symbol) + if not current_snapshot: + return None + + mid_price = current_snapshot.mid_price + price_step = mid_price * 0.0001 # 1 basis point steps + + # Matrix: time x price levels + time_window = min(600, len(self.order_heatmaps[symbol])) + heatmap_matrix = np.zeros((time_window, levels)) + + # Fill matrix + for t, entry in enumerate(list(self.order_heatmaps[symbol])[-time_window:]): + for price_offset, level_data in entry['levels'].items(): + level_idx = int((price_offset + (levels/2) * price_step) / price_step) + + if 0 <= level_idx < levels: + size_weight = 1.0 if level_data['side'] == 'bid' else -1.0 + heatmap_matrix[t, level_idx] = level_data['size'] * size_weight + + return heatmap_matrix + + except Exception as e: + logger.error(f"Error generating heatmap matrix for {symbol}: {e}") + return None + + def get_dashboard_data(self, symbol: str) -> Optional[Dict]: + """Get data for dashboard visualization""" + try: + if symbol not in self.order_books: + return None + + snapshot = self.order_books[symbol] + + return { + 'timestamp': snapshot.timestamp.isoformat(), + 'symbol': symbol, + 'mid_price': snapshot.mid_price, + 'spread': snapshot.spread, + 'bids': [{'price': l.price, 'size': l.size} for l in snapshot.bids[:20]], + 'asks': [{'price': l.price, 'size': l.size} for l in snapshot.asks[:20]], + 'liquidity_metrics': self.liquidity_metrics.get(symbol, {}), + 'volume_profile': self.get_volume_profile_data(symbol), + 'heatmap_matrix': self.get_order_heatmap_matrix(symbol).tolist() if self.get_order_heatmap_matrix(symbol) is not None else None, + 'enhanced_order_flow': self.get_enhanced_order_flow_metrics(symbol), + 'recent_signals': [ + { + 'type': s.signal_type, + 'price': s.price, + 'volume': s.volume, + 'confidence': s.confidence, + 'timestamp': s.timestamp.isoformat(), + 'description': s.description + } + for s in list(self.flow_signals[symbol])[-10:] + ] + } + + except Exception as e: + logger.error(f"Error getting dashboard data for {symbol}: {e}") + return None + + def get_volume_profile_data(self, symbol: str) -> Optional[List[Dict]]: + """Get rolling volume profile data (10-minute window)""" + try: + if symbol not in self.volume_profiles: + return None + + profile_data = [] + for level in sorted(self.volume_profiles[symbol], key=lambda x: x.price): + profile_data.append({ + 'price': level.price, + 'volume': level.volume, + 'buy_volume': level.buy_volume, + 'sell_volume': level.sell_volume, + 'trades_count': level.trades_count, + 'vwap': level.vwap, + 'net_volume': level.buy_volume - level.sell_volume + }) + + return profile_data + + except Exception as e: + logger.error(f"Error getting volume profile for {symbol}: {e}") + return None + + def get_session_volume_profile_data(self, symbol: str) -> Optional[List[Dict]]: + """Get Session Volume Profile (SVP) data - full session data""" + try: + if symbol not in self.price_level_cache: + return None + + session_data = [] + total_volume = sum(level.volume for level in self.price_level_cache[symbol].values()) + + for price_key, level in sorted(self.price_level_cache[symbol].items()): + volume_percentage = (level.volume / total_volume * 100) if total_volume > 0 else 0 + + session_data.append({ + 'price': level.price, + 'volume': level.volume, + 'buy_volume': level.buy_volume, + 'sell_volume': level.sell_volume, + 'trades_count': level.trades_count, + 'vwap': level.vwap, + 'net_volume': level.buy_volume - level.sell_volume, + 'volume_percentage': volume_percentage, + 'is_high_volume_node': volume_percentage > 2.0, # Mark significant price levels + 'buy_sell_ratio': level.buy_volume / level.sell_volume if level.sell_volume > 0 else float('inf') + }) + + return session_data + + except Exception as e: + logger.error(f"Error getting Session Volume Profile for {symbol}: {e}") + return None + + def get_session_statistics(self, symbol: str) -> Optional[Dict]: + """Get session trading statistics""" + try: + if symbol not in self.price_level_cache: + return None + + levels = list(self.price_level_cache[symbol].values()) + if not levels: + return None + + total_volume = sum(level.volume for level in levels) + total_buy_volume = sum(level.buy_volume for level in levels) + total_sell_volume = sum(level.sell_volume for level in levels) + total_trades = sum(level.trades_count for level in levels) + + # Calculate session VWAP + session_vwap = sum(level.vwap * level.volume for level in levels) / total_volume if total_volume > 0 else 0 + + # Find price extremes + prices = [level.price for level in levels] + session_high = max(prices) if prices else 0 + session_low = min(prices) if prices else 0 + + # Find Point of Control (POC) - price level with highest volume + poc_level = max(levels, key=lambda x: x.volume) if levels else None + poc_price = poc_level.price if poc_level else 0 + poc_volume = poc_level.volume if poc_level else 0 + + # Calculate Value Area (70% of volume around POC) + sorted_levels = sorted(levels, key=lambda x: x.volume, reverse=True) + value_area_volume = total_volume * 0.7 + value_area_levels = [] + current_volume = 0 + + for level in sorted_levels: + value_area_levels.append(level) + current_volume += level.volume + if current_volume >= value_area_volume: + break + + value_area_high = max(level.price for level in value_area_levels) if value_area_levels else 0 + value_area_low = min(level.price for level in value_area_levels) if value_area_levels else 0 + + session_start = self.session_start_time.get(symbol, datetime.now()) + session_duration = (datetime.now() - session_start).total_seconds() / 3600 # Hours + + return { + 'symbol': symbol, + 'session_start': session_start.isoformat(), + 'session_duration_hours': session_duration, + 'total_volume': total_volume, + 'total_buy_volume': total_buy_volume, + 'total_sell_volume': total_sell_volume, + 'total_trades': total_trades, + 'session_vwap': session_vwap, + 'session_high': session_high, + 'session_low': session_low, + 'poc_price': poc_price, + 'poc_volume': poc_volume, + 'value_area_high': value_area_high, + 'value_area_low': value_area_low, + 'value_area_range': value_area_high - value_area_low, + 'buy_sell_ratio': total_buy_volume / total_sell_volume if total_sell_volume > 0 else float('inf'), + 'price_levels_traded': len(levels), + 'avg_trade_size': total_volume / total_trades if total_trades > 0 else 0 + } + + except Exception as e: + logger.error(f"Error getting session statistics for {symbol}: {e}") + return None + + def get_market_profile_analysis(self, symbol: str) -> Optional[Dict]: + """Get detailed market profile analysis""" + try: + current_snapshot = self.order_books.get(symbol) + session_stats = self.get_session_statistics(symbol) + svp_data = self.get_session_volume_profile_data(symbol) + + if not all([current_snapshot, session_stats, svp_data]): + return None + + current_price = current_snapshot.mid_price + session_vwap = session_stats['session_vwap'] + poc_price = session_stats['poc_price'] + value_area_high = session_stats['value_area_high'] + value_area_low = session_stats['value_area_low'] + + # Market structure analysis + price_vs_vwap = "above" if current_price > session_vwap else "below" + price_vs_poc = "above" if current_price > poc_price else "below" + + in_value_area = value_area_low <= current_price <= value_area_high + + # Find support and resistance levels from high volume nodes + high_volume_nodes = [item for item in svp_data if item['is_high_volume_node']] + resistance_levels = [node['price'] for node in high_volume_nodes if node['price'] > current_price] + support_levels = [node['price'] for node in high_volume_nodes if node['price'] < current_price] + + # Sort to get nearest levels + resistance_levels.sort() + support_levels.sort(reverse=True) + + return { + 'symbol': symbol, + 'current_price': current_price, + 'market_structure': { + 'price_vs_vwap': price_vs_vwap, + 'price_vs_poc': price_vs_poc, + 'in_value_area': in_value_area, + 'distance_from_vwap_bps': int(abs(current_price - session_vwap) / session_vwap * 10000), + 'distance_from_poc_bps': int(abs(current_price - poc_price) / poc_price * 10000) + }, + 'key_levels': { + 'session_vwap': session_vwap, + 'poc_price': poc_price, + 'value_area_high': value_area_high, + 'value_area_low': value_area_low, + 'nearest_resistance': resistance_levels[0] if resistance_levels else None, + 'nearest_support': support_levels[0] if support_levels else None + }, + 'volume_analysis': { + 'total_high_volume_nodes': len(high_volume_nodes), + 'resistance_levels': resistance_levels[:3], # Top 3 resistance + 'support_levels': support_levels[:3], # Top 3 support + 'poc_strength': session_stats['poc_volume'] / session_stats['total_volume'] * 100 + }, + 'session_statistics': session_stats + } + + except Exception as e: + logger.error(f"Error getting market profile analysis for {symbol}: {e}") + return None + + def get_enhanced_order_flow_metrics(self, symbol: str) -> Optional[Dict]: + """Get enhanced order flow metrics including aggressive vs passive ratios""" + try: + if symbol not in self.current_flow_ratios: + return None + + current_ratios = self.current_flow_ratios.get(symbol, {}) + + # Get recent trade size distribution + recent_trades = list(self.trade_size_distributions[symbol])[-100:] # Last 100 trades + if not recent_trades: + return None + + # Calculate institutional vs retail breakdown + institutional_trades = [t for t in recent_trades if t['is_institutional']] + retail_trades = [t for t in recent_trades if not t['is_institutional']] + block_trades = [t for t in recent_trades if t['is_block_trade']] + + institutional_volume = sum(t['trade_value'] for t in institutional_trades) + retail_volume = sum(t['trade_value'] for t in retail_trades) + total_volume = institutional_volume + retail_volume + + # Size category breakdown + size_breakdown = { + 'micro': len([t for t in recent_trades if t['size_category'] == 'micro']), + 'small': len([t for t in recent_trades if t['size_category'] == 'small']), + 'medium': len([t for t in recent_trades if t['size_category'] == 'medium']), + 'large': len([t for t in recent_trades if t['size_category'] == 'large']), + 'block': len([t for t in recent_trades if t['size_category'] == 'block']) + } + + # Get recent order flow intensity + recent_intensity = list(self.order_flow_intensity[symbol])[-10:] + avg_intensity = sum(i['intensity_score'] for i in recent_intensity) / max(1, len(recent_intensity)) + + # Get recent liquidity consumption + recent_consumption = list(self.liquidity_consumption_rates[symbol])[-20:] + avg_consumption_rate = sum(c['consumption_rate'] for c in recent_consumption) / max(1, len(recent_consumption)) + + # Get recent price impact + recent_impacts = list(self.price_impact_measurements[symbol])[-20:] + avg_price_impact = sum(i['price_impact'] for i in recent_impacts) / max(1, len(recent_impacts)) + + # Impact distribution + impact_distribution = {} + for impact in recent_impacts: + category = impact['impact_category'] + impact_distribution[category] = impact_distribution.get(category, 0) + 1 + + # Market maker vs taker flow analysis + recent_flows = list(self.market_maker_taker_flows[symbol])[-50:] + buy_aggressive_volume = sum(f['trade_value'] for f in recent_flows if f['flow_direction'] == 'buy_aggressive') + sell_aggressive_volume = sum(f['trade_value'] for f in recent_flows if f['flow_direction'] == 'sell_aggressive') + + return { + 'symbol': symbol, + 'timestamp': datetime.now().isoformat(), + + # Aggressive vs Passive Analysis + 'aggressive_passive': { + 'aggressive_ratio': current_ratios.get('aggressive_ratio', 0), + 'passive_ratio': current_ratios.get('passive_ratio', 0), + 'aggressive_volume': current_ratios.get('aggressive_volume', 0), + 'passive_volume': current_ratios.get('passive_volume', 0), + 'avg_aggressive_size': current_ratios.get('avg_aggressive_size', 0), + 'avg_passive_size': current_ratios.get('avg_passive_size', 0), + 'trade_count': current_ratios.get('trade_count', 0) + }, + + # Institutional vs Retail Analysis + 'institutional_retail': { + 'institutional_ratio': institutional_volume / total_volume if total_volume > 0 else 0, + 'retail_ratio': retail_volume / total_volume if total_volume > 0 else 0, + 'institutional_volume': institutional_volume, + 'retail_volume': retail_volume, + 'institutional_trade_count': len(institutional_trades), + 'retail_trade_count': len(retail_trades), + 'block_trade_count': len(block_trades), + 'avg_institutional_size': institutional_volume / max(1, len(institutional_trades)), + 'avg_retail_size': retail_volume / max(1, len(retail_trades)) + }, + + # Trade Size Distribution + 'size_distribution': size_breakdown, + + # Order Flow Intensity + 'flow_intensity': { + 'current_intensity': avg_intensity, + 'intensity_category': 'high' if avg_intensity > 5 else 'medium' if avg_intensity > 2 else 'low' + }, + + # Liquidity Analysis + 'liquidity': { + 'avg_consumption_rate': avg_consumption_rate, + 'consumption_category': 'high' if avg_consumption_rate > 0.8 else 'medium' if avg_consumption_rate > 0.5 else 'low' + }, + + # Price Impact Analysis + 'price_impact': { + 'avg_impact': avg_price_impact * 10000, # in basis points + 'impact_distribution': impact_distribution, + 'impact_category': 'high' if avg_price_impact > 0.005 else 'medium' if avg_price_impact > 0.001 else 'low' + }, + + # Market Maker vs Taker Flow + 'maker_taker_flow': { + 'buy_aggressive_volume': buy_aggressive_volume, + 'sell_aggressive_volume': sell_aggressive_volume, + 'buy_pressure': buy_aggressive_volume / (buy_aggressive_volume + sell_aggressive_volume) if (buy_aggressive_volume + sell_aggressive_volume) > 0 else 0.5, + 'sell_pressure': sell_aggressive_volume / (buy_aggressive_volume + sell_aggressive_volume) if (buy_aggressive_volume + sell_aggressive_volume) > 0 else 0.5 + }, + + # 24h Volume Statistics (if available) + 'volume_stats': self.volume_stats.get(symbol, {}) + } + + except Exception as e: + logger.error(f"Error getting enhanced order flow metrics for {symbol}: {e}") + return None \ No newline at end of file diff --git a/core/cob_integration.py b/core/cob_integration.py new file mode 100644 index 0000000..b0ef2df --- /dev/null +++ b/core/cob_integration.py @@ -0,0 +1,597 @@ +""" +Consolidated Order Book (COB) Integration Module + +This module integrates the Multi-Exchange COB Provider with the existing +gogo2 trading system architecture, providing: + +- Integration with existing DataProvider +- CNN/DQN model data feeding +- Dashboard data formatting +- Trading signal generation based on COB analysis +- Enhanced market microstructure analysis + +Connects to the main trading dashboard and AI models. +""" + +import asyncio +import logging +import numpy as np +import pandas as pd +from datetime import datetime, timedelta +from typing import Dict, List, Optional, Any, Callable +from threading import Thread +import json +import math +from collections import defaultdict + +from .multi_exchange_cob_provider import MultiExchangeCOBProvider, COBSnapshot, ConsolidatedOrderBookLevel +from .data_provider import DataProvider, MarketTick + +logger = logging.getLogger(__name__) + +class COBIntegration: + """ + Integration layer for Multi-Exchange COB data with gogo2 trading system + """ + + def __init__(self, data_provider: DataProvider = None, symbols: List[str] = None): + """ + Initialize COB Integration + + Args: + data_provider: Existing DataProvider instance + symbols: List of symbols to monitor + """ + self.data_provider = data_provider + self.symbols = symbols or ['BTC/USDT', 'ETH/USDT'] + + # Initialize COB provider + self.cob_provider = MultiExchangeCOBProvider( + symbols=self.symbols, + bucket_size_bps=1.0 # 1 basis point granularity + ) + + # Register callbacks + self.cob_provider.subscribe_to_cob_updates(self._on_cob_update) + self.cob_provider.subscribe_to_bucket_updates(self._on_bucket_update) + + # CNN/DQN integration + self.cnn_callbacks: List[Callable] = [] + self.dqn_callbacks: List[Callable] = [] + self.dashboard_callbacks: List[Callable] = [] + + # COB analysis and signals + self.cob_signals: Dict[str, List[Dict]] = {} + self.liquidity_alerts: Dict[str, List[Dict]] = {} + self.arbitrage_opportunities: Dict[str, List[Dict]] = {} + + # Performance tracking + self.cob_feature_cache: Dict[str, np.ndarray] = {} + self.last_cob_features_update: Dict[str, datetime] = {} + + # Initialize signal tracking + for symbol in self.symbols: + self.cob_signals[symbol] = [] + self.liquidity_alerts[symbol] = [] + self.arbitrage_opportunities[symbol] = [] + + logger.info("COB Integration initialized") + logger.info(f"Symbols: {self.symbols}") + + async def start(self): + """Start COB integration""" + logger.info("Starting COB Integration") + + # Start COB provider + await self.cob_provider.start_streaming() + + # Start analysis threads + asyncio.create_task(self._continuous_cob_analysis()) + asyncio.create_task(self._continuous_signal_generation()) + + logger.info("COB Integration started successfully") + + async def stop(self): + """Stop COB integration""" + logger.info("Stopping COB Integration") + await self.cob_provider.stop_streaming() + logger.info("COB Integration stopped") + + def add_cnn_callback(self, callback: Callable[[str, Dict], None]): + """Add CNN model callback for COB features""" + self.cnn_callbacks.append(callback) + logger.info(f"Added CNN callback: {len(self.cnn_callbacks)} total") + + def add_dqn_callback(self, callback: Callable[[str, Dict], None]): + """Add DQN model callback for COB state features""" + self.dqn_callbacks.append(callback) + logger.info(f"Added DQN callback: {len(self.dqn_callbacks)} total") + + def add_dashboard_callback(self, callback: Callable[[str, Dict], None]): + """Add dashboard callback for COB visualization data""" + self.dashboard_callbacks.append(callback) + logger.info(f"Added dashboard callback: {len(self.dashboard_callbacks)} total") + + async def _on_cob_update(self, symbol: str, cob_snapshot: COBSnapshot): + """Handle COB update from provider""" + try: + # Generate CNN features + cnn_features = self._generate_cnn_features(symbol, cob_snapshot) + if cnn_features is not None: + self.cob_feature_cache[symbol] = cnn_features + self.last_cob_features_update[symbol] = datetime.now() + + # Notify CNN callbacks + for callback in self.cnn_callbacks: + try: + callback(symbol, { + 'features': cnn_features, + 'timestamp': cob_snapshot.timestamp, + 'type': 'cob_features' + }) + except Exception as e: + logger.warning(f"Error in CNN callback: {e}") + + # Generate DQN state features + dqn_features = self._generate_dqn_features(symbol, cob_snapshot) + if dqn_features is not None: + for callback in self.dqn_callbacks: + try: + callback(symbol, { + 'state': dqn_features, + 'timestamp': cob_snapshot.timestamp, + 'type': 'cob_state' + }) + except Exception as e: + logger.warning(f"Error in DQN callback: {e}") + + # Generate dashboard data + dashboard_data = self._generate_dashboard_data(symbol, cob_snapshot) + for callback in self.dashboard_callbacks: + try: + if asyncio.iscoroutinefunction(callback): + asyncio.create_task(callback(symbol, dashboard_data)) + else: + callback(symbol, dashboard_data) + except Exception as e: + logger.warning(f"Error in dashboard callback: {e}") + + except Exception as e: + logger.error(f"Error processing COB update for {symbol}: {e}") + + async def _on_bucket_update(self, symbol: str, price_buckets: Dict): + """Handle price bucket update from provider""" + try: + # Analyze bucket distribution and generate alerts + await self._analyze_bucket_distribution(symbol, price_buckets) + + except Exception as e: + logger.error(f"Error processing bucket update for {symbol}: {e}") + + def _generate_cnn_features(self, symbol: str, cob_snapshot: COBSnapshot) -> Optional[np.ndarray]: + """Generate CNN input features from COB data""" + try: + features = [] + + # Order book depth features (200 features: 20 levels x 5 features x 2 sides) + max_levels = 20 + + # Process bids + for i in range(max_levels): + if i < len(cob_snapshot.consolidated_bids): + level = cob_snapshot.consolidated_bids[i] + price_offset = (level.price - cob_snapshot.volume_weighted_mid) / cob_snapshot.volume_weighted_mid + features.extend([ + price_offset, + level.total_volume_usd / 1000000, # Normalize to millions + level.total_size / 1000, # Normalize to thousands + len(level.exchange_breakdown), + level.liquidity_score + ]) + else: + features.extend([0.0, 0.0, 0.0, 0.0, 0.0]) + + # Process asks + for i in range(max_levels): + if i < len(cob_snapshot.consolidated_asks): + level = cob_snapshot.consolidated_asks[i] + price_offset = (level.price - cob_snapshot.volume_weighted_mid) / cob_snapshot.volume_weighted_mid + features.extend([ + price_offset, + level.total_volume_usd / 1000000, + level.total_size / 1000, + len(level.exchange_breakdown), + level.liquidity_score + ]) + else: + features.extend([0.0, 0.0, 0.0, 0.0, 0.0]) + + # Market microstructure features (20 features) + features.extend([ + cob_snapshot.spread_bps / 100, # Normalize spread + cob_snapshot.liquidity_imbalance, + cob_snapshot.total_bid_liquidity / 1000000, + cob_snapshot.total_ask_liquidity / 1000000, + len(cob_snapshot.exchanges_active) / 5, # Normalize to max 5 exchanges + cob_snapshot.volume_weighted_mid / 100000, # Normalize price + + # Exchange diversity metrics + self._calculate_exchange_diversity(cob_snapshot.consolidated_bids), + self._calculate_exchange_diversity(cob_snapshot.consolidated_asks), + + # Price bucket concentration + self._calculate_bucket_concentration(cob_snapshot.price_buckets, 'bids'), + self._calculate_bucket_concentration(cob_snapshot.price_buckets, 'asks'), + + # Liquidity depth metrics + self._calculate_liquidity_depth_ratio(cob_snapshot.consolidated_bids, 5), + self._calculate_liquidity_depth_ratio(cob_snapshot.consolidated_asks, 5), + + # Time-based features + cob_snapshot.timestamp.hour / 24, + cob_snapshot.timestamp.minute / 60, + cob_snapshot.timestamp.weekday() / 7, + + # Additional features + 0.0, 0.0, 0.0, 0.0, 0.0 + ]) + + return np.array(features, dtype=np.float32) + + except Exception as e: + logger.error(f"Error generating CNN features for {symbol}: {e}") + return None + + def _generate_dqn_features(self, symbol: str, cob_snapshot: COBSnapshot) -> Optional[np.ndarray]: + """Generate DQN state features from COB data""" + try: + state_features = [] + + # Normalized order book state (20 features) + total_liquidity = cob_snapshot.total_bid_liquidity + cob_snapshot.total_ask_liquidity + + if total_liquidity > 0: + # Top 10 bid levels (normalized by total liquidity) + for i in range(10): + if i < len(cob_snapshot.consolidated_bids): + level = cob_snapshot.consolidated_bids[i] + state_features.append(level.total_volume_usd / total_liquidity) + else: + state_features.append(0.0) + + # Top 10 ask levels (normalized by total liquidity) + for i in range(10): + if i < len(cob_snapshot.consolidated_asks): + level = cob_snapshot.consolidated_asks[i] + state_features.append(level.total_volume_usd / total_liquidity) + else: + state_features.append(0.0) + else: + state_features.extend([0.0] * 20) + + # Market state indicators (10 features) + state_features.extend([ + cob_snapshot.spread_bps / 1000, # Normalized spread + cob_snapshot.liquidity_imbalance, + len(cob_snapshot.exchanges_active) / 5, # Exchange count ratio + min(1.0, total_liquidity / 10000000), # Liquidity abundance + 0.5, # Price efficiency placeholder + min(1.0, total_liquidity / 5000000), # Market impact resistance + 0.0, # Arbitrage score placeholder + 0.0, # Liquidity fragmentation placeholder + (datetime.now().hour * 60 + datetime.now().minute) / 1440, # Time of day + 0.5 # Market regime indicator placeholder + ]) + + return np.array(state_features, dtype=np.float32) + + except Exception as e: + logger.error(f"Error generating DQN features for {symbol}: {e}") + return None + + def _generate_dashboard_data(self, symbol: str, cob_snapshot: COBSnapshot) -> Dict: + """Generate formatted data for dashboard visualization""" + try: + # Get fixed bucket size for the symbol + bucket_size = self.cob_provider.fixed_usd_buckets.get(symbol, 1.0) + + # Calculate price range for buckets + mid_price = cob_snapshot.volume_weighted_mid + price_range = 100 # Show 100 price levels on each side + + # Initialize bucket arrays + bid_buckets = defaultdict(float) + ask_buckets = defaultdict(float) + + # Process bids into fixed USD buckets + for bid in cob_snapshot.consolidated_bids: + bucket_price = math.floor(bid.price / bucket_size) * bucket_size + bid_buckets[bucket_price] += bid.total_volume_usd + + # Process asks into fixed USD buckets + for ask in cob_snapshot.consolidated_asks: + bucket_price = math.floor(ask.price / bucket_size) * bucket_size + ask_buckets[bucket_price] += ask.total_volume_usd + + # Convert to sorted arrays for visualization + bid_data = [] + ask_data = [] + + # Generate price levels + min_price = math.floor((mid_price - (price_range * bucket_size)) / bucket_size) * bucket_size + max_price = math.ceil((mid_price + (price_range * bucket_size)) / bucket_size) * bucket_size + + # Fill bid data + current_price = mid_price + while current_price >= min_price: + bucket_price = math.floor(current_price / bucket_size) * bucket_size + volume = bid_buckets.get(bucket_price, 0) + if volume > 0: + bid_data.append({ + 'price': bucket_price, + 'volume': volume, + 'side': 'bid' + }) + current_price -= bucket_size + + # Fill ask data + current_price = mid_price + while current_price <= max_price: + bucket_price = math.floor(current_price / bucket_size) * bucket_size + volume = ask_buckets.get(bucket_price, 0) + if volume > 0: + ask_data.append({ + 'price': bucket_price, + 'volume': volume, + 'side': 'ask' + }) + current_price += bucket_size + + # Get actual Session Volume Profile (SVP) from trade data + svp_data = [] + try: + svp_result = self.cob_provider.get_session_volume_profile(symbol, bucket_size) + if svp_result and 'data' in svp_result: + svp_data = svp_result['data'] + logger.debug(f"Retrieved SVP data for {symbol}: {len(svp_data)} price levels") + else: + logger.warning(f"No SVP data available for {symbol}") + except Exception as e: + logger.error(f"Error getting SVP data for {symbol}: {e}") + + # Generate market stats + stats = { + 'symbol': symbol, + 'timestamp': cob_snapshot.timestamp.isoformat(), + 'mid_price': cob_snapshot.volume_weighted_mid, + 'spread_bps': cob_snapshot.spread_bps, + 'total_bid_liquidity': cob_snapshot.total_bid_liquidity, + 'total_ask_liquidity': cob_snapshot.total_ask_liquidity, + 'liquidity_imbalance': cob_snapshot.liquidity_imbalance, + 'exchanges_active': cob_snapshot.exchanges_active, + 'bucket_size': bucket_size + } + + # Add exchange diversity metrics + stats['bid_exchange_diversity'] = self._calculate_exchange_diversity(cob_snapshot.consolidated_bids[:20]) + stats['ask_exchange_diversity'] = self._calculate_exchange_diversity(cob_snapshot.consolidated_asks[:20]) + + # Add SVP statistics + if svp_data: + total_traded_volume = sum(item['total_volume'] for item in svp_data) + stats['total_traded_volume'] = total_traded_volume + stats['svp_price_levels'] = len(svp_data) + stats['session_start'] = svp_result.get('session_start', '') + else: + stats['total_traded_volume'] = 0 + stats['svp_price_levels'] = 0 + stats['session_start'] = '' + + # Add real-time statistics for NN models + try: + realtime_stats = self.cob_provider.get_realtime_stats(symbol) + if realtime_stats: + stats['realtime_1s'] = realtime_stats.get('1s_stats', {}) + stats['realtime_5s'] = realtime_stats.get('5s_stats', {}) + else: + stats['realtime_1s'] = {} + stats['realtime_5s'] = {} + except Exception as e: + logger.error(f"Error getting real-time stats for {symbol}: {e}") + stats['realtime_1s'] = {} + stats['realtime_5s'] = {} + + return { + 'type': 'cob_update', + 'data': { + 'bids': bid_data, + 'asks': ask_data, + 'svp': svp_data, + 'stats': stats + } + } + + except Exception as e: + logger.error(f"Error generating dashboard data for {symbol}: {e}") + return { + 'type': 'error', + 'data': {'error': str(e)} + } + + def _calculate_exchange_diversity(self, levels: List[ConsolidatedOrderBookLevel]) -> float: + """Calculate exchange diversity in order book levels""" + if not levels: + return 0.0 + + exchange_counts = {} + total_volume = 0 + + for level in levels[:10]: # Top 10 levels + total_volume += level.total_volume_usd + for exchange in level.exchange_breakdown: + exchange_counts[exchange] = exchange_counts.get(exchange, 0) + level.exchange_breakdown[exchange].volume_usd + + if total_volume == 0: + return 0.0 + + # Calculate diversity score + hhi = sum((volume / total_volume) ** 2 for volume in exchange_counts.values()) + return 1 - hhi + + def _calculate_bucket_concentration(self, price_buckets: Dict, side: str) -> float: + """Calculate concentration of liquidity in price buckets""" + buckets = price_buckets.get(side, {}) + if not buckets: + return 0.0 + + volumes = [bucket['volume_usd'] for bucket in buckets.values()] + total_volume = sum(volumes) + + if total_volume == 0: + return 0.0 + + sorted_volumes = sorted(volumes, reverse=True) + top_20_percent = int(len(sorted_volumes) * 0.2) or 1 + return sum(sorted_volumes[:top_20_percent]) / total_volume + + def _calculate_liquidity_depth_ratio(self, levels: List[ConsolidatedOrderBookLevel], top_n: int) -> float: + """Calculate ratio of top N levels liquidity to total""" + if not levels: + return 0.0 + + top_n_volume = sum(level.total_volume_usd for level in levels[:top_n]) + total_volume = sum(level.total_volume_usd for level in levels) + + return top_n_volume / total_volume if total_volume > 0 else 0.0 + + async def _continuous_cob_analysis(self): + """Continuously analyze COB data for patterns and signals""" + while True: + try: + for symbol in self.symbols: + cob_snapshot = self.cob_provider.get_consolidated_orderbook(symbol) + if cob_snapshot: + await self._analyze_cob_patterns(symbol, cob_snapshot) + + await asyncio.sleep(1) + + except Exception as e: + logger.error(f"Error in COB analysis loop: {e}") + await asyncio.sleep(5) + + async def _analyze_cob_patterns(self, symbol: str, cob_snapshot: COBSnapshot): + """Analyze COB data for trading patterns and signals""" + try: + # Large liquidity imbalance detection + if abs(cob_snapshot.liquidity_imbalance) > 0.4: + signal = { + 'timestamp': cob_snapshot.timestamp.isoformat(), + 'type': 'liquidity_imbalance', + 'side': 'buy' if cob_snapshot.liquidity_imbalance > 0 else 'sell', + 'strength': abs(cob_snapshot.liquidity_imbalance), + 'confidence': min(1.0, abs(cob_snapshot.liquidity_imbalance) * 2) + } + self.cob_signals[symbol].append(signal) + + # Cleanup old signals + self.cob_signals[symbol] = self.cob_signals[symbol][-100:] + + except Exception as e: + logger.error(f"Error analyzing COB patterns for {symbol}: {e}") + + async def _analyze_bucket_distribution(self, symbol: str, price_buckets: Dict): + """Analyze price bucket distribution for patterns""" + try: + # Placeholder for bucket analysis + pass + + except Exception as e: + logger.error(f"Error analyzing bucket distribution for {symbol}: {e}") + + async def _continuous_signal_generation(self): + """Continuously generate trading signals based on COB analysis""" + while True: + try: + await asyncio.sleep(5) + + except Exception as e: + logger.error(f"Error in signal generation loop: {e}") + await asyncio.sleep(10) + + # Public interface methods + + def get_cob_features(self, symbol: str) -> Optional[np.ndarray]: + """Get latest CNN features for a symbol""" + return self.cob_feature_cache.get(symbol) + + def get_cob_snapshot(self, symbol: str) -> Optional[COBSnapshot]: + """Get latest COB snapshot for a symbol""" + return self.cob_provider.get_consolidated_orderbook(symbol) + + def get_market_depth_analysis(self, symbol: str) -> Optional[Dict]: + """Get detailed market depth analysis""" + return self.cob_provider.get_market_depth_analysis(symbol) + + def get_exchange_breakdown(self, symbol: str) -> Optional[Dict]: + """Get liquidity breakdown by exchange""" + return self.cob_provider.get_exchange_breakdown(symbol) + + def get_price_buckets(self, symbol: str) -> Optional[Dict]: + """Get fine-grain price buckets""" + return self.cob_provider.get_price_buckets(symbol) + + def get_recent_signals(self, symbol: str, count: int = 20) -> List[Dict]: + """Get recent COB-based trading signals""" + return self.cob_signals.get(symbol, [])[-count:] + + def get_statistics(self) -> Dict[str, Any]: + """Get COB integration statistics""" + provider_stats = self.cob_provider.get_statistics() + + return { + **provider_stats, + 'cnn_callbacks': len(self.cnn_callbacks), + 'dqn_callbacks': len(self.dqn_callbacks), + 'dashboard_callbacks': len(self.dashboard_callbacks), + 'cached_features': list(self.cob_feature_cache.keys()), + 'total_signals': {symbol: len(signals) for symbol, signals in self.cob_signals.items()} + } + + def get_realtime_stats_for_nn(self, symbol: str) -> Dict: + """Get real-time statistics formatted for NN models""" + try: + realtime_stats = self.cob_provider.get_realtime_stats(symbol) + if not realtime_stats: + return {} + + # Format for NN consumption + nn_stats = { + 'symbol': symbol, + 'timestamp': datetime.now().isoformat(), + 'current': { + 'mid_price': 0.0, + 'spread_bps': 0.0, + 'bid_liquidity': 0.0, + 'ask_liquidity': 0.0, + 'imbalance': 0.0 + }, + '1s_window': realtime_stats.get('1s_stats', {}), + '5s_window': realtime_stats.get('5s_stats', {}) + } + + # Get current values from latest COB snapshot + cob_snapshot = self.cob_provider.get_consolidated_orderbook(symbol) + if cob_snapshot: + nn_stats['current'] = { + 'mid_price': cob_snapshot.volume_weighted_mid, + 'spread_bps': cob_snapshot.spread_bps, + 'bid_liquidity': cob_snapshot.total_bid_liquidity, + 'ask_liquidity': cob_snapshot.total_ask_liquidity, + 'imbalance': cob_snapshot.liquidity_imbalance + } + + return nn_stats + + except Exception as e: + logger.error(f"Error getting NN stats for {symbol}: {e}") + return {} \ No newline at end of file diff --git a/core/data_provider.py b/core/data_provider.py index 1f8fdca..01f80f2 100644 --- a/core/data_provider.py +++ b/core/data_provider.py @@ -180,6 +180,37 @@ class DataProvider: logger.info("Centralized data distribution enabled") logger.info("Pivot-based normalization system enabled") + def _ensure_datetime_index(self, df: pd.DataFrame) -> pd.DataFrame: + """Ensure dataframe has proper datetime index""" + if df is None or df.empty: + return df + + try: + # If we already have a proper DatetimeIndex, return as is + if isinstance(df.index, pd.DatetimeIndex): + return df + + # If timestamp column exists, use it as index + if 'timestamp' in df.columns: + df['timestamp'] = pd.to_datetime(df['timestamp']) + df.set_index('timestamp', inplace=True) + return df + + # If we have a RangeIndex or other non-datetime index, create datetime index + if isinstance(df.index, pd.RangeIndex) or not isinstance(df.index, pd.DatetimeIndex): + # Use current time and work backwards for realistic timestamps + from datetime import datetime, timedelta + end_time = datetime.now() + start_time = end_time - timedelta(minutes=len(df)) + df.index = pd.date_range(start=start_time, end=end_time, periods=len(df)) + logger.debug(f"Converted RangeIndex to DatetimeIndex for {len(df)} records") + + return df + + except Exception as e: + logger.warning(f"Error ensuring datetime index: {e}") + return df + def get_historical_data(self, symbol: str, timeframe: str, limit: int = 1000, refresh: bool = False) -> Optional[pd.DataFrame]: """Get historical OHLCV data for a symbol and timeframe""" try: @@ -188,6 +219,8 @@ class DataProvider: if self.cache_enabled: cached_data = self._load_from_cache(symbol, timeframe) if cached_data is not None and len(cached_data) >= limit * 0.8: + # Ensure proper datetime index for cached data + cached_data = self._ensure_datetime_index(cached_data) # logger.info(f"Using cached data for {symbol} {timeframe}") return cached_data.tail(limit) @@ -208,8 +241,11 @@ class DataProvider: df = self._fetch_from_mexc(symbol, timeframe, limit) if df is not None and not df.empty: - # Add technical indicators - df = self._add_technical_indicators(df) + # Ensure proper datetime index + df = self._ensure_datetime_index(df) + + # Add technical indicators. temporarily disabled to save time as it is not working as expected. + # df = self._add_technical_indicators(df) # Cache the data if self.cache_enabled: @@ -1151,9 +1187,21 @@ class DataProvider: try: cache_file = self.monthly_data_cache_dir / f"{symbol.replace('/', '')}_monthly_1m.parquet" if cache_file.exists(): - df = pd.read_parquet(cache_file) - logger.info(f"Loaded {len(df)} 1m candles from cache for {symbol}") - return df + try: + df = pd.read_parquet(cache_file) + logger.info(f"Loaded {len(df)} 1m candles from cache for {symbol}") + return df + except Exception as parquet_e: + # Handle corrupted Parquet file + if "Parquet magic bytes not found" in str(parquet_e) or "corrupted" in str(parquet_e).lower(): + logger.warning(f"Corrupted Parquet cache file for {symbol}, removing and returning None: {parquet_e}") + try: + cache_file.unlink() # Delete corrupted file + except Exception: + pass + return None + else: + raise parquet_e return None @@ -1240,9 +1288,21 @@ class DataProvider: # Check if cache is recent (less than 1 hour old) cache_age = time.time() - cache_file.stat().st_mtime if cache_age < 3600: # 1 hour - df = pd.read_parquet(cache_file) - logger.debug(f"Loaded {len(df)} rows from cache for {symbol} {timeframe}") - return df + try: + df = pd.read_parquet(cache_file) + logger.debug(f"Loaded {len(df)} rows from cache for {symbol} {timeframe}") + return df + except Exception as parquet_e: + # Handle corrupted Parquet file + if "Parquet magic bytes not found" in str(parquet_e) or "corrupted" in str(parquet_e).lower(): + logger.warning(f"Corrupted Parquet cache file for {symbol} {timeframe}, removing and returning None: {parquet_e}") + try: + cache_file.unlink() # Delete corrupted file + except Exception: + pass + return None + else: + raise parquet_e else: logger.debug(f"Cache for {symbol} {timeframe} is too old ({cache_age/3600:.1f}h)") return None diff --git a/core/enhanced_orchestrator.py b/core/enhanced_orchestrator.py index 3691cd3..1e56399 100644 --- a/core/enhanced_orchestrator.py +++ b/core/enhanced_orchestrator.py @@ -2324,7 +2324,14 @@ class EnhancedTradingOrchestrator: # 4. Return threshold adjustment (0.0 to 0.1 typically) # For now, return small adjustment to demonstrate concept - if hasattr(self.pivot_rl_trainer.williams, 'cnn_model') and self.pivot_rl_trainer.williams.cnn_model: + # Check if CNN models are available in the model registry + cnn_available = False + for model_key, model in self.model_registry.items(): + if hasattr(model, 'cnn_model') and model.cnn_model: + cnn_available = True + break + + if cnn_available: # CNN is available, could provide small threshold reduction for better entries return 0.05 # 5% threshold reduction when CNN available @@ -2337,17 +2344,27 @@ class EnhancedTradingOrchestrator: def update_dynamic_thresholds(self): """Update thresholds based on recent performance""" try: - # Update thresholds in pivot trainer - self.pivot_rl_trainer.update_thresholds_based_on_performance() + # Internal threshold update based on recent performance + # This orchestrator handles thresholds internally without external trainer - # Get updated thresholds - thresholds = self.pivot_rl_trainer.get_current_thresholds() old_entry = self.entry_threshold old_exit = self.exit_threshold - self.entry_threshold = thresholds['entry_threshold'] - self.exit_threshold = thresholds['exit_threshold'] - self.uninvested_threshold = thresholds['uninvested_threshold'] + # Simple performance-based threshold adjustment + if len(self.completed_trades) >= 10: + recent_trades = list(self.completed_trades)[-10:] + win_rate = sum(1 for trade in recent_trades if trade.get('pnl_percentage', 0) > 0) / len(recent_trades) + + # Adjust thresholds based on recent performance + if win_rate > 0.7: # High win rate - can be more aggressive + self.entry_threshold = max(0.5, self.entry_threshold - 0.02) + self.exit_threshold = min(0.5, self.exit_threshold + 0.02) + elif win_rate < 0.3: # Low win rate - be more conservative + self.entry_threshold = min(0.8, self.entry_threshold + 0.02) + self.exit_threshold = max(0.2, self.exit_threshold - 0.02) + + # Update uninvested threshold based on activity + self.uninvested_threshold = (self.entry_threshold + self.exit_threshold) / 2 # Log changes if significant if abs(old_entry - self.entry_threshold) > 0.01 or abs(old_exit - self.exit_threshold) > 0.01: @@ -2362,9 +2379,32 @@ class EnhancedTradingOrchestrator: trade_outcome: Dict[str, Any]) -> float: """Calculate reward using the enhanced pivot-based system""" try: - return self.pivot_rl_trainer.calculate_pivot_based_reward( - trade_decision, market_data, trade_outcome - ) + # Simplified pivot-based reward calculation without external trainer + # This orchestrator handles pivot logic internally via dynamic thresholds + + if not trade_outcome or 'pnl_percentage' not in trade_outcome: + return 0.0 + + pnl_percentage = trade_outcome['pnl_percentage'] + confidence = trade_decision.get('confidence', 0.5) + + # Base reward from PnL + base_reward = pnl_percentage * 10 # Scale PnL to reasonable reward range + + # Bonus for high-confidence decisions that work out + confidence_bonus = 0.0 + if pnl_percentage > 0 and confidence > self.entry_threshold: + confidence_bonus = (confidence - self.entry_threshold) * 5.0 + + # Penalty for low-confidence losses + confidence_penalty = 0.0 + if pnl_percentage < 0 and confidence < self.exit_threshold: + confidence_penalty = abs(pnl_percentage) * 2.0 + + total_reward = base_reward + confidence_bonus - confidence_penalty + + return total_reward + except Exception as e: logger.error(f"Error calculating enhanced pivot reward: {e}") return 0.0 diff --git a/core/multi_exchange_cob_provider.py b/core/multi_exchange_cob_provider.py new file mode 100644 index 0000000..beaa622 --- /dev/null +++ b/core/multi_exchange_cob_provider.py @@ -0,0 +1,1268 @@ +""" +Multi-Exchange Consolidated Order Book (COB) Data Provider + +This module aggregates order book data from multiple cryptocurrency exchanges to provide: +- Consolidated Order Book (COB) data across multiple exchanges +- Fine-grain volume buckets at configurable price levels +- Real-time order book depth aggregation +- Volume-weighted consolidated pricing +- Exchange-specific order flow analysis +- Liquidity distribution metrics + +Supported Exchanges: +- Binance (via WebSocket depth streams) +- Coinbase Pro (via WebSocket level2 updates) +- Kraken (via WebSocket book updates) +- Huobi (via WebSocket mbp updates) +- Bitfinex (via WebSocket book updates) + +Data is structured for consumption by CNN/DQN models and trading dashboards. +""" + +import asyncio +import json +import logging +import time +import websockets +import numpy as np +import pandas as pd +from datetime import datetime, timedelta +from typing import Dict, List, Optional, Tuple, Any, Callable, Union +from collections import deque, defaultdict +from dataclasses import dataclass, field +from threading import Thread, Lock +import requests +import ccxt +from enum import Enum +import math +import aiohttp +import aiohttp.resolver + +logger = logging.getLogger(__name__) + +class ExchangeType(Enum): + BINANCE = "binance" + COINBASE = "coinbase" + KRAKEN = "kraken" + HUOBI = "huobi" + BITFINEX = "bitfinex" + +@dataclass +class ExchangeOrderBookLevel: + """Single order book level with exchange attribution""" + exchange: str + price: float + size: float + volume_usd: float + orders_count: int + side: str # 'bid' or 'ask' + timestamp: datetime + raw_data: Dict[str, Any] = field(default_factory=dict) + +@dataclass +class ConsolidatedOrderBookLevel: + """Consolidated order book level across multiple exchanges""" + price: float + total_size: float + total_volume_usd: float + total_orders: int + side: str + exchange_breakdown: Dict[str, ExchangeOrderBookLevel] + dominant_exchange: str + liquidity_score: float + timestamp: datetime + +@dataclass +class COBSnapshot: + """Complete Consolidated Order Book snapshot""" + symbol: str + timestamp: datetime + consolidated_bids: List[ConsolidatedOrderBookLevel] + consolidated_asks: List[ConsolidatedOrderBookLevel] + exchanges_active: List[str] + volume_weighted_mid: float + total_bid_liquidity: float + total_ask_liquidity: float + spread_bps: float + liquidity_imbalance: float + price_buckets: Dict[str, Dict[str, float]] # Fine-grain volume buckets + +@dataclass +class ExchangeConfig: + """Exchange configuration for COB aggregation""" + exchange_type: ExchangeType + weight: float = 1.0 + enabled: bool = True + websocket_url: str = "" + rest_api_url: str = "" + symbols_mapping: Dict[str, str] = field(default_factory=dict) + rate_limits: Dict[str, int] = field(default_factory=dict) + +class MultiExchangeCOBProvider: + """ + Multi-Exchange Consolidated Order Book Data Provider + + Aggregates real-time order book data from multiple cryptocurrency exchanges + to create a consolidated view of market liquidity and pricing. + """ + + def __init__(self, symbols: List[str] = None, bucket_size_bps: float = 1.0): + """ + Initialize Multi-Exchange COB Provider + + Args: + symbols: List of symbols to monitor (e.g., ['BTC/USDT', 'ETH/USDT']) + bucket_size_bps: Price bucket size in basis points for fine-grain analysis + """ + self.symbols = symbols or ['BTC/USDT', 'ETH/USDT'] + self.bucket_size_bps = bucket_size_bps + self.bucket_update_frequency = 100 # ms + self.consolidation_frequency = 100 # ms + + # REST API configuration for deep order book + self.rest_api_frequency = 5000 # ms - full snapshot every 5 seconds + self.rest_depth_limit = 1000 # Get up to 1000 levels via REST + + # Exchange configurations + self.exchange_configs = self._initialize_exchange_configs() + + # Order book storage - now with deep and live separation + self.exchange_order_books = { + symbol: { + exchange.value: { + 'bids': {}, + 'asks': {}, + 'timestamp': None, + 'connected': False, + 'deep_bids': {}, # Full depth from REST API + 'deep_asks': {}, # Full depth from REST API + 'deep_timestamp': None, + 'last_update_id': None # For managing diff updates + } + for exchange in ExchangeType + } + for symbol in self.symbols + } + + # Consolidated order books + self.consolidated_order_books: Dict[str, COBSnapshot] = {} + + # Real-time statistics tracking + self.realtime_stats: Dict[str, Dict] = {symbol: {} for symbol in self.symbols} + self.realtime_snapshots: Dict[str, deque] = { + symbol: deque(maxlen=1000) for symbol in self.symbols + } + + # Session tracking for SVP + self.session_start_time = datetime.now() + self.session_trades: Dict[str, List[Dict]] = {symbol: [] for symbol in self.symbols} + self.svp_cache: Dict[str, Dict] = {symbol: {} for symbol in self.symbols} + + # Fixed USD bucket sizes for different symbols + self.fixed_usd_buckets = { + 'BTC/USDT': 10.0, # $10 buckets for BTC + 'ETH/USDT': 1.0, # $1 buckets for ETH + } + + # WebSocket management + self.is_streaming = False + self.active_exchanges = ['binance'] # Start with Binance only + + # Callbacks for real-time updates + self.cob_update_callbacks = [] + self.bucket_update_callbacks = [] + + # Performance tracking + self.exchange_update_counts = {exchange.value: 0 for exchange in ExchangeType} + self.consolidation_stats = { + symbol: { + 'total_updates': 0, + 'avg_consolidation_time_ms': 0, + 'total_liquidity_usd': 0, + 'last_update': None + } + for symbol in self.symbols + } + self.processing_times = {'consolidation': deque(maxlen=100), 'rest_api': deque(maxlen=100)} + + # Thread safety + self.data_lock = asyncio.Lock() + + # Create REST API session + # Fix for Windows aiodns issue - use ThreadedResolver instead + connector = aiohttp.TCPConnector( + resolver=aiohttp.ThreadedResolver(), + use_dns_cache=False + ) + self.rest_session = aiohttp.ClientSession(connector=connector) + + # Initialize data structures + for symbol in self.symbols: + self.exchange_order_books[symbol]['binance']['connected'] = False + self.exchange_order_books[symbol]['binance']['deep_bids'] = {} + self.exchange_order_books[symbol]['binance']['deep_asks'] = {} + self.exchange_order_books[symbol]['binance']['deep_timestamp'] = None + self.exchange_order_books[symbol]['binance']['last_update_id'] = None + self.realtime_snapshots[symbol].append(COBSnapshot( + symbol=symbol, + timestamp=datetime.now(), + consolidated_bids=[], + consolidated_asks=[], + exchanges_active=[], + volume_weighted_mid=0.0, + total_bid_liquidity=0.0, + total_ask_liquidity=0.0, + spread_bps=0.0, + liquidity_imbalance=0.0, + price_buckets={} + )) + + logger.info(f"Multi-Exchange COB Provider initialized") + logger.info(f"Symbols: {self.symbols}") + logger.info(f"Bucket size: {bucket_size_bps} bps") + logger.info(f"Fixed USD buckets: {self.fixed_usd_buckets}") + logger.info(f"Configured exchanges: {[e.value for e in ExchangeType]}") + + def _initialize_exchange_configs(self) -> Dict[str, ExchangeConfig]: + """Initialize exchange configurations""" + configs = {} + + # Binance configuration + configs[ExchangeType.BINANCE.value] = ExchangeConfig( + exchange_type=ExchangeType.BINANCE, + weight=0.3, # Higher weight due to volume + websocket_url="wss://stream.binance.com:9443/ws/", + rest_api_url="https://api.binance.com", + symbols_mapping={'BTC/USDT': 'BTCUSDT', 'ETH/USDT': 'ETHUSDT'}, + rate_limits={'requests_per_minute': 1200, 'weight_per_minute': 6000} + ) + + # Coinbase Pro configuration + configs[ExchangeType.COINBASE.value] = ExchangeConfig( + exchange_type=ExchangeType.COINBASE, + weight=0.25, + websocket_url="wss://ws-feed.exchange.coinbase.com", + rest_api_url="https://api.exchange.coinbase.com", + symbols_mapping={'BTC/USDT': 'BTC-USD', 'ETH/USDT': 'ETH-USD'}, + rate_limits={'requests_per_minute': 600} + ) + + # Kraken configuration + configs[ExchangeType.KRAKEN.value] = ExchangeConfig( + exchange_type=ExchangeType.KRAKEN, + weight=0.2, + websocket_url="wss://ws.kraken.com", + rest_api_url="https://api.kraken.com", + symbols_mapping={'BTC/USDT': 'XBT/USDT', 'ETH/USDT': 'ETH/USDT'}, + rate_limits={'requests_per_minute': 900} + ) + + # Huobi configuration + configs[ExchangeType.HUOBI.value] = ExchangeConfig( + exchange_type=ExchangeType.HUOBI, + weight=0.15, + websocket_url="wss://api.huobi.pro/ws", + rest_api_url="https://api.huobi.pro", + symbols_mapping={'BTC/USDT': 'btcusdt', 'ETH/USDT': 'ethusdt'}, + rate_limits={'requests_per_minute': 2000} + ) + + # Bitfinex configuration + configs[ExchangeType.BITFINEX.value] = ExchangeConfig( + exchange_type=ExchangeType.BITFINEX, + weight=0.1, + websocket_url="wss://api-pub.bitfinex.com/ws/2", + rest_api_url="https://api-pub.bitfinex.com", + symbols_mapping={'BTC/USDT': 'tBTCUST', 'ETH/USDT': 'tETHUST'}, + rate_limits={'requests_per_minute': 1000} + ) + + return configs + + async def start_streaming(self): + """Start streaming from all configured exchanges""" + if self.is_streaming: + logger.warning("COB streaming already active") + return + + logger.info("Starting Multi-Exchange COB streaming") + self.is_streaming = True + + # Start streaming tasks for each exchange and symbol + tasks = [] + + for exchange_name in self.active_exchanges: + for symbol in self.symbols: + # WebSocket task for real-time top 20 levels + task = asyncio.create_task( + self._stream_exchange_orderbook(exchange_name, symbol) + ) + tasks.append(task) + + # REST API task for deep order book snapshots + deep_task = asyncio.create_task( + self._stream_deep_orderbook(exchange_name, symbol) + ) + tasks.append(deep_task) + + # Trade stream task for SVP + if exchange_name == 'binance': + trade_task = asyncio.create_task( + self._stream_binance_trades(symbol) + ) + tasks.append(trade_task) + + # Start consolidation and analysis tasks + tasks.extend([ + asyncio.create_task(self._continuous_consolidation()), + asyncio.create_task(self._continuous_bucket_updates()) + ]) + + # Wait for all tasks + try: + await asyncio.gather(*tasks) + except Exception as e: + logger.error(f"Error in streaming tasks: {e}") + finally: + self.is_streaming = False + + async def stop_streaming(self): + """Stop streaming from all exchanges""" + logger.info("Stopping Multi-Exchange COB streaming") + self.is_streaming = False + + # Close REST API session + if self.rest_session: + await self.rest_session.close() + self.rest_session = None + + # Wait a bit for tasks to stop gracefully + await asyncio.sleep(1) + + async def _stream_deep_orderbook(self, exchange_name: str, symbol: str): + """Fetch deep order book data via REST API periodically""" + while self.is_streaming: + try: + start_time = time.time() + + if exchange_name == 'binance': + await self._fetch_binance_deep_orderbook(symbol) + # Add other exchanges here as needed + + processing_time = (time.time() - start_time) * 1000 + self.processing_times['rest_api'].append(processing_time) + + logger.debug(f"Deep order book fetch for {symbol} took {processing_time:.2f}ms") + + # Wait before next fetch + await asyncio.sleep(self.rest_api_frequency / 1000) + + except Exception as e: + logger.error(f"Error fetching deep order book for {symbol}: {e}") + await asyncio.sleep(5) # Wait 5 seconds on error + + async def _fetch_binance_deep_orderbook(self, symbol: str): + """Fetch deep order book from Binance REST API""" + try: + if not self.rest_session: + return + + # Convert symbol format for Binance + binance_symbol = symbol.replace('/', '').upper() + url = f"https://api.binance.com/api/v3/depth" + params = { + 'symbol': binance_symbol, + 'limit': self.rest_depth_limit + } + + async with self.rest_session.get(url, params=params) as response: + if response.status == 200: + data = await response.json() + await self._process_binance_deep_orderbook(symbol, data) + else: + logger.error(f"Binance REST API error {response.status} for {symbol}") + + except Exception as e: + logger.error(f"Error fetching Binance deep order book for {symbol}: {e}") + + async def _process_binance_deep_orderbook(self, symbol: str, data: Dict): + """Process deep order book data from Binance REST API""" + try: + timestamp = datetime.now() + exchange_name = 'binance' + + # Parse deep bids and asks + deep_bids = {} + deep_asks = {} + + for bid_data in data.get('bids', []): + price = float(bid_data[0]) + size = float(bid_data[1]) + if size > 0: + deep_bids[price] = ExchangeOrderBookLevel( + exchange=exchange_name, + price=price, + size=size, + volume_usd=price * size, + orders_count=1, + side='bid', + timestamp=timestamp + ) + + for ask_data in data.get('asks', []): + price = float(ask_data[0]) + size = float(ask_data[1]) + if size > 0: + deep_asks[price] = ExchangeOrderBookLevel( + exchange=exchange_name, + price=price, + size=size, + volume_usd=price * size, + orders_count=1, + side='ask', + timestamp=timestamp + ) + + # Update deep order book storage + async with self.data_lock: + self.exchange_order_books[symbol][exchange_name]['deep_bids'] = deep_bids + self.exchange_order_books[symbol][exchange_name]['deep_asks'] = deep_asks + self.exchange_order_books[symbol][exchange_name]['deep_timestamp'] = timestamp + self.exchange_order_books[symbol][exchange_name]['last_update_id'] = data.get('lastUpdateId') + + logger.debug(f"Updated deep order book for {symbol}: {len(deep_bids)} bids, {len(deep_asks)} asks") + + except Exception as e: + logger.error(f"Error processing deep order book for {symbol}: {e}") + + async def _stream_exchange_orderbook(self, exchange_name: str, symbol: str): + """Stream order book data from specific exchange""" + config = self.exchange_configs[exchange_name] + + try: + if exchange_name == ExchangeType.BINANCE.value: + await self._stream_binance_orderbook(symbol, config) + elif exchange_name == ExchangeType.COINBASE.value: + await self._stream_coinbase_orderbook(symbol, config) + elif exchange_name == ExchangeType.KRAKEN.value: + await self._stream_kraken_orderbook(symbol, config) + elif exchange_name == ExchangeType.HUOBI.value: + await self._stream_huobi_orderbook(symbol, config) + elif exchange_name == ExchangeType.BITFINEX.value: + await self._stream_bitfinex_orderbook(symbol, config) + + except Exception as e: + logger.error(f"Error streaming {exchange_name} for {symbol}: {e}") + await asyncio.sleep(5) # Wait before reconnecting + + async def _stream_binance_orderbook(self, symbol: str, config: ExchangeConfig): + """Stream order book data from Binance""" + try: + ws_url = f"{config.websocket_url}{config.symbols_mapping[symbol].lower()}@depth20@100ms" + logger.info(f"Connecting to Binance WebSocket: {ws_url}") + + async with websockets.connect(ws_url) as websocket: + self.exchange_order_books[symbol]['binance']['connected'] = True + logger.info(f"Connected to Binance order book stream for {symbol}") + + async for message in websocket: + if not self.is_streaming: + break + + try: + data = json.loads(message) + await self._process_binance_orderbook(symbol, data) + + # Also track trades for SVP + await self._track_binance_trades(symbol, data) + + except json.JSONDecodeError as e: + logger.error(f"Error parsing Binance message: {e}") + except Exception as e: + logger.error(f"Error processing Binance data: {e}") + + except Exception as e: + logger.error(f"Binance WebSocket error for {symbol}: {e}") + finally: + self.exchange_order_books[symbol]['binance']['connected'] = False + logger.info(f"Disconnected from Binance order book stream for {symbol}") + + async def _track_binance_trades(self, symbol: str, data: Dict): + """Track executed trades from Binance for SVP calculation""" + try: + # Binance depth stream doesn't include trades, so we need to connect to trade stream + if 'e' in data and data['e'] == 'trade': + trade = { + 'exchange': 'binance', + 'symbol': symbol, + 'price': float(data['p']), + 'quantity': float(data['q']), + 'side': 'buy' if data['m'] else 'sell', # m is true for maker sell + 'timestamp': datetime.fromtimestamp(data['T'] / 1000), + 'volume_usd': float(data['p']) * float(data['q']) + } + + await self._add_trade_to_svp(symbol, trade) + + except Exception as e: + logger.error(f"Error tracking Binance trade: {e}") + + async def _add_trade_to_svp(self, symbol: str, trade: Dict): + """Add trade to session volume profile""" + try: + async with self.data_lock: + # Add to session trades + self.session_trades[symbol].append(trade) + + # Update SVP cache + price = trade['price'] + side = trade['side'] + volume = trade['volume_usd'] + + if price not in self.svp_cache[symbol]: + self.svp_cache[symbol][price] = {'buy_volume': 0.0, 'sell_volume': 0.0} + + if side == 'buy': + self.svp_cache[symbol][price]['buy_volume'] += volume + else: + self.svp_cache[symbol][price]['sell_volume'] += volume + + # Keep only recent trades (last 24 hours) + cutoff_time = datetime.now() - timedelta(hours=24) + self.session_trades[symbol] = [ + t for t in self.session_trades[symbol] + if t['timestamp'] > cutoff_time + ] + + except Exception as e: + logger.error(f"Error adding trade to SVP: {e}") + + def get_session_volume_profile(self, symbol: str, bucket_size: float = None) -> Dict: + """Get session volume profile for a symbol""" + try: + if bucket_size is None: + bucket_size = self.fixed_usd_buckets.get(symbol, 1.0) + + svp_data = {} + + # Access SVP cache without lock for read-only operations (generally safe) + try: + for price, volumes in self.svp_cache[symbol].items(): + bucket_price = math.floor(price / bucket_size) * bucket_size + + if bucket_price not in svp_data: + svp_data[bucket_price] = { + 'buy_volume': 0.0, + 'sell_volume': 0.0, + 'total_volume': 0.0, + 'trade_count': 0 + } + + svp_data[bucket_price]['buy_volume'] += volumes['buy_volume'] + svp_data[bucket_price]['sell_volume'] += volumes['sell_volume'] + svp_data[bucket_price]['total_volume'] += volumes['buy_volume'] + volumes['sell_volume'] + svp_data[bucket_price]['trade_count'] += 1 + except Exception as e: + logger.error(f"Error accessing SVP cache for {symbol}: {e}") + return {} + + # Convert to sorted list + svp_list = [] + for price in sorted(svp_data.keys()): + data = svp_data[price] + if data['total_volume'] > 0: + svp_list.append({ + 'price': price, + 'buy_volume': data['buy_volume'], + 'sell_volume': data['sell_volume'], + 'total_volume': data['total_volume'], + 'trade_count': data['trade_count'], + 'buy_percent': (data['buy_volume'] / data['total_volume']) * 100 if data['total_volume'] > 0 else 0, + 'sell_percent': (data['sell_volume'] / data['total_volume']) * 100 if data['total_volume'] > 0 else 0 + }) + + return { + 'symbol': symbol, + 'session_start': self.session_start_time.isoformat(), + 'bucket_size': bucket_size, + 'data': svp_list + } + + except Exception as e: + logger.error(f"Error getting session volume profile for {symbol}: {e}") + return {} + + async def _process_binance_orderbook(self, symbol: str, data: Dict): + """Process Binance order book update""" + try: + timestamp = datetime.now() + exchange_name = ExchangeType.BINANCE.value + + # Parse bids and asks + bids = {} + asks = {} + + for bid_data in data.get('bids', []): + price = float(bid_data[0]) + size = float(bid_data[1]) + if size > 0: # Only include non-zero sizes + bids[price] = ExchangeOrderBookLevel( + exchange=exchange_name, + price=price, + size=size, + volume_usd=price * size, + orders_count=1, + side='bid', + timestamp=timestamp + ) + + for ask_data in data.get('asks', []): + price = float(ask_data[0]) + size = float(ask_data[1]) + if size > 0: + asks[price] = ExchangeOrderBookLevel( + exchange=exchange_name, + price=price, + size=size, + volume_usd=price * size, + orders_count=1, + side='ask', + timestamp=timestamp + ) + + # Update exchange order book + async with self.data_lock: + self.exchange_order_books[symbol][exchange_name].update({ + 'bids': bids, + 'asks': asks, + 'timestamp': timestamp, + 'connected': True + }) + + logger.debug(f"Updated Binance order book for {symbol}: {len(bids)} bids, {len(asks)} asks") + + self.exchange_update_counts[exchange_name] += 1 + + # Log every 100th update + if self.exchange_update_counts[exchange_name] % 100 == 0: + logger.info(f"Processed {self.exchange_update_counts[exchange_name]} Binance updates for {symbol}") + + except Exception as e: + logger.error(f"Error processing Binance order book for {symbol}: {e}", exc_info=True) + + async def _stream_coinbase_orderbook(self, symbol: str, config: ExchangeConfig): + """Stream Coinbase order book data (placeholder implementation)""" + try: + # For now, just log that Coinbase streaming is not implemented + logger.info(f"Coinbase streaming for {symbol} not yet implemented") + await asyncio.sleep(60) # Sleep to prevent spam + except Exception as e: + logger.error(f"Error streaming Coinbase order book for {symbol}: {e}") + + async def _stream_kraken_orderbook(self, symbol: str, config: ExchangeConfig): + """Stream Kraken order book data (placeholder implementation)""" + try: + logger.info(f"Kraken streaming for {symbol} not yet implemented") + await asyncio.sleep(60) # Sleep to prevent spam + except Exception as e: + logger.error(f"Error streaming Kraken order book for {symbol}: {e}") + + async def _stream_huobi_orderbook(self, symbol: str, config: ExchangeConfig): + """Stream Huobi order book data (placeholder implementation)""" + try: + logger.info(f"Huobi streaming for {symbol} not yet implemented") + await asyncio.sleep(60) # Sleep to prevent spam + except Exception as e: + logger.error(f"Error streaming Huobi order book for {symbol}: {e}") + + async def _stream_bitfinex_orderbook(self, symbol: str, config: ExchangeConfig): + """Stream Bitfinex order book data (placeholder implementation)""" + try: + logger.info(f"Bitfinex streaming for {symbol} not yet implemented") + await asyncio.sleep(60) # Sleep to prevent spam + except Exception as e: + logger.error(f"Error streaming Bitfinex order book for {symbol}: {e}") + + async def _stream_binance_trades(self, symbol: str): + """Stream trade data from Binance for SVP calculation""" + try: + config = self.exchange_configs[ExchangeType.BINANCE.value] + ws_url = f"{config.websocket_url}{config.symbols_mapping[symbol].lower()}@trade" + logger.info(f"Connecting to Binance trade stream: {ws_url}") + + async with websockets.connect(ws_url) as websocket: + logger.info(f"Connected to Binance trade stream for {symbol}") + + async for message in websocket: + if not self.is_streaming: + break + + try: + data = json.loads(message) + await self._process_binance_trade(symbol, data) + + except json.JSONDecodeError as e: + logger.error(f"Error parsing Binance trade message: {e}") + except Exception as e: + logger.error(f"Error processing Binance trade: {e}") + + except Exception as e: + logger.error(f"Binance trade stream error for {symbol}: {e}") + finally: + logger.info(f"Disconnected from Binance trade stream for {symbol}") + + async def _process_binance_trade(self, symbol: str, data: Dict): + """Process Binance trade data for SVP calculation""" + try: + if 'e' in data and data['e'] == 'trade': + trade = { + 'exchange': 'binance', + 'symbol': symbol, + 'price': float(data['p']), + 'quantity': float(data['q']), + 'side': 'buy' if not data['m'] else 'sell', # m is true for maker sell + 'timestamp': datetime.fromtimestamp(data['T'] / 1000), + 'volume_usd': float(data['p']) * float(data['q']) + } + + await self._add_trade_to_svp(symbol, trade) + + # Log every 100th trade + if len(self.session_trades[symbol]) % 100 == 0: + logger.info(f"Tracked {len(self.session_trades[symbol])} trades for {symbol}") + + except Exception as e: + logger.error(f"Error processing Binance trade for {symbol}: {e}") + + async def _continuous_consolidation(self): + """Continuously consolidate order books from all exchanges""" + while self.is_streaming: + try: + start_time = time.time() + + for symbol in self.symbols: + logger.debug(f"Starting consolidation for {symbol}") + await self._consolidate_symbol_orderbook(symbol) + + processing_time = (time.time() - start_time) * 1000 + self.processing_times['consolidation'].append(processing_time) + + # Log consolidation performance every 100 iterations + if len(self.processing_times['consolidation']) % 100 == 0: + avg_time = sum(self.processing_times['consolidation']) / len(self.processing_times['consolidation']) + logger.info(f"Average consolidation time: {avg_time:.2f}ms") + + await asyncio.sleep(0.1) # 100ms consolidation frequency + + except Exception as e: + logger.error(f"Error in consolidation loop: {e}", exc_info=True) + await asyncio.sleep(1) + + async def _consolidate_symbol_orderbook(self, symbol: str): + """Consolidate order book for a specific symbol across all exchanges""" + try: + timestamp = datetime.now() + consolidated_bids = {} + consolidated_asks = {} + active_exchanges = [] + + # Collect order book data from all connected exchanges + async with self.data_lock: + logger.debug(f"Collecting order book data for {symbol}") + for exchange_name, exchange_data in self.exchange_order_books[symbol].items(): + if exchange_data.get('connected', False): + active_exchanges.append(exchange_name) + + # Get real-time WebSocket data (top 20 levels) + live_bids = exchange_data.get('bids', {}) + live_asks = exchange_data.get('asks', {}) + + # Get deep REST API data (up to 1000 levels) + deep_bids = exchange_data.get('deep_bids', {}) + deep_asks = exchange_data.get('deep_asks', {}) + + # Merge data: prioritize live data for top levels, add deep data for others + merged_bids = self._merge_orderbook_data(live_bids, deep_bids, 'bid') + merged_asks = self._merge_orderbook_data(live_asks, deep_asks, 'ask') + + bid_count = len(merged_bids) + ask_count = len(merged_asks) + logger.debug(f"{exchange_name} data for {symbol}: {bid_count} bids ({len(live_bids)} live), {ask_count} asks ({len(live_asks)} live)") + + # Process merged bids + for price, level in merged_bids.items(): + if price not in consolidated_bids: + consolidated_bids[price] = ConsolidatedOrderBookLevel( + price=price, + total_size=0, + total_volume_usd=0, + total_orders=0, + side='bid', + exchange_breakdown={}, + dominant_exchange=exchange_name, + liquidity_score=0, + timestamp=timestamp + ) + + consolidated_bids[price].total_size += level.size + consolidated_bids[price].total_volume_usd += level.volume_usd + consolidated_bids[price].total_orders += level.orders_count + consolidated_bids[price].exchange_breakdown[exchange_name] = level + + # Update dominant exchange based on volume + if level.volume_usd > consolidated_bids[price].exchange_breakdown.get( + consolidated_bids[price].dominant_exchange, + type('obj', (object,), {'volume_usd': 0})() + ).volume_usd: + consolidated_bids[price].dominant_exchange = exchange_name + + # Process merged asks (similar logic) + for price, level in merged_asks.items(): + if price not in consolidated_asks: + consolidated_asks[price] = ConsolidatedOrderBookLevel( + price=price, + total_size=0, + total_volume_usd=0, + total_orders=0, + side='ask', + exchange_breakdown={}, + dominant_exchange=exchange_name, + liquidity_score=0, + timestamp=timestamp + ) + + consolidated_asks[price].total_size += level.size + consolidated_asks[price].total_volume_usd += level.volume_usd + consolidated_asks[price].total_orders += level.orders_count + consolidated_asks[price].exchange_breakdown[exchange_name] = level + + if level.volume_usd > consolidated_asks[price].exchange_breakdown.get( + consolidated_asks[price].dominant_exchange, + type('obj', (object,), {'volume_usd': 0})() + ).volume_usd: + consolidated_asks[price].dominant_exchange = exchange_name + + logger.debug(f"Consolidated {len(consolidated_bids)} bids and {len(consolidated_asks)} asks for {symbol}") + + # Sort and calculate consolidated metrics + sorted_bids = sorted(consolidated_bids.values(), key=lambda x: x.price, reverse=True) + sorted_asks = sorted(consolidated_asks.values(), key=lambda x: x.price) + + # Calculate consolidated metrics + volume_weighted_mid = self._calculate_volume_weighted_mid(sorted_bids, sorted_asks) + total_bid_liquidity = sum(level.total_volume_usd for level in sorted_bids) + total_ask_liquidity = sum(level.total_volume_usd for level in sorted_asks) + + spread_bps = 0 + liquidity_imbalance = 0 + + if sorted_bids and sorted_asks: + best_bid = sorted_bids[0].price + best_ask = sorted_asks[0].price + spread_bps = ((best_ask - best_bid) / volume_weighted_mid) * 10000 + + if total_bid_liquidity + total_ask_liquidity > 0: + liquidity_imbalance = (total_bid_liquidity - total_ask_liquidity) / (total_bid_liquidity + total_ask_liquidity) + + logger.debug(f"{symbol} metrics - Mid: ${volume_weighted_mid:.2f}, Spread: {spread_bps:.1f}bps, " + + f"Imbalance: {liquidity_imbalance:.2%}") + + # Generate fine-grain price buckets + price_buckets = self._generate_price_buckets(symbol, sorted_bids, sorted_asks, volume_weighted_mid) + + # Create consolidated snapshot + cob_snapshot = COBSnapshot( + symbol=symbol, + timestamp=timestamp, + consolidated_bids=sorted_bids[:50], # Top 50 levels + consolidated_asks=sorted_asks[:50], + exchanges_active=active_exchanges, + volume_weighted_mid=volume_weighted_mid, + total_bid_liquidity=total_bid_liquidity, + total_ask_liquidity=total_ask_liquidity, + spread_bps=spread_bps, + liquidity_imbalance=liquidity_imbalance, + price_buckets=price_buckets + ) + + # Store consolidated order book + self.consolidated_order_books[symbol] = cob_snapshot + self.realtime_snapshots[symbol].append(cob_snapshot) + + # Update real-time statistics + self._update_realtime_stats(symbol, cob_snapshot) + + # Update consolidation statistics + async with self.data_lock: + self.consolidation_stats[symbol]['total_updates'] += 1 + self.consolidation_stats[symbol]['active_price_levels'] = len(sorted_bids) + len(sorted_asks) + self.consolidation_stats[symbol]['total_liquidity_usd'] = total_bid_liquidity + total_ask_liquidity + + # Notify callbacks with real-time data + for callback in self.cob_update_callbacks: + try: + if asyncio.iscoroutinefunction(callback): + asyncio.create_task(callback(symbol, cob_snapshot)) + else: + callback(symbol, cob_snapshot) + except Exception as e: + logger.error(f"Error in COB update callback: {e}") + + logger.debug(f"Notified {len(self.cob_update_callbacks)} COB callbacks for {symbol}") + + logger.debug(f"Completed consolidation for {symbol} - {len(active_exchanges)} exchanges active") + + except Exception as e: + logger.error(f"Error consolidating order book for {symbol}: {e}", exc_info=True) + + def _merge_orderbook_data(self, live_data: Dict, deep_data: Dict, side: str) -> Dict: + """ + Merge live WebSocket data with deep REST API data + Strategy: Use live data for top levels (lowest latency), deep data for additional depth + """ + try: + merged = {} + + # Always prioritize live WebSocket data (top 20 levels) + for price, level in live_data.items(): + merged[price] = level + + # Add deep data that's not already covered by live data + for price, level in deep_data.items(): + if price not in merged: + # Mark this as deep data (older timestamp but more comprehensive) + level.timestamp = level.timestamp # Keep original timestamp + merged[price] = level + + # Sort to find the cutoff point for live vs deep data + if side == 'bid': + # For bids, higher prices are better (closer to mid) + sorted_prices = sorted(merged.keys(), reverse=True) + else: + # For asks, lower prices are better (closer to mid) + sorted_prices = sorted(merged.keys()) + + # Limit total depth to prevent memory issues (keep top 200 levels) + max_levels = 200 + if len(sorted_prices) > max_levels: + cutoff_price = sorted_prices[max_levels - 1] + if side == 'bid': + merged = {p: level for p, level in merged.items() if p >= cutoff_price} + else: + merged = {p: level for p, level in merged.items() if p <= cutoff_price} + + return merged + + except Exception as e: + logger.error(f"Error merging order book data: {e}") + return live_data # Fallback to live data only + + def _generate_price_buckets(self, symbol: str, bids: List[ConsolidatedOrderBookLevel], + asks: List[ConsolidatedOrderBookLevel], mid_price: float) -> Dict[str, Dict[str, float]]: + """Generate fine-grain price buckets for volume analysis""" + try: + buckets = {'bids': {}, 'asks': {}} + + # Use fixed USD bucket size if configured for this symbol + if symbol in self.fixed_usd_buckets: + bucket_size = self.fixed_usd_buckets[symbol] + logger.debug(f"Using fixed USD bucket size {bucket_size} for {symbol}") + else: + bucket_size = mid_price * (self.bucket_size_bps / 10000) # Convert bps to decimal + + # Process bids (below mid price) + for level in bids: + if level.price <= mid_price: + bucket_key = int((mid_price - level.price) / bucket_size) + bucket_price = mid_price - (bucket_key * bucket_size) + + if bucket_key not in buckets['bids']: + buckets['bids'][bucket_key] = { + 'price': bucket_price, + 'volume_usd': 0, + 'size': 0, + 'orders': 0, + 'exchanges': set() + } + + buckets['bids'][bucket_key]['volume_usd'] += level.total_volume_usd + buckets['bids'][bucket_key]['size'] += level.total_size + buckets['bids'][bucket_key]['orders'] += level.total_orders + buckets['bids'][bucket_key]['exchanges'].update(level.exchange_breakdown.keys()) + + # Process asks (above mid price) + for level in asks: + if level.price >= mid_price: + bucket_key = int((level.price - mid_price) / bucket_size) + bucket_price = mid_price + (bucket_key * bucket_size) + + if bucket_key not in buckets['asks']: + buckets['asks'][bucket_key] = { + 'price': bucket_price, + 'volume_usd': 0, + 'size': 0, + 'orders': 0, + 'exchanges': set() + } + + buckets['asks'][bucket_key]['volume_usd'] += level.total_volume_usd + buckets['asks'][bucket_key]['size'] += level.total_size + buckets['asks'][bucket_key]['orders'] += level.total_orders + buckets['asks'][bucket_key]['exchanges'].update(level.exchange_breakdown.keys()) + + # Convert sets to lists for JSON serialization + for side in ['bids', 'asks']: + for bucket_key in buckets[side]: + buckets[side][bucket_key]['exchanges'] = list(buckets[side][bucket_key]['exchanges']) + + return buckets + + except Exception as e: + logger.error(f"Error generating price buckets for {symbol}: {e}") + return {'bids': {}, 'asks': {}} + + def _calculate_volume_weighted_mid(self, bids: List[ConsolidatedOrderBookLevel], + asks: List[ConsolidatedOrderBookLevel]) -> float: + """Calculate volume-weighted mid price across all exchanges""" + if not bids or not asks: + return 0.0 + + try: + # Take top 5 levels for volume weighting + top_bids = bids[:5] + top_asks = asks[:5] + + total_bid_volume = sum(level.total_volume_usd for level in top_bids) + total_ask_volume = sum(level.total_volume_usd for level in top_asks) + + if total_bid_volume + total_ask_volume == 0: + return (bids[0].price + asks[0].price) / 2 + + weighted_bid = sum(level.price * level.total_volume_usd for level in top_bids) / total_bid_volume if total_bid_volume > 0 else bids[0].price + weighted_ask = sum(level.price * level.total_volume_usd for level in top_asks) / total_ask_volume if total_ask_volume > 0 else asks[0].price + + bid_weight = total_bid_volume / (total_bid_volume + total_ask_volume) + ask_weight = total_ask_volume / (total_bid_volume + total_ask_volume) + + return (weighted_bid * ask_weight) + (weighted_ask * bid_weight) + + except Exception as e: + logger.error(f"Error calculating volume weighted mid: {e}") + return (bids[0].price + asks[0].price) / 2 if bids and asks else 0.0 + + async def _continuous_bucket_updates(self): + """Continuously update and optimize price buckets""" + while self.is_streaming: + try: + for symbol in self.symbols: + if symbol in self.consolidated_order_books: + cob = self.consolidated_order_books[symbol] + + # Notify bucket update callbacks + for callback in self.bucket_update_callbacks: + try: + if asyncio.iscoroutinefunction(callback): + asyncio.create_task(callback(symbol, cob.price_buckets)) + else: + callback(symbol, cob.price_buckets) + except Exception as e: + logger.warning(f"Error in bucket update callback: {e}") + + await asyncio.sleep(self.bucket_update_frequency / 1000) # Convert ms to seconds + + except Exception as e: + logger.error(f"Error in bucket update loop: {e}") + await asyncio.sleep(1) + + # Public interface methods + + def subscribe_to_cob_updates(self, callback: Callable[[str, COBSnapshot], None]): + """Subscribe to consolidated order book updates""" + self.cob_update_callbacks.append(callback) + logger.info(f"Added COB update callback: {len(self.cob_update_callbacks)} total") + + def subscribe_to_bucket_updates(self, callback: Callable[[str, Dict], None]): + """Subscribe to price bucket updates""" + self.bucket_update_callbacks.append(callback) + logger.info(f"Added bucket update callback: {len(self.bucket_update_callbacks)} total") + + def get_consolidated_orderbook(self, symbol: str) -> Optional[COBSnapshot]: + """Get current consolidated order book snapshot""" + return self.consolidated_order_books.get(symbol) + + def get_price_buckets(self, symbol: str, bucket_count: int = 100) -> Optional[Dict]: + """Get fine-grain price buckets for a symbol""" + if symbol not in self.consolidated_order_books: + return None + + cob = self.consolidated_order_books[symbol] + return cob.price_buckets + + def get_exchange_breakdown(self, symbol: str) -> Optional[Dict]: + """Get breakdown of liquidity by exchange""" + if symbol not in self.consolidated_order_books: + return None + + cob = self.consolidated_order_books[symbol] + breakdown = {} + + for exchange in cob.exchanges_active: + breakdown[exchange] = { + 'bid_liquidity': 0, + 'ask_liquidity': 0, + 'total_liquidity': 0, + 'market_share': 0 + } + + # Calculate liquidity by exchange + for level in cob.consolidated_bids + cob.consolidated_asks: + for exchange, exchange_level in level.exchange_breakdown.items(): + if level.side == 'bid': + breakdown[exchange]['bid_liquidity'] += exchange_level.volume_usd + else: + breakdown[exchange]['ask_liquidity'] += exchange_level.volume_usd + breakdown[exchange]['total_liquidity'] += exchange_level.volume_usd + + # Calculate market share + total_market_liquidity = sum(data['total_liquidity'] for data in breakdown.values()) + if total_market_liquidity > 0: + for exchange in breakdown: + breakdown[exchange]['market_share'] = breakdown[exchange]['total_liquidity'] / total_market_liquidity + + return breakdown + + def get_statistics(self) -> Dict[str, Any]: + """Get provider statistics""" + return { + 'symbols': self.symbols, + 'is_streaming': self.is_streaming, + 'active_exchanges': self.active_exchanges, + 'exchange_update_counts': dict(self.exchange_update_counts), + 'consolidation_stats': dict(self.consolidation_stats), + 'bucket_size_bps': self.bucket_size_bps, + 'cob_update_callbacks': len(self.cob_update_callbacks), + 'bucket_update_callbacks': len(self.bucket_update_callbacks), + 'avg_processing_time_ms': np.mean(self.processing_times.get('consolidation', [0])) if self.processing_times.get('consolidation') else 0 + } + + def get_market_depth_analysis(self, symbol: str, depth_levels: int = 20) -> Optional[Dict]: + """Get detailed market depth analysis""" + if symbol not in self.consolidated_order_books: + return None + + cob = self.consolidated_order_books[symbol] + + # Analyze depth distribution + bid_levels = cob.consolidated_bids[:depth_levels] + ask_levels = cob.consolidated_asks[:depth_levels] + + analysis = { + 'symbol': symbol, + 'timestamp': cob.timestamp.isoformat(), + 'volume_weighted_mid': cob.volume_weighted_mid, + 'spread_bps': cob.spread_bps, + 'total_bid_liquidity': cob.total_bid_liquidity, + 'total_ask_liquidity': cob.total_ask_liquidity, + 'liquidity_imbalance': cob.liquidity_imbalance, + 'exchanges_active': cob.exchanges_active, + 'depth_analysis': { + 'bid_levels': len(bid_levels), + 'ask_levels': len(ask_levels), + 'bid_liquidity_distribution': [], + 'ask_liquidity_distribution': [], + 'dominant_exchanges': {} + } + } + + # Analyze liquidity distribution + for i, level in enumerate(bid_levels): + analysis['depth_analysis']['bid_liquidity_distribution'].append({ + 'level': i + 1, + 'price': level.price, + 'volume_usd': level.total_volume_usd, + 'size': level.total_size, + 'dominant_exchange': level.dominant_exchange, + 'exchange_count': len(level.exchange_breakdown) + }) + + for i, level in enumerate(ask_levels): + analysis['depth_analysis']['ask_liquidity_distribution'].append({ + 'level': i + 1, + 'price': level.price, + 'volume_usd': level.total_volume_usd, + 'size': level.total_size, + 'dominant_exchange': level.dominant_exchange, + 'exchange_count': len(level.exchange_breakdown) + }) + + # Count dominant exchanges + for level in bid_levels + ask_levels: + exchange = level.dominant_exchange + if exchange not in analysis['depth_analysis']['dominant_exchanges']: + analysis['depth_analysis']['dominant_exchanges'][exchange] = 0 + analysis['depth_analysis']['dominant_exchanges'][exchange] += 1 + + return analysis + + def _update_realtime_stats(self, symbol: str, cob_snapshot: COBSnapshot): + """Update real-time statistics for 1s and 5s windows""" + try: + current_time = datetime.now() + + # Add to history + self.realtime_snapshots[symbol].append(cob_snapshot) + + # Calculate 1s and 5s windows + window_1s = current_time - timedelta(seconds=1) + window_5s = current_time - timedelta(seconds=5) + + # Get data within windows + data_1s = [snapshot for snapshot in self.realtime_snapshots[symbol] + if snapshot.timestamp >= window_1s] + data_5s = [snapshot for snapshot in self.realtime_snapshots[symbol] + if snapshot.timestamp >= window_5s] + + # Update 1s stats + if data_1s: + self.realtime_stats[symbol]['1s_stats'] = self._calculate_window_stats(data_1s) + + # Update 5s stats + if data_5s: + self.realtime_stats[symbol]['5s_stats'] = self._calculate_window_stats(data_5s) + + except Exception as e: + logger.error(f"Error updating real-time stats for {symbol}: {e}") + + def _calculate_window_stats(self, snapshots: List[COBSnapshot]) -> Dict: + """Calculate statistics for a time window""" + if not snapshots: + return {} + + mid_prices = [s.volume_weighted_mid for s in snapshots] + spreads = [s.spread_bps for s in snapshots] + bid_liquidity = [s.total_bid_liquidity for s in snapshots] + ask_liquidity = [s.total_ask_liquidity for s in snapshots] + imbalances = [s.liquidity_imbalance for s in snapshots] + + return { + 'max_mid_price': max(mid_prices), + 'min_mid_price': min(mid_prices), + 'avg_mid_price': sum(mid_prices) / len(mid_prices), + 'max_spread_bps': max(spreads), + 'avg_spread_bps': sum(spreads) / len(spreads), + 'max_bid_liquidity': max(bid_liquidity), + 'avg_bid_liquidity': sum(bid_liquidity) / len(bid_liquidity), + 'max_ask_liquidity': max(ask_liquidity), + 'avg_ask_liquidity': sum(ask_liquidity) / len(ask_liquidity), + 'max_imbalance': max(imbalances), + 'avg_imbalance': sum(imbalances) / len(imbalances), + 'update_count': len(snapshots) + } + + def get_realtime_stats(self, symbol: str) -> Dict: + """Get current real-time statistics for a symbol""" + try: + return self.realtime_stats.get(symbol, {}) + except Exception as e: + logger.error(f"Error getting real-time stats for {symbol}: {e}") + return {} \ No newline at end of file diff --git a/run_cob_dashboard.py b/run_cob_dashboard.py new file mode 100644 index 0000000..adfb0ab --- /dev/null +++ b/run_cob_dashboard.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +""" +Simple runner for COB Dashboard +""" + +import asyncio +import logging +import sys + +# Add the project root to the path +sys.path.insert(0, '.') + +from web.cob_realtime_dashboard import main + +if __name__ == "__main__": + # Set up logging + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler('cob_dashboard.log') + ] + ) + + logger = logging.getLogger(__name__) + logger.info("Starting COB Dashboard...") + + try: + asyncio.run(main()) + except KeyboardInterrupt: + logger.info("COB Dashboard stopped by user") + except Exception as e: + logger.error(f"COB Dashboard failed: {e}", exc_info=True) + sys.exit(1) \ No newline at end of file diff --git a/simple_cob_dashboard.py b/simple_cob_dashboard.py new file mode 100644 index 0000000..d5b89ab --- /dev/null +++ b/simple_cob_dashboard.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +""" +Simple Windows-compatible COB Dashboard +""" + +import asyncio +import json +import logging +import time +from datetime import datetime +from http.server import HTTPServer, SimpleHTTPRequestHandler +from socketserver import ThreadingMixIn +import threading +import webbrowser +from urllib.parse import urlparse, parse_qs + +from core.multi_exchange_cob_provider import MultiExchangeCOBProvider + +logger = logging.getLogger(__name__) + +class COBHandler(SimpleHTTPRequestHandler): + """HTTP handler for COB dashboard""" + + def __init__(self, *args, cob_provider=None, **kwargs): + self.cob_provider = cob_provider + super().__init__(*args, **kwargs) + + def do_GET(self): + """Handle GET requests""" + path = urlparse(self.path).path + + if path == '/': + self.serve_dashboard() + elif path.startswith('/api/cob/'): + self.serve_cob_data() + elif path == '/api/status': + self.serve_status() + else: + super().do_GET() + + def serve_dashboard(self): + """Serve the dashboard HTML""" + html_content = """ + + + + COB Dashboard + + + +
+

Consolidated Order Book Dashboard

+
Hybrid WebSocket + REST API | Real-time + Deep Market Data
+
+ +
+ + +
+ +
Loading...
+ +
+
+

Market Analysis

+
+

Chart data will be displayed here

+
Current implementation shows:
+
    +
  • āœ“ Real-time order book data (WebSocket)
    • +
  • āœ“ Deep market data (REST API)
    • +
  • āœ“ Session Volume Profile
    • +
  • āœ“ Hybrid data merging
    • +
+
+
+ +
+

Order Book Ladder

+ +
+
Price
+
Size
+
Total
+
+ +
+ +
$--
+ +
+
+
+ +
+
+
Total Liquidity
+
--
+
+
+
Book Depth
+
--
+
+
+
Spread
+
-- bps
+
+
+ + + + + """ + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(html_content.encode()) + + def serve_cob_data(self): + """Serve COB data""" + try: + # Extract symbol from path + symbol = self.path.split('/')[-1].replace('%2F', '/') + + if not self.cob_provider: + data = self.get_mock_data(symbol) + else: + data = self.get_real_data(symbol) + + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.send_header('Access-Control-Allow-Origin', '*') + self.end_headers() + self.wfile.write(json.dumps(data).encode()) + + except Exception as e: + logger.error(f"Error serving COB data: {e}") + self.send_error(500, str(e)) + + def serve_status(self): + """Serve status""" + status = { + 'server': 'running', + 'timestamp': datetime.now().isoformat(), + 'cob_provider': 'active' if self.cob_provider else 'mock' + } + + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.send_header('Access-Control-Allow-Origin', '*') + self.end_headers() + self.wfile.write(json.dumps(status).encode()) + + def get_real_data(self, symbol): + """Get real data from COB provider""" + try: + cob_snapshot = self.cob_provider.get_consolidated_orderbook(symbol) + if not cob_snapshot: + return self.get_mock_data(symbol) + + # Convert to dashboard format + bids = [] + asks = [] + + for level in cob_snapshot.consolidated_bids[:20]: + bids.append({ + 'price': level.price, + 'size': level.total_size, + 'volume': level.total_volume_usd + }) + + for level in cob_snapshot.consolidated_asks[:20]: + asks.append({ + 'price': level.price, + 'size': level.total_size, + 'volume': level.total_volume_usd + }) + + return { + 'symbol': symbol, + 'bids': bids, + 'asks': asks, + 'stats': { + 'mid_price': cob_snapshot.volume_weighted_mid, + 'spread_bps': cob_snapshot.spread_bps, + 'bid_liquidity': cob_snapshot.total_bid_liquidity, + 'ask_liquidity': cob_snapshot.total_ask_liquidity, + 'bid_levels': len(cob_snapshot.consolidated_bids), + 'ask_levels': len(cob_snapshot.consolidated_asks), + 'imbalance': cob_snapshot.liquidity_imbalance + } + } + + except Exception as e: + logger.error(f"Error getting real data: {e}") + return self.get_mock_data(symbol) + + def get_mock_data(self, symbol): + """Get mock data for testing""" + base_price = 50000 if 'BTC' in symbol else 3000 + + bids = [] + asks = [] + + # Generate mock bids + for i in range(20): + price = base_price - (i * 10) + size = 1.0 + (i * 0.1) + bids.append({ + 'price': price, + 'size': size, + 'volume': price * size + }) + + # Generate mock asks + for i in range(20): + price = base_price + 10 + (i * 10) + size = 1.0 + (i * 0.1) + asks.append({ + 'price': price, + 'size': size, + 'volume': price * size + }) + + return { + 'symbol': symbol, + 'bids': bids, + 'asks': asks, + 'stats': { + 'mid_price': base_price + 5, + 'spread_bps': 2.5, + 'bid_liquidity': sum(b['volume'] for b in bids), + 'ask_liquidity': sum(a['volume'] for a in asks), + 'bid_levels': len(bids), + 'ask_levels': len(asks), + 'imbalance': 0.1 + } + } + + +class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): + """Thread pool server""" + allow_reuse_address = True + + +def start_cob_dashboard(): + """Start the COB dashboard""" + print("Starting Simple COB Dashboard...") + + # Initialize COB provider + cob_provider = None + try: + print("Initializing COB provider...") + cob_provider = MultiExchangeCOBProvider(symbols=['BTC/USDT', 'ETH/USDT']) + + # Start in background thread + def run_provider(): + asyncio.run(cob_provider.start_streaming()) + + provider_thread = threading.Thread(target=run_provider, daemon=True) + provider_thread.start() + + time.sleep(2) # Give it time to connect + print("COB provider started") + + except Exception as e: + print(f"Warning: COB provider failed to start: {e}") + print("Running in mock mode...") + + # Start HTTP server + def handler(*args, **kwargs): + COBHandler(*args, cob_provider=cob_provider, **kwargs) + + port = 8053 + server = ThreadedHTTPServer(('localhost', port), handler) + + print(f"COB Dashboard running at http://localhost:{port}") + print("Press Ctrl+C to stop") + + # Open browser + try: + webbrowser.open(f'http://localhost:{port}') + except: + pass + + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nStopping dashboard...") + server.shutdown() + if cob_provider: + asyncio.run(cob_provider.stop_streaming()) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + start_cob_dashboard() \ No newline at end of file diff --git a/test_enhanced_order_flow_integration.py b/test_enhanced_order_flow_integration.py new file mode 100644 index 0000000..2d197b2 --- /dev/null +++ b/test_enhanced_order_flow_integration.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +""" +Test Enhanced Order Flow Integration + +Tests the enhanced order flow analysis capabilities including: +- Aggressive vs passive participant ratios +- Institutional vs retail trade detection +- Market maker vs taker flow analysis +- Order flow intensity measurements +- Liquidity consumption and price impact analysis +- Block trade and iceberg order detection +- High-frequency trading activity detection + +Usage: + python test_enhanced_order_flow_integration.py +""" + +import asyncio +import logging +import time +import json +from datetime import datetime, timedelta +from core.bookmap_integration import BookmapIntegration + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(), + logging.FileHandler('enhanced_order_flow_test.log') + ] +) +logger = logging.getLogger(__name__) + +class EnhancedOrderFlowTester: + """Test enhanced order flow analysis features""" + + def __init__(self): + self.bookmap = None + self.symbols = ['ETHUSDT', 'BTCUSDT'] + self.test_duration = 300 # 5 minutes + self.metrics_history = [] + + async def setup_integration(self): + """Initialize the Bookmap integration""" + try: + logger.info("Setting up Enhanced Order Flow Integration...") + self.bookmap = BookmapIntegration(symbols=self.symbols) + + # Add callbacks for testing + self.bookmap.add_cnn_callback(self._cnn_callback) + self.bookmap.add_dqn_callback(self._dqn_callback) + + logger.info(f"Integration setup complete for symbols: {self.symbols}") + return True + + except Exception as e: + logger.error(f"Failed to setup integration: {e}") + return False + + def _cnn_callback(self, symbol: str, features: dict): + """CNN callback for testing""" + logger.debug(f"CNN features received for {symbol}: {len(features.get('features', []))} dimensions") + + def _dqn_callback(self, symbol: str, state: dict): + """DQN callback for testing""" + logger.debug(f"DQN state received for {symbol}: {len(state.get('state', []))} dimensions") + + async def start_streaming(self): + """Start real-time data streaming""" + try: + logger.info("Starting enhanced order flow streaming...") + await self.bookmap.start_streaming() + logger.info("Streaming started successfully") + return True + + except Exception as e: + logger.error(f"Failed to start streaming: {e}") + return False + + async def monitor_order_flow(self): + """Monitor and analyze order flow for test duration""" + logger.info(f"Monitoring enhanced order flow for {self.test_duration} seconds...") + + start_time = time.time() + iteration = 0 + + while time.time() - start_time < self.test_duration: + try: + iteration += 1 + + # Test each symbol + for symbol in self.symbols: + await self._analyze_symbol_flow(symbol, iteration) + + # Wait 10 seconds between analyses + await asyncio.sleep(10) + + except Exception as e: + logger.error(f"Error during monitoring iteration {iteration}: {e}") + await asyncio.sleep(5) + + logger.info("Order flow monitoring completed") + + async def _analyze_symbol_flow(self, symbol: str, iteration: int): + """Analyze order flow for a specific symbol""" + try: + # Get enhanced order flow metrics + flow_metrics = self.bookmap.get_enhanced_order_flow_metrics(symbol) + if not flow_metrics: + logger.warning(f"No flow metrics available for {symbol}") + return + + # Log key metrics + aggressive_passive = flow_metrics['aggressive_passive'] + institutional_retail = flow_metrics['institutional_retail'] + flow_intensity = flow_metrics['flow_intensity'] + price_impact = flow_metrics['price_impact'] + maker_taker = flow_metrics['maker_taker_flow'] + + logger.info(f"\n=== {symbol} Order Flow Analysis (Iteration {iteration}) ===") + logger.info(f"Aggressive Ratio: {aggressive_passive['aggressive_ratio']:.2%}") + logger.info(f"Passive Ratio: {aggressive_passive['passive_ratio']:.2%}") + logger.info(f"Institutional Ratio: {institutional_retail['institutional_ratio']:.2%}") + logger.info(f"Retail Ratio: {institutional_retail['retail_ratio']:.2%}") + logger.info(f"Flow Intensity: {flow_intensity['current_intensity']:.2f} ({flow_intensity['intensity_category']})") + logger.info(f"Price Impact: {price_impact['avg_impact']:.2f} bps ({price_impact['impact_category']})") + logger.info(f"Buy Pressure: {maker_taker['buy_pressure']:.2%}") + logger.info(f"Sell Pressure: {maker_taker['sell_pressure']:.2%}") + + # Trade size analysis + size_dist = flow_metrics['size_distribution'] + total_trades = sum(size_dist.values()) + if total_trades > 0: + logger.info(f"Trade Size Distribution (last 100 trades):") + logger.info(f" Micro (<$1K): {size_dist.get('micro', 0)} ({size_dist.get('micro', 0)/total_trades:.1%})") + logger.info(f" Small ($1K-$10K): {size_dist.get('small', 0)} ({size_dist.get('small', 0)/total_trades:.1%})") + logger.info(f" Medium ($10K-$50K): {size_dist.get('medium', 0)} ({size_dist.get('medium', 0)/total_trades:.1%})") + logger.info(f" Large ($50K-$100K): {size_dist.get('large', 0)} ({size_dist.get('large', 0)/total_trades:.1%})") + logger.info(f" Block (>$100K): {size_dist.get('block', 0)} ({size_dist.get('block', 0)/total_trades:.1%})") + + # Volume analysis + if 'volume_stats' in flow_metrics and flow_metrics['volume_stats']: + volume_stats = flow_metrics['volume_stats'] + logger.info(f"24h Volume: {volume_stats.get('volume_24h', 0):,.0f}") + logger.info(f"24h Quote Volume: ${volume_stats.get('quote_volume_24h', 0):,.0f}") + + # Store metrics for analysis + self.metrics_history.append({ + 'timestamp': datetime.now(), + 'symbol': symbol, + 'iteration': iteration, + 'metrics': flow_metrics + }) + + # Test CNN and DQN features + await self._test_model_features(symbol) + + except Exception as e: + logger.error(f"Error analyzing flow for {symbol}: {e}") + + async def _test_model_features(self, symbol: str): + """Test CNN and DQN feature extraction""" + try: + # Test CNN features + cnn_features = self.bookmap.get_cnn_features(symbol) + if cnn_features is not None: + logger.info(f"CNN Features: {len(cnn_features)} dimensions") + logger.info(f" Order book features: {cnn_features[:80].mean():.4f} (avg)") + logger.info(f" Liquidity metrics: {cnn_features[80:90].mean():.4f} (avg)") + logger.info(f" Imbalance features: {cnn_features[90:95].mean():.4f} (avg)") + logger.info(f" Enhanced flow features: {cnn_features[95:].mean():.4f} (avg)") + + # Test DQN features + dqn_features = self.bookmap.get_dqn_state_features(symbol) + if dqn_features is not None: + logger.info(f"DQN State: {len(dqn_features)} dimensions") + logger.info(f" Order book state: {dqn_features[:20].mean():.4f} (avg)") + logger.info(f" Market indicators: {dqn_features[20:30].mean():.4f} (avg)") + logger.info(f" Enhanced flow state: {dqn_features[30:].mean():.4f} (avg)") + + # Test dashboard data + dashboard_data = self.bookmap.get_dashboard_data(symbol) + if dashboard_data and 'enhanced_order_flow' in dashboard_data: + logger.info("Dashboard data includes enhanced order flow metrics") + + except Exception as e: + logger.error(f"Error testing model features for {symbol}: {e}") + + async def stop_streaming(self): + """Stop data streaming""" + try: + logger.info("Stopping order flow streaming...") + await self.bookmap.stop_streaming() + logger.info("Streaming stopped") + + except Exception as e: + logger.error(f"Error stopping streaming: {e}") + + def generate_summary_report(self): + """Generate a summary report of the test""" + try: + logger.info("\n" + "="*60) + logger.info("ENHANCED ORDER FLOW ANALYSIS SUMMARY") + logger.info("="*60) + + if not self.metrics_history: + logger.warning("No metrics data collected during test") + return + + # Group by symbol + symbol_data = {} + for entry in self.metrics_history: + symbol = entry['symbol'] + if symbol not in symbol_data: + symbol_data[symbol] = [] + symbol_data[symbol].append(entry) + + # Analyze each symbol + for symbol, data in symbol_data.items(): + logger.info(f"\n--- {symbol} Analysis ---") + logger.info(f"Data points collected: {len(data)}") + + if len(data) > 0: + # Calculate averages + avg_aggressive = sum(d['metrics']['aggressive_passive']['aggressive_ratio'] for d in data) / len(data) + avg_institutional = sum(d['metrics']['institutional_retail']['institutional_ratio'] for d in data) / len(data) + avg_intensity = sum(d['metrics']['flow_intensity']['current_intensity'] for d in data) / len(data) + avg_impact = sum(d['metrics']['price_impact']['avg_impact'] for d in data) / len(data) + + logger.info(f"Average Aggressive Ratio: {avg_aggressive:.2%}") + logger.info(f"Average Institutional Ratio: {avg_institutional:.2%}") + logger.info(f"Average Flow Intensity: {avg_intensity:.2f}") + logger.info(f"Average Price Impact: {avg_impact:.2f} bps") + + # Detect trends + first_half = data[:len(data)//2] if len(data) > 1 else data + second_half = data[len(data)//2:] if len(data) > 1 else data + + if len(first_half) > 0 and len(second_half) > 0: + first_aggressive = sum(d['metrics']['aggressive_passive']['aggressive_ratio'] for d in first_half) / len(first_half) + second_aggressive = sum(d['metrics']['aggressive_passive']['aggressive_ratio'] for d in second_half) / len(second_half) + + trend = "increasing" if second_aggressive > first_aggressive else "decreasing" + logger.info(f"Aggressive trading trend: {trend}") + + logger.info("\n" + "="*60) + logger.info("Test completed successfully!") + logger.info("Enhanced order flow analysis is working correctly.") + logger.info("="*60) + + except Exception as e: + logger.error(f"Error generating summary report: {e}") + +async def run_enhanced_order_flow_test(): + """Run the complete enhanced order flow test""" + tester = EnhancedOrderFlowTester() + + try: + # Setup + logger.info("Starting Enhanced Order Flow Integration Test") + logger.info("This test will demonstrate:") + logger.info("- Aggressive vs Passive participant analysis") + logger.info("- Institutional vs Retail trade detection") + logger.info("- Order flow intensity measurements") + logger.info("- Price impact and liquidity consumption analysis") + logger.info("- Block trade and iceberg order detection") + logger.info("- Enhanced CNN and DQN feature extraction") + + if not await tester.setup_integration(): + logger.error("Failed to setup integration") + return False + + # Start streaming + if not await tester.start_streaming(): + logger.error("Failed to start streaming") + return False + + # Wait for initial data + logger.info("Waiting 30 seconds for initial data...") + await asyncio.sleep(30) + + # Monitor order flow + await tester.monitor_order_flow() + + # Generate report + tester.generate_summary_report() + + return True + + except Exception as e: + logger.error(f"Test failed: {e}") + return False + + finally: + # Cleanup + try: + await tester.stop_streaming() + except Exception as e: + logger.error(f"Error during cleanup: {e}") + +if __name__ == "__main__": + try: + # Run the test + success = asyncio.run(run_enhanced_order_flow_test()) + + if success: + print("\nāœ… Enhanced Order Flow Integration Test PASSED") + print("All enhanced order flow analysis features are working correctly!") + else: + print("\nāŒ Enhanced Order Flow Integration Test FAILED") + print("Check the logs for details.") + + except KeyboardInterrupt: + print("\nāš ļø Test interrupted by user") + except Exception as e: + print(f"\nšŸ’„ Test crashed: {e}") \ No newline at end of file diff --git a/test_free_orderbook_integration.py b/test_free_orderbook_integration.py new file mode 100644 index 0000000000000000000000000000000000000000..abb90630dcc1ef495fa6470eb94b87716fdb2913 GIT binary patch literal 15216 zcmc(m-Etg9703IA8>%Q?VIq@~49Sv^gbPb7Ok>(wSJv}|pZ*Uv_m%204>{-~+i2ZOBsR941hym&hBhV{hIuy=IcMtXdRQI5H zPh*VwHNlvQ!l~f#WD=>}zL)`)-FVNaa1LMT-ck5i@J_;6q!Fm65sq`at^06$CTL*U ziTq9C^^0&La`dsDZAJ_Sg2qha|4?{cs?SBcN!54+CeMNS;sVMd@~py&}irT-_1mN|<-UpTyOEyqbg$BDQ4B zKtSF{QJPS@t$Am{wxj!padi0{?qBGWBws#v{WT#^MF*I$1+Y}YauhLOi-&sVwuju@ z3QFi^o9X#Sn&V?lRwsfD?gVou^0pgB!L7(FcN}?WN)>765-IKdPOm1eiJo=gn?}EV zA&${N6>X-JS}8|pLnix@dp>_9@=^%`AL794Xc4P5h$HFyfg;kw@KG^iFY*q*Ct)=# zhYw}-;4z!U&#p&JVC6hZ7_iaibM6qawq*e}ce#e~rJhY?g+Myi$kp()peXT@+tEnA zNYo+_eR6fBmw4UC1)BKhd8ESH4`%W>$>;xD`B+OinG z8g)_9k33GoJz+x6j9Up~eO!)Q8!T`T2hjswkfMuQTDy&H2rALfqNY=6zU)Bn?k3;qE1ihWBN*0;2v_*n;dBaIg-|7Wtb;?V-X;N~m zI53SeB3l^+x21o$ww&s7{5zMl-@B%nH|1GGc$7!(Sx9i6FO2s*936&A2a`%70p2J~?OZ6ZMhi znNHs*#_<#BeOdT&JD=%JyT0~gOstQlqOf}BP`p;67|rc=#7C#DPxsFE?HXN4GH5+n z!W-ej63C8Iaa5=9dU&JfXoCzt)m&SXAiZ`@hfv9wl-;U}`U@&cc^Y-~yH7>+r{ROU7v!)$kGZ1{iJZd;vSmq54e;!i|gs!K9jo7JW<{q~y1UA?6`ms=UV z?!Emsny7kz)>%^3W|N5utnrIW*dleFt*FqFRZUhAjN@-|d1-%>m`~F>bS$qVZ=6Pb z&!R6|<}}@ZF(-a*M$ciT^fKBDRDd-_iei#fcPr|`(!Rn;(Q%|)(q%=ip$cZF~13kK(DnG14fyP3X4@ztEqL$A=H}Yg^B9Utz^v_uRUFU7qT16+vu`?fUGZ z^-Iy(GFZ7wMYkgN&XR$7%V>-F$nWIU#J(&}MyW_=%AX|!#IsZCF;Cf;qSbX~vXYWhp8 z(&y!>cKfDVM82Z7AflC#A=h8BAIrw{e)_$`y{La}H_wmT)>1Fb-!G^PFV_n zkOZg`h_J(IhEkvC0gt6eK#yuupIchzJr=wz{lBR{`dM9>ryJU3>le?D`6|%%5%TDp zKc%kBHSSjC-(56fzsPU({O|v?ZbQEA^a_a{5g588;8^-7AUdtdTCGoB#Ry3rq!ZwM zUE>k%`ydfTXUb*FvdBX%js>yr#d^xE*ZM!rT>|e8*t{-S)b&XkrFS{U{%yU3KFa#_ z<8b=)#F+J3revRyE({T@3O|eWdAD@v&r)wtyRfrmGnch4x}oamSMq}9%G@cdf6t?I zD`Fr%O$O6OwzhqQi6}LyKEG0`*e_*$bWakq-56VurS3|f4w2vU2!SrrGez91lh;jI z8#~XJrwqN*L0*)yK3{XKUlV1Uk{OofHQK&bgqFejD+z~zW_>=@SUx{h&|Th$?E4_V z^$xRJc(xYQLh#A{geeGiCdq<2vLQv1h;KYSES<IBDfWt-Dx$?1IF~iq-6nz8BAvT~Oy$ zG0x{a3tK(^o2r?*G_BhM)rt1`HO)|*jC-PWAm=wc7kBfC#n+#{jy38 zI{NDaspC}1&i|lJFu5hK9ws|Z5K-?Ph{EoeFt1)+M7Q+WsAcdhj}Tp6X7MahNXO6T zT70`o%GOj9p6Ton_ReaOOzr*WA(TFk^NA8Re;ldBPY&OsEkbRIQS^l90Xf~%K{Bp; zP_uW&`8nWPRfJc48neaAcaGKNIWg++bl#7>eyfb~b7F37^>5BG_=&TzuFIK=M(h$A zxi8VxguZy}F5%ay9ab?|5ByZ|#ZMm?m;JEus&YgV4&F7SX5oC_nPLYW6ki7nrJ-%^ zP`pEEo0r|bJpae{KmDQ78G>Xtwo<3>mSlRhxeW`q^0plWLS2Wh-j=CwlCsxo5tv*p&Oq?ys=q}WxXL@V(Yjz zU&QX{yBluol8i>SvC7Zw37Uq*t3|BkL4ny;&2`LP0easbe`8l3JnW5j?~Xd{oW)u6JY)Or1Gw8dtN7cY=&@C&dB5+d z*Y_yq*-5^)!up$&-S7@xx_D@NM<1&sd4IunQ$Dz;?$B|YZmA>S0E)^OU|J-%^4#`I z{Jcf^enfMx!m=H5VK0GCZey$GA4+?-UvGT9)`QR;w<=M-TkfZ zv-5CA_c8P;}&E4{S5Qu9rnb&UYuzPg?KTzdyp5579RR&w4hBLO< zPcAbksjo)*-sXXV?zd0OG7~Ln?T1cmm!gjSp=1Py?1l8bqXOocG-+oTX7OzGwm3u1 z$5B3h@{hBcXx~l*Qf0rY?GM6rSwUjGX|2$F*Cl`Z$toZlgD19C3Uw|wMK_f}{+da^=w9~9CCiW$U8Yz`11U(-{-T< z^UM9B{CVjM$Iu)yw$`t|ZJ&f2tL?azQ%?S^!0k?nFh!TIy=Zj9@SH|$xkFEjK#lyoi+nX!7YGdEbOVd_4Y zo6F-}&n}c)rIQKuGYfkeh1MEdvp6$l`J3ydT88iQl2j|$i7~5|IHi^9h0?Q2J>3<>mpKzt rRt~l!e^)*1Wb0m6u}6X}^`C$#sk*iB$(NiS&IsHq{4VLRX)*r=-3j7t literal 0 HcmV?d00001 diff --git a/test_multi_exchange_cob.py b/test_multi_exchange_cob.py new file mode 100644 index 0000000..0c035da --- /dev/null +++ b/test_multi_exchange_cob.py @@ -0,0 +1,327 @@ +""" +Test Multi-Exchange Consolidated Order Book (COB) Provider + +This script demonstrates the functionality of the new multi-exchange COB data provider: +1. Real-time order book aggregation from multiple exchanges +2. Fine-grain price bucket generation +3. CNN/DQN feature generation +4. Dashboard integration +5. Market analysis and signal generation + +Run this to test the COB provider with live data streams. +""" + +import asyncio +import logging +import time +from datetime import datetime +from core.multi_exchange_cob_provider import MultiExchangeCOBProvider +from core.cob_integration import COBIntegration +from core.data_provider import DataProvider + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +class COBTester: + """Test harness for Multi-Exchange COB Provider""" + + def __init__(self): + self.symbols = ['BTC/USDT', 'ETH/USDT'] + self.data_provider = None + self.cob_integration = None + self.test_duration = 300 # 5 minutes + + # Statistics tracking + self.stats = { + 'cob_updates_received': 0, + 'bucket_updates_received': 0, + 'cnn_features_generated': 0, + 'dqn_features_generated': 0, + 'signals_generated': 0, + 'start_time': None + } + + async def run_test(self): + """Run comprehensive COB provider test""" + logger.info("Starting Multi-Exchange COB Provider Test") + logger.info(f"Testing symbols: {self.symbols}") + logger.info(f"Test duration: {self.test_duration} seconds") + + try: + # Initialize components + await self._initialize_components() + + # Run test scenarios + await self._run_basic_functionality_test() + await self._run_feature_generation_test() + await self._run_dashboard_integration_test() + await self._run_signal_analysis_test() + + # Monitor for specified duration + await self._monitor_live_data() + + # Generate final report + self._generate_test_report() + + except Exception as e: + logger.error(f"Test failed: {e}") + finally: + await self._cleanup() + + async def _initialize_components(self): + """Initialize COB provider and integration components""" + logger.info("Initializing COB components...") + + # Create data provider (optional - for integration testing) + self.data_provider = DataProvider(symbols=self.symbols) + + # Create COB integration + self.cob_integration = COBIntegration( + data_provider=self.data_provider, + symbols=self.symbols + ) + + # Register test callbacks + self.cob_integration.add_cnn_callback(self._cnn_callback) + self.cob_integration.add_dqn_callback(self._dqn_callback) + self.cob_integration.add_dashboard_callback(self._dashboard_callback) + + # Start COB integration + await self.cob_integration.start() + + # Allow time for connections + await asyncio.sleep(5) + + self.stats['start_time'] = datetime.now() + logger.info("COB components initialized successfully") + + async def _run_basic_functionality_test(self): + """Test basic COB provider functionality""" + logger.info("Testing basic COB functionality...") + + # Wait for order book data + await asyncio.sleep(10) + + for symbol in self.symbols: + # Test consolidated order book retrieval + cob_snapshot = self.cob_integration.get_cob_snapshot(symbol) + if cob_snapshot: + logger.info(f"{symbol} COB Status:") + logger.info(f" Exchanges active: {cob_snapshot.exchanges_active}") + logger.info(f" Volume weighted mid: ${cob_snapshot.volume_weighted_mid:.2f}") + logger.info(f" Spread: {cob_snapshot.spread_bps:.2f} bps") + logger.info(f" Bid liquidity: ${cob_snapshot.total_bid_liquidity:,.0f}") + logger.info(f" Ask liquidity: ${cob_snapshot.total_ask_liquidity:,.0f}") + logger.info(f" Liquidity imbalance: {cob_snapshot.liquidity_imbalance:.3f}") + + # Test price buckets + price_buckets = self.cob_integration.get_price_buckets(symbol) + if price_buckets: + bid_buckets = len(price_buckets.get('bids', {})) + ask_buckets = len(price_buckets.get('asks', {})) + logger.info(f" Price buckets: {bid_buckets} bids, {ask_buckets} asks") + + # Test exchange breakdown + exchange_breakdown = self.cob_integration.get_exchange_breakdown(symbol) + if exchange_breakdown: + logger.info(f" Exchange breakdown:") + for exchange, data in exchange_breakdown.items(): + market_share = data.get('market_share', 0) * 100 + logger.info(f" {exchange}: {market_share:.1f}% market share") + else: + logger.warning(f"No COB data available for {symbol}") + + logger.info("Basic functionality test completed") + + async def _run_feature_generation_test(self): + """Test CNN and DQN feature generation""" + logger.info("Testing feature generation...") + + for symbol in self.symbols: + # Test CNN features + cnn_features = self.cob_integration.get_cob_features(symbol) + if cnn_features is not None: + logger.info(f"{symbol} CNN features: shape={cnn_features.shape}, " + f"min={cnn_features.min():.4f}, max={cnn_features.max():.4f}") + else: + logger.warning(f"No CNN features available for {symbol}") + + # Test market depth analysis + depth_analysis = self.cob_integration.get_market_depth_analysis(symbol) + if depth_analysis: + logger.info(f"{symbol} Market Depth Analysis:") + logger.info(f" Depth levels: {depth_analysis['depth_analysis']['bid_levels']} bids, " + f"{depth_analysis['depth_analysis']['ask_levels']} asks") + + dominant_exchanges = depth_analysis['depth_analysis'].get('dominant_exchanges', {}) + logger.info(f" Dominant exchanges: {dominant_exchanges}") + + logger.info("Feature generation test completed") + + async def _run_dashboard_integration_test(self): + """Test dashboard data generation""" + logger.info("Testing dashboard integration...") + + # Dashboard integration is tested via callbacks + # Statistics are tracked in the callback functions + await asyncio.sleep(5) + + logger.info("Dashboard integration test completed") + + async def _run_signal_analysis_test(self): + """Test signal generation and analysis""" + logger.info("Testing signal analysis...") + + for symbol in self.symbols: + # Get recent signals + recent_signals = self.cob_integration.get_recent_signals(symbol, count=10) + logger.info(f"{symbol} recent signals: {len(recent_signals)} generated") + + for signal in recent_signals[-3:]: # Show last 3 signals + logger.info(f" Signal: {signal.get('type')} - {signal.get('side')} - " + f"Confidence: {signal.get('confidence', 0):.3f}") + + logger.info("Signal analysis test completed") + + async def _monitor_live_data(self): + """Monitor live data for the specified duration""" + logger.info(f"Monitoring live data for {self.test_duration} seconds...") + + start_time = time.time() + last_stats_time = start_time + + while time.time() - start_time < self.test_duration: + # Print periodic statistics + current_time = time.time() + if current_time - last_stats_time >= 30: # Every 30 seconds + self._print_periodic_stats() + last_stats_time = current_time + + await asyncio.sleep(1) + + logger.info("Live data monitoring completed") + + def _print_periodic_stats(self): + """Print periodic statistics during monitoring""" + elapsed = (datetime.now() - self.stats['start_time']).total_seconds() + + logger.info("Periodic Statistics:") + logger.info(f" Elapsed time: {elapsed:.0f} seconds") + logger.info(f" COB updates: {self.stats['cob_updates_received']}") + logger.info(f" Bucket updates: {self.stats['bucket_updates_received']}") + logger.info(f" CNN features: {self.stats['cnn_features_generated']}") + logger.info(f" DQN features: {self.stats['dqn_features_generated']}") + logger.info(f" Signals: {self.stats['signals_generated']}") + + # Calculate rates + if elapsed > 0: + cob_rate = self.stats['cob_updates_received'] / elapsed + logger.info(f" COB update rate: {cob_rate:.2f}/sec") + + def _generate_test_report(self): + """Generate final test report""" + elapsed = (datetime.now() - self.stats['start_time']).total_seconds() + + logger.info("=" * 60) + logger.info("MULTI-EXCHANGE COB PROVIDER TEST REPORT") + logger.info("=" * 60) + logger.info(f"Test Duration: {elapsed:.0f} seconds") + logger.info(f"Symbols Tested: {', '.join(self.symbols)}") + logger.info("") + + # Data Reception Statistics + logger.info("Data Reception:") + logger.info(f" COB Updates Received: {self.stats['cob_updates_received']}") + logger.info(f" Bucket Updates Received: {self.stats['bucket_updates_received']}") + logger.info(f" Average COB Rate: {self.stats['cob_updates_received'] / elapsed:.2f}/sec") + logger.info("") + + # Feature Generation Statistics + logger.info("Feature Generation:") + logger.info(f" CNN Features Generated: {self.stats['cnn_features_generated']}") + logger.info(f" DQN Features Generated: {self.stats['dqn_features_generated']}") + logger.info("") + + # Signal Generation Statistics + logger.info("Signal Analysis:") + logger.info(f" Signals Generated: {self.stats['signals_generated']}") + logger.info("") + + # Component Statistics + cob_stats = self.cob_integration.get_statistics() + logger.info("Component Statistics:") + logger.info(f" Active Exchanges: {', '.join(cob_stats.get('active_exchanges', []))}") + logger.info(f" Streaming Status: {cob_stats.get('is_streaming', False)}") + logger.info(f" Bucket Size: {cob_stats.get('bucket_size_bps', 0)} bps") + logger.info(f" Average Processing Time: {cob_stats.get('avg_processing_time_ms', 0):.2f} ms") + logger.info("") + + # Per-Symbol Analysis + logger.info("Per-Symbol Analysis:") + for symbol in self.symbols: + cob_snapshot = self.cob_integration.get_cob_snapshot(symbol) + if cob_snapshot: + logger.info(f" {symbol}:") + logger.info(f" Active Exchanges: {len(cob_snapshot.exchanges_active)}") + logger.info(f" Spread: {cob_snapshot.spread_bps:.2f} bps") + logger.info(f" Total Liquidity: ${(cob_snapshot.total_bid_liquidity + cob_snapshot.total_ask_liquidity):,.0f}") + + recent_signals = self.cob_integration.get_recent_signals(symbol) + logger.info(f" Signals Generated: {len(recent_signals)}") + + logger.info("=" * 60) + logger.info("Test completed successfully!") + + async def _cleanup(self): + """Cleanup resources""" + logger.info("Cleaning up resources...") + + if self.cob_integration: + await self.cob_integration.stop() + + if self.data_provider and hasattr(self.data_provider, 'stop_real_time_streaming'): + await self.data_provider.stop_real_time_streaming() + + logger.info("Cleanup completed") + + # Callback functions for testing + + def _cnn_callback(self, symbol: str, data: dict): + """CNN feature callback for testing""" + self.stats['cnn_features_generated'] += 1 + if self.stats['cnn_features_generated'] % 100 == 0: + logger.debug(f"CNN features generated: {self.stats['cnn_features_generated']}") + + def _dqn_callback(self, symbol: str, data: dict): + """DQN feature callback for testing""" + self.stats['dqn_features_generated'] += 1 + if self.stats['dqn_features_generated'] % 100 == 0: + logger.debug(f"DQN features generated: {self.stats['dqn_features_generated']}") + + def _dashboard_callback(self, symbol: str, data: dict): + """Dashboard data callback for testing""" + self.stats['cob_updates_received'] += 1 + + # Check for signals in dashboard data + signals = data.get('recent_signals', []) + self.stats['signals_generated'] += len(signals) + +async def main(): + """Main test function""" + logger.info("Multi-Exchange COB Provider Test Starting...") + + try: + tester = COBTester() + await tester.run_test() + except KeyboardInterrupt: + logger.info("Test interrupted by user") + except Exception as e: + logger.error(f"Test failed with error: {e}") + raise + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/test_realtime_cob.py b/test_realtime_cob.py new file mode 100644 index 0000000..aedf212 --- /dev/null +++ b/test_realtime_cob.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +Test script for real-time COB functionality +""" + +import asyncio +import aiohttp +import json +import time +from datetime import datetime + +async def test_realtime_cob(): + """Test real-time COB data streaming""" + + # Test API endpoints + base_url = "http://localhost:8053" + + async with aiohttp.ClientSession() as session: + print("Testing COB Dashboard API endpoints...") + + # Test symbols endpoint + try: + async with session.get(f"{base_url}/api/symbols") as response: + if response.status == 200: + data = await response.json() + print(f"āœ“ Symbols: {data}") + else: + print(f"āœ— Symbols endpoint failed: {response.status}") + except Exception as e: + print(f"āœ— Error testing symbols endpoint: {e}") + + # Test real-time stats for BTC/USDT + try: + async with session.get(f"{base_url}/api/realtime/BTC/USDT") as response: + if response.status == 200: + data = await response.json() + print(f"āœ“ Real-time stats for BTC/USDT:") + print(f" Current mid price: {data.get('current', {}).get('mid_price', 'N/A')}") + print(f" 1s window updates: {data.get('1s_window', {}).get('update_count', 'N/A')}") + print(f" 5s window updates: {data.get('5s_window', {}).get('update_count', 'N/A')}") + else: + print(f"āœ— Real-time stats endpoint failed: {response.status}") + error_data = await response.text() + print(f" Error: {error_data}") + except Exception as e: + print(f"āœ— Error testing real-time stats endpoint: {e}") + + # Test WebSocket connection + print("\nTesting WebSocket connection...") + try: + async with session.ws_connect(f"{base_url.replace('http', 'ws')}/ws") as ws: + print("āœ“ WebSocket connected") + + # Wait for some data + message_count = 0 + start_time = time.time() + + async for msg in ws: + if msg.type == aiohttp.WSMsgType.TEXT: + data = json.loads(msg.data) + message_count += 1 + + if data.get('type') == 'cob_update': + symbol = data.get('data', {}).get('stats', {}).get('symbol', 'Unknown') + mid_price = data.get('data', {}).get('stats', {}).get('mid_price', 0) + print(f"āœ“ Received COB update for {symbol}: ${mid_price:.2f}") + + # Check for real-time stats + if 'realtime_1s' in data.get('data', {}).get('stats', {}): + print(f" āœ“ Real-time 1s stats available") + if 'realtime_5s' in data.get('data', {}).get('stats', {}): + print(f" āœ“ Real-time 5s stats available") + + # Stop after 5 messages or 10 seconds + if message_count >= 5 or (time.time() - start_time) > 10: + break + elif msg.type == aiohttp.WSMsgType.ERROR: + print(f"āœ— WebSocket error: {ws.exception()}") + break + + print(f"āœ“ Received {message_count} WebSocket messages") + + except Exception as e: + print(f"āœ— WebSocket connection failed: {e}") + +if __name__ == "__main__": + print("Testing Real-time COB Dashboard") + print("=" * 40) + + asyncio.run(test_realtime_cob()) + + print("\nTest completed!") \ No newline at end of file diff --git a/web/cob_dashboard.html b/web/cob_dashboard.html new file mode 100644 index 0000000..867c99a --- /dev/null +++ b/web/cob_dashboard.html @@ -0,0 +1,689 @@ + + + + + + Consolidated Order Book Dashboard + + + + +
+

Consolidated Order Book Dashboard

+
Hybrid WebSocket + REST API | Real-time + Deep Market Data
+
+ +
+
+ + +
+
+ + +
+
+ +
+ Disconnected - Click Toggle Connection to start +
+ +
+ +
+ +
+
Price & Volume Analysis
+
+
+ + +
+
Session Volume Profile (SVP)
+
+
+
+ + +
+
Order Book Ladder
+ +
+
Price
+
Size
+
Total
+
+ +
+ +
+
Loading asks...
+
+ + +
+
$--
+
Spread: -- bps
+
+ + +
+
Loading bids...
+
+
+
+ + +
+
+
Total Liquidity
+
--
+
--
+
+ +
+
Book Depth
+
--
+
--
+
+ +
+
Imbalance
+
--
+
bid/ask ratio
+
+ +
+
Update Rate
+
--
+
updates/sec
+
+ +
+
Best Bid
+
--
+
--
+
+ +
+
Best Ask
+
--
+
--
+
+
+
+ + + + \ No newline at end of file diff --git a/web/cob_realtime_dashboard.py b/web/cob_realtime_dashboard.py new file mode 100644 index 0000000..bb7c1dd --- /dev/null +++ b/web/cob_realtime_dashboard.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +""" +Consolidated Order Book (COB) Real-time Dashboard Server + +Provides a web interface for visualizing: +- Consolidated order book across multiple exchanges +- Session Volume Profile (SVP) from actual trades +- Real-time statistics for neural network models +- Hybrid WebSocket + REST API order book data + +Windows-compatible implementation with proper error handling. +""" + +import asyncio +import json +import logging +import weakref +from datetime import datetime, timedelta +from collections import deque +from typing import Dict, List, Optional, Any +import traceback + +# Windows-compatible imports +try: + from aiohttp import web, WSMsgType + import aiohttp_cors +except ImportError as e: + logging.error(f"Required dependencies missing: {e}") + raise + +from core.cob_integration import COBIntegration + +logger = logging.getLogger(__name__) + +class COBDashboardServer: + """ + Real-time COB Dashboard Server with Windows compatibility + """ + + def __init__(self, host: str = 'localhost', port: int = 8053): + self.host = host + self.port = port + self.app = web.Application() + self.symbols = ['BTC/USDT', 'ETH/USDT'] + + # COB components + self.cob_integration: Optional[COBIntegration] = None + + # Web server components + self.runner = None + self.site = None + + # WebSocket connections for real-time updates + self.websocket_connections = weakref.WeakSet() + + # Latest data cache for quick serving + self.latest_cob_data: Dict[str, Dict] = {} + self.latest_stats: Dict = {} + + # Update timestamps for monitoring + self.update_timestamps: Dict[str, deque] = { + symbol: deque(maxlen=100) for symbol in self.symbols + } + + # Setup routes and CORS + self._setup_routes() + self._setup_cors() + + logger.info(f"COB Dashboard Server initialized for {self.symbols}") + + def _setup_routes(self): + """Setup HTTP routes""" + # Static files + self.app.router.add_get('/', self.serve_dashboard) + + # API endpoints + self.app.router.add_get('/api/symbols', self.get_symbols) + self.app.router.add_get('/api/cob/{symbol}', self.get_cob_data) + self.app.router.add_get('/api/realtime/{symbol}', self.get_realtime_stats) + self.app.router.add_get('/api/status', self.get_status) + + # WebSocket endpoint + self.app.router.add_get('/ws', self.websocket_handler) + + def _setup_cors(self): + """Setup CORS for cross-origin requests""" + cors = aiohttp_cors.setup(self.app, defaults={ + "*": aiohttp_cors.ResourceOptions( + allow_credentials=True, + expose_headers="*", + allow_headers="*", + allow_methods="*" + ) + }) + + # Add CORS to all routes + for route in list(self.app.router.routes()): + cors.add(route) + + async def start(self): + """Start the dashboard server""" + try: + logger.info(f"Starting COB Dashboard Server on {self.host}:{self.port}") + + # Start web server first + self.runner = web.AppRunner(self.app) + await self.runner.setup() + + self.site = web.TCPSite(self.runner, self.host, self.port) + await self.site.start() + + logger.info(f"COB Dashboard Server running at http://{self.host}:{self.port}") + + # Initialize COB integration + self.cob_integration = COBIntegration(symbols=self.symbols) + self.cob_integration.add_dashboard_callback(self._on_cob_update) + + # Start COB data streaming as background task + asyncio.create_task(self.cob_integration.start()) + + # Start periodic tasks as background tasks + asyncio.create_task(self._periodic_stats_update()) + asyncio.create_task(self._cleanup_old_data()) + + # Keep the server running + while True: + await asyncio.sleep(1) + + except Exception as e: + logger.error(f"Error starting COB Dashboard Server: {e}") + logger.error(traceback.format_exc()) + raise + + async def stop(self): + """Stop the dashboard server""" + logger.info("Stopping COB Dashboard Server") + + # Close all WebSocket connections + for ws in list(self.websocket_connections): + try: + await ws.close() + except Exception as e: + logger.warning(f"Error closing WebSocket: {e}") + + # Stop web server + if self.site: + await self.site.stop() + if self.runner: + await self.runner.cleanup() + + # Stop COB integration + if self.cob_integration: + await self.cob_integration.stop() + + logger.info("COB Dashboard Server stopped") + + async def serve_dashboard(self, request): + """Serve the main dashboard HTML page""" + try: + return web.FileResponse('web/cob_dashboard.html') + except FileNotFoundError: + return web.Response( + text="Dashboard HTML file not found", + status=404, + content_type='text/plain' + ) + + async def get_symbols(self, request): + """Get available symbols""" + return web.json_response({ + 'symbols': self.symbols, + 'default': self.symbols[0] if self.symbols else None + }) + + async def get_cob_data(self, request): + """Get consolidated order book data for a symbol""" + try: + symbol = request.match_info['symbol'] + symbol = symbol.replace('%2F', '/') # URL decode + + if symbol not in self.symbols: + return web.json_response({ + 'error': f'Symbol {symbol} not supported', + 'available_symbols': self.symbols + }, status=400) + + # Get latest data from cache or COB integration + if symbol in self.latest_cob_data: + data = self.latest_cob_data[symbol] + elif self.cob_integration: + data = await self._generate_dashboard_data(symbol) + else: + data = self._get_empty_data(symbol) + + return web.json_response({ + 'symbol': symbol, + 'timestamp': datetime.now().isoformat(), + 'data': data + }) + + except Exception as e: + logger.error(f"Error getting COB data: {e}") + return web.json_response({ + 'error': str(e) + }, status=500) + + async def get_realtime_stats(self, request): + """Get real-time statistics for neural network models""" + try: + symbol = request.match_info['symbol'] + symbol = symbol.replace('%2F', '/') + + if symbol not in self.symbols: + return web.json_response({ + 'error': f'Symbol {symbol} not supported' + }, status=400) + + stats = {} + if self.cob_integration: + stats = self.cob_integration.get_realtime_stats_for_nn(symbol) + + return web.json_response({ + 'symbol': symbol, + 'timestamp': datetime.now().isoformat(), + 'stats': stats + }) + + except Exception as e: + logger.error(f"Error getting realtime stats: {e}") + return web.json_response({ + 'error': str(e) + }, status=500) + + async def get_status(self, request): + """Get server status""" + status = { + 'server': 'running', + 'symbols': self.symbols, + 'websocket_connections': len(self.websocket_connections), + 'cob_integration': 'active' if self.cob_integration else 'inactive', + 'last_updates': {} + } + + # Add last update times + for symbol in self.symbols: + if symbol in self.update_timestamps and self.update_timestamps[symbol]: + status['last_updates'][symbol] = self.update_timestamps[symbol][-1].isoformat() + + return web.json_response(status) + + async def websocket_handler(self, request): + """Handle WebSocket connections""" + ws = web.WebSocketResponse() + await ws.prepare(request) + + # Add to connections + self.websocket_connections.add(ws) + logger.info(f"WebSocket connected. Total connections: {len(self.websocket_connections)}") + + try: + # Send initial data + for symbol in self.symbols: + if symbol in self.latest_cob_data: + await self._send_websocket_data(ws, 'cob_update', symbol, self.latest_cob_data[symbol]) + + # Handle incoming messages + async for msg in ws: + if msg.type == WSMsgType.TEXT: + try: + data = json.loads(msg.data) + await self._handle_websocket_message(ws, data) + except json.JSONDecodeError: + await ws.send_str(json.dumps({ + 'type': 'error', + 'message': 'Invalid JSON' + })) + elif msg.type == WSMsgType.ERROR: + logger.error(f'WebSocket error: {ws.exception()}') + break + + except Exception as e: + logger.error(f"WebSocket error: {e}") + + finally: + # Remove from connections + self.websocket_connections.discard(ws) + logger.info(f"WebSocket disconnected. Remaining connections: {len(self.websocket_connections)}") + + return ws + + async def _handle_websocket_message(self, ws, data): + """Handle incoming WebSocket messages""" + try: + message_type = data.get('type') + + if message_type == 'subscribe': + symbol = data.get('symbol') + if symbol in self.symbols and symbol in self.latest_cob_data: + await self._send_websocket_data(ws, 'cob_update', symbol, self.latest_cob_data[symbol]) + + elif message_type == 'ping': + await ws.send_str(json.dumps({ + 'type': 'pong', + 'timestamp': datetime.now().isoformat() + })) + + except Exception as e: + logger.error(f"Error handling WebSocket message: {e}") + + async def _on_cob_update(self, symbol: str, data: Dict): + """Handle COB updates from integration""" + try: + logger.debug(f"Received COB update for {symbol}") + + # Update cache + self.latest_cob_data[symbol] = data + self.update_timestamps[symbol].append(datetime.now()) + + # Broadcast to WebSocket clients + await self._broadcast_cob_update(symbol, data) + + logger.debug(f"Broadcasted COB update for {symbol} to {len(self.websocket_connections)} connections") + + except Exception as e: + logger.error(f"Error handling COB update for {symbol}: {e}") + + async def _broadcast_cob_update(self, symbol: str, data: Dict): + """Broadcast COB update to all connected WebSocket clients""" + if not self.websocket_connections: + return + + message = { + 'type': 'cob_update', + 'symbol': symbol, + 'timestamp': datetime.now().isoformat(), + 'data': data + } + + # Send to all connections + dead_connections = [] + for ws in self.websocket_connections: + try: + await ws.send_str(json.dumps(message)) + except Exception as e: + logger.warning(f"Failed to send to WebSocket: {e}") + dead_connections.append(ws) + + # Clean up dead connections + for ws in dead_connections: + self.websocket_connections.discard(ws) + + async def _send_websocket_data(self, ws, msg_type: str, symbol: str, data: Dict): + """Send data to a specific WebSocket connection""" + try: + message = { + 'type': msg_type, + 'symbol': symbol, + 'timestamp': datetime.now().isoformat(), + 'data': data + } + await ws.send_str(json.dumps(message)) + except Exception as e: + logger.error(f"Error sending WebSocket data: {e}") + + async def _generate_dashboard_data(self, symbol: str) -> Dict: + """Generate dashboard data for a symbol""" + try: + # Return cached data from COB integration callbacks + if symbol in self.latest_cob_data: + return self.latest_cob_data[symbol] + else: + return self._get_empty_data(symbol) + + except Exception as e: + logger.error(f"Error generating dashboard data for {symbol}: {e}") + return self._get_empty_data(symbol) + + def _get_empty_data(self, symbol: str) -> Dict: + """Get empty data structure""" + return { + 'symbol': symbol, + 'bids': [], + 'asks': [], + 'svp': {'data': []}, + 'stats': { + 'mid_price': 0, + 'spread_bps': 0, + 'bid_liquidity': 0, + 'ask_liquidity': 0, + 'bid_levels': 0, + 'ask_levels': 0, + 'imbalance': 0 + } + } + + async def _periodic_stats_update(self): + """Periodically update and broadcast statistics""" + while True: + try: + # Calculate update frequencies + update_frequencies = {} + for symbol in self.symbols: + if symbol in self.update_timestamps and len(self.update_timestamps[symbol]) > 1: + timestamps = list(self.update_timestamps[symbol]) + if len(timestamps) >= 2: + time_diff = (timestamps[-1] - timestamps[-2]).total_seconds() + if time_diff > 0: + update_frequencies[symbol] = 1.0 / time_diff + + # Broadcast stats if needed + if update_frequencies: + stats_message = { + 'type': 'stats_update', + 'timestamp': datetime.now().isoformat(), + 'update_frequencies': update_frequencies + } + + for ws in list(self.websocket_connections): + try: + await ws.send_str(json.dumps(stats_message)) + except Exception: + self.websocket_connections.discard(ws) + + await asyncio.sleep(5) # Update every 5 seconds + + except Exception as e: + logger.error(f"Error in periodic stats update: {e}") + await asyncio.sleep(5) + + async def _cleanup_old_data(self): + """Clean up old data to prevent memory leaks""" + while True: + try: + cutoff_time = datetime.now() - timedelta(hours=1) + + # Clean up old timestamps + for symbol in self.symbols: + if symbol in self.update_timestamps: + timestamps = self.update_timestamps[symbol] + while timestamps and timestamps[0] < cutoff_time: + timestamps.popleft() + + await asyncio.sleep(300) # Clean up every 5 minutes + + except Exception as e: + logger.error(f"Error in cleanup: {e}") + await asyncio.sleep(300) + + +async def main(): + """Main entry point""" + # Set up logging + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + logger.info("Starting COB Dashboard Server") + + try: + # Windows event loop policy fix + if hasattr(asyncio, 'WindowsProactorEventLoopPolicy'): + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + + server = COBDashboardServer() + await server.start() + + except KeyboardInterrupt: + logger.info("COB Dashboard Server interrupted by user") + except Exception as e: + logger.error(f"COB Dashboard Server failed: {e}") + logger.error(traceback.format_exc()) + finally: + if 'server' in locals(): + await server.stop() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file