fix model mappings,dash updates, trading

This commit is contained in:
Dobromir Popov
2025-07-22 15:44:59 +03:00
parent 3e35b9cddb
commit 1a54fb1d56
32 changed files with 6168 additions and 857 deletions

View File

@ -97,7 +97,7 @@ class EnhancedCOBWebSocket:
logger.info(f"Enhanced COB WebSocket initialized for symbols: {self.symbols}")
if not WEBSOCKETS_AVAILABLE:
logger.error("⚠️ WebSockets module not available - COB data will be limited to REST API")
logger.error("WebSockets module not available - COB data will be limited to REST API")
def add_cob_callback(self, callback: Callable):
"""Add callback for COB data updates"""
@ -109,7 +109,7 @@ class EnhancedCOBWebSocket:
async def start(self):
"""Start COB WebSocket connections"""
logger.info("🚀 Starting Enhanced COB WebSocket system")
logger.info("Starting Enhanced COB WebSocket system")
# Initialize REST session for fallback
await self._init_rest_session()
@ -121,11 +121,11 @@ class EnhancedCOBWebSocket:
# Start monitoring task
asyncio.create_task(self._monitor_connections())
logger.info("Enhanced COB WebSocket system started")
logger.info("Enhanced COB WebSocket system started")
async def stop(self):
"""Stop all WebSocket connections"""
logger.info("🛑 Stopping Enhanced COB WebSocket system")
logger.info("Stopping Enhanced COB WebSocket system")
# Cancel all WebSocket tasks
for symbol, task in self.websocket_tasks.items():
@ -149,21 +149,161 @@ class EnhancedCOBWebSocket:
if self.rest_session:
await self.rest_session.close()
logger.info("Enhanced COB WebSocket system stopped")
logger.info("Enhanced COB WebSocket system stopped")
async def _init_rest_session(self):
"""Initialize REST API session for fallback"""
"""Initialize REST API session for fallback and snapshots"""
try:
timeout = aiohttp.ClientTimeout(total=10)
self.rest_session = aiohttp.ClientSession(timeout=timeout)
logger.info("✅ REST API session initialized for fallback")
# Windows-compatible configuration without aiodns
timeout = aiohttp.ClientTimeout(total=10, connect=5)
connector = aiohttp.TCPConnector(
limit=100,
limit_per_host=10,
enable_cleanup_closed=True,
use_dns_cache=False, # Disable DNS cache to avoid aiodns
family=0 # Use default family
)
self.rest_session = aiohttp.ClientSession(
timeout=timeout,
connector=connector,
headers={'User-Agent': 'Enhanced-COB-WebSocket/1.0'}
)
logger.info("✅ REST API session initialized (Windows compatible)")
except Exception as e:
logger.error(f" Failed to initialize REST session: {e}")
logger.warning(f"⚠️ Failed to initialize REST session: {e}")
# Try with minimal configuration
try:
self.rest_session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=10),
connector=aiohttp.TCPConnector(use_dns_cache=False)
)
logger.info("✅ REST API session initialized with minimal config")
except Exception as e2:
logger.warning(f"⚠️ Failed to initialize minimal REST session: {e2}")
# Continue without REST session - WebSocket only
self.rest_session = None
async def _get_order_book_snapshot(self, symbol: str):
"""Get initial order book snapshot from REST API
This is necessary for properly maintaining the order book state
with the WebSocket depth stream.
"""
try:
# Ensure REST session is available
if not self.rest_session:
await self._init_rest_session()
if not self.rest_session:
logger.warning(f"⚠️ Cannot get order book snapshot for {symbol} - REST session not available, will use WebSocket data only")
return
# Convert symbol format for Binance API
binance_symbol = symbol.replace('/', '')
# Get order book snapshot with maximum depth
url = f"https://api.binance.com/api/v3/depth?symbol={binance_symbol}&limit=1000"
logger.debug(f"🔍 Getting order book snapshot for {symbol} from {url}")
async with self.rest_session.get(url) as response:
if response.status == 200:
data = await response.json()
# Validate response structure
if not isinstance(data, dict) or 'bids' not in data or 'asks' not in data:
logger.error(f"❌ Invalid order book snapshot response for {symbol}: missing bids/asks")
return
# Initialize order book state for proper WebSocket synchronization
self.order_books[symbol] = {
'bids': {float(price): float(qty) for price, qty in data['bids']},
'asks': {float(price): float(qty) for price, qty in data['asks']}
}
# Store last update ID for synchronization
if 'lastUpdateId' in data:
self.last_update_ids[symbol] = data['lastUpdateId']
logger.info(f"✅ Got order book snapshot for {symbol}: {len(data['bids'])} bids, {len(data['asks'])} asks")
# Create initial COB data from snapshot
bids = [{'price': float(price), 'size': float(qty)} for price, qty in data['bids'] if float(qty) > 0]
asks = [{'price': float(price), 'size': float(qty)} for price, qty in data['asks'] if float(qty) > 0]
# Sort bids (descending) and asks (ascending)
bids.sort(key=lambda x: x['price'], reverse=True)
asks.sort(key=lambda x: x['price'])
# Create COB data structure if we have valid data
if bids and asks:
best_bid = bids[0]
best_ask = asks[0]
mid_price = (best_bid['price'] + best_ask['price']) / 2
spread = best_ask['price'] - best_bid['price']
spread_bps = (spread / mid_price) * 10000 if mid_price > 0 else 0
# Calculate volumes
bid_volume = sum(bid['size'] * bid['price'] for bid in bids)
ask_volume = sum(ask['size'] * ask['price'] for ask in asks)
total_volume = bid_volume + ask_volume
cob_data = {
'symbol': symbol,
'timestamp': datetime.now(),
'bids': bids,
'asks': asks,
'source': 'rest_snapshot',
'exchange': 'binance',
'stats': {
'best_bid': best_bid['price'],
'best_ask': best_ask['price'],
'mid_price': mid_price,
'spread': spread,
'spread_bps': spread_bps,
'bid_volume': bid_volume,
'ask_volume': ask_volume,
'total_bid_volume': bid_volume,
'total_ask_volume': ask_volume,
'imbalance': (bid_volume - ask_volume) / total_volume if total_volume > 0 else 0,
'bid_levels': len(bids),
'ask_levels': len(asks),
'timestamp': datetime.now().isoformat()
}
}
# Update cache
self.latest_cob_data[symbol] = cob_data
# Notify callbacks
for callback in self.cob_callbacks:
try:
await callback(symbol, cob_data)
except Exception as e:
logger.error(f"❌ Error in COB callback: {e}")
logger.debug(f"📊 Initial snapshot for {symbol}: ${mid_price:.2f}, spread: {spread_bps:.1f} bps")
else:
logger.warning(f"⚠️ No valid bid/ask data in snapshot for {symbol}")
elif response.status == 429:
logger.warning(f"⚠️ Rate limited getting snapshot for {symbol}, will continue with WebSocket only")
else:
logger.error(f"❌ Failed to get order book snapshot for {symbol}: HTTP {response.status}")
response_text = await response.text()
logger.debug(f"Response: {response_text}")
except asyncio.TimeoutError:
logger.warning(f"⚠️ Timeout getting order book snapshot for {symbol}, will continue with WebSocket only")
except Exception as e:
logger.warning(f"⚠️ Error getting order book snapshot for {symbol}: {e}, will continue with WebSocket only")
logger.debug(f"Snapshot error details: {e}")
# Don't fail the entire connection due to snapshot issues
async def _start_symbol_websocket(self, symbol: str):
"""Start WebSocket connection for a specific symbol"""
if not WEBSOCKETS_AVAILABLE:
logger.warning(f"⚠️ WebSockets not available for {symbol}, starting REST fallback")
logger.warning(f"WebSockets not available for {symbol}, starting REST fallback")
await self._start_rest_fallback(symbol)
return
@ -176,22 +316,25 @@ class EnhancedCOBWebSocket:
self._websocket_connection_loop(symbol)
)
logger.info(f"🔌 Started WebSocket task for {symbol}")
logger.info(f"Started WebSocket task for {symbol}")
async def _websocket_connection_loop(self, symbol: str):
"""Main WebSocket connection loop with reconnection logic"""
"""Main WebSocket connection loop with reconnection logic
Uses depth@100ms for fastest updates with maximum depth.
"""
status = self.status[symbol]
while True:
try:
logger.info(f"🔌 Attempting WebSocket connection for {symbol} (attempt {status.connection_attempts + 1})")
logger.info(f"Attempting WebSocket connection for {symbol} (attempt {status.connection_attempts + 1})")
status.connection_attempts += 1
# Create WebSocket URL with maximum depth
# Create WebSocket URL with maximum depth - use depth@100ms for fastest updates
ws_symbol = symbol.replace('/', '').lower() # BTCUSDT, ETHUSDT
ws_url = f"wss://stream.binance.com:9443/ws/{ws_symbol}@depth@{self.update_speed}"
ws_url = f"wss://stream.binance.com:9443/ws/{ws_symbol}@depth@100ms"
logger.info(f"🔗 Connecting to: {ws_url}")
logger.info(f"Connecting to: {ws_url}")
async with websockets_connect(ws_url) as websocket:
# Connection successful
@ -199,7 +342,7 @@ class EnhancedCOBWebSocket:
status.last_error = None
status.reset_reconnect_delay()
logger.info(f"WebSocket connected for {symbol}")
logger.info(f"WebSocket connected for {symbol}")
await self._notify_dashboard_status(symbol, "connected", "WebSocket connected")
# Deactivate REST fallback
@ -216,24 +359,24 @@ class EnhancedCOBWebSocket:
status.messages_received += 1
except json.JSONDecodeError as e:
logger.warning(f"⚠️ Invalid JSON from {symbol} WebSocket: {e}")
logger.warning(f"Invalid JSON from {symbol} WebSocket: {e}")
except Exception as e:
logger.error(f"Error processing WebSocket message for {symbol}: {e}")
logger.error(f"Error processing WebSocket message for {symbol}: {e}")
except ConnectionClosed as e:
status.connected = False
status.last_error = f"Connection closed: {e}"
logger.warning(f"🔌 WebSocket connection closed for {symbol}: {e}")
logger.warning(f"WebSocket connection closed for {symbol}: {e}")
except WebSocketException as e:
status.connected = False
status.last_error = f"WebSocket error: {e}"
logger.error(f"WebSocket error for {symbol}: {e}")
logger.error(f"WebSocket error for {symbol}: {e}")
except Exception as e:
status.connected = False
status.last_error = f"Unexpected error: {e}"
logger.error(f"Unexpected WebSocket error for {symbol}: {e}")
logger.error(f"Unexpected WebSocket error for {symbol}: {e}")
logger.error(traceback.format_exc())
# Connection failed or closed - start REST fallback
@ -242,51 +385,163 @@ class EnhancedCOBWebSocket:
# Wait before reconnecting
status.increase_reconnect_delay()
logger.info(f"Waiting {status.reconnect_delay:.1f}s before reconnecting {symbol}")
logger.info(f"Waiting {status.reconnect_delay:.1f}s before reconnecting {symbol}")
await asyncio.sleep(status.reconnect_delay)
async def _process_websocket_message(self, symbol: str, data: Dict):
"""Process WebSocket message and convert to COB format"""
try:
# Binance depth stream format
if 'b' in data and 'a' in data: # bids and asks
cob_data = {
'symbol': symbol,
'timestamp': datetime.now(),
'bids': [{'price': float(bid[0]), 'size': float(bid[1])} for bid in data['b']],
'asks': [{'price': float(ask[0]), 'size': float(ask[1])} for ask in data['a']],
'source': 'websocket',
'exchange': 'binance'
}
# Calculate stats
if cob_data['bids'] and cob_data['asks']:
best_bid = max(cob_data['bids'], key=lambda x: x['price'])
best_ask = min(cob_data['asks'], key=lambda x: x['price'])
cob_data['stats'] = {
'best_bid': best_bid['price'],
'best_ask': best_ask['price'],
'spread': best_ask['price'] - best_bid['price'],
'mid_price': (best_bid['price'] + best_ask['price']) / 2,
'bid_volume': sum(bid['size'] for bid in cob_data['bids']),
'ask_volume': sum(ask['size'] for ask in cob_data['asks'])
}
# Update cache
self.latest_cob_data[symbol] = cob_data
# Notify callbacks
for callback in self.cob_callbacks:
try:
await callback(symbol, cob_data)
except Exception as e:
logger.error(f"❌ Error in COB callback: {e}")
logger.debug(f"📊 Processed WebSocket COB data for {symbol}: {len(cob_data['bids'])} bids, {len(cob_data['asks'])} asks")
"""Process WebSocket message and convert to COB format
Based on the working implementation from cob_realtime_dashboard.py
Using maximum depth for best performance - no order book maintenance needed.
"""
try:
# Extract bids and asks from the message - handle all possible formats
bids_data = data.get('b', [])
asks_data = data.get('a', [])
# Process the order book data - filter out zero quantities
# Binance uses 0 quantity to indicate removal from the book
valid_bids = []
valid_asks = []
# Process bids
for bid in bids_data:
try:
if len(bid) >= 2:
price = float(bid[0])
size = float(bid[1])
if size > 0: # Only include non-zero quantities
valid_bids.append({'price': price, 'size': size})
except (IndexError, ValueError, TypeError):
continue
# Process asks
for ask in asks_data:
try:
if len(ask) >= 2:
price = float(ask[0])
size = float(ask[1])
if size > 0: # Only include non-zero quantities
valid_asks.append({'price': price, 'size': size})
except (IndexError, ValueError, TypeError):
continue
# Sort bids (descending) and asks (ascending) for proper order book
valid_bids.sort(key=lambda x: x['price'], reverse=True)
valid_asks.sort(key=lambda x: x['price'])
# Limit to maximum depth (1000 levels for maximum DOM)
max_depth = 1000
if len(valid_bids) > max_depth:
valid_bids = valid_bids[:max_depth]
if len(valid_asks) > max_depth:
valid_asks = valid_asks[:max_depth]
# Create COB data structure matching the working dashboard format
cob_data = {
'symbol': symbol,
'timestamp': datetime.now(),
'bids': valid_bids,
'asks': valid_asks,
'source': 'enhanced_websocket',
'exchange': 'binance'
}
# Calculate comprehensive stats if we have valid data
if valid_bids and valid_asks:
best_bid = valid_bids[0] # Already sorted, first is highest
best_ask = valid_asks[0] # Already sorted, first is lowest
# Core price metrics
mid_price = (best_bid['price'] + best_ask['price']) / 2
spread = best_ask['price'] - best_bid['price']
spread_bps = (spread / mid_price) * 10000 if mid_price > 0 else 0
# Volume calculations (notional value) - limit to top 20 levels for performance
top_bids = valid_bids[:20]
top_asks = valid_asks[:20]
bid_volume = sum(bid['size'] * bid['price'] for bid in top_bids)
ask_volume = sum(ask['size'] * ask['price'] for ask in top_asks)
# Size calculations (base currency)
bid_size = sum(bid['size'] for bid in top_bids)
ask_size = sum(ask['size'] for ask in top_asks)
# Imbalance calculations
total_volume = bid_volume + ask_volume
volume_imbalance = (bid_volume - ask_volume) / total_volume if total_volume > 0 else 0
total_size = bid_size + ask_size
size_imbalance = (bid_size - ask_size) / total_size if total_size > 0 else 0
cob_data['stats'] = {
'best_bid': best_bid['price'],
'best_ask': best_ask['price'],
'mid_price': mid_price,
'spread': spread,
'spread_bps': spread_bps,
'bid_volume': bid_volume,
'ask_volume': ask_volume,
'total_bid_volume': bid_volume,
'total_ask_volume': ask_volume,
'bid_liquidity': bid_volume, # Add liquidity fields
'ask_liquidity': ask_volume,
'total_bid_liquidity': bid_volume,
'total_ask_liquidity': ask_volume,
'bid_size': bid_size,
'ask_size': ask_size,
'volume_imbalance': volume_imbalance,
'size_imbalance': size_imbalance,
'imbalance': volume_imbalance, # Default to volume imbalance
'bid_levels': len(valid_bids),
'ask_levels': len(valid_asks),
'timestamp': datetime.now().isoformat(),
'update_id': data.get('u', 0), # Binance update ID
'event_time': data.get('E', 0) # Binance event time
}
else:
# Provide default stats if no valid data
cob_data['stats'] = {
'best_bid': 0,
'best_ask': 0,
'mid_price': 0,
'spread': 0,
'spread_bps': 0,
'bid_volume': 0,
'ask_volume': 0,
'total_bid_volume': 0,
'total_ask_volume': 0,
'bid_size': 0,
'ask_size': 0,
'volume_imbalance': 0,
'size_imbalance': 0,
'imbalance': 0,
'bid_levels': 0,
'ask_levels': 0,
'timestamp': datetime.now().isoformat(),
'update_id': data.get('u', 0),
'event_time': data.get('E', 0)
}
# Update cache
self.latest_cob_data[symbol] = cob_data
# Notify callbacks
for callback in self.cob_callbacks:
try:
await callback(symbol, cob_data)
except Exception as e:
logger.error(f"Error in COB callback: {e}")
# Log success with key metrics (only for non-empty updates)
if valid_bids and valid_asks:
logger.debug(f"{symbol}: ${cob_data['stats']['mid_price']:.2f}, {len(valid_bids)} bids, {len(valid_asks)} asks, spread: {cob_data['stats']['spread_bps']:.1f} bps")
except Exception as e:
logger.error(f"Error processing WebSocket message for {symbol}: {e}")
logger.error(f"Error processing WebSocket message for {symbol}: {e}")
import traceback
logger.debug(traceback.format_exc())
async def _start_rest_fallback(self, symbol: str):
"""Start REST API fallback for a symbol"""
@ -304,7 +559,7 @@ class EnhancedCOBWebSocket:
self._rest_fallback_loop(symbol)
)
logger.warning(f"⚠️ Started REST API fallback for {symbol}")
logger.warning(f"Started REST API fallback for {symbol}")
await self._notify_dashboard_status(symbol, "fallback", "Using REST API fallback")
async def _stop_rest_fallback(self, symbol: str):
@ -317,7 +572,7 @@ class EnhancedCOBWebSocket:
if symbol in self.rest_tasks and not self.rest_tasks[symbol].done():
self.rest_tasks[symbol].cancel()
logger.info(f"Stopped REST API fallback for {symbol}")
logger.info(f"Stopped REST API fallback for {symbol}")
async def _rest_fallback_loop(self, symbol: str):
"""REST API fallback loop"""
@ -328,7 +583,7 @@ class EnhancedCOBWebSocket:
except asyncio.CancelledError:
break
except Exception as e:
logger.error(f"REST fallback error for {symbol}: {e}")
logger.error(f"REST fallback error for {symbol}: {e}")
await asyncio.sleep(5) # Wait longer on error
async def _fetch_rest_orderbook(self, symbol: str):
@ -381,10 +636,10 @@ class EnhancedCOBWebSocket:
logger.debug(f"📊 Fetched REST COB data for {symbol}: {len(cob_data['bids'])} bids, {len(cob_data['asks'])} asks")
else:
logger.warning(f"⚠️ REST API error for {symbol}: HTTP {response.status}")
logger.warning(f"REST API error for {symbol}: HTTP {response.status}")
except Exception as e:
logger.error(f"Error fetching REST order book for {symbol}: {e}")
logger.error(f"Error fetching REST order book for {symbol}: {e}")
async def _monitor_connections(self):
"""Monitor WebSocket connections and provide status updates"""
@ -399,33 +654,40 @@ class EnhancedCOBWebSocket:
if status.connected and status.last_message_time:
time_since_last = datetime.now() - status.last_message_time
if time_since_last > timedelta(seconds=30):
logger.warning(f"⚠️ No messages from {symbol} WebSocket for {time_since_last.total_seconds():.0f}s")
logger.warning(f"No messages from {symbol} WebSocket for {time_since_last.total_seconds():.0f}s")
await self._notify_dashboard_status(symbol, "stale", "No recent messages")
# Log status
if status.connected:
logger.debug(f"{symbol}: Connected, {status.messages_received} messages received")
logger.debug(f"{symbol}: Connected, {status.messages_received} messages received")
elif self.rest_fallback_active[symbol]:
logger.debug(f"⚠️ {symbol}: Using REST fallback")
logger.debug(f"{symbol}: Using REST fallback")
else:
logger.debug(f"{symbol}: Disconnected, last error: {status.last_error}")
logger.debug(f"{symbol}: Disconnected, last error: {status.last_error}")
except Exception as e:
logger.error(f"Error in connection monitor: {e}")
logger.error(f"Error in connection monitor: {e}")
async def _notify_dashboard_status(self, symbol: str, status: str, message: str):
"""Notify dashboard of status changes"""
try:
if self.dashboard_callback:
await self.dashboard_callback({
status_data = {
'type': 'cob_status',
'symbol': symbol,
'status': status,
'message': message,
'timestamp': datetime.now().isoformat()
})
}
# Check if callback is async or sync
if asyncio.iscoroutinefunction(self.dashboard_callback):
await self.dashboard_callback(status_data)
else:
# Call sync function directly
self.dashboard_callback(status_data)
except Exception as e:
logger.error(f"Error notifying dashboard: {e}")
logger.error(f"Error notifying dashboard: {e}")
def get_status_summary(self) -> Dict[str, Any]:
"""Get status summary for all symbols"""