COB data and dash
This commit is contained in:
Dobromir Popov
952
core/bookmap_data_provider.py
Normal file
952
core/bookmap_data_provider.py
Normal file
@ -0,0 +1,952 @@
|
||||
"""
|
||||
Bookmap Order Book Data Provider
|
||||
|
||||
This module integrates with Bookmap to gather:
|
||||
- Current Order Book (COB) data
|
||||
- Session Volume Profile (SVP) data
|
||||
- Order book sweeps and momentum trades detection
|
||||
- Real-time order size heatmap matrix (last 10 minutes)
|
||||
- Level 2 market depth analysis
|
||||
|
||||
The data is processed and fed to CNN and DQN networks for enhanced trading decisions.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import websockets
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Tuple, Any, Callable
|
||||
from collections import deque, defaultdict
|
||||
from dataclasses import dataclass
|
||||
from threading import Thread, Lock
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class OrderBookLevel:
|
||||
"""Represents a single order book level"""
|
||||
price: float
|
||||
size: float
|
||||
orders: int
|
||||
side: str # 'bid' or 'ask'
|
||||
timestamp: datetime
|
||||
|
||||
@dataclass
|
||||
class OrderBookSnapshot:
|
||||
"""Complete order book snapshot"""
|
||||
symbol: str
|
||||
timestamp: datetime
|
||||
bids: List[OrderBookLevel]
|
||||
asks: List[OrderBookLevel]
|
||||
spread: float
|
||||
mid_price: float
|
||||
|
||||
@dataclass
|
||||
class VolumeProfileLevel:
|
||||
"""Volume profile level data"""
|
||||
price: float
|
||||
volume: float
|
||||
buy_volume: float
|
||||
sell_volume: float
|
||||
trades_count: int
|
||||
vwap: float
|
||||
|
||||
@dataclass
|
||||
class OrderFlowSignal:
|
||||
"""Order flow signal detection"""
|
||||
timestamp: datetime
|
||||
signal_type: str # 'sweep', 'absorption', 'iceberg', 'momentum'
|
||||
price: float
|
||||
volume: float
|
||||
confidence: float
|
||||
description: str
|
||||
|
||||
class BookmapDataProvider:
|
||||
"""
|
||||
Real-time order book data provider using Bookmap-style analysis
|
||||
|
||||
Features:
|
||||
- Level 2 order book monitoring
|
||||
- Order flow detection (sweeps, absorptions)
|
||||
- Volume profile analysis
|
||||
- Order size heatmap generation
|
||||
- Market microstructure analysis
|
||||
"""
|
||||
|
||||
def __init__(self, symbols: List[str] = None, depth_levels: int = 20):
|
||||
"""
|
||||
Initialize Bookmap data provider
|
||||
|
||||
Args:
|
||||
symbols: List of symbols to monitor
|
||||
depth_levels: Number of order book levels to track
|
||||
"""
|
||||
self.symbols = symbols or ['ETHUSDT', 'BTCUSDT']
|
||||
self.depth_levels = depth_levels
|
||||
self.is_streaming = False
|
||||
|
||||
# Order book data storage
|
||||
self.order_books: Dict[str, OrderBookSnapshot] = {}
|
||||
self.order_book_history: Dict[str, deque] = {}
|
||||
self.volume_profiles: Dict[str, List[VolumeProfileLevel]] = {}
|
||||
|
||||
# Heatmap data (10-minute rolling window)
|
||||
self.heatmap_window = timedelta(minutes=10)
|
||||
self.order_heatmaps: Dict[str, deque] = {}
|
||||
self.price_levels: Dict[str, List[float]] = {}
|
||||
|
||||
# Order flow detection
|
||||
self.flow_signals: Dict[str, deque] = {}
|
||||
self.sweep_threshold = 0.8 # Minimum confidence for sweep detection
|
||||
self.absorption_threshold = 0.7 # Minimum confidence for absorption
|
||||
|
||||
# Market microstructure metrics
|
||||
self.bid_ask_spreads: Dict[str, deque] = {}
|
||||
self.order_book_imbalances: Dict[str, deque] = {}
|
||||
self.liquidity_metrics: Dict[str, Dict] = {}
|
||||
|
||||
# WebSocket connections
|
||||
self.websocket_tasks: Dict[str, asyncio.Task] = {}
|
||||
self.data_lock = Lock()
|
||||
|
||||
# Callbacks for CNN/DQN integration
|
||||
self.cnn_callbacks: List[Callable] = []
|
||||
self.dqn_callbacks: List[Callable] = []
|
||||
|
||||
# Performance tracking
|
||||
self.update_counts = defaultdict(int)
|
||||
self.last_update_times = {}
|
||||
|
||||
# Initialize data structures
|
||||
for symbol in self.symbols:
|
||||
self.order_book_history[symbol] = deque(maxlen=1000)
|
||||
self.order_heatmaps[symbol] = deque(maxlen=600) # 10 min at 1s intervals
|
||||
self.flow_signals[symbol] = deque(maxlen=500)
|
||||
self.bid_ask_spreads[symbol] = deque(maxlen=1000)
|
||||
self.order_book_imbalances[symbol] = deque(maxlen=1000)
|
||||
self.liquidity_metrics[symbol] = {
|
||||
'total_bid_size': 0.0,
|
||||
'total_ask_size': 0.0,
|
||||
'weighted_mid': 0.0,
|
||||
'liquidity_ratio': 1.0
|
||||
}
|
||||
|
||||
logger.info(f"BookmapDataProvider initialized for {len(self.symbols)} symbols")
|
||||
logger.info(f"Tracking {depth_levels} order book levels per side")
|
||||
|
||||
def add_cnn_callback(self, callback: Callable[[str, Dict], None]):
|
||||
"""Add callback for CNN model updates"""
|
||||
self.cnn_callbacks.append(callback)
|
||||
logger.info(f"Added CNN callback: {len(self.cnn_callbacks)} total")
|
||||
|
||||
def add_dqn_callback(self, callback: Callable[[str, Dict], None]):
|
||||
"""Add callback for DQN model updates"""
|
||||
self.dqn_callbacks.append(callback)
|
||||
logger.info(f"Added DQN callback: {len(self.dqn_callbacks)} total")
|
||||
|
||||
async def start_streaming(self):
|
||||
"""Start real-time order book streaming"""
|
||||
if self.is_streaming:
|
||||
logger.warning("Bookmap streaming already active")
|
||||
return
|
||||
|
||||
self.is_streaming = True
|
||||
logger.info("Starting Bookmap order book streaming")
|
||||
|
||||
# Start order book streams for each symbol
|
||||
for symbol in self.symbols:
|
||||
# Order book depth stream
|
||||
depth_task = asyncio.create_task(self._stream_order_book_depth(symbol))
|
||||
self.websocket_tasks[f"{symbol}_depth"] = depth_task
|
||||
|
||||
# Trade stream for order flow analysis
|
||||
trade_task = asyncio.create_task(self._stream_trades(symbol))
|
||||
self.websocket_tasks[f"{symbol}_trades"] = trade_task
|
||||
|
||||
# Start analysis threads
|
||||
analysis_task = asyncio.create_task(self._continuous_analysis())
|
||||
self.websocket_tasks["analysis"] = analysis_task
|
||||
|
||||
logger.info(f"Started streaming for {len(self.symbols)} symbols")
|
||||
|
||||
async def stop_streaming(self):
|
||||
"""Stop order book streaming"""
|
||||
if not self.is_streaming:
|
||||
return
|
||||
|
||||
logger.info("Stopping Bookmap streaming")
|
||||
self.is_streaming = False
|
||||
|
||||
# Cancel all tasks
|
||||
for name, task in self.websocket_tasks.items():
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
self.websocket_tasks.clear()
|
||||
logger.info("Bookmap streaming stopped")
|
||||
|
||||
async def _stream_order_book_depth(self, symbol: str):
|
||||
"""Stream order book depth data"""
|
||||
binance_symbol = symbol.lower()
|
||||
url = f"wss://stream.binance.com:9443/ws/{binance_symbol}@depth20@100ms"
|
||||
|
||||
while self.is_streaming:
|
||||
try:
|
||||
async with websockets.connect(url) as websocket:
|
||||
logger.info(f"Order book depth WebSocket connected for {symbol}")
|
||||
|
||||
async for message in websocket:
|
||||
if not self.is_streaming:
|
||||
break
|
||||
|
||||
try:
|
||||
data = json.loads(message)
|
||||
await self._process_depth_update(symbol, data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error processing depth for {symbol}: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Depth WebSocket error for {symbol}: {e}")
|
||||
if self.is_streaming:
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def _stream_trades(self, symbol: str):
|
||||
"""Stream trade data for order flow analysis"""
|
||||
binance_symbol = symbol.lower()
|
||||
url = f"wss://stream.binance.com:9443/ws/{binance_symbol}@trade"
|
||||
|
||||
while self.is_streaming:
|
||||
try:
|
||||
async with websockets.connect(url) as websocket:
|
||||
logger.info(f"Trade WebSocket connected for {symbol}")
|
||||
|
||||
async for message in websocket:
|
||||
if not self.is_streaming:
|
||||
break
|
||||
|
||||
try:
|
||||
data = json.loads(message)
|
||||
await self._process_trade_update(symbol, data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error processing trade for {symbol}: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Trade WebSocket error for {symbol}: {e}")
|
||||
if self.is_streaming:
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def _process_depth_update(self, symbol: str, data: Dict):
|
||||
"""Process order book depth update"""
|
||||
try:
|
||||
timestamp = datetime.now()
|
||||
|
||||
# Parse bids and asks
|
||||
bids = []
|
||||
asks = []
|
||||
|
||||
for bid_data in data.get('bids', []):
|
||||
price = float(bid_data[0])
|
||||
size = float(bid_data[1])
|
||||
bids.append(OrderBookLevel(
|
||||
price=price,
|
||||
size=size,
|
||||
orders=1, # Binance doesn't provide order count
|
||||
side='bid',
|
||||
timestamp=timestamp
|
||||
))
|
||||
|
||||
for ask_data in data.get('asks', []):
|
||||
price = float(ask_data[0])
|
||||
size = float(ask_data[1])
|
||||
asks.append(OrderBookLevel(
|
||||
price=price,
|
||||
size=size,
|
||||
orders=1,
|
||||
side='ask',
|
||||
timestamp=timestamp
|
||||
))
|
||||
|
||||
# Sort order book levels
|
||||
bids.sort(key=lambda x: x.price, reverse=True)
|
||||
asks.sort(key=lambda x: x.price)
|
||||
|
||||
# Calculate spread and mid price
|
||||
if bids and asks:
|
||||
best_bid = bids[0].price
|
||||
best_ask = asks[0].price
|
||||
spread = best_ask - best_bid
|
||||
mid_price = (best_bid + best_ask) / 2
|
||||
else:
|
||||
spread = 0.0
|
||||
mid_price = 0.0
|
||||
|
||||
# Create order book snapshot
|
||||
snapshot = OrderBookSnapshot(
|
||||
symbol=symbol,
|
||||
timestamp=timestamp,
|
||||
bids=bids,
|
||||
asks=asks,
|
||||
spread=spread,
|
||||
mid_price=mid_price
|
||||
)
|
||||
|
||||
with self.data_lock:
|
||||
self.order_books[symbol] = snapshot
|
||||
self.order_book_history[symbol].append(snapshot)
|
||||
|
||||
# Update liquidity metrics
|
||||
self._update_liquidity_metrics(symbol, snapshot)
|
||||
|
||||
# Update order book imbalance
|
||||
self._calculate_order_book_imbalance(symbol, snapshot)
|
||||
|
||||
# Update heatmap data
|
||||
self._update_order_heatmap(symbol, snapshot)
|
||||
|
||||
# Update counters
|
||||
self.update_counts[f"{symbol}_depth"] += 1
|
||||
self.last_update_times[f"{symbol}_depth"] = timestamp
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing depth update for {symbol}: {e}")
|
||||
|
||||
async def _process_trade_update(self, symbol: str, data: Dict):
|
||||
"""Process trade data for order flow analysis"""
|
||||
try:
|
||||
timestamp = datetime.fromtimestamp(int(data['T']) / 1000)
|
||||
price = float(data['p'])
|
||||
quantity = float(data['q'])
|
||||
is_buyer_maker = data['m']
|
||||
|
||||
# Analyze for order flow signals
|
||||
await self._analyze_order_flow(symbol, timestamp, price, quantity, is_buyer_maker)
|
||||
|
||||
# Update volume profile
|
||||
self._update_volume_profile(symbol, price, quantity, is_buyer_maker)
|
||||
|
||||
self.update_counts[f"{symbol}_trades"] += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing trade for {symbol}: {e}")
|
||||
|
||||
def _update_liquidity_metrics(self, symbol: str, snapshot: OrderBookSnapshot):
|
||||
"""Update liquidity metrics from order book snapshot"""
|
||||
try:
|
||||
total_bid_size = sum(level.size for level in snapshot.bids)
|
||||
total_ask_size = sum(level.size for level in snapshot.asks)
|
||||
|
||||
# Calculate weighted mid price
|
||||
if snapshot.bids and snapshot.asks:
|
||||
bid_weight = total_bid_size / (total_bid_size + total_ask_size)
|
||||
ask_weight = total_ask_size / (total_bid_size + total_ask_size)
|
||||
weighted_mid = (snapshot.bids[0].price * ask_weight +
|
||||
snapshot.asks[0].price * bid_weight)
|
||||
else:
|
||||
weighted_mid = snapshot.mid_price
|
||||
|
||||
# Liquidity ratio (bid/ask balance)
|
||||
if total_ask_size > 0:
|
||||
liquidity_ratio = total_bid_size / total_ask_size
|
||||
else:
|
||||
liquidity_ratio = 1.0
|
||||
|
||||
self.liquidity_metrics[symbol] = {
|
||||
'total_bid_size': total_bid_size,
|
||||
'total_ask_size': total_ask_size,
|
||||
'weighted_mid': weighted_mid,
|
||||
'liquidity_ratio': liquidity_ratio,
|
||||
'spread_bps': (snapshot.spread / snapshot.mid_price) * 10000 if snapshot.mid_price > 0 else 0
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating liquidity metrics for {symbol}: {e}")
|
||||
|
||||
def _calculate_order_book_imbalance(self, symbol: str, snapshot: OrderBookSnapshot):
|
||||
"""Calculate order book imbalance ratio"""
|
||||
try:
|
||||
if not snapshot.bids or not snapshot.asks:
|
||||
return
|
||||
|
||||
# Calculate imbalance for top N levels
|
||||
n_levels = min(5, len(snapshot.bids), len(snapshot.asks))
|
||||
|
||||
total_bid_size = sum(snapshot.bids[i].size for i in range(n_levels))
|
||||
total_ask_size = sum(snapshot.asks[i].size for i in range(n_levels))
|
||||
|
||||
if total_bid_size + total_ask_size > 0:
|
||||
imbalance = (total_bid_size - total_ask_size) / (total_bid_size + total_ask_size)
|
||||
else:
|
||||
imbalance = 0.0
|
||||
|
||||
self.order_book_imbalances[symbol].append({
|
||||
'timestamp': snapshot.timestamp,
|
||||
'imbalance': imbalance,
|
||||
'bid_size': total_bid_size,
|
||||
'ask_size': total_ask_size
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating imbalance for {symbol}: {e}")
|
||||
|
||||
def _update_order_heatmap(self, symbol: str, snapshot: OrderBookSnapshot):
|
||||
"""Update order size heatmap matrix"""
|
||||
try:
|
||||
# Create heatmap entry
|
||||
heatmap_entry = {
|
||||
'timestamp': snapshot.timestamp,
|
||||
'mid_price': snapshot.mid_price,
|
||||
'levels': {}
|
||||
}
|
||||
|
||||
# Add bid levels
|
||||
for level in snapshot.bids:
|
||||
price_offset = level.price - snapshot.mid_price
|
||||
heatmap_entry['levels'][price_offset] = {
|
||||
'side': 'bid',
|
||||
'size': level.size,
|
||||
'price': level.price
|
||||
}
|
||||
|
||||
# Add ask levels
|
||||
for level in snapshot.asks:
|
||||
price_offset = level.price - snapshot.mid_price
|
||||
heatmap_entry['levels'][price_offset] = {
|
||||
'side': 'ask',
|
||||
'size': level.size,
|
||||
'price': level.price
|
||||
}
|
||||
|
||||
self.order_heatmaps[symbol].append(heatmap_entry)
|
||||
|
||||
# Clean old entries (keep 10 minutes)
|
||||
cutoff_time = snapshot.timestamp - self.heatmap_window
|
||||
while (self.order_heatmaps[symbol] and
|
||||
self.order_heatmaps[symbol][0]['timestamp'] < cutoff_time):
|
||||
self.order_heatmaps[symbol].popleft()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating heatmap for {symbol}: {e}")
|
||||
|
||||
def _update_volume_profile(self, symbol: str, price: float, quantity: float, is_buyer_maker: bool):
|
||||
"""Update volume profile with new trade"""
|
||||
try:
|
||||
# Initialize if not exists
|
||||
if symbol not in self.volume_profiles:
|
||||
self.volume_profiles[symbol] = []
|
||||
|
||||
# Find or create price level
|
||||
price_level = None
|
||||
for level in self.volume_profiles[symbol]:
|
||||
if abs(level.price - price) < 0.01: # Price tolerance
|
||||
price_level = level
|
||||
break
|
||||
|
||||
if not price_level:
|
||||
price_level = VolumeProfileLevel(
|
||||
price=price,
|
||||
volume=0.0,
|
||||
buy_volume=0.0,
|
||||
sell_volume=0.0,
|
||||
trades_count=0,
|
||||
vwap=price
|
||||
)
|
||||
self.volume_profiles[symbol].append(price_level)
|
||||
|
||||
# Update volume profile
|
||||
volume = price * quantity
|
||||
old_total = price_level.volume
|
||||
|
||||
price_level.volume += volume
|
||||
price_level.trades_count += 1
|
||||
|
||||
if is_buyer_maker:
|
||||
price_level.sell_volume += volume
|
||||
else:
|
||||
price_level.buy_volume += volume
|
||||
|
||||
# Update VWAP
|
||||
if price_level.volume > 0:
|
||||
price_level.vwap = ((price_level.vwap * old_total) + (price * volume)) / price_level.volume
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating volume profile for {symbol}: {e}")
|
||||
|
||||
async def _analyze_order_flow(self, symbol: str, timestamp: datetime, price: float,
|
||||
quantity: float, is_buyer_maker: bool):
|
||||
"""Analyze order flow for sweep and absorption patterns"""
|
||||
try:
|
||||
# Get recent order book data
|
||||
if symbol not in self.order_book_history or not self.order_book_history[symbol]:
|
||||
return
|
||||
|
||||
recent_snapshots = list(self.order_book_history[symbol])[-10:] # Last 10 snapshots
|
||||
|
||||
# Check for order book sweeps
|
||||
sweep_signal = self._detect_order_sweep(symbol, recent_snapshots, price, quantity, is_buyer_maker)
|
||||
if sweep_signal:
|
||||
self.flow_signals[symbol].append(sweep_signal)
|
||||
await self._notify_flow_signal(symbol, sweep_signal)
|
||||
|
||||
# Check for absorption patterns
|
||||
absorption_signal = self._detect_absorption(symbol, recent_snapshots, price, quantity)
|
||||
if absorption_signal:
|
||||
self.flow_signals[symbol].append(absorption_signal)
|
||||
await self._notify_flow_signal(symbol, absorption_signal)
|
||||
|
||||
# Check for momentum trades
|
||||
momentum_signal = self._detect_momentum_trade(symbol, price, quantity, is_buyer_maker)
|
||||
if momentum_signal:
|
||||
self.flow_signals[symbol].append(momentum_signal)
|
||||
await self._notify_flow_signal(symbol, momentum_signal)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error analyzing order flow for {symbol}: {e}")
|
||||
|
||||
def _detect_order_sweep(self, symbol: str, snapshots: List[OrderBookSnapshot],
|
||||
price: float, quantity: float, is_buyer_maker: bool) -> Optional[OrderFlowSignal]:
|
||||
"""Detect order book sweep patterns"""
|
||||
try:
|
||||
if len(snapshots) < 2:
|
||||
return None
|
||||
|
||||
before_snapshot = snapshots[-2]
|
||||
after_snapshot = snapshots[-1]
|
||||
|
||||
# Check if multiple levels were consumed
|
||||
if is_buyer_maker: # Sell order, check ask side
|
||||
levels_consumed = 0
|
||||
total_consumed_size = 0
|
||||
|
||||
for level in before_snapshot.asks[:5]: # Check top 5 levels
|
||||
if level.price <= price:
|
||||
levels_consumed += 1
|
||||
total_consumed_size += level.size
|
||||
|
||||
if levels_consumed >= 2 and total_consumed_size > quantity * 1.5:
|
||||
confidence = min(0.9, levels_consumed / 5.0 + 0.3)
|
||||
|
||||
return OrderFlowSignal(
|
||||
timestamp=datetime.now(),
|
||||
signal_type='sweep',
|
||||
price=price,
|
||||
volume=quantity * price,
|
||||
confidence=confidence,
|
||||
description=f"Sell sweep: {levels_consumed} levels, {total_consumed_size:.2f} size"
|
||||
)
|
||||
else: # Buy order, check bid side
|
||||
levels_consumed = 0
|
||||
total_consumed_size = 0
|
||||
|
||||
for level in before_snapshot.bids[:5]:
|
||||
if level.price >= price:
|
||||
levels_consumed += 1
|
||||
total_consumed_size += level.size
|
||||
|
||||
if levels_consumed >= 2 and total_consumed_size > quantity * 1.5:
|
||||
confidence = min(0.9, levels_consumed / 5.0 + 0.3)
|
||||
|
||||
return OrderFlowSignal(
|
||||
timestamp=datetime.now(),
|
||||
signal_type='sweep',
|
||||
price=price,
|
||||
volume=quantity * price,
|
||||
confidence=confidence,
|
||||
description=f"Buy sweep: {levels_consumed} levels, {total_consumed_size:.2f} size"
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error detecting sweep for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def _detect_absorption(self, symbol: str, snapshots: List[OrderBookSnapshot],
|
||||
price: float, quantity: float) -> Optional[OrderFlowSignal]:
|
||||
"""Detect absorption patterns where large orders are absorbed without price movement"""
|
||||
try:
|
||||
if len(snapshots) < 3:
|
||||
return None
|
||||
|
||||
# Check if large order was absorbed with minimal price impact
|
||||
volume_threshold = 10000 # $10K minimum for absorption
|
||||
price_impact_threshold = 0.001 # 0.1% max price impact
|
||||
|
||||
trade_value = price * quantity
|
||||
if trade_value < volume_threshold:
|
||||
return None
|
||||
|
||||
# Calculate price impact
|
||||
price_before = snapshots[-3].mid_price
|
||||
price_after = snapshots[-1].mid_price
|
||||
price_impact = abs(price_after - price_before) / price_before
|
||||
|
||||
if price_impact < price_impact_threshold:
|
||||
confidence = min(0.8, (trade_value / 50000) * 0.5 + 0.3) # Scale with size
|
||||
|
||||
return OrderFlowSignal(
|
||||
timestamp=datetime.now(),
|
||||
signal_type='absorption',
|
||||
price=price,
|
||||
volume=trade_value,
|
||||
confidence=confidence,
|
||||
description=f"Absorption: ${trade_value:.0f} with {price_impact*100:.3f}% impact"
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error detecting absorption for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def _detect_momentum_trade(self, symbol: str, price: float, quantity: float,
|
||||
is_buyer_maker: bool) -> Optional[OrderFlowSignal]:
|
||||
"""Detect momentum trades based on size and direction"""
|
||||
try:
|
||||
trade_value = price * quantity
|
||||
momentum_threshold = 25000 # $25K minimum for momentum classification
|
||||
|
||||
if trade_value < momentum_threshold:
|
||||
return None
|
||||
|
||||
# Calculate confidence based on trade size
|
||||
confidence = min(0.9, trade_value / 100000 * 0.6 + 0.3)
|
||||
|
||||
direction = "sell" if is_buyer_maker else "buy"
|
||||
|
||||
return OrderFlowSignal(
|
||||
timestamp=datetime.now(),
|
||||
signal_type='momentum',
|
||||
price=price,
|
||||
volume=trade_value,
|
||||
confidence=confidence,
|
||||
description=f"Large {direction}: ${trade_value:.0f}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error detecting momentum for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
async def _notify_flow_signal(self, symbol: str, signal: OrderFlowSignal):
|
||||
"""Notify CNN and DQN models of order flow signals"""
|
||||
try:
|
||||
signal_data = {
|
||||
'signal_type': signal.signal_type,
|
||||
'price': signal.price,
|
||||
'volume': signal.volume,
|
||||
'confidence': signal.confidence,
|
||||
'timestamp': signal.timestamp,
|
||||
'description': signal.description
|
||||
}
|
||||
|
||||
# Notify CNN callbacks
|
||||
for callback in self.cnn_callbacks:
|
||||
try:
|
||||
callback(symbol, signal_data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error in CNN callback: {e}")
|
||||
|
||||
# Notify DQN callbacks
|
||||
for callback in self.dqn_callbacks:
|
||||
try:
|
||||
callback(symbol, signal_data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error in DQN callback: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error notifying flow signal: {e}")
|
||||
|
||||
async def _continuous_analysis(self):
|
||||
"""Continuous analysis of market microstructure"""
|
||||
while self.is_streaming:
|
||||
try:
|
||||
await asyncio.sleep(1) # Analyze every second
|
||||
|
||||
for symbol in self.symbols:
|
||||
# Generate CNN features
|
||||
cnn_features = self.get_cnn_features(symbol)
|
||||
if cnn_features is not None:
|
||||
for callback in self.cnn_callbacks:
|
||||
try:
|
||||
callback(symbol, {'features': cnn_features, 'type': 'orderbook'})
|
||||
except Exception as e:
|
||||
logger.warning(f"Error in CNN feature callback: {e}")
|
||||
|
||||
# Generate DQN state features
|
||||
dqn_features = self.get_dqn_state_features(symbol)
|
||||
if dqn_features is not None:
|
||||
for callback in self.dqn_callbacks:
|
||||
try:
|
||||
callback(symbol, {'state': dqn_features, 'type': 'orderbook'})
|
||||
except Exception as e:
|
||||
logger.warning(f"Error in DQN state callback: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in continuous analysis: {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
def get_cnn_features(self, symbol: str) -> Optional[np.ndarray]:
|
||||
"""Generate CNN input features from order book data"""
|
||||
try:
|
||||
if symbol not in self.order_books:
|
||||
return None
|
||||
|
||||
snapshot = self.order_books[symbol]
|
||||
features = []
|
||||
|
||||
# Order book features (40 features: 20 levels x 2 sides)
|
||||
for i in range(min(20, len(snapshot.bids))):
|
||||
bid = snapshot.bids[i]
|
||||
features.append(bid.size)
|
||||
features.append(bid.price - snapshot.mid_price) # Price offset
|
||||
|
||||
# Pad if not enough bid levels
|
||||
while len(features) < 40:
|
||||
features.extend([0.0, 0.0])
|
||||
|
||||
for i in range(min(20, len(snapshot.asks))):
|
||||
ask = snapshot.asks[i]
|
||||
features.append(ask.size)
|
||||
features.append(ask.price - snapshot.mid_price) # Price offset
|
||||
|
||||
# Pad if not enough ask levels
|
||||
while len(features) < 80:
|
||||
features.extend([0.0, 0.0])
|
||||
|
||||
# Liquidity metrics (10 features)
|
||||
metrics = self.liquidity_metrics.get(symbol, {})
|
||||
features.extend([
|
||||
metrics.get('total_bid_size', 0.0),
|
||||
metrics.get('total_ask_size', 0.0),
|
||||
metrics.get('liquidity_ratio', 1.0),
|
||||
metrics.get('spread_bps', 0.0),
|
||||
snapshot.spread,
|
||||
metrics.get('weighted_mid', snapshot.mid_price) - snapshot.mid_price,
|
||||
len(snapshot.bids),
|
||||
len(snapshot.asks),
|
||||
snapshot.mid_price,
|
||||
time.time() % 86400 # Time of day
|
||||
])
|
||||
|
||||
# Order book imbalance features (5 features)
|
||||
if self.order_book_imbalances[symbol]:
|
||||
latest_imbalance = self.order_book_imbalances[symbol][-1]
|
||||
features.extend([
|
||||
latest_imbalance['imbalance'],
|
||||
latest_imbalance['bid_size'],
|
||||
latest_imbalance['ask_size'],
|
||||
latest_imbalance['bid_size'] + latest_imbalance['ask_size'],
|
||||
abs(latest_imbalance['imbalance'])
|
||||
])
|
||||
else:
|
||||
features.extend([0.0, 0.0, 0.0, 0.0, 0.0])
|
||||
|
||||
# Flow signal features (5 features)
|
||||
recent_signals = [s for s in self.flow_signals[symbol]
|
||||
if (datetime.now() - s.timestamp).seconds < 60]
|
||||
|
||||
sweep_count = sum(1 for s in recent_signals if s.signal_type == 'sweep')
|
||||
absorption_count = sum(1 for s in recent_signals if s.signal_type == 'absorption')
|
||||
momentum_count = sum(1 for s in recent_signals if s.signal_type == 'momentum')
|
||||
|
||||
max_confidence = max([s.confidence for s in recent_signals], default=0.0)
|
||||
total_flow_volume = sum(s.volume for s in recent_signals)
|
||||
|
||||
features.extend([
|
||||
sweep_count,
|
||||
absorption_count,
|
||||
momentum_count,
|
||||
max_confidence,
|
||||
total_flow_volume
|
||||
])
|
||||
|
||||
return np.array(features, dtype=np.float32)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating CNN features for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def get_dqn_state_features(self, symbol: str) -> Optional[np.ndarray]:
|
||||
"""Generate DQN state features from order book data"""
|
||||
try:
|
||||
if symbol not in self.order_books:
|
||||
return None
|
||||
|
||||
snapshot = self.order_books[symbol]
|
||||
state_features = []
|
||||
|
||||
# Normalized order book state (20 features)
|
||||
total_bid_size = sum(level.size for level in snapshot.bids[:10])
|
||||
total_ask_size = sum(level.size for level in snapshot.asks[:10])
|
||||
total_size = total_bid_size + total_ask_size
|
||||
|
||||
if total_size > 0:
|
||||
for i in range(min(10, len(snapshot.bids))):
|
||||
state_features.append(snapshot.bids[i].size / total_size)
|
||||
|
||||
# Pad bids
|
||||
while len(state_features) < 10:
|
||||
state_features.append(0.0)
|
||||
|
||||
for i in range(min(10, len(snapshot.asks))):
|
||||
state_features.append(snapshot.asks[i].size / total_size)
|
||||
|
||||
# Pad asks
|
||||
while len(state_features) < 20:
|
||||
state_features.append(0.0)
|
||||
else:
|
||||
state_features.extend([0.0] * 20)
|
||||
|
||||
# Market state indicators (10 features)
|
||||
metrics = self.liquidity_metrics.get(symbol, {})
|
||||
|
||||
# Normalize spread as percentage
|
||||
spread_pct = (snapshot.spread / snapshot.mid_price) if snapshot.mid_price > 0 else 0
|
||||
|
||||
# Liquidity imbalance
|
||||
liquidity_ratio = metrics.get('liquidity_ratio', 1.0)
|
||||
liquidity_imbalance = (liquidity_ratio - 1) / (liquidity_ratio + 1)
|
||||
|
||||
# Recent flow signals strength
|
||||
recent_signals = [s for s in self.flow_signals[symbol]
|
||||
if (datetime.now() - s.timestamp).seconds < 30]
|
||||
flow_strength = sum(s.confidence for s in recent_signals) / max(len(recent_signals), 1)
|
||||
|
||||
# Price volatility (from recent snapshots)
|
||||
if len(self.order_book_history[symbol]) >= 10:
|
||||
recent_prices = [s.mid_price for s in list(self.order_book_history[symbol])[-10:]]
|
||||
price_volatility = np.std(recent_prices) / np.mean(recent_prices) if recent_prices else 0
|
||||
else:
|
||||
price_volatility = 0
|
||||
|
||||
state_features.extend([
|
||||
spread_pct * 10000, # Spread in basis points
|
||||
liquidity_imbalance,
|
||||
flow_strength,
|
||||
price_volatility * 100, # Volatility as percentage
|
||||
min(len(snapshot.bids), 20) / 20, # Book depth ratio
|
||||
min(len(snapshot.asks), 20) / 20,
|
||||
sweep_count / 10 if 'sweep_count' in locals() else 0, # From CNN features
|
||||
absorption_count / 5 if 'absorption_count' in locals() else 0,
|
||||
momentum_count / 5 if 'momentum_count' in locals() else 0,
|
||||
(datetime.now().hour * 60 + datetime.now().minute) / 1440 # Time of day normalized
|
||||
])
|
||||
|
||||
return np.array(state_features, dtype=np.float32)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating DQN features for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def get_order_heatmap_matrix(self, symbol: str, levels: int = 40) -> Optional[np.ndarray]:
|
||||
"""Generate order size heatmap matrix for dashboard visualization"""
|
||||
try:
|
||||
if symbol not in self.order_heatmaps or not self.order_heatmaps[symbol]:
|
||||
return None
|
||||
|
||||
# Create price levels around current mid price
|
||||
current_snapshot = self.order_books.get(symbol)
|
||||
if not current_snapshot:
|
||||
return None
|
||||
|
||||
mid_price = current_snapshot.mid_price
|
||||
price_step = mid_price * 0.0001 # 1 basis point steps
|
||||
|
||||
# Create matrix: time x price levels
|
||||
time_window = min(600, len(self.order_heatmaps[symbol])) # 10 minutes max
|
||||
heatmap_matrix = np.zeros((time_window, levels))
|
||||
|
||||
# Fill matrix with order sizes
|
||||
for t, entry in enumerate(list(self.order_heatmaps[symbol])[-time_window:]):
|
||||
for price_offset, level_data in entry['levels'].items():
|
||||
# Convert price offset to matrix index
|
||||
level_idx = int((price_offset + (levels/2) * price_step) / price_step)
|
||||
|
||||
if 0 <= level_idx < levels:
|
||||
size_weight = 1.0 if level_data['side'] == 'bid' else -1.0
|
||||
heatmap_matrix[t, level_idx] = level_data['size'] * size_weight
|
||||
|
||||
return heatmap_matrix
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating heatmap matrix for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def get_volume_profile_data(self, symbol: str) -> Optional[List[Dict]]:
|
||||
"""Get session volume profile data"""
|
||||
try:
|
||||
if symbol not in self.volume_profiles:
|
||||
return None
|
||||
|
||||
profile_data = []
|
||||
for level in sorted(self.volume_profiles[symbol], key=lambda x: x.price):
|
||||
profile_data.append({
|
||||
'price': level.price,
|
||||
'volume': level.volume,
|
||||
'buy_volume': level.buy_volume,
|
||||
'sell_volume': level.sell_volume,
|
||||
'trades_count': level.trades_count,
|
||||
'vwap': level.vwap,
|
||||
'net_volume': level.buy_volume - level.sell_volume
|
||||
})
|
||||
|
||||
return profile_data
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting volume profile for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def get_current_order_book(self, symbol: str) -> Optional[Dict]:
|
||||
"""Get current order book snapshot"""
|
||||
try:
|
||||
if symbol not in self.order_books:
|
||||
return None
|
||||
|
||||
snapshot = self.order_books[symbol]
|
||||
|
||||
return {
|
||||
'timestamp': snapshot.timestamp.isoformat(),
|
||||
'symbol': symbol,
|
||||
'mid_price': snapshot.mid_price,
|
||||
'spread': snapshot.spread,
|
||||
'bids': [{'price': l.price, 'size': l.size} for l in snapshot.bids[:20]],
|
||||
'asks': [{'price': l.price, 'size': l.size} for l in snapshot.asks[:20]],
|
||||
'liquidity_metrics': self.liquidity_metrics.get(symbol, {}),
|
||||
'recent_signals': [
|
||||
{
|
||||
'type': s.signal_type,
|
||||
'price': s.price,
|
||||
'volume': s.volume,
|
||||
'confidence': s.confidence,
|
||||
'timestamp': s.timestamp.isoformat()
|
||||
}
|
||||
for s in list(self.flow_signals[symbol])[-5:] # Last 5 signals
|
||||
]
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting order book for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""Get provider statistics"""
|
||||
return {
|
||||
'symbols': self.symbols,
|
||||
'is_streaming': self.is_streaming,
|
||||
'update_counts': dict(self.update_counts),
|
||||
'last_update_times': {k: v.isoformat() if isinstance(v, datetime) else v
|
||||
for k, v in self.last_update_times.items()},
|
||||
'order_books_active': len(self.order_books),
|
||||
'flow_signals_total': sum(len(signals) for signals in self.flow_signals.values()),
|
||||
'cnn_callbacks': len(self.cnn_callbacks),
|
||||
'dqn_callbacks': len(self.dqn_callbacks),
|
||||
'websocket_tasks': len(self.websocket_tasks)
|
||||
}
|
||||
1839
core/bookmap_integration.py
Normal file
1839
core/bookmap_integration.py
Normal file
File diff suppressed because it is too large
Load Diff
597
core/cob_integration.py
Normal file
597
core/cob_integration.py
Normal file
@ -0,0 +1,597 @@
|
||||
"""
|
||||
Consolidated Order Book (COB) Integration Module
|
||||
|
||||
This module integrates the Multi-Exchange COB Provider with the existing
|
||||
gogo2 trading system architecture, providing:
|
||||
|
||||
- Integration with existing DataProvider
|
||||
- CNN/DQN model data feeding
|
||||
- Dashboard data formatting
|
||||
- Trading signal generation based on COB analysis
|
||||
- Enhanced market microstructure analysis
|
||||
|
||||
Connects to the main trading dashboard and AI models.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Any, Callable
|
||||
from threading import Thread
|
||||
import json
|
||||
import math
|
||||
from collections import defaultdict
|
||||
|
||||
from .multi_exchange_cob_provider import MultiExchangeCOBProvider, COBSnapshot, ConsolidatedOrderBookLevel
|
||||
from .data_provider import DataProvider, MarketTick
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class COBIntegration:
|
||||
"""
|
||||
Integration layer for Multi-Exchange COB data with gogo2 trading system
|
||||
"""
|
||||
|
||||
def __init__(self, data_provider: DataProvider = None, symbols: List[str] = None):
|
||||
"""
|
||||
Initialize COB Integration
|
||||
|
||||
Args:
|
||||
data_provider: Existing DataProvider instance
|
||||
symbols: List of symbols to monitor
|
||||
"""
|
||||
self.data_provider = data_provider
|
||||
self.symbols = symbols or ['BTC/USDT', 'ETH/USDT']
|
||||
|
||||
# Initialize COB provider
|
||||
self.cob_provider = MultiExchangeCOBProvider(
|
||||
symbols=self.symbols,
|
||||
bucket_size_bps=1.0 # 1 basis point granularity
|
||||
)
|
||||
|
||||
# Register callbacks
|
||||
self.cob_provider.subscribe_to_cob_updates(self._on_cob_update)
|
||||
self.cob_provider.subscribe_to_bucket_updates(self._on_bucket_update)
|
||||
|
||||
# CNN/DQN integration
|
||||
self.cnn_callbacks: List[Callable] = []
|
||||
self.dqn_callbacks: List[Callable] = []
|
||||
self.dashboard_callbacks: List[Callable] = []
|
||||
|
||||
# COB analysis and signals
|
||||
self.cob_signals: Dict[str, List[Dict]] = {}
|
||||
self.liquidity_alerts: Dict[str, List[Dict]] = {}
|
||||
self.arbitrage_opportunities: Dict[str, List[Dict]] = {}
|
||||
|
||||
# Performance tracking
|
||||
self.cob_feature_cache: Dict[str, np.ndarray] = {}
|
||||
self.last_cob_features_update: Dict[str, datetime] = {}
|
||||
|
||||
# Initialize signal tracking
|
||||
for symbol in self.symbols:
|
||||
self.cob_signals[symbol] = []
|
||||
self.liquidity_alerts[symbol] = []
|
||||
self.arbitrage_opportunities[symbol] = []
|
||||
|
||||
logger.info("COB Integration initialized")
|
||||
logger.info(f"Symbols: {self.symbols}")
|
||||
|
||||
async def start(self):
|
||||
"""Start COB integration"""
|
||||
logger.info("Starting COB Integration")
|
||||
|
||||
# Start COB provider
|
||||
await self.cob_provider.start_streaming()
|
||||
|
||||
# Start analysis threads
|
||||
asyncio.create_task(self._continuous_cob_analysis())
|
||||
asyncio.create_task(self._continuous_signal_generation())
|
||||
|
||||
logger.info("COB Integration started successfully")
|
||||
|
||||
async def stop(self):
|
||||
"""Stop COB integration"""
|
||||
logger.info("Stopping COB Integration")
|
||||
await self.cob_provider.stop_streaming()
|
||||
logger.info("COB Integration stopped")
|
||||
|
||||
def add_cnn_callback(self, callback: Callable[[str, Dict], None]):
|
||||
"""Add CNN model callback for COB features"""
|
||||
self.cnn_callbacks.append(callback)
|
||||
logger.info(f"Added CNN callback: {len(self.cnn_callbacks)} total")
|
||||
|
||||
def add_dqn_callback(self, callback: Callable[[str, Dict], None]):
|
||||
"""Add DQN model callback for COB state features"""
|
||||
self.dqn_callbacks.append(callback)
|
||||
logger.info(f"Added DQN callback: {len(self.dqn_callbacks)} total")
|
||||
|
||||
def add_dashboard_callback(self, callback: Callable[[str, Dict], None]):
|
||||
"""Add dashboard callback for COB visualization data"""
|
||||
self.dashboard_callbacks.append(callback)
|
||||
logger.info(f"Added dashboard callback: {len(self.dashboard_callbacks)} total")
|
||||
|
||||
async def _on_cob_update(self, symbol: str, cob_snapshot: COBSnapshot):
|
||||
"""Handle COB update from provider"""
|
||||
try:
|
||||
# Generate CNN features
|
||||
cnn_features = self._generate_cnn_features(symbol, cob_snapshot)
|
||||
if cnn_features is not None:
|
||||
self.cob_feature_cache[symbol] = cnn_features
|
||||
self.last_cob_features_update[symbol] = datetime.now()
|
||||
|
||||
# Notify CNN callbacks
|
||||
for callback in self.cnn_callbacks:
|
||||
try:
|
||||
callback(symbol, {
|
||||
'features': cnn_features,
|
||||
'timestamp': cob_snapshot.timestamp,
|
||||
'type': 'cob_features'
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning(f"Error in CNN callback: {e}")
|
||||
|
||||
# Generate DQN state features
|
||||
dqn_features = self._generate_dqn_features(symbol, cob_snapshot)
|
||||
if dqn_features is not None:
|
||||
for callback in self.dqn_callbacks:
|
||||
try:
|
||||
callback(symbol, {
|
||||
'state': dqn_features,
|
||||
'timestamp': cob_snapshot.timestamp,
|
||||
'type': 'cob_state'
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning(f"Error in DQN callback: {e}")
|
||||
|
||||
# Generate dashboard data
|
||||
dashboard_data = self._generate_dashboard_data(symbol, cob_snapshot)
|
||||
for callback in self.dashboard_callbacks:
|
||||
try:
|
||||
if asyncio.iscoroutinefunction(callback):
|
||||
asyncio.create_task(callback(symbol, dashboard_data))
|
||||
else:
|
||||
callback(symbol, dashboard_data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error in dashboard callback: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing COB update for {symbol}: {e}")
|
||||
|
||||
async def _on_bucket_update(self, symbol: str, price_buckets: Dict):
|
||||
"""Handle price bucket update from provider"""
|
||||
try:
|
||||
# Analyze bucket distribution and generate alerts
|
||||
await self._analyze_bucket_distribution(symbol, price_buckets)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing bucket update for {symbol}: {e}")
|
||||
|
||||
def _generate_cnn_features(self, symbol: str, cob_snapshot: COBSnapshot) -> Optional[np.ndarray]:
|
||||
"""Generate CNN input features from COB data"""
|
||||
try:
|
||||
features = []
|
||||
|
||||
# Order book depth features (200 features: 20 levels x 5 features x 2 sides)
|
||||
max_levels = 20
|
||||
|
||||
# Process bids
|
||||
for i in range(max_levels):
|
||||
if i < len(cob_snapshot.consolidated_bids):
|
||||
level = cob_snapshot.consolidated_bids[i]
|
||||
price_offset = (level.price - cob_snapshot.volume_weighted_mid) / cob_snapshot.volume_weighted_mid
|
||||
features.extend([
|
||||
price_offset,
|
||||
level.total_volume_usd / 1000000, # Normalize to millions
|
||||
level.total_size / 1000, # Normalize to thousands
|
||||
len(level.exchange_breakdown),
|
||||
level.liquidity_score
|
||||
])
|
||||
else:
|
||||
features.extend([0.0, 0.0, 0.0, 0.0, 0.0])
|
||||
|
||||
# Process asks
|
||||
for i in range(max_levels):
|
||||
if i < len(cob_snapshot.consolidated_asks):
|
||||
level = cob_snapshot.consolidated_asks[i]
|
||||
price_offset = (level.price - cob_snapshot.volume_weighted_mid) / cob_snapshot.volume_weighted_mid
|
||||
features.extend([
|
||||
price_offset,
|
||||
level.total_volume_usd / 1000000,
|
||||
level.total_size / 1000,
|
||||
len(level.exchange_breakdown),
|
||||
level.liquidity_score
|
||||
])
|
||||
else:
|
||||
features.extend([0.0, 0.0, 0.0, 0.0, 0.0])
|
||||
|
||||
# Market microstructure features (20 features)
|
||||
features.extend([
|
||||
cob_snapshot.spread_bps / 100, # Normalize spread
|
||||
cob_snapshot.liquidity_imbalance,
|
||||
cob_snapshot.total_bid_liquidity / 1000000,
|
||||
cob_snapshot.total_ask_liquidity / 1000000,
|
||||
len(cob_snapshot.exchanges_active) / 5, # Normalize to max 5 exchanges
|
||||
cob_snapshot.volume_weighted_mid / 100000, # Normalize price
|
||||
|
||||
# Exchange diversity metrics
|
||||
self._calculate_exchange_diversity(cob_snapshot.consolidated_bids),
|
||||
self._calculate_exchange_diversity(cob_snapshot.consolidated_asks),
|
||||
|
||||
# Price bucket concentration
|
||||
self._calculate_bucket_concentration(cob_snapshot.price_buckets, 'bids'),
|
||||
self._calculate_bucket_concentration(cob_snapshot.price_buckets, 'asks'),
|
||||
|
||||
# Liquidity depth metrics
|
||||
self._calculate_liquidity_depth_ratio(cob_snapshot.consolidated_bids, 5),
|
||||
self._calculate_liquidity_depth_ratio(cob_snapshot.consolidated_asks, 5),
|
||||
|
||||
# Time-based features
|
||||
cob_snapshot.timestamp.hour / 24,
|
||||
cob_snapshot.timestamp.minute / 60,
|
||||
cob_snapshot.timestamp.weekday() / 7,
|
||||
|
||||
# Additional features
|
||||
0.0, 0.0, 0.0, 0.0, 0.0
|
||||
])
|
||||
|
||||
return np.array(features, dtype=np.float32)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating CNN features for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def _generate_dqn_features(self, symbol: str, cob_snapshot: COBSnapshot) -> Optional[np.ndarray]:
|
||||
"""Generate DQN state features from COB data"""
|
||||
try:
|
||||
state_features = []
|
||||
|
||||
# Normalized order book state (20 features)
|
||||
total_liquidity = cob_snapshot.total_bid_liquidity + cob_snapshot.total_ask_liquidity
|
||||
|
||||
if total_liquidity > 0:
|
||||
# Top 10 bid levels (normalized by total liquidity)
|
||||
for i in range(10):
|
||||
if i < len(cob_snapshot.consolidated_bids):
|
||||
level = cob_snapshot.consolidated_bids[i]
|
||||
state_features.append(level.total_volume_usd / total_liquidity)
|
||||
else:
|
||||
state_features.append(0.0)
|
||||
|
||||
# Top 10 ask levels (normalized by total liquidity)
|
||||
for i in range(10):
|
||||
if i < len(cob_snapshot.consolidated_asks):
|
||||
level = cob_snapshot.consolidated_asks[i]
|
||||
state_features.append(level.total_volume_usd / total_liquidity)
|
||||
else:
|
||||
state_features.append(0.0)
|
||||
else:
|
||||
state_features.extend([0.0] * 20)
|
||||
|
||||
# Market state indicators (10 features)
|
||||
state_features.extend([
|
||||
cob_snapshot.spread_bps / 1000, # Normalized spread
|
||||
cob_snapshot.liquidity_imbalance,
|
||||
len(cob_snapshot.exchanges_active) / 5, # Exchange count ratio
|
||||
min(1.0, total_liquidity / 10000000), # Liquidity abundance
|
||||
0.5, # Price efficiency placeholder
|
||||
min(1.0, total_liquidity / 5000000), # Market impact resistance
|
||||
0.0, # Arbitrage score placeholder
|
||||
0.0, # Liquidity fragmentation placeholder
|
||||
(datetime.now().hour * 60 + datetime.now().minute) / 1440, # Time of day
|
||||
0.5 # Market regime indicator placeholder
|
||||
])
|
||||
|
||||
return np.array(state_features, dtype=np.float32)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating DQN features for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def _generate_dashboard_data(self, symbol: str, cob_snapshot: COBSnapshot) -> Dict:
|
||||
"""Generate formatted data for dashboard visualization"""
|
||||
try:
|
||||
# Get fixed bucket size for the symbol
|
||||
bucket_size = self.cob_provider.fixed_usd_buckets.get(symbol, 1.0)
|
||||
|
||||
# Calculate price range for buckets
|
||||
mid_price = cob_snapshot.volume_weighted_mid
|
||||
price_range = 100 # Show 100 price levels on each side
|
||||
|
||||
# Initialize bucket arrays
|
||||
bid_buckets = defaultdict(float)
|
||||
ask_buckets = defaultdict(float)
|
||||
|
||||
# Process bids into fixed USD buckets
|
||||
for bid in cob_snapshot.consolidated_bids:
|
||||
bucket_price = math.floor(bid.price / bucket_size) * bucket_size
|
||||
bid_buckets[bucket_price] += bid.total_volume_usd
|
||||
|
||||
# Process asks into fixed USD buckets
|
||||
for ask in cob_snapshot.consolidated_asks:
|
||||
bucket_price = math.floor(ask.price / bucket_size) * bucket_size
|
||||
ask_buckets[bucket_price] += ask.total_volume_usd
|
||||
|
||||
# Convert to sorted arrays for visualization
|
||||
bid_data = []
|
||||
ask_data = []
|
||||
|
||||
# Generate price levels
|
||||
min_price = math.floor((mid_price - (price_range * bucket_size)) / bucket_size) * bucket_size
|
||||
max_price = math.ceil((mid_price + (price_range * bucket_size)) / bucket_size) * bucket_size
|
||||
|
||||
# Fill bid data
|
||||
current_price = mid_price
|
||||
while current_price >= min_price:
|
||||
bucket_price = math.floor(current_price / bucket_size) * bucket_size
|
||||
volume = bid_buckets.get(bucket_price, 0)
|
||||
if volume > 0:
|
||||
bid_data.append({
|
||||
'price': bucket_price,
|
||||
'volume': volume,
|
||||
'side': 'bid'
|
||||
})
|
||||
current_price -= bucket_size
|
||||
|
||||
# Fill ask data
|
||||
current_price = mid_price
|
||||
while current_price <= max_price:
|
||||
bucket_price = math.floor(current_price / bucket_size) * bucket_size
|
||||
volume = ask_buckets.get(bucket_price, 0)
|
||||
if volume > 0:
|
||||
ask_data.append({
|
||||
'price': bucket_price,
|
||||
'volume': volume,
|
||||
'side': 'ask'
|
||||
})
|
||||
current_price += bucket_size
|
||||
|
||||
# Get actual Session Volume Profile (SVP) from trade data
|
||||
svp_data = []
|
||||
try:
|
||||
svp_result = self.cob_provider.get_session_volume_profile(symbol, bucket_size)
|
||||
if svp_result and 'data' in svp_result:
|
||||
svp_data = svp_result['data']
|
||||
logger.debug(f"Retrieved SVP data for {symbol}: {len(svp_data)} price levels")
|
||||
else:
|
||||
logger.warning(f"No SVP data available for {symbol}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting SVP data for {symbol}: {e}")
|
||||
|
||||
# Generate market stats
|
||||
stats = {
|
||||
'symbol': symbol,
|
||||
'timestamp': cob_snapshot.timestamp.isoformat(),
|
||||
'mid_price': cob_snapshot.volume_weighted_mid,
|
||||
'spread_bps': cob_snapshot.spread_bps,
|
||||
'total_bid_liquidity': cob_snapshot.total_bid_liquidity,
|
||||
'total_ask_liquidity': cob_snapshot.total_ask_liquidity,
|
||||
'liquidity_imbalance': cob_snapshot.liquidity_imbalance,
|
||||
'exchanges_active': cob_snapshot.exchanges_active,
|
||||
'bucket_size': bucket_size
|
||||
}
|
||||
|
||||
# Add exchange diversity metrics
|
||||
stats['bid_exchange_diversity'] = self._calculate_exchange_diversity(cob_snapshot.consolidated_bids[:20])
|
||||
stats['ask_exchange_diversity'] = self._calculate_exchange_diversity(cob_snapshot.consolidated_asks[:20])
|
||||
|
||||
# Add SVP statistics
|
||||
if svp_data:
|
||||
total_traded_volume = sum(item['total_volume'] for item in svp_data)
|
||||
stats['total_traded_volume'] = total_traded_volume
|
||||
stats['svp_price_levels'] = len(svp_data)
|
||||
stats['session_start'] = svp_result.get('session_start', '')
|
||||
else:
|
||||
stats['total_traded_volume'] = 0
|
||||
stats['svp_price_levels'] = 0
|
||||
stats['session_start'] = ''
|
||||
|
||||
# Add real-time statistics for NN models
|
||||
try:
|
||||
realtime_stats = self.cob_provider.get_realtime_stats(symbol)
|
||||
if realtime_stats:
|
||||
stats['realtime_1s'] = realtime_stats.get('1s_stats', {})
|
||||
stats['realtime_5s'] = realtime_stats.get('5s_stats', {})
|
||||
else:
|
||||
stats['realtime_1s'] = {}
|
||||
stats['realtime_5s'] = {}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting real-time stats for {symbol}: {e}")
|
||||
stats['realtime_1s'] = {}
|
||||
stats['realtime_5s'] = {}
|
||||
|
||||
return {
|
||||
'type': 'cob_update',
|
||||
'data': {
|
||||
'bids': bid_data,
|
||||
'asks': ask_data,
|
||||
'svp': svp_data,
|
||||
'stats': stats
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating dashboard data for {symbol}: {e}")
|
||||
return {
|
||||
'type': 'error',
|
||||
'data': {'error': str(e)}
|
||||
}
|
||||
|
||||
def _calculate_exchange_diversity(self, levels: List[ConsolidatedOrderBookLevel]) -> float:
|
||||
"""Calculate exchange diversity in order book levels"""
|
||||
if not levels:
|
||||
return 0.0
|
||||
|
||||
exchange_counts = {}
|
||||
total_volume = 0
|
||||
|
||||
for level in levels[:10]: # Top 10 levels
|
||||
total_volume += level.total_volume_usd
|
||||
for exchange in level.exchange_breakdown:
|
||||
exchange_counts[exchange] = exchange_counts.get(exchange, 0) + level.exchange_breakdown[exchange].volume_usd
|
||||
|
||||
if total_volume == 0:
|
||||
return 0.0
|
||||
|
||||
# Calculate diversity score
|
||||
hhi = sum((volume / total_volume) ** 2 for volume in exchange_counts.values())
|
||||
return 1 - hhi
|
||||
|
||||
def _calculate_bucket_concentration(self, price_buckets: Dict, side: str) -> float:
|
||||
"""Calculate concentration of liquidity in price buckets"""
|
||||
buckets = price_buckets.get(side, {})
|
||||
if not buckets:
|
||||
return 0.0
|
||||
|
||||
volumes = [bucket['volume_usd'] for bucket in buckets.values()]
|
||||
total_volume = sum(volumes)
|
||||
|
||||
if total_volume == 0:
|
||||
return 0.0
|
||||
|
||||
sorted_volumes = sorted(volumes, reverse=True)
|
||||
top_20_percent = int(len(sorted_volumes) * 0.2) or 1
|
||||
return sum(sorted_volumes[:top_20_percent]) / total_volume
|
||||
|
||||
def _calculate_liquidity_depth_ratio(self, levels: List[ConsolidatedOrderBookLevel], top_n: int) -> float:
|
||||
"""Calculate ratio of top N levels liquidity to total"""
|
||||
if not levels:
|
||||
return 0.0
|
||||
|
||||
top_n_volume = sum(level.total_volume_usd for level in levels[:top_n])
|
||||
total_volume = sum(level.total_volume_usd for level in levels)
|
||||
|
||||
return top_n_volume / total_volume if total_volume > 0 else 0.0
|
||||
|
||||
async def _continuous_cob_analysis(self):
|
||||
"""Continuously analyze COB data for patterns and signals"""
|
||||
while True:
|
||||
try:
|
||||
for symbol in self.symbols:
|
||||
cob_snapshot = self.cob_provider.get_consolidated_orderbook(symbol)
|
||||
if cob_snapshot:
|
||||
await self._analyze_cob_patterns(symbol, cob_snapshot)
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in COB analysis loop: {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
async def _analyze_cob_patterns(self, symbol: str, cob_snapshot: COBSnapshot):
|
||||
"""Analyze COB data for trading patterns and signals"""
|
||||
try:
|
||||
# Large liquidity imbalance detection
|
||||
if abs(cob_snapshot.liquidity_imbalance) > 0.4:
|
||||
signal = {
|
||||
'timestamp': cob_snapshot.timestamp.isoformat(),
|
||||
'type': 'liquidity_imbalance',
|
||||
'side': 'buy' if cob_snapshot.liquidity_imbalance > 0 else 'sell',
|
||||
'strength': abs(cob_snapshot.liquidity_imbalance),
|
||||
'confidence': min(1.0, abs(cob_snapshot.liquidity_imbalance) * 2)
|
||||
}
|
||||
self.cob_signals[symbol].append(signal)
|
||||
|
||||
# Cleanup old signals
|
||||
self.cob_signals[symbol] = self.cob_signals[symbol][-100:]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error analyzing COB patterns for {symbol}: {e}")
|
||||
|
||||
async def _analyze_bucket_distribution(self, symbol: str, price_buckets: Dict):
|
||||
"""Analyze price bucket distribution for patterns"""
|
||||
try:
|
||||
# Placeholder for bucket analysis
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error analyzing bucket distribution for {symbol}: {e}")
|
||||
|
||||
async def _continuous_signal_generation(self):
|
||||
"""Continuously generate trading signals based on COB analysis"""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(5)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in signal generation loop: {e}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
# Public interface methods
|
||||
|
||||
def get_cob_features(self, symbol: str) -> Optional[np.ndarray]:
|
||||
"""Get latest CNN features for a symbol"""
|
||||
return self.cob_feature_cache.get(symbol)
|
||||
|
||||
def get_cob_snapshot(self, symbol: str) -> Optional[COBSnapshot]:
|
||||
"""Get latest COB snapshot for a symbol"""
|
||||
return self.cob_provider.get_consolidated_orderbook(symbol)
|
||||
|
||||
def get_market_depth_analysis(self, symbol: str) -> Optional[Dict]:
|
||||
"""Get detailed market depth analysis"""
|
||||
return self.cob_provider.get_market_depth_analysis(symbol)
|
||||
|
||||
def get_exchange_breakdown(self, symbol: str) -> Optional[Dict]:
|
||||
"""Get liquidity breakdown by exchange"""
|
||||
return self.cob_provider.get_exchange_breakdown(symbol)
|
||||
|
||||
def get_price_buckets(self, symbol: str) -> Optional[Dict]:
|
||||
"""Get fine-grain price buckets"""
|
||||
return self.cob_provider.get_price_buckets(symbol)
|
||||
|
||||
def get_recent_signals(self, symbol: str, count: int = 20) -> List[Dict]:
|
||||
"""Get recent COB-based trading signals"""
|
||||
return self.cob_signals.get(symbol, [])[-count:]
|
||||
|
||||
def get_statistics(self) -> Dict[str, Any]:
|
||||
"""Get COB integration statistics"""
|
||||
provider_stats = self.cob_provider.get_statistics()
|
||||
|
||||
return {
|
||||
**provider_stats,
|
||||
'cnn_callbacks': len(self.cnn_callbacks),
|
||||
'dqn_callbacks': len(self.dqn_callbacks),
|
||||
'dashboard_callbacks': len(self.dashboard_callbacks),
|
||||
'cached_features': list(self.cob_feature_cache.keys()),
|
||||
'total_signals': {symbol: len(signals) for symbol, signals in self.cob_signals.items()}
|
||||
}
|
||||
|
||||
def get_realtime_stats_for_nn(self, symbol: str) -> Dict:
|
||||
"""Get real-time statistics formatted for NN models"""
|
||||
try:
|
||||
realtime_stats = self.cob_provider.get_realtime_stats(symbol)
|
||||
if not realtime_stats:
|
||||
return {}
|
||||
|
||||
# Format for NN consumption
|
||||
nn_stats = {
|
||||
'symbol': symbol,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'current': {
|
||||
'mid_price': 0.0,
|
||||
'spread_bps': 0.0,
|
||||
'bid_liquidity': 0.0,
|
||||
'ask_liquidity': 0.0,
|
||||
'imbalance': 0.0
|
||||
},
|
||||
'1s_window': realtime_stats.get('1s_stats', {}),
|
||||
'5s_window': realtime_stats.get('5s_stats', {})
|
||||
}
|
||||
|
||||
# Get current values from latest COB snapshot
|
||||
cob_snapshot = self.cob_provider.get_consolidated_orderbook(symbol)
|
||||
if cob_snapshot:
|
||||
nn_stats['current'] = {
|
||||
'mid_price': cob_snapshot.volume_weighted_mid,
|
||||
'spread_bps': cob_snapshot.spread_bps,
|
||||
'bid_liquidity': cob_snapshot.total_bid_liquidity,
|
||||
'ask_liquidity': cob_snapshot.total_ask_liquidity,
|
||||
'imbalance': cob_snapshot.liquidity_imbalance
|
||||
}
|
||||
|
||||
return nn_stats
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting NN stats for {symbol}: {e}")
|
||||
return {}
|
||||
@ -180,6 +180,37 @@ class DataProvider:
|
||||
logger.info("Centralized data distribution enabled")
|
||||
logger.info("Pivot-based normalization system enabled")
|
||||
|
||||
def _ensure_datetime_index(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Ensure dataframe has proper datetime index"""
|
||||
if df is None or df.empty:
|
||||
return df
|
||||
|
||||
try:
|
||||
# If we already have a proper DatetimeIndex, return as is
|
||||
if isinstance(df.index, pd.DatetimeIndex):
|
||||
return df
|
||||
|
||||
# If timestamp column exists, use it as index
|
||||
if 'timestamp' in df.columns:
|
||||
df['timestamp'] = pd.to_datetime(df['timestamp'])
|
||||
df.set_index('timestamp', inplace=True)
|
||||
return df
|
||||
|
||||
# If we have a RangeIndex or other non-datetime index, create datetime index
|
||||
if isinstance(df.index, pd.RangeIndex) or not isinstance(df.index, pd.DatetimeIndex):
|
||||
# Use current time and work backwards for realistic timestamps
|
||||
from datetime import datetime, timedelta
|
||||
end_time = datetime.now()
|
||||
start_time = end_time - timedelta(minutes=len(df))
|
||||
df.index = pd.date_range(start=start_time, end=end_time, periods=len(df))
|
||||
logger.debug(f"Converted RangeIndex to DatetimeIndex for {len(df)} records")
|
||||
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error ensuring datetime index: {e}")
|
||||
return df
|
||||
|
||||
def get_historical_data(self, symbol: str, timeframe: str, limit: int = 1000, refresh: bool = False) -> Optional[pd.DataFrame]:
|
||||
"""Get historical OHLCV data for a symbol and timeframe"""
|
||||
try:
|
||||
@ -188,6 +219,8 @@ class DataProvider:
|
||||
if self.cache_enabled:
|
||||
cached_data = self._load_from_cache(symbol, timeframe)
|
||||
if cached_data is not None and len(cached_data) >= limit * 0.8:
|
||||
# Ensure proper datetime index for cached data
|
||||
cached_data = self._ensure_datetime_index(cached_data)
|
||||
# logger.info(f"Using cached data for {symbol} {timeframe}")
|
||||
return cached_data.tail(limit)
|
||||
|
||||
@ -208,8 +241,11 @@ class DataProvider:
|
||||
df = self._fetch_from_mexc(symbol, timeframe, limit)
|
||||
|
||||
if df is not None and not df.empty:
|
||||
# Add technical indicators
|
||||
df = self._add_technical_indicators(df)
|
||||
# Ensure proper datetime index
|
||||
df = self._ensure_datetime_index(df)
|
||||
|
||||
# Add technical indicators. temporarily disabled to save time as it is not working as expected.
|
||||
# df = self._add_technical_indicators(df)
|
||||
|
||||
# Cache the data
|
||||
if self.cache_enabled:
|
||||
@ -1151,9 +1187,21 @@ class DataProvider:
|
||||
try:
|
||||
cache_file = self.monthly_data_cache_dir / f"{symbol.replace('/', '')}_monthly_1m.parquet"
|
||||
if cache_file.exists():
|
||||
df = pd.read_parquet(cache_file)
|
||||
logger.info(f"Loaded {len(df)} 1m candles from cache for {symbol}")
|
||||
return df
|
||||
try:
|
||||
df = pd.read_parquet(cache_file)
|
||||
logger.info(f"Loaded {len(df)} 1m candles from cache for {symbol}")
|
||||
return df
|
||||
except Exception as parquet_e:
|
||||
# Handle corrupted Parquet file
|
||||
if "Parquet magic bytes not found" in str(parquet_e) or "corrupted" in str(parquet_e).lower():
|
||||
logger.warning(f"Corrupted Parquet cache file for {symbol}, removing and returning None: {parquet_e}")
|
||||
try:
|
||||
cache_file.unlink() # Delete corrupted file
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
else:
|
||||
raise parquet_e
|
||||
|
||||
return None
|
||||
|
||||
@ -1240,9 +1288,21 @@ class DataProvider:
|
||||
# Check if cache is recent (less than 1 hour old)
|
||||
cache_age = time.time() - cache_file.stat().st_mtime
|
||||
if cache_age < 3600: # 1 hour
|
||||
df = pd.read_parquet(cache_file)
|
||||
logger.debug(f"Loaded {len(df)} rows from cache for {symbol} {timeframe}")
|
||||
return df
|
||||
try:
|
||||
df = pd.read_parquet(cache_file)
|
||||
logger.debug(f"Loaded {len(df)} rows from cache for {symbol} {timeframe}")
|
||||
return df
|
||||
except Exception as parquet_e:
|
||||
# Handle corrupted Parquet file
|
||||
if "Parquet magic bytes not found" in str(parquet_e) or "corrupted" in str(parquet_e).lower():
|
||||
logger.warning(f"Corrupted Parquet cache file for {symbol} {timeframe}, removing and returning None: {parquet_e}")
|
||||
try:
|
||||
cache_file.unlink() # Delete corrupted file
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
else:
|
||||
raise parquet_e
|
||||
else:
|
||||
logger.debug(f"Cache for {symbol} {timeframe} is too old ({cache_age/3600:.1f}h)")
|
||||
return None
|
||||
|
||||
@ -2324,7 +2324,14 @@ class EnhancedTradingOrchestrator:
|
||||
# 4. Return threshold adjustment (0.0 to 0.1 typically)
|
||||
|
||||
# For now, return small adjustment to demonstrate concept
|
||||
if hasattr(self.pivot_rl_trainer.williams, 'cnn_model') and self.pivot_rl_trainer.williams.cnn_model:
|
||||
# Check if CNN models are available in the model registry
|
||||
cnn_available = False
|
||||
for model_key, model in self.model_registry.items():
|
||||
if hasattr(model, 'cnn_model') and model.cnn_model:
|
||||
cnn_available = True
|
||||
break
|
||||
|
||||
if cnn_available:
|
||||
# CNN is available, could provide small threshold reduction for better entries
|
||||
return 0.05 # 5% threshold reduction when CNN available
|
||||
|
||||
@ -2337,17 +2344,27 @@ class EnhancedTradingOrchestrator:
|
||||
def update_dynamic_thresholds(self):
|
||||
"""Update thresholds based on recent performance"""
|
||||
try:
|
||||
# Update thresholds in pivot trainer
|
||||
self.pivot_rl_trainer.update_thresholds_based_on_performance()
|
||||
# Internal threshold update based on recent performance
|
||||
# This orchestrator handles thresholds internally without external trainer
|
||||
|
||||
# Get updated thresholds
|
||||
thresholds = self.pivot_rl_trainer.get_current_thresholds()
|
||||
old_entry = self.entry_threshold
|
||||
old_exit = self.exit_threshold
|
||||
|
||||
self.entry_threshold = thresholds['entry_threshold']
|
||||
self.exit_threshold = thresholds['exit_threshold']
|
||||
self.uninvested_threshold = thresholds['uninvested_threshold']
|
||||
# Simple performance-based threshold adjustment
|
||||
if len(self.completed_trades) >= 10:
|
||||
recent_trades = list(self.completed_trades)[-10:]
|
||||
win_rate = sum(1 for trade in recent_trades if trade.get('pnl_percentage', 0) > 0) / len(recent_trades)
|
||||
|
||||
# Adjust thresholds based on recent performance
|
||||
if win_rate > 0.7: # High win rate - can be more aggressive
|
||||
self.entry_threshold = max(0.5, self.entry_threshold - 0.02)
|
||||
self.exit_threshold = min(0.5, self.exit_threshold + 0.02)
|
||||
elif win_rate < 0.3: # Low win rate - be more conservative
|
||||
self.entry_threshold = min(0.8, self.entry_threshold + 0.02)
|
||||
self.exit_threshold = max(0.2, self.exit_threshold - 0.02)
|
||||
|
||||
# Update uninvested threshold based on activity
|
||||
self.uninvested_threshold = (self.entry_threshold + self.exit_threshold) / 2
|
||||
|
||||
# Log changes if significant
|
||||
if abs(old_entry - self.entry_threshold) > 0.01 or abs(old_exit - self.exit_threshold) > 0.01:
|
||||
@ -2362,9 +2379,32 @@ class EnhancedTradingOrchestrator:
|
||||
trade_outcome: Dict[str, Any]) -> float:
|
||||
"""Calculate reward using the enhanced pivot-based system"""
|
||||
try:
|
||||
return self.pivot_rl_trainer.calculate_pivot_based_reward(
|
||||
trade_decision, market_data, trade_outcome
|
||||
)
|
||||
# Simplified pivot-based reward calculation without external trainer
|
||||
# This orchestrator handles pivot logic internally via dynamic thresholds
|
||||
|
||||
if not trade_outcome or 'pnl_percentage' not in trade_outcome:
|
||||
return 0.0
|
||||
|
||||
pnl_percentage = trade_outcome['pnl_percentage']
|
||||
confidence = trade_decision.get('confidence', 0.5)
|
||||
|
||||
# Base reward from PnL
|
||||
base_reward = pnl_percentage * 10 # Scale PnL to reasonable reward range
|
||||
|
||||
# Bonus for high-confidence decisions that work out
|
||||
confidence_bonus = 0.0
|
||||
if pnl_percentage > 0 and confidence > self.entry_threshold:
|
||||
confidence_bonus = (confidence - self.entry_threshold) * 5.0
|
||||
|
||||
# Penalty for low-confidence losses
|
||||
confidence_penalty = 0.0
|
||||
if pnl_percentage < 0 and confidence < self.exit_threshold:
|
||||
confidence_penalty = abs(pnl_percentage) * 2.0
|
||||
|
||||
total_reward = base_reward + confidence_bonus - confidence_penalty
|
||||
|
||||
return total_reward
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating enhanced pivot reward: {e}")
|
||||
return 0.0
|
||||
|
||||
1268
core/multi_exchange_cob_provider.py
Normal file
1268
core/multi_exchange_cob_provider.py
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user