better cob integration
This commit is contained in:
Dobromir Popov
@ -421,29 +421,17 @@ class CleanTradingDashboard:
|
||||
[Input('interval-component', 'n_intervals')]
|
||||
)
|
||||
def update_cob_data(n):
|
||||
"""Update COB data displays with price buckets"""
|
||||
"""Update COB data displays with real order book ladders"""
|
||||
try:
|
||||
# ETH/USDT COB with $1 price buckets
|
||||
eth_cob = self._get_cob_snapshot('ETH/USDT')
|
||||
eth_buckets = self.get_cob_price_buckets('ETH/USDT')
|
||||
eth_memory_stats = self.get_cob_memory_stats('ETH/USDT')
|
||||
eth_components = self.component_manager.format_cob_data_with_buckets(
|
||||
eth_cob, 'ETH/USDT', eth_buckets, eth_memory_stats, bucket_size=1.0
|
||||
)
|
||||
|
||||
# BTC/USDT COB with $10 price buckets - Reference data for ETH models
|
||||
btc_cob = self._get_cob_snapshot('BTC/USDT')
|
||||
btc_buckets = self.get_cob_price_buckets('BTC/USDT')
|
||||
btc_memory_stats = self.get_cob_memory_stats('BTC/USDT')
|
||||
btc_components = self.component_manager.format_cob_data_with_buckets(
|
||||
btc_cob, 'BTC/USDT', btc_buckets, btc_memory_stats, bucket_size=10.0
|
||||
)
|
||||
# Get real COB data from the working integration
|
||||
eth_components = self._create_cob_ladder_display('ETH/USDT')
|
||||
btc_components = self._create_cob_ladder_display('BTC/USDT')
|
||||
|
||||
return eth_components, btc_components
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating COB data: {e}")
|
||||
error_msg = html.P(f"Error: {str(e)}", className="text-danger")
|
||||
error_msg = html.P(f"COB Error: {str(e)}", className="text-danger small")
|
||||
return error_msg, error_msg
|
||||
|
||||
@self.app.callback(
|
||||
@ -2947,47 +2935,243 @@ class CleanTradingDashboard:
|
||||
self.training_system = None
|
||||
|
||||
def _initialize_cob_integration(self):
|
||||
"""Initialize COB integration with high-frequency data handling"""
|
||||
"""Initialize COB integration with high-frequency data handling - LAZY INITIALIZATION"""
|
||||
try:
|
||||
if not COB_INTEGRATION_AVAILABLE:
|
||||
logger.warning("COB integration not available - skipping")
|
||||
return
|
||||
logger.info("Setting up COB integration for lazy initialization (will start when dashboard runs)")
|
||||
|
||||
# Initialize COB integration with dashboard callback
|
||||
self.cob_integration = COBIntegration(
|
||||
data_provider=self.data_provider,
|
||||
symbols=['ETH/USDT', 'BTC/USDT']
|
||||
)
|
||||
# Don't initialize COB here - just set up for lazy initialization
|
||||
self.cob_integration = None
|
||||
self.cob_integration_started = False
|
||||
self.latest_cob_data = {}
|
||||
self.cob_update_timestamps = {}
|
||||
|
||||
# Register dashboard callback for COB updates
|
||||
self.cob_integration.add_dashboard_callback(self._on_high_frequency_cob_update)
|
||||
|
||||
# Register CNN callback for COB features (for next price prediction)
|
||||
self.cob_integration.add_cnn_callback(self._on_cob_cnn_features)
|
||||
|
||||
# Register DQN callback for COB state features (for RL training)
|
||||
self.cob_integration.add_dqn_callback(self._on_cob_dqn_features)
|
||||
|
||||
# Start COB integration in background thread
|
||||
import threading
|
||||
def start_cob():
|
||||
import asyncio
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
loop.run_until_complete(self.cob_integration.start())
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting COB integration: {e}")
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
cob_thread = threading.Thread(target=start_cob, daemon=True)
|
||||
cob_thread.start()
|
||||
|
||||
logger.info("High-frequency COB integration initialized (50-100 Hz data handling)")
|
||||
logger.info("COB integration setup complete - will initialize when event loop is available")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error initializing COB integration: {e}")
|
||||
logger.error(f"Error setting up COB integration: {e}")
|
||||
self.cob_integration = None
|
||||
|
||||
def _start_cob_integration_lazy(self):
|
||||
"""Start COB integration when dashboard is running (lazy initialization)"""
|
||||
if self.cob_integration_started:
|
||||
return
|
||||
|
||||
try:
|
||||
logger.info("Starting COB integration with lazy initialization pattern")
|
||||
|
||||
# Import COB integration directly (same as working dashboard)
|
||||
from core.cob_integration import COBIntegration
|
||||
|
||||
# Start COB integration in a background thread with proper event loop
|
||||
def start_cob_worker():
|
||||
"""Start COB integration using the exact same pattern as working dashboard"""
|
||||
try:
|
||||
# Create new event loop for COB (same as working dashboard)
|
||||
import asyncio
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
async def cob_main():
|
||||
"""Main COB loop (same pattern as working dashboard)"""
|
||||
try:
|
||||
# Initialize COB integration with our symbols (same pattern as working dashboard)
|
||||
self.cob_integration = COBIntegration(symbols=['ETH/USDT', 'BTC/USDT'])
|
||||
|
||||
# Register callback to receive real-time COB data (same as working dashboard)
|
||||
self.cob_integration.add_dashboard_callback(self._on_cob_update)
|
||||
|
||||
# Start COB data streaming as background task (same as working dashboard)
|
||||
await self.cob_integration.start()
|
||||
|
||||
logger.info("COB integration started successfully with lazy initialization")
|
||||
logger.info("High-frequency COB data streaming active")
|
||||
|
||||
# Keep running (same as working dashboard)
|
||||
while True:
|
||||
await asyncio.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in COB main loop: {e}")
|
||||
|
||||
# Run the COB integration (same as working dashboard)
|
||||
loop.run_until_complete(cob_main())
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in COB worker thread: {e}")
|
||||
finally:
|
||||
try:
|
||||
loop.close()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Start COB worker in background thread
|
||||
import threading
|
||||
self.cob_thread = threading.Thread(target=start_cob_worker, daemon=True)
|
||||
self.cob_thread.start()
|
||||
|
||||
self.cob_integration_started = True
|
||||
logger.info("COB integration lazy initialization completed")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in lazy COB initialization: {e}")
|
||||
self.cob_integration = None
|
||||
|
||||
def _on_cob_update(self, symbol: str, data: Dict):
|
||||
"""Handle COB data updates (same callback pattern as working dashboard)"""
|
||||
try:
|
||||
# Store latest COB data
|
||||
self.latest_cob_data[symbol] = data
|
||||
self.cob_update_timestamps[symbol] = datetime.now()
|
||||
|
||||
# Provide data to orchestrator models
|
||||
if hasattr(self.orchestrator, '_on_cob_dashboard_data'):
|
||||
self.orchestrator._on_cob_dashboard_data(symbol, data)
|
||||
|
||||
# Provide data to enhanced training system
|
||||
if hasattr(self, 'training_system') and self.training_system:
|
||||
# Add COB snapshot to training system
|
||||
if hasattr(self.training_system, 'real_time_data'):
|
||||
cob_snapshot = {
|
||||
'timestamp': time.time(),
|
||||
'symbol': symbol,
|
||||
'stats': data.get('stats', {}),
|
||||
'levels': len(data.get('bids', [])) + len(data.get('asks', [])),
|
||||
'imbalance': data.get('stats', {}).get('imbalance', 0),
|
||||
'spread_bps': data.get('stats', {}).get('spread_bps', 0)
|
||||
}
|
||||
self.training_system.real_time_data['cob_snapshots'].append(cob_snapshot)
|
||||
|
||||
logger.debug(f"COB update processed: {symbol} - {len(data.get('bids', []))} bids, {len(data.get('asks', []))} asks")
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error processing COB update: {e}")
|
||||
|
||||
def get_cob_data(self, symbol: str) -> Optional[Dict]:
|
||||
"""Get latest COB data for a symbol"""
|
||||
try:
|
||||
return self.latest_cob_data.get(symbol)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error getting COB data: {e}")
|
||||
return None
|
||||
|
||||
def get_cob_statistics(self, symbol: str) -> Optional[Dict]:
|
||||
"""Get COB statistics for a symbol"""
|
||||
try:
|
||||
if symbol in self.latest_cob_data:
|
||||
return self.latest_cob_data[symbol].get('stats', {})
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"Error getting COB statistics: {e}")
|
||||
return None
|
||||
|
||||
def _create_cob_ladder_display(self, symbol: str) -> List:
|
||||
"""Create real COB ladder display showing order book"""
|
||||
try:
|
||||
# Get COB data from the working integration
|
||||
cob_data = self.get_cob_data(symbol)
|
||||
|
||||
if not cob_data:
|
||||
return [
|
||||
html.Div([
|
||||
html.H6(f"{symbol} - COB", className="text-muted mb-2"),
|
||||
html.P("COB data not available", className="text-warning small"),
|
||||
html.P("Initializing connections...", className="text-muted small")
|
||||
])
|
||||
]
|
||||
|
||||
components = []
|
||||
|
||||
# Header with symbol and status
|
||||
components.append(html.Div([
|
||||
html.H6(f"{symbol} - Order Book", className="text-info mb-2"),
|
||||
html.Small(f"Last update: {datetime.now().strftime('%H:%M:%S')}", className="text-muted")
|
||||
]))
|
||||
|
||||
# Get order book data
|
||||
bids = cob_data.get('bids', [])
|
||||
asks = cob_data.get('asks', [])
|
||||
stats = cob_data.get('stats', {})
|
||||
|
||||
# Display key statistics
|
||||
if stats:
|
||||
spread = stats.get('spread_bps', 0)
|
||||
imbalance = stats.get('imbalance', 0)
|
||||
|
||||
components.append(html.Div([
|
||||
html.P([
|
||||
html.Span("Spread: ", className="text-muted small"),
|
||||
html.Span(f"{spread:.1f} bps", className="text-warning small fw-bold")
|
||||
], className="mb-1"),
|
||||
html.P([
|
||||
html.Span("Imbalance: ", className="text-muted small"),
|
||||
html.Span(f"{imbalance:.3f}", className="text-info small fw-bold")
|
||||
], className="mb-2")
|
||||
]))
|
||||
|
||||
# Order book ladder - Asks (top, descending)
|
||||
if asks:
|
||||
components.append(html.Div([
|
||||
html.H6("ASKS", className="text-danger small mb-1"),
|
||||
html.Div([
|
||||
html.Div([
|
||||
html.Span(f"${ask['price']:.2f}", className="text-danger small me-2"),
|
||||
html.Span(f"{ask['size']:.4f}", className="text-muted small")
|
||||
], className="d-flex justify-content-between mb-1")
|
||||
for ask in asks[:5] # Top 5 asks
|
||||
], className="border-start border-danger ps-2 mb-2")
|
||||
]))
|
||||
|
||||
# Current price (mid)
|
||||
if bids and asks:
|
||||
mid_price = (bids[0]['price'] + asks[0]['price']) / 2
|
||||
components.append(html.Div([
|
||||
html.Hr(className="my-1"),
|
||||
html.P([
|
||||
html.Strong(f"${mid_price:.2f}", className="text-primary")
|
||||
], className="text-center mb-1"),
|
||||
html.Hr(className="my-1")
|
||||
]))
|
||||
|
||||
# Order book ladder - Bids (bottom, descending)
|
||||
if bids:
|
||||
components.append(html.Div([
|
||||
html.H6("BIDS", className="text-success small mb-1"),
|
||||
html.Div([
|
||||
html.Div([
|
||||
html.Span(f"${bid['price']:.2f}", className="text-success small me-2"),
|
||||
html.Span(f"{bid['size']:.4f}", className="text-muted small")
|
||||
], className="d-flex justify-content-between mb-1")
|
||||
for bid in bids[:5] # Top 5 bids
|
||||
], className="border-start border-success ps-2")
|
||||
]))
|
||||
|
||||
# Summary stats
|
||||
if bids and asks:
|
||||
total_bid_volume = sum(bid['size'] * bid['price'] for bid in bids[:10])
|
||||
total_ask_volume = sum(ask['size'] * ask['price'] for ask in asks[:10])
|
||||
|
||||
components.append(html.Div([
|
||||
html.Hr(className="my-2"),
|
||||
html.P([
|
||||
html.Span("Bid Vol: ", className="text-muted small"),
|
||||
html.Span(f"${total_bid_volume:,.0f}", className="text-success small")
|
||||
], className="mb-1"),
|
||||
html.P([
|
||||
html.Span("Ask Vol: ", className="text-muted small"),
|
||||
html.Span(f"${total_ask_volume:,.0f}", className="text-danger small")
|
||||
], className="mb-1")
|
||||
]))
|
||||
|
||||
return components
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating COB ladder for {symbol}: {e}")
|
||||
return [
|
||||
html.Div([
|
||||
html.H6(f"{symbol} - COB", className="text-muted mb-2"),
|
||||
html.P(f"Error: {str(e)}", className="text-danger small")
|
||||
])
|
||||
]
|
||||
|
||||
def _initialize_unified_orchestrator_features(self):
|
||||
"""Initialize unified orchestrator features including COB integration"""
|
||||
@ -3068,6 +3252,10 @@ class CleanTradingDashboard:
|
||||
logging.getLogger('werkzeug').setLevel(logging.ERROR)
|
||||
|
||||
logger.info(f"Starting Clean Trading Dashboard at http://{host}:{port}")
|
||||
|
||||
# Start lazy COB integration now that dashboard is running
|
||||
self._start_cob_integration_lazy()
|
||||
|
||||
self.app.run(host=host, port=port, debug=debug, dev_tools_silence_routes_logging=True)
|
||||
|
||||
def stop(self):
|
||||
@ -3479,47 +3667,7 @@ class CleanTradingDashboard:
|
||||
except Exception as e:
|
||||
logger.error(f"Error broadcasting COB update to UI: {e}")
|
||||
|
||||
def get_cob_price_buckets(self, symbol: str) -> List[Dict]:
|
||||
"""Get price buckets for display in dashboard"""
|
||||
try:
|
||||
if symbol not in self.cob_price_buckets:
|
||||
return []
|
||||
|
||||
buckets = self.cob_price_buckets[symbol]
|
||||
|
||||
# Sort buckets by price and return as list
|
||||
sorted_buckets = []
|
||||
for price_key in sorted(buckets.keys(), key=float):
|
||||
bucket = buckets[price_key]
|
||||
if bucket['total_volume'] > 0: # Only return buckets with volume
|
||||
sorted_buckets.append(bucket)
|
||||
|
||||
return sorted_buckets
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting COB price buckets for {symbol}: {e}")
|
||||
return []
|
||||
|
||||
def get_cob_memory_stats(self, symbol: str) -> Dict:
|
||||
"""Get COB memory statistics for debugging"""
|
||||
try:
|
||||
if symbol not in self.cob_memory:
|
||||
return {}
|
||||
|
||||
memory = self.cob_memory[symbol]
|
||||
buffer = self.cob_data_buffer[symbol]
|
||||
|
||||
return {
|
||||
'memory_snapshots': len(memory),
|
||||
'buffer_updates': len(buffer),
|
||||
'total_updates': self.cob_update_count,
|
||||
'last_update': self.last_cob_broadcast.get(symbol, 0),
|
||||
'bucket_count': len(self.cob_price_buckets.get(symbol, {}))
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting COB memory stats: {e}")
|
||||
return {}
|
||||
# REMOVED: Complex COB bucket methods - using simplified real order book display instead
|
||||
|
||||
def _on_cob_cnn_features(self, symbol: str, cob_features: Dict):
|
||||
"""Handle COB features for CNN models (next price prediction)"""
|
||||
|
||||
Reference in New Issue
Block a user