popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix model mappings,dash updates, trading

Browse Source
This commit is contained in:
Dobromir Popov
2025-07-22 15:44:59 +03:00
parent 3e35b9cddb
commit 1a54fb1d56
32 changed files with 6168 additions and 857 deletions
Download Patch File Download Diff File Expand all files Collapse all files

357
web/clean_dashboard.py
View File

@ -88,10 +88,23 @@ except ImportError:
logger.warning("Universal Data Adapter not available")
# Import RL COB trader for 1B parameter model integration
from core.realtime_rl_cob_trader import RealtimeRLCOBTrader, PredictionResult
try:
from core.realtime_rl_cob_trader import RealtimeRLCOBTrader, PredictionResult
REALTIME_RL_AVAILABLE = True
except ImportError:
REALTIME_RL_AVAILABLE = False
logger.warning("Realtime RL COB trader not available")
RealtimeRLCOBTrader = None
PredictionResult = None
# Import overnight training coordinator
from core.overnight_training_coordinator import OvernightTrainingCoordinator
try:
from core.overnight_training_coordinator import OvernightTrainingCoordinator
OVERNIGHT_TRAINING_AVAILABLE = True
except ImportError:
OVERNIGHT_TRAINING_AVAILABLE = False
logger.warning("Overnight training coordinator not available")
OvernightTrainingCoordinator = None
# Single unified orchestrator with full ML capabilities
@ -231,6 +244,19 @@ class CleanTradingDashboard:
# Initialize COB integration with enhanced WebSocket
self._initialize_cob_integration() # Use the working COB integration method
# Subscribe to COB data updates from data provider and start collection
if self.data_provider:
try:
# Start COB collection first
self.data_provider.start_cob_collection()
logger.info("Started COB collection in data provider")
# Then subscribe to updates
self.data_provider.subscribe_to_cob(self._on_cob_data_update)
logger.info("Subscribed to COB data updates from data provider")
except Exception as e:
logger.error(f"Failed to start COB collection or subscribe: {e}")
# Start signal generation loop to ensure continuous trading signals
self._start_signal_generation_loop()
@ -251,6 +277,75 @@ class CleanTradingDashboard:
logger.debug("Clean Trading Dashboard initialized with HIGH-FREQUENCY COB integration and signal generation")
logger.info("🌙 Overnight Training Coordinator ready - call start_overnight_training() to begin")
def _on_cob_data_update(self, symbol: str, cob_data: dict):
"""Handle COB data updates from data provider"""
try:
# Update latest COB data cache
if not hasattr(self, 'latest_cob_data'):
self.latest_cob_data = {}
# Ensure cob_data is a dictionary with the expected structure
if not isinstance(cob_data, dict):
logger.warning(f"Received non-dict COB data for {symbol}: {type(cob_data)}")
# Try to convert to dict if possible
if hasattr(cob_data, '__dict__'):
cob_data = vars(cob_data)
else:
# Create a minimal valid structure
cob_data = {
'symbol': symbol,
'timestamp': datetime.now(),
'stats': {
'mid_price': 0.0,
'imbalance': 0.0,
'imbalance_5s': 0.0,
'imbalance_15s': 0.0,
'imbalance_60s': 0.0
}
}
# Ensure stats is present
if 'stats' not in cob_data:
cob_data['stats'] = {
'mid_price': 0.0,
'imbalance': 0.0,
'imbalance_5s': 0.0,
'imbalance_15s': 0.0,
'imbalance_60s': 0.0
}
self.latest_cob_data[symbol] = cob_data
# Update last update timestamp
if not hasattr(self, 'cob_last_update'):
self.cob_last_update = {}
import time
self.cob_last_update[symbol] = time.time()
# Update current price from COB data
if 'stats' in cob_data and 'mid_price' in cob_data['stats']:
mid_price = cob_data['stats']['mid_price']
if mid_price > 0:
self.current_prices[symbol] = mid_price
# Log successful price update
logger.debug(f"Updated price for {symbol}: ${mid_price:.2f}")
# Store in history for moving average calculations
if not hasattr(self, 'cob_data_history'):
self.cob_data_history = {
'ETH/USDT': deque(maxlen=61),
'BTC/USDT': deque(maxlen=61)
}
if symbol in self.cob_data_history:
self.cob_data_history[symbol].append(cob_data)
logger.debug(f"Updated COB data for {symbol}: mid_price=${cob_data.get('stats', {}).get('mid_price', 0):.2f}")
except Exception as e:
logger.error(f"Error handling COB data update for {symbol}: {e}")
def start_overnight_training(self):
"""Start the overnight training session"""
@ -525,13 +620,17 @@ class CleanTradingDashboard:
# Try to get price from COB data as fallback
if hasattr(self, 'latest_cob_data') and 'ETH/USDT' in self.latest_cob_data:
cob_data = self.latest_cob_data['ETH/USDT']
if 'stats' in cob_data and 'mid_price' in cob_data['stats']:
if isinstance(cob_data, dict) and 'stats' in cob_data and 'mid_price' in cob_data['stats']:
current_price = cob_data['stats']['mid_price']
price_str = f"${current_price:.2f}"
else:
price_str = "Loading..."
# Debug log to help diagnose the issue
logger.debug(f"COB data format issue: {type(cob_data)}, keys: {cob_data.keys() if isinstance(cob_data, dict) else 'N/A'}")
else:
price_str = "Loading..."
# Debug log to help diagnose the issue
logger.debug(f"No COB data available for ETH/USDT. Latest COB data keys: {self.latest_cob_data.keys() if hasattr(self, 'latest_cob_data') else 'N/A'}")
# Calculate session P&L including unrealized P&L from current position
total_session_pnl = self.session_pnl # Start with realized P&L
@ -542,6 +641,34 @@ class CleanTradingDashboard:
size = self.current_position.get('size', 0)
entry_price = self.current_position.get('price', 0)
if entry_price and size > 0:
# Calculate unrealized P&L with current leverage
if side.upper() == 'LONG' or side.upper() == 'BUY':
raw_pnl_per_unit = current_price - entry_price
else: # SHORT or SELL
raw_pnl_per_unit = entry_price - current_price
# Apply current leverage to unrealized P&L
leveraged_unrealized_pnl = raw_pnl_per_unit * size * self.current_leverage
total_session_pnl += leveraged_unrealized_pnlent_position and current_price:ent_position and current_price:ent_position and current_price:ent_position and current_price:ent_position and current_price:
side = self.current_position.get('side', 'UNKNOWN')
size = self.current_position.get('size', 0)
entry_price = self.current_position.get('price', 0)
if entry_price and size > 0:
# Calculate unrealized P&L with current leverage
if side.upper() == 'LONG' or side.upper() == 'BUY':
raw_pnl_per_unit = current_price - entry_price
else: # SHORT or SELL
raw_pnl_per_unit = entry_price - current_price
# Apply current leverage to unrealized P&L
leveraged_unrealized_pnl = raw_pnl_per_unit * size * self.current_leverage
total_session_pnl += leveraged_unrealized_pnlent_position and current_price:
side = self.current_position.get('side', 'UNKNOWN')
size = self.current_position.get('size', 0)
entry_price = self.current_position.get('price', 0)
if entry_price and size > 0:
# Calculate unrealized P&L with current leverage
if side.upper() == 'LONG' or side.upper() == 'BUY':
@ -772,9 +899,24 @@ class CleanTradingDashboard:
eth_snapshot = self._get_cob_snapshot('ETH/USDT')
btc_snapshot = self._get_cob_snapshot('BTC/USDT')
# Debug: Log COB data availability - OPTIMIZED: Less frequent logging
if n % 30 == 0: # Log every 30 seconds to reduce spam and improve performance
logger.info(f"COB Update #{n % 100}: ETH snapshot: {eth_snapshot is not None}, BTC snapshot: {btc_snapshot is not None}")
# Debug: Log COB data availability more frequently to debug the issue
if n % 10 == 0: # Log every 10 seconds to debug
logger.info(f"COB Update #{n}: ETH snapshot: {eth_snapshot is not None}, BTC snapshot: {btc_snapshot is not None}")
# Check data provider COB data directly
if self.data_provider:
eth_cob = self.data_provider.get_latest_cob_data('ETH/USDT')
btc_cob = self.data_provider.get_latest_cob_data('BTC/USDT')
logger.info(f"Data Provider COB: ETH={eth_cob is not None}, BTC={btc_cob is not None}")
if eth_cob:
eth_stats = eth_cob.get('stats', {})
logger.info(f"ETH COB stats: mid_price=${eth_stats.get('mid_price', 0):.2f}")
if btc_cob:
btc_stats = btc_cob.get('stats', {})
logger.info(f"BTC COB stats: mid_price=${btc_stats.get('mid_price', 0):.2f}")
if hasattr(self, 'latest_cob_data'):
eth_data_time = self.cob_last_update.get('ETH/USDT', 0) if hasattr(self, 'cob_last_update') else 0
btc_data_time = self.cob_last_update.get('BTC/USDT', 0) if hasattr(self, 'cob_last_update') else 0
@ -2340,18 +2482,18 @@ class CleanTradingDashboard:
return {'error': str(e), 'cob_status': 'Error Getting Status', 'orchestrator_type': 'Unknown'}
def _get_cob_snapshot(self, symbol: str) -> Optional[Any]:
"""Get COB snapshot for symbol - CENTRALIZED: Use data provider's COB data"""
"""Get COB snapshot for symbol - ENHANCED: Use data provider's WebSocket COB data"""
try:
# Priority 1: Use data provider's centralized COB data (primary source)
# Priority 1: Use data provider's latest COB data (WebSocket or REST)
if self.data_provider:
try:
cob_data = self.data_provider.get_latest_cob_data(symbol)
logger.debug(f"COB data type for {symbol}: {type(cob_data)}, data: {cob_data}")
if cob_data and isinstance(cob_data, dict):
# Validate COB data structure
if 'stats' in cob_data and cob_data['stats']:
logger.debug(f"COB snapshot available for {symbol} from centralized data provider")
stats = cob_data.get('stats', {})
if stats and stats.get('mid_price', 0) > 0:
logger.debug(f"COB snapshot available for {symbol} from data provider")
# Create a snapshot object from the data provider's data
class COBSnapshot:
@ -2381,58 +2523,107 @@ class CleanTradingDashboard:
'total_volume_usd': ask[0] * ask[1]
})
self.stats = data.get('stats', {})
# Add direct attributes for new format compatibility
self.volume_weighted_mid = self.stats.get('mid_price', 0)
self.spread_bps = self.stats.get('spread_bps', 0)
self.liquidity_imbalance = self.stats.get('imbalance', 0)
self.total_bid_liquidity = self.stats.get('bid_liquidity', 0)
self.total_ask_liquidity = self.stats.get('ask_liquidity', 0)
# Use stats from data and calculate liquidity properly
self.stats = stats.copy()
# Calculate total liquidity from order book if not provided
bid_liquidity = stats.get('bid_liquidity', 0) or stats.get('total_bid_liquidity', 0)
ask_liquidity = stats.get('ask_liquidity', 0) or stats.get('total_ask_liquidity', 0)
# If liquidity is still 0, calculate from order book data
if bid_liquidity == 0 and self.consolidated_bids:
bid_liquidity = sum(bid['total_volume_usd'] for bid in self.consolidated_bids)
if ask_liquidity == 0 and self.consolidated_asks:
ask_liquidity = sum(ask['total_volume_usd'] for ask in self.consolidated_asks)
# Update stats with calculated liquidity
self.stats['total_bid_liquidity'] = bid_liquidity
self.stats['total_ask_liquidity'] = ask_liquidity
self.stats['bid_liquidity'] = bid_liquidity
self.stats['ask_liquidity'] = ask_liquidity
# Add direct attributes for compatibility
self.volume_weighted_mid = stats.get('mid_price', 0)
self.spread_bps = stats.get('spread_bps', 0)
self.liquidity_imbalance = stats.get('imbalance', 0)
self.total_bid_liquidity = bid_liquidity
self.total_ask_liquidity = ask_liquidity
self.exchanges_active = ['Binance'] # Default for now
return COBSnapshot(cob_data)
else:
# Data exists but no stats - this is the "Invalid COB data" case
logger.debug(f"COB data for {symbol} missing stats structure: {type(cob_data)}, keys: {list(cob_data.keys()) if isinstance(cob_data, dict) else 'not dict'}")
logger.debug(f"COB data for {symbol} missing valid stats: {stats}")
return None
else:
logger.debug(f"No COB data available for {symbol} from data provider")
logger.debug(f"No valid COB data for {symbol} from data provider")
return None
except Exception as e:
logger.error(f"Error getting COB data from data provider: {e}")
# Priority 2: Use orchestrator's COB integration (secondary source)
if hasattr(self.orchestrator, 'cob_integration') and self.orchestrator.cob_integration:
# Try to get snapshot from orchestrator's COB integration
snapshot = self.orchestrator.cob_integration.get_cob_snapshot(symbol)
if snapshot:
logger.debug(f"COB snapshot available for {symbol} from orchestrator COB integration, type: {type(snapshot)}")
# Check if it's a list (which would cause the error)
if isinstance(snapshot, list):
logger.warning(f"Orchestrator returned list instead of COB snapshot for {symbol}")
# Don't return the list, continue to other sources
else:
return snapshot
# If no snapshot, try to get from orchestrator's cached data
if hasattr(self.orchestrator, 'latest_cob_data') and symbol in self.orchestrator.latest_cob_data:
cob_data = self.orchestrator.latest_cob_data[symbol]
logger.debug(f"COB snapshot available for {symbol} from orchestrator cached data")
# Create a simple snapshot object from the cached data
class COBSnapshot:
def __init__(self, data):
self.consolidated_bids = data.get('bids', [])
self.consolidated_asks = data.get('asks', [])
self.stats = data.get('stats', {})
return COBSnapshot(cob_data)
# Priority 2: Try to get raw WebSocket data directly
if self.data_provider and hasattr(self.data_provider, 'cob_raw_ticks'):
try:
raw_ticks = self.data_provider.get_cob_raw_ticks(symbol, count=1)
if raw_ticks:
latest_tick = raw_ticks[-1]
stats = latest_tick.get('stats', {})
if stats and stats.get('mid_price', 0) > 0:
logger.debug(f"Using raw WebSocket tick for {symbol}")
# Create snapshot from raw tick
class COBSnapshot:
def __init__(self, tick_data):
bids = tick_data.get('bids', [])
asks = tick_data.get('asks', [])
self.consolidated_bids = []
for bid in bids[:20]: # Top 20 levels
self.consolidated_bids.append({
'price': bid['price'],
'size': bid['size'],
'total_size': bid['size'],
'total_volume_usd': bid['price'] * bid['size']
})
self.consolidated_asks = []
for ask in asks[:20]: # Top 20 levels
self.consolidated_asks.append({
'price': ask['price'],
'size': ask['size'],
'total_size': ask['size'],
'total_volume_usd': ask['price'] * ask['size']
})
self.stats = stats
self.volume_weighted_mid = stats.get('mid_price', 0)
self.spread_bps = stats.get('spread_bps', 0)
self.liquidity_imbalance = stats.get('imbalance', 0)
self.total_bid_liquidity = stats.get('bid_volume', 0)
self.total_ask_liquidity = stats.get('ask_volume', 0)
self.exchanges_active = ['Binance']
return COBSnapshot(latest_tick)
except Exception as e:
logger.debug(f"Error getting raw WebSocket data: {e}")
# Priority 3: Use dashboard's cached COB data (last resort fallback)
# Priority 3: Use orchestrator's COB integration (fallback)
if hasattr(self.orchestrator, 'cob_integration') and self.orchestrator.cob_integration:
try:
snapshot = self.orchestrator.cob_integration.get_cob_snapshot(symbol)
if snapshot and not isinstance(snapshot, list):
logger.debug(f"COB snapshot from orchestrator for {symbol}")
return snapshot
except Exception as e:
logger.debug(f"Error getting COB from orchestrator: {e}")
# Priority 4: Use dashboard's cached COB data (last resort)
if symbol in self.latest_cob_data and self.latest_cob_data[symbol]:
cob_data = self.latest_cob_data[symbol]
logger.debug(f"COB snapshot available for {symbol} from dashboard cached data (fallback)")
logger.debug(f"Using dashboard cached COB data for {symbol}")
# Create a simple snapshot object from the cached data
class COBSnapshot:
@ -2460,18 +2651,40 @@ class CleanTradingDashboard:
def _get_cob_mode(self) -> str:
"""Get current COB data collection mode"""
try:
# Check if orchestrator COB integration is working
if hasattr(self.orchestrator, 'cob_integration') and self.orchestrator.cob_integration:
# Try to get a snapshot from orchestrator
snapshot = self.orchestrator.cob_integration.get_cob_snapshot('ETH/USDT')
if snapshot and hasattr(snapshot, 'consolidated_bids') and snapshot.consolidated_bids:
return "WS" # WebSocket/Advanced mode
# Check if data provider has WebSocket COB integration
if self.data_provider and hasattr(self.data_provider, 'cob_websocket'):
# Check WebSocket status
if hasattr(self.data_provider.cob_websocket, 'status'):
eth_status = self.data_provider.cob_websocket.status.get('ETH/USDT')
if eth_status and eth_status.connected:
return "WS" # WebSocket mode
# Check if we have recent WebSocket data
if hasattr(self.data_provider, 'cob_raw_ticks'):
eth_ticks = self.data_provider.cob_raw_ticks.get('ETH/USDT', [])
if eth_ticks:
import time
latest_tick = eth_ticks[-1]
tick_time = latest_tick.get('timestamp', 0)
if isinstance(tick_time, (int, float)) and (time.time() - tick_time) < 10:
return "WS" # Recent WebSocket data
# Check if fallback data is available
# Check if we have any COB data (REST fallback)
if hasattr(self, 'latest_cob_data') and 'ETH/USDT' in self.latest_cob_data:
if self.latest_cob_data['ETH/USDT']:
return "REST" # REST API fallback mode
# Check data provider cache
if self.data_provider:
latest_cob = self.data_provider.get_latest_cob_data('ETH/USDT')
if latest_cob and latest_cob.get('stats', {}).get('mid_price', 0) > 0:
# Check source to determine mode
source = latest_cob.get('source', 'unknown')
if 'websocket' in source.lower() or 'enhanced' in source.lower():
return "WS"
else:
return "REST"
return "None" # No data available
except Exception as e:
@ -6313,16 +6526,26 @@ class CleanTradingDashboard:
"""Connect to orchestrator for real trading signals"""
try:
if self.orchestrator and hasattr(self.orchestrator, 'add_decision_callback'):
def connect_worker():
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(self.orchestrator.add_decision_callback(self._on_trading_decision))
logger.info("Successfully connected to orchestrator for trading signals.")
except Exception as e:
logger.error(f"Orchestrator connection worker failed: {e}")
thread = threading.Thread(target=connect_worker, daemon=True)
thread.start()
# Directly add the callback to the orchestrator's decision_callbacks list
# This is a simpler approach that avoids async/threading issues
if hasattr(self.orchestrator, 'decision_callbacks'):
if self._on_trading_decision not in self.orchestrator.decision_callbacks:
self.orchestrator.decision_callbacks.append(self._on_trading_decision)
logger.info("Successfully connected to orchestrator for trading signals (direct method).")
else:
logger.info("Trading decision callback already registered.")
else:
# Fallback to async method if needed
def connect_worker():
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(self.orchestrator.add_decision_callback(self._on_trading_decision))
logger.info("Successfully connected to orchestrator for trading signals (async method).")
except Exception as e:
logger.error(f"Orchestrator connection worker failed: {e}")
thread = threading.Thread(target=connect_worker, daemon=True)
thread.start()
else:
logger.warning("Orchestrator not available or doesn't support callbacks")
except Exception as e: