improve training and model data
This commit is contained in:
Dobromir Popov
@ -504,13 +504,23 @@ class CleanTradingDashboard:
|
||||
def update_cob_data(n):
|
||||
"""Update COB data displays with real order book ladders and cumulative stats"""
|
||||
try:
|
||||
# Update less frequently to reduce flickering
|
||||
if n % self.update_batch_interval != 0:
|
||||
raise PreventUpdate
|
||||
# COB data is critical - update every second (no batching)
|
||||
# if n % self.update_batch_interval != 0:
|
||||
# raise PreventUpdate
|
||||
|
||||
eth_snapshot = self._get_cob_snapshot('ETH/USDT')
|
||||
btc_snapshot = self._get_cob_snapshot('BTC/USDT')
|
||||
|
||||
# Debug: Log COB data availability
|
||||
if n % 5 == 0: # Log every 5 seconds to avoid spam
|
||||
logger.info(f"COB Update #{n}: ETH snapshot: {eth_snapshot is not None}, BTC snapshot: {btc_snapshot is not None}")
|
||||
if hasattr(self, 'latest_cob_data'):
|
||||
eth_data_time = self.cob_last_update.get('ETH/USDT', 0) if hasattr(self, 'cob_last_update') else 0
|
||||
btc_data_time = self.cob_last_update.get('BTC/USDT', 0) if hasattr(self, 'cob_last_update') else 0
|
||||
import time
|
||||
current_time = time.time()
|
||||
logger.info(f"COB Data Age: ETH: {current_time - eth_data_time:.1f}s, BTC: {current_time - btc_data_time:.1f}s")
|
||||
|
||||
eth_imbalance_stats = self._calculate_cumulative_imbalance('ETH/USDT')
|
||||
btc_imbalance_stats = self._calculate_cumulative_imbalance('BTC/USDT')
|
||||
|
||||
@ -1155,27 +1165,11 @@ class CleanTradingDashboard:
|
||||
def _add_cob_rl_predictions_to_chart(self, fig: go.Figure, symbol: str, df_main: pd.DataFrame, row: int = 1):
|
||||
"""Add COB_RL microstructure predictions as diamond markers"""
|
||||
try:
|
||||
# Get recent COB_RL predictions (simulated for now since model is FRESH)
|
||||
current_time = datetime.now()
|
||||
current_price = self._get_current_price(symbol) or 3500.0
|
||||
# Get real COB_RL predictions from orchestrator or enhanced training system
|
||||
cob_predictions = self._get_real_cob_rl_predictions(symbol)
|
||||
|
||||
# Generate sample COB_RL predictions for visualization
|
||||
cob_predictions = []
|
||||
for i in range(10): # Generate 10 sample predictions over last 5 minutes
|
||||
pred_time = current_time - timedelta(minutes=i * 0.5)
|
||||
price_variation = (i % 3 - 1) * 2.0 # Small price variations
|
||||
|
||||
# Simulate COB_RL predictions based on microstructure analysis
|
||||
direction = (i % 3) # 0=DOWN, 1=SIDEWAYS, 2=UP
|
||||
confidence = 0.65 + (i % 4) * 0.08 # Varying confidence
|
||||
|
||||
cob_predictions.append({
|
||||
'timestamp': pred_time,
|
||||
'direction': direction,
|
||||
'confidence': confidence,
|
||||
'price': current_price + price_variation,
|
||||
'microstructure_signal': ['SELL_PRESSURE', 'BALANCED', 'BUY_PRESSURE'][direction]
|
||||
})
|
||||
if not cob_predictions:
|
||||
return # No real predictions to display
|
||||
|
||||
# Separate predictions by direction
|
||||
up_predictions = [p for p in cob_predictions if p['direction'] == 2]
|
||||
@ -1346,6 +1340,61 @@ class CleanTradingDashboard:
|
||||
except Exception as e:
|
||||
logger.debug(f"Error adding prediction accuracy feedback to chart: {e}")
|
||||
|
||||
def _get_real_cob_rl_predictions(self, symbol: str) -> List[Dict]:
|
||||
"""Get real COB RL predictions from the model"""
|
||||
try:
|
||||
cob_predictions = []
|
||||
|
||||
# Get predictions from enhanced training system
|
||||
if hasattr(self, 'enhanced_training_system') and self.enhanced_training_system:
|
||||
if hasattr(self.enhanced_training_system, 'get_prediction_summary'):
|
||||
summary = self.enhanced_training_system.get_prediction_summary(symbol)
|
||||
if summary and 'cob_rl_predictions' in summary:
|
||||
raw_predictions = summary['cob_rl_predictions'][-10:] # Last 10 predictions
|
||||
for pred in raw_predictions:
|
||||
if 'timestamp' in pred and 'direction' in pred:
|
||||
cob_predictions.append({
|
||||
'timestamp': pred['timestamp'],
|
||||
'direction': pred['direction'],
|
||||
'confidence': pred.get('confidence', 0.5),
|
||||
'price': pred.get('price', self._get_current_price(symbol) or 3500.0),
|
||||
'microstructure_signal': pred.get('signal', ['SELL_PRESSURE', 'BALANCED', 'BUY_PRESSURE'][pred['direction']])
|
||||
})
|
||||
|
||||
# Fallback to orchestrator COB RL agent predictions
|
||||
if not cob_predictions and self.orchestrator:
|
||||
if hasattr(self.orchestrator, 'cob_rl_agent') and self.orchestrator.cob_rl_agent:
|
||||
agent = self.orchestrator.cob_rl_agent
|
||||
# Check if agent has recent predictions stored
|
||||
if hasattr(agent, 'recent_predictions'):
|
||||
for pred in agent.recent_predictions[-10:]:
|
||||
cob_predictions.append({
|
||||
'timestamp': pred.get('timestamp', datetime.now()),
|
||||
'direction': pred.get('action', 1), # 0=SELL, 1=HOLD, 2=BUY
|
||||
'confidence': pred.get('confidence', 0.5),
|
||||
'price': pred.get('price', self._get_current_price(symbol) or 3500.0),
|
||||
'microstructure_signal': ['SELL_PRESSURE', 'BALANCED', 'BUY_PRESSURE'][pred.get('action', 1)]
|
||||
})
|
||||
|
||||
# Alternative: Try getting predictions from RL agent (DQN can handle COB features)
|
||||
elif hasattr(self.orchestrator, 'rl_agent') and self.orchestrator.rl_agent:
|
||||
agent = self.orchestrator.rl_agent
|
||||
if hasattr(agent, 'recent_predictions'):
|
||||
for pred in agent.recent_predictions[-10:]:
|
||||
cob_predictions.append({
|
||||
'timestamp': pred.get('timestamp', datetime.now()),
|
||||
'direction': pred.get('action', 1),
|
||||
'confidence': pred.get('confidence', 0.5),
|
||||
'price': pred.get('price', self._get_current_price(symbol) or 3500.0),
|
||||
'microstructure_signal': ['SELL_PRESSURE', 'BALANCED', 'BUY_PRESSURE'][pred.get('action', 1)]
|
||||
})
|
||||
|
||||
return cob_predictions
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error getting real COB RL predictions: {e}")
|
||||
return []
|
||||
|
||||
def _get_recent_dqn_predictions(self, symbol: str) -> List[Dict]:
|
||||
"""Get recent DQN predictions from orchestrator with sample generation"""
|
||||
try:
|
||||
@ -5202,12 +5251,24 @@ class CleanTradingDashboard:
|
||||
logger.error(f"Error updating session metrics: {e}")
|
||||
|
||||
def _start_actual_training_if_needed(self):
|
||||
"""Connect to centralized training system in orchestrator (following architecture)"""
|
||||
"""Connect to centralized training system in orchestrator and start training"""
|
||||
try:
|
||||
if not self.orchestrator:
|
||||
logger.warning("No orchestrator available for training connection")
|
||||
return
|
||||
|
||||
logger.info("DASHBOARD: Connected to orchestrator's centralized training system")
|
||||
|
||||
# Actually start the orchestrator's enhanced training system
|
||||
if hasattr(self.orchestrator, 'start_enhanced_training'):
|
||||
training_started = self.orchestrator.start_enhanced_training()
|
||||
if training_started:
|
||||
logger.info("TRAINING: Orchestrator enhanced training system started successfully")
|
||||
else:
|
||||
logger.warning("TRAINING: Failed to start orchestrator enhanced training system")
|
||||
else:
|
||||
logger.warning("TRAINING: Orchestrator does not have enhanced training system")
|
||||
|
||||
# Dashboard only displays training status - actual training happens in orchestrator
|
||||
# Training is centralized in the orchestrator as per architecture design
|
||||
except Exception as e:
|
||||
@ -5971,30 +6032,7 @@ class CleanTradingDashboard:
|
||||
cob_rl_agent = self.orchestrator.cob_rl_agent
|
||||
|
||||
if not cob_rl_agent:
|
||||
# Create a simple checkpoint to prevent recreation if no agent available
|
||||
try:
|
||||
from utils.checkpoint_manager import save_checkpoint
|
||||
checkpoint_data = {
|
||||
'model_state_dict': {},
|
||||
'training_samples': len(market_data),
|
||||
'cob_features_processed': True
|
||||
}
|
||||
performance_metrics = {
|
||||
'loss': 0.356,
|
||||
'training_samples': len(market_data),
|
||||
'model_parameters': 0
|
||||
}
|
||||
metadata = save_checkpoint(
|
||||
model=checkpoint_data,
|
||||
model_name="cob_rl",
|
||||
model_type="cob_rl",
|
||||
performance_metrics=performance_metrics,
|
||||
training_metadata={'cob_data_processed': True}
|
||||
)
|
||||
if metadata:
|
||||
logger.info(f"COB RL placeholder checkpoint saved: {metadata.checkpoint_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error saving COB RL placeholder checkpoint: {e}")
|
||||
logger.debug("No COB RL agent available for training")
|
||||
return
|
||||
|
||||
# Perform actual COB RL training
|
||||
|
||||
@ -286,11 +286,11 @@ class DashboardComponentManager:
|
||||
if hasattr(cob_snapshot, 'stats'):
|
||||
# Old format with stats attribute
|
||||
stats = cob_snapshot.stats
|
||||
mid_price = stats.get('mid_price', 0)
|
||||
spread_bps = stats.get('spread_bps', 0)
|
||||
imbalance = stats.get('imbalance', 0)
|
||||
bids = getattr(cob_snapshot, 'consolidated_bids', [])
|
||||
asks = getattr(cob_snapshot, 'consolidated_asks', [])
|
||||
mid_price = stats.get('mid_price', 0)
|
||||
spread_bps = stats.get('spread_bps', 0)
|
||||
imbalance = stats.get('imbalance', 0)
|
||||
bids = getattr(cob_snapshot, 'consolidated_bids', [])
|
||||
asks = getattr(cob_snapshot, 'consolidated_asks', [])
|
||||
else:
|
||||
# New COBSnapshot format with direct attributes
|
||||
mid_price = getattr(cob_snapshot, 'volume_weighted_mid', 0)
|
||||
@ -421,10 +421,10 @@ class DashboardComponentManager:
|
||||
volume_usd = order.total_volume_usd
|
||||
else:
|
||||
# Dictionary format (legacy)
|
||||
price = order.get('price', 0)
|
||||
# Handle both old format (size) and new format (total_size)
|
||||
size = order.get('total_size', order.get('size', 0))
|
||||
volume_usd = order.get('total_volume_usd', size * price)
|
||||
price = order.get('price', 0)
|
||||
# Handle both old format (size) and new format (total_size)
|
||||
size = order.get('total_size', order.get('size', 0))
|
||||
volume_usd = order.get('total_volume_usd', size * price)
|
||||
|
||||
if price > 0:
|
||||
bucket_key = round(price / bucket_size) * bucket_size
|
||||
|
||||
Reference in New Issue
Block a user