inf/trn toggles UI
This commit is contained in:
Dobromir Popov
@ -1433,17 +1433,81 @@ class TradingOrchestrator:
|
||||
return self.model_toggle_states.get(model_name, {"inference_enabled": True, "training_enabled": True})
|
||||
|
||||
def set_model_toggle_state(self, model_name: str, inference_enabled: bool = None, training_enabled: bool = None):
|
||||
"""Set toggle state for a model"""
|
||||
"""Set toggle state for a model - Universal handler for any model"""
|
||||
# Initialize model toggle state if it doesn't exist
|
||||
if model_name not in self.model_toggle_states:
|
||||
self.model_toggle_states[model_name] = {"inference_enabled": True, "training_enabled": True}
|
||||
logger.info(f"Initialized toggle state for new model: {model_name}")
|
||||
|
||||
# Update the toggle states
|
||||
if inference_enabled is not None:
|
||||
self.model_toggle_states[model_name]["inference_enabled"] = inference_enabled
|
||||
if training_enabled is not None:
|
||||
self.model_toggle_states[model_name]["training_enabled"] = training_enabled
|
||||
|
||||
# Save the updated state
|
||||
self._save_ui_state()
|
||||
|
||||
# Log the change
|
||||
logger.info(f"Model {model_name} toggle state updated: inference={self.model_toggle_states[model_name]['inference_enabled']}, training={self.model_toggle_states[model_name]['training_enabled']}")
|
||||
|
||||
# Notify any listeners about the toggle change
|
||||
self._notify_model_toggle_change(model_name, self.model_toggle_states[model_name])
|
||||
|
||||
def _notify_model_toggle_change(self, model_name: str, toggle_state: Dict[str, bool]):
|
||||
"""Notify components about model toggle changes"""
|
||||
try:
|
||||
# This can be extended to notify other components
|
||||
# For now, just log the change
|
||||
logger.debug(f"Model toggle change notification: {model_name} -> {toggle_state}")
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error notifying model toggle change: {e}")
|
||||
|
||||
def register_model_dynamically(self, model_name: str, model_interface):
|
||||
"""Register a new model dynamically and set up its toggle state"""
|
||||
try:
|
||||
# Register with model registry
|
||||
if self.model_registry.register_model(model_interface):
|
||||
# Initialize toggle state for the new model
|
||||
if model_name not in self.model_toggle_states:
|
||||
self.model_toggle_states[model_name] = {
|
||||
"inference_enabled": True,
|
||||
"training_enabled": True
|
||||
}
|
||||
logger.info(f"Registered new model dynamically: {model_name}")
|
||||
self._save_ui_state()
|
||||
return True
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error registering model {model_name} dynamically: {e}")
|
||||
return False
|
||||
|
||||
def get_all_registered_models(self):
|
||||
"""Get all registered models from registry and toggle states"""
|
||||
try:
|
||||
all_models = {}
|
||||
|
||||
# Get models from registry
|
||||
if hasattr(self, 'model_registry') and self.model_registry:
|
||||
registry_models = self.model_registry.get_all_models()
|
||||
all_models.update(registry_models)
|
||||
|
||||
# Add any models that have toggle states but aren't in registry
|
||||
for model_name in self.model_toggle_states.keys():
|
||||
if model_name not in all_models:
|
||||
all_models[model_name] = {
|
||||
'name': model_name,
|
||||
'type': 'toggle_only',
|
||||
'registered': False
|
||||
}
|
||||
|
||||
return all_models
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting all registered models: {e}")
|
||||
return {}
|
||||
|
||||
def is_model_inference_enabled(self, model_name: str) -> bool:
|
||||
"""Check if model inference is enabled"""
|
||||
@ -5371,18 +5435,21 @@ class TradingOrchestrator:
|
||||
if 'model_state_dict' in checkpoint:
|
||||
self.decision_fusion_network.load_state_dict(checkpoint['model_state_dict'])
|
||||
|
||||
# Update model states
|
||||
self.model_states["decision"]["initial_loss"] = (
|
||||
# Update model states - FIX: Use correct key "decision_fusion"
|
||||
if "decision_fusion" not in self.model_states:
|
||||
self.model_states["decision_fusion"] = {}
|
||||
|
||||
self.model_states["decision_fusion"]["initial_loss"] = (
|
||||
metadata.performance_metrics.get("loss", 0.0)
|
||||
)
|
||||
self.model_states["decision"]["current_loss"] = (
|
||||
self.model_states["decision_fusion"]["current_loss"] = (
|
||||
metadata.performance_metrics.get("loss", 0.0)
|
||||
)
|
||||
self.model_states["decision"]["best_loss"] = (
|
||||
self.model_states["decision_fusion"]["best_loss"] = (
|
||||
metadata.performance_metrics.get("loss", 0.0)
|
||||
)
|
||||
self.model_states["decision"]["checkpoint_loaded"] = True
|
||||
self.model_states["decision"][
|
||||
self.model_states["decision_fusion"]["checkpoint_loaded"] = True
|
||||
self.model_states["decision_fusion"][
|
||||
"checkpoint_filename"
|
||||
] = metadata.checkpoint_id
|
||||
|
||||
@ -5398,8 +5465,15 @@ class TradingOrchestrator:
|
||||
logger.warning(f"Error loading decision fusion checkpoint: {e}")
|
||||
logger.info("Decision fusion network starting fresh")
|
||||
|
||||
# Initialize optimizer for decision fusion training
|
||||
self.decision_fusion_optimizer = torch.optim.Adam(
|
||||
self.decision_fusion_network.parameters(),
|
||||
lr=decision_fusion_config.get("learning_rate", 0.001)
|
||||
)
|
||||
|
||||
logger.info(f"Decision fusion network initialized on device: {self.device}")
|
||||
logger.info(f"Decision fusion mode: {self.decision_fusion_mode}")
|
||||
logger.info(f"Decision fusion optimizer initialized with lr={decision_fusion_config.get('learning_rate', 0.001)}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Decision fusion initialization failed: {e}")
|
||||
@ -5454,6 +5528,64 @@ class TradingOrchestrator:
|
||||
except Exception as e:
|
||||
logger.error(f"Error training decision fusion in programmatic mode: {e}")
|
||||
|
||||
def _save_decision_fusion_checkpoint(self):
|
||||
"""Save decision fusion model checkpoint"""
|
||||
try:
|
||||
if not self.decision_fusion_network or not self.checkpoint_manager:
|
||||
return
|
||||
|
||||
# Get current performance score
|
||||
model_stats = self.model_statistics.get('decision_fusion')
|
||||
performance_score = 0.5 # Default score
|
||||
|
||||
if model_stats and model_stats.accuracy is not None:
|
||||
performance_score = model_stats.accuracy
|
||||
elif hasattr(self, 'decision_fusion_performance_score'):
|
||||
performance_score = self.decision_fusion_performance_score
|
||||
|
||||
# Create checkpoint data
|
||||
checkpoint_data = {
|
||||
'model_state_dict': self.decision_fusion_network.state_dict(),
|
||||
'optimizer_state_dict': self.decision_fusion_optimizer.state_dict() if hasattr(self, 'decision_fusion_optimizer') else None,
|
||||
'epoch': self.decision_fusion_decisions_count,
|
||||
'loss': 1.0 - performance_score, # Convert performance to loss
|
||||
'performance_score': performance_score,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'model_name': 'decision_fusion',
|
||||
'training_data_count': len(self.decision_fusion_training_data)
|
||||
}
|
||||
|
||||
# Save checkpoint using checkpoint manager
|
||||
checkpoint_path = self.checkpoint_manager.save_model_checkpoint(
|
||||
model_name="decision_fusion",
|
||||
model_data=checkpoint_data,
|
||||
loss=1.0 - performance_score,
|
||||
performance_score=performance_score
|
||||
)
|
||||
|
||||
if checkpoint_path:
|
||||
logger.info(f"Decision fusion checkpoint saved: {checkpoint_path}")
|
||||
|
||||
# Update model state
|
||||
if 'decision_fusion' not in self.model_states:
|
||||
self.model_states['decision_fusion'] = {}
|
||||
|
||||
self.model_states['decision_fusion'].update({
|
||||
'checkpoint_loaded': True,
|
||||
'checkpoint_filename': checkpoint_path.name if hasattr(checkpoint_path, 'name') else str(checkpoint_path),
|
||||
'current_loss': 1.0 - performance_score,
|
||||
'best_loss': min(self.model_states['decision_fusion'].get('best_loss', float('inf')), 1.0 - performance_score),
|
||||
'last_training': datetime.now(),
|
||||
'performance_score': performance_score
|
||||
})
|
||||
|
||||
logger.info(f"Decision fusion model state updated with checkpoint info")
|
||||
else:
|
||||
logger.warning("Failed to save decision fusion checkpoint")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error saving decision fusion checkpoint: {e}")
|
||||
|
||||
def _create_decision_fusion_input(
|
||||
self,
|
||||
symbol: str,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"model_toggle_states": {
|
||||
"dqn": {
|
||||
"inference_enabled": true,
|
||||
"inference_enabled": false,
|
||||
"training_enabled": true
|
||||
},
|
||||
"cnn": {
|
||||
@ -21,5 +21,5 @@
|
||||
"training_enabled": true
|
||||
}
|
||||
},
|
||||
"timestamp": "2025-07-29T15:16:02.752760"
|
||||
"timestamp": "2025-07-29T15:48:22.223668"
|
||||
}
|
||||
@ -252,16 +252,9 @@ class CleanTradingDashboard:
|
||||
# Live balance caching for real-time portfolio updates
|
||||
self._cached_live_balance: float = 0.0
|
||||
|
||||
# ENHANCED: Model control toggles - separate inference and training
|
||||
# Initialize with defaults, will be updated from orchestrator's persisted state
|
||||
self.dqn_inference_enabled = True # Default: enabled
|
||||
self.dqn_training_enabled = True # Default: enabled
|
||||
self.cnn_inference_enabled = True
|
||||
self.cnn_training_enabled = True
|
||||
self.cob_rl_inference_enabled = True # Default: enabled
|
||||
self.cob_rl_training_enabled = True # Default: enabled
|
||||
self.decision_fusion_inference_enabled = True # Default: enabled
|
||||
self.decision_fusion_training_enabled = True # Default: enabled
|
||||
# ENHANCED: Dynamic model control toggles - works with any model
|
||||
# Model toggle states are now managed dynamically through orchestrator
|
||||
self.model_toggle_states = {} # Dynamic storage for model toggle states
|
||||
|
||||
# Load persisted UI state from orchestrator
|
||||
self._sync_ui_state_from_orchestrator()
|
||||
@ -785,6 +778,123 @@ class CleanTradingDashboard:
|
||||
"""Setup the dashboard layout using layout manager"""
|
||||
self.app.layout = self.layout_manager.create_main_layout()
|
||||
|
||||
def _setup_universal_model_callbacks(self):
|
||||
"""Setup universal model toggle callbacks that work with any model in the registry"""
|
||||
try:
|
||||
# Get all available models from orchestrator's model registry
|
||||
available_models = self._get_available_models()
|
||||
|
||||
logger.info(f"Setting up universal callbacks for {len(available_models)} models: {list(available_models.keys())}")
|
||||
|
||||
# Create callbacks for each model dynamically
|
||||
for model_name in available_models.keys():
|
||||
self._create_model_toggle_callbacks(model_name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error setting up universal model callbacks: {e}")
|
||||
|
||||
def _get_available_models(self):
|
||||
"""Get all available models from orchestrator and model registry"""
|
||||
available_models = {}
|
||||
|
||||
try:
|
||||
# Get models from orchestrator's model registry
|
||||
if self.orchestrator and hasattr(self.orchestrator, 'model_registry'):
|
||||
registry_models = self.orchestrator.model_registry.get_all_models()
|
||||
available_models.update(registry_models)
|
||||
logger.debug(f"Found {len(registry_models)} models in orchestrator registry")
|
||||
|
||||
# Get models from orchestrator's toggle states (includes all known models)
|
||||
if self.orchestrator and hasattr(self.orchestrator, 'model_toggle_states'):
|
||||
toggle_models = self.orchestrator.model_toggle_states.keys()
|
||||
for model_name in toggle_models:
|
||||
if model_name not in available_models:
|
||||
available_models[model_name] = {'name': model_name, 'type': 'unknown'}
|
||||
logger.debug(f"Found {len(toggle_models)} models in toggle states")
|
||||
|
||||
# Fallback: Add known models if none found
|
||||
if not available_models:
|
||||
fallback_models = ['dqn', 'cnn', 'cob_rl', 'decision_fusion', 'transformer']
|
||||
for model_name in fallback_models:
|
||||
available_models[model_name] = {'name': model_name, 'type': 'fallback'}
|
||||
logger.warning(f"Using fallback models: {fallback_models}")
|
||||
|
||||
return available_models
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting available models: {e}")
|
||||
# Return fallback models
|
||||
return {
|
||||
'dqn': {'name': 'dqn', 'type': 'fallback'},
|
||||
'cnn': {'name': 'cnn', 'type': 'fallback'},
|
||||
'cob_rl': {'name': 'cob_rl', 'type': 'fallback'},
|
||||
'decision_fusion': {'name': 'decision_fusion', 'type': 'fallback'},
|
||||
'transformer': {'name': 'transformer', 'type': 'fallback'}
|
||||
}
|
||||
|
||||
def _create_model_toggle_callbacks(self, model_name):
|
||||
"""Create inference and training toggle callbacks for a specific model"""
|
||||
try:
|
||||
# Create inference toggle callback
|
||||
@self.app.callback(
|
||||
Output(f'{model_name}-inference-toggle', 'value'),
|
||||
[Input(f'{model_name}-inference-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_model_inference_toggle(value):
|
||||
return self._handle_model_toggle(model_name, 'inference', value)
|
||||
|
||||
# Create training toggle callback
|
||||
@self.app.callback(
|
||||
Output(f'{model_name}-training-toggle', 'value'),
|
||||
[Input(f'{model_name}-training-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_model_training_toggle(value):
|
||||
return self._handle_model_toggle(model_name, 'training', value)
|
||||
|
||||
logger.debug(f"Created toggle callbacks for model: {model_name}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating callbacks for model {model_name}: {e}")
|
||||
|
||||
def _handle_model_toggle(self, model_name, toggle_type, value):
|
||||
"""Universal handler for model toggle changes"""
|
||||
try:
|
||||
enabled = bool(value and len(value) > 0) # Convert list to boolean
|
||||
|
||||
if self.orchestrator:
|
||||
# Update orchestrator toggle state
|
||||
if toggle_type == 'inference':
|
||||
self.orchestrator.set_model_toggle_state(model_name, inference_enabled=enabled)
|
||||
elif toggle_type == 'training':
|
||||
self.orchestrator.set_model_toggle_state(model_name, training_enabled=enabled)
|
||||
|
||||
logger.info(f"Model {model_name} {toggle_type} toggle: {enabled}")
|
||||
|
||||
# Update dashboard state variables for backward compatibility
|
||||
self._update_dashboard_state_variable(model_name, toggle_type, enabled)
|
||||
|
||||
return value
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error handling toggle for {model_name} {toggle_type}: {e}")
|
||||
return value
|
||||
|
||||
def _update_dashboard_state_variable(self, model_name, toggle_type, enabled):
|
||||
"""Update dashboard state variables for backward compatibility"""
|
||||
try:
|
||||
# Map model names to dashboard state variables
|
||||
state_var_name = f"{model_name}_{toggle_type}_enabled"
|
||||
|
||||
# Set the state variable if it exists
|
||||
if hasattr(self, state_var_name):
|
||||
setattr(self, state_var_name, enabled)
|
||||
logger.debug(f"Updated dashboard state: {state_var_name} = {enabled}")
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error updating dashboard state variable: {e}")
|
||||
|
||||
def _setup_callbacks(self):
|
||||
"""Setup dashboard callbacks"""
|
||||
|
||||
@ -1280,11 +1390,32 @@ class CleanTradingDashboard:
|
||||
def handle_store_models(n_clicks):
|
||||
"""Handle store all models button click"""
|
||||
if n_clicks:
|
||||
success = self._store_all_models()
|
||||
if success:
|
||||
return [html.I(className="fas fa-save me-1"), "Models Stored"]
|
||||
else:
|
||||
return [html.I(className="fas fa-exclamation-triangle me-1"), "Store Failed"]
|
||||
try:
|
||||
success = self._store_all_models()
|
||||
if success:
|
||||
# Check if all models were successfully stored and verified
|
||||
stored_count = 0
|
||||
verified_count = 0
|
||||
|
||||
# Count stored models by checking model states
|
||||
if self.orchestrator:
|
||||
for model_key in ['dqn', 'cnn', 'cob_rl', 'decision_fusion']:
|
||||
if (model_key in self.orchestrator.model_states and
|
||||
self.orchestrator.model_states[model_key].get('session_stored', False)):
|
||||
stored_count += 1
|
||||
if self.orchestrator.model_states[model_key].get('checkpoint_loaded', False):
|
||||
verified_count += 1
|
||||
|
||||
if stored_count > 0:
|
||||
return [html.I(className="fas fa-check-circle me-1 text-success"),
|
||||
f"Stored & Verified ({stored_count}/{verified_count})"]
|
||||
else:
|
||||
return [html.I(className="fas fa-save me-1 text-success"), "Models Stored"]
|
||||
else:
|
||||
return [html.I(className="fas fa-exclamation-triangle me-1 text-warning"), "Store Failed"]
|
||||
except Exception as e:
|
||||
logger.error(f"Error in store models callback: {e}")
|
||||
return [html.I(className="fas fa-times me-1 text-danger"), "Error"]
|
||||
return [html.I(className="fas fa-save me-1"), "Store All Models"]
|
||||
|
||||
# Trading Mode Toggle
|
||||
@ -1344,171 +1475,16 @@ class CleanTradingDashboard:
|
||||
self.cold_start_enabled = False
|
||||
return "Cold Start: OFF", "badge bg-secondary"
|
||||
|
||||
# Model toggle callbacks
|
||||
@self.app.callback(
|
||||
Output('dqn-inference-toggle', 'value'),
|
||||
[Input('dqn-inference-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_dqn_inference_toggle(value):
|
||||
if self.orchestrator:
|
||||
enabled = bool(value and len(value) > 0) # Convert list to boolean
|
||||
self.orchestrator.set_model_toggle_state("dqn", inference_enabled=enabled)
|
||||
# Update dashboard state variable
|
||||
self.dqn_inference_enabled = enabled
|
||||
logger.info(f"DQN inference toggle: {enabled}")
|
||||
return value
|
||||
# Universal Model Toggle Callbacks - Dynamic for all models
|
||||
self._setup_universal_model_callbacks()
|
||||
|
||||
# Cold Start Toggle Callback (proper function definition)
|
||||
@self.app.callback(
|
||||
Output('dqn-training-toggle', 'value'),
|
||||
[Input('dqn-training-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
Output('cold-start-display', 'children'),
|
||||
Output('cold-start-display', 'className'),
|
||||
[Input('cold-start-switch', 'value')]
|
||||
)
|
||||
def update_dqn_training_toggle(value):
|
||||
if self.orchestrator:
|
||||
enabled = bool(value and len(value) > 0) # Convert list to boolean
|
||||
self.orchestrator.set_model_toggle_state("dqn", training_enabled=enabled)
|
||||
# Update dashboard state variable
|
||||
self.dqn_training_enabled = enabled
|
||||
logger.info(f"DQN training toggle: {enabled}")
|
||||
return value
|
||||
|
||||
@self.app.callback(
|
||||
Output('cnn-inference-toggle', 'value'),
|
||||
[Input('cnn-inference-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_cnn_inference_toggle(value):
|
||||
if self.orchestrator:
|
||||
enabled = bool(value and len(value) > 0) # Convert list to boolean
|
||||
self.orchestrator.set_model_toggle_state("cnn", inference_enabled=enabled)
|
||||
# Update dashboard state variable
|
||||
self.cnn_inference_enabled = enabled
|
||||
logger.info(f"CNN inference toggle: {enabled}")
|
||||
return value
|
||||
|
||||
@self.app.callback(
|
||||
Output('cnn-training-toggle', 'value'),
|
||||
[Input('cnn-training-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_cnn_training_toggle(value):
|
||||
if self.orchestrator:
|
||||
enabled = bool(value and len(value) > 0) # Convert list to boolean
|
||||
self.orchestrator.set_model_toggle_state("cnn", training_enabled=enabled)
|
||||
# Update dashboard state variable
|
||||
self.cnn_training_enabled = enabled
|
||||
logger.info(f"CNN training toggle: {enabled}")
|
||||
return value
|
||||
|
||||
@self.app.callback(
|
||||
Output('cob-rl-inference-toggle', 'value'),
|
||||
[Input('cob-rl-inference-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_cob_rl_inference_toggle(value):
|
||||
if self.orchestrator:
|
||||
enabled = bool(value and len(value) > 0) # Convert list to boolean
|
||||
self.orchestrator.set_model_toggle_state("cob_rl", inference_enabled=enabled)
|
||||
# Update dashboard state variable
|
||||
self.cob_rl_inference_enabled = enabled
|
||||
logger.info(f"COB RL inference toggle: {enabled}")
|
||||
return value
|
||||
|
||||
@self.app.callback(
|
||||
Output('cob-rl-training-toggle', 'value'),
|
||||
[Input('cob-rl-training-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_cob_rl_training_toggle(value):
|
||||
if self.orchestrator:
|
||||
enabled = bool(value and len(value) > 0) # Convert list to boolean
|
||||
self.orchestrator.set_model_toggle_state("cob_rl", training_enabled=enabled)
|
||||
# Update dashboard state variable
|
||||
self.cob_rl_training_enabled = enabled
|
||||
logger.info(f"COB RL training toggle: {enabled}")
|
||||
return value
|
||||
|
||||
@self.app.callback(
|
||||
Output('decision-fusion-inference-toggle', 'value'),
|
||||
[Input('decision-fusion-inference-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_decision_fusion_inference_toggle(value):
|
||||
if self.orchestrator:
|
||||
enabled = bool(value and len(value) > 0) # Convert list to boolean
|
||||
self.orchestrator.set_model_toggle_state("decision_fusion", inference_enabled=enabled)
|
||||
# Update dashboard state variable
|
||||
self.decision_fusion_inference_enabled = enabled
|
||||
logger.info(f"Decision Fusion inference toggle: {enabled}")
|
||||
return value
|
||||
|
||||
@self.app.callback(
|
||||
Output('decision-fusion-training-toggle', 'value'),
|
||||
[Input('decision-fusion-training-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_decision_fusion_training_toggle(value):
|
||||
if self.orchestrator:
|
||||
enabled = bool(value and len(value) > 0) # Convert list to boolean
|
||||
self.orchestrator.set_model_toggle_state("decision_fusion", training_enabled=enabled)
|
||||
# Update dashboard state variable
|
||||
self.decision_fusion_training_enabled = enabled
|
||||
logger.info(f"Decision Fusion training toggle: {enabled}")
|
||||
return value
|
||||
|
||||
# NEW: Callback to sync toggle states from orchestrator on page load
|
||||
@self.app.callback(
|
||||
[Output('dqn-inference-toggle', 'value'),
|
||||
Output('dqn-training-toggle', 'value'),
|
||||
Output('cnn-inference-toggle', 'value'),
|
||||
Output('cnn-training-toggle', 'value'),
|
||||
Output('cob-rl-inference-toggle', 'value'),
|
||||
Output('cob-rl-training-toggle', 'value'),
|
||||
Output('decision-fusion-inference-toggle', 'value'),
|
||||
Output('decision-fusion-training-toggle', 'value')],
|
||||
[Input('interval-component', 'n_intervals')],
|
||||
prevent_initial_call=False
|
||||
)
|
||||
def sync_toggle_states_from_orchestrator(n):
|
||||
"""Sync toggle states from orchestrator to ensure UI consistency"""
|
||||
if not self.orchestrator:
|
||||
return [], [], [], [], [], [], [], []
|
||||
|
||||
try:
|
||||
# Get toggle states from orchestrator
|
||||
dqn_state = self.orchestrator.get_model_toggle_state("dqn")
|
||||
cnn_state = self.orchestrator.get_model_toggle_state("cnn")
|
||||
cob_rl_state = self.orchestrator.get_model_toggle_state("cob_rl")
|
||||
decision_fusion_state = self.orchestrator.get_model_toggle_state("decision_fusion")
|
||||
|
||||
# Convert to checklist values (list with 'enabled' if True, empty list if False)
|
||||
dqn_inf = ['enabled'] if dqn_state.get('inference_enabled', True) else []
|
||||
dqn_trn = ['enabled'] if dqn_state.get('training_enabled', True) else []
|
||||
cnn_inf = ['enabled'] if cnn_state.get('inference_enabled', True) else []
|
||||
cnn_trn = ['enabled'] if cnn_state.get('training_enabled', True) else []
|
||||
cob_rl_inf = ['enabled'] if cob_rl_state.get('inference_enabled', True) else []
|
||||
cob_rl_trn = ['enabled'] if cob_rl_state.get('training_enabled', True) else []
|
||||
decision_inf = ['enabled'] if decision_fusion_state.get('inference_enabled', True) else []
|
||||
decision_trn = ['enabled'] if decision_fusion_state.get('training_enabled', True) else []
|
||||
|
||||
# Update dashboard state variables
|
||||
self.dqn_inference_enabled = bool(dqn_inf)
|
||||
self.dqn_training_enabled = bool(dqn_trn)
|
||||
self.cnn_inference_enabled = bool(cnn_inf)
|
||||
self.cnn_training_enabled = bool(cnn_trn)
|
||||
self.cob_rl_inference_enabled = bool(cob_rl_inf)
|
||||
self.cob_rl_training_enabled = bool(cob_rl_trn)
|
||||
self.decision_fusion_inference_enabled = bool(decision_inf)
|
||||
self.decision_fusion_training_enabled = bool(decision_trn)
|
||||
|
||||
logger.debug(f"Synced toggle states from orchestrator: DQN(inf:{self.dqn_inference_enabled}, trn:{self.dqn_training_enabled}), CNN(inf:{self.cnn_inference_enabled}, trn:{self.cnn_training_enabled}), COB_RL(inf:{self.cob_rl_inference_enabled}, trn:{self.cob_rl_training_enabled}), Decision_Fusion(inf:{self.decision_fusion_inference_enabled}, trn:{self.decision_fusion_training_enabled})")
|
||||
|
||||
return dqn_inf, dqn_trn, cnn_inf, cnn_trn, cob_rl_inf, cob_rl_trn, decision_inf, decision_trn
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error syncing toggle states from orchestrator: {e}")
|
||||
return [], [], [], [], [], [], [], []
|
||||
def update_cold_start_mode(switch_value):
|
||||
"""Update cold start training mode"""
|
||||
logger.debug(f"Cold start callback triggered with value: {switch_value}")
|
||||
try:
|
||||
@ -3474,14 +3450,23 @@ class CleanTradingDashboard:
|
||||
except Exception as e:
|
||||
logger.debug(f"Error getting orchestrator model statistics: {e}")
|
||||
|
||||
# Ensure toggle_states are available - use dashboard state variables as fallback
|
||||
# Ensure toggle_states are available - get from orchestrator or use defaults
|
||||
if toggle_states is None:
|
||||
toggle_states = {
|
||||
"dqn": {"inference_enabled": self.dqn_inference_enabled, "training_enabled": self.dqn_training_enabled},
|
||||
"cnn": {"inference_enabled": self.cnn_inference_enabled, "training_enabled": self.cnn_training_enabled},
|
||||
"cob_rl": {"inference_enabled": self.cob_rl_inference_enabled, "training_enabled": self.cob_rl_training_enabled},
|
||||
"decision_fusion": {"inference_enabled": self.decision_fusion_inference_enabled, "training_enabled": self.decision_fusion_training_enabled}
|
||||
}
|
||||
if self.orchestrator:
|
||||
# Get all available models and their toggle states
|
||||
available_models = self._get_available_models()
|
||||
toggle_states = {}
|
||||
for model_name in available_models.keys():
|
||||
toggle_states[model_name] = self.orchestrator.get_model_toggle_state(model_name)
|
||||
else:
|
||||
# Fallback to default states for known models
|
||||
toggle_states = {
|
||||
"dqn": {"inference_enabled": True, "training_enabled": True},
|
||||
"cnn": {"inference_enabled": True, "training_enabled": True},
|
||||
"cob_rl": {"inference_enabled": True, "training_enabled": True},
|
||||
"decision_fusion": {"inference_enabled": True, "training_enabled": True},
|
||||
"transformer": {"inference_enabled": True, "training_enabled": True}
|
||||
}
|
||||
|
||||
# Helper function to safely calculate improvement percentage
|
||||
def safe_improvement_calc(initial, current, default_improvement=0.0):
|
||||
@ -6375,97 +6360,244 @@ class CleanTradingDashboard:
|
||||
logger.error(f"Error forcing dashboard refresh: {e}")
|
||||
|
||||
def _store_all_models(self) -> bool:
|
||||
"""Store all current models to persistent storage"""
|
||||
"""Store all current models to persistent storage and verify loading"""
|
||||
try:
|
||||
if not self.orchestrator:
|
||||
logger.warning("No orchestrator available for model storage")
|
||||
return False
|
||||
|
||||
stored_models = []
|
||||
if not hasattr(self.orchestrator, 'checkpoint_manager') or not self.orchestrator.checkpoint_manager:
|
||||
logger.warning("No checkpoint manager available for model storage")
|
||||
return False
|
||||
|
||||
# 1. Store DQN model
|
||||
stored_models = []
|
||||
verification_results = []
|
||||
|
||||
logger.info("🔄 Starting comprehensive model storage and verification...")
|
||||
|
||||
# Get current model statistics for checkpoint saving
|
||||
current_performance = 0.8 # Default performance score
|
||||
if hasattr(self.orchestrator, 'get_model_statistics'):
|
||||
all_stats = self.orchestrator.get_model_statistics()
|
||||
if all_stats:
|
||||
# Calculate average accuracy across all models
|
||||
accuracies = [stats.accuracy for stats in all_stats.values() if stats.accuracy is not None]
|
||||
if accuracies:
|
||||
current_performance = sum(accuracies) / len(accuracies)
|
||||
|
||||
# 1. Store DQN model using checkpoint manager
|
||||
if hasattr(self.orchestrator, 'rl_agent') and self.orchestrator.rl_agent:
|
||||
try:
|
||||
if hasattr(self.orchestrator.rl_agent, 'save'):
|
||||
save_path = self.orchestrator.rl_agent.save('models/saved/dqn_agent_session')
|
||||
stored_models.append(('DQN', save_path))
|
||||
logger.info(f"Stored DQN model: {save_path}")
|
||||
logger.info("💾 Saving DQN model checkpoint...")
|
||||
dqn_stats = self.orchestrator.get_model_statistics('dqn')
|
||||
performance_score = dqn_stats.accuracy if dqn_stats and dqn_stats.accuracy else current_performance
|
||||
|
||||
from datetime import datetime
|
||||
checkpoint_data = {
|
||||
'model_state_dict': self.orchestrator.rl_agent.get_model_state() if hasattr(self.orchestrator.rl_agent, 'get_model_state') else None,
|
||||
'performance_score': performance_score,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'model_name': 'dqn_agent',
|
||||
'session_storage': True
|
||||
}
|
||||
|
||||
save_path = self.orchestrator.checkpoint_manager.save_model_checkpoint(
|
||||
model_name="dqn_agent",
|
||||
model_data=checkpoint_data,
|
||||
loss=1.0 - performance_score,
|
||||
performance_score=performance_score
|
||||
)
|
||||
|
||||
if save_path:
|
||||
stored_models.append(('DQN', str(save_path)))
|
||||
logger.info(f"✅ Stored DQN model checkpoint: {save_path}")
|
||||
|
||||
# Update model state to [LOADED]
|
||||
if 'dqn' not in self.orchestrator.model_states:
|
||||
self.orchestrator.model_states['dqn'] = {}
|
||||
self.orchestrator.model_states['dqn']['checkpoint_loaded'] = True
|
||||
self.orchestrator.model_states['dqn']['session_stored'] = True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to store DQN model: {e}")
|
||||
logger.warning(f"❌ Failed to store DQN model: {e}")
|
||||
|
||||
# 2. Store CNN model
|
||||
# 2. Store CNN model using checkpoint manager
|
||||
if hasattr(self.orchestrator, 'cnn_model') and self.orchestrator.cnn_model:
|
||||
try:
|
||||
if hasattr(self.orchestrator.cnn_model, 'save'):
|
||||
save_path = self.orchestrator.cnn_model.save('models/saved/cnn_model_session')
|
||||
stored_models.append(('CNN', save_path))
|
||||
logger.info(f"Stored CNN model: {save_path}")
|
||||
logger.info("💾 Saving CNN model checkpoint...")
|
||||
cnn_stats = self.orchestrator.get_model_statistics('enhanced_cnn')
|
||||
performance_score = cnn_stats.accuracy if cnn_stats and cnn_stats.accuracy else current_performance
|
||||
|
||||
checkpoint_data = {
|
||||
'model_state_dict': self.orchestrator.cnn_model.state_dict() if hasattr(self.orchestrator.cnn_model, 'state_dict') else None,
|
||||
'performance_score': performance_score,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'model_name': 'enhanced_cnn',
|
||||
'session_storage': True
|
||||
}
|
||||
|
||||
save_path = self.orchestrator.checkpoint_manager.save_model_checkpoint(
|
||||
model_name="enhanced_cnn",
|
||||
model_data=checkpoint_data,
|
||||
loss=1.0 - performance_score,
|
||||
performance_score=performance_score
|
||||
)
|
||||
|
||||
if save_path:
|
||||
stored_models.append(('CNN', str(save_path)))
|
||||
logger.info(f"✅ Stored CNN model checkpoint: {save_path}")
|
||||
|
||||
# Update model state to [LOADED]
|
||||
if 'cnn' not in self.orchestrator.model_states:
|
||||
self.orchestrator.model_states['cnn'] = {}
|
||||
self.orchestrator.model_states['cnn']['checkpoint_loaded'] = True
|
||||
self.orchestrator.model_states['cnn']['session_stored'] = True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to store CNN model: {e}")
|
||||
logger.warning(f"❌ Failed to store CNN model: {e}")
|
||||
|
||||
# 3. Store Transformer model
|
||||
if hasattr(self.orchestrator, 'primary_transformer') and self.orchestrator.primary_transformer:
|
||||
try:
|
||||
if hasattr(self.orchestrator.primary_transformer, 'save'):
|
||||
save_path = self.orchestrator.primary_transformer.save('models/saved/transformer_model_session')
|
||||
stored_models.append(('Transformer', save_path))
|
||||
logger.info(f"Stored Transformer model: {save_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to store Transformer model: {e}")
|
||||
|
||||
# 4. Store COB RL model
|
||||
# 3. Store COB RL model using checkpoint manager
|
||||
if hasattr(self.orchestrator, 'cob_rl_agent') and self.orchestrator.cob_rl_agent:
|
||||
try:
|
||||
if hasattr(self.orchestrator.cob_rl_agent, 'save'):
|
||||
save_path = self.orchestrator.cob_rl_agent.save('models/saved/cob_rl_agent_session')
|
||||
stored_models.append(('COB RL', save_path))
|
||||
logger.info(f"Stored COB RL model: {save_path}")
|
||||
logger.info("💾 Saving COB RL model checkpoint...")
|
||||
cob_stats = self.orchestrator.get_model_statistics('cob_rl_model')
|
||||
performance_score = cob_stats.accuracy if cob_stats and cob_stats.accuracy else current_performance
|
||||
|
||||
checkpoint_data = {
|
||||
'model_state_dict': self.orchestrator.cob_rl_agent.state_dict() if hasattr(self.orchestrator.cob_rl_agent, 'state_dict') else None,
|
||||
'performance_score': performance_score,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'model_name': 'cob_rl_model',
|
||||
'session_storage': True
|
||||
}
|
||||
|
||||
save_path = self.orchestrator.checkpoint_manager.save_model_checkpoint(
|
||||
model_name="cob_rl_model",
|
||||
model_data=checkpoint_data,
|
||||
loss=1.0 - performance_score,
|
||||
performance_score=performance_score
|
||||
)
|
||||
|
||||
if save_path:
|
||||
stored_models.append(('COB RL', str(save_path)))
|
||||
logger.info(f"✅ Stored COB RL model checkpoint: {save_path}")
|
||||
|
||||
# Update model state to [LOADED]
|
||||
if 'cob_rl' not in self.orchestrator.model_states:
|
||||
self.orchestrator.model_states['cob_rl'] = {}
|
||||
self.orchestrator.model_states['cob_rl']['checkpoint_loaded'] = True
|
||||
self.orchestrator.model_states['cob_rl']['session_stored'] = True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to store COB RL model: {e}")
|
||||
logger.warning(f"❌ Failed to store COB RL model: {e}")
|
||||
|
||||
# 5. Store Decision Fusion model
|
||||
# 4. Store Decision Fusion model using orchestrator's save method
|
||||
if hasattr(self.orchestrator, 'decision_fusion_network') and self.orchestrator.decision_fusion_network:
|
||||
try:
|
||||
if hasattr(self.orchestrator.decision_fusion_network, 'save'):
|
||||
save_path = self.orchestrator.decision_fusion_network.save('models/saved/decision_fusion_session')
|
||||
stored_models.append(('Decision Fusion', save_path))
|
||||
logger.info(f"Stored Decision Fusion model: {save_path}")
|
||||
logger.info("💾 Saving Decision Fusion model checkpoint...")
|
||||
# Use the orchestrator's decision fusion checkpoint method
|
||||
self.orchestrator._save_decision_fusion_checkpoint()
|
||||
|
||||
stored_models.append(('Decision Fusion', 'checkpoint_manager'))
|
||||
logger.info(f"✅ Stored Decision Fusion model checkpoint")
|
||||
|
||||
# Update model state to [LOADED]
|
||||
if 'decision_fusion' not in self.orchestrator.model_states:
|
||||
self.orchestrator.model_states['decision_fusion'] = {}
|
||||
self.orchestrator.model_states['decision_fusion']['checkpoint_loaded'] = True
|
||||
self.orchestrator.model_states['decision_fusion']['session_stored'] = True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to store Decision Fusion model: {e}")
|
||||
logger.warning(f"❌ Failed to store Decision Fusion model: {e}")
|
||||
|
||||
# 6. Store model metadata and training state
|
||||
# 5. Verification Step - Try to load checkpoints to verify they work
|
||||
logger.info("🔍 Verifying stored checkpoints...")
|
||||
|
||||
for model_name, checkpoint_path in stored_models:
|
||||
try:
|
||||
if model_name == 'Decision Fusion':
|
||||
# Decision fusion verification is handled by the orchestrator
|
||||
verification_results.append((model_name, True, "Checkpoint saved successfully"))
|
||||
continue
|
||||
|
||||
# Try to get checkpoint metadata to verify it exists and is valid
|
||||
from utils.checkpoint_manager import load_best_checkpoint
|
||||
|
||||
model_key = {
|
||||
'DQN': 'dqn_agent',
|
||||
'CNN': 'enhanced_cnn',
|
||||
'COB RL': 'cob_rl_model'
|
||||
}.get(model_name)
|
||||
|
||||
if model_key:
|
||||
result = load_best_checkpoint(model_key)
|
||||
if result:
|
||||
file_path, metadata = result
|
||||
verification_results.append((model_name, True, f"Verified: {metadata.checkpoint_id}"))
|
||||
logger.info(f"✅ Verified {model_name} checkpoint: {metadata.checkpoint_id}")
|
||||
else:
|
||||
verification_results.append((model_name, False, "Checkpoint not found after save"))
|
||||
logger.warning(f"⚠️ Could not verify {model_name} checkpoint")
|
||||
|
||||
except Exception as e:
|
||||
verification_results.append((model_name, False, f"Verification failed: {str(e)}"))
|
||||
logger.warning(f"⚠️ Failed to verify {model_name}: {e}")
|
||||
|
||||
# 6. Store session metadata
|
||||
try:
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
metadata = {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'session_pnl': self.session_pnl,
|
||||
'trade_count': len(self.closed_trades),
|
||||
'session_pnl': getattr(self, 'session_pnl', 0.0),
|
||||
'trade_count': len(getattr(self, 'closed_trades', [])),
|
||||
'stored_models': stored_models,
|
||||
'training_iterations': getattr(self, 'training_iteration', 0),
|
||||
'model_performance': self.get_model_performance_metrics()
|
||||
'verification_results': verification_results,
|
||||
'training_iterations': getattr(self.orchestrator, 'training_iterations', 0) if self.orchestrator else 0,
|
||||
'model_performance': self.get_model_performance_metrics() if hasattr(self, 'get_model_performance_metrics') else {},
|
||||
'storage_method': 'checkpoint_manager_with_verification'
|
||||
}
|
||||
|
||||
import os
|
||||
os.makedirs('models/saved', exist_ok=True)
|
||||
metadata_path = 'models/saved/session_metadata.json'
|
||||
with open(metadata_path, 'w') as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
|
||||
logger.info(f"Stored session metadata: {metadata_path}")
|
||||
logger.info(f"📋 Stored session metadata: {metadata_path}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to store metadata: {e}")
|
||||
|
||||
# Log summary
|
||||
# 7. Save orchestrator UI state to persist model states
|
||||
if hasattr(self.orchestrator, '_save_ui_state'):
|
||||
try:
|
||||
self.orchestrator._save_ui_state()
|
||||
logger.info("💾 Saved orchestrator UI state")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save UI state: {e}")
|
||||
|
||||
# Summary
|
||||
successful_stores = len(stored_models)
|
||||
successful_verifications = len([r for r in verification_results if r[1]])
|
||||
|
||||
if stored_models:
|
||||
logger.info(f"Successfully stored {len(stored_models)} models: {[name for name, _ in stored_models]}")
|
||||
logger.info(f"📊 STORAGE SUMMARY:")
|
||||
logger.info(f" ✅ Models stored: {successful_stores}")
|
||||
logger.info(f" ✅ Verifications passed: {successful_verifications}/{len(verification_results)}")
|
||||
logger.info(f" 📋 Models: {[name for name, _ in stored_models]}")
|
||||
|
||||
# Update button display with success info
|
||||
return True
|
||||
else:
|
||||
logger.warning("No models were stored - no models available or save methods not found")
|
||||
logger.warning("❌ No models were stored - no models available")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error storing models: {e}")
|
||||
logger.error(f"❌ Error in store all models operation: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
def _get_signal_attribute(self, signal, attr_name, default=None):
|
||||
|
||||
Reference in New Issue
Block a user