models CKPts auto save
This commit is contained in:
@ -7930,6 +7930,42 @@ class TradingOrchestrator:
|
||||
logger.error(f"Error initializing checkpoint manager: {e}")
|
||||
self.checkpoint_manager = None
|
||||
|
||||
def autosave_models(self):
|
||||
"""Attempt to autosave best model checkpoints periodically."""
|
||||
try:
|
||||
if not self.checkpoint_manager:
|
||||
return
|
||||
# CNN autosave when current_loss equals best_loss
|
||||
try:
|
||||
cnn_stats = self.model_states.get('cnn', {})
|
||||
if cnn_stats and cnn_stats.get('current_loss') is not None:
|
||||
if cnn_stats.get('best_loss') is not None and cnn_stats['current_loss'] <= cnn_stats['best_loss']:
|
||||
path = self.checkpoint_manager.save_model_checkpoint(
|
||||
model_name='enhanced_cnn',
|
||||
model=self.cnn_model,
|
||||
metrics={'loss': float(cnn_stats['current_loss'])},
|
||||
metadata={'source': 'autosave'}
|
||||
)
|
||||
if path:
|
||||
logger.info(f"Autosaved CNN checkpoint: {path}")
|
||||
except Exception:
|
||||
pass
|
||||
# COB RL autosave
|
||||
try:
|
||||
cob_stats = self.model_states.get('cob_rl', {})
|
||||
if cob_stats and cob_stats.get('current_loss') is not None:
|
||||
if cob_stats.get('best_loss') is not None and cob_stats['current_loss'] <= cob_stats['best_loss']:
|
||||
self.checkpoint_manager.save_model_checkpoint(
|
||||
model_name='cob_rl',
|
||||
model=self.cob_rl_agent,
|
||||
metrics={'loss': float(cob_stats['current_loss'])},
|
||||
metadata={'source': 'autosave'}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.debug(f"Autosave models skipped: {e}")
|
||||
|
||||
def _schedule_database_cleanup(self):
|
||||
"""Schedule periodic database cleanup"""
|
||||
try:
|
||||
|
@ -958,14 +958,7 @@ class CleanTradingDashboard:
|
||||
def update_model_inference_toggle(value):
|
||||
return self._handle_model_toggle(model_name, 'inference', value)
|
||||
|
||||
# Create training toggle callback
|
||||
@self.app.callback(
|
||||
Output(f'{model_name}-training-toggle', 'value'),
|
||||
[Input(f'{model_name}-training-toggle', 'value')],
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_model_training_toggle(value):
|
||||
return self._handle_model_toggle(model_name, 'training', value)
|
||||
# Skip training toggle callback registration until components exist in layout
|
||||
|
||||
logger.debug(f"Created toggle callbacks for model: {model_name}")
|
||||
|
||||
|
Reference in New Issue
Block a user