models CKPts auto save

This commit is contained in:
Dobromir Popov
2025-08-10 03:42:03 +03:00
parent 1de536322a
commit e05163deb7
2 changed files with 37 additions and 8 deletions

View File

@ -7930,6 +7930,42 @@ class TradingOrchestrator:
logger.error(f"Error initializing checkpoint manager: {e}")
self.checkpoint_manager = None
def autosave_models(self):
"""Attempt to autosave best model checkpoints periodically."""
try:
if not self.checkpoint_manager:
return
# CNN autosave when current_loss equals best_loss
try:
cnn_stats = self.model_states.get('cnn', {})
if cnn_stats and cnn_stats.get('current_loss') is not None:
if cnn_stats.get('best_loss') is not None and cnn_stats['current_loss'] <= cnn_stats['best_loss']:
path = self.checkpoint_manager.save_model_checkpoint(
model_name='enhanced_cnn',
model=self.cnn_model,
metrics={'loss': float(cnn_stats['current_loss'])},
metadata={'source': 'autosave'}
)
if path:
logger.info(f"Autosaved CNN checkpoint: {path}")
except Exception:
pass
# COB RL autosave
try:
cob_stats = self.model_states.get('cob_rl', {})
if cob_stats and cob_stats.get('current_loss') is not None:
if cob_stats.get('best_loss') is not None and cob_stats['current_loss'] <= cob_stats['best_loss']:
self.checkpoint_manager.save_model_checkpoint(
model_name='cob_rl',
model=self.cob_rl_agent,
metrics={'loss': float(cob_stats['current_loss'])},
metadata={'source': 'autosave'}
)
except Exception:
pass
except Exception as e:
logger.debug(f"Autosave models skipped: {e}")
def _schedule_database_cleanup(self):
"""Schedule periodic database cleanup"""
try:

View File

@ -958,14 +958,7 @@ class CleanTradingDashboard:
def update_model_inference_toggle(value):
return self._handle_model_toggle(model_name, 'inference', value)
# Create training toggle callback
@self.app.callback(
Output(f'{model_name}-training-toggle', 'value'),
[Input(f'{model_name}-training-toggle', 'value')],
prevent_initial_call=True
)
def update_model_training_toggle(value):
return self._handle_model_toggle(model_name, 'training', value)
# Skip training toggle callback registration until components exist in layout
logger.debug(f"Created toggle callbacks for model: {model_name}")