popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

models CKPts auto save

Browse Source
This commit is contained in:
Dobromir Popov
2025-08-10 03:42:03 +03:00
parent 1de536322a
commit e05163deb7
2 changed files with 37 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
core/orchestrator.py
View File

@ -7930,6 +7930,42 @@ class TradingOrchestrator:
logger.error(f"Error initializing checkpoint manager: {e}")
self.checkpoint_manager = None
def autosave_models(self):
"""Attempt to autosave best model checkpoints periodically."""
try:
if not self.checkpoint_manager:
return
# CNN autosave when current_loss equals best_loss
try:
cnn_stats = self.model_states.get('cnn', {})
if cnn_stats and cnn_stats.get('current_loss') is not None:
if cnn_stats.get('best_loss') is not None and cnn_stats['current_loss'] <= cnn_stats['best_loss']:
path = self.checkpoint_manager.save_model_checkpoint(
model_name='enhanced_cnn',
model=self.cnn_model,
metrics={'loss': float(cnn_stats['current_loss'])},
metadata={'source': 'autosave'}
)
if path:
logger.info(f"Autosaved CNN checkpoint: {path}")
except Exception:
pass
# COB RL autosave
try:
cob_stats = self.model_states.get('cob_rl', {})
if cob_stats and cob_stats.get('current_loss') is not None:
if cob_stats.get('best_loss') is not None and cob_stats['current_loss'] <= cob_stats['best_loss']:
self.checkpoint_manager.save_model_checkpoint(
model_name='cob_rl',
model=self.cob_rl_agent,
metrics={'loss': float(cob_stats['current_loss'])},
metadata={'source': 'autosave'}
)
except Exception:
pass
except Exception as e:
logger.debug(f"Autosave models skipped: {e}")
def _schedule_database_cleanup(self):
"""Schedule periodic database cleanup"""
try: