popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more training fixes

Browse Source
This commit is contained in:
Dobromir Popov
2025-12-08 19:37:07 +02:00
parent 7e8722aa57
commit 5383ac7df8
1 changed files with 14 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
ANNOTATE/core/real_training_adapter.py
View File

@@ -2507,10 +2507,11 @@ class RealTrainingAdapter:
for batch in cached_batches: for batch in cached_batches:
grouped_batches.append(batch) grouped_batches.append(batch)
# Clear cached_batches to free memory # Don't clear cached_batches yet - grouped_batches contains references to them
cached_batches.clear() # We'll clear after training completes
del cached_batches # cached_batches.clear()
gc.collect() # del cached_batches
# gc.collect()
def batch_generator(): def batch_generator():
""" """
@@ -2577,6 +2578,15 @@ class RealTrainingAdapter:
# Generate batches fresh for each epoch # Generate batches fresh for each epoch
for i, batch in enumerate(batch_generator()): for i, batch in enumerate(batch_generator()):
try: try:
# DEBUG: Check if batch has timeframe data
if epoch > 0 and i == 0:
has_1m = batch.get('price_data_1m') is not None
has_1h = batch.get('price_data_1h') is not None
has_1d = batch.get('price_data_1d') is not None
logger.debug(f"Epoch {epoch+1}, Batch 1: has_1m={has_1m}, has_1h={has_1h}, has_1d={has_1d}")
if has_1m:
logger.debug(f" price_data_1m shape: {batch['price_data_1m'].shape}")
# Store prediction before training (for visualization) # Store prediction before training (for visualization)
# Only store predictions on first epoch and every 10th batch to avoid clutter # Only store predictions on first epoch and every 10th batch to avoid clutter
if epoch == 0 and i % 10 == 0 and self.orchestrator: if epoch == 0 and i % 10 == 0 and self.orchestrator: