popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More T train fixes

Browse Source
This commit is contained in:
Dobromir Popov
2025-11-17 21:12:20 +02:00
parent a8d59a946e
commit 2d1d036c07
1 changed files with 10 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
ANNOTATE/core/real_training_adapter.py
View File

@@ -2128,13 +2128,16 @@ class RealTrainingAdapter:
# MEMORY FIX: Final cleanup # MEMORY FIX: Final cleanup
logger.info(" Final memory cleanup...") logger.info(" Final memory cleanup...")
# Clear cached batches # Clear grouped batches (cached_batches was already cleared earlier)
for batch in cached_batches: # Note: Don't delete batch contents as they may be referenced elsewhere
for key in list(batch.keys()): # Just clear the list reference - Python GC will handle cleanup
if isinstance(batch[key], torch.Tensor): try:
del batch[key] if grouped_batches:
cached_batches.clear() grouped_batches.clear()
del cached_batches del grouped_batches
except NameError:
# grouped_batches already cleaned up or doesn't exist
pass
gc.collect() gc.collect()
if torch.cuda.is_available(): if torch.cuda.is_available():