popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

model training WIP

Browse Source
This commit is contained in:
Dobromir Popov
2025-10-25 00:17:53 +03:00
parent c28ee2c432
commit e816cb9795
4 changed files with 516 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
ANNOTATE/web/app.py
View File

@@ -174,58 +174,88 @@ class AnnotationDashboard:
logger.info("Annotation Dashboard initialized")
def _start_async_model_loading(self):
"""Load ML models asynchronously in background thread"""
"""Load ML models asynchronously in background thread with retry logic"""
import threading
import time
def load_models():
try:
logger.info("🔄 Starting async model loading...")
# Initialize orchestrator with models
if TradingOrchestrator:
max_retries = 3
retry_delay = 5 # seconds
for attempt in range(max_retries):
try:
if attempt > 0:
logger.info(f"🔄 Retry attempt {attempt + 1}/{max_retries} for model loading...")
time.sleep(retry_delay)
else:
logger.info("🔄 Starting async model loading...")
# Check if TradingOrchestrator is available
if not TradingOrchestrator:
logger.error("❌ TradingOrchestrator class not available")
self.models_loading = False
self.available_models = []
return
# Initialize orchestrator with models
logger.info(" Creating TradingOrchestrator instance...")
self.orchestrator = TradingOrchestrator(
data_provider=self.data_provider,
enhanced_rl_training=True
)
logger.info(" ✅ Orchestrator created")
# Initialize ML models
logger.info("Initializing ML models...")
logger.info(" Initializing ML models...")
self.orchestrator._initialize_ml_models()
logger.info(" ✅ ML models initialized")
# Update training adapter with orchestrator
self.training_adapter.orchestrator = self.orchestrator
logger.info(" ✅ Training adapter updated")
# Get available models from orchestrator
available = []
if hasattr(self.orchestrator, 'rl_agent') and self.orchestrator.rl_agent:
available.append('DQN')
logger.info(" ✅ DQN model available")
if hasattr(self.orchestrator, 'cnn_model') and self.orchestrator.cnn_model:
available.append('CNN')
logger.info(" ✅ CNN model available")
if hasattr(self.orchestrator, 'transformer_model') and self.orchestrator.transformer_model:
available.append('Transformer')
logger.info(" ✅ Transformer model available")
self.available_models = available
if available:
logger.info(f"✅ Models loaded: {', '.join(available)}")
logger.info(f"✅ Models loaded successfully: {', '.join(available)}")
else:
logger.warning("⚠️ No models were initialized")
logger.warning("⚠️ No models were initialized (this might be normal if models aren't configured)")
self.models_loading = False
logger.info("✅ Async model loading complete")
else:
logger.warning("⚠️ TradingOrchestrator not available")
self.models_loading = False
return # Success - exit retry loop
except Exception as e:
logger.error(f"❌ Error loading models: {e}")
self.models_loading = False
self.available_models = []
except Exception as e:
logger.error(f"❌ Error loading models (attempt {attempt + 1}/{max_retries}): {e}")
import traceback
logger.error(f"Traceback:\n{traceback.format_exc()}")
if attempt == max_retries - 1:
# Final attempt failed
logger.error(f"❌ Model loading failed after {max_retries} attempts")
self.models_loading = False
self.available_models = []
else:
logger.info(f" Will retry in {retry_delay} seconds...")
# Start loading in background thread
thread = threading.Thread(target=load_models, daemon=True)
thread = threading.Thread(target=load_models, daemon=True, name="ModelLoader")
thread.start()
logger.info("🚀 Model loading started in background (UI remains responsive)")
logger.info(f"🚀 Model loading started in background thread (ID: {thread.ident}, Name: {thread.name})")
logger.info(" UI remains responsive while models load...")
logger.info(" Will retry up to 3 times if loading fails")
def _enable_unified_storage_async(self):
"""Enable unified storage system in background thread"""