From e9edf2c5f21ee73a32351cbb45d250c1fcaada85 Mon Sep 17 00:00:00 2001
From: Dobromir Popov <dobromir.popov@gateway.one>
Date: Fri, 24 Oct 2025 23:13:28 +0300
Subject: [PATCH] fix async model loading

---
 ANNOTATE/web/app.py                           |  76 ++++++++-
 .../templates/components/annotation_list.html | 152 +++++++++---------
 .../templates/components/training_panel.html  |  61 +++++--
 3 files changed, 197 insertions(+), 92 deletions(-)

diff --git a/ANNOTATE/web/app.py b/ANNOTATE/web/app.py
index 441f8c9..7297bb9 100644
--- a/ANNOTATE/web/app.py
+++ b/ANNOTATE/web/app.py
@@ -147,14 +147,19 @@ class AnnotationDashboard:
         if self.data_provider:
             self._enable_unified_storage_async()
         
-        # ANNOTATE doesn't need orchestrator - skip ML model loading for fast startup
+        # ANNOTATE doesn't need orchestrator immediately - load async for fast startup
         self.orchestrator = None
+        self.models_loading = True
+        self.available_models = []
         
         # Initialize ANNOTATE components
         self.annotation_manager = AnnotationManager()
         # Use REAL training adapter - NO SIMULATION!
         self.training_adapter = RealTrainingAdapter(None, self.data_provider)
         
+        # Start async model loading in background
+        self._start_async_model_loading()
+        
         # Initialize data loader with existing DataProvider
         self.data_loader = HistoricalDataLoader(self.data_provider) if self.data_provider else None
         self.time_range_manager = TimeRangeManager(self.data_loader) if self.data_loader else None
@@ -168,6 +173,60 @@ class AnnotationDashboard:
         
         logger.info("Annotation Dashboard initialized")
     
+    def _start_async_model_loading(self):
+        """Load ML models asynchronously in background thread"""
+        import threading
+        
+        def load_models():
+            try:
+                logger.info("🔄 Starting async model loading...")
+                
+                # Initialize orchestrator with models
+                if TradingOrchestrator:
+                    self.orchestrator = TradingOrchestrator(
+                        data_provider=self.data_provider,
+                        enhanced_rl_training=True
+                    )
+                    
+                    # Initialize ML models
+                    logger.info("Initializing ML models...")
+                    self.orchestrator._initialize_ml_models()
+                    
+                    # Update training adapter with orchestrator
+                    self.training_adapter.orchestrator = self.orchestrator
+                    
+                    # Get available models from orchestrator
+                    available = []
+                    if hasattr(self.orchestrator, 'rl_agent') and self.orchestrator.rl_agent:
+                        available.append('DQN')
+                    if hasattr(self.orchestrator, 'cnn_model') and self.orchestrator.cnn_model:
+                        available.append('CNN')
+                    if hasattr(self.orchestrator, 'transformer_model') and self.orchestrator.transformer_model:
+                        available.append('Transformer')
+                    
+                    self.available_models = available
+                    
+                    if available:
+                        logger.info(f"✅ Models loaded: {', '.join(available)}")
+                    else:
+                        logger.warning("⚠️ No models were initialized")
+                    
+                    self.models_loading = False
+                    logger.info("✅ Async model loading complete")
+                else:
+                    logger.warning("⚠️ TradingOrchestrator not available")
+                    self.models_loading = False
+                    
+            except Exception as e:
+                logger.error(f"❌ Error loading models: {e}")
+                self.models_loading = False
+                self.available_models = []
+        
+        # Start loading in background thread
+        thread = threading.Thread(target=load_models, daemon=True)
+        thread.start()
+        logger.info("🚀 Model loading started in background (UI remains responsive)")
+    
     def _enable_unified_storage_async(self):
         """Enable unified storage system in background thread"""
         def enable_storage():
@@ -967,21 +1026,33 @@ class AnnotationDashboard:
         
         @self.server.route('/api/available-models', methods=['GET'])
         def get_available_models():
-            """Get list of available models"""
+            """Get list of available models with loading status"""
             try:
                 if not self.training_adapter:
                     return jsonify({
                         'success': False,
+                        'loading': False,
                         'error': {
                             'code': 'TRAINING_UNAVAILABLE',
                             'message': 'Real training adapter not available'
                         }
                     })
                 
+                # Check if models are still loading
+                if self.models_loading:
+                    return jsonify({
+                        'success': True,
+                        'loading': True,
+                        'models': [],
+                        'message': 'Models are loading in background...'
+                    })
+                
+                # Models loaded - get the list
                 models = self.training_adapter.get_available_models()
                 
                 return jsonify({
                     'success': True,
+                    'loading': False,
                     'models': models
                 })
                 
@@ -989,6 +1060,7 @@ class AnnotationDashboard:
                 logger.error(f"Error getting available models: {e}")
                 return jsonify({
                     'success': False,
+                    'loading': False,
                     'error': {
                         'code': 'MODEL_LIST_ERROR',
                         'message': str(e)
diff --git a/ANNOTATE/web/templates/components/annotation_list.html b/ANNOTATE/web/templates/components/annotation_list.html
index b2e0e94..ec42ce9 100644
--- a/ANNOTATE/web/templates/components/annotation_list.html
+++ b/ANNOTATE/web/templates/components/annotation_list.html
@@ -27,102 +27,102 @@
 
 <script>
     // Export annotations
-    document.getElementById('export-annotations-btn').addEventListener('click', function() {
+    document.getElementById('export-annotations-btn').addEventListener('click', function () {
         fetch('/api/export-annotations', {
             method: 'POST',
-            headers: {'Content-Type': 'application/json'},
+            headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({
                 symbol: appState.currentSymbol,
                 format: 'json'
             })
         })
-        .then(response => response.blob())
-        .then(blob => {
-            const url = window.URL.createObjectURL(blob);
-            const a = document.createElement('a');
-            a.href = url;
-            a.download = `annotations_${appState.currentSymbol}_${Date.now()}.json`;
-            document.body.appendChild(a);
-            a.click();
-            window.URL.revokeObjectURL(url);
-            a.remove();
-            showSuccess('Annotations exported successfully');
-        })
-        .catch(error => {
-            showError('Failed to export annotations: ' + error.message);
-        });
+            .then(response => response.blob())
+            .then(blob => {
+                const url = window.URL.createObjectURL(blob);
+                const a = document.createElement('a');
+                a.href = url;
+                a.download = `annotations_${appState.currentSymbol}_${Date.now()}.json`;
+                document.body.appendChild(a);
+                a.click();
+                window.URL.revokeObjectURL(url);
+                a.remove();
+                showSuccess('Annotations exported successfully');
+            })
+            .catch(error => {
+                showError('Failed to export annotations: ' + error.message);
+            });
     });
-    
+
     // Clear all annotations
-    document.getElementById('clear-all-annotations-btn').addEventListener('click', function() {
+    document.getElementById('clear-all-annotations-btn').addEventListener('click', function () {
         if (appState.annotations.length === 0) {
             showError('No annotations to clear');
             return;
         }
-        
+
         if (!confirm(`Are you sure you want to delete all ${appState.annotations.length} annotations? This action cannot be undone.`)) {
             return;
         }
-        
+
         fetch('/api/clear-all-annotations', {
             method: 'POST',
-            headers: {'Content-Type': 'application/json'},
+            headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({
                 symbol: appState.currentSymbol
             })
         })
-        .then(response => response.json())
-        .then(data => {
-            if (data.success) {
-                // Clear from app state
-                appState.annotations = [];
-                
-                // Update UI
-                renderAnnotationsList(appState.annotations);
-                
-                // Clear from chart
-                if (appState.chartManager) {
-                    appState.chartManager.clearAllAnnotations();
+            .then(response => response.json())
+            .then(data => {
+                if (data.success) {
+                    // Clear from app state
+                    appState.annotations = [];
+
+                    // Update UI
+                    renderAnnotationsList(appState.annotations);
+
+                    // Clear from chart
+                    if (appState.chartManager) {
+                        appState.chartManager.clearAllAnnotations();
+                    }
+
+                    showSuccess(`Cleared ${data.deleted_count} annotations`);
+                } else {
+                    showError('Failed to clear annotations: ' + data.error.message);
                 }
-                
-                showSuccess(`Cleared ${data.deleted_count} annotations`);
-            } else {
-                showError('Failed to clear annotations: ' + data.error.message);
-            }
-        })
-        .catch(error => {
-            showError('Network error: ' + error.message);
-        });
+            })
+            .catch(error => {
+                showError('Network error: ' + error.message);
+            });
     });
-    
+
     // Function to render annotations list
     function renderAnnotationsList(annotations) {
         const listContainer = document.getElementById('annotations-list');
         const noAnnotationsMsg = document.getElementById('no-annotations-msg');
-        
+
         if (annotations.length === 0) {
             noAnnotationsMsg.style.display = 'block';
             return;
         }
-        
+
         noAnnotationsMsg.style.display = 'none';
-        
+
         // Clear existing items (except the no-annotations message)
         Array.from(listContainer.children).forEach(child => {
             if (child.id !== 'no-annotations-msg') {
                 child.remove();
             }
         });
-        
+
         // Add annotation items
         annotations.forEach(annotation => {
             const item = document.createElement('div');
             item.className = 'list-group-item list-group-item-action p-2';
             item.setAttribute('data-annotation-id', annotation.annotation_id);
-            
+
             const profitClass = annotation.profit_loss_pct >= 0 ? 'text-success' : 'text-danger';
             const directionIcon = annotation.direction === 'LONG' ? 'fa-arrow-up' : 'fa-arrow-down';
-            
+
             item.innerHTML = `
                 <div class="d-flex justify-content-between align-items-start">
                     <div class="flex-grow-1">
@@ -151,25 +151,25 @@
                     </div>
                 </div>
             `;
-            
+
             // Add event listeners
-            item.querySelector('.view-annotation-btn').addEventListener('click', function(e) {
+            item.querySelector('.view-annotation-btn').addEventListener('click', function (e) {
                 e.stopPropagation();
                 viewAnnotation(annotation);
             });
-            
-            item.querySelector('.generate-testcase-btn').addEventListener('click', function(e) {
+
+            item.querySelector('.generate-testcase-btn').addEventListener('click', function (e) {
                 e.stopPropagation();
                 generateTestCase(annotation.annotation_id);
             });
-            
-            item.querySelector('.delete-annotation-btn').addEventListener('click', function(e) {
+
+            item.querySelector('.delete-annotation-btn').addEventListener('click', function (e) {
                 e.stopPropagation();
                 console.log('=== Delete annotation button clicked ===');
                 console.log('Annotation ID:', annotation.annotation_id);
                 console.log('window.deleteAnnotation type:', typeof window.deleteAnnotation);
                 console.log('window object keys containing delete:', Object.keys(window).filter(k => k.includes('delete')));
-                
+
                 // Use window.deleteAnnotation to ensure we get the global function
                 if (typeof window.deleteAnnotation === 'function') {
                     console.log('Calling window.deleteAnnotation...');
@@ -193,14 +193,14 @@
                     }
                 }
             });
-            
+
             listContainer.appendChild(item);
         });
-        
+
         // Update annotation count
         document.getElementById('annotation-count').textContent = annotations.length;
     }
-    
+
     function viewAnnotation(annotation) {
         // Navigate to annotation time and highlight it
         if (appState.timeNavigator) {
@@ -210,25 +210,25 @@
             appState.chartManager.highlightAnnotation(annotation.annotation_id);
         }
     }
-    
+
     function generateTestCase(annotationId) {
         fetch('/api/generate-test-case', {
             method: 'POST',
-            headers: {'Content-Type': 'application/json'},
-            body: JSON.stringify({annotation_id: annotationId})
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ annotation_id: annotationId })
         })
-        .then(response => response.json())
-        .then(data => {
-            if (data.success) {
-                showSuccess('Test case generated successfully');
-            } else {
-                showError('Failed to generate test case: ' + data.error.message);
-            }
-        })
-        .catch(error => {
-            showError('Network error: ' + error.message);
-        });
+            .then(response => response.json())
+            .then(data => {
+                if (data.success) {
+                    showSuccess('Test case generated successfully');
+                } else {
+                    showError('Failed to generate test case: ' + data.error.message);
+                }
+            })
+            .catch(error => {
+                showError('Network error: ' + error.message);
+            });
     }
-    
+
     // Note: deleteAnnotation is defined in annotation_dashboard.html to avoid duplication
-</script>
+</script>
\ No newline at end of file
diff --git a/ANNOTATE/web/templates/components/training_panel.html b/ANNOTATE/web/templates/components/training_panel.html
index 6e4d733..0b186ef 100644
--- a/ANNOTATE/web/templates/components/training_panel.html
+++ b/ANNOTATE/web/templates/components/training_panel.html
@@ -102,32 +102,65 @@
 </div>
 
 <script>
-    // Load available models on page load
+    // Load available models on page load with polling for async loading
+    let modelLoadingPollInterval = null;
+    
     function loadAvailableModels() {
         fetch('/api/available-models')
             .then(response => response.json())
             .then(data => {
                 const modelSelect = document.getElementById('model-select');
-                modelSelect.innerHTML = '';
-
-                if (data.success && data.models.length > 0) {
-                    data.models.forEach(model => {
-                        const option = document.createElement('option');
-                        option.value = model;
-                        option.textContent = model;
-                        modelSelect.appendChild(option);
-                    });
+                
+                if (data.loading) {
+                    // Models still loading - show loading message and poll
+                    modelSelect.innerHTML = '<option value="">🔄 Loading models...</option>';
+                    
+                    // Start polling if not already polling
+                    if (!modelLoadingPollInterval) {
+                        console.log('Models loading in background, will poll for completion...');
+                        modelLoadingPollInterval = setInterval(loadAvailableModels, 2000); // Poll every 2 seconds
+                    }
                 } else {
-                    const option = document.createElement('option');
-                    option.value = '';
-                    option.textContent = 'No models available';
-                    modelSelect.appendChild(option);
+                    // Models loaded - stop polling
+                    if (modelLoadingPollInterval) {
+                        clearInterval(modelLoadingPollInterval);
+                        modelLoadingPollInterval = null;
+                    }
+                    
+                    modelSelect.innerHTML = '';
+
+                    if (data.success && data.models.length > 0) {
+                        // Show success notification
+                        if (window.showSuccess) {
+                            window.showSuccess(`✅ ${data.models.length} models loaded and ready for training`);
+                        }
+                        
+                        data.models.forEach(model => {
+                            const option = document.createElement('option');
+                            option.value = model;
+                            option.textContent = model;
+                            modelSelect.appendChild(option);
+                        });
+                        
+                        console.log(`✅ Models loaded: ${data.models.join(', ')}`);
+                    } else {
+                        const option = document.createElement('option');
+                        option.value = '';
+                        option.textContent = 'No models available';
+                        modelSelect.appendChild(option);
+                    }
                 }
             })
             .catch(error => {
                 console.error('Error loading models:', error);
                 const modelSelect = document.getElementById('model-select');
                 modelSelect.innerHTML = '<option value="">Error loading models</option>';
+                
+                // Stop polling on error
+                if (modelLoadingPollInterval) {
+                    clearInterval(modelLoadingPollInterval);
+                    modelLoadingPollInterval = null;
+                }
             });
     }