wip
This commit is contained in:
Dobromir Popov
@@ -9,6 +9,7 @@ import sys
|
||||
import logging
|
||||
import torch.nn.functional as F
|
||||
import time
|
||||
from typing import Tuple, List, Dict, Any, Optional
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
@@ -305,6 +305,7 @@ class TradingOrchestrator:
|
||||
self.data_provider = data_provider or DataProvider()
|
||||
self.universal_adapter = UniversalDataAdapter(self.data_provider)
|
||||
self.model_manager = None # Will be initialized later if needed
|
||||
self.model_registry = model_registry # Model registry for dynamic model management
|
||||
self.enhanced_rl_training = enhanced_rl_training
|
||||
|
||||
# Set primary trading symbol
|
||||
@@ -462,6 +463,45 @@ class TradingOrchestrator:
|
||||
self.inference_logger = None # Will be initialized later if needed
|
||||
self.db_manager = None # Will be initialized later if needed
|
||||
|
||||
# CRITICAL: Initialize model_states dictionary to track model performance
|
||||
self.model_states: Dict[str, Dict[str, Any]] = {
|
||||
"dqn": {
|
||||
"initial_loss": None,
|
||||
"current_loss": None,
|
||||
"best_loss": None,
|
||||
"checkpoint_loaded": False,
|
||||
"checkpoint_filename": None
|
||||
},
|
||||
"cnn": {
|
||||
"initial_loss": None,
|
||||
"current_loss": None,
|
||||
"best_loss": None,
|
||||
"checkpoint_loaded": False,
|
||||
"checkpoint_filename": None
|
||||
},
|
||||
"extrema_trainer": {
|
||||
"initial_loss": None,
|
||||
"current_loss": None,
|
||||
"best_loss": None,
|
||||
"checkpoint_loaded": False,
|
||||
"checkpoint_filename": None
|
||||
},
|
||||
"decision_fusion": {
|
||||
"initial_loss": None,
|
||||
"current_loss": None,
|
||||
"best_loss": None,
|
||||
"checkpoint_loaded": False,
|
||||
"checkpoint_filename": None
|
||||
},
|
||||
"transformer": {
|
||||
"initial_loss": None,
|
||||
"current_loss": None,
|
||||
"best_loss": None,
|
||||
"checkpoint_loaded": False,
|
||||
"checkpoint_filename": None
|
||||
}
|
||||
}
|
||||
|
||||
# ENHANCED: Real-time Training System Integration
|
||||
self.enhanced_training_system = (
|
||||
None # Will be set to EnhancedRealtimeTrainingSystem if available
|
||||
@@ -578,9 +618,16 @@ class TradingOrchestrator:
|
||||
# Wait 24 hours before next cleanup
|
||||
await asyncio.sleep(24 * 60 * 60) # 24 hours in seconds
|
||||
|
||||
# Create and start the cleanup task
|
||||
self._db_cleanup_task = asyncio.create_task(cleanup_task())
|
||||
logger.info("Database cleanup scheduler started - will run every 24 hours")
|
||||
# Try to get or create event loop
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
# Create and start the cleanup task
|
||||
self._db_cleanup_task = loop.create_task(cleanup_task())
|
||||
logger.info("Database cleanup scheduler started - will run every 24 hours")
|
||||
except RuntimeError:
|
||||
# No running event loop - schedule for later
|
||||
logger.info("No event loop available yet - database cleanup will be scheduled when loop starts")
|
||||
self._db_cleanup_task = None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to schedule database cleanup: {e}")
|
||||
@@ -639,7 +686,7 @@ class TradingOrchestrator:
|
||||
"""
|
||||
try:
|
||||
if self.cob_integration is None:
|
||||
logger.warning("COB integration not initialized - cannot start sync")
|
||||
logger.info("COB integration not initialized - skipping sync")
|
||||
return
|
||||
|
||||
# Create async task to start COB integration
|
||||
@@ -651,9 +698,16 @@ class TradingOrchestrator:
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start COB integration sync: {e}")
|
||||
|
||||
# Create the task (will be executed when event loop is running)
|
||||
self._cob_sync_task = asyncio.create_task(start_cob_task())
|
||||
logger.info("COB integration sync task created - will start when event loop is available")
|
||||
# Try to get or create event loop
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
# Create the task (will be executed when event loop is running)
|
||||
self._cob_sync_task = loop.create_task(start_cob_task())
|
||||
logger.info("COB integration sync task created - will start when event loop is available")
|
||||
except RuntimeError:
|
||||
# No running event loop - schedule for later
|
||||
logger.info("No event loop available yet - COB integration will be started when loop starts")
|
||||
self._cob_sync_task = None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize COB integration sync: {e}")
|
||||
@@ -941,50 +995,55 @@ class TradingOrchestrator:
|
||||
self.cob_rl_agent = None
|
||||
|
||||
|
||||
# CRITICAL: Register models with the model registry
|
||||
logger.info("Registering models with model registry...")
|
||||
logger.info(
|
||||
f"Model registry before registration: {len(self.model_registry.models)} models"
|
||||
)
|
||||
# CRITICAL: Register models with the model registry (if available)
|
||||
if self.model_registry is not None:
|
||||
logger.info("Registering models with model registry...")
|
||||
logger.info(
|
||||
f"Model registry before registration: {len(self.model_registry.models)} models"
|
||||
)
|
||||
|
||||
# Import model interfaces
|
||||
# These are now imported at the top of the file
|
||||
# Import model interfaces
|
||||
# These are now imported at the top of the file
|
||||
|
||||
# Register RL Agent
|
||||
if self.rl_agent:
|
||||
try:
|
||||
rl_interface = RLAgentInterface(self.rl_agent, name="dqn_agent")
|
||||
if self.model_registry.register_model(rl_interface):
|
||||
logger.info("RL Agent registered successfully")
|
||||
else:
|
||||
logger.error("Failed to register RL Agent with registry")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to register RL Agent: {e}")
|
||||
# Register RL Agent
|
||||
if self.rl_agent:
|
||||
try:
|
||||
rl_interface = RLAgentInterface(self.rl_agent, name="dqn_agent")
|
||||
if self.model_registry.register_model(rl_interface):
|
||||
logger.info("RL Agent registered successfully")
|
||||
else:
|
||||
logger.error("Failed to register RL Agent with registry")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to register RL Agent: {e}")
|
||||
|
||||
# Register CNN Model
|
||||
if self.cnn_model:
|
||||
try:
|
||||
cnn_interface = CNNModelInterface(self.cnn_model, name="cnn_model")
|
||||
if self.model_registry.register_model(cnn_interface):
|
||||
logger.info("CNN Model registered successfully")
|
||||
else:
|
||||
logger.error("Failed to register CNN Model with registry")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to register CNN Model: {e}")
|
||||
# Register CNN Model
|
||||
if self.cnn_model:
|
||||
try:
|
||||
cnn_interface = CNNModelInterface(self.cnn_model, name="cnn_model")
|
||||
if self.model_registry.register_model(cnn_interface):
|
||||
logger.info("CNN Model registered successfully")
|
||||
else:
|
||||
logger.error("Failed to register CNN Model with registry")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to register CNN Model: {e}")
|
||||
|
||||
# Register Extrema Trainer
|
||||
if self.extrema_trainer:
|
||||
try:
|
||||
extrema_interface = ExtremaTrainerInterface(self.extrema_trainer, name="extrema_trainer")
|
||||
if self.model_registry.register_model(extrema_interface):
|
||||
logger.info("Extrema Trainer registered successfully")
|
||||
else:
|
||||
logger.error("Failed to register Extrema Trainer with registry")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to register Extrema Trainer: {e}")
|
||||
# Register Extrema Trainer
|
||||
if self.extrema_trainer:
|
||||
try:
|
||||
extrema_interface = ExtremaTrainerInterface(self.extrema_trainer, name="extrema_trainer")
|
||||
if self.model_registry.register_model(extrema_interface):
|
||||
logger.info("Extrema Trainer registered successfully")
|
||||
else:
|
||||
logger.error("Failed to register Extrema Trainer with registry")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to register Extrema Trainer: {e}")
|
||||
else:
|
||||
logger.info("Model registry not available - skipping model registration")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error initializing ML models: {e}")
|
||||
import traceback
|
||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||
|
||||
def get_model_training_stats(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Get current model training statistics for dashboard display"""
|
||||
@@ -1362,8 +1421,8 @@ class TradingOrchestrator:
|
||||
def register_model_dynamically(self, model_name: str, model_interface):
|
||||
"""Register a new model dynamically and set up its toggle state"""
|
||||
try:
|
||||
# Register with model registry
|
||||
if self.model_registry.register_model(model_interface):
|
||||
# Register with model registry (if available)
|
||||
if self.model_registry is not None and self.model_registry.register_model(model_interface):
|
||||
# Initialize toggle state for the new model
|
||||
if model_name not in self.model_toggle_states:
|
||||
self.model_toggle_states[model_name] = {
|
||||
@@ -1373,6 +1432,8 @@ class TradingOrchestrator:
|
||||
logger.info(f"Registered new model dynamically: {model_name}")
|
||||
self._save_ui_state()
|
||||
return True
|
||||
elif self.model_registry is None:
|
||||
logger.warning(f"Cannot register model {model_name} - model registry not available")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error registering model {model_name} dynamically: {e}")
|
||||
|
||||
Reference in New Issue
Block a user