UI and stability
This commit is contained in:
Dobromir Popov
@ -2589,24 +2589,11 @@ class TradingOrchestrator:
|
||||
|
||||
# Method 3: Dictionary with feature data
|
||||
if isinstance(model_input, dict):
|
||||
# Check if dictionary is empty
|
||||
# Check if dictionary is empty - this is the main issue!
|
||||
if not model_input:
|
||||
logger.warning(f"Empty dictionary passed as model_input for {model_name}, using fallback")
|
||||
# Try to use data provider to build state as fallback
|
||||
if hasattr(self, 'data_provider'):
|
||||
try:
|
||||
base_data = self.data_provider.build_base_data_input('ETH/USDT')
|
||||
if base_data and hasattr(base_data, 'get_feature_vector'):
|
||||
state = base_data.get_feature_vector()
|
||||
if isinstance(state, np.ndarray):
|
||||
logger.debug(f"Used data provider fallback for empty dict in {model_name}")
|
||||
return state
|
||||
except Exception as e:
|
||||
logger.debug(f"Data provider fallback failed for empty dict in {model_name}: {e}")
|
||||
|
||||
# Final fallback: return default state
|
||||
logger.warning(f"Using default state for empty dict in {model_name}")
|
||||
return np.zeros(403, dtype=np.float32) # Default state size
|
||||
logger.warning(f"Empty dictionary passed as model_input for {model_name}, using data provider fallback")
|
||||
# Use data provider to build proper state as fallback
|
||||
return self._generate_fresh_state_fallback(model_name)
|
||||
|
||||
# Try to extract features from dictionary
|
||||
if 'features' in model_input:
|
||||
@ -2629,7 +2616,8 @@ class TradingOrchestrator:
|
||||
if feature_list:
|
||||
return np.array(feature_list, dtype=np.float32)
|
||||
else:
|
||||
logger.warning(f"No numerical features found in dictionary for {model_name}, using fallback")
|
||||
logger.warning(f"No numerical features found in dictionary for {model_name}, using data provider fallback")
|
||||
return self._generate_fresh_state_fallback(model_name)
|
||||
|
||||
# Method 4: List or tuple
|
||||
if isinstance(model_input, (list, tuple)):
|
||||
@ -2642,24 +2630,57 @@ class TradingOrchestrator:
|
||||
if isinstance(model_input, (int, float)):
|
||||
return np.array([model_input], dtype=np.float32)
|
||||
|
||||
# Method 6: Try to use data provider to build state
|
||||
if hasattr(self, 'data_provider'):
|
||||
try:
|
||||
base_data = self.data_provider.build_base_data_input('ETH/USDT')
|
||||
if base_data and hasattr(base_data, 'get_feature_vector'):
|
||||
state = base_data.get_feature_vector()
|
||||
if isinstance(state, np.ndarray):
|
||||
logger.debug(f"Used data provider fallback for {model_name}")
|
||||
return state
|
||||
except Exception as e:
|
||||
logger.debug(f"Data provider fallback failed for {model_name}: {e}")
|
||||
|
||||
logger.warning(f"Cannot convert model_input to RL state for {model_name}: {type(model_input)}")
|
||||
return None
|
||||
# Method 6: Final fallback - generate fresh state
|
||||
logger.warning(f"Cannot convert model_input to RL state for {model_name}: {type(model_input)}, using fresh state fallback")
|
||||
return self._generate_fresh_state_fallback(model_name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error converting model_input to RL state for {model_name}: {e}")
|
||||
return None
|
||||
return self._generate_fresh_state_fallback(model_name)
|
||||
|
||||
def _generate_fresh_state_fallback(self, model_name: str) -> np.ndarray:
|
||||
"""Generate a fresh state from current market data when model_input is empty/invalid"""
|
||||
try:
|
||||
# Try to use data provider to build fresh state
|
||||
if hasattr(self, 'data_provider') and self.data_provider:
|
||||
try:
|
||||
# Build fresh BaseDataInput with current market data
|
||||
base_data = self.data_provider.build_base_data_input('ETH/USDT')
|
||||
if base_data and hasattr(base_data, 'get_feature_vector'):
|
||||
state = base_data.get_feature_vector()
|
||||
if isinstance(state, np.ndarray) and state.size > 0:
|
||||
logger.info(f"Generated fresh state for {model_name} from data provider: shape={state.shape}")
|
||||
return state
|
||||
except Exception as e:
|
||||
logger.debug(f"Data provider fresh state generation failed for {model_name}: {e}")
|
||||
|
||||
# Try to get state from model registry
|
||||
if hasattr(self, 'model_registry') and self.model_registry:
|
||||
try:
|
||||
model_interface = self.model_registry.models.get(model_name)
|
||||
if model_interface and hasattr(model_interface, 'get_current_state'):
|
||||
state = model_interface.get_current_state()
|
||||
if isinstance(state, np.ndarray) and state.size > 0:
|
||||
logger.info(f"Generated fresh state for {model_name} from model interface: shape={state.shape}")
|
||||
return state
|
||||
except Exception as e:
|
||||
logger.debug(f"Model interface fresh state generation failed for {model_name}: {e}")
|
||||
|
||||
# Final fallback: create a reasonable default state with proper dimensions
|
||||
# Use the expected state size for DQN models (403 features)
|
||||
default_state_size = 403
|
||||
if 'cnn' in model_name.lower():
|
||||
default_state_size = 500 # Larger for CNN models
|
||||
elif 'cob' in model_name.lower():
|
||||
default_state_size = 2000 # Much larger for COB models
|
||||
|
||||
logger.warning(f"Using default zero state for {model_name} with size {default_state_size}")
|
||||
return np.zeros(default_state_size, dtype=np.float32)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating fresh state fallback for {model_name}: {e}")
|
||||
# Ultimate fallback
|
||||
return np.zeros(403, dtype=np.float32)
|
||||
|
||||
async def _train_cnn_model(self, model, model_name: str, record: Dict, prediction: Dict, reward: float) -> bool:
|
||||
"""Train CNN model directly (no adapter)"""
|
||||
@ -3785,6 +3806,35 @@ class TradingOrchestrator:
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error setting training dashboard: {e}")
|
||||
|
||||
def set_cold_start_training_enabled(self, enabled: bool) -> bool:
|
||||
"""Enable or disable cold start training (excessive training during cold start)
|
||||
|
||||
Args:
|
||||
enabled: Whether to enable cold start training
|
||||
|
||||
Returns:
|
||||
bool: True if setting was applied successfully
|
||||
"""
|
||||
try:
|
||||
# Store the setting
|
||||
self.cold_start_enabled = enabled
|
||||
|
||||
# Adjust training frequency based on cold start mode
|
||||
if enabled:
|
||||
# High frequency training during cold start
|
||||
self.training_frequency = 'high'
|
||||
logger.info("ORCHESTRATOR: Cold start training ENABLED - Excessive training on every signal")
|
||||
else:
|
||||
# Normal training frequency
|
||||
self.training_frequency = 'normal'
|
||||
logger.info("ORCHESTRATOR: Cold start training DISABLED - Normal training frequency")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error setting cold start training: {e}")
|
||||
return False
|
||||
|
||||
def get_universal_data_stream(self, current_time: Optional[datetime] = None):
|
||||
"""Get universal data stream for external consumers like dashboard - DELEGATED to data provider"""
|
||||
|
||||
Reference in New Issue
Block a user