popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cob ma data for models

Browse Source
This commit is contained in:
Dobromir Popov
2025-09-09 02:07:04 +03:00
parent 317c703ea0
commit 729e0bccb1
3 changed files with 132 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
core/orchestrator.py
View File

@@ -220,6 +220,7 @@ class TradingOrchestrator:
# Load historical data for models and RL training
self._load_historical_data_for_models()
# SINGLE-USE FUNCTION - Called only once in codebase
def _initialize_ml_models(self):
"""Initialize ML models for enhanced trading"""
try:
@@ -592,6 +593,7 @@ class TradingOrchestrator:
logger.error(f"Error in extrema trainer prediction: {e}")
return None
# UNUSED FUNCTION - Not called anywhere in codebase
def get_memory_usage(self) -> float:
return 30.0 # MB
@@ -623,6 +625,7 @@ class TradingOrchestrator:
logger.error(f"Error in transformer prediction: {e}")
return None
# UNUSED FUNCTION - Not called anywhere in codebase
def get_memory_usage(self) -> float:
return 60.0 # MB estimate for transformer
@@ -649,6 +652,7 @@ class TradingOrchestrator:
logger.error(f"Error in decision model prediction: {e}")
return None
# UNUSED FUNCTION - Not called anywhere in codebase
def get_memory_usage(self) -> float:
return 40.0 # MB estimate for decision model
@@ -666,6 +670,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error initializing ML models: {e}")
# UNUSED FUNCTION - Not called anywhere in codebase
def update_model_loss(self, model_name: str, current_loss: float, best_loss: float = None):
"""Update model loss and potentially best loss"""
if model_name in self.model_states:
@@ -676,6 +681,7 @@ class TradingOrchestrator:
self.model_states[model_name]['best_loss'] = current_loss
logger.debug(f"Updated {model_name} loss: current={current_loss:.4f}, best={self.model_states[model_name]['best_loss']:.4f}")
# UNUSED FUNCTION - Not called anywhere in codebase
def checkpoint_saved(self, model_name: str, checkpoint_data: Dict[str, Any]):
"""Callback when a model checkpoint is saved"""
if model_name in self.model_states:
@@ -689,6 +695,7 @@ class TradingOrchestrator:
self.model_states[model_name]['best_loss'] = saved_loss
logger.info(f"New best loss for {model_name}: {saved_loss:.4f}")
# UNUSED FUNCTION - Not called anywhere in codebase
def get_recent_predictions(self, limit: int = 10) -> List[Dict[str, Any]]:
"""Get recent predictions from all models for data streaming"""
try:
@@ -728,6 +735,7 @@ class TradingOrchestrator:
logger.debug(f"Error getting recent predictions: {e}")
return []
# UNUSED FUNCTION - Not called anywhere in codebase
def _save_orchestrator_state(self):
"""Save the current state of the orchestrator, including model states."""
state = {
@@ -742,6 +750,7 @@ class TradingOrchestrator:
json.dump(state, f, indent=4)
logger.info(f"Orchestrator state saved to {save_path}")
# UNUSED FUNCTION - Not called anywhere in codebase
def _load_orchestrator_state(self):
"""Load the orchestrator state from a saved file."""
save_path = os.path.join(self.config.paths.get('checkpoint_dir', './models/saved'), 'orchestrator_state.json')
@@ -777,6 +786,7 @@ class TradingOrchestrator:
self.trade_loop_task = asyncio.create_task(self._trading_decision_loop())
logger.info("Continuous trading loop initiated.")
# UNUSED FUNCTION - Not called anywhere in codebase
def _initialize_cob_integration(self):
"""Initialize COB integration for real-time market microstructure data"""
if COB_INTEGRATION_AVAILABLE:
@@ -807,12 +817,14 @@ class TradingOrchestrator:
else:
logger.warning("COB Integration not initialized. Cannot start streaming.")
# UNUSED FUNCTION - Not called anywhere in codebase
def _start_cob_matrix_worker(self):
"""Start a background worker to continuously update COB matrices for models"""
if not self.cob_integration:
logger.warning("COB Integration not available, cannot start COB matrix worker.")
return
# UNUSED FUNCTION - Not called anywhere in codebase
def matrix_worker():
logger.info("COB Matrix Worker started.")
while self.realtime_processing:
@@ -851,6 +863,7 @@ class TradingOrchestrator:
matrix_thread = threading.Thread(target=matrix_worker, daemon=True)
matrix_thread.start()
# UNUSED FUNCTION - Not called anywhere in codebase
def _update_cob_matrix_for_symbol(self, symbol: str):
"""Updates the COB matrix and features for a specific symbol."""
if not self.cob_integration:
@@ -967,6 +980,7 @@ class TradingOrchestrator:
logger.error(f"Error generating COB DQN features for {symbol}: {e}")
return None
# UNUSED FUNCTION - Not called anywhere in codebase
def _on_cob_cnn_features(self, symbol: str, cob_data: Dict):
"""Callback for when new COB CNN features are available"""
if not self.realtime_processing:
@@ -984,6 +998,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error in _on_cob_cnn_features for {symbol}: {e}")
# UNUSED FUNCTION - Not called anywhere in codebase
def _on_cob_dqn_features(self, symbol: str, cob_data: Dict):
"""Callback for when new COB DQN features are available"""
if not self.realtime_processing:
@@ -1001,6 +1016,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error in _on_cob_dqn_features for {symbol}: {e}")
# UNUSED FUNCTION - Not called anywhere in codebase
def _on_cob_dashboard_data(self, symbol: str, cob_data: Dict):
"""Callback for when new COB data is available for the dashboard"""
if not self.realtime_processing:
@@ -1013,20 +1029,24 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error in _on_cob_dashboard_data for {symbol}: {e}")
# UNUSED FUNCTION - Not called anywhere in codebase
def get_cob_features(self, symbol: str) -> Optional[np.ndarray]:
"""Get the latest COB features for CNN model"""
return self.latest_cob_features.get(symbol)
# UNUSED FUNCTION - Not called anywhere in codebase
def get_cob_state(self, symbol: str) -> Optional[np.ndarray]:
"""Get the latest COB state for DQN model"""
return self.latest_cob_state.get(symbol)
# SINGLE-USE FUNCTION - Called only once in codebase
def get_cob_snapshot(self, symbol: str) -> Optional[COBSnapshot]:
"""Get the latest raw COB snapshot for a symbol"""
if self.cob_integration:
return self.cob_integration.get_latest_cob_snapshot(symbol)
return None
# SINGLE-USE FUNCTION - Called only once in codebase
def get_cob_feature_matrix(self, symbol: str, sequence_length: int = 60) -> Optional[np.ndarray]:
"""Get a sequence of COB CNN features for sequence models"""
if symbol not in self.cob_feature_history or not self.cob_feature_history[symbol]:
@@ -1059,6 +1079,7 @@ class TradingOrchestrator:
# Weight normalization removed - handled by ModelManager
# UNUSED FUNCTION - Not called anywhere in codebase
def add_decision_callback(self, callback):
"""Add a callback function to be called when decisions are made"""
self.decision_callbacks.append(callback)
@@ -1322,6 +1343,7 @@ class TradingOrchestrator:
logger.debug(f"Error building RL state for {symbol}: {e}")
return None
# SINGLE-USE FUNCTION - Called only once in codebase
def _get_cob_state(self, symbol: str) -> Optional[np.ndarray]:
"""Build COB state vector for COB RL agent"""
try:
@@ -1478,6 +1500,7 @@ class TradingOrchestrator:
logger.error(f"Error creating RL state for {symbol}: {e}")
return None
# SINGLE-USE FUNCTION - Called only once in codebase
def _combine_predictions(self, symbol: str, price: float,
predictions: List[Prediction],
timestamp: datetime) -> TradingDecision:
@@ -1593,6 +1616,7 @@ class TradingOrchestrator:
current_position_pnl=0.0
)
# SINGLE-USE FUNCTION - Called only once in codebase
def _get_timeframe_weight(self, timeframe: str) -> float:
"""Get importance weight for a timeframe"""
# Higher timeframes get more weight in decision making
@@ -1605,12 +1629,14 @@ class TradingOrchestrator:
# Model performance and weight adaptation removed - handled by ModelManager
# Use self.model_manager for all model performance tracking
# UNUSED FUNCTION - Not called anywhere in codebase
def get_recent_decisions(self, symbol: str, limit: int = 10) -> List[TradingDecision]:
"""Get recent decisions for a symbol"""
if symbol in self.recent_decisions:
return self.recent_decisions[symbol][-limit:]
return []
# UNUSED FUNCTION - Not called anywhere in codebase
def get_performance_metrics(self) -> Dict[str, Any]:
"""Get performance metrics for the orchestrator"""
return {
@@ -1625,6 +1651,7 @@ class TradingOrchestrator:
}
}
# UNUSED FUNCTION - Not called anywhere in codebase
def get_model_states(self) -> Dict[str, Dict]:
"""Get current model states with REAL checkpoint data - SSOT for dashboard"""
try:
@@ -1749,6 +1776,7 @@ class TradingOrchestrator:
'extrema_trainer': {'initial_loss': None, 'current_loss': None, 'best_loss': None, 'checkpoint_loaded': False}
}
# SINGLE-USE FUNCTION - Called only once in codebase
def _initialize_decision_fusion(self):
"""Initialize the decision fusion neural network for learning model effectiveness"""
try:
@@ -1767,6 +1795,7 @@ class TradingOrchestrator:
self.fc3 = nn.Linear(hidden_size, 3) # BUY, SELL, HOLD
self.dropout = nn.Dropout(0.2)
# UNUSED FUNCTION - Not called anywhere in codebase
def forward(self, x):
x = torch.relu(self.fc1(x))
x = self.dropout(x)
@@ -1781,6 +1810,7 @@ class TradingOrchestrator:
logger.warning(f"Decision fusion initialization failed: {e}")
self.decision_fusion_enabled = False
# SINGLE-USE FUNCTION - Called only once in codebase
def _initialize_enhanced_training_system(self):
"""Initialize the enhanced real-time training system"""
try:
@@ -1825,6 +1855,7 @@ class TradingOrchestrator:
self.training_enabled = False
self.enhanced_training_system = None
# SINGLE-USE FUNCTION - Called only once in codebase
def start_enhanced_training(self):
"""Start the enhanced real-time training system"""
try:
@@ -1845,6 +1876,7 @@ class TradingOrchestrator:
logger.error(f"Error starting enhanced training: {e}")
return False
# UNUSED FUNCTION - Not called anywhere in codebase
def stop_enhanced_training(self):
"""Stop the enhanced real-time training system"""
try:
@@ -1858,6 +1890,7 @@ class TradingOrchestrator:
logger.error(f"Error stopping enhanced training: {e}")
return False
# UNUSED FUNCTION - Not called anywhere in codebase
def get_enhanced_training_stats(self) -> Dict[str, Any]:
"""Get enhanced training system statistics with orchestrator integration"""
try:
@@ -1954,6 +1987,7 @@ class TradingOrchestrator:
'error': str(e)
}
# UNUSED FUNCTION - Not called anywhere in codebase
def set_training_dashboard(self, dashboard):
"""Set the dashboard reference for the training system"""
try:
@@ -1972,6 +2006,7 @@ class TradingOrchestrator:
logger.error(f"Error getting universal data stream: {e}")
return None
# UNUSED FUNCTION - Not called anywhere in codebase
def get_universal_data_for_model(self, model_type: str = 'cnn') -> Optional[Dict[str, Any]]:
"""Get formatted universal data for specific model types"""
try:
@@ -2014,6 +2049,7 @@ class TradingOrchestrator:
except Exception:
return False
# SINGLE-USE FUNCTION - Called only once in codebase
def _calculate_aggressiveness_thresholds(self, current_pnl: float, symbol: str) -> tuple:
"""Calculate confidence thresholds based on aggressiveness settings"""
# Base thresholds
@@ -2036,6 +2072,7 @@ class TradingOrchestrator:
return entry_threshold, exit_threshold
# SINGLE-USE FUNCTION - Called only once in codebase
def _apply_pnl_feedback(self, action: str, confidence: float, current_pnl: float,
symbol: str, reasoning: dict) -> tuple:
"""Apply P&L-based feedback to decision making"""
@@ -2069,6 +2106,7 @@ class TradingOrchestrator:
logger.debug(f"Error applying P&L feedback: {e}")
return action, confidence
# SINGLE-USE FUNCTION - Called only once in codebase
def _calculate_dynamic_entry_aggressiveness(self, symbol: str) -> float:
"""Calculate dynamic entry aggressiveness based on recent performance"""
try:
@@ -2097,6 +2135,7 @@ class TradingOrchestrator:
logger.debug(f"Error calculating dynamic entry aggressiveness: {e}")
return 0.5
# SINGLE-USE FUNCTION - Called only once in codebase
def _calculate_dynamic_exit_aggressiveness(self, symbol: str, current_pnl: float) -> float:
"""Calculate dynamic exit aggressiveness based on P&L and market conditions"""
try:
@@ -2119,11 +2158,13 @@ class TradingOrchestrator:
logger.debug(f"Error calculating dynamic exit aggressiveness: {e}")
return 0.5
# UNUSED FUNCTION - Not called anywhere in codebase
def set_trading_executor(self, trading_executor):
"""Set the trading executor for position tracking"""
self.trading_executor = trading_executor
logger.info("Trading executor set for position tracking and P&L feedback")
# SINGLE-USE FUNCTION - Called only once in codebase
def _get_current_price(self, symbol: str) -> float:
"""Get current price for symbol"""
try:
@@ -2169,6 +2210,7 @@ class TradingOrchestrator:
else:
return 1000.0
# SINGLE-USE FUNCTION - Called only once in codebase
def _generate_fallback_prediction(self, symbol: str) -> Dict[str, Any]:
"""Generate fallback prediction when models fail"""
try:
@@ -2189,6 +2231,7 @@ class TradingOrchestrator:
'model': 'fallback'
}
# UNUSED FUNCTION - Not called anywhere in codebase
def capture_dqn_prediction(self, symbol: str, action_idx: int, confidence: float, price: float, q_values: List[float] = None):
"""Capture DQN prediction for dashboard visualization"""
try:
@@ -2205,6 +2248,7 @@ class TradingOrchestrator:
except Exception as e:
logger.debug(f"Error capturing DQN prediction: {e}")
# UNUSED FUNCTION - Not called anywhere in codebase
def capture_cnn_prediction(self, symbol: str, direction: int, confidence: float, current_price: float, predicted_price: float):
"""Capture CNN prediction for dashboard visualization"""
try:
@@ -2270,6 +2314,7 @@ class TradingOrchestrator:
logger.warning(f"Data stream monitor initialization failed: {e}")
self.data_stream_monitor = None
# UNUSED FUNCTION - Not called anywhere in codebase
def start_data_stream(self) -> bool:
"""Start data streaming if not already active."""
try:
@@ -2282,6 +2327,7 @@ class TradingOrchestrator:
logger.error(f"Failed to start data stream: {e}")
return False
# UNUSED FUNCTION - Not called anywhere in codebase
def stop_data_stream(self) -> bool:
"""Stop data streaming if active."""
try:
@@ -2292,6 +2338,7 @@ class TradingOrchestrator:
logger.error(f"Failed to stop data stream: {e}")
return False
# SINGLE-USE FUNCTION - Called only once in codebase
def get_data_stream_status(self) -> Dict[str, any]:
"""Return current data stream status and buffer sizes."""
status = {
@@ -2310,6 +2357,7 @@ class TradingOrchestrator:
pass
return status
# UNUSED FUNCTION - Not called anywhere in codebase
def save_data_snapshot(self, filepath: str = None) -> str:
"""Save a snapshot of current data stream buffers to a file.
@@ -2337,6 +2385,7 @@ class TradingOrchestrator:
logger.error(f"Failed to save data snapshot: {e}")
raise
# UNUSED FUNCTION - Not called anywhere in codebase
def get_stream_summary(self) -> Dict[str, any]:
"""Get a summary of current data stream activity."""
status = self.get_data_stream_status()
@@ -2360,6 +2409,7 @@ class TradingOrchestrator:
return summary
# UNUSED FUNCTION - Not called anywhere in codebase
def get_cob_data(self, symbol: str, limit: int = 300) -> List:
"""Get COB data for a symbol with specified limit."""
try:
@@ -2370,6 +2420,7 @@ class TradingOrchestrator:
logger.error(f"Error getting COB data: {e}")
return []
# SINGLE-USE FUNCTION - Called only once in codebase
def _load_historical_data_for_models(self):
"""Load 300 historical candles for all required timeframes and symbols for model training"""
logger.info("Loading 300 historical candles for model training and RL context...")
@@ -2425,6 +2476,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error in historical data loading: {e}")
# SINGLE-USE FUNCTION - Called only once in codebase
def _initialize_models_with_historical_data(self, symbols_timeframes: List[Tuple[str, str]]):
"""Initialize all NN models with historical data using data provider's normalized methods"""
try:
@@ -2458,6 +2510,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error initializing models with historical data: {e}")
# SINGLE-USE FUNCTION - Called only once in codebase
def _initialize_cnn_with_provider_data(self):
"""Initialize CNN using data provider's normalized feature extraction"""
try:
@@ -2488,6 +2541,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error initializing CNN with provider data: {e}")
# SINGLE-USE FUNCTION - Called only once in codebase
def _initialize_dqn_with_provider_data(self, symbols_timeframes: List[Tuple[str, str]]):
"""Initialize DQN using data provider's normalized state vector creation"""
try:
@@ -2505,6 +2559,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error initializing DQN with provider data: {e}")
# SINGLE-USE FUNCTION - Called only once in codebase
def _initialize_transformer_with_provider_data(self, symbols_timeframes: List[Tuple[str, str]]):
"""Initialize Transformer using data provider's normalized sequence creation"""
try:
@@ -2522,6 +2577,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error initializing Transformer with provider data: {e}")
# SINGLE-USE FUNCTION - Called only once in codebase
def _initialize_decision_with_provider_data(self, symbol_features: Dict[str, Dict[str, pd.DataFrame]]):
"""Initialize Decision Fusion using data provider's feature aggregation"""
try:
@@ -2551,6 +2607,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error initializing Decision Fusion with provider data: {e}")
# UNUSED FUNCTION - Not called anywhere in codebase
def get_ohlcv_data(self, symbol: str, timeframe: str, limit: int = 300) -> List:
"""Get OHLCV data for a symbol with specified timeframe and limit."""
try: