data stream

core/orchestrator.py

@@ -192,6 +192,9 @@ class TradingOrchestrator:
         self._initialize_cob_integration()
         self._initialize_decision_fusion()  # Initialize fusion system
         self._initialize_enhanced_training_system()  # Initialize real-time training
+        
+        # Initialize and start data stream monitor (single source of truth)
+        self._initialize_data_stream_monitor()
     
     def _initialize_ml_models(self):
         """Initialize ML models for enhanced trading"""
@@ -2192,4 +2195,112 @@ class TradingOrchestrator:
             return None
         except Exception as e:
             logger.error(f"Error getting COB RL prediction: {e}")
-            return None
+            return None
+
+    def _initialize_data_stream_monitor(self) -> None:
+        """Initialize the data stream monitor and start streaming immediately.
+        Managed by orchestrator to avoid external process control.
+        """
+        try:
+            from data_stream_monitor import get_data_stream_monitor
+            self.data_stream_monitor = get_data_stream_monitor(
+                orchestrator=self,
+                data_provider=self.data_provider,
+                training_system=getattr(self, 'training_manager', None)
+            )
+            if not getattr(self.data_stream_monitor, 'is_streaming', False):
+                self.data_stream_monitor.start_streaming()
+            logger.info("Data stream monitor initialized and started by orchestrator")
+        except Exception as e:
+            logger.warning(f"Data stream monitor initialization failed: {e}")
+            self.data_stream_monitor = None
+
+    def start_data_stream(self) -> bool:
+        """Start data streaming if not already active."""
+        try:
+            if not getattr(self, 'data_stream_monitor', None):
+                self._initialize_data_stream_monitor()
+            if self.data_stream_monitor and not self.data_stream_monitor.is_streaming:
+                self.data_stream_monitor.start_streaming()
+            return True
+        except Exception as e:
+            logger.error(f"Failed to start data stream: {e}")
+            return False
+
+    def stop_data_stream(self) -> bool:
+        """Stop data streaming if active."""
+        try:
+            if getattr(self, 'data_stream_monitor', None) and self.data_stream_monitor.is_streaming:
+                self.data_stream_monitor.stop_streaming()
+            return True
+        except Exception as e:
+            logger.error(f"Failed to stop data stream: {e}")
+            return False
+
+    def get_data_stream_status(self) -> Dict[str, any]:
+        """Return current data stream status and buffer sizes."""
+        status = {
+            'connected': False,
+            'streaming': False,
+            'buffers': {}
+        }
+        monitor = getattr(self, 'data_stream_monitor', None)
+        if not monitor:
+            return status
+        try:
+            status['connected'] = monitor.orchestrator is not None and monitor.data_provider is not None
+            status['streaming'] = bool(monitor.is_streaming)
+            status['buffers'] = {name: len(buf) for name, buf in monitor.data_streams.items()}
+        except Exception:
+            pass
+        return status
+
+    def save_data_snapshot(self, filepath: str = None) -> str:
+        """Save a snapshot of current data stream buffers to a file.
+        
+        Args:
+            filepath: Optional path for the snapshot file. If None, generates timestamped name.
+            
+        Returns:
+            Path to the saved snapshot file.
+        """
+        if not getattr(self, 'data_stream_monitor', None):
+            raise RuntimeError("Data stream monitor not initialized")
+        
+        if not filepath:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filepath = f"data_snapshots/snapshot_{timestamp}.json"
+        
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(filepath), exist_ok=True)
+        
+        try:
+            snapshot_data = self.data_stream_monitor.save_snapshot(filepath)
+            logger.info(f"Data snapshot saved to: {filepath}")
+            return filepath
+        except Exception as e:
+            logger.error(f"Failed to save data snapshot: {e}")
+            raise
+
+    def get_stream_summary(self) -> Dict[str, any]:
+        """Get a summary of current data stream activity."""
+        status = self.get_data_stream_status()
+        summary = {
+            'status': status,
+            'total_samples': sum(status.get('buffers', {}).values()),
+            'active_streams': [name for name, count in status.get('buffers', {}).items() if count > 0],
+            'last_update': datetime.now().isoformat()
+        }
+        
+        # Add some sample data if available
+        if getattr(self, 'data_stream_monitor', None):
+            try:
+                sample_data = {}
+                for stream_name, buffer in self.data_stream_monitor.data_streams.items():
+                    if len(buffer) > 0:
+                        sample_data[stream_name] = buffer[-1]  # Latest sample
+                summary['sample_data'] = sample_data
+            except Exception:
+                pass
+        
+        return summary