wip training and inference stats

core/data_provider.py

@@ -512,18 +512,23 @@ class DataProvider:
             # Get raw ticks for the target second
             target_ticks = []
             
-            for tick in self.cob_raw_ticks[symbol]:
-                tick_timestamp = tick['timestamp']
+            # FIXED: Create a copy of the deque to avoid mutation during iteration
+            if symbol in self.cob_raw_ticks:
+                # Create a safe copy of the deque to iterate over
+                ticks_copy = list(self.cob_raw_ticks[symbol])
                 
-                # Handle both datetime and float timestamps
-                if isinstance(tick_timestamp, datetime):
-                    tick_time = tick_timestamp.timestamp()
-                else:
-                    tick_time = float(tick_timestamp)
-                
-                # Check if tick is in target second
-                if target_second <= tick_time < target_second + 1:
-                    target_ticks.append(tick)
+                for tick in ticks_copy:
+                    tick_timestamp = tick['timestamp']
+                    
+                    # Handle both datetime and float timestamps
+                    if isinstance(tick_timestamp, datetime):
+                        tick_time = tick_timestamp.timestamp()
+                    else:
+                        tick_time = float(tick_timestamp)
+                    
+                    # Check if tick is in target second
+                    if target_second <= tick_time < target_second + 1:
+                        target_ticks.append(tick)
             
             if not target_ticks:
                 return
@@ -563,7 +568,14 @@ class DataProvider:
                 current_imbalance = self._calculate_cob_imbalance(latest_cob, price_range)
             
             # Get historical COB data for timeframe calculations
-            historical_cob_data = list(self.cob_raw_ticks[symbol]) if symbol in self.cob_raw_ticks else []
+            # FIXED: Create a safe copy to avoid deque mutation during iteration
+            historical_cob_data = []
+            if symbol in self.cob_raw_ticks:
+                try:
+                    historical_cob_data = list(self.cob_raw_ticks[symbol])
+                except Exception as e:
+                    logger.debug(f"Error copying COB raw ticks for {symbol}: {e}")
+                    historical_cob_data = []
             
             # Calculate imbalances for different timeframes using COB data
             imbalances = {
@@ -4112,12 +4124,18 @@ class DataProvider:
             target_ticks = []
             
             # Filter ticks for the target second
-            for tick in self.cob_raw_ticks[symbol]:
-                tick_time = tick.get('timestamp', 0)
-                if isinstance(tick_time, (int, float)):
-                    tick_second = int(tick_time)
-                    if tick_second == target_second:
-                        target_ticks.append(tick)
+            # FIXED: Create a safe copy to avoid deque mutation during iteration
+            if symbol in self.cob_raw_ticks:
+                try:
+                    ticks_copy = list(self.cob_raw_ticks[symbol])
+                    for tick in ticks_copy:
+                        tick_time = tick.get('timestamp', 0)
+                        if isinstance(tick_time, (int, float)):
+                            tick_second = int(tick_time)
+                            if tick_second == target_second:
+                                target_ticks.append(tick)
+                except Exception as e:
+                    logger.debug(f"Error copying COB raw ticks for {symbol}: {e}")
             
             if not target_ticks:
                 return