folder stricture reorganize

core/data_provider.py

@@ -189,6 +189,12 @@ class DataProvider:
         logger.info(f"Timeframes: {self.timeframes}")
         logger.info("Centralized data distribution enabled")
         logger.info("Pivot-based normalization system enabled")
+        
+        # Rate limiting
+        self.last_request_time = {}
+        self.request_interval = 0.2  # 200ms between requests
+        self.retry_delay = 60  # 1 minute retry delay for 451 errors
+        self.max_retries = 3
     
     def _ensure_datetime_index(self, df: pd.DataFrame) -> pd.DataFrame:
         """Ensure dataframe has proper datetime index"""
@@ -2466,4 +2472,46 @@ class DataProvider:
             # For now, return synthetic features since full implementation would be complex
             return self.generate_synthetic_bom_features(symbol)[70:]  # Last 50 features
         except:
-            return [0.0] * 50
+            return [0.0] * 50
+    
+    def _handle_rate_limit(self, url: str):
+        """Handle rate limiting with exponential backoff"""
+        current_time = time.time()
+        
+        # Check if we need to wait
+        if url in self.last_request_time:
+            time_since_last = current_time - self.last_request_time[url]
+            if time_since_last < self.request_interval:
+                sleep_time = self.request_interval - time_since_last
+                logger.info(f"Rate limiting: sleeping {sleep_time:.2f}s")
+                time.sleep(sleep_time)
+        
+        self.last_request_time[url] = time.time()
+    
+    def _make_request_with_retry(self, url: str, params: dict = None):
+        """Make HTTP request with retry logic for 451 errors"""
+        for attempt in range(self.max_retries):
+            try:
+                self._handle_rate_limit(url)
+                response = requests.get(url, params=params, timeout=30)
+                
+                if response.status_code == 451:
+                    logger.warning(f"Rate limit hit (451), attempt {attempt + 1}/{self.max_retries}")
+                    if attempt < self.max_retries - 1:
+                        sleep_time = self.retry_delay * (2 ** attempt)  # Exponential backoff
+                        logger.info(f"Waiting {sleep_time}s before retry...")
+                        time.sleep(sleep_time)
+                        continue
+                    else:
+                        logger.error("Max retries reached, using cached data")
+                        return None
+                
+                response.raise_for_status()
+                return response
+                
+            except Exception as e:
+                logger.error(f"Request failed (attempt {attempt + 1}): {e}")
+                if attempt < self.max_retries - 1:
+                    time.sleep(5 * (attempt + 1))
+                
+        return None