COB data and dash

core/data_provider.py

@@ -180,6 +180,37 @@ class DataProvider:
         logger.info("Centralized data distribution enabled")
         logger.info("Pivot-based normalization system enabled")
     
+    def _ensure_datetime_index(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Ensure dataframe has proper datetime index"""
+        if df is None or df.empty:
+            return df
+            
+        try:
+            # If we already have a proper DatetimeIndex, return as is
+            if isinstance(df.index, pd.DatetimeIndex):
+                return df
+                
+            # If timestamp column exists, use it as index
+            if 'timestamp' in df.columns:
+                df['timestamp'] = pd.to_datetime(df['timestamp'])
+                df.set_index('timestamp', inplace=True)
+                return df
+                
+            # If we have a RangeIndex or other non-datetime index, create datetime index
+            if isinstance(df.index, pd.RangeIndex) or not isinstance(df.index, pd.DatetimeIndex):
+                # Use current time and work backwards for realistic timestamps
+                from datetime import datetime, timedelta
+                end_time = datetime.now()
+                start_time = end_time - timedelta(minutes=len(df))
+                df.index = pd.date_range(start=start_time, end=end_time, periods=len(df))
+                logger.debug(f"Converted RangeIndex to DatetimeIndex for {len(df)} records")
+                
+            return df
+            
+        except Exception as e:
+            logger.warning(f"Error ensuring datetime index: {e}")
+            return df
+    
     def get_historical_data(self, symbol: str, timeframe: str, limit: int = 1000, refresh: bool = False) -> Optional[pd.DataFrame]:
         """Get historical OHLCV data for a symbol and timeframe"""
         try:
@@ -188,6 +219,8 @@ class DataProvider:
                 if self.cache_enabled:
                     cached_data = self._load_from_cache(symbol, timeframe)
                     if cached_data is not None and len(cached_data) >= limit * 0.8:
+                        # Ensure proper datetime index for cached data
+                        cached_data = self._ensure_datetime_index(cached_data)
                         # logger.info(f"Using cached data for {symbol} {timeframe}")
                         return cached_data.tail(limit)
 
@@ -208,8 +241,11 @@ class DataProvider:
                     df = self._fetch_from_mexc(symbol, timeframe, limit)
 
             if df is not None and not df.empty:
-                # Add technical indicators
-                df = self._add_technical_indicators(df)
+                # Ensure proper datetime index
+                df = self._ensure_datetime_index(df)
+                
+                # Add technical indicators. temporarily disabled to save time as it is not working as expected.
+                # df = self._add_technical_indicators(df)
 
                 # Cache the data
                 if self.cache_enabled:
@@ -1151,9 +1187,21 @@ class DataProvider:
         try:
             cache_file = self.monthly_data_cache_dir / f"{symbol.replace('/', '')}_monthly_1m.parquet"
             if cache_file.exists():
-                df = pd.read_parquet(cache_file)
-                logger.info(f"Loaded {len(df)} 1m candles from cache for {symbol}")
-                return df
+                try:
+                    df = pd.read_parquet(cache_file)
+                    logger.info(f"Loaded {len(df)} 1m candles from cache for {symbol}")
+                    return df
+                except Exception as parquet_e:
+                    # Handle corrupted Parquet file
+                    if "Parquet magic bytes not found" in str(parquet_e) or "corrupted" in str(parquet_e).lower():
+                        logger.warning(f"Corrupted Parquet cache file for {symbol}, removing and returning None: {parquet_e}")
+                        try:
+                            cache_file.unlink()  # Delete corrupted file
+                        except Exception:
+                            pass
+                        return None
+                    else:
+                        raise parquet_e
             
             return None
             
@@ -1240,9 +1288,21 @@ class DataProvider:
                 # Check if cache is recent (less than 1 hour old)
                 cache_age = time.time() - cache_file.stat().st_mtime
                 if cache_age < 3600:  # 1 hour
-                    df = pd.read_parquet(cache_file)
-                    logger.debug(f"Loaded {len(df)} rows from cache for {symbol} {timeframe}")
-                    return df
+                    try:
+                        df = pd.read_parquet(cache_file)
+                        logger.debug(f"Loaded {len(df)} rows from cache for {symbol} {timeframe}")
+                        return df
+                    except Exception as parquet_e:
+                        # Handle corrupted Parquet file
+                        if "Parquet magic bytes not found" in str(parquet_e) or "corrupted" in str(parquet_e).lower():
+                            logger.warning(f"Corrupted Parquet cache file for {symbol} {timeframe}, removing and returning None: {parquet_e}")
+                            try:
+                                cache_file.unlink()  # Delete corrupted file
+                            except Exception:
+                                pass
+                            return None
+                        else:
+                            raise parquet_e
                 else:
                     logger.debug(f"Cache for {symbol} {timeframe} is too old ({cache_age/3600:.1f}h)")
             return None