wip

core/data_provider.py

@@ -1623,7 +1623,7 @@ class DataProvider:
             logger.error(f"Error getting market state at time: {e}")
             return {}
 
-    def get_historical_data(self, symbol: str, timeframe: str, limit: int = 1000, refresh: bool = False, allow_stale_cache: bool = False) -> Optional[pd.DataFrame]:
+    def get_historical_data(self, symbol: str, timeframe: str, limit: int = 1000, refresh: bool = False, allow_stale_cache: bool = False, persist: bool = True) -> Optional[pd.DataFrame]:
         """Get historical OHLCV data.
         - Prefer cached data for low latency.
         - If cache is empty or refresh=True, fetch real data from exchanges.
@@ -1635,6 +1635,7 @@ class DataProvider:
             limit: Number of candles to return
             refresh: Force refresh from exchange
             allow_stale_cache: Allow loading stale cache (for startup performance)
+            persist: Whether to save fetched data to DuckDB (default: True). Set False for high-frequency polling.
         """
         try:
             # Serve from cache when available
@@ -1644,7 +1645,7 @@ class DataProvider:
                     return cached_df.tail(limit)
 
             # Try loading from DuckDB first (fast Parquet queries)
-            if allow_stale_cache:
+            if allow_stale_cache and not refresh:
                 cached_df = self._load_from_duckdb(symbol, timeframe, limit=1500)
                 if cached_df is not None and not cached_df.empty:
                     logger.info(f"Loaded {len(cached_df)} candles from DuckDB for {symbol} {timeframe} (startup mode)")
@@ -1662,8 +1663,8 @@ class DataProvider:
             if df is not None and not df.empty:
                 df = self._ensure_datetime_index(df)
                 
-                # Store in DuckDB (Parquet + SQL in one)
-                if self.duckdb_storage:
+                # Store in DuckDB (Parquet + SQL in one) - Only if persist is True
+                if self.duckdb_storage and persist:
                     try:
                         self.duckdb_storage.store_ohlcv_data(symbol, timeframe, df)
                     except Exception as e:
@@ -1680,7 +1681,12 @@ class DataProvider:
                     combined_df = combined_df.sort_index()
                     self.cached_data[symbol][timeframe] = combined_df.tail(1500)
                 
-                logger.info(f"Stored {len(df)} candles for {symbol} {timeframe} (DuckDB + memory cache)")
+                if persist:
+                    logger.info(f"Stored {len(df)} candles for {symbol} {timeframe} (DuckDB + memory cache)")
+                else:
+                    # Less verbose for high-frequency polling
+                    logger.debug(f"Updated memory cache with {len(df)} candles for {symbol} {timeframe}")
+                    
                 return self.cached_data[symbol][timeframe].tail(limit)
 
             logger.warning(f"No real data available for {symbol} {timeframe} at request time")