popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more training

Browse Source
This commit is contained in:
Dobromir Popov
2025-05-27 01:46:15 +03:00
parent 97d348d517
commit 2ba0406b9f
7 changed files with 1708 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
core/data_provider.py
View File

@ -137,9 +137,16 @@ class DataProvider:
logger.info(f"Using cached data for {symbol} {timeframe}")
return cached_data.tail(limit)
# Fetch from API
logger.info(f"Fetching historical data for {symbol} {timeframe}")
df = self._fetch_from_binance(symbol, timeframe, limit)
# Check if we need to preload 300s of data for first load
should_preload = self._should_preload_data(symbol, timeframe, limit)
if should_preload:
logger.info(f"Preloading 300s of data for {symbol} {timeframe}")
df = self._preload_300s_data(symbol, timeframe)
else:
# Fetch from API with requested limit
logger.info(f"Fetching historical data for {symbol} {timeframe}")
df = self._fetch_from_binance(symbol, timeframe, limit)
if df is not None and not df.empty:
# Add technical indicators
@ -154,7 +161,8 @@ class DataProvider:
self.historical_data[symbol] = {}
self.historical_data[symbol][timeframe] = df
return df
# Return requested amount
return df.tail(limit)
logger.warning(f"No data received for {symbol} {timeframe}")
return None
@ -163,6 +171,124 @@ class DataProvider:
logger.error(f"Error fetching historical data for {symbol} {timeframe}: {e}")
return None
def _should_preload_data(self, symbol: str, timeframe: str, limit: int) -> bool:
"""Determine if we should preload 300s of data"""
try:
# Check if we have any cached data
if self.cache_enabled:
cached_data = self._load_from_cache(symbol, timeframe)
if cached_data is not None and len(cached_data) > 0:
return False # Already have some data
# Check if we have data in memory
if (symbol in self.historical_data and
timeframe in self.historical_data[symbol] and
len(self.historical_data[symbol][timeframe]) > 0):
return False # Already have data in memory
# Calculate if 300s worth of data would be more than requested limit
timeframe_seconds = self.timeframe_seconds.get(timeframe, 60)
candles_in_300s = 300 // timeframe_seconds
# Preload if we need more than the requested limit or if it's a short timeframe
if candles_in_300s > limit or timeframe in ['1s', '1m']:
return True
return False
except Exception as e:
logger.error(f"Error determining if should preload data: {e}")
return False
def _preload_300s_data(self, symbol: str, timeframe: str) -> Optional[pd.DataFrame]:
"""Preload 300 seconds worth of data for better initial performance"""
try:
# Calculate how many candles we need for 300 seconds
timeframe_seconds = self.timeframe_seconds.get(timeframe, 60)
candles_needed = max(300 // timeframe_seconds, 100) # At least 100 candles
# For very short timeframes, limit to reasonable amount
if timeframe == '1s':
candles_needed = min(candles_needed, 300) # Max 300 1s candles
elif timeframe == '1m':
candles_needed = min(candles_needed, 60) # Max 60 1m candles (1 hour)
else:
candles_needed = min(candles_needed, 500) # Max 500 candles for other timeframes
logger.info(f"Preloading {candles_needed} candles for {symbol} {timeframe} (300s worth)")
# Fetch the data
df = self._fetch_from_binance(symbol, timeframe, candles_needed)
if df is not None and not df.empty:
logger.info(f"Successfully preloaded {len(df)} candles for {symbol} {timeframe}")
return df
else:
logger.warning(f"Failed to preload data for {symbol} {timeframe}")
return None
except Exception as e:
logger.error(f"Error preloading 300s data for {symbol} {timeframe}: {e}")
return None
def preload_all_symbols_data(self, timeframes: List[str] = None) -> Dict[str, Dict[str, bool]]:
"""Preload 300s of data for all symbols and timeframes"""
try:
if timeframes is None:
timeframes = self.timeframes
preload_results = {}
for symbol in self.symbols:
preload_results[symbol] = {}
for timeframe in timeframes:
try:
logger.info(f"Preloading data for {symbol} {timeframe}")
# Check if we should preload
if self._should_preload_data(symbol, timeframe, 100):
df = self._preload_300s_data(symbol, timeframe)
if df is not None and not df.empty:
# Add technical indicators
df = self._add_technical_indicators(df)
# Cache the data
if self.cache_enabled:
self._save_to_cache(df, symbol, timeframe)
# Store in memory
if symbol not in self.historical_data:
self.historical_data[symbol] = {}
self.historical_data[symbol][timeframe] = df
preload_results[symbol][timeframe] = True
logger.info(f"✅ Preloaded {len(df)} candles for {symbol} {timeframe}")
else:
preload_results[symbol][timeframe] = False
logger.warning(f"❌ Failed to preload {symbol} {timeframe}")
else:
preload_results[symbol][timeframe] = True # Already have data
logger.info(f"⏭️ Skipped preloading {symbol} {timeframe} (already have data)")
except Exception as e:
logger.error(f"Error preloading {symbol} {timeframe}: {e}")
preload_results[symbol][timeframe] = False
# Log summary
total_pairs = len(self.symbols) * len(timeframes)
successful_pairs = sum(1 for symbol_results in preload_results.values()
for success in symbol_results.values() if success)
logger.info(f"Preloading completed: {successful_pairs}/{total_pairs} symbol-timeframe pairs loaded")
return preload_results
except Exception as e:
logger.error(f"Error in preload_all_symbols_data: {e}")
return {}
def _fetch_from_binance(self, symbol: str, timeframe: str, limit: int) -> Optional[pd.DataFrame]:
"""Fetch data from Binance API"""
try: