infinite lowad WIP
This commit is contained in:
Dobromir Popov
@@ -79,11 +79,14 @@ class HistoricalDataLoader:
|
||||
if len(cached_df) >= min(limit, 100): # Use cached if we have at least 100 candles
|
||||
logger.debug(f"Using DataProvider cached data for {symbol} {timeframe} ({len(cached_df)} candles)")
|
||||
|
||||
# Filter by time range if specified
|
||||
if start_time or end_time:
|
||||
filtered_df = self._filter_by_time_range(cached_df.copy(), start_time, end_time)
|
||||
else:
|
||||
filtered_df = cached_df.tail(limit).copy()
|
||||
# Filter by time range with direction support
|
||||
filtered_df = self._filter_by_time_range(
|
||||
cached_df.copy(),
|
||||
start_time,
|
||||
end_time,
|
||||
direction,
|
||||
limit
|
||||
)
|
||||
|
||||
# Cache in memory
|
||||
self.memory_cache[cache_key] = (filtered_df, datetime.now())
|
||||
@@ -140,13 +143,14 @@ class HistoricalDataLoader:
|
||||
df = self.data_provider.cached_data[symbol][timeframe]
|
||||
|
||||
if df is not None and not df.empty:
|
||||
# Filter by time range if specified
|
||||
if start_time or end_time:
|
||||
df = self._filter_by_time_range(df, start_time, end_time)
|
||||
|
||||
# Limit number of candles
|
||||
if len(df) > limit:
|
||||
df = df.tail(limit)
|
||||
# Filter by time range with direction support
|
||||
df = self._filter_by_time_range(
|
||||
df.copy(),
|
||||
start_time,
|
||||
end_time,
|
||||
direction,
|
||||
limit
|
||||
)
|
||||
|
||||
# Cache in memory
|
||||
self.memory_cache[cache_key] = (df.copy(), datetime.now())
|
||||
@@ -182,10 +186,37 @@ class HistoricalDataLoader:
|
||||
self.memory_cache[cache_key] = (df.copy(), datetime.now())
|
||||
return df
|
||||
else:
|
||||
logger.info(f"No data in DuckDB, fetching from API for {symbol} {timeframe}")
|
||||
logger.info(f"📡 No data in DuckDB, fetching from exchange API for {symbol} {timeframe}")
|
||||
|
||||
# Fetch from exchange API with time range
|
||||
df = self._fetch_from_exchange_api(
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=limit,
|
||||
direction=direction
|
||||
)
|
||||
|
||||
if df is not None and not df.empty:
|
||||
# Store in DuckDB for future use
|
||||
if self.data_provider.duckdb_storage:
|
||||
stored_count = self.data_provider.duckdb_storage.store_ohlcv_data(
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
df=df
|
||||
)
|
||||
logger.info(f"💾 Stored {stored_count} new candles in DuckDB")
|
||||
|
||||
# Cache in memory
|
||||
self.memory_cache[cache_key] = (df.copy(), datetime.now())
|
||||
return df
|
||||
else:
|
||||
logger.warning(f"No data available from exchange API for {symbol} {timeframe}")
|
||||
return None
|
||||
|
||||
# Fetch from API and store in DuckDB
|
||||
logger.info(f"Fetching data from API for {symbol} {timeframe}")
|
||||
# Fetch from API and store in DuckDB (no time range specified)
|
||||
logger.info(f"Fetching latest data from API for {symbol} {timeframe}")
|
||||
df = self.data_provider.get_historical_data(
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
@@ -194,9 +225,14 @@ class HistoricalDataLoader:
|
||||
)
|
||||
|
||||
if df is not None and not df.empty:
|
||||
# Filter by time range if specified
|
||||
if start_time or end_time:
|
||||
df = self._filter_by_time_range(df, start_time, end_time)
|
||||
# Filter by time range with direction support
|
||||
df = self._filter_by_time_range(
|
||||
df.copy(),
|
||||
start_time,
|
||||
end_time,
|
||||
direction,
|
||||
limit
|
||||
)
|
||||
|
||||
# Cache in memory
|
||||
self.memory_cache[cache_key] = (df.copy(), datetime.now())
|
||||
@@ -211,14 +247,156 @@ class HistoricalDataLoader:
|
||||
logger.error(f"Error loading data for {symbol} {timeframe}: {e}")
|
||||
return None
|
||||
|
||||
def _fetch_from_exchange_api(self, symbol: str, timeframe: str,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
limit: int = 1000,
|
||||
direction: str = 'latest') -> Optional[pd.DataFrame]:
|
||||
"""
|
||||
Fetch historical data from exchange API (Binance/MEXC) with time range support
|
||||
|
||||
Args:
|
||||
symbol: Trading pair
|
||||
timeframe: Timeframe
|
||||
start_time: Start time for data range
|
||||
end_time: End time for data range
|
||||
limit: Maximum number of candles
|
||||
direction: 'latest', 'before', or 'after'
|
||||
|
||||
Returns:
|
||||
DataFrame with OHLCV data or None
|
||||
"""
|
||||
try:
|
||||
import requests
|
||||
from core.api_rate_limiter import get_rate_limiter
|
||||
|
||||
# Convert symbol format for Binance
|
||||
binance_symbol = symbol.replace('/', '').upper()
|
||||
|
||||
# Convert timeframe
|
||||
timeframe_map = {
|
||||
'1s': '1s', '1m': '1m', '5m': '5m', '15m': '15m', '30m': '30m',
|
||||
'1h': '1h', '4h': '4h', '1d': '1d'
|
||||
}
|
||||
binance_timeframe = timeframe_map.get(timeframe, '1m')
|
||||
|
||||
# Build API parameters
|
||||
params = {
|
||||
'symbol': binance_symbol,
|
||||
'interval': binance_timeframe,
|
||||
'limit': min(limit, 1000) # Binance max is 1000
|
||||
}
|
||||
|
||||
# Add time range parameters if specified
|
||||
if direction == 'before' and end_time:
|
||||
# Get data ending at end_time
|
||||
params['endTime'] = int(end_time.timestamp() * 1000)
|
||||
elif direction == 'after' and start_time:
|
||||
# Get data starting at start_time
|
||||
params['startTime'] = int(start_time.timestamp() * 1000)
|
||||
elif start_time:
|
||||
params['startTime'] = int(start_time.timestamp() * 1000)
|
||||
if end_time and direction != 'before':
|
||||
params['endTime'] = int(end_time.timestamp() * 1000)
|
||||
|
||||
# Use rate limiter
|
||||
rate_limiter = get_rate_limiter()
|
||||
url = "https://api.binance.com/api/v3/klines"
|
||||
|
||||
logger.info(f"Fetching from Binance: {symbol} {timeframe} (direction={direction}, limit={limit})")
|
||||
|
||||
response = rate_limiter.make_request('binance_api', url, 'GET', params=params)
|
||||
|
||||
if response is None or response.status_code != 200:
|
||||
logger.warning(f"Binance API failed, trying MEXC...")
|
||||
# Try MEXC as fallback
|
||||
return self._fetch_from_mexc_with_time_range(
|
||||
symbol, timeframe, start_time, end_time, limit, direction
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
|
||||
if not data:
|
||||
logger.warning(f"No data returned from Binance for {symbol} {timeframe}")
|
||||
return None
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(data, columns=[
|
||||
'timestamp', 'open', 'high', 'low', 'close', 'volume',
|
||||
'close_time', 'quote_volume', 'trades', 'taker_buy_base',
|
||||
'taker_buy_quote', 'ignore'
|
||||
])
|
||||
|
||||
# Process columns
|
||||
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
|
||||
for col in ['open', 'high', 'low', 'close', 'volume']:
|
||||
df[col] = df[col].astype(float)
|
||||
|
||||
# Keep only OHLCV columns
|
||||
df = df[['timestamp', 'open', 'high', 'low', 'close', 'volume']]
|
||||
df = df.set_index('timestamp')
|
||||
df = df.sort_index()
|
||||
|
||||
logger.info(f"✅ Fetched {len(df)} candles from Binance for {symbol} {timeframe}")
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching from exchange API: {e}")
|
||||
return None
|
||||
|
||||
def _fetch_from_mexc_with_time_range(self, symbol: str, timeframe: str,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
limit: int = 1000,
|
||||
direction: str = 'latest') -> Optional[pd.DataFrame]:
|
||||
"""Fetch from MEXC with time range support (fallback)"""
|
||||
try:
|
||||
# MEXC implementation would go here
|
||||
# For now, just return None to indicate unavailable
|
||||
logger.warning("MEXC time range fetch not implemented yet")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching from MEXC: {e}")
|
||||
return None
|
||||
|
||||
def _filter_by_time_range(self, df: pd.DataFrame,
|
||||
start_time: Optional[datetime],
|
||||
end_time: Optional[datetime]) -> pd.DataFrame:
|
||||
"""Filter DataFrame by time range"""
|
||||
if start_time:
|
||||
df = df[df.index >= start_time]
|
||||
if end_time:
|
||||
df = df[df.index <= end_time]
|
||||
end_time: Optional[datetime],
|
||||
direction: str = 'latest',
|
||||
limit: int = 500) -> pd.DataFrame:
|
||||
"""
|
||||
Filter DataFrame by time range with direction support
|
||||
|
||||
Args:
|
||||
df: DataFrame to filter
|
||||
start_time: Start time filter
|
||||
end_time: End time filter
|
||||
direction: 'latest', 'before', or 'after'
|
||||
limit: Maximum number of candles
|
||||
|
||||
Returns:
|
||||
Filtered DataFrame
|
||||
"""
|
||||
if direction == 'before' and end_time:
|
||||
# Get candles BEFORE end_time
|
||||
df = df[df.index < end_time]
|
||||
# Return the most recent N candles before end_time
|
||||
df = df.tail(limit)
|
||||
elif direction == 'after' and start_time:
|
||||
# Get candles AFTER start_time
|
||||
df = df[df.index > start_time]
|
||||
# Return the oldest N candles after start_time
|
||||
df = df.head(limit)
|
||||
else:
|
||||
# Default: filter by range
|
||||
if start_time:
|
||||
df = df[df.index >= start_time]
|
||||
if end_time:
|
||||
df = df[df.index <= end_time]
|
||||
# Return most recent candles
|
||||
if len(df) > limit:
|
||||
df = df.tail(limit)
|
||||
|
||||
return df
|
||||
|
||||
def get_multi_timeframe_data(self, symbol: str,
|
||||
|
||||
Reference in New Issue
Block a user