UI
This commit is contained in:
Dobromir Popov
@@ -310,19 +310,16 @@ class HistoricalDataLoader:
|
||||
}
|
||||
binance_timeframe = timeframe_map.get(timeframe, '1m')
|
||||
|
||||
# Build API parameters
|
||||
# Build initial API parameters
|
||||
params = {
|
||||
'symbol': binance_symbol,
|
||||
'interval': binance_timeframe,
|
||||
'limit': min(limit, 1000) # Binance max is 1000
|
||||
'interval': binance_timeframe
|
||||
}
|
||||
|
||||
# Add time range parameters if specified
|
||||
if direction == 'before' and end_time:
|
||||
# Get data ending at end_time
|
||||
params['endTime'] = int(end_time.timestamp() * 1000)
|
||||
elif direction == 'after' and start_time:
|
||||
# Get data starting at start_time
|
||||
params['startTime'] = int(start_time.timestamp() * 1000)
|
||||
elif start_time:
|
||||
params['startTime'] = int(start_time.timestamp() * 1000)
|
||||
@@ -335,40 +332,91 @@ class HistoricalDataLoader:
|
||||
|
||||
logger.info(f"Fetching from Binance: {symbol} {timeframe} (direction={direction}, limit={limit})")
|
||||
|
||||
response = rate_limiter.make_request('binance_api', url, 'GET', params=params)
|
||||
# Pagination variables
|
||||
all_dfs = []
|
||||
total_fetched = 0
|
||||
is_fetching_forward = (direction == 'after')
|
||||
|
||||
if response is None or response.status_code != 200:
|
||||
logger.warning(f"Binance API failed, trying MEXC...")
|
||||
# Try MEXC as fallback
|
||||
return self._fetch_from_mexc_with_time_range(
|
||||
symbol, timeframe, start_time, end_time, limit, direction
|
||||
)
|
||||
# Fetch loop
|
||||
while total_fetched < limit:
|
||||
# Calculate batch limit (max 1000 per request)
|
||||
batch_limit = min(limit - total_fetched, 1000)
|
||||
params['limit'] = batch_limit
|
||||
|
||||
response = rate_limiter.make_request('binance_api', url, 'GET', params=params)
|
||||
|
||||
if response is None or response.status_code != 200:
|
||||
if total_fetched == 0:
|
||||
logger.warning(f"Binance API failed, trying MEXC...")
|
||||
return self._fetch_from_mexc_with_time_range(
|
||||
symbol, timeframe, start_time, end_time, limit, direction
|
||||
)
|
||||
else:
|
||||
logger.warning("Binance API failed during pagination, returning partial data")
|
||||
break
|
||||
|
||||
data = response.json()
|
||||
|
||||
if not data:
|
||||
if total_fetched == 0:
|
||||
logger.warning(f"No data returned from Binance for {symbol} {timeframe}")
|
||||
return None
|
||||
else:
|
||||
break
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(data, columns=[
|
||||
'timestamp', 'open', 'high', 'low', 'close', 'volume',
|
||||
'close_time', 'quote_volume', 'trades', 'taker_buy_base',
|
||||
'taker_buy_quote', 'ignore'
|
||||
])
|
||||
|
||||
# Process columns
|
||||
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
|
||||
for col in ['open', 'high', 'low', 'close', 'volume']:
|
||||
df[col] = df[col].astype(float)
|
||||
|
||||
# Keep only OHLCV columns
|
||||
df = df[['timestamp', 'open', 'high', 'low', 'close', 'volume']]
|
||||
df = df.set_index('timestamp')
|
||||
df = df.sort_index()
|
||||
|
||||
if df.empty:
|
||||
break
|
||||
|
||||
all_dfs.append(df)
|
||||
total_fetched += len(df)
|
||||
|
||||
# Prepare for next batch
|
||||
if total_fetched >= limit:
|
||||
break
|
||||
|
||||
# Update params for next iteration
|
||||
if is_fetching_forward:
|
||||
# Next batch starts after the last candle
|
||||
last_ts = df.index[-1]
|
||||
params['startTime'] = int(last_ts.value / 10**6) + 1
|
||||
# Check if we exceeded end_time
|
||||
if 'endTime' in params and params['startTime'] > params['endTime']:
|
||||
break
|
||||
else:
|
||||
# Next batch ends before the first candle
|
||||
first_ts = df.index[0]
|
||||
params['endTime'] = int(first_ts.value / 10**6) - 1
|
||||
# Check if we exceeded start_time
|
||||
if 'startTime' in params and params['endTime'] < params['startTime']:
|
||||
break
|
||||
|
||||
data = response.json()
|
||||
|
||||
if not data:
|
||||
logger.warning(f"No data returned from Binance for {symbol} {timeframe}")
|
||||
# Combine all batches
|
||||
if not all_dfs:
|
||||
return None
|
||||
|
||||
final_df = pd.concat(all_dfs)
|
||||
final_df = final_df.sort_index()
|
||||
final_df = final_df[~final_df.index.duplicated(keep='first')]
|
||||
|
||||
# Convert to DataFrame
|
||||
df = pd.DataFrame(data, columns=[
|
||||
'timestamp', 'open', 'high', 'low', 'close', 'volume',
|
||||
'close_time', 'quote_volume', 'trades', 'taker_buy_base',
|
||||
'taker_buy_quote', 'ignore'
|
||||
])
|
||||
|
||||
# Process columns
|
||||
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
|
||||
for col in ['open', 'high', 'low', 'close', 'volume']:
|
||||
df[col] = df[col].astype(float)
|
||||
|
||||
# Keep only OHLCV columns
|
||||
df = df[['timestamp', 'open', 'high', 'low', 'close', 'volume']]
|
||||
df = df.set_index('timestamp')
|
||||
df = df.sort_index()
|
||||
|
||||
logger.info(f" Fetched {len(df)} candles from Binance for {symbol} {timeframe}")
|
||||
return df
|
||||
logger.info(f" Fetched {len(final_df)} candles from Binance for {symbol} {timeframe} (requested {limit})")
|
||||
return final_df
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching from exchange API: {e}")
|
||||
|
||||
Reference in New Issue
Block a user