popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replay system

Browse Source
This commit is contained in:
Dobromir Popov
2025-07-20 12:37:02 +03:00
parent 469269e809
commit 12865fd3ef
13 changed files with 6132 additions and 465 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
core/data_provider.py
View File

@ -503,8 +503,10 @@ class DataProvider:
return None
def _fetch_from_binance(self, symbol: str, timeframe: str, limit: int) -> Optional[pd.DataFrame]:
"""Fetch data from Binance API (primary data source) with HTTP 451 error handling"""
"""Fetch data from Binance API with robust rate limiting and error handling"""
try:
from .api_rate_limiter import get_rate_limiter
# Convert symbol format
binance_symbol = symbol.replace('/', '').upper()
@ -515,7 +517,18 @@ class DataProvider:
}
binance_timeframe = timeframe_map.get(timeframe, '1h')
# API request with timeout and better headers
# Use rate limiter for API requests
rate_limiter = get_rate_limiter()
# Check if we can make request
can_request, wait_time = rate_limiter.can_make_request('binance_api')
if not can_request:
logger.debug(f"Binance rate limited, waiting {wait_time:.1f}s for {symbol} {timeframe}")
if wait_time > 30: # If wait is too long, use fallback
return self._get_fallback_data(symbol, timeframe, limit)
time.sleep(min(wait_time, 5)) # Cap wait at 5 seconds
# API request with rate limiter
url = "https://api.binance.com/api/v3/klines"
params = {
'symbol': binance_symbol,
@ -523,20 +536,15 @@ class DataProvider:
'limit': limit
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept': 'application/json',
'Connection': 'keep-alive'
}
response = rate_limiter.make_request('binance_api', url, 'GET', params=params)
response = requests.get(url, params=params, headers=headers, timeout=10)
# Handle HTTP 451 (Unavailable For Legal Reasons) specifically
if response.status_code == 451:
logger.warning(f"Binance API returned 451 (blocked) for {symbol} {timeframe} - using fallback")
if response is None:
logger.warning(f"Binance API request failed for {symbol} {timeframe} - using fallback")
return self._get_fallback_data(symbol, timeframe, limit)
response.raise_for_status()
if response.status_code != 200:
logger.warning(f"Binance API returned {response.status_code} for {symbol} {timeframe}")
return self._get_fallback_data(symbol, timeframe, limit)
data = response.json()
@ -2619,35 +2627,57 @@ class DataProvider:
logger.info("Centralized data collection stopped")
def start_cob_data_collection(self):
"""Start COB (Consolidated Order Book) data collection"""
"""Start COB (Consolidated Order Book) data collection prioritizing WebSocket"""
if self.cob_collection_active:
logger.warning("COB data collection already active")
return
# Start real-time WebSocket streaming first (no rate limits)
if not self.is_streaming:
logger.info("Auto-starting WebSocket streaming for COB data (rate limit free)")
self.start_real_time_streaming()
self.cob_collection_active = True
self.cob_collection_thread = Thread(target=self._cob_collection_worker, daemon=True)
self.cob_collection_thread.start()
logger.info("COB data collection started")
logger.info("COB data collection started (WebSocket priority, minimal REST API)")
def _cob_collection_worker(self):
"""Worker thread for COB data collection"""
"""Worker thread for COB data collection with WebSocket priority"""
import requests
import time
import threading
logger.info("COB data collection worker started")
logger.info("COB data collection worker started (WebSocket-first approach)")
# Use separate threads for each symbol to achieve higher update frequency
# Significantly reduced frequency for REST API fallback only
def collect_symbol_data(symbol):
rest_api_fallback_count = 0
while self.cob_collection_active:
try:
self._collect_cob_data_for_symbol(symbol)
# Sleep for a very short time to achieve ~120 updates/sec across all symbols
# With 2 symbols, each can update at ~60/sec
time.sleep(0.016) # ~60 updates per second per symbol
# PRIORITY 1: Try to use WebSocket data first
ws_data = self._get_websocket_cob_data(symbol)
if ws_data and len(ws_data) > 0:
# Distribute WebSocket COB data
self._distribute_cob_data(symbol, ws_data)
rest_api_fallback_count = 0 # Reset fallback counter
# Much longer sleep since WebSocket provides real-time data
time.sleep(10.0) # Only check every 10 seconds when WS is working
else:
# FALLBACK: Only use REST API if WebSocket fails
rest_api_fallback_count += 1
if rest_api_fallback_count <= 3: # Limited fallback attempts
logger.warning(f"WebSocket COB data unavailable for {symbol}, using REST API fallback #{rest_api_fallback_count}")
self._collect_cob_data_for_symbol(symbol)
else:
logger.debug(f"Skipping REST API for {symbol} to prevent rate limits (WS data preferred)")
# Much longer sleep when using REST API fallback
time.sleep(30.0) # 30 seconds between REST calls
except Exception as e:
logger.error(f"Error collecting COB data for {symbol}: {e}")
time.sleep(1) # Short recovery time
time.sleep(10) # Longer recovery time
# Start a thread for each symbol
threads = []
@ -2664,22 +2694,84 @@ class DataProvider:
for thread in threads:
thread.join(timeout=1)
def _get_websocket_cob_data(self, symbol: str) -> Optional[dict]:
"""Get COB data from WebSocket streams (rate limit free)"""
try:
binance_symbol = symbol.replace('/', '').upper()
# Check if we have recent WebSocket tick data
if binance_symbol in self.tick_buffers and len(self.tick_buffers[binance_symbol]) > 10:
recent_ticks = list(self.tick_buffers[binance_symbol])[-50:] # Last 50 ticks
if recent_ticks:
# Calculate COB data from WebSocket ticks
latest_tick = recent_ticks[-1]
# Calculate bid/ask liquidity from recent tick patterns
buy_volume = sum(tick.volume for tick in recent_ticks if tick.side == 'buy')
sell_volume = sum(tick.volume for tick in recent_ticks if tick.side == 'sell')
total_volume = buy_volume + sell_volume
# Calculate metrics
imbalance = (buy_volume - sell_volume) / total_volume if total_volume > 0 else 0
avg_price = sum(tick.price for tick in recent_ticks) / len(recent_ticks)
# Create synthetic COB snapshot from WebSocket data
cob_snapshot = {
'symbol': symbol,
'timestamp': datetime.now(),
'source': 'websocket', # Mark as WebSocket source
'stats': {
'mid_price': latest_tick.price,
'avg_price': avg_price,
'imbalance': imbalance,
'buy_volume': buy_volume,
'sell_volume': sell_volume,
'total_volume': total_volume,
'tick_count': len(recent_ticks),
'best_bid': latest_tick.price - 0.01, # Approximate
'best_ask': latest_tick.price + 0.01, # Approximate
'spread_bps': 10 # Approximate spread
}
}
return cob_snapshot
return None
except Exception as e:
logger.debug(f"Error getting WebSocket COB data for {symbol}: {e}")
return None
def _collect_cob_data_for_symbol(self, symbol: str):
"""Collect COB data for a specific symbol using Binance REST API"""
"""Collect COB data for a specific symbol using Binance REST API with rate limiting"""
try:
import requests
import time
# Basic rate limiting check
if not self._handle_rate_limit(f"https://api.binance.com/api/v3/depth"):
logger.debug(f"Rate limited for {symbol}, skipping COB collection")
return
# Convert symbol format
binance_symbol = symbol.replace('/', '').upper()
# Get order book data
# Get order book data with reduced limit to minimize load
url = f"https://api.binance.com/api/v3/depth"
params = {
'symbol': binance_symbol,
'limit': 100 # Get top 100 levels
'limit': 50 # Reduced from 100 to 50 levels to reduce load
}
response = requests.get(url, params=params, timeout=5)
# Add headers to reduce detection
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept': 'application/json'
}
response = requests.get(url, params=params, headers=headers, timeout=10)
if response.status_code == 200:
order_book = response.json()
@ -2695,6 +2787,10 @@ class DataProvider:
# Distribute to COB data subscribers
self._distribute_cob_data(symbol, cob_snapshot)
elif response.status_code in [418, 429, 451]:
logger.warning(f"Rate limited (HTTP {response.status_code}) for {symbol} COB collection")
# Don't retry immediately, let the sleep in the worker handle it
else:
logger.debug(f"Failed to fetch COB data for {symbol}: {response.status_code}")
@ -2980,13 +3076,38 @@ class DataProvider:
import requests
import time
# Use Binance REST API for order book data
binance_symbol = symbol.replace('/', '')
url = f"https://api.binance.com/api/v3/depth?symbol={binance_symbol}&limit=500"
# Check rate limits before making request
if not self._handle_rate_limit(f"https://api.binance.com/api/v3/depth"):
logger.warning(f"Rate limited for {symbol}, using cached data")
# Return cached data if available
binance_symbol = symbol.replace('/', '').upper()
if binance_symbol in self.cob_data_cache and self.cob_data_cache[binance_symbol]:
return self.cob_data_cache[binance_symbol][-1]
return {}
response = requests.get(url, timeout=5)
# Use Binance REST API for order book data with reduced limit
binance_symbol = symbol.replace('/', '')
url = f"https://api.binance.com/api/v3/depth?symbol={binance_symbol}&limit=100" # Reduced from 500
# Add headers to reduce detection
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept': 'application/json'
}
response = requests.get(url, headers=headers, timeout=10)
if response.status_code == 200:
data = response.json()
elif response.status_code in [418, 429, 451]:
logger.warning(f"Rate limited (HTTP {response.status_code}) for {symbol}, using cached data")
# Return cached data if available
binance_symbol = symbol.replace('/', '').upper()
if binance_symbol in self.cob_data_cache and self.cob_data_cache[binance_symbol]:
return self.cob_data_cache[binance_symbol][-1]
return {}
else:
logger.warning(f"Failed to fetch COB data for {symbol}: {response.status_code}")
return {}
# Process order book data
bids = [[float(price), float(qty)] for price, qty in data.get('bids', [])]