popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip COB data source - not ready

Browse Source
This commit is contained in:
Dobromir Popov
2025-08-08 00:49:13 +03:00
parent 0ce6e2691b
commit bd15bdc87d
7 changed files with 327 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
core/data_provider.py
View File

@ -1137,8 +1137,19 @@ class DataProvider:
# Extract timestamp and price from tick
if isinstance(tick, dict):
timestamp = tick.get('timestamp')
price = tick.get('price', tick.get('mid_price', 0))
volume = tick.get('volume', 1.0) # Default volume if not available
# Prefer explicit price if available, fallback to stats.mid_price
stats = tick.get('stats', {}) if isinstance(tick.get('stats', {}), dict) else {}
price = tick.get('price')
if not price:
price = tick.get('mid_price') or stats.get('mid_price', 0)
# Derive a volume proxy if not provided (use bid+ask volume from stats)
volume = tick.get('volume')
if volume is None:
bid_vol = stats.get('bid_volume', 0) or 0
ask_vol = stats.get('ask_volume', 0) or 0
volume = float(bid_vol) + float(ask_vol)
if volume == 0:
volume = 1.0 # Minimal placeholder to avoid zero-volume bars
else:
continue
@ -2221,14 +2232,40 @@ class DataProvider:
# Get latest COB data from cache
cob_data = self.get_latest_cob_data(symbol)
if cob_data and 'current_price' in cob_data:
if cob_data:
# Determine current price (prefer explicit field, fallback to stats.mid_price)
stats = cob_data.get('stats', {}) if isinstance(cob_data.get('stats', {}), dict) else {}
current_price = cob_data.get('current_price') or stats.get('mid_price', 0.0)
bucket_size = 1.0 if 'ETH' in symbol else 10.0
# Ensure price buckets exist; compute from bids/asks if missing
price_buckets = cob_data.get('price_buckets') or {}
if (not price_buckets) and current_price:
price_buckets = self._compute_price_buckets_from_snapshot(
current_price=current_price,
bucket_size=bucket_size,
bids=cob_data.get('bids', []),
asks=cob_data.get('asks', [])
)
# Build imbalance map (price -> imbalance) if not provided
bid_ask_imbalance = cob_data.get('bid_ask_imbalance') or {}
if not bid_ask_imbalance and price_buckets:
tmp = {}
for price, bucket in price_buckets.items():
bid_vol = float(bucket.get('bid_volume', 0.0) or 0.0)
ask_vol = float(bucket.get('ask_volume', 0.0) or 0.0)
denom = bid_vol + ask_vol
tmp[price] = (bid_vol - ask_vol) / denom if denom > 0 else 0.0
bid_ask_imbalance = tmp
return COBData(
symbol=symbol,
timestamp=datetime.now(),
current_price=cob_data['current_price'],
bucket_size=1.0 if 'ETH' in symbol else 10.0,
price_buckets=cob_data.get('price_buckets', {}),
bid_ask_imbalance=cob_data.get('bid_ask_imbalance', {}),
current_price=float(current_price or 0.0),
bucket_size=bucket_size,
price_buckets=price_buckets,
bid_ask_imbalance=bid_ask_imbalance,
volume_weighted_prices=cob_data.get('volume_weighted_prices', {}),
order_flow_metrics=cob_data.get('order_flow_metrics', {}),
ma_1s_imbalance=cob_data.get('ma_1s_imbalance', {}),
@ -2241,6 +2278,67 @@ class DataProvider:
logger.error(f"Error getting COB data object for {symbol}: {e}")
return None
def _compute_price_buckets_from_snapshot(
self,
current_price: float,
bucket_size: float,
bids: List[List[float]],
asks: List[List[float]]
) -> Dict[float, Dict[str, float]]:
"""Compute ±20 price buckets around current price from raw bids/asks.
Returns dict: price -> {bid_volume, ask_volume, total_volume, imbalance}
"""
try:
# Initialize bucket map for ±20 buckets
bucket_map: Dict[float, Dict[str, float]] = {}
if not current_price or bucket_size <= 0:
return bucket_map
# Center-aligned bucket prices
bucket_count = 20
for i in range(-bucket_count, bucket_count + 1):
price = (round(current_price / bucket_size) * bucket_size) + (i * bucket_size)
bucket_map[price] = {
'bid_volume': 0.0,
'ask_volume': 0.0,
'total_volume': 0.0,
'imbalance': 0.0,
}
# Aggregate bids
for level in (bids or [])[:200]:
try:
price, size = float(level[0]), float(level[1])
except Exception:
continue
bucket_price = round(price / bucket_size) * bucket_size
if bucket_price in bucket_map:
bucket_map[bucket_price]['bid_volume'] += size
# Aggregate asks
for level in (asks or [])[:200]:
try:
price, size = float(level[0]), float(level[1])
except Exception:
continue
bucket_price = round(price / bucket_size) * bucket_size
if bucket_price in bucket_map:
bucket_map[bucket_price]['ask_volume'] += size
# Compute totals and imbalance
for price, bucket in bucket_map.items():
bid_vol = float(bucket['bid_volume'])
ask_vol = float(bucket['ask_volume'])
total = bid_vol + ask_vol
bucket['total_volume'] = total
bucket['imbalance'] = (bid_vol - ask_vol) / total if total > 0 else 0.0
return bucket_map
except Exception as e:
logger.debug(f"Error computing price buckets: {e}")
return {}
def _add_basic_indicators(self, df: pd.DataFrame) -> pd.DataFrame:
@ -4278,13 +4376,46 @@ class DataProvider:
if symbol not in self.cob_data_cache:
self.cob_data_cache[symbol] = []
# Convert WebSocket format to standard format
# Convert WebSocket format to standard format and enrich stats if missing
bids_arr = [[bid['price'], bid['size']] for bid in cob_data.get('bids', [])[:50]]
asks_arr = [[ask['price'], ask['size']] for ask in cob_data.get('asks', [])[:50]]
stats_in = cob_data.get('stats', {}) if isinstance(cob_data.get('stats', {}), dict) else {}
# Derive stats when not provided by source
best_bid = max([b[0] for b in bids_arr], default=0)
best_ask = min([a[0] for a in asks_arr], default=0)
mid = stats_in.get('mid_price') or ((best_bid + best_ask) / 2.0 if best_bid > 0 and best_ask > 0 else 0)
total_bid_liq = sum([b[0] * b[1] for b in bids_arr]) # price*size USD approx
total_ask_liq = sum([a[0] * a[1] for a in asks_arr])
spread_bps = 0
if best_bid > 0 and best_ask > 0 and mid > 0:
spread_bps = ((best_ask - best_bid) / mid) * 10000
imbalance = 0.0
denom = (total_bid_liq + total_ask_liq)
if denom > 0:
imbalance = (total_bid_liq - total_ask_liq) / denom
stats_out = {
'mid_price': mid,
'spread_bps': spread_bps,
'imbalance': imbalance,
'best_bid': best_bid,
'best_ask': best_ask,
'bid_volume': total_bid_liq,
'ask_volume': total_ask_liq,
'bid_levels': len(bids_arr),
'ask_levels': len(asks_arr)
}
# Merge any provided stats atop computed defaults
stats_out.update(stats_in or {})
standard_cob_data = {
'symbol': symbol,
'timestamp': int(cob_data['timestamp'] * 1000), # Convert to milliseconds
'bids': [[bid['price'], bid['size']] for bid in cob_data.get('bids', [])[:50]],
'asks': [[ask['price'], ask['size']] for ask in cob_data.get('asks', [])[:50]],
'stats': cob_data.get('stats', {})
'bids': bids_arr,
'asks': asks_arr,
'stats': stats_out
}
# Add to cache

1
core/multi_exchange_cob_provider.py
View File

@ -99,6 +99,7 @@ class ExchangeType(Enum):
KRAKEN = "kraken"
HUOBI = "huobi"
BITFINEX = "bitfinex"
COINAPI = "coinapi"
@dataclass
class ExchangeOrderBookLevel:

130
core/standardized_data_provider.py
View File

@ -86,6 +86,15 @@ class StandardizedDataProvider(DataProvider):
enabled=True,
websocket_url="wss://stream.binance.com:9443/ws/",
symbols_mapping={symbol: symbol.replace('/', '').lower() for symbol in self.symbols}
),
# CoinAPI REST for supplemental depth snapshots (merged with WS streams)
'coinapi': ExchangeConfig(
exchange_type=ExchangeType.COINAPI,
weight=0.6,
enabled=True,
rest_api_url="https://rest.coinapi.io/v1/",
symbols_mapping={symbol: symbol.replace('/', '_').replace('USDT', 'USD') for symbol in self.symbols},
rate_limits={"min_interval_ms": 500}
)
}
@ -229,69 +238,24 @@ class StandardizedDataProvider(DataProvider):
COBData: COB data with price buckets and moving averages
"""
try:
if not self.cob_provider:
# Use real-time COB snapshot from parent and convert to COBData
cob_obj = self._get_latest_cob_data_object(symbol)
if cob_obj is None:
return None
# Get current price
current_price = self.current_prices.get(symbol.replace('/', '').upper(), 0.0)
if current_price <= 0:
return None
# Determine bucket size based on symbol
bucket_size = 1.0 if 'ETH' in symbol else 10.0 # $1 for ETH, $10 for BTC
# Calculate price range (±20 buckets)
price_range = 20 * bucket_size
min_price = current_price - price_range
max_price = current_price + price_range
# Create price buckets
price_buckets = {}
bid_ask_imbalance = {}
volume_weighted_prices = {}
# Generate mock COB data for now (will be replaced with real COB provider data)
for i in range(-20, 21):
price = current_price + (i * bucket_size)
if price > 0:
# Mock data - replace with real COB provider data
bid_volume = max(0, 1000 - abs(i) * 50) # More volume near current price
ask_volume = max(0, 1000 - abs(i) * 50)
total_volume = bid_volume + ask_volume
imbalance = (bid_volume - ask_volume) / max(total_volume, 1)
price_buckets[price] = {
'bid_volume': bid_volume,
'ask_volume': ask_volume,
'total_volume': total_volume,
'imbalance': imbalance
}
bid_ask_imbalance[price] = imbalance
volume_weighted_prices[price] = price # Simplified VWAP
# Calculate moving averages of imbalance for ±5 buckets
ma_data = self._calculate_cob_moving_averages(symbol, bid_ask_imbalance, timestamp)
cob_data = COBData(
symbol=symbol,
timestamp=timestamp,
current_price=current_price,
bucket_size=bucket_size,
price_buckets=price_buckets,
bid_ask_imbalance=bid_ask_imbalance,
volume_weighted_prices=volume_weighted_prices,
order_flow_metrics={},
ma_1s_imbalance=ma_data.get('1s', {}),
ma_5s_imbalance=ma_data.get('5s', {}),
ma_15s_imbalance=ma_data.get('15s', {}),
ma_60s_imbalance=ma_data.get('60s', {})
)
# Cache the COB data
self.cob_data_cache[symbol] = cob_data
return cob_data
ma_data = self._calculate_cob_moving_averages(symbol, cob_obj.bid_ask_imbalance, timestamp)
# Update MA fields
cob_obj.ma_1s_imbalance = ma_data.get('1s', {})
cob_obj.ma_5s_imbalance = ma_data.get('5s', {})
cob_obj.ma_15s_imbalance = ma_data.get('15s', {})
cob_obj.ma_60s_imbalance = ma_data.get('60s', {})
# Cache and return
self.cob_data_cache[symbol] = cob_obj
return cob_obj
except Exception as e:
logger.error(f"Error getting COB data for {symbol}: {e}")
return None
@ -379,16 +343,40 @@ class StandardizedDataProvider(DataProvider):
def _get_pivot_points(self, symbol: str) -> List[PivotPoint]:
"""Get pivot points for a symbol"""
try:
pivot_points = []
# Get pivot points from Williams Market Structure if available
if symbol in self.williams_structure:
williams = self.williams_structure[symbol]
# This would need to be implemented based on the actual Williams structure
# For now, return empty list
pass
return pivot_points
results: List[PivotPoint] = []
# Prefer DataProvider's Williams calculation (1s OHLCV based)
try:
levels = self.calculate_williams_pivot_points(symbol)
except Exception:
levels = {}
# Flatten levels into standardized PivotPoint list
if levels:
for level_idx, trend_level in levels.items():
# Expect trend_level to have an iterable of pivot points
pivots = getattr(trend_level, 'pivots', None)
if not pivots:
# Some implementations may expose as list directly
pivots = getattr(trend_level, 'points', [])
for p in pivots or []:
# Map fields defensively
ts = getattr(p, 'timestamp', None)
price = float(getattr(p, 'price', 0.0) or 0.0)
ptype = getattr(p, 'pivot_type', getattr(p, 'type', 'low'))
ptype = 'high' if str(ptype).lower() == 'high' else 'low'
lvl = int(getattr(p, 'level', level_idx) or level_idx)
if ts and price > 0:
results.append(PivotPoint(
symbol=symbol,
timestamp=ts,
price=price,
type=ptype,
level=lvl,
confidence=1.0
))
return results
except Exception as e:
logger.error(f"Error getting pivot points for {symbol}: {e}")