popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve training and model data

Browse Source
This commit is contained in:
Dobromir Popov
2025-07-07 15:48:25 +03:00
parent 271e7d59b5
commit 2d8f763eeb
16 changed files with 2047 additions and 1699 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
core/cob_integration.py
View File

@ -88,7 +88,7 @@ class COBIntegration:
# Start COB provider streaming
try:
logger.info("Starting COB provider streaming...")
await self.cob_provider.start_streaming()
await self.cob_provider.start_streaming()
except Exception as e:
logger.error(f"Error starting COB provider streaming: {e}")
# Start a background task instead
@ -112,7 +112,7 @@ class COBIntegration:
"""Stop COB integration"""
logger.info("Stopping COB Integration")
if self.cob_provider:
await self.cob_provider.stop_streaming()
await self.cob_provider.stop_streaming()
logger.info("COB Integration stopped")
def add_cnn_callback(self, callback: Callable[[str, Dict], None]):
@ -313,7 +313,7 @@ class COBIntegration:
# Get fixed bucket size for the symbol
bucket_size = 1.0 # Default bucket size
if self.cob_provider:
bucket_size = self.cob_provider.fixed_usd_buckets.get(symbol, 1.0)
bucket_size = self.cob_provider.fixed_usd_buckets.get(symbol, 1.0)
# Calculate price range for buckets
mid_price = cob_snapshot.volume_weighted_mid
@ -359,15 +359,15 @@ class COBIntegration:
# Get actual Session Volume Profile (SVP) from trade data
svp_data = []
if self.cob_provider:
try:
svp_result = self.cob_provider.get_session_volume_profile(symbol, bucket_size)
if svp_result and 'data' in svp_result:
svp_data = svp_result['data']
logger.debug(f"Retrieved SVP data for {symbol}: {len(svp_data)} price levels")
else:
logger.warning(f"No SVP data available for {symbol}")
except Exception as e:
logger.error(f"Error getting SVP data for {symbol}: {e}")
try:
svp_result = self.cob_provider.get_session_volume_profile(symbol, bucket_size)
if svp_result and 'data' in svp_result:
svp_data = svp_result['data']
logger.debug(f"Retrieved SVP data for {symbol}: {len(svp_data)} price levels")
else:
logger.warning(f"No SVP data available for {symbol}")
except Exception as e:
logger.error(f"Error getting SVP data for {symbol}: {e}")
# Generate market stats
stats = {
@ -405,18 +405,18 @@ class COBIntegration:
# Get additional real-time stats
realtime_stats = {}
if self.cob_provider:
try:
realtime_stats = self.cob_provider.get_realtime_stats(symbol)
if realtime_stats:
stats['realtime_1s'] = realtime_stats.get('1s_stats', {})
stats['realtime_5s'] = realtime_stats.get('5s_stats', {})
else:
stats['realtime_1s'] = {}
stats['realtime_5s'] = {}
except Exception as e:
logger.error(f"Error getting real-time stats for {symbol}: {e}")
try:
realtime_stats = self.cob_provider.get_realtime_stats(symbol)
if realtime_stats:
stats['realtime_1s'] = realtime_stats.get('1s_stats', {})
stats['realtime_5s'] = realtime_stats.get('5s_stats', {})
else:
stats['realtime_1s'] = {}
stats['realtime_5s'] = {}
except Exception as e:
logger.error(f"Error getting real-time stats for {symbol}: {e}")
stats['realtime_1s'] = {}
stats['realtime_5s'] = {}
return {
'type': 'cob_update',
@ -487,9 +487,9 @@ class COBIntegration:
try:
for symbol in self.symbols:
if self.cob_provider:
cob_snapshot = self.cob_provider.get_consolidated_orderbook(symbol)
if cob_snapshot:
await self._analyze_cob_patterns(symbol, cob_snapshot)
cob_snapshot = self.cob_provider.get_consolidated_orderbook(symbol)
if cob_snapshot:
await self._analyze_cob_patterns(symbol, cob_snapshot)
await asyncio.sleep(1)
@ -655,7 +655,7 @@ class COBIntegration:
except Exception as e:
logger.error(f"Error getting NN stats for {symbol}: {e}")
return {}
return {}
def get_realtime_stats(self):
# Added null check to ensure the COB provider is initialized

313
core/multi_exchange_cob_provider.py
View File

@ -661,22 +661,315 @@ class MultiExchangeCOBProvider:
except Exception as e:
logger.error(f"Error processing Binance order book for {symbol}: {e}", exc_info=True)
async def _stream_coinbase_orderbook(self, symbol: str, config: ExchangeConfig):
"""Stream Coinbase order book data (placeholder implementation)"""
async def _process_coinbase_orderbook(self, symbol: str, data: Dict):
"""Process Coinbase order book data"""
try:
# For now, just log that Coinbase streaming is not implemented
logger.info(f"Coinbase streaming for {symbol} not yet implemented")
await asyncio.sleep(60) # Sleep to prevent spam
if data.get('type') == 'snapshot':
# Initial snapshot
bids = {}
asks = {}
for bid_data in data.get('bids', []):
price, size = float(bid_data[0]), float(bid_data[1])
if size > 0:
bids[price] = ExchangeOrderBookLevel(
exchange='coinbase',
price=price,
size=size,
volume_usd=price * size,
orders_count=1, # Coinbase doesn't provide order count
side='bid',
timestamp=datetime.now(),
raw_data=bid_data
)
for ask_data in data.get('asks', []):
price, size = float(ask_data[0]), float(ask_data[1])
if size > 0:
asks[price] = ExchangeOrderBookLevel(
exchange='coinbase',
price=price,
size=size,
volume_usd=price * size,
orders_count=1,
side='ask',
timestamp=datetime.now(),
raw_data=ask_data
)
# Update order book
async with self.data_lock:
if symbol not in self.exchange_order_books:
self.exchange_order_books[symbol] = {}
self.exchange_order_books[symbol]['coinbase'] = {
'bids': bids,
'asks': asks,
'last_update': datetime.now(),
'connected': True
}
logger.info(f"Coinbase snapshot for {symbol}: {len(bids)} bids, {len(asks)} asks")
elif data.get('type') == 'l2update':
# Level 2 update
async with self.data_lock:
if symbol in self.exchange_order_books and 'coinbase' in self.exchange_order_books[symbol]:
coinbase_data = self.exchange_order_books[symbol]['coinbase']
for change in data.get('changes', []):
side, price_str, size_str = change
price, size = float(price_str), float(size_str)
if side == 'buy':
if size == 0:
# Remove level
coinbase_data['bids'].pop(price, None)
else:
# Update level
coinbase_data['bids'][price] = ExchangeOrderBookLevel(
exchange='coinbase',
price=price,
size=size,
volume_usd=price * size,
orders_count=1,
side='bid',
timestamp=datetime.now(),
raw_data=change
)
elif side == 'sell':
if size == 0:
# Remove level
coinbase_data['asks'].pop(price, None)
else:
# Update level
coinbase_data['asks'][price] = ExchangeOrderBookLevel(
exchange='coinbase',
price=price,
size=size,
volume_usd=price * size,
orders_count=1,
side='ask',
timestamp=datetime.now(),
raw_data=change
)
coinbase_data['last_update'] = datetime.now()
# Update exchange count
exchange_name = 'coinbase'
if exchange_name not in self.exchange_update_counts:
self.exchange_update_counts[exchange_name] = 0
self.exchange_update_counts[exchange_name] += 1
# Log every 1000th update
if self.exchange_update_counts[exchange_name] % 1000 == 0:
logger.info(f"Processed {self.exchange_update_counts[exchange_name]} Coinbase updates for {symbol}")
except Exception as e:
logger.error(f"Error streaming Coinbase order book for {symbol}: {e}")
logger.error(f"Error processing Coinbase order book for {symbol}: {e}", exc_info=True)
async def _process_kraken_orderbook(self, symbol: str, data: Dict):
"""Process Kraken order book data"""
try:
# Kraken sends different message types
if isinstance(data, list) and len(data) > 1:
# Order book update format: [channel_id, data, channel_name, pair]
if len(data) >= 4 and data[2] == "book-25":
book_data = data[1]
# Check for snapshot vs update
if 'bs' in book_data and 'as' in book_data:
# Snapshot
bids = {}
asks = {}
for bid_data in book_data.get('bs', []):
price, volume, timestamp = float(bid_data[0]), float(bid_data[1]), float(bid_data[2])
if volume > 0:
bids[price] = ExchangeOrderBookLevel(
exchange='kraken',
price=price,
size=volume,
volume_usd=price * volume,
orders_count=1, # Kraken doesn't provide order count in book feed
side='bid',
timestamp=datetime.fromtimestamp(timestamp),
raw_data=bid_data
)
for ask_data in book_data.get('as', []):
price, volume, timestamp = float(ask_data[0]), float(ask_data[1]), float(ask_data[2])
if volume > 0:
asks[price] = ExchangeOrderBookLevel(
exchange='kraken',
price=price,
size=volume,
volume_usd=price * volume,
orders_count=1,
side='ask',
timestamp=datetime.fromtimestamp(timestamp),
raw_data=ask_data
)
# Update order book
async with self.data_lock:
if symbol not in self.exchange_order_books:
self.exchange_order_books[symbol] = {}
self.exchange_order_books[symbol]['kraken'] = {
'bids': bids,
'asks': asks,
'last_update': datetime.now(),
'connected': True
}
logger.info(f"Kraken snapshot for {symbol}: {len(bids)} bids, {len(asks)} asks")
else:
# Incremental update
async with self.data_lock:
if symbol in self.exchange_order_books and 'kraken' in self.exchange_order_books[symbol]:
kraken_data = self.exchange_order_books[symbol]['kraken']
# Process bid updates
for bid_update in book_data.get('b', []):
price, volume, timestamp = float(bid_update[0]), float(bid_update[1]), float(bid_update[2])
if volume == 0:
# Remove level
kraken_data['bids'].pop(price, None)
else:
# Update level
kraken_data['bids'][price] = ExchangeOrderBookLevel(
exchange='kraken',
price=price,
size=volume,
volume_usd=price * volume,
orders_count=1,
side='bid',
timestamp=datetime.fromtimestamp(timestamp),
raw_data=bid_update
)
# Process ask updates
for ask_update in book_data.get('a', []):
price, volume, timestamp = float(ask_update[0]), float(ask_update[1]), float(ask_update[2])
if volume == 0:
# Remove level
kraken_data['asks'].pop(price, None)
else:
# Update level
kraken_data['asks'][price] = ExchangeOrderBookLevel(
exchange='kraken',
price=price,
size=volume,
volume_usd=price * volume,
orders_count=1,
side='ask',
timestamp=datetime.fromtimestamp(timestamp),
raw_data=ask_update
)
kraken_data['last_update'] = datetime.now()
# Update exchange count
exchange_name = 'kraken'
if exchange_name not in self.exchange_update_counts:
self.exchange_update_counts[exchange_name] = 0
self.exchange_update_counts[exchange_name] += 1
# Log every 1000th update
if self.exchange_update_counts[exchange_name] % 1000 == 0:
logger.info(f"Processed {self.exchange_update_counts[exchange_name]} Kraken updates for {symbol}")
except Exception as e:
logger.error(f"Error processing Kraken order book for {symbol}: {e}", exc_info=True)
async def _stream_coinbase_orderbook(self, symbol: str, config: ExchangeConfig):
"""Stream Coinbase order book data via WebSocket"""
try:
import json
if websockets is None or websockets_connect is None:
raise ImportError("websockets module not available")
# Coinbase Pro WebSocket URL
ws_url = "wss://ws-feed.pro.coinbase.com"
coinbase_symbol = config.symbols_mapping.get(symbol, symbol.replace('/', '-'))
# Subscribe message for level2 order book updates
subscribe_message = {
"type": "subscribe",
"product_ids": [coinbase_symbol],
"channels": ["level2"]
}
logger.info(f"Connecting to Coinbase order book stream for {symbol}")
async with websockets_connect(ws_url) as websocket:
# Send subscription
await websocket.send(json.dumps(subscribe_message))
logger.info(f"Subscribed to Coinbase level2 for {coinbase_symbol}")
async for message in websocket:
if not self.is_streaming:
break
try:
data = json.loads(message)
await self._process_coinbase_orderbook(symbol, data)
except json.JSONDecodeError as e:
logger.error(f"Error parsing Coinbase message: {e}")
except Exception as e:
logger.error(f"Error processing Coinbase orderbook: {e}")
except Exception as e:
logger.error(f"Coinbase order book stream error for {symbol}: {e}")
finally:
logger.info(f"Disconnected from Coinbase order book stream for {symbol}")
async def _stream_kraken_orderbook(self, symbol: str, config: ExchangeConfig):
"""Stream Kraken order book data (placeholder implementation)"""
"""Stream Kraken order book data via WebSocket"""
try:
logger.info(f"Kraken streaming for {symbol} not yet implemented")
await asyncio.sleep(60) # Sleep to prevent spam
import json
if websockets is None or websockets_connect is None:
raise ImportError("websockets module not available")
# Kraken WebSocket URL
ws_url = "wss://ws.kraken.com"
kraken_symbol = config.symbols_mapping.get(symbol, symbol.replace('/', ''))
# Subscribe message for book updates
subscribe_message = {
"event": "subscribe",
"pair": [kraken_symbol],
"subscription": {"name": "book", "depth": 25}
}
logger.info(f"Connecting to Kraken order book stream for {symbol}")
async with websockets_connect(ws_url) as websocket:
# Send subscription
await websocket.send(json.dumps(subscribe_message))
logger.info(f"Subscribed to Kraken book for {kraken_symbol}")
async for message in websocket:
if not self.is_streaming:
break
try:
data = json.loads(message)
await self._process_kraken_orderbook(symbol, data)
except json.JSONDecodeError as e:
logger.error(f"Error parsing Kraken message: {e}")
except Exception as e:
logger.error(f"Error processing Kraken orderbook: {e}")
except Exception as e:
logger.error(f"Error streaming Kraken order book for {symbol}: {e}")
logger.error(f"Kraken order book stream error for {symbol}: {e}")
finally:
logger.info(f"Disconnected from Kraken order book stream for {symbol}")
async def _stream_huobi_orderbook(self, symbol: str, config: ExchangeConfig):
"""Stream Huobi order book data (placeholder implementation)"""

1834
core/orchestrator.py
View File

File diff suppressed because it is too large Load Diff