COBY : specs + task 1

This commit is contained in:
Dobromir Popov
2025-08-04 15:50:54 +03:00
parent e223bc90e9
commit de9fa4a421
28 changed files with 4165 additions and 1 deletions

View File

@ -0,0 +1,17 @@
"""
Interface definitions for the multi-exchange data aggregation system.
"""
from .exchange_connector import ExchangeConnector
from .data_processor import DataProcessor
from .aggregation_engine import AggregationEngine
from .storage_manager import StorageManager
from .replay_manager import ReplayManager
__all__ = [
'ExchangeConnector',
'DataProcessor',
'AggregationEngine',
'StorageManager',
'ReplayManager'
]

View File

@ -0,0 +1,139 @@
"""
Interface for data aggregation and heatmap generation.
"""
from abc import ABC, abstractmethod
from typing import Dict, List
from ..models.core import (
OrderBookSnapshot, PriceBuckets, HeatmapData,
ImbalanceMetrics, ConsolidatedOrderBook
)
class AggregationEngine(ABC):
"""Aggregates data into price buckets and heatmaps"""
@abstractmethod
def create_price_buckets(self, orderbook: OrderBookSnapshot,
bucket_size: float) -> PriceBuckets:
"""
Convert order book data to price buckets.
Args:
orderbook: Order book snapshot
bucket_size: Size of each price bucket
Returns:
PriceBuckets: Aggregated price bucket data
"""
pass
@abstractmethod
def update_heatmap(self, symbol: str, buckets: PriceBuckets) -> HeatmapData:
"""
Update heatmap data with new price buckets.
Args:
symbol: Trading symbol
buckets: Price bucket data
Returns:
HeatmapData: Updated heatmap visualization data
"""
pass
@abstractmethod
def calculate_imbalances(self, orderbook: OrderBookSnapshot) -> ImbalanceMetrics:
"""
Calculate order book imbalance metrics.
Args:
orderbook: Order book snapshot
Returns:
ImbalanceMetrics: Calculated imbalance metrics
"""
pass
@abstractmethod
def aggregate_across_exchanges(self, symbol: str,
orderbooks: List[OrderBookSnapshot]) -> ConsolidatedOrderBook:
"""
Aggregate order book data from multiple exchanges.
Args:
symbol: Trading symbol
orderbooks: List of order book snapshots from different exchanges
Returns:
ConsolidatedOrderBook: Consolidated order book data
"""
pass
@abstractmethod
def calculate_volume_weighted_price(self, orderbooks: List[OrderBookSnapshot]) -> float:
"""
Calculate volume-weighted average price across exchanges.
Args:
orderbooks: List of order book snapshots
Returns:
float: Volume-weighted average price
"""
pass
@abstractmethod
def get_market_depth(self, orderbook: OrderBookSnapshot,
depth_levels: List[float]) -> Dict[float, Dict[str, float]]:
"""
Calculate market depth at different price levels.
Args:
orderbook: Order book snapshot
depth_levels: List of depth percentages (e.g., [0.1, 0.5, 1.0])
Returns:
Dict: Market depth data {level: {'bid_volume': x, 'ask_volume': y}}
"""
pass
@abstractmethod
def smooth_heatmap(self, heatmap: HeatmapData, smoothing_factor: float) -> HeatmapData:
"""
Apply smoothing to heatmap data to reduce noise.
Args:
heatmap: Raw heatmap data
smoothing_factor: Smoothing factor (0.0 to 1.0)
Returns:
HeatmapData: Smoothed heatmap data
"""
pass
@abstractmethod
def calculate_liquidity_score(self, orderbook: OrderBookSnapshot) -> float:
"""
Calculate liquidity score for an order book.
Args:
orderbook: Order book snapshot
Returns:
float: Liquidity score (0.0 to 1.0)
"""
pass
@abstractmethod
def detect_support_resistance(self, heatmap: HeatmapData) -> Dict[str, List[float]]:
"""
Detect support and resistance levels from heatmap data.
Args:
heatmap: Heatmap data
Returns:
Dict: {'support': [prices], 'resistance': [prices]}
"""
pass

View File

@ -0,0 +1,119 @@
"""
Interface for data processing and normalization.
"""
from abc import ABC, abstractmethod
from typing import Dict, Union, List, Optional
from ..models.core import OrderBookSnapshot, TradeEvent, OrderBookMetrics
class DataProcessor(ABC):
"""Processes and normalizes raw exchange data"""
@abstractmethod
def normalize_orderbook(self, raw_data: Dict, exchange: str) -> OrderBookSnapshot:
"""
Normalize raw order book data to standard format.
Args:
raw_data: Raw order book data from exchange
exchange: Exchange name
Returns:
OrderBookSnapshot: Normalized order book data
"""
pass
@abstractmethod
def normalize_trade(self, raw_data: Dict, exchange: str) -> TradeEvent:
"""
Normalize raw trade data to standard format.
Args:
raw_data: Raw trade data from exchange
exchange: Exchange name
Returns:
TradeEvent: Normalized trade data
"""
pass
@abstractmethod
def validate_data(self, data: Union[OrderBookSnapshot, TradeEvent]) -> bool:
"""
Validate normalized data for quality and consistency.
Args:
data: Normalized data to validate
Returns:
bool: True if data is valid, False otherwise
"""
pass
@abstractmethod
def calculate_metrics(self, orderbook: OrderBookSnapshot) -> OrderBookMetrics:
"""
Calculate metrics from order book data.
Args:
orderbook: Order book snapshot
Returns:
OrderBookMetrics: Calculated metrics
"""
pass
@abstractmethod
def detect_anomalies(self, data: Union[OrderBookSnapshot, TradeEvent]) -> List[str]:
"""
Detect anomalies in the data.
Args:
data: Data to analyze for anomalies
Returns:
List[str]: List of detected anomaly descriptions
"""
pass
@abstractmethod
def filter_data(self, data: Union[OrderBookSnapshot, TradeEvent],
criteria: Dict) -> bool:
"""
Filter data based on criteria.
Args:
data: Data to filter
criteria: Filtering criteria
Returns:
bool: True if data passes filter, False otherwise
"""
pass
@abstractmethod
def enrich_data(self, data: Union[OrderBookSnapshot, TradeEvent]) -> Dict:
"""
Enrich data with additional metadata.
Args:
data: Data to enrich
Returns:
Dict: Enriched data with metadata
"""
pass
@abstractmethod
def get_data_quality_score(self, data: Union[OrderBookSnapshot, TradeEvent]) -> float:
"""
Calculate data quality score.
Args:
data: Data to score
Returns:
float: Quality score between 0.0 and 1.0
"""
pass

View File

@ -0,0 +1,189 @@
"""
Base interface for exchange WebSocket connectors.
"""
from abc import ABC, abstractmethod
from typing import Callable, List, Optional
from ..models.core import ConnectionStatus, OrderBookSnapshot, TradeEvent
class ExchangeConnector(ABC):
"""Base interface for exchange WebSocket connectors"""
def __init__(self, exchange_name: str):
self.exchange_name = exchange_name
self._data_callbacks: List[Callable] = []
self._status_callbacks: List[Callable] = []
self._connection_status = ConnectionStatus.DISCONNECTED
@abstractmethod
async def connect(self) -> bool:
"""
Establish connection to the exchange WebSocket.
Returns:
bool: True if connection successful, False otherwise
"""
pass
@abstractmethod
async def disconnect(self) -> None:
"""Disconnect from the exchange WebSocket."""
pass
@abstractmethod
async def subscribe_orderbook(self, symbol: str) -> None:
"""
Subscribe to order book updates for a symbol.
Args:
symbol: Trading symbol (e.g., 'BTCUSDT')
"""
pass
@abstractmethod
async def subscribe_trades(self, symbol: str) -> None:
"""
Subscribe to trade updates for a symbol.
Args:
symbol: Trading symbol (e.g., 'BTCUSDT')
"""
pass
@abstractmethod
async def unsubscribe_orderbook(self, symbol: str) -> None:
"""
Unsubscribe from order book updates for a symbol.
Args:
symbol: Trading symbol (e.g., 'BTCUSDT')
"""
pass
@abstractmethod
async def unsubscribe_trades(self, symbol: str) -> None:
"""
Unsubscribe from trade updates for a symbol.
Args:
symbol: Trading symbol (e.g., 'BTCUSDT')
"""
pass
def get_connection_status(self) -> ConnectionStatus:
"""
Get current connection status.
Returns:
ConnectionStatus: Current connection status
"""
return self._connection_status
def add_data_callback(self, callback: Callable) -> None:
"""
Add callback for data updates.
Args:
callback: Function to call when data is received
Signature: callback(data: Union[OrderBookSnapshot, TradeEvent])
"""
if callback not in self._data_callbacks:
self._data_callbacks.append(callback)
def remove_data_callback(self, callback: Callable) -> None:
"""
Remove data callback.
Args:
callback: Callback function to remove
"""
if callback in self._data_callbacks:
self._data_callbacks.remove(callback)
def add_status_callback(self, callback: Callable) -> None:
"""
Add callback for status updates.
Args:
callback: Function to call when status changes
Signature: callback(exchange: str, status: ConnectionStatus)
"""
if callback not in self._status_callbacks:
self._status_callbacks.append(callback)
def remove_status_callback(self, callback: Callable) -> None:
"""
Remove status callback.
Args:
callback: Callback function to remove
"""
if callback in self._status_callbacks:
self._status_callbacks.remove(callback)
def _notify_data_callbacks(self, data):
"""Notify all data callbacks of new data."""
for callback in self._data_callbacks:
try:
callback(data)
except Exception as e:
# Log error but don't stop other callbacks
print(f"Error in data callback: {e}")
def _notify_status_callbacks(self, status: ConnectionStatus):
"""Notify all status callbacks of status change."""
self._connection_status = status
for callback in self._status_callbacks:
try:
callback(self.exchange_name, status)
except Exception as e:
# Log error but don't stop other callbacks
print(f"Error in status callback: {e}")
@abstractmethod
async def get_symbols(self) -> List[str]:
"""
Get list of available trading symbols.
Returns:
List[str]: List of available symbols
"""
pass
@abstractmethod
def normalize_symbol(self, symbol: str) -> str:
"""
Normalize symbol to exchange format.
Args:
symbol: Standard symbol format (e.g., 'BTCUSDT')
Returns:
str: Exchange-specific symbol format
"""
pass
@abstractmethod
async def get_orderbook_snapshot(self, symbol: str, depth: int = 20) -> Optional[OrderBookSnapshot]:
"""
Get current order book snapshot.
Args:
symbol: Trading symbol
depth: Number of price levels to retrieve
Returns:
OrderBookSnapshot: Current order book or None if unavailable
"""
pass
@property
def name(self) -> str:
"""Get exchange name."""
return self.exchange_name
@property
def is_connected(self) -> bool:
"""Check if connector is connected."""
return self._connection_status == ConnectionStatus.CONNECTED

View File

@ -0,0 +1,212 @@
"""
Interface for historical data replay functionality.
"""
from abc import ABC, abstractmethod
from datetime import datetime
from typing import List, Optional, Callable, Dict, Any
from ..models.core import ReplaySession, ReplayStatus
class ReplayManager(ABC):
"""Provides historical data replay functionality"""
@abstractmethod
def create_replay_session(self, start_time: datetime, end_time: datetime,
speed: float = 1.0, symbols: Optional[List[str]] = None,
exchanges: Optional[List[str]] = None) -> str:
"""
Create a new replay session.
Args:
start_time: Replay start time
end_time: Replay end time
speed: Playback speed multiplier (1.0 = real-time)
symbols: List of symbols to replay (None = all)
exchanges: List of exchanges to replay (None = all)
Returns:
str: Session ID
"""
pass
@abstractmethod
async def start_replay(self, session_id: str) -> None:
"""
Start replay session.
Args:
session_id: Session ID to start
"""
pass
@abstractmethod
async def pause_replay(self, session_id: str) -> None:
"""
Pause replay session.
Args:
session_id: Session ID to pause
"""
pass
@abstractmethod
async def resume_replay(self, session_id: str) -> None:
"""
Resume paused replay session.
Args:
session_id: Session ID to resume
"""
pass
@abstractmethod
async def stop_replay(self, session_id: str) -> None:
"""
Stop replay session.
Args:
session_id: Session ID to stop
"""
pass
@abstractmethod
def get_replay_status(self, session_id: str) -> Optional[ReplaySession]:
"""
Get replay session status.
Args:
session_id: Session ID
Returns:
ReplaySession: Session status or None if not found
"""
pass
@abstractmethod
def list_replay_sessions(self) -> List[ReplaySession]:
"""
List all replay sessions.
Returns:
List[ReplaySession]: List of all sessions
"""
pass
@abstractmethod
def delete_replay_session(self, session_id: str) -> bool:
"""
Delete replay session.
Args:
session_id: Session ID to delete
Returns:
bool: True if deleted successfully, False otherwise
"""
pass
@abstractmethod
def set_replay_speed(self, session_id: str, speed: float) -> bool:
"""
Change replay speed for active session.
Args:
session_id: Session ID
speed: New playback speed multiplier
Returns:
bool: True if speed changed successfully, False otherwise
"""
pass
@abstractmethod
def seek_replay(self, session_id: str, timestamp: datetime) -> bool:
"""
Seek to specific timestamp in replay.
Args:
session_id: Session ID
timestamp: Target timestamp
Returns:
bool: True if seek successful, False otherwise
"""
pass
@abstractmethod
def add_data_callback(self, session_id: str, callback: Callable) -> bool:
"""
Add callback for replay data.
Args:
session_id: Session ID
callback: Function to call with replay data
Signature: callback(data: Union[OrderBookSnapshot, TradeEvent])
Returns:
bool: True if callback added successfully, False otherwise
"""
pass
@abstractmethod
def remove_data_callback(self, session_id: str, callback: Callable) -> bool:
"""
Remove data callback from replay session.
Args:
session_id: Session ID
callback: Callback function to remove
Returns:
bool: True if callback removed successfully, False otherwise
"""
pass
@abstractmethod
def add_status_callback(self, session_id: str, callback: Callable) -> bool:
"""
Add callback for replay status changes.
Args:
session_id: Session ID
callback: Function to call on status change
Signature: callback(session_id: str, status: ReplayStatus)
Returns:
bool: True if callback added successfully, False otherwise
"""
pass
@abstractmethod
async def get_available_data_range(self, symbol: str,
exchange: Optional[str] = None) -> Optional[Dict[str, datetime]]:
"""
Get available data time range for replay.
Args:
symbol: Trading symbol
exchange: Exchange name (None = all exchanges)
Returns:
Dict: {'start': datetime, 'end': datetime} or None if no data
"""
pass
@abstractmethod
def validate_replay_request(self, start_time: datetime, end_time: datetime,
symbols: Optional[List[str]] = None,
exchanges: Optional[List[str]] = None) -> List[str]:
"""
Validate replay request parameters.
Args:
start_time: Requested start time
end_time: Requested end time
symbols: Requested symbols
exchanges: Requested exchanges
Returns:
List[str]: List of validation errors (empty if valid)
"""
pass

View File

@ -0,0 +1,215 @@
"""
Interface for data storage and retrieval.
"""
from abc import ABC, abstractmethod
from datetime import datetime
from typing import List, Dict, Optional, Any
from ..models.core import OrderBookSnapshot, TradeEvent, HeatmapData, SystemMetrics
class StorageManager(ABC):
"""Manages data persistence and retrieval"""
@abstractmethod
async def store_orderbook(self, data: OrderBookSnapshot) -> bool:
"""
Store order book snapshot to database.
Args:
data: Order book snapshot to store
Returns:
bool: True if stored successfully, False otherwise
"""
pass
@abstractmethod
async def store_trade(self, data: TradeEvent) -> bool:
"""
Store trade event to database.
Args:
data: Trade event to store
Returns:
bool: True if stored successfully, False otherwise
"""
pass
@abstractmethod
async def store_heatmap(self, data: HeatmapData) -> bool:
"""
Store heatmap data to database.
Args:
data: Heatmap data to store
Returns:
bool: True if stored successfully, False otherwise
"""
pass
@abstractmethod
async def store_metrics(self, data: SystemMetrics) -> bool:
"""
Store system metrics to database.
Args:
data: System metrics to store
Returns:
bool: True if stored successfully, False otherwise
"""
pass
@abstractmethod
async def get_historical_orderbooks(self, symbol: str, exchange: str,
start: datetime, end: datetime,
limit: Optional[int] = None) -> List[OrderBookSnapshot]:
"""
Retrieve historical order book data.
Args:
symbol: Trading symbol
exchange: Exchange name
start: Start timestamp
end: End timestamp
limit: Maximum number of records to return
Returns:
List[OrderBookSnapshot]: Historical order book data
"""
pass
@abstractmethod
async def get_historical_trades(self, symbol: str, exchange: str,
start: datetime, end: datetime,
limit: Optional[int] = None) -> List[TradeEvent]:
"""
Retrieve historical trade data.
Args:
symbol: Trading symbol
exchange: Exchange name
start: Start timestamp
end: End timestamp
limit: Maximum number of records to return
Returns:
List[TradeEvent]: Historical trade data
"""
pass
@abstractmethod
async def get_latest_orderbook(self, symbol: str, exchange: str) -> Optional[OrderBookSnapshot]:
"""
Get latest order book snapshot.
Args:
symbol: Trading symbol
exchange: Exchange name
Returns:
OrderBookSnapshot: Latest order book or None if not found
"""
pass
@abstractmethod
async def get_latest_heatmap(self, symbol: str, bucket_size: float) -> Optional[HeatmapData]:
"""
Get latest heatmap data.
Args:
symbol: Trading symbol
bucket_size: Price bucket size
Returns:
HeatmapData: Latest heatmap or None if not found
"""
pass
@abstractmethod
async def get_ohlcv_data(self, symbol: str, exchange: str, timeframe: str,
start: datetime, end: datetime) -> List[Dict[str, Any]]:
"""
Get OHLCV candlestick data.
Args:
symbol: Trading symbol
exchange: Exchange name
timeframe: Timeframe (e.g., '1m', '5m', '1h')
start: Start timestamp
end: End timestamp
Returns:
List[Dict]: OHLCV data
"""
pass
@abstractmethod
async def batch_store_orderbooks(self, data: List[OrderBookSnapshot]) -> int:
"""
Store multiple order book snapshots in batch.
Args:
data: List of order book snapshots
Returns:
int: Number of records stored successfully
"""
pass
@abstractmethod
async def batch_store_trades(self, data: List[TradeEvent]) -> int:
"""
Store multiple trade events in batch.
Args:
data: List of trade events
Returns:
int: Number of records stored successfully
"""
pass
@abstractmethod
def setup_database_schema(self) -> None:
"""
Set up database schema and tables.
Should be idempotent - safe to call multiple times.
"""
pass
@abstractmethod
async def cleanup_old_data(self, retention_days: int) -> int:
"""
Clean up old data based on retention policy.
Args:
retention_days: Number of days to retain data
Returns:
int: Number of records deleted
"""
pass
@abstractmethod
async def get_storage_stats(self) -> Dict[str, Any]:
"""
Get storage statistics.
Returns:
Dict: Storage statistics (table sizes, record counts, etc.)
"""
pass
@abstractmethod
async def health_check(self) -> bool:
"""
Check storage system health.
Returns:
bool: True if healthy, False otherwise
"""
pass