COBY : specs + task 1
This commit is contained in:
Dobromir Popov
176
COBY/config.py
Normal file
176
COBY/config.py
Normal file
@ -0,0 +1,176 @@
|
||||
"""
|
||||
Configuration management for the multi-exchange data aggregation system.
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Dict, Any
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatabaseConfig:
|
||||
"""Database configuration settings"""
|
||||
host: str = os.getenv('DB_HOST', '192.168.0.10')
|
||||
port: int = int(os.getenv('DB_PORT', '5432'))
|
||||
name: str = os.getenv('DB_NAME', 'market_data')
|
||||
user: str = os.getenv('DB_USER', 'market_user')
|
||||
password: str = os.getenv('DB_PASSWORD', 'market_data_secure_pass_2024')
|
||||
schema: str = os.getenv('DB_SCHEMA', 'market_data')
|
||||
pool_size: int = int(os.getenv('DB_POOL_SIZE', '10'))
|
||||
max_overflow: int = int(os.getenv('DB_MAX_OVERFLOW', '20'))
|
||||
pool_timeout: int = int(os.getenv('DB_POOL_TIMEOUT', '30'))
|
||||
|
||||
|
||||
@dataclass
|
||||
class RedisConfig:
|
||||
"""Redis configuration settings"""
|
||||
host: str = os.getenv('REDIS_HOST', '192.168.0.10')
|
||||
port: int = int(os.getenv('REDIS_PORT', '6379'))
|
||||
password: str = os.getenv('REDIS_PASSWORD', 'market_data_redis_2024')
|
||||
db: int = int(os.getenv('REDIS_DB', '0'))
|
||||
max_connections: int = int(os.getenv('REDIS_MAX_CONNECTIONS', '50'))
|
||||
socket_timeout: int = int(os.getenv('REDIS_SOCKET_TIMEOUT', '5'))
|
||||
socket_connect_timeout: int = int(os.getenv('REDIS_CONNECT_TIMEOUT', '5'))
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExchangeConfig:
|
||||
"""Exchange configuration settings"""
|
||||
exchanges: List[str] = field(default_factory=lambda: [
|
||||
'binance', 'coinbase', 'kraken', 'bybit', 'okx',
|
||||
'huobi', 'kucoin', 'gateio', 'bitfinex', 'mexc'
|
||||
])
|
||||
symbols: List[str] = field(default_factory=lambda: ['BTCUSDT', 'ETHUSDT'])
|
||||
max_connections_per_exchange: int = int(os.getenv('MAX_CONNECTIONS_PER_EXCHANGE', '5'))
|
||||
reconnect_delay: int = int(os.getenv('RECONNECT_DELAY', '5'))
|
||||
max_reconnect_attempts: int = int(os.getenv('MAX_RECONNECT_ATTEMPTS', '10'))
|
||||
heartbeat_interval: int = int(os.getenv('HEARTBEAT_INTERVAL', '30'))
|
||||
|
||||
|
||||
@dataclass
|
||||
class AggregationConfig:
|
||||
"""Data aggregation configuration"""
|
||||
btc_bucket_size: float = float(os.getenv('BTC_BUCKET_SIZE', '10.0')) # $10 USD buckets
|
||||
eth_bucket_size: float = float(os.getenv('ETH_BUCKET_SIZE', '1.0')) # $1 USD buckets
|
||||
default_bucket_size: float = float(os.getenv('DEFAULT_BUCKET_SIZE', '1.0'))
|
||||
heatmap_depth: int = int(os.getenv('HEATMAP_DEPTH', '50')) # Number of price levels
|
||||
update_frequency: float = float(os.getenv('UPDATE_FREQUENCY', '0.5')) # Seconds
|
||||
volume_threshold: float = float(os.getenv('VOLUME_THRESHOLD', '0.01')) # Minimum volume
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceConfig:
|
||||
"""Performance and optimization settings"""
|
||||
data_buffer_size: int = int(os.getenv('DATA_BUFFER_SIZE', '10000'))
|
||||
batch_write_size: int = int(os.getenv('BATCH_WRITE_SIZE', '1000'))
|
||||
max_memory_usage: int = int(os.getenv('MAX_MEMORY_USAGE', '2048')) # MB
|
||||
gc_threshold: float = float(os.getenv('GC_THRESHOLD', '0.8')) # 80% of max memory
|
||||
processing_timeout: int = int(os.getenv('PROCESSING_TIMEOUT', '10')) # Seconds
|
||||
max_queue_size: int = int(os.getenv('MAX_QUEUE_SIZE', '50000'))
|
||||
|
||||
|
||||
@dataclass
|
||||
class APIConfig:
|
||||
"""API server configuration"""
|
||||
host: str = os.getenv('API_HOST', '0.0.0.0')
|
||||
port: int = int(os.getenv('API_PORT', '8080'))
|
||||
websocket_port: int = int(os.getenv('WS_PORT', '8081'))
|
||||
cors_origins: List[str] = field(default_factory=lambda: ['*'])
|
||||
rate_limit: int = int(os.getenv('RATE_LIMIT', '100')) # Requests per minute
|
||||
max_connections: int = int(os.getenv('MAX_WS_CONNECTIONS', '1000'))
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoggingConfig:
|
||||
"""Logging configuration"""
|
||||
level: str = os.getenv('LOG_LEVEL', 'INFO')
|
||||
format: str = os.getenv('LOG_FORMAT', '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
file_path: str = os.getenv('LOG_FILE', 'logs/coby.log')
|
||||
max_file_size: int = int(os.getenv('LOG_MAX_SIZE', '100')) # MB
|
||||
backup_count: int = int(os.getenv('LOG_BACKUP_COUNT', '5'))
|
||||
enable_correlation_id: bool = os.getenv('ENABLE_CORRELATION_ID', 'true').lower() == 'true'
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
"""Main configuration class"""
|
||||
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
||||
redis: RedisConfig = field(default_factory=RedisConfig)
|
||||
exchanges: ExchangeConfig = field(default_factory=ExchangeConfig)
|
||||
aggregation: AggregationConfig = field(default_factory=AggregationConfig)
|
||||
performance: PerformanceConfig = field(default_factory=PerformanceConfig)
|
||||
api: APIConfig = field(default_factory=APIConfig)
|
||||
logging: LoggingConfig = field(default_factory=LoggingConfig)
|
||||
|
||||
# Environment
|
||||
environment: str = os.getenv('ENVIRONMENT', 'development')
|
||||
debug: bool = os.getenv('DEBUG', 'false').lower() == 'true'
|
||||
|
||||
def __post_init__(self):
|
||||
"""Post-initialization validation and setup"""
|
||||
# Create logs directory if it doesn't exist
|
||||
log_dir = Path(self.logging.file_path).parent
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Validate bucket sizes
|
||||
if self.aggregation.btc_bucket_size <= 0:
|
||||
raise ValueError("BTC bucket size must be positive")
|
||||
if self.aggregation.eth_bucket_size <= 0:
|
||||
raise ValueError("ETH bucket size must be positive")
|
||||
|
||||
def get_bucket_size(self, symbol: str) -> float:
|
||||
"""Get bucket size for a specific symbol"""
|
||||
symbol_upper = symbol.upper()
|
||||
if 'BTC' in symbol_upper:
|
||||
return self.aggregation.btc_bucket_size
|
||||
elif 'ETH' in symbol_upper:
|
||||
return self.aggregation.eth_bucket_size
|
||||
else:
|
||||
return self.aggregation.default_bucket_size
|
||||
|
||||
def get_database_url(self) -> str:
|
||||
"""Get database connection URL"""
|
||||
return (f"postgresql://{self.database.user}:{self.database.password}"
|
||||
f"@{self.database.host}:{self.database.port}/{self.database.name}")
|
||||
|
||||
def get_redis_url(self) -> str:
|
||||
"""Get Redis connection URL"""
|
||||
auth = f":{self.redis.password}@" if self.redis.password else ""
|
||||
return f"redis://{auth}{self.redis.host}:{self.redis.port}/{self.redis.db}"
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert configuration to dictionary"""
|
||||
return {
|
||||
'database': {
|
||||
'host': self.database.host,
|
||||
'port': self.database.port,
|
||||
'name': self.database.name,
|
||||
'schema': self.database.schema,
|
||||
},
|
||||
'redis': {
|
||||
'host': self.redis.host,
|
||||
'port': self.redis.port,
|
||||
'db': self.redis.db,
|
||||
},
|
||||
'exchanges': {
|
||||
'count': len(self.exchanges.exchanges),
|
||||
'symbols': self.exchanges.symbols,
|
||||
},
|
||||
'aggregation': {
|
||||
'btc_bucket_size': self.aggregation.btc_bucket_size,
|
||||
'eth_bucket_size': self.aggregation.eth_bucket_size,
|
||||
'heatmap_depth': self.aggregation.heatmap_depth,
|
||||
},
|
||||
'api': {
|
||||
'host': self.api.host,
|
||||
'port': self.api.port,
|
||||
'websocket_port': self.api.websocket_port,
|
||||
},
|
||||
'environment': self.environment,
|
||||
'debug': self.debug,
|
||||
}
|
||||
|
||||
|
||||
# Global configuration instance
|
||||
config = Config()
|
||||
Reference in New Issue
Block a user