167 lines
6.7 KiB
Python
167 lines
6.7 KiB
Python
"""
|
|
Configuration management for the multi-exchange data aggregation system.
|
|
"""
|
|
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
from typing import List, Dict, Any
|
|
from pathlib import Path
|
|
|
|
|
|
@dataclass
|
|
class DatabaseConfig:
|
|
"""Database configuration settings"""
|
|
host: str = os.getenv('DB_HOST', '192.168.0.10')
|
|
port: int = int(os.getenv('DB_PORT', '5432'))
|
|
name: str = os.getenv('DB_NAME', 'market_data')
|
|
user: str = os.getenv('DB_USER', 'market_user')
|
|
password: str = os.getenv('DB_PASSWORD', 'market_data_secure_pass_2024')
|
|
schema: str = os.getenv('DB_SCHEMA', 'market_data')
|
|
pool_size: int = int(os.getenv('DB_POOL_SIZE', '10'))
|
|
max_overflow: int = int(os.getenv('DB_MAX_OVERFLOW', '20'))
|
|
pool_timeout: int = int(os.getenv('DB_POOL_TIMEOUT', '30'))
|
|
|
|
|
|
@dataclass
|
|
class RedisConfig:
|
|
"""Redis configuration settings"""
|
|
host: str = os.getenv('REDIS_HOST', '192.168.0.10')
|
|
port: int = int(os.getenv('REDIS_PORT', '6379'))
|
|
password: str = os.getenv('REDIS_PASSWORD', 'market_data_redis_2024')
|
|
db: int = int(os.getenv('REDIS_DB', '0'))
|
|
max_connections: int = int(os.getenv('REDIS_MAX_CONNECTIONS', '50'))
|
|
socket_timeout: int = int(os.getenv('REDIS_SOCKET_TIMEOUT', '5'))
|
|
socket_connect_timeout: int = int(os.getenv('REDIS_CONNECT_TIMEOUT', '5'))
|
|
|
|
|
|
@dataclass
|
|
class ExchangeConfig:
|
|
"""Exchange configuration settings"""
|
|
exchanges: List[str] = field(default_factory=lambda: [
|
|
'binance', 'coinbase', 'kraken', 'bybit', 'okx',
|
|
'huobi', 'kucoin', 'gateio', 'bitfinex', 'mexc'
|
|
])
|
|
symbols: List[str] = field(default_factory=lambda: ['BTCUSDT', 'ETHUSDT'])
|
|
max_connections_per_exchange: int = int(os.getenv('MAX_CONNECTIONS_PER_EXCHANGE', '5'))
|
|
reconnect_delay: int = int(os.getenv('RECONNECT_DELAY', '5'))
|
|
max_reconnect_attempts: int = int(os.getenv('MAX_RECONNECT_ATTEMPTS', '10'))
|
|
heartbeat_interval: int = int(os.getenv('HEARTBEAT_INTERVAL', '30'))
|
|
|
|
|
|
@dataclass
|
|
class AggregationConfig:
|
|
"""Data aggregation configuration"""
|
|
bucket_size: float = float(os.getenv('BUCKET_SIZE', '1.0')) # $1 USD buckets for all symbols
|
|
heatmap_depth: int = int(os.getenv('HEATMAP_DEPTH', '50')) # Number of price levels
|
|
update_frequency: float = float(os.getenv('UPDATE_FREQUENCY', '0.5')) # Seconds
|
|
volume_threshold: float = float(os.getenv('VOLUME_THRESHOLD', '0.01')) # Minimum volume
|
|
|
|
|
|
@dataclass
|
|
class PerformanceConfig:
|
|
"""Performance and optimization settings"""
|
|
data_buffer_size: int = int(os.getenv('DATA_BUFFER_SIZE', '10000'))
|
|
batch_write_size: int = int(os.getenv('BATCH_WRITE_SIZE', '1000'))
|
|
max_memory_usage: int = int(os.getenv('MAX_MEMORY_USAGE', '2048')) # MB
|
|
gc_threshold: float = float(os.getenv('GC_THRESHOLD', '0.8')) # 80% of max memory
|
|
processing_timeout: int = int(os.getenv('PROCESSING_TIMEOUT', '10')) # Seconds
|
|
max_queue_size: int = int(os.getenv('MAX_QUEUE_SIZE', '50000'))
|
|
|
|
|
|
@dataclass
|
|
class APIConfig:
|
|
"""API server configuration"""
|
|
host: str = os.getenv('API_HOST', '0.0.0.0')
|
|
port: int = int(os.getenv('API_PORT', '8080'))
|
|
websocket_port: int = int(os.getenv('WS_PORT', '8081'))
|
|
cors_origins: List[str] = field(default_factory=lambda: ['*'])
|
|
rate_limit: int = int(os.getenv('RATE_LIMIT', '100')) # Requests per minute
|
|
max_connections: int = int(os.getenv('MAX_WS_CONNECTIONS', '1000'))
|
|
|
|
|
|
@dataclass
|
|
class LoggingConfig:
|
|
"""Logging configuration"""
|
|
level: str = os.getenv('LOG_LEVEL', 'INFO')
|
|
format: str = os.getenv('LOG_FORMAT', '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
file_path: str = os.getenv('LOG_FILE', 'logs/coby.log')
|
|
max_file_size: int = int(os.getenv('LOG_MAX_SIZE', '100')) # MB
|
|
backup_count: int = int(os.getenv('LOG_BACKUP_COUNT', '5'))
|
|
enable_correlation_id: bool = os.getenv('ENABLE_CORRELATION_ID', 'true').lower() == 'true'
|
|
|
|
|
|
@dataclass
|
|
class Config:
|
|
"""Main configuration class"""
|
|
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
|
redis: RedisConfig = field(default_factory=RedisConfig)
|
|
exchanges: ExchangeConfig = field(default_factory=ExchangeConfig)
|
|
aggregation: AggregationConfig = field(default_factory=AggregationConfig)
|
|
performance: PerformanceConfig = field(default_factory=PerformanceConfig)
|
|
api: APIConfig = field(default_factory=APIConfig)
|
|
logging: LoggingConfig = field(default_factory=LoggingConfig)
|
|
|
|
# Environment
|
|
environment: str = os.getenv('ENVIRONMENT', 'development')
|
|
debug: bool = os.getenv('DEBUG', 'false').lower() == 'true'
|
|
|
|
def __post_init__(self):
|
|
"""Post-initialization validation and setup"""
|
|
# Create logs directory if it doesn't exist
|
|
log_dir = Path(self.logging.file_path).parent
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Validate bucket sizes
|
|
if self.aggregation.btc_bucket_size <= 0:
|
|
raise ValueError("BTC bucket size must be positive")
|
|
if self.aggregation.eth_bucket_size <= 0:
|
|
raise ValueError("ETH bucket size must be positive")
|
|
|
|
def get_bucket_size(self, symbol: str = None) -> float:
|
|
"""Get bucket size (now universal $1 for all symbols)"""
|
|
return self.aggregation.bucket_size
|
|
|
|
def get_database_url(self) -> str:
|
|
"""Get database connection URL"""
|
|
return (f"postgresql://{self.database.user}:{self.database.password}"
|
|
f"@{self.database.host}:{self.database.port}/{self.database.name}")
|
|
|
|
def get_redis_url(self) -> str:
|
|
"""Get Redis connection URL"""
|
|
auth = f":{self.redis.password}@" if self.redis.password else ""
|
|
return f"redis://{auth}{self.redis.host}:{self.redis.port}/{self.redis.db}"
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
"""Convert configuration to dictionary"""
|
|
return {
|
|
'database': {
|
|
'host': self.database.host,
|
|
'port': self.database.port,
|
|
'name': self.database.name,
|
|
'schema': self.database.schema,
|
|
},
|
|
'redis': {
|
|
'host': self.redis.host,
|
|
'port': self.redis.port,
|
|
'db': self.redis.db,
|
|
},
|
|
'exchanges': {
|
|
'count': len(self.exchanges.exchanges),
|
|
'symbols': self.exchanges.symbols,
|
|
},
|
|
'aggregation': {
|
|
'bucket_size': self.aggregation.bucket_size,
|
|
'heatmap_depth': self.aggregation.heatmap_depth,
|
|
},
|
|
'api': {
|
|
'host': self.api.host,
|
|
'port': self.api.port,
|
|
'websocket_port': self.api.websocket_port,
|
|
},
|
|
'environment': self.environment,
|
|
'debug': self.debug,
|
|
}
|
|
|
|
|
|
# Global configuration instance
|
|
config = Config() |