515 lines
19 KiB
Python
515 lines
19 KiB
Python
"""
|
|
REST API server for COBY system.
|
|
"""
|
|
|
|
from fastapi import FastAPI, HTTPException, Request, Query, Path, WebSocket, WebSocketDisconnect
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.responses import JSONResponse, HTMLResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from typing import Optional, List
|
|
import asyncio
|
|
import os
|
|
import json
|
|
import time
|
|
try:
|
|
from ..simple_config import config
|
|
from ..caching.redis_manager import redis_manager
|
|
from ..utils.logging import get_logger, set_correlation_id
|
|
from ..utils.validation import validate_symbol
|
|
from .rate_limiter import RateLimiter
|
|
from .response_formatter import ResponseFormatter
|
|
except ImportError:
|
|
from simple_config import config
|
|
from caching.redis_manager import redis_manager
|
|
from utils.logging import get_logger, set_correlation_id
|
|
from utils.validation import validate_symbol
|
|
from api.rate_limiter import RateLimiter
|
|
from api.response_formatter import ResponseFormatter
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
class ConnectionManager:
|
|
"""Manage WebSocket connections for dashboard updates"""
|
|
|
|
def __init__(self):
|
|
self.active_connections: List[WebSocket] = []
|
|
|
|
async def connect(self, websocket: WebSocket):
|
|
await websocket.accept()
|
|
self.active_connections.append(websocket)
|
|
logger.info(f"WebSocket client connected. Total connections: {len(self.active_connections)}")
|
|
|
|
def disconnect(self, websocket: WebSocket):
|
|
if websocket in self.active_connections:
|
|
self.active_connections.remove(websocket)
|
|
logger.info(f"WebSocket client disconnected. Total connections: {len(self.active_connections)}")
|
|
|
|
async def send_personal_message(self, message: str, websocket: WebSocket):
|
|
try:
|
|
await websocket.send_text(message)
|
|
except Exception as e:
|
|
logger.error(f"Error sending personal message: {e}")
|
|
self.disconnect(websocket)
|
|
|
|
async def broadcast(self, message: str):
|
|
disconnected = []
|
|
for connection in self.active_connections:
|
|
try:
|
|
await connection.send_text(message)
|
|
except Exception as e:
|
|
logger.error(f"Error broadcasting to connection: {e}")
|
|
disconnected.append(connection)
|
|
|
|
# Remove disconnected clients
|
|
for connection in disconnected:
|
|
self.disconnect(connection)
|
|
|
|
|
|
def create_app(config_obj=None) -> FastAPI:
|
|
"""Create and configure FastAPI application"""
|
|
|
|
app = FastAPI(
|
|
title="COBY Market Data API",
|
|
description="Real-time cryptocurrency market data aggregation API",
|
|
version="1.0.0",
|
|
docs_url="/docs",
|
|
redoc_url="/redoc"
|
|
)
|
|
|
|
# Add CORS middleware
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=config.api.cors_origins,
|
|
allow_credentials=True,
|
|
allow_methods=["GET", "POST", "PUT", "DELETE"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# Initialize components
|
|
rate_limiter = RateLimiter(
|
|
requests_per_minute=config.api.rate_limit,
|
|
burst_size=20
|
|
)
|
|
response_formatter = ResponseFormatter()
|
|
connection_manager = ConnectionManager()
|
|
|
|
@app.websocket("/ws/dashboard")
|
|
async def websocket_endpoint(websocket: WebSocket):
|
|
"""WebSocket endpoint for real-time dashboard updates"""
|
|
await connection_manager.connect(websocket)
|
|
try:
|
|
while True:
|
|
# Send periodic status updates
|
|
status_data = {
|
|
"type": "status",
|
|
"timestamp": time.time(),
|
|
"connections": len(connection_manager.active_connections),
|
|
"system": "healthy"
|
|
}
|
|
await connection_manager.send_personal_message(
|
|
json.dumps(status_data), websocket
|
|
)
|
|
await asyncio.sleep(30) # Send update every 30 seconds
|
|
except WebSocketDisconnect:
|
|
connection_manager.disconnect(websocket)
|
|
except Exception as e:
|
|
logger.error(f"WebSocket error: {e}")
|
|
connection_manager.disconnect(websocket)
|
|
|
|
@app.get("/api/health")
|
|
async def api_health_check():
|
|
"""API Health check endpoint for dashboard"""
|
|
try:
|
|
# Check Redis connection
|
|
redis_healthy = await redis_manager.ping()
|
|
|
|
health_data = {
|
|
'status': 'healthy' if redis_healthy else 'degraded',
|
|
'redis': 'connected' if redis_healthy else 'disconnected',
|
|
'version': '1.0.0',
|
|
'timestamp': time.time()
|
|
}
|
|
|
|
return response_formatter.status_response(health_data)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Health check failed: {e}")
|
|
return JSONResponse(
|
|
status_code=503,
|
|
content=response_formatter.error("Service unavailable", "HEALTH_CHECK_FAILED")
|
|
)
|
|
|
|
@app.get("/health")
|
|
async def health_check():
|
|
"""Health check endpoint"""
|
|
try:
|
|
# Check Redis connection
|
|
redis_healthy = await redis_manager.ping()
|
|
|
|
health_data = {
|
|
'status': 'healthy' if redis_healthy else 'degraded',
|
|
'redis': 'connected' if redis_healthy else 'disconnected',
|
|
'version': '1.0.0'
|
|
}
|
|
|
|
return response_formatter.status_response(health_data)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Health check failed: {e}")
|
|
return JSONResponse(
|
|
status_code=503,
|
|
content=response_formatter.error("Service unavailable", "HEALTH_CHECK_FAILED")
|
|
)
|
|
|
|
@app.get("/", response_class=HTMLResponse)
|
|
async def root():
|
|
"""Root endpoint - serve dashboard HTML"""
|
|
static_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "web", "static")
|
|
index_path = os.path.join(static_path, "index.html")
|
|
|
|
if os.path.exists(index_path):
|
|
with open(index_path, 'r', encoding='utf-8') as f:
|
|
return HTMLResponse(content=f.read())
|
|
else:
|
|
# Fallback if index.html doesn't exist
|
|
return HTMLResponse(content="""
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head><title>COBY System</title></head>
|
|
<body>
|
|
<h1>COBY Multi-Exchange Data Aggregation System</h1>
|
|
<p>System is running. Dashboard files not found.</p>
|
|
<p><a href="/api/health">Health Check</a></p>
|
|
</body>
|
|
</html>
|
|
""")
|
|
|
|
@app.middleware("http")
|
|
async def rate_limit_middleware(request: Request, call_next):
|
|
"""Rate limiting middleware"""
|
|
client_ip = request.client.host
|
|
|
|
if not rate_limiter.is_allowed(client_ip):
|
|
client_stats = rate_limiter.get_client_stats(client_ip)
|
|
error_response = response_formatter.rate_limit_error(client_stats)
|
|
return JSONResponse(
|
|
status_code=429,
|
|
content=error_response,
|
|
headers={
|
|
"X-RateLimit-Remaining": str(int(client_stats['remaining_tokens'])),
|
|
"X-RateLimit-Reset": str(int(client_stats['reset_time']))
|
|
}
|
|
)
|
|
|
|
response = await call_next(request)
|
|
|
|
# Add rate limit headers
|
|
client_stats = rate_limiter.get_client_stats(client_ip)
|
|
response.headers["X-RateLimit-Remaining"] = str(int(client_stats['remaining_tokens']))
|
|
response.headers["X-RateLimit-Reset"] = str(int(client_stats['reset_time']))
|
|
|
|
return response
|
|
|
|
@app.middleware("http")
|
|
async def correlation_middleware(request: Request, call_next):
|
|
"""Add correlation ID to requests"""
|
|
set_correlation_id()
|
|
response = await call_next(request)
|
|
return response
|
|
|
|
@app.on_event("startup")
|
|
async def startup_event():
|
|
"""Initialize services on startup"""
|
|
try:
|
|
await redis_manager.initialize()
|
|
logger.info("API server startup completed")
|
|
except Exception as e:
|
|
logger.error(f"API server startup failed: {e}")
|
|
raise
|
|
|
|
@app.on_event("shutdown")
|
|
async def shutdown_event():
|
|
"""Cleanup on shutdown"""
|
|
try:
|
|
await redis_manager.close()
|
|
logger.info("API server shutdown completed")
|
|
except Exception as e:
|
|
logger.error(f"API server shutdown error: {e}")
|
|
|
|
# Heatmap endpoints
|
|
@app.get("/api/v1/heatmap/{symbol}")
|
|
async def get_heatmap(
|
|
symbol: str = Path(..., description="Trading symbol (e.g., BTCUSDT)"),
|
|
exchange: Optional[str] = Query(None, description="Exchange name (None for consolidated)")
|
|
):
|
|
"""Get heatmap data for a symbol"""
|
|
try:
|
|
# Validate symbol
|
|
if not validate_symbol(symbol):
|
|
return JSONResponse(
|
|
status_code=400,
|
|
content=response_formatter.validation_error("symbol", "Invalid symbol format")
|
|
)
|
|
|
|
# Get heatmap from cache
|
|
heatmap_data = await redis_manager.get_heatmap(symbol.upper(), exchange)
|
|
|
|
return response_formatter.heatmap_response(heatmap_data, symbol.upper(), exchange)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting heatmap for {symbol}: {e}")
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=response_formatter.error("Internal server error", "HEATMAP_ERROR")
|
|
)
|
|
|
|
# Order book endpoints
|
|
@app.get("/api/v1/orderbook/{symbol}/{exchange}")
|
|
async def get_orderbook(
|
|
symbol: str = Path(..., description="Trading symbol"),
|
|
exchange: str = Path(..., description="Exchange name")
|
|
):
|
|
"""Get order book data for a symbol on an exchange"""
|
|
try:
|
|
# Validate symbol
|
|
if not validate_symbol(symbol):
|
|
return JSONResponse(
|
|
status_code=400,
|
|
content=response_formatter.validation_error("symbol", "Invalid symbol format")
|
|
)
|
|
|
|
# Get order book from cache
|
|
orderbook_data = await redis_manager.get_orderbook(symbol.upper(), exchange.lower())
|
|
|
|
return response_formatter.orderbook_response(orderbook_data, symbol.upper(), exchange.lower())
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting order book for {symbol}@{exchange}: {e}")
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=response_formatter.error("Internal server error", "ORDERBOOK_ERROR")
|
|
)
|
|
|
|
# Metrics endpoints
|
|
@app.get("/api/v1/metrics/{symbol}/{exchange}")
|
|
async def get_metrics(
|
|
symbol: str = Path(..., description="Trading symbol"),
|
|
exchange: str = Path(..., description="Exchange name")
|
|
):
|
|
"""Get metrics data for a symbol on an exchange"""
|
|
try:
|
|
# Validate symbol
|
|
if not validate_symbol(symbol):
|
|
return JSONResponse(
|
|
status_code=400,
|
|
content=response_formatter.validation_error("symbol", "Invalid symbol format")
|
|
)
|
|
|
|
# Get metrics from cache
|
|
metrics_data = await redis_manager.get_metrics(symbol.upper(), exchange.lower())
|
|
|
|
return response_formatter.metrics_response(metrics_data, symbol.upper(), exchange.lower())
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting metrics for {symbol}@{exchange}: {e}")
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=response_formatter.error("Internal server error", "METRICS_ERROR")
|
|
)
|
|
|
|
# Exchange status endpoints
|
|
@app.get("/api/v1/status/{exchange}")
|
|
async def get_exchange_status(
|
|
exchange: str = Path(..., description="Exchange name")
|
|
):
|
|
"""Get status for an exchange"""
|
|
try:
|
|
# Get status from cache
|
|
status_data = await redis_manager.get_exchange_status(exchange.lower())
|
|
|
|
if not status_data:
|
|
return JSONResponse(
|
|
status_code=404,
|
|
content=response_formatter.error("Exchange status not found", "STATUS_NOT_FOUND")
|
|
)
|
|
|
|
return response_formatter.success(
|
|
data=status_data,
|
|
message=f"Status for {exchange}"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting status for {exchange}: {e}")
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=response_formatter.error("Internal server error", "STATUS_ERROR")
|
|
)
|
|
|
|
# List endpoints
|
|
@app.get("/api/v1/symbols")
|
|
async def list_symbols():
|
|
"""List available trading symbols"""
|
|
try:
|
|
# Get symbols from cache (this would be populated by exchange connectors)
|
|
symbols_pattern = "symbols:*"
|
|
symbol_keys = await redis_manager.keys(symbols_pattern)
|
|
|
|
all_symbols = set()
|
|
for key in symbol_keys:
|
|
symbols_data = await redis_manager.get(key)
|
|
if symbols_data and isinstance(symbols_data, list):
|
|
all_symbols.update(symbols_data)
|
|
|
|
return response_formatter.success(
|
|
data=sorted(list(all_symbols)),
|
|
message="Available trading symbols",
|
|
metadata={'total_symbols': len(all_symbols)}
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error listing symbols: {e}")
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=response_formatter.error("Internal server error", "SYMBOLS_ERROR")
|
|
)
|
|
|
|
@app.get("/api/v1/exchanges")
|
|
async def list_exchanges():
|
|
"""List available exchanges"""
|
|
try:
|
|
# Get exchange status keys
|
|
status_pattern = "st:*"
|
|
status_keys = await redis_manager.keys(status_pattern)
|
|
|
|
exchanges = []
|
|
for key in status_keys:
|
|
# Extract exchange name from key (st:exchange_name)
|
|
exchange_name = key.split(':', 1)[1] if ':' in key else key
|
|
exchanges.append(exchange_name)
|
|
|
|
return response_formatter.success(
|
|
data=sorted(exchanges),
|
|
message="Available exchanges",
|
|
metadata={'total_exchanges': len(exchanges)}
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error listing exchanges: {e}")
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=response_formatter.error("Internal server error", "EXCHANGES_ERROR")
|
|
)
|
|
|
|
# Statistics endpoints
|
|
@app.get("/api/v1/stats/cache")
|
|
async def get_cache_stats():
|
|
"""Get cache statistics"""
|
|
try:
|
|
cache_stats = redis_manager.get_stats()
|
|
redis_health = await redis_manager.health_check()
|
|
|
|
stats_data = {
|
|
'cache_performance': cache_stats,
|
|
'redis_health': redis_health
|
|
}
|
|
|
|
return response_formatter.success(
|
|
data=stats_data,
|
|
message="Cache statistics"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting cache stats: {e}")
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=response_formatter.error("Internal server error", "STATS_ERROR")
|
|
)
|
|
|
|
@app.get("/api/v1/stats/api")
|
|
async def get_api_stats():
|
|
"""Get API statistics"""
|
|
try:
|
|
api_stats = {
|
|
'rate_limiter': rate_limiter.get_global_stats(),
|
|
'response_formatter': response_formatter.get_stats()
|
|
}
|
|
|
|
return response_formatter.success(
|
|
data=api_stats,
|
|
message="API statistics"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting API stats: {e}")
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=response_formatter.error("Internal server error", "API_STATS_ERROR")
|
|
)
|
|
|
|
# Batch endpoints for efficiency
|
|
@app.get("/api/v1/batch/heatmaps")
|
|
async def get_batch_heatmaps(
|
|
symbols: str = Query(..., description="Comma-separated list of symbols"),
|
|
exchange: Optional[str] = Query(None, description="Exchange name (None for consolidated)")
|
|
):
|
|
"""Get heatmaps for multiple symbols"""
|
|
try:
|
|
symbol_list = [s.strip().upper() for s in symbols.split(',')]
|
|
|
|
# Validate all symbols
|
|
for symbol in symbol_list:
|
|
if not validate_symbol(symbol):
|
|
return JSONResponse(
|
|
status_code=400,
|
|
content=response_formatter.validation_error("symbols", f"Invalid symbol: {symbol}")
|
|
)
|
|
|
|
# Get heatmaps in batch
|
|
heatmaps = {}
|
|
for symbol in symbol_list:
|
|
heatmap_data = await redis_manager.get_heatmap(symbol, exchange)
|
|
if heatmap_data:
|
|
heatmaps[symbol] = {
|
|
'symbol': heatmap_data.symbol,
|
|
'timestamp': heatmap_data.timestamp.isoformat(),
|
|
'bucket_size': heatmap_data.bucket_size,
|
|
'points': [
|
|
{
|
|
'price': point.price,
|
|
'volume': point.volume,
|
|
'intensity': point.intensity,
|
|
'side': point.side
|
|
}
|
|
for point in heatmap_data.data
|
|
]
|
|
}
|
|
|
|
return response_formatter.success(
|
|
data=heatmaps,
|
|
message=f"Batch heatmaps for {len(symbol_list)} symbols",
|
|
metadata={
|
|
'requested_symbols': len(symbol_list),
|
|
'found_heatmaps': len(heatmaps),
|
|
'exchange': exchange or 'consolidated'
|
|
}
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting batch heatmaps: {e}")
|
|
return JSONResponse(
|
|
status_code=500,
|
|
content=response_formatter.error("Internal server error", "BATCH_HEATMAPS_ERROR")
|
|
)
|
|
|
|
# Mount static files for web dashboard AFTER all API routes are defined
|
|
static_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "web", "static")
|
|
if os.path.exists(static_path):
|
|
app.mount("/static", StaticFiles(directory=static_path), name="static")
|
|
# Don't mount at root to avoid conflicts with WebSocket and API routes
|
|
|
|
return app
|
|
|
|
|
|
# Create the FastAPI app instance
|
|
app = create_app() |