160 lines
5.0 KiB
Python
160 lines
5.0 KiB
Python
"""
|
|
Simple shared API rate limiter and HTTP client wrapper.
|
|
|
|
Provides a singleton via get_rate_limiter() used by data providers
|
|
to throttle external API calls (e.g., Binance) and centralize retries.
|
|
|
|
Windows-safe, ASCII-only logging.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import time
|
|
import logging
|
|
from typing import Any, Dict, Optional, Tuple
|
|
|
|
try:
|
|
import requests
|
|
except Exception: # requests should be available; fail lazily in make_request
|
|
requests = None # type: ignore
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class RateLimiter:
|
|
"""Basic per-key interval limiter with retry/backoff HTTP wrapper.
|
|
|
|
- Per-key min interval between calls
|
|
- Optional simple error backoff per key
|
|
- Thread-safe
|
|
"""
|
|
|
|
def __init__(self, config: Optional[Dict[str, Dict[str, Any]]] = None) -> None:
|
|
# Defaults tailored for typical public REST endpoints
|
|
self._config: Dict[str, Dict[str, Any]] = config or {
|
|
# Binance REST defaults
|
|
"binance_api": {
|
|
"min_interval": 0.25, # seconds between calls (~4 req/s)
|
|
"timeout": 10.0,
|
|
"max_retries": 2,
|
|
"backoff_base": 0.75, # seconds
|
|
"backoff_factor": 2.0,
|
|
}
|
|
}
|
|
self._lock = threading.Lock()
|
|
self._last_request_ts: Dict[str, float] = {}
|
|
self._error_backoff_until: Dict[str, float] = {}
|
|
|
|
def _now(self) -> float:
|
|
return time.monotonic()
|
|
|
|
def _get_key_config(self, key: str) -> Dict[str, Any]:
|
|
return self._config.get(key, {
|
|
"min_interval": 0.5,
|
|
"timeout": 10.0,
|
|
"max_retries": 1,
|
|
"backoff_base": 0.5,
|
|
"backoff_factor": 2.0,
|
|
})
|
|
|
|
def can_make_request(self, key: str) -> Tuple[bool, float]:
|
|
"""Return (can_request, wait_time_seconds)."""
|
|
cfg = self._get_key_config(key)
|
|
now = self._now()
|
|
with self._lock:
|
|
min_interval = float(cfg.get("min_interval", 0.5))
|
|
last = self._last_request_ts.get(key, 0.0)
|
|
ready_at = last + min_interval
|
|
|
|
# Respect error backoff if set
|
|
backoff_until = self._error_backoff_until.get(key, 0.0)
|
|
ready_at = max(ready_at, backoff_until)
|
|
|
|
if now >= ready_at:
|
|
return True, 0.0
|
|
return False, max(0.0, ready_at - now)
|
|
|
|
def _note_request(self, key: str) -> None:
|
|
with self._lock:
|
|
self._last_request_ts[key] = self._now()
|
|
|
|
def _note_error(self, key: str, attempt_index: int) -> None:
|
|
cfg = self._get_key_config(key)
|
|
base = float(cfg.get("backoff_base", 0.5))
|
|
factor = float(cfg.get("backoff_factor", 2.0))
|
|
delay = base * (factor ** max(0, attempt_index))
|
|
with self._lock:
|
|
self._error_backoff_until[key] = max(self._error_backoff_until.get(key, 0.0), self._now() + delay)
|
|
|
|
def make_request(
|
|
self,
|
|
key: str,
|
|
url: str,
|
|
method: str = "GET",
|
|
params: Optional[Dict[str, Any]] = None,
|
|
data: Optional[Any] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
) -> Optional["requests.Response"]:
|
|
"""Perform an HTTP request with per-key rate limiting and retries.
|
|
|
|
Returns Response or None on failure.
|
|
"""
|
|
cfg = self._get_key_config(key)
|
|
timeout = float(cfg.get("timeout", 10.0))
|
|
max_retries = int(cfg.get("max_retries", 1))
|
|
|
|
if requests is None:
|
|
logger.error("requests library not available")
|
|
return None
|
|
|
|
method_upper = (method or "GET").upper()
|
|
|
|
for attempt in range(max_retries + 1):
|
|
can, wait = self.can_make_request(key)
|
|
if not can:
|
|
time.sleep(min(wait, 5.0)) # cap local wait
|
|
|
|
self._note_request(key)
|
|
try:
|
|
resp = requests.request(
|
|
method=method_upper,
|
|
url=url,
|
|
params=params,
|
|
data=data,
|
|
headers=headers,
|
|
timeout=timeout,
|
|
)
|
|
|
|
# If server imposes rate limit or transient error, backoff and retry
|
|
if resp is None or resp.status_code >= 500 or resp.status_code in (408, 429):
|
|
logger.warning(
|
|
"HTTP retry %s for key %s status %s", attempt, key, getattr(resp, "status_code", "n/a")
|
|
)
|
|
self._note_error(key, attempt)
|
|
if attempt < max_retries:
|
|
continue
|
|
return resp
|
|
|
|
return resp
|
|
|
|
except Exception as ex:
|
|
logger.warning("HTTP error on %s %s: %s", method_upper, url, ex)
|
|
self._note_error(key, attempt)
|
|
if attempt < max_retries:
|
|
continue
|
|
return None
|
|
|
|
|
|
_singleton: Optional[RateLimiter] = None
|
|
|
|
|
|
def get_rate_limiter() -> RateLimiter:
|
|
global _singleton
|
|
if _singleton is None:
|
|
_singleton = RateLimiter()
|
|
return _singleton
|
|
|
|
|