"""GPU hardware detection with graceful CPU fallback.""" import os import subprocess import time def _detect_ram_mb() -> int: """Return host physical RAM in MB, or 0 if unavailable.""" try: pages = os.sysconf("SC_PHYS_PAGES") page_size = os.sysconf("SC_PAGE_SIZE") return int((pages * page_size) // (1024 * 1024)) except (AttributeError, OSError, ValueError): return 0 def detect_hardware() -> dict: """Detect GPU model and available VRAM. Returns hardware profile dict.""" ram_mb = _detect_ram_mb() try: import torch # type: ignore[import] if torch.cuda.is_available(): idx = torch.cuda.current_device() name = torch.cuda.get_device_name(idx) props = torch.cuda.get_device_properties(idx) vram_mb = props.total_memory // (1024 * 1024) return {"device": "cuda", "gpu_name": name, "vram_mb": vram_mb, "ram_mb": ram_mb} except ImportError: pass try: result = subprocess.run( ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5, ) if result.returncode == 0 and result.stdout.strip(): line = result.stdout.strip().splitlines()[0] parts = line.split(",", 1) gpu_name = parts[0].strip() vram_mb = int(parts[1].strip()) if len(parts) > 1 else 0 return {"device": "cuda", "gpu_name": gpu_name, "vram_mb": vram_mb, "ram_mb": ram_mb} except (FileNotFoundError, subprocess.TimeoutExpired, ValueError, IndexError): pass return {"device": "cpu", "gpu_name": None, "vram_mb": 0, "ram_mb": ram_mb} def benchmark_throughput(device_str: str = "cpu") -> float: """ Estimate compute throughput via a synthetic transformer GEMM benchmark. Runs hidden_size × (hidden_size*4) matmul — the dominant op in FFN layers — and returns iterations/second as a relative speed index. Higher = faster. The value is used as benchmark_tokens_per_sec in tracker registration for routing tiebreaks; it is not an absolute token rate. Falls back to 1.0 if torch is unavailable. """ try: import torch # type: ignore[import] device = torch.device(device_str) # bfloat16 on CUDA matches real inference dtype; float32 on CPU avoids # precision-downcast surprises on older hardware without bfloat16 support. dtype = torch.bfloat16 if device_str == "cuda" else torch.float32 # hidden_size=2048 is representative of a mid-sized model; large enough # that BLAS finds an efficient kernel on both GPU and CPU. hidden_size = 2048 a = torch.randn(1, hidden_size, dtype=dtype, device=device) b = torch.randn(hidden_size, hidden_size * 4, dtype=dtype, device=device) def _sync() -> None: if device_str == "cuda": torch.cuda.synchronize() # Warmup: prime caches and JIT compilation. for _ in range(10): torch.matmul(a, b) _sync() n_iters = 50 t0 = time.perf_counter() for _ in range(n_iters): torch.matmul(a, b) _sync() elapsed = time.perf_counter() - t0 return round(n_iters / max(elapsed, 1e-9), 2) except Exception: return 1.0