93 lines
3.3 KiB
Python
93 lines
3.3 KiB
Python
"""GPU hardware detection with graceful CPU fallback."""
|
||
|
||
import os
|
||
import subprocess
|
||
import time
|
||
|
||
|
||
def _detect_ram_mb() -> int:
|
||
"""Return host physical RAM in MB, or 0 if unavailable."""
|
||
try:
|
||
pages = os.sysconf("SC_PHYS_PAGES")
|
||
page_size = os.sysconf("SC_PAGE_SIZE")
|
||
return int((pages * page_size) // (1024 * 1024))
|
||
except (AttributeError, OSError, ValueError):
|
||
return 0
|
||
|
||
|
||
def detect_hardware() -> dict:
|
||
"""Detect GPU model and available VRAM. Returns hardware profile dict."""
|
||
ram_mb = _detect_ram_mb()
|
||
try:
|
||
import torch # type: ignore[import]
|
||
if torch.cuda.is_available():
|
||
idx = torch.cuda.current_device()
|
||
name = torch.cuda.get_device_name(idx)
|
||
props = torch.cuda.get_device_properties(idx)
|
||
vram_mb = props.total_memory // (1024 * 1024)
|
||
return {"device": "cuda", "gpu_name": name, "vram_mb": vram_mb, "ram_mb": ram_mb}
|
||
except ImportError:
|
||
pass
|
||
|
||
try:
|
||
result = subprocess.run(
|
||
["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"],
|
||
capture_output=True, text=True, timeout=5,
|
||
)
|
||
if result.returncode == 0 and result.stdout.strip():
|
||
line = result.stdout.strip().splitlines()[0]
|
||
parts = line.split(",", 1)
|
||
gpu_name = parts[0].strip()
|
||
vram_mb = int(parts[1].strip()) if len(parts) > 1 else 0
|
||
return {"device": "cuda", "gpu_name": gpu_name, "vram_mb": vram_mb, "ram_mb": ram_mb}
|
||
except (FileNotFoundError, subprocess.TimeoutExpired, ValueError, IndexError):
|
||
pass
|
||
|
||
return {"device": "cpu", "gpu_name": None, "vram_mb": 0, "ram_mb": ram_mb}
|
||
|
||
|
||
def benchmark_throughput(device_str: str = "cpu") -> float:
|
||
"""
|
||
Estimate compute throughput via a synthetic transformer GEMM benchmark.
|
||
|
||
Runs hidden_size × (hidden_size*4) matmul — the dominant op in FFN layers —
|
||
and returns iterations/second as a relative speed index. Higher = faster.
|
||
The value is used as benchmark_tokens_per_sec in tracker registration for
|
||
routing tiebreaks; it is not an absolute token rate.
|
||
|
||
Falls back to 1.0 if torch is unavailable.
|
||
"""
|
||
try:
|
||
import torch # type: ignore[import]
|
||
|
||
device = torch.device(device_str)
|
||
# bfloat16 on CUDA matches real inference dtype; float32 on CPU avoids
|
||
# precision-downcast surprises on older hardware without bfloat16 support.
|
||
dtype = torch.bfloat16 if device_str == "cuda" else torch.float32
|
||
|
||
# hidden_size=2048 is representative of a mid-sized model; large enough
|
||
# that BLAS finds an efficient kernel on both GPU and CPU.
|
||
hidden_size = 2048
|
||
a = torch.randn(1, hidden_size, dtype=dtype, device=device)
|
||
b = torch.randn(hidden_size, hidden_size * 4, dtype=dtype, device=device)
|
||
|
||
def _sync() -> None:
|
||
if device_str == "cuda":
|
||
torch.cuda.synchronize()
|
||
|
||
# Warmup: prime caches and JIT compilation.
|
||
for _ in range(10):
|
||
torch.matmul(a, b)
|
||
_sync()
|
||
|
||
n_iters = 50
|
||
t0 = time.perf_counter()
|
||
for _ in range(n_iters):
|
||
torch.matmul(a, b)
|
||
_sync()
|
||
elapsed = time.perf_counter() - t0
|
||
|
||
return round(n_iters / max(elapsed, 1e-9), 2)
|
||
except Exception:
|
||
return 1.0
|