Require executable CUDA for GPU mode

packages/node/meshnet_node/hardware.py

@@ -105,12 +105,57 @@ def _detect_windows_gpu_memory() -> dict | None:
     return best
 
 
+def _detect_nvidia_smi_gpu_memory() -> dict | None:
+    """Return NVIDIA GPU memory metadata from nvidia-smi, if available."""
+    try:
+        result = subprocess.run(
+            ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"],
+            capture_output=True, text=True, timeout=5,
+        )
+        if result.returncode == 0 and result.stdout.strip():
+            line = result.stdout.strip().splitlines()[0]
+            parts = line.split(",", 1)
+            gpu_name = parts[0].strip()
+            vram_mb = int(parts[1].strip()) if len(parts) > 1 else 0
+            return {"gpu_name": gpu_name, "vram_mb": max(0, vram_mb)}
+    except (FileNotFoundError, subprocess.TimeoutExpired, ValueError, IndexError):
+        pass
+    return None
+
+
+def _torch_cuda_is_executable(torch_module) -> bool:
+    """Return True only if this Python process can execute a CUDA tensor op."""
+    try:
+        if not torch_module.cuda.is_available():
+            return False
+        probe = torch_module.empty((1,), device="cuda")
+        probe += 1
+        torch_module.cuda.synchronize()
+        return True
+    except Exception:
+        return False
+
+
+def _gpu_inventory_profile(gpu: dict | None, ram_mb: int) -> dict | None:
+    if gpu is None:
+        return None
+    return {
+        "device": "cpu",
+        "gpu_name": gpu["gpu_name"],
+        "vram_mb": gpu["vram_mb"],
+        "dedicated_vram_mb": gpu["vram_mb"],
+        "shared_vram_mb": max(0, ram_mb // 2),
+        "ram_mb": ram_mb,
+        "cuda_available": False,
+    }
+
+
 def detect_hardware() -> dict:
     """Detect GPU model and available VRAM. Returns hardware profile dict."""
     ram_mb = _detect_ram_mb()
     try:
         import torch  # type: ignore[import]
-        if torch.cuda.is_available():
+        if _torch_cuda_is_executable(torch):
             idx = torch.cuda.current_device()
             name = torch.cuda.get_device_name(idx)
             props = torch.cuda.get_device_properties(idx)
@@ -123,42 +168,18 @@ def detect_hardware() -> dict:
                 "dedicated_vram_mb": vram_mb,
                 "shared_vram_mb": shared_vram_mb,
                 "ram_mb": ram_mb,
+                "cuda_available": True,
             }
     except ImportError:
         pass
 
-    try:
-        result = subprocess.run(
-            ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"],
-            capture_output=True, text=True, timeout=5,
-        )
-        if result.returncode == 0 and result.stdout.strip():
-            line = result.stdout.strip().splitlines()[0]
-            parts = line.split(",", 1)
-            gpu_name = parts[0].strip()
-            vram_mb = int(parts[1].strip()) if len(parts) > 1 else 0
-            shared_vram_mb = max(0, ram_mb // 2)
-            return {
-                "device": "cuda",
-                "gpu_name": gpu_name,
-                "vram_mb": vram_mb,
-                "dedicated_vram_mb": vram_mb,
-                "shared_vram_mb": shared_vram_mb,
-                "ram_mb": ram_mb,
-            }
-    except (FileNotFoundError, subprocess.TimeoutExpired, ValueError, IndexError):
-        pass
+    nvidia_gpu = _gpu_inventory_profile(_detect_nvidia_smi_gpu_memory(), ram_mb)
+    if nvidia_gpu is not None:
+        return nvidia_gpu
 
-    windows_gpu = _detect_windows_gpu_memory()
+    windows_gpu = _gpu_inventory_profile(_detect_windows_gpu_memory(), ram_mb)
     if windows_gpu is not None:
-        return {
-            "device": "cpu",
-            "gpu_name": windows_gpu["gpu_name"],
-            "vram_mb": windows_gpu["vram_mb"],
-            "dedicated_vram_mb": windows_gpu["vram_mb"],
-            "shared_vram_mb": max(0, ram_mb // 2),
-            "ram_mb": ram_mb,
-        }
+        return windows_gpu
 
     return {
         "device": "cpu",
@@ -167,6 +188,7 @@ def detect_hardware() -> dict:
         "dedicated_vram_mb": 0,
         "shared_vram_mb": 0,
         "ram_mb": ram_mb,
+        "cuda_available": False,
     }
 
 

packages/node/meshnet_node/startup.py

@@ -33,6 +33,14 @@ def _memory_budget(device: str, vram_mb: int, ram_mb: int, shared_vram_mb: int =
     return max(0, ram_mb), "RAM"
 
 
+def _hardware_label(device: str, gpu_name: str | None = None) -> str:
+    if device == "cuda":
+        return "CUDA"
+    if gpu_name:
+        return "CPU (CUDA inactive)"
+    return "CPU"
+
+
 def _max_assignable_layers(memory_mb: int, total_layers: int | None) -> int:
     if total_layers is None or total_layers <= 0 or memory_mb <= 0:
         return 0
@@ -360,7 +368,10 @@ def run_startup(
     elif device == "cpu":
         gpu_suffix = ""
         if gpu_name and vram_mb > 0:
-            gpu_suffix = f"; detected {gpu_name} ({vram_mb / 1024:.1f} GB dedicated VRAM, {shared_vram_mb / 1024:.1f} GB shared)"
+            gpu_suffix = (
+                f"; CUDA inactive; detected {gpu_name} "
+                f"({vram_mb / 1024:.1f} GB dedicated VRAM, {shared_vram_mb / 1024:.1f} GB shared)"
+            )
         print(f"  WARNING: No CUDA GPU detected — running in CPU mode ({ram_mb / 1024:.1f} GB RAM{gpu_suffix})", flush=True)
     else:
         shared_suffix = f", {shared_vram_mb / 1024:.1f} GB shared" if shared_vram_mb > 0 else ""
@@ -497,7 +508,7 @@ def run_startup(
             f"  Quantization: {quantization}\n"
             f"  Endpoint:     {endpoint}\n"
             f"  Node ID:      {tracker_node_id or 'unregistered'}\n"
-            f"  Hardware:     {device.upper()}\n"
+            f"  Hardware:     {_hardware_label(device, gpu_name)}\n"
             f"  Benchmark:    {bench_tps:,.0f} (throughput index)\n"
             f"{'=' * 32}",
             flush=True,
@@ -590,7 +601,7 @@ def run_startup(
             f"  Quantization: {quantization}\n"
             f"  Endpoint:     {endpoint}\n"
             f"  Node ID:      {tracker_node_id or 'unregistered'}\n"
-            f"  Hardware:     {device.upper()}\n"
+            f"  Hardware:     {_hardware_label(device, gpu_name)}\n"
             f"  Benchmark:    {bench_tps:,.0f} (throughput index)\n"
             f"{'=' * 32}",
             flush=True,