"""Offline behavior tests for DGR-030's native CI/build matrix orchestration. These tests never fetch or compile llama.cpp: `llama_cpp_dependency`'s fetch/ apply/reverse/build/smoke/ctest_lane/accelerator_status/accelerator_build are stubbed so the matrix's own lane-reporting and cleanup contract is exercised in isolation. The real compile path is covered separately by `tests/test_llama_cpp_dependency.py`'s `accelerator_build`/CPU-lane tests and by a live run recorded in the DGR-030 evidence README. """ from __future__ import annotations import importlib.util import json import pathlib import sys ROOT = pathlib.Path(__file__).resolve().parents[1] MATRIX_SCRIPT = ROOT / "scripts/native_accelerator_matrix.py" DEP_SCRIPT = ROOT / "scripts/llama_cpp_dependency.py" def _load_matrix_module(monkeypatch): """Load private copies of both modules with a controllable `dep`. `native_accelerator_matrix.py` does `import llama_cpp_dependency as dep` after inserting `scripts/` onto `sys.path`; pre-registering our own module instance under that name in `sys.modules` (undone by monkeypatch at teardown) makes the matrix module bind to the stub instead of importing a fresh copy of the real dependency module. """ dep_spec = importlib.util.spec_from_file_location("llama_cpp_dependency_matrix_dep", DEP_SCRIPT) dep = importlib.util.module_from_spec(dep_spec) dep_spec.loader.exec_module(dep) monkeypatch.setitem(sys.modules, "llama_cpp_dependency", dep) matrix_spec = importlib.util.spec_from_file_location("native_accelerator_matrix", MATRIX_SCRIPT) matrix = importlib.util.module_from_spec(matrix_spec) matrix_spec.loader.exec_module(matrix) return matrix, dep def test_matrix_reports_unavailable_accelerator_sdks_as_skipped_not_false_success(tmp_path, monkeypatch) -> None: matrix, dep = _load_matrix_module(monkeypatch) workspace = tmp_path / "llama.cpp" source = workspace / "source" source.mkdir(parents=True) calls: list = [] monkeypatch.setattr(dep, "fetch", lambda ws: source) monkeypatch.setattr(dep, "apply", lambda src: calls.append(("apply", src))) monkeypatch.setattr(dep, "reverse", lambda src: calls.append(("reverse", src))) monkeypatch.setattr( dep, "_load_lock", lambda: {"accelerator_presets": {"cuda": {}, "rocm": {}, "vulkan": {}, "metal": {}}}, ) def _cpu_build(src, build_dir): build_dir.mkdir(parents=True) (build_dir / "meshnet-build-metadata.json").write_text(json.dumps({"lane": "cpu"})) return build_dir / "bin/llama-gguf-hash" monkeypatch.setattr(dep, "build", _cpu_build) monkeypatch.setattr(dep, "smoke", lambda binary: calls.append(("smoke", binary))) monkeypatch.setattr(dep, "ctest_lane", lambda build_dir: calls.append(("ctest", build_dir))) monkeypatch.setattr( dep, "accelerator_status", lambda name, lock: {"lane": name, "available": False, "reason": f"{name} SDK is unavailable on PATH"}, ) report = matrix.run_matrix(workspace) assert report["lanes"][0] == { "lane": "cpu", "status": "built", "build_dir": str((workspace / "build").resolve()), "metadata": {"lane": "cpu"}, } accelerator_lanes = {lane["lane"]: lane for lane in report["lanes"][1:]} assert set(accelerator_lanes) == {"cuda", "rocm", "vulkan", "metal"} for name, lane in accelerator_lanes.items(): assert lane["status"] == "skipped" assert "unavailable" in lane["reason"] assert report["failed_lanes"] == [] assert report["hardware_certified"] is False assert ("reverse", source) in calls # cleanup always runs # Only the CPU lane is ever smoke-tested/ctested; skipped accelerator lanes are not. smoke_calls = [call for call in calls if call[0] == "smoke"] ctest_calls = [call for call in calls if call[0] == "ctest"] assert smoke_calls == [("smoke", (workspace.resolve() / "build" / "bin/llama-gguf-hash"))] assert ctest_calls == [("ctest", (workspace.resolve() / "build"))] def test_matrix_compiles_an_available_accelerator_lane_without_smoke_or_ctest(tmp_path, monkeypatch) -> None: matrix, dep = _load_matrix_module(monkeypatch) workspace = tmp_path / "llama.cpp" source = workspace / "source" source.mkdir(parents=True) calls: list = [] monkeypatch.setattr(dep, "fetch", lambda ws: source) monkeypatch.setattr(dep, "apply", lambda src: None) monkeypatch.setattr(dep, "reverse", lambda src: calls.append("reverse")) monkeypatch.setattr(dep, "_load_lock", lambda: {"accelerator_presets": {"cuda": {}}}) monkeypatch.setattr( matrix, "_cpu_lane", lambda src, ws: {"lane": "cpu", "status": "skipped", "reason": "pre-existing build dir"}, ) monkeypatch.setattr( dep, "accelerator_status", lambda name, lock: {"lane": name, "available": True, "sdk_binary": "/fake/nvcc"} ) def _accelerator_build(src, name, build_dir): calls.append(("accelerator_build", name)) build_dir.mkdir(parents=True) (build_dir / "meshnet-build-metadata.json").write_text( json.dumps({"lane": name, "hardware_certified": False}) ) return build_dir monkeypatch.setattr(dep, "accelerator_build", _accelerator_build) monkeypatch.setattr(dep, "smoke", lambda binary: calls.append(("smoke", binary))) monkeypatch.setattr(dep, "ctest_lane", lambda build_dir: calls.append(("ctest", build_dir))) report = matrix.run_matrix(workspace) assert report["lanes"][1]["lane"] == "cuda" assert report["lanes"][1]["status"] == "built" assert report["lanes"][1]["metadata"]["hardware_certified"] is False assert ("accelerator_build", "cuda") in calls assert not any(call[0] in ("smoke", "ctest") for call in calls if isinstance(call, tuple)) assert "reverse" in calls def test_matrix_reports_a_lane_failure_without_aborting_the_others_or_skipping_reverse(tmp_path, monkeypatch) -> None: matrix, dep = _load_matrix_module(monkeypatch) workspace = tmp_path / "llama.cpp" source = workspace / "source" source.mkdir(parents=True) calls: list = [] monkeypatch.setattr(dep, "fetch", lambda ws: source) monkeypatch.setattr(dep, "apply", lambda src: None) monkeypatch.setattr(dep, "reverse", lambda src: calls.append("reverse")) monkeypatch.setattr(dep, "_load_lock", lambda: {"accelerator_presets": {"cuda": {}, "vulkan": {}}}) def _cpu_lane_raises(src, ws): raise dep.DependencyError("simulated cpu compile failure") monkeypatch.setattr(matrix, "_cpu_lane", _cpu_lane_raises) monkeypatch.setattr( dep, "accelerator_status", lambda name, lock: {"lane": name, "available": False, "reason": f"{name} SDK is unavailable on PATH"}, ) report = matrix.run_matrix(workspace) assert report["lanes"][0] == {"lane": "cpu", "status": "failed", "reason": "simulated cpu compile failure"} assert report["failed_lanes"] == ["cpu"] accelerator_statuses = {lane["lane"]: lane["status"] for lane in report["lanes"][1:]} assert accelerator_statuses == {"cuda": "skipped", "vulkan": "skipped"} assert "reverse" in calls