story: DGR-030 Add accelerator build presets and native CI matrix
This commit is contained in:
Dobromir Popov
@@ -334,3 +334,157 @@ def test_ctest_lane_raises_an_actionable_error_for_a_failing_named_test(tmp_path
|
||||
assert "meshnet-fixture-fail" in str(error)
|
||||
else:
|
||||
raise AssertionError("a failing named CTest lane must raise DependencyError")
|
||||
|
||||
|
||||
def test_accelerator_presets_isolate_one_backend_without_touching_the_cpu_default() -> None:
|
||||
dependency = _load_dependency_module()
|
||||
lock = json.loads((LLAMA_DIR / "UPSTREAM_LOCK.json").read_text())
|
||||
presets = lock["accelerator_presets"]
|
||||
|
||||
assert set(presets) == {"cuda", "rocm", "vulkan", "metal"}
|
||||
base_flags = list(lock["build"]["configure_flags"])
|
||||
base_values = dict(flag[len("-D"):].split("=", 1) for flag in base_flags)
|
||||
|
||||
for name, preset in presets.items():
|
||||
flags = dependency.accelerator_configure_flags(lock, name)
|
||||
|
||||
# The CPU default's own flag list is never mutated by building a preset.
|
||||
assert lock["build"]["configure_flags"] == base_flags
|
||||
|
||||
new_values = dict(flag[len("-D"):].split("=", 1) for flag in flags)
|
||||
backend_flag = preset["backend_flag"]
|
||||
assert base_values[backend_flag] == "OFF"
|
||||
assert new_values[backend_flag] == "ON"
|
||||
for other_flag, value in base_values.items():
|
||||
if other_flag != backend_flag:
|
||||
assert new_values[other_flag] == value, f"{name}: {other_flag} drifted from the CPU default"
|
||||
|
||||
|
||||
def test_accelerator_configure_flags_rejects_an_unknown_lane() -> None:
|
||||
dependency = _load_dependency_module()
|
||||
lock = json.loads((LLAMA_DIR / "UPSTREAM_LOCK.json").read_text())
|
||||
try:
|
||||
dependency.accelerator_configure_flags(lock, "bogus")
|
||||
except dependency.DependencyError as error:
|
||||
assert "unknown accelerator lane" in str(error)
|
||||
else:
|
||||
raise AssertionError("an unknown accelerator lane must be refused")
|
||||
|
||||
|
||||
def test_accelerator_status_reports_unavailable_sdks_without_raising(monkeypatch) -> None:
|
||||
dependency = _load_dependency_module()
|
||||
lock = json.loads((LLAMA_DIR / "UPSTREAM_LOCK.json").read_text())
|
||||
monkeypatch.setattr(dependency.shutil, "which", lambda name: None)
|
||||
|
||||
for name in ("cuda", "rocm", "vulkan"):
|
||||
env_var = lock["accelerator_presets"][name]["sdk_probe"]["env_var"]
|
||||
monkeypatch.delenv(env_var, raising=False)
|
||||
binary = lock["accelerator_presets"][name]["sdk_probe"]["binary"]
|
||||
assert dependency.accelerator_status(name, lock) == {
|
||||
"lane": name,
|
||||
"available": False,
|
||||
"reason": f"{binary} is unavailable on PATH",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(dependency.sys, "platform", "linux")
|
||||
assert dependency.accelerator_status("metal", lock) == {
|
||||
"lane": "metal",
|
||||
"available": False,
|
||||
"reason": "platform 'linux' is not 'darwin'",
|
||||
}
|
||||
|
||||
|
||||
def test_accelerator_status_honors_an_explicit_sdk_override(tmp_path, monkeypatch) -> None:
|
||||
dependency = _load_dependency_module()
|
||||
lock = json.loads((LLAMA_DIR / "UPSTREAM_LOCK.json").read_text())
|
||||
fake_nvcc = tmp_path / "nvcc"
|
||||
fake_nvcc.write_text("#!/bin/sh\nexit 0\n")
|
||||
fake_nvcc.chmod(0o755)
|
||||
monkeypatch.setenv("CUDACXX", str(fake_nvcc))
|
||||
|
||||
assert dependency.accelerator_status("cuda", lock) == {
|
||||
"lane": "cuda",
|
||||
"available": True,
|
||||
"sdk_binary": str(fake_nvcc),
|
||||
}
|
||||
|
||||
|
||||
def test_accelerator_status_rejects_an_unknown_lane() -> None:
|
||||
dependency = _load_dependency_module()
|
||||
lock = json.loads((LLAMA_DIR / "UPSTREAM_LOCK.json").read_text())
|
||||
try:
|
||||
dependency.accelerator_status("bogus", lock)
|
||||
except dependency.DependencyError as error:
|
||||
assert "unknown accelerator lane" in str(error)
|
||||
else:
|
||||
raise AssertionError("an unknown accelerator lane must be refused")
|
||||
|
||||
|
||||
def test_accelerator_build_refuses_to_compile_an_unavailable_lane(tmp_path, monkeypatch) -> None:
|
||||
dependency = _load_dependency_module()
|
||||
source = tmp_path / "source"
|
||||
(source / "cmake").mkdir(parents=True)
|
||||
(source / "cmake" / "meshnet-patch-stack.cmake").write_text("# marker\n")
|
||||
monkeypatch.setattr(dependency, "_verify_source", lambda *a, **k: None)
|
||||
monkeypatch.setattr(dependency, "_verify_patched_source", lambda *a, **k: None)
|
||||
monkeypatch.setattr(dependency.shutil, "which", lambda name: None)
|
||||
monkeypatch.delenv("CUDACXX", raising=False)
|
||||
|
||||
build_dir = tmp_path / "build-cuda"
|
||||
try:
|
||||
dependency.accelerator_build(source, "cuda", build_dir)
|
||||
except dependency.DependencyError as error:
|
||||
assert "SDK is unavailable" in str(error)
|
||||
else:
|
||||
raise AssertionError("accelerator_build must refuse to compile an unavailable lane")
|
||||
assert not build_dir.exists()
|
||||
|
||||
|
||||
@requires_cmake
|
||||
def test_accelerator_build_compiles_the_available_lane_with_isolated_evidence(tmp_path, monkeypatch) -> None:
|
||||
dependency = _load_dependency_module()
|
||||
|
||||
# A tiny synthetic project stands in for the patched llama.cpp checkout —
|
||||
# it only needs the meshnet patch-stack marker and one target, proving
|
||||
# accelerator_build's configure/build/evidence wiring without a multi-minute
|
||||
# llama.cpp compile or a real GPU SDK.
|
||||
source = tmp_path / "source"
|
||||
(source / "cmake").mkdir(parents=True)
|
||||
(source / "cmake" / "meshnet-patch-stack.cmake").write_text("# marker\n")
|
||||
(source / "CMakeLists.txt").write_text(
|
||||
"cmake_minimum_required(VERSION 3.14)\n"
|
||||
"project(accelerator_lane_fixture NONE)\n"
|
||||
"option(GGML_CUDA \"\" OFF)\n"
|
||||
"if(GGML_CUDA)\n"
|
||||
" file(WRITE ${CMAKE_BINARY_DIR}/lane-flag-on.txt \"on\")\n"
|
||||
"endif()\n"
|
||||
"add_custom_target(fixture-target ALL COMMAND ${CMAKE_COMMAND} -E true)\n"
|
||||
)
|
||||
|
||||
base_lock = json.loads((LLAMA_DIR / "UPSTREAM_LOCK.json").read_text())
|
||||
fake_lock = dict(base_lock)
|
||||
fake_lock["build"] = {
|
||||
**base_lock["build"],
|
||||
"generator": "Unix Makefiles",
|
||||
"configure_flags": ["-DGGML_CUDA=OFF"],
|
||||
"native_targets": ["fixture-target"],
|
||||
}
|
||||
monkeypatch.setattr(dependency, "_load_lock", lambda: fake_lock)
|
||||
monkeypatch.setattr(dependency, "_patches", lambda lock: [])
|
||||
monkeypatch.setattr(dependency, "_verify_source", lambda *a, **k: None)
|
||||
monkeypatch.setattr(dependency, "_verify_patched_source", lambda *a, **k: None)
|
||||
monkeypatch.setenv("CUDACXX", str(dependency._cmake()))
|
||||
|
||||
build_dir = tmp_path / "build-cuda"
|
||||
result = dependency.accelerator_build(source, "cuda", build_dir)
|
||||
|
||||
assert result == build_dir
|
||||
assert (build_dir / "lane-flag-on.txt").is_file()
|
||||
metadata = json.loads((build_dir / "meshnet-build-metadata.json").read_text())
|
||||
assert metadata["lane"] == "cuda"
|
||||
assert metadata["backend_flag"] == "GGML_CUDA"
|
||||
assert metadata["configure_flags"] == ["-DGGML_CUDA=ON"]
|
||||
assert metadata["hardware_execution"] is False
|
||||
assert metadata["hardware_certified"] is False
|
||||
assert metadata["semantic_certification"] is False
|
||||
assert "registered-dark" in metadata["note"]
|
||||
|
||||
171
tests/test_native_accelerator_matrix.py
Normal file
171
tests/test_native_accelerator_matrix.py
Normal file
@@ -0,0 +1,171 @@
|
||||
"""Offline behavior tests for DGR-030's native CI/build matrix orchestration.
|
||||
|
||||
These tests never fetch or compile llama.cpp: `llama_cpp_dependency`'s fetch/
|
||||
apply/reverse/build/smoke/ctest_lane/accelerator_status/accelerator_build are
|
||||
stubbed so the matrix's own lane-reporting and cleanup contract is exercised
|
||||
in isolation. The real compile path is covered separately by
|
||||
`tests/test_llama_cpp_dependency.py`'s `accelerator_build`/CPU-lane tests and
|
||||
by a live run recorded in the DGR-030 evidence README.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
MATRIX_SCRIPT = ROOT / "scripts/native_accelerator_matrix.py"
|
||||
DEP_SCRIPT = ROOT / "scripts/llama_cpp_dependency.py"
|
||||
|
||||
|
||||
def _load_matrix_module(monkeypatch):
|
||||
"""Load private copies of both modules with a controllable `dep`.
|
||||
|
||||
`native_accelerator_matrix.py` does `import llama_cpp_dependency as dep`
|
||||
after inserting `scripts/` onto `sys.path`; pre-registering our own module
|
||||
instance under that name in `sys.modules` (undone by monkeypatch at
|
||||
teardown) makes the matrix module bind to the stub instead of importing a
|
||||
fresh copy of the real dependency module.
|
||||
"""
|
||||
dep_spec = importlib.util.spec_from_file_location("llama_cpp_dependency_matrix_dep", DEP_SCRIPT)
|
||||
dep = importlib.util.module_from_spec(dep_spec)
|
||||
dep_spec.loader.exec_module(dep)
|
||||
monkeypatch.setitem(sys.modules, "llama_cpp_dependency", dep)
|
||||
|
||||
matrix_spec = importlib.util.spec_from_file_location("native_accelerator_matrix", MATRIX_SCRIPT)
|
||||
matrix = importlib.util.module_from_spec(matrix_spec)
|
||||
matrix_spec.loader.exec_module(matrix)
|
||||
return matrix, dep
|
||||
|
||||
|
||||
def test_matrix_reports_unavailable_accelerator_sdks_as_skipped_not_false_success(tmp_path, monkeypatch) -> None:
|
||||
matrix, dep = _load_matrix_module(monkeypatch)
|
||||
|
||||
workspace = tmp_path / "llama.cpp"
|
||||
source = workspace / "source"
|
||||
source.mkdir(parents=True)
|
||||
calls: list = []
|
||||
|
||||
monkeypatch.setattr(dep, "fetch", lambda ws: source)
|
||||
monkeypatch.setattr(dep, "apply", lambda src: calls.append(("apply", src)))
|
||||
monkeypatch.setattr(dep, "reverse", lambda src: calls.append(("reverse", src)))
|
||||
monkeypatch.setattr(
|
||||
dep,
|
||||
"_load_lock",
|
||||
lambda: {"accelerator_presets": {"cuda": {}, "rocm": {}, "vulkan": {}, "metal": {}}},
|
||||
)
|
||||
|
||||
def _cpu_build(src, build_dir):
|
||||
build_dir.mkdir(parents=True)
|
||||
(build_dir / "meshnet-build-metadata.json").write_text(json.dumps({"lane": "cpu"}))
|
||||
return build_dir / "bin/llama-gguf-hash"
|
||||
|
||||
monkeypatch.setattr(dep, "build", _cpu_build)
|
||||
monkeypatch.setattr(dep, "smoke", lambda binary: calls.append(("smoke", binary)))
|
||||
monkeypatch.setattr(dep, "ctest_lane", lambda build_dir: calls.append(("ctest", build_dir)))
|
||||
monkeypatch.setattr(
|
||||
dep,
|
||||
"accelerator_status",
|
||||
lambda name, lock: {"lane": name, "available": False, "reason": f"{name} SDK is unavailable on PATH"},
|
||||
)
|
||||
|
||||
report = matrix.run_matrix(workspace)
|
||||
|
||||
assert report["lanes"][0] == {
|
||||
"lane": "cpu",
|
||||
"status": "built",
|
||||
"build_dir": str((workspace / "build").resolve()),
|
||||
"metadata": {"lane": "cpu"},
|
||||
}
|
||||
accelerator_lanes = {lane["lane"]: lane for lane in report["lanes"][1:]}
|
||||
assert set(accelerator_lanes) == {"cuda", "rocm", "vulkan", "metal"}
|
||||
for name, lane in accelerator_lanes.items():
|
||||
assert lane["status"] == "skipped"
|
||||
assert "unavailable" in lane["reason"]
|
||||
|
||||
assert report["failed_lanes"] == []
|
||||
assert report["hardware_certified"] is False
|
||||
assert ("reverse", source) in calls # cleanup always runs
|
||||
# Only the CPU lane is ever smoke-tested/ctested; skipped accelerator lanes are not.
|
||||
smoke_calls = [call for call in calls if call[0] == "smoke"]
|
||||
ctest_calls = [call for call in calls if call[0] == "ctest"]
|
||||
assert smoke_calls == [("smoke", (workspace.resolve() / "build" / "bin/llama-gguf-hash"))]
|
||||
assert ctest_calls == [("ctest", (workspace.resolve() / "build"))]
|
||||
|
||||
|
||||
def test_matrix_compiles_an_available_accelerator_lane_without_smoke_or_ctest(tmp_path, monkeypatch) -> None:
|
||||
matrix, dep = _load_matrix_module(monkeypatch)
|
||||
|
||||
workspace = tmp_path / "llama.cpp"
|
||||
source = workspace / "source"
|
||||
source.mkdir(parents=True)
|
||||
calls: list = []
|
||||
|
||||
monkeypatch.setattr(dep, "fetch", lambda ws: source)
|
||||
monkeypatch.setattr(dep, "apply", lambda src: None)
|
||||
monkeypatch.setattr(dep, "reverse", lambda src: calls.append("reverse"))
|
||||
monkeypatch.setattr(dep, "_load_lock", lambda: {"accelerator_presets": {"cuda": {}}})
|
||||
monkeypatch.setattr(
|
||||
matrix,
|
||||
"_cpu_lane",
|
||||
lambda src, ws: {"lane": "cpu", "status": "skipped", "reason": "pre-existing build dir"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
dep, "accelerator_status", lambda name, lock: {"lane": name, "available": True, "sdk_binary": "/fake/nvcc"}
|
||||
)
|
||||
|
||||
def _accelerator_build(src, name, build_dir):
|
||||
calls.append(("accelerator_build", name))
|
||||
build_dir.mkdir(parents=True)
|
||||
(build_dir / "meshnet-build-metadata.json").write_text(
|
||||
json.dumps({"lane": name, "hardware_certified": False})
|
||||
)
|
||||
return build_dir
|
||||
|
||||
monkeypatch.setattr(dep, "accelerator_build", _accelerator_build)
|
||||
monkeypatch.setattr(dep, "smoke", lambda binary: calls.append(("smoke", binary)))
|
||||
monkeypatch.setattr(dep, "ctest_lane", lambda build_dir: calls.append(("ctest", build_dir)))
|
||||
|
||||
report = matrix.run_matrix(workspace)
|
||||
|
||||
assert report["lanes"][1]["lane"] == "cuda"
|
||||
assert report["lanes"][1]["status"] == "built"
|
||||
assert report["lanes"][1]["metadata"]["hardware_certified"] is False
|
||||
assert ("accelerator_build", "cuda") in calls
|
||||
assert not any(call[0] in ("smoke", "ctest") for call in calls if isinstance(call, tuple))
|
||||
assert "reverse" in calls
|
||||
|
||||
|
||||
def test_matrix_reports_a_lane_failure_without_aborting_the_others_or_skipping_reverse(tmp_path, monkeypatch) -> None:
|
||||
matrix, dep = _load_matrix_module(monkeypatch)
|
||||
|
||||
workspace = tmp_path / "llama.cpp"
|
||||
source = workspace / "source"
|
||||
source.mkdir(parents=True)
|
||||
calls: list = []
|
||||
|
||||
monkeypatch.setattr(dep, "fetch", lambda ws: source)
|
||||
monkeypatch.setattr(dep, "apply", lambda src: None)
|
||||
monkeypatch.setattr(dep, "reverse", lambda src: calls.append("reverse"))
|
||||
monkeypatch.setattr(dep, "_load_lock", lambda: {"accelerator_presets": {"cuda": {}, "vulkan": {}}})
|
||||
|
||||
def _cpu_lane_raises(src, ws):
|
||||
raise dep.DependencyError("simulated cpu compile failure")
|
||||
|
||||
monkeypatch.setattr(matrix, "_cpu_lane", _cpu_lane_raises)
|
||||
monkeypatch.setattr(
|
||||
dep,
|
||||
"accelerator_status",
|
||||
lambda name, lock: {"lane": name, "available": False, "reason": f"{name} SDK is unavailable on PATH"},
|
||||
)
|
||||
|
||||
report = matrix.run_matrix(workspace)
|
||||
|
||||
assert report["lanes"][0] == {"lane": "cpu", "status": "failed", "reason": "simulated cpu compile failure"}
|
||||
assert report["failed_lanes"] == ["cpu"]
|
||||
accelerator_statuses = {lane["lane"]: lane["status"] for lane in report["lanes"][1:]}
|
||||
assert accelerator_statuses == {"cuda": "skipped", "vulkan": "skipped"}
|
||||
assert "reverse" in calls
|
||||
Reference in New Issue
Block a user