story: DGR-030 Add accelerator build presets and native CI matrix

2026-07-23 10:51:08 +03:00
parent 254297660a
commit fd742d35c0
13 changed files with 2576 additions and 14 deletions
--- a/scripts/llama_cpp_dependency.py
+++ b/scripts/llama_cpp_dependency.py
@@ -109,9 +109,41 @@ def _load_lock() -> dict[str, Any]:
        "workspace": "build/llama.cpp",
    }:
        raise DependencyError("retrieval must use the locked detached-commit build workspace")
+    _verify_accelerator_presets(lock)
    return lock


+def _verify_accelerator_presets(lock: dict[str, Any]) -> None:
+    """Each preset must isolate one backend that the CPU default leaves OFF.
+
+    This is what keeps DGR-030's presets from ever being able to change the
+    deterministic CPU default recorded in ``build.configure_flags``: a preset
+    can only exist for a flag this lock already pins OFF, and
+    ``accelerator_configure_flags`` only ever returns a fresh list, never
+    mutates ``build.configure_flags`` in place.
+    """
+    presets = lock.get("accelerator_presets", {})
+    if not isinstance(presets, dict):
+        raise DependencyError("accelerator_presets must be a JSON object")
+    if not presets:
+        return
+    base_flags = dict(flag[len("-D"):].split("=", 1) for flag in lock["build"]["configure_flags"])
+    for name, preset in presets.items():
+        if not isinstance(preset, dict):
+            raise DependencyError(f"accelerator_presets.{name} must be a JSON object")
+        backend_flag = preset.get("backend_flag")
+        if not isinstance(backend_flag, str) or not backend_flag:
+            raise DependencyError(f"accelerator_presets.{name} is missing backend_flag")
+        if base_flags.get(backend_flag) != "OFF":
+            raise DependencyError(
+                f"accelerator_presets.{name} backend flag {backend_flag} must be OFF in "
+                "the deterministic CPU default build.configure_flags"
+            )
+        probe = preset.get("sdk_probe")
+        if not isinstance(probe, dict) or not isinstance(probe.get("binary"), str) or not probe["binary"]:
+            raise DependencyError(f"accelerator_presets.{name} is missing an sdk_probe.binary")
+
+
 def _patches(lock: dict[str, Any]) -> list[pathlib.Path]:
    series = [line for line in (PATCH_DIR / "series").read_text().splitlines() if line]
    if series != lock["patch_series"] or series != sorted(series) or not series:
@@ -507,6 +539,116 @@ def ctest_lane(build_dir: pathlib.Path) -> None:
    print(_run(_ctest(), "--test-dir", str(build_dir), "-R", regex, "--output-on-failure"))


+def _sdk_probe(probe: dict[str, Any]) -> str | None:
+    """Resolve one accelerator lane's SDK binary, or None if it is unavailable."""
+    platform_only = probe.get("platform_only")
+    if platform_only and sys.platform != platform_only:
+        return None
+    env_var = probe.get("env_var")
+    if env_var:
+        override = os.environ.get(env_var)
+        if override:
+            return override
+    return shutil.which(probe["binary"])
+
+
+def accelerator_status(name: str, lock: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Report whether lane `name`'s SDK is present, never raising for absence.
+
+    This is the single source of truth for DGR-030's "unavailable/skipped, not
+    false success" contract: absence is reported as data, not swallowed and
+    not escalated into a build attempt.
+    """
+    lock = lock if lock is not None else _load_lock()
+    presets = lock.get("accelerator_presets", {})
+    if name not in presets:
+        raise DependencyError(f"unknown accelerator lane: {name}")
+    probe = presets[name]["sdk_probe"]
+    resolved = _sdk_probe(probe)
+    if resolved is None:
+        platform_only = probe.get("platform_only")
+        if platform_only and sys.platform != platform_only:
+            reason = f"platform {sys.platform!r} is not {platform_only!r}"
+        else:
+            reason = f"{probe['binary']} is unavailable on PATH"
+        return {"lane": name, "available": False, "reason": reason}
+    return {"lane": name, "available": True, "sdk_binary": resolved}
+
+
+def accelerator_configure_flags(lock: dict[str, Any], name: str) -> list[str]:
+    """The CPU default's configure flags with exactly one backend flag flipped ON.
+
+    Returns a new list; `lock["build"]["configure_flags"]` (the deterministic
+    CPU default DGR-029 locked) is never mutated.
+    """
+    presets = lock.get("accelerator_presets", {})
+    if name not in presets:
+        raise DependencyError(f"unknown accelerator lane: {name}")
+    backend_flag = presets[name]["backend_flag"]
+    target = f"-D{backend_flag}="
+    flags: list[str] = []
+    replaced = False
+    for flag in lock["build"]["configure_flags"]:
+        if flag.startswith(target):
+            flags.append(f"-D{backend_flag}=ON")
+            replaced = True
+        else:
+            flags.append(flag)
+    if not replaced:
+        raise DependencyError(f"accelerator lane {name} backend flag {backend_flag} is not a locked base flag")
+    return flags
+
+
+def accelerator_build(source: pathlib.Path, name: str, build_dir: pathlib.Path) -> pathlib.Path:
+    """Compile lane `name` into its own out-of-tree directory. Compile-only.
+
+    This never runs `smoke`/`ctest_lane`: exercising a binary linked against an
+    accelerator backend would touch real hardware, and DGR-030 keeps every
+    backend/model/recipe lane registered-dark (compiled, never certified)
+    until a separate real-hardware certification record exists.
+    """
+    lock = _load_lock()
+    _patches(lock)
+    _verify_source(source, lock, require_clean=False)
+    _verify_patched_source(source, lock)
+    expected_marker = source / "cmake/meshnet-patch-stack.cmake"
+    if not expected_marker.is_file():
+        raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent")
+    if build_dir.exists():
+        raise DependencyError(f"accelerator build directory already exists; use a clean build dir: {build_dir}")
+    status = accelerator_status(name, lock)
+    if not status["available"]:
+        raise DependencyError(f"accelerator lane {name} SDK is unavailable: {status['reason']}")
+    flags = accelerator_configure_flags(lock, name)
+    cmake = _cmake()
+    _run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags)
+    for target in lock["build"]["native_targets"]:
+        _run(cmake, "--build", str(build_dir), "--target", target, "-j2")
+    metadata = {
+        "lane": name,
+        "backend_flag": lock["accelerator_presets"][name]["backend_flag"],
+        "commit": lock["commit"],
+        "commit_tree": lock["commit_tree"],
+        "patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)},
+        "configure_flags": flags,
+        "cmake": _run(cmake, "--version").splitlines()[0],
+        "cxx": _run("c++", "--version").splitlines()[0],
+        "sdk_binary": status["sdk_binary"],
+        "model_downloads": False,
+        "hardware_execution": False,
+        "hardware_certified": False,
+        "semantic_certification": False,
+        "note": (
+            "compiled only; no accelerator device was exercised or driven. "
+            "Backend/model/recipe capability remains registered-dark until a "
+            "separate real-hardware certification record exists (see "
+            "DGR-041/053/067)."
+        ),
+    }
+    (build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
+    return build_dir
+
+
 def verify(workspace: pathlib.Path) -> None:
    """Apply, verify, reverse, and leave the exact cached pin pristine."""
    source = fetch(workspace)
@@ -562,6 +704,12 @@ def main() -> int:
    smoke_parser.add_argument("--binary", type=pathlib.Path, required=True)
    ctest_parser = subcommands.add_parser("ctest")
    ctest_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
+    accel_status_parser = subcommands.add_parser("accelerator-status")
+    accel_status_parser.add_argument("--name", required=True)
+    accel_build_parser = subcommands.add_parser("accelerator-build")
+    accel_build_parser.add_argument("--name", required=True)
+    accel_build_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
+    accel_build_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
    reproduce_parser = subcommands.add_parser("reproduce")
    reproduce_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
    args = parser.parse_args()
@@ -582,6 +730,10 @@ def main() -> int:
            smoke(args.binary)
        elif args.command == "ctest":
            ctest_lane(args.build_dir)
+        elif args.command == "accelerator-status":
+            print(json.dumps(accelerator_status(args.name), indent=2, sort_keys=True))
+        elif args.command == "accelerator-build":
+            accelerator_build(args.source_dir, args.name, args.build_dir)
        else:
            reproduce(args.workspace)
    except DependencyError as error:
--- a/scripts/native_accelerator_matrix.py
+++ b/scripts/native_accelerator_matrix.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""DGR-030: native CI/build matrix over the CPU default plus accelerator lanes.
+
+Runs the exact deterministic CPU lane DGR-029 locked (unchanged), then probes
+each accelerator preset (CUDA, ROCm, Vulkan, Metal) from `UPSTREAM_LOCK.json`
+and compiles the ones whose SDK is present on this machine into their own
+out-of-tree build directory.
+
+A lane whose SDK is absent is reported as `skipped` with the exact probe
+reason, never treated as a false pass. A lane that compiles is reported as
+`built`, carrying exact compiler/SDK/upstream-pin/patch-stack/build-option
+evidence — never as a certified capability. This script never runs an
+accelerator binary and never certifies a backend/model/recipe: real-hardware
+certification is separate future work (DGR-041/053/067).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import pathlib
+import sys
+from typing import Any
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "scripts"))
+import llama_cpp_dependency as dep  # noqa: E402
+
+
+def _cpu_lane(source: pathlib.Path, workspace: pathlib.Path) -> dict[str, Any]:
+    build_dir = workspace.resolve() / "build"
+    if build_dir.exists():
+        return {
+            "lane": "cpu",
+            "status": "skipped",
+            "reason": f"build directory already exists; remove for a clean rebuild: {build_dir}",
+        }
+    binary = dep.build(source, build_dir)
+    dep.smoke(binary)
+    dep.ctest_lane(build_dir)
+    metadata = json.loads((build_dir / "meshnet-build-metadata.json").read_text())
+    return {"lane": "cpu", "status": "built", "build_dir": str(build_dir), "metadata": metadata}
+
+
+def _accelerator_lane(source: pathlib.Path, workspace: pathlib.Path, name: str, lock: dict[str, Any]) -> dict[str, Any]:
+    status = dep.accelerator_status(name, lock)
+    if not status["available"]:
+        return {"lane": name, "status": "skipped", "reason": status["reason"]}
+    build_dir = workspace.resolve() / f"build-{name}"
+    if build_dir.exists():
+        return {
+            "lane": name,
+            "status": "skipped",
+            "reason": f"build directory already exists; remove for a clean rebuild: {build_dir}",
+        }
+    dep.accelerator_build(source, name, build_dir)
+    metadata = json.loads((build_dir / "meshnet-build-metadata.json").read_text())
+    return {"lane": name, "status": "built", "build_dir": str(build_dir), "metadata": metadata}
+
+
+def run_matrix(workspace: pathlib.Path) -> dict[str, Any]:
+    """Fetch/apply once, run every lane, then always reverse the checkout."""
+    source = dep.fetch(workspace)
+    dep.apply(source)
+    lanes: list[dict[str, Any]] = []
+    try:
+        lock = dep._load_lock()
+        try:
+            lanes.append(_cpu_lane(source, workspace))
+        except dep.DependencyError as error:
+            lanes.append({"lane": "cpu", "status": "failed", "reason": str(error)})
+        for name in lock.get("accelerator_presets", {}):
+            try:
+                lanes.append(_accelerator_lane(source, workspace, name, lock))
+            except dep.DependencyError as error:
+                lanes.append({"lane": name, "status": "failed", "reason": str(error)})
+    finally:
+        dep.reverse(source)
+    failed_lanes = [lane["lane"] for lane in lanes if lane["status"] == "failed"]
+    return {
+        "lanes": lanes,
+        "hardware_certified": False,
+        "note": (
+            "A `built` lane means it compiled with the exact recorded compiler/SDK/"
+            "upstream-pin/patch-stack/build-option evidence — it never means an "
+            "accelerator device was exercised. Every backend/model/recipe lane "
+            "stays registered-dark until a separate real-hardware certification "
+            "record exists."
+        ),
+        "failed_lanes": failed_lanes,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
+    parser.add_argument("--out", type=pathlib.Path, default=None, help="also write the JSON report here")
+    args = parser.parse_args()
+    try:
+        report = run_matrix(args.workspace)
+    except dep.DependencyError as error:
+        print(f"DGR-030 dependency error: {error}", file=sys.stderr)
+        return 2
+    text = json.dumps(report, indent=2, sort_keys=True)
+    print(text)
+    if args.out:
+        args.out.write_text(text + "\n")
+    return 1 if report["failed_lanes"] else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())