popov/neuron-tai
1
0
Fork 0
Code Issues 42 Pull Requests Actions Packages Projects Releases Wiki Activity

story: DGR-030 Add accelerator build presets and native CI matrix

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-23 10:51:08 +03:00
parent 254297660a
commit fd742d35c0
13 changed files with 2576 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
scripts/llama_cpp_dependency.py
View File

@@ -109,9 +109,41 @@ def _load_lock() -> dict[str, Any]:
"workspace": "build/llama.cpp",
}:
raise DependencyError("retrieval must use the locked detached-commit build workspace")
_verify_accelerator_presets(lock)
return lock
def _verify_accelerator_presets(lock: dict[str, Any]) -> None:
"""Each preset must isolate one backend that the CPU default leaves OFF.
This is what keeps DGR-030's presets from ever being able to change the
deterministic CPU default recorded in ``build.configure_flags``: a preset
can only exist for a flag this lock already pins OFF, and
``accelerator_configure_flags`` only ever returns a fresh list, never
mutates ``build.configure_flags`` in place.
"""
presets = lock.get("accelerator_presets", {})
if not isinstance(presets, dict):
raise DependencyError("accelerator_presets must be a JSON object")
if not presets:
return
base_flags = dict(flag[len("-D"):].split("=", 1) for flag in lock["build"]["configure_flags"])
for name, preset in presets.items():
if not isinstance(preset, dict):
raise DependencyError(f"accelerator_presets.{name} must be a JSON object")
backend_flag = preset.get("backend_flag")
if not isinstance(backend_flag, str) or not backend_flag:
raise DependencyError(f"accelerator_presets.{name} is missing backend_flag")
if base_flags.get(backend_flag) != "OFF":
raise DependencyError(
f"accelerator_presets.{name} backend flag {backend_flag} must be OFF in "
"the deterministic CPU default build.configure_flags"
)
probe = preset.get("sdk_probe")
if not isinstance(probe, dict) or not isinstance(probe.get("binary"), str) or not probe["binary"]:
raise DependencyError(f"accelerator_presets.{name} is missing an sdk_probe.binary")
def _patches(lock: dict[str, Any]) -> list[pathlib.Path]:
series = [line for line in (PATCH_DIR / "series").read_text().splitlines() if line]
if series != lock["patch_series"] or series != sorted(series) or not series:
@@ -507,6 +539,116 @@ def ctest_lane(build_dir: pathlib.Path) -> None:
print(_run(_ctest(), "--test-dir", str(build_dir), "-R", regex, "--output-on-failure"))
def _sdk_probe(probe: dict[str, Any]) -> str | None:
"""Resolve one accelerator lane's SDK binary, or None if it is unavailable."""
platform_only = probe.get("platform_only")
if platform_only and sys.platform != platform_only:
return None
env_var = probe.get("env_var")
if env_var:
override = os.environ.get(env_var)
if override:
return override
return shutil.which(probe["binary"])
def accelerator_status(name: str, lock: dict[str, Any] | None = None) -> dict[str, Any]:
"""Report whether lane `name`'s SDK is present, never raising for absence.
This is the single source of truth for DGR-030's "unavailable/skipped, not
false success" contract: absence is reported as data, not swallowed and
not escalated into a build attempt.
"""
lock = lock if lock is not None else _load_lock()
presets = lock.get("accelerator_presets", {})
if name not in presets:
raise DependencyError(f"unknown accelerator lane: {name}")
probe = presets[name]["sdk_probe"]
resolved = _sdk_probe(probe)
if resolved is None:
platform_only = probe.get("platform_only")
if platform_only and sys.platform != platform_only:
reason = f"platform {sys.platform!r} is not {platform_only!r}"
else:
reason = f"{probe['binary']} is unavailable on PATH"
return {"lane": name, "available": False, "reason": reason}
return {"lane": name, "available": True, "sdk_binary": resolved}
def accelerator_configure_flags(lock: dict[str, Any], name: str) -> list[str]:
"""The CPU default's configure flags with exactly one backend flag flipped ON.
Returns a new list; `lock["build"]["configure_flags"]` (the deterministic
CPU default DGR-029 locked) is never mutated.
"""
presets = lock.get("accelerator_presets", {})
if name not in presets:
raise DependencyError(f"unknown accelerator lane: {name}")
backend_flag = presets[name]["backend_flag"]
target = f"-D{backend_flag}="
flags: list[str] = []
replaced = False
for flag in lock["build"]["configure_flags"]:
if flag.startswith(target):
flags.append(f"-D{backend_flag}=ON")
replaced = True
else:
flags.append(flag)
if not replaced:
raise DependencyError(f"accelerator lane {name} backend flag {backend_flag} is not a locked base flag")
return flags
def accelerator_build(source: pathlib.Path, name: str, build_dir: pathlib.Path) -> pathlib.Path:
"""Compile lane `name` into its own out-of-tree directory. Compile-only.
This never runs `smoke`/`ctest_lane`: exercising a binary linked against an
accelerator backend would touch real hardware, and DGR-030 keeps every
backend/model/recipe lane registered-dark (compiled, never certified)
until a separate real-hardware certification record exists.
"""
lock = _load_lock()
_patches(lock)
_verify_source(source, lock, require_clean=False)
_verify_patched_source(source, lock)
expected_marker = source / "cmake/meshnet-patch-stack.cmake"
if not expected_marker.is_file():
raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent")
if build_dir.exists():
raise DependencyError(f"accelerator build directory already exists; use a clean build dir: {build_dir}")
status = accelerator_status(name, lock)
if not status["available"]:
raise DependencyError(f"accelerator lane {name} SDK is unavailable: {status['reason']}")
flags = accelerator_configure_flags(lock, name)
cmake = _cmake()
_run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags)
for target in lock["build"]["native_targets"]:
_run(cmake, "--build", str(build_dir), "--target", target, "-j2")
metadata = {
"lane": name,
"backend_flag": lock["accelerator_presets"][name]["backend_flag"],
"commit": lock["commit"],
"commit_tree": lock["commit_tree"],
"patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)},
"configure_flags": flags,
"cmake": _run(cmake, "--version").splitlines()[0],
"cxx": _run("c++", "--version").splitlines()[0],
"sdk_binary": status["sdk_binary"],
"model_downloads": False,
"hardware_execution": False,
"hardware_certified": False,
"semantic_certification": False,
"note": (
"compiled only; no accelerator device was exercised or driven. "
"Backend/model/recipe capability remains registered-dark until a "
"separate real-hardware certification record exists (see "
"DGR-041/053/067)."
),
}
(build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
return build_dir
def verify(workspace: pathlib.Path) -> None:
"""Apply, verify, reverse, and leave the exact cached pin pristine."""
source = fetch(workspace)
@@ -562,6 +704,12 @@ def main() -> int:
smoke_parser.add_argument("--binary", type=pathlib.Path, required=True)
ctest_parser = subcommands.add_parser("ctest")
ctest_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
accel_status_parser = subcommands.add_parser("accelerator-status")
accel_status_parser.add_argument("--name", required=True)
accel_build_parser = subcommands.add_parser("accelerator-build")
accel_build_parser.add_argument("--name", required=True)
accel_build_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
accel_build_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
reproduce_parser = subcommands.add_parser("reproduce")
reproduce_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
args = parser.parse_args()
@@ -582,6 +730,10 @@ def main() -> int:
smoke(args.binary)
elif args.command == "ctest":
ctest_lane(args.build_dir)
elif args.command == "accelerator-status":
print(json.dumps(accelerator_status(args.name), indent=2, sort_keys=True))
elif args.command == "accelerator-build":
accelerator_build(args.source_dir, args.name, args.build_dir)
else:
reproduce(args.workspace)
except DependencyError as error: