#!/usr/bin/env python3 """Materialize, verify, build, and smoke-test DGR-004's llama.cpp pin. This tool deliberately owns only a source dependency boundary. It never downloads a model, invokes inference, or interprets generated text. """ from __future__ import annotations import argparse import hashlib import json import os import pathlib import shutil import subprocess import sys from typing import Any ROOT = pathlib.Path(__file__).resolve().parents[1] LLAMA_DIR = ROOT / "packages/node/native/llama" LOCK_PATH = LLAMA_DIR / "UPSTREAM_LOCK.json" PATCH_DIR = LLAMA_DIR / "patches" def _cmake() -> str: """Use an explicit override, PATH, or the active Python environment.""" configured = os.environ.get("CMAKE") if configured: return configured on_path = shutil.which("cmake") if on_path: return on_path sibling = pathlib.Path(sys.executable).parent / "cmake" if sibling.is_file(): return str(sibling) raise DependencyError("cmake is unavailable; set CMAKE or activate the project toolchain") class DependencyError(RuntimeError): """A fail-closed reproducibility or source-integrity failure.""" def _run(*args: str, cwd: pathlib.Path | None = None) -> str: try: completed = subprocess.run( args, cwd=cwd, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) except FileNotFoundError as error: raise DependencyError(f"required executable is unavailable: {args[0]}") from error except subprocess.CalledProcessError as error: detail = (error.stderr or error.stdout).strip() raise DependencyError(f"command failed: {' '.join(args)}\n{detail}") from error return completed.stdout.strip() def _load_lock() -> dict[str, Any]: try: lock = json.loads(LOCK_PATH.read_text()) except (OSError, json.JSONDecodeError) as error: raise DependencyError(f"invalid upstream lock: {LOCK_PATH}: {error}") from error required = { "upstream", "commit", "commit_tree", "patched_tree", "patch_series", "required_upstream_blobs", "patched_paths", "build", } missing = sorted(required - lock.keys()) if missing: raise DependencyError(f"upstream lock is missing fields: {', '.join(missing)}") commit_file = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip() if lock["commit"] != commit_file or len(commit_file) != 40: raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on a full commit") return lock def _patches(lock: dict[str, Any]) -> list[pathlib.Path]: series = [line for line in (PATCH_DIR / "series").read_text().splitlines() if line] if series != lock["patch_series"] or series != sorted(series) or not series: raise DependencyError("patches/series is empty, unordered, or disagrees with UPSTREAM_LOCK.json") sums: dict[str, str] = {} for line in (PATCH_DIR / "SHA256SUMS").read_text().splitlines(): if line and not line.startswith("#"): digest, name = line.split(maxsplit=1) sums[name] = digest patches: list[pathlib.Path] = [] for name in series: patch = PATCH_DIR / name if not patch.is_file(): raise DependencyError(f"patch listed in series is missing: {name}") actual = hashlib.sha256(patch.read_bytes()).hexdigest() if sums.get(name) != actual: raise DependencyError(f"patch digest mismatch for {name}: expected {sums.get(name)}, got {actual}") patches.append(patch) return patches def _git(source: pathlib.Path, *args: str) -> str: return _run("git", "-C", str(source), *args) def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean: bool) -> None: if not (source / ".git").exists(): raise DependencyError(f"not a materialized git checkout: {source}") if _git(source, "rev-parse", "HEAD") != lock["commit"]: raise DependencyError("upstream drift: checkout HEAD does not equal the locked commit") if _git(source, "rev-parse", "HEAD^{tree}") != lock["commit_tree"]: raise DependencyError("upstream drift: checkout tree does not equal the locked tree") if require_clean and _git(source, "status", "--porcelain"): raise DependencyError("local edits detected in materialized llama.cpp checkout") for relative, expected in lock["required_upstream_blobs"].items(): actual = _git(source, "rev-parse", f"HEAD:{relative}") if actual != expected: raise DependencyError( f"upstream ABI/context drift for {relative}: expected {expected}, got {actual}" ) if not (source / "LICENSE").is_file(): raise DependencyError("upstream LICENSE is missing; refusing to drop required attribution") def materialize(source: pathlib.Path, repository: str) -> None: lock = _load_lock() _patches(lock) if source.exists(): raise DependencyError(f"destination already exists; refusing to reuse possibly edited source: {source}") source.parent.mkdir(parents=True, exist_ok=True) _run("git", "clone", "--no-checkout", repository, str(source)) _git(source, "checkout", "--detach", lock["commit"]) _verify_source(source, lock, require_clean=True) def apply(source: pathlib.Path) -> None: lock = _load_lock() patches = _patches(lock) _verify_source(source, lock, require_clean=True) for patch in patches: _git(source, "apply", "--check", str(patch)) _git(source, "apply", "--index", str(patch)) _verify_patched_source(source, lock) def _verify_patched_source(source: pathlib.Path, lock: dict[str, Any]) -> None: if _git(source, "diff", "--quiet"): raise DependencyError("local unstaged edits detected after applying patch stack") changed_paths = _git(source, "diff", "--cached", "--name-only").splitlines() if changed_paths != lock["patched_paths"]: raise DependencyError(f"patched paths drifted: expected {lock['patched_paths']}, got {changed_paths}") if _git(source, "write-tree") != lock["patched_tree"]: raise DependencyError("patched source tree differs from the locked patch stack") if _git(source, "diff", "--name-only"): raise DependencyError("local unstaged edits detected after applying patch stack") untracked = _git(source, "ls-files", "--others", "--exclude-standard").splitlines() if untracked: raise DependencyError(f"untracked files detected after applying patch stack: {untracked}") def build(source: pathlib.Path, build_dir: pathlib.Path) -> pathlib.Path: lock = _load_lock() _patches(lock) _verify_source(source, lock, require_clean=False) _verify_patched_source(source, lock) expected_marker = source / "cmake/meshnet-patch-stack.cmake" if not expected_marker.is_file(): raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent") if build_dir.exists(): raise DependencyError(f"build directory already exists; use reproduce for a clean rebuild: {build_dir}") flags = lock["build"]["configure_flags"] cmake = _cmake() _run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags) for target in lock["build"]["native_targets"]: _run(cmake, "--build", str(build_dir), "--target", target, "-j2") metadata = { "commit": lock["commit"], "commit_tree": lock["commit_tree"], "patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)}, "configure_flags": flags, "cmake": _run(cmake, "--version").splitlines()[0], "cxx": _run("c++", "--version").splitlines()[0], "model_downloads": False, "semantic_certification": False, } (build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") return build_dir / lock["build"]["smoke_binary"] def smoke(binary: pathlib.Path) -> None: if not binary.is_file(): raise DependencyError(f"native smoke binary is missing: {binary}") smoke_config = _load_lock()["build"] output = _run(str(binary), *smoke_config["smoke_args"]) expected = smoke_config["smoke_output_token"] if expected not in output.lower(): raise DependencyError(f"native smoke output did not contain {expected!r}: {output}") print(output) def reproduce(work_dir: pathlib.Path, repository: str) -> None: resolved = work_dir.resolve() build_root = (ROOT / "build").resolve() if build_root not in resolved.parents: raise DependencyError(f"--work-dir must be below {build_root}: {resolved}") if resolved.exists(): raise DependencyError( f"work directory already exists; refusing to erase possible local edits: {resolved}" ) source = resolved / "source" build_dir = resolved / "build" materialize(source, repository) apply(source) smoke(build(source, build_dir)) def inspect() -> None: lock = _load_lock() patches = _patches(lock) print(json.dumps({ "commit": lock["commit"], "patch_count": len(patches), "patches": [patch.name for patch in patches], "model_downloads": False, "semantic_certification": False, "glm_stock_limitations": lock["stock_glm_limitations"], }, indent=2, sort_keys=True)) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) subcommands = parser.add_subparsers(dest="command", required=True) subcommands.add_parser("inspect") materialize_parser = subcommands.add_parser("materialize") materialize_parser.add_argument("--source-dir", type=pathlib.Path, required=True) materialize_parser.add_argument("--source-repository", default=_load_lock()["upstream"]) apply_parser = subcommands.add_parser("apply") apply_parser.add_argument("--source-dir", type=pathlib.Path, required=True) build_parser = subcommands.add_parser("build") build_parser.add_argument("--source-dir", type=pathlib.Path, required=True) build_parser.add_argument("--build-dir", type=pathlib.Path, required=True) smoke_parser = subcommands.add_parser("smoke") smoke_parser.add_argument("--binary", type=pathlib.Path, required=True) reproduce_parser = subcommands.add_parser("reproduce") reproduce_parser.add_argument("--work-dir", type=pathlib.Path, default=ROOT / "build/dgr-004-smoke") reproduce_parser.add_argument("--source-repository", default=_load_lock()["upstream"]) args = parser.parse_args() try: if args.command == "inspect": inspect() elif args.command == "materialize": materialize(args.source_dir, args.source_repository) elif args.command == "apply": apply(args.source_dir) elif args.command == "build": build(args.source_dir, args.build_dir) elif args.command == "smoke": smoke(args.binary) else: reproduce(args.work_dir, args.source_repository) except DependencyError as error: print(f"DGR-004 dependency error: {error}", file=sys.stderr) return 2 return 0 if __name__ == "__main__": raise SystemExit(main())