#!/usr/bin/env python3 """Fetch, verify, build, and smoke-test DGR-027's exact llama.cpp pin. This tool deliberately owns only a source dependency boundary. It never downloads a model, invokes inference, or interprets generated text. DGR-028 adds the numbered patch-stack contract: the ordered series applies, checks, and reverses deterministically against the exact manifest pin, each patch's recorded upstream file/API assumptions are enforced before it is attempted, the first incompatible patch is named on failure, and the stack is refused if it carries license/attribution damage or Meshnet control-plane (routing, billing, relay, authentication) code. """ from __future__ import annotations import argparse import hashlib import json import os import pathlib import re import shutil import subprocess import sys from typing import Any ROOT = pathlib.Path(__file__).resolve().parents[1] LLAMA_DIR = ROOT / "packages/node/native/llama" LOCK_PATH = LLAMA_DIR / "UPSTREAM_LOCK.json" PATCH_DIR = LLAMA_DIR / "patches" def _cmake() -> str: """Use an explicit override, PATH, or the active Python environment.""" configured = os.environ.get("CMAKE") if configured: return configured on_path = shutil.which("cmake") if on_path: return on_path sibling = pathlib.Path(sys.executable).parent / "cmake" if sibling.is_file(): return str(sibling) raise DependencyError("cmake is unavailable; set CMAKE or activate the project toolchain") class DependencyError(RuntimeError): """A fail-closed reproducibility or source-integrity failure.""" def _run(*args: str, cwd: pathlib.Path | None = None) -> str: try: completed = subprocess.run( args, cwd=cwd, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) except FileNotFoundError as error: raise DependencyError(f"required executable is unavailable: {args[0]}") from error except subprocess.CalledProcessError as error: detail = (error.stderr or error.stdout).strip() raise DependencyError(f"command failed: {' '.join(args)}\n{detail}") from error return completed.stdout.strip() def _git(source: pathlib.Path, *args: str) -> str: """Run Git against the materialized upstream checkout.""" return _run("git", "-C", str(source), *args) def _load_lock() -> dict[str, Any]: try: lock = json.loads(LOCK_PATH.read_text()) except (OSError, json.JSONDecodeError) as error: raise DependencyError(f"invalid upstream lock: {LOCK_PATH}: {error}") from error required = { "upstream", "commit", "commit_tree", "patched_tree", "patch_series", "required_upstream_blobs", "patched_paths", "build", "upstream_license", "expected_source", "retrieval", } missing = sorted(required - lock.keys()) if missing: raise DependencyError(f"upstream lock is missing fields: {', '.join(missing)}") commit_file = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip() object_ids = [commit_file, lock["commit"], lock["commit_tree"], lock["patched_tree"]] if lock["commit"] != commit_file or not all( isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40}", value) for value in object_ids ): raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on full hexadecimal object IDs") if lock["expected_source"] != {"git_tree": lock["commit_tree"]}: raise DependencyError("expected_source must record the locked git tree") retrieval = lock["retrieval"] if retrieval != { "method": "git-clone-detached-commit", "workspace": "build/llama.cpp", }: raise DependencyError("retrieval must use the locked detached-commit build workspace") return lock def _patches(lock: dict[str, Any]) -> list[pathlib.Path]: series = [line for line in (PATCH_DIR / "series").read_text().splitlines() if line] if series != lock["patch_series"] or series != sorted(series) or not series: raise DependencyError("patches/series is empty, unordered, or disagrees with UPSTREAM_LOCK.json") sums: dict[str, str] = {} for line in (PATCH_DIR / "SHA256SUMS").read_text().splitlines(): if line and not line.startswith("#"): digest, name = line.split(maxsplit=1) sums[name] = digest patches: list[pathlib.Path] = [] for name in series: patch = PATCH_DIR / name if not patch.is_file(): raise DependencyError(f"patch listed in series is missing: {name}") actual = hashlib.sha256(patch.read_bytes()).hexdigest() if sums.get(name) != actual: raise DependencyError(f"patch digest mismatch for {name}: expected {sums.get(name)}, got {actual}") patches.append(patch) return patches def _parse_patch_files(patch: pathlib.Path) -> dict[str, tuple[str | None, str | None]]: """Parse one patch into {path: (before-short, after-short)}. Short object IDs come from the patch's ``index`` lines; an all-zero side means the file is created (``before is None``) or deleted (``after is None``). Patch order is preserved. """ files: dict[str, tuple[str | None, str | None]] = {} current: str | None = None for line in patch.read_text().splitlines(): header = re.match(r"^diff --git a/(.+) b/(.+)$", line) if header: if header.group(1) != header.group(2): raise DependencyError(f"{patch.name}: rename/copy diffs are unsupported: {line}") current = header.group(1) files[current] = (None, None) continue index = re.match(r"^index ([0-9a-f]{7,40})\.\.([0-9a-f]{7,40})(?:\s|$)", line) if index and current is not None: before, after = index.group(1), index.group(2) files[current] = ( None if set(before) == {"0"} else before, None if set(after) == {"0"} else after, ) if not files: raise DependencyError(f"{patch.name}: no file diffs found") return files _ASSUMPTIONS_DEFAULT = "patches/UPSTREAM-ASSUMPTIONS.json" def _assumptions(lock: dict[str, Any], patches: list[pathlib.Path]) -> dict[str, Any]: """Load and validate the recorded upstream file/API assumptions. The record must cover exactly the ordered series, each recorded file must match the patch's parsed diff headers, and each recorded full object ID must agree with the patch's abbreviated ``index`` IDs. A stale or edited record is a fail-closed error, never a warning. """ configured = lock.get("patch_assumptions", _ASSUMPTIONS_DEFAULT) if not isinstance(configured, str) or not configured: raise DependencyError("patch_assumptions must name a manifest-relative JSON path") relative = pathlib.Path(configured) if relative.is_absolute() or ".." in relative.parts: raise DependencyError("patch_assumptions must stay inside the repository manifest tree") candidate = (LLAMA_DIR / relative).absolute() try: candidate.relative_to(LLAMA_DIR.absolute()) except ValueError as error: raise DependencyError("patch_assumptions must live under packages/node/native/llama") from error if not candidate.is_file(): raise DependencyError(f"recorded upstream assumptions are missing: {candidate}") try: doc = json.loads(candidate.read_text()) except (OSError, json.JSONDecodeError) as error: raise DependencyError(f"invalid upstream assumptions: {candidate}: {error}") from error if not isinstance(doc, dict) or doc.get("schema_version") != 1: raise DependencyError("upstream assumptions must declare schema_version 1") if doc.get("upstream_commit") != lock["commit"]: raise DependencyError("upstream assumptions disagree with the locked commit") recorded = doc.get("patches") if not isinstance(recorded, dict): raise DependencyError("upstream assumptions must record a patches object") names = [patch.name for patch in patches] if list(recorded.keys()) != names: raise DependencyError( f"upstream assumptions do not cover exactly the ordered series: " f"expected {names}, got {list(recorded.keys())}" ) for patch in patches: entry = recorded[patch.name] if not isinstance(entry.get("concern"), str) or not entry["concern"]: raise DependencyError(f"{patch.name}: assumptions are missing the scoped concern") symbols = entry.get("api_assumptions") if not isinstance(symbols, list) or not symbols or not all(isinstance(s, str) and s for s in symbols): raise DependencyError(f"{patch.name}: assumptions must record upstream file/API assumptions") files = entry.get("files") if not isinstance(files, dict): raise DependencyError(f"{patch.name}: assumptions must record a files object") parsed = _parse_patch_files(patch) if list(files.keys()) != list(parsed.keys()): raise DependencyError( f"{patch.name}: recorded files {list(files.keys())} disagree with the patch bytes {list(parsed.keys())}" ) for path, (before_short, after_short) in parsed.items(): blobs = files[path] if not isinstance(blobs, dict): raise DependencyError(f"{patch.name}: recorded blobs for {path} must be an object") for side, short in (("before", before_short), ("after", after_short)): value = blobs.get(side) if short is None: if value is not None: raise DependencyError(f"{patch.name}: {path} {side} must be null for a created/deleted file") elif not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{40}", value) or not value.startswith(short): raise DependencyError( f"{patch.name}: recorded {side} blob for {path} does not match the patch index ID {short}" ) return doc # Control-plane vocabulary that must never enter the upstream patch stack: # Meshnet routing, billing, relay, authentication, and transport semantics are # backend-agnostic and live outside the llama.cpp fork boundary (ADR-0024). _CONTROL_PLANE_TERMS = re.compile( r"\b(tracker|routing|route session|grpc|billing|wallet|relay|telemetry|" r"auth|authentication|load balanc\w*)\b", re.IGNORECASE, ) _LICENSE_PATH = re.compile(r"(^|/)(license|copying|notice)(\..*)?$", re.IGNORECASE) _LICENSE_TEXT = re.compile( r"copyright|permission is hereby granted|mit license|apache license|gnu general public", re.IGNORECASE, ) def _verify_patch_stack_boundaries(patches: list[pathlib.Path]) -> None: """Refuse license/attribution damage or control-plane code in the stack.""" for patch in patches: for line in patch.read_text().splitlines(): header = re.match(r"^diff --git a/(.+) b/(.+)$", line) if header and (_LICENSE_PATH.search(header.group(1)) or _LICENSE_PATH.search(header.group(2))): raise DependencyError(f"{patch.name}: license/attribution files may not be patched: {line}") if line.startswith("-") and not line.startswith("---") and _LICENSE_TEXT.search(line[1:]): raise DependencyError(f"{patch.name}: removing license or attribution text is refused: {line}") body = patch.read_text() match = _CONTROL_PLANE_TERMS.search(body) if match: raise DependencyError( f"{patch.name}: Meshnet control-plane term {match.group(0)!r} must not enter the patch stack" ) def _index_blob(source: pathlib.Path, path: str) -> str | None: """Return the staged blob object ID for path, or None when absent.""" records = _git(source, "ls-files", "-s", "-z", "--", path).split("\0") entries = [record for record in records if record] if not entries: return None if len(entries) != 1: raise DependencyError(f"unmerged index entry blocks patch verification: {path}") metadata, _ = entries[0].split("\t", 1) mode, blob, stage = metadata.split() if stage != "0": raise DependencyError(f"unmerged index entry blocks patch verification: {path}") return blob def _check_assumption_blobs( source: pathlib.Path, patch: pathlib.Path, files: dict[str, Any], side: str, ) -> None: """Fail on the first recorded pre-/post-image blob the index disagrees with.""" for path, blobs in files.items(): expected = blobs[side] actual = _index_blob(source, path) if actual != expected: raise DependencyError( f"first incompatible patch: {patch.name}: recorded {side} blob for {path} is " f"{expected}, found {actual}" ) def _verify_tracked_content(source: pathlib.Path, lock: dict[str, Any]) -> None: if _git(source, "write-tree") != lock["commit_tree"]: raise DependencyError("materialized checkout index differs from the locked tree") filemode_trusted = _git(source, "config", "--bool", "core.filemode") == "true" records = _git(source, "ls-files", "-s", "-z").split("\0") paths: list[str] = [] expected: list[str] = [] for record in records: if not record: continue metadata, path = record.split("\t", 1) mode, blob, stage = metadata.split() if stage != "0" or mode not in {"100644", "100755", "120000"}: raise DependencyError(f"unsupported tracked entry in materialized checkout: {record!r}") if "\n" in path: raise DependencyError(f"newline-bearing tracked path is unsupported: {path!r}") candidate = source / path cursor = source for part in pathlib.Path(path).parts[:-1]: cursor = cursor / part if cursor.is_symlink(): raise DependencyError(f"tracked path traverses a symlink: {path!r}") if mode == "120000": if not candidate.is_symlink(): raise DependencyError(f"tracked symlink type differs from the locked tree: {path!r}") else: if candidate.is_symlink() or not candidate.is_file(): raise DependencyError(f"tracked file type differs from the locked tree: {path!r}") executable = bool(candidate.stat().st_mode & 0o111) if filemode_trusted and executable != (mode == "100755"): raise DependencyError(f"tracked executable mode differs from the locked tree: {path!r}") paths.append(path) expected.append(blob) try: completed = subprocess.run( ["git", "hash-object", "--stdin-paths"], cwd=source, input="".join(f"{path}\n" for path in paths), check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) except subprocess.CalledProcessError as error: raise DependencyError(f"unable to hash tracked source content: {error.stderr.strip()}") from error actual = completed.stdout.splitlines() if len(actual) != len(expected) or actual != expected: raise DependencyError("tracked source content differs from the locked tree") def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean: bool) -> None: if not (source / ".git").exists(): raise DependencyError(f"not a materialized git checkout: {source}") if _git(source, "rev-parse", "HEAD") != lock["commit"]: raise DependencyError("upstream drift: checkout HEAD does not equal the locked commit") if _git(source, "rev-parse", "HEAD^{tree}") != lock["commit_tree"]: raise DependencyError("upstream drift: checkout tree does not equal the locked tree") if _git(source, "rev-parse", "--abbrev-ref", "HEAD") != "HEAD": raise DependencyError("materialized llama.cpp checkout must have a detached HEAD") if require_clean: _verify_tracked_content(source, lock) tracked = _git(source, "status", "--porcelain", "--untracked-files=no") untracked = _git(source, "ls-files", "--others", "--exclude-standard") ignored = _git(source, "ls-files", "--others", "--ignored", "--exclude-standard") if tracked or untracked or ignored: raise DependencyError("local edits or unmanifested files detected in materialized llama.cpp checkout") for relative, expected in lock["required_upstream_blobs"].items(): actual = _git(source, "rev-parse", f"HEAD:{relative}") if actual != expected: raise DependencyError( f"upstream ABI/context drift for {relative}: expected {expected}, got {actual}" ) if not (source / "LICENSE").is_file(): raise DependencyError("upstream LICENSE is missing; refusing to drop required attribution") def _workspace_source(workspace: pathlib.Path, lock: dict[str, Any]) -> pathlib.Path: relative = pathlib.Path(lock["retrieval"]["workspace"]) expected = (ROOT / relative).absolute() supplied = workspace.absolute() if supplied != expected: raise DependencyError(f"--workspace must equal the locked ignored build root: {expected}") cursor = ROOT for part in relative.parts: cursor = cursor / part if cursor.is_symlink(): raise DependencyError(f"locked build workspace may not traverse a symlink: {cursor}") resolved = workspace.resolve() try: resolved.relative_to(ROOT.resolve()) except ValueError as error: raise DependencyError("locked build workspace escapes the repository root") from error source = resolved / "source" if source.is_symlink(): raise DependencyError(f"locked source checkout may not be a symlink: {source}") return source def fetch(workspace: pathlib.Path) -> pathlib.Path: """Fetch once, or verify an exact clean cached checkout for offline reuse.""" lock = _load_lock() _patches(lock) source = _workspace_source(workspace, lock) if source.exists(): _verify_source(source, lock, require_clean=True) print(f"reused verified offline cache: {source}") return source source.parent.mkdir(parents=True, exist_ok=True) _run("git", "clone", "--no-checkout", lock["upstream"], str(source)) _git(source, "checkout", "--detach", lock["commit"]) _verify_source(source, lock, require_clean=True) print(f"fetched and verified exact source: {source}") return source def apply(source: pathlib.Path) -> None: lock = _load_lock() patches = _patches(lock) assumptions = _assumptions(lock, patches) _verify_patch_stack_boundaries(patches) _verify_source(source, lock, require_clean=True) for patch in patches: files = assumptions["patches"][patch.name]["files"] _check_assumption_blobs(source, patch, files, "before") _git(source, "apply", "--check", str(patch)) _git(source, "apply", "--index", str(patch)) _check_assumption_blobs(source, patch, files, "after") _verify_patched_source(source, lock) def reverse(source: pathlib.Path) -> None: """Reverse the complete verified stack and recover the exact clean pin.""" lock = _load_lock() patches = _patches(lock) assumptions = _assumptions(lock, patches) _verify_patch_stack_boundaries(patches) _verify_source(source, lock, require_clean=False) _verify_patched_source(source, lock) for patch in reversed(patches): files = assumptions["patches"][patch.name]["files"] _check_assumption_blobs(source, patch, files, "after") _git(source, "apply", "--reverse", "--check", str(patch)) _git(source, "apply", "--reverse", "--index", str(patch)) _check_assumption_blobs(source, patch, files, "before") _verify_source(source, lock, require_clean=True) def _verify_patched_source(source: pathlib.Path, lock: dict[str, Any]) -> None: changed_paths = _git(source, "diff", "--cached", "--name-only").splitlines() if changed_paths != lock["patched_paths"]: raise DependencyError(f"patched paths drifted: expected {lock['patched_paths']}, got {changed_paths}") if _git(source, "write-tree") != lock["patched_tree"]: raise DependencyError("patched source tree differs from the locked patch stack") if _git(source, "diff", "--name-only"): raise DependencyError("local unstaged edits detected after applying patch stack") untracked = _git(source, "ls-files", "--others", "--exclude-standard").splitlines() if untracked: raise DependencyError(f"untracked files detected after applying patch stack: {untracked}") def build(source: pathlib.Path, build_dir: pathlib.Path) -> pathlib.Path: lock = _load_lock() _patches(lock) _verify_source(source, lock, require_clean=False) _verify_patched_source(source, lock) expected_marker = source / "cmake/meshnet-patch-stack.cmake" if not expected_marker.is_file(): raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent") if build_dir.exists(): raise DependencyError(f"build directory already exists; use reproduce for a clean rebuild: {build_dir}") flags = lock["build"]["configure_flags"] cmake = _cmake() _run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags) for target in lock["build"]["native_targets"]: _run(cmake, "--build", str(build_dir), "--target", target, "-j2") metadata = { "commit": lock["commit"], "commit_tree": lock["commit_tree"], "patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)}, "configure_flags": flags, "cmake": _run(cmake, "--version").splitlines()[0], "cxx": _run("c++", "--version").splitlines()[0], "model_downloads": False, "semantic_certification": False, } (build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") return build_dir / lock["build"]["smoke_binary"] def smoke(binary: pathlib.Path) -> None: if not binary.is_file(): raise DependencyError(f"native smoke binary is missing: {binary}") smoke_config = _load_lock()["build"] output = _run(str(binary), *smoke_config["smoke_args"]) expected = smoke_config["smoke_output_token"] if expected not in output.lower(): raise DependencyError(f"native smoke output did not contain {expected!r}: {output}") print(output) def verify(workspace: pathlib.Path) -> None: """Apply, verify, reverse, and leave the exact cached pin pristine.""" source = fetch(workspace) apply(source) reverse(source) def reproduce(workspace: pathlib.Path) -> None: source = fetch(workspace) build_dir = workspace.resolve() / "build" if build_dir.exists(): raise DependencyError(f"build directory already exists; refusing to erase possible local edits: {build_dir}") apply(source) smoke(build(source, build_dir)) def inspect() -> None: lock = _load_lock() patches = _patches(lock) print(json.dumps({ "commit": lock["commit"], "commit_tree": lock["commit_tree"], "retrieval": lock["retrieval"], "upstream_license": lock["upstream_license"], "patch_count": len(patches), "patches": [patch.name for patch in patches], "model_downloads": False, "semantic_certification": False, "glm_stock_limitations": lock["stock_glm_limitations"], }, indent=2, sort_keys=True)) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) subcommands = parser.add_subparsers(dest="command", required=True) subcommands.add_parser("inspect") fetch_parser = subcommands.add_parser("fetch") fetch_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp") apply_parser = subcommands.add_parser("apply") apply_parser.add_argument("--source-dir", type=pathlib.Path, required=True) reverse_parser = subcommands.add_parser("reverse") reverse_parser.add_argument("--source-dir", type=pathlib.Path, required=True) verify_parser = subcommands.add_parser("verify") verify_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp") build_parser = subcommands.add_parser("build") build_parser.add_argument("--source-dir", type=pathlib.Path, required=True) build_parser.add_argument("--build-dir", type=pathlib.Path, required=True) smoke_parser = subcommands.add_parser("smoke") smoke_parser.add_argument("--binary", type=pathlib.Path, required=True) reproduce_parser = subcommands.add_parser("reproduce") reproduce_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp") args = parser.parse_args() try: if args.command == "inspect": inspect() elif args.command == "fetch": fetch(args.workspace) elif args.command == "apply": apply(args.source_dir) elif args.command == "reverse": reverse(args.source_dir) elif args.command == "verify": verify(args.workspace) elif args.command == "build": build(args.source_dir, args.build_dir) elif args.command == "smoke": smoke(args.binary) else: reproduce(args.workspace) except DependencyError as error: print(f"DGR-027 dependency error: {error}", file=sys.stderr) return 2 return 0 if __name__ == "__main__": raise SystemExit(main())