269 lines
11 KiB
Python
269 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Materialize, verify, build, and smoke-test DGR-004's llama.cpp pin.
|
|
|
|
This tool deliberately owns only a source dependency boundary. It never
|
|
downloads a model, invokes inference, or interprets generated text.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
LLAMA_DIR = ROOT / "packages/node/native/llama"
|
|
LOCK_PATH = LLAMA_DIR / "UPSTREAM_LOCK.json"
|
|
PATCH_DIR = LLAMA_DIR / "patches"
|
|
|
|
|
|
def _cmake() -> str:
|
|
"""Use an explicit override, PATH, or the active Python environment."""
|
|
configured = os.environ.get("CMAKE")
|
|
if configured:
|
|
return configured
|
|
on_path = shutil.which("cmake")
|
|
if on_path:
|
|
return on_path
|
|
sibling = pathlib.Path(sys.executable).parent / "cmake"
|
|
if sibling.is_file():
|
|
return str(sibling)
|
|
raise DependencyError("cmake is unavailable; set CMAKE or activate the project toolchain")
|
|
|
|
|
|
class DependencyError(RuntimeError):
|
|
"""A fail-closed reproducibility or source-integrity failure."""
|
|
|
|
|
|
def _run(*args: str, cwd: pathlib.Path | None = None) -> str:
|
|
try:
|
|
completed = subprocess.run(
|
|
args,
|
|
cwd=cwd,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
except FileNotFoundError as error:
|
|
raise DependencyError(f"required executable is unavailable: {args[0]}") from error
|
|
except subprocess.CalledProcessError as error:
|
|
detail = (error.stderr or error.stdout).strip()
|
|
raise DependencyError(f"command failed: {' '.join(args)}\n{detail}") from error
|
|
return completed.stdout.strip()
|
|
|
|
|
|
def _load_lock() -> dict[str, Any]:
|
|
try:
|
|
lock = json.loads(LOCK_PATH.read_text())
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
raise DependencyError(f"invalid upstream lock: {LOCK_PATH}: {error}") from error
|
|
required = {
|
|
"upstream", "commit", "commit_tree", "patched_tree", "patch_series",
|
|
"required_upstream_blobs", "patched_paths", "build",
|
|
}
|
|
missing = sorted(required - lock.keys())
|
|
if missing:
|
|
raise DependencyError(f"upstream lock is missing fields: {', '.join(missing)}")
|
|
commit_file = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip()
|
|
if lock["commit"] != commit_file or len(commit_file) != 40:
|
|
raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on a full commit")
|
|
return lock
|
|
|
|
|
|
def _patches(lock: dict[str, Any]) -> list[pathlib.Path]:
|
|
series = [line for line in (PATCH_DIR / "series").read_text().splitlines() if line]
|
|
if series != lock["patch_series"] or series != sorted(series) or not series:
|
|
raise DependencyError("patches/series is empty, unordered, or disagrees with UPSTREAM_LOCK.json")
|
|
sums: dict[str, str] = {}
|
|
for line in (PATCH_DIR / "SHA256SUMS").read_text().splitlines():
|
|
if line and not line.startswith("#"):
|
|
digest, name = line.split(maxsplit=1)
|
|
sums[name] = digest
|
|
patches: list[pathlib.Path] = []
|
|
for name in series:
|
|
patch = PATCH_DIR / name
|
|
if not patch.is_file():
|
|
raise DependencyError(f"patch listed in series is missing: {name}")
|
|
actual = hashlib.sha256(patch.read_bytes()).hexdigest()
|
|
if sums.get(name) != actual:
|
|
raise DependencyError(f"patch digest mismatch for {name}: expected {sums.get(name)}, got {actual}")
|
|
patches.append(patch)
|
|
return patches
|
|
|
|
|
|
def _git(source: pathlib.Path, *args: str) -> str:
|
|
return _run("git", "-C", str(source), *args)
|
|
|
|
|
|
def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean: bool) -> None:
|
|
if not (source / ".git").exists():
|
|
raise DependencyError(f"not a materialized git checkout: {source}")
|
|
if _git(source, "rev-parse", "HEAD") != lock["commit"]:
|
|
raise DependencyError("upstream drift: checkout HEAD does not equal the locked commit")
|
|
if _git(source, "rev-parse", "HEAD^{tree}") != lock["commit_tree"]:
|
|
raise DependencyError("upstream drift: checkout tree does not equal the locked tree")
|
|
if require_clean and _git(source, "status", "--porcelain"):
|
|
raise DependencyError("local edits detected in materialized llama.cpp checkout")
|
|
for relative, expected in lock["required_upstream_blobs"].items():
|
|
actual = _git(source, "rev-parse", f"HEAD:{relative}")
|
|
if actual != expected:
|
|
raise DependencyError(
|
|
f"upstream ABI/context drift for {relative}: expected {expected}, got {actual}"
|
|
)
|
|
if not (source / "LICENSE").is_file():
|
|
raise DependencyError("upstream LICENSE is missing; refusing to drop required attribution")
|
|
|
|
|
|
def materialize(source: pathlib.Path, repository: str) -> None:
|
|
lock = _load_lock()
|
|
_patches(lock)
|
|
if source.exists():
|
|
raise DependencyError(f"destination already exists; refusing to reuse possibly edited source: {source}")
|
|
source.parent.mkdir(parents=True, exist_ok=True)
|
|
_run("git", "clone", "--no-checkout", repository, str(source))
|
|
_git(source, "checkout", "--detach", lock["commit"])
|
|
_verify_source(source, lock, require_clean=True)
|
|
|
|
|
|
def apply(source: pathlib.Path) -> None:
|
|
lock = _load_lock()
|
|
patches = _patches(lock)
|
|
_verify_source(source, lock, require_clean=True)
|
|
for patch in patches:
|
|
_git(source, "apply", "--check", str(patch))
|
|
_git(source, "apply", "--index", str(patch))
|
|
_verify_patched_source(source, lock)
|
|
|
|
|
|
def _verify_patched_source(source: pathlib.Path, lock: dict[str, Any]) -> None:
|
|
if _git(source, "diff", "--quiet"):
|
|
raise DependencyError("local unstaged edits detected after applying patch stack")
|
|
changed_paths = _git(source, "diff", "--cached", "--name-only").splitlines()
|
|
if changed_paths != lock["patched_paths"]:
|
|
raise DependencyError(f"patched paths drifted: expected {lock['patched_paths']}, got {changed_paths}")
|
|
if _git(source, "write-tree") != lock["patched_tree"]:
|
|
raise DependencyError("patched source tree differs from the locked patch stack")
|
|
if _git(source, "diff", "--name-only"):
|
|
raise DependencyError("local unstaged edits detected after applying patch stack")
|
|
untracked = _git(source, "ls-files", "--others", "--exclude-standard").splitlines()
|
|
if untracked:
|
|
raise DependencyError(f"untracked files detected after applying patch stack: {untracked}")
|
|
|
|
|
|
def build(source: pathlib.Path, build_dir: pathlib.Path) -> pathlib.Path:
|
|
lock = _load_lock()
|
|
_patches(lock)
|
|
_verify_source(source, lock, require_clean=False)
|
|
_verify_patched_source(source, lock)
|
|
expected_marker = source / "cmake/meshnet-patch-stack.cmake"
|
|
if not expected_marker.is_file():
|
|
raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent")
|
|
if build_dir.exists():
|
|
raise DependencyError(f"build directory already exists; use reproduce for a clean rebuild: {build_dir}")
|
|
flags = lock["build"]["configure_flags"]
|
|
cmake = _cmake()
|
|
_run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags)
|
|
for target in lock["build"]["native_targets"]:
|
|
_run(cmake, "--build", str(build_dir), "--target", target, "-j2")
|
|
metadata = {
|
|
"commit": lock["commit"],
|
|
"commit_tree": lock["commit_tree"],
|
|
"patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)},
|
|
"configure_flags": flags,
|
|
"cmake": _run(cmake, "--version").splitlines()[0],
|
|
"cxx": _run("c++", "--version").splitlines()[0],
|
|
"model_downloads": False,
|
|
"semantic_certification": False,
|
|
}
|
|
(build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
|
|
return build_dir / lock["build"]["smoke_binary"]
|
|
|
|
|
|
def smoke(binary: pathlib.Path) -> None:
|
|
if not binary.is_file():
|
|
raise DependencyError(f"native smoke binary is missing: {binary}")
|
|
smoke_config = _load_lock()["build"]
|
|
output = _run(str(binary), *smoke_config["smoke_args"])
|
|
expected = smoke_config["smoke_output_token"]
|
|
if expected not in output.lower():
|
|
raise DependencyError(f"native smoke output did not contain {expected!r}: {output}")
|
|
print(output)
|
|
|
|
|
|
def reproduce(work_dir: pathlib.Path, repository: str) -> None:
|
|
resolved = work_dir.resolve()
|
|
build_root = (ROOT / "build").resolve()
|
|
if build_root not in resolved.parents:
|
|
raise DependencyError(f"--work-dir must be below {build_root}: {resolved}")
|
|
if resolved.exists():
|
|
raise DependencyError(
|
|
f"work directory already exists; refusing to erase possible local edits: {resolved}"
|
|
)
|
|
source = resolved / "source"
|
|
build_dir = resolved / "build"
|
|
materialize(source, repository)
|
|
apply(source)
|
|
smoke(build(source, build_dir))
|
|
|
|
|
|
def inspect() -> None:
|
|
lock = _load_lock()
|
|
patches = _patches(lock)
|
|
print(json.dumps({
|
|
"commit": lock["commit"],
|
|
"patch_count": len(patches),
|
|
"patches": [patch.name for patch in patches],
|
|
"model_downloads": False,
|
|
"semantic_certification": False,
|
|
"glm_stock_limitations": lock["stock_glm_limitations"],
|
|
}, indent=2, sort_keys=True))
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
subcommands = parser.add_subparsers(dest="command", required=True)
|
|
subcommands.add_parser("inspect")
|
|
materialize_parser = subcommands.add_parser("materialize")
|
|
materialize_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
|
materialize_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
|
|
apply_parser = subcommands.add_parser("apply")
|
|
apply_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
|
build_parser = subcommands.add_parser("build")
|
|
build_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
|
build_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
|
|
smoke_parser = subcommands.add_parser("smoke")
|
|
smoke_parser.add_argument("--binary", type=pathlib.Path, required=True)
|
|
reproduce_parser = subcommands.add_parser("reproduce")
|
|
reproduce_parser.add_argument("--work-dir", type=pathlib.Path, default=ROOT / "build/dgr-004-smoke")
|
|
reproduce_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.command == "inspect":
|
|
inspect()
|
|
elif args.command == "materialize":
|
|
materialize(args.source_dir, args.source_repository)
|
|
elif args.command == "apply":
|
|
apply(args.source_dir)
|
|
elif args.command == "build":
|
|
build(args.source_dir, args.build_dir)
|
|
elif args.command == "smoke":
|
|
smoke(args.binary)
|
|
else:
|
|
reproduce(args.work_dir, args.source_repository)
|
|
except DependencyError as error:
|
|
print(f"DGR-004 dependency error: {error}", file=sys.stderr)
|
|
return 2
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|