feat: DGR-004-CHAIN - Execute chained DGR-004/005/003-emission with anchor
This commit is contained in:
Dobromir Popov
267
scripts/llama_cpp_dependency.py
Normal file
267
scripts/llama_cpp_dependency.py
Normal file
@@ -0,0 +1,267 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Materialize, verify, build, and smoke-test DGR-004's llama.cpp pin.
|
||||
|
||||
This tool deliberately owns only a source dependency boundary. It never
|
||||
downloads a model, invokes inference, or interprets generated text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
LLAMA_DIR = ROOT / "packages/node/native/llama"
|
||||
LOCK_PATH = LLAMA_DIR / "UPSTREAM_LOCK.json"
|
||||
PATCH_DIR = LLAMA_DIR / "patches"
|
||||
|
||||
|
||||
def _cmake() -> str:
|
||||
"""Use an explicit override, PATH, or the active Python environment."""
|
||||
configured = os.environ.get("CMAKE")
|
||||
if configured:
|
||||
return configured
|
||||
on_path = shutil.which("cmake")
|
||||
if on_path:
|
||||
return on_path
|
||||
sibling = pathlib.Path(sys.executable).parent / "cmake"
|
||||
if sibling.is_file():
|
||||
return str(sibling)
|
||||
raise DependencyError("cmake is unavailable; set CMAKE or activate the project toolchain")
|
||||
|
||||
|
||||
class DependencyError(RuntimeError):
|
||||
"""A fail-closed reproducibility or source-integrity failure."""
|
||||
|
||||
|
||||
def _run(*args: str, cwd: pathlib.Path | None = None) -> str:
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
args,
|
||||
cwd=cwd,
|
||||
check=True,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
except FileNotFoundError as error:
|
||||
raise DependencyError(f"required executable is unavailable: {args[0]}") from error
|
||||
except subprocess.CalledProcessError as error:
|
||||
detail = (error.stderr or error.stdout).strip()
|
||||
raise DependencyError(f"command failed: {' '.join(args)}\n{detail}") from error
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
def _load_lock() -> dict[str, Any]:
|
||||
try:
|
||||
lock = json.loads(LOCK_PATH.read_text())
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise DependencyError(f"invalid upstream lock: {LOCK_PATH}: {error}") from error
|
||||
required = {
|
||||
"upstream", "commit", "commit_tree", "patched_tree", "patch_series",
|
||||
"required_upstream_blobs", "patched_paths", "build",
|
||||
}
|
||||
missing = sorted(required - lock.keys())
|
||||
if missing:
|
||||
raise DependencyError(f"upstream lock is missing fields: {', '.join(missing)}")
|
||||
commit_file = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip()
|
||||
if lock["commit"] != commit_file or len(commit_file) != 40:
|
||||
raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on a full commit")
|
||||
return lock
|
||||
|
||||
|
||||
def _patches(lock: dict[str, Any]) -> list[pathlib.Path]:
|
||||
series = [line for line in (PATCH_DIR / "series").read_text().splitlines() if line]
|
||||
if series != lock["patch_series"] or series != sorted(series) or not series:
|
||||
raise DependencyError("patches/series is empty, unordered, or disagrees with UPSTREAM_LOCK.json")
|
||||
sums: dict[str, str] = {}
|
||||
for line in (PATCH_DIR / "SHA256SUMS").read_text().splitlines():
|
||||
if line and not line.startswith("#"):
|
||||
digest, name = line.split(maxsplit=1)
|
||||
sums[name] = digest
|
||||
patches: list[pathlib.Path] = []
|
||||
for name in series:
|
||||
patch = PATCH_DIR / name
|
||||
if not patch.is_file():
|
||||
raise DependencyError(f"patch listed in series is missing: {name}")
|
||||
actual = hashlib.sha256(patch.read_bytes()).hexdigest()
|
||||
if sums.get(name) != actual:
|
||||
raise DependencyError(f"patch digest mismatch for {name}: expected {sums.get(name)}, got {actual}")
|
||||
patches.append(patch)
|
||||
return patches
|
||||
|
||||
|
||||
def _git(source: pathlib.Path, *args: str) -> str:
|
||||
return _run("git", "-C", str(source), *args)
|
||||
|
||||
|
||||
def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean: bool) -> None:
|
||||
if not (source / ".git").exists():
|
||||
raise DependencyError(f"not a materialized git checkout: {source}")
|
||||
if _git(source, "rev-parse", "HEAD") != lock["commit"]:
|
||||
raise DependencyError("upstream drift: checkout HEAD does not equal the locked commit")
|
||||
if _git(source, "rev-parse", "HEAD^{tree}") != lock["commit_tree"]:
|
||||
raise DependencyError("upstream drift: checkout tree does not equal the locked tree")
|
||||
if require_clean and _git(source, "status", "--porcelain"):
|
||||
raise DependencyError("local edits detected in materialized llama.cpp checkout")
|
||||
for relative, expected in lock["required_upstream_blobs"].items():
|
||||
actual = _git(source, "rev-parse", f"HEAD:{relative}")
|
||||
if actual != expected:
|
||||
raise DependencyError(
|
||||
f"upstream ABI/context drift for {relative}: expected {expected}, got {actual}"
|
||||
)
|
||||
if not (source / "LICENSE").is_file():
|
||||
raise DependencyError("upstream LICENSE is missing; refusing to drop required attribution")
|
||||
|
||||
|
||||
def materialize(source: pathlib.Path, repository: str) -> None:
|
||||
lock = _load_lock()
|
||||
_patches(lock)
|
||||
if source.exists():
|
||||
raise DependencyError(f"destination already exists; refusing to reuse possibly edited source: {source}")
|
||||
source.parent.mkdir(parents=True, exist_ok=True)
|
||||
_run("git", "clone", "--no-checkout", repository, str(source))
|
||||
_git(source, "checkout", "--detach", lock["commit"])
|
||||
_verify_source(source, lock, require_clean=True)
|
||||
|
||||
|
||||
def apply(source: pathlib.Path) -> None:
|
||||
lock = _load_lock()
|
||||
patches = _patches(lock)
|
||||
_verify_source(source, lock, require_clean=True)
|
||||
for patch in patches:
|
||||
_git(source, "apply", "--check", str(patch))
|
||||
_git(source, "apply", "--index", str(patch))
|
||||
_verify_patched_source(source, lock)
|
||||
|
||||
|
||||
def _verify_patched_source(source: pathlib.Path, lock: dict[str, Any]) -> None:
|
||||
if _git(source, "diff", "--quiet"):
|
||||
raise DependencyError("local unstaged edits detected after applying patch stack")
|
||||
changed_paths = _git(source, "diff", "--cached", "--name-only").splitlines()
|
||||
if changed_paths != lock["patched_paths"]:
|
||||
raise DependencyError(f"patched paths drifted: expected {lock['patched_paths']}, got {changed_paths}")
|
||||
if _git(source, "write-tree") != lock["patched_tree"]:
|
||||
raise DependencyError("patched source tree differs from the locked patch stack")
|
||||
status = _git(source, "status", "--porcelain", "--untracked-files=all").splitlines()
|
||||
expected_status = [f"M {path}" if path == "CMakeLists.txt" else f"A {path}" for path in lock["patched_paths"]]
|
||||
if status != expected_status:
|
||||
raise DependencyError(f"local edits detected after applying patch stack: {status}")
|
||||
|
||||
|
||||
def build(source: pathlib.Path, build_dir: pathlib.Path) -> pathlib.Path:
|
||||
lock = _load_lock()
|
||||
_patches(lock)
|
||||
_verify_source(source, lock, require_clean=False)
|
||||
_verify_patched_source(source, lock)
|
||||
expected_marker = source / "cmake/meshnet-patch-stack.cmake"
|
||||
if not expected_marker.is_file():
|
||||
raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent")
|
||||
if build_dir.exists():
|
||||
raise DependencyError(f"build directory already exists; use reproduce for a clean rebuild: {build_dir}")
|
||||
flags = lock["build"]["configure_flags"]
|
||||
cmake = _cmake()
|
||||
_run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags)
|
||||
for target in lock["build"]["native_targets"]:
|
||||
_run(cmake, "--build", str(build_dir), "--target", target, "-j2")
|
||||
metadata = {
|
||||
"commit": lock["commit"],
|
||||
"commit_tree": lock["commit_tree"],
|
||||
"patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)},
|
||||
"configure_flags": flags,
|
||||
"cmake": _run(cmake, "--version").splitlines()[0],
|
||||
"cxx": _run("c++", "--version").splitlines()[0],
|
||||
"model_downloads": False,
|
||||
"semantic_certification": False,
|
||||
}
|
||||
(build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
|
||||
return build_dir / lock["build"]["smoke_binary"]
|
||||
|
||||
|
||||
def smoke(binary: pathlib.Path) -> None:
|
||||
if not binary.is_file():
|
||||
raise DependencyError(f"native smoke binary is missing: {binary}")
|
||||
smoke_config = _load_lock()["build"]
|
||||
output = _run(str(binary), *smoke_config["smoke_args"])
|
||||
expected = smoke_config["smoke_output_token"]
|
||||
if expected not in output.lower():
|
||||
raise DependencyError(f"native smoke output did not contain {expected!r}: {output}")
|
||||
print(output)
|
||||
|
||||
|
||||
def reproduce(work_dir: pathlib.Path, repository: str) -> None:
|
||||
resolved = work_dir.resolve()
|
||||
build_root = (ROOT / "build").resolve()
|
||||
if build_root not in resolved.parents:
|
||||
raise DependencyError(f"--work-dir must be below {build_root}: {resolved}")
|
||||
if resolved.exists():
|
||||
raise DependencyError(
|
||||
f"work directory already exists; refusing to erase possible local edits: {resolved}"
|
||||
)
|
||||
source = resolved / "source"
|
||||
build_dir = resolved / "build"
|
||||
materialize(source, repository)
|
||||
apply(source)
|
||||
smoke(build(source, build_dir))
|
||||
|
||||
|
||||
def inspect() -> None:
|
||||
lock = _load_lock()
|
||||
patches = _patches(lock)
|
||||
print(json.dumps({
|
||||
"commit": lock["commit"],
|
||||
"patch_count": len(patches),
|
||||
"patches": [patch.name for patch in patches],
|
||||
"model_downloads": False,
|
||||
"semantic_certification": False,
|
||||
"glm_stock_limitations": lock["stock_glm_limitations"],
|
||||
}, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subcommands = parser.add_subparsers(dest="command", required=True)
|
||||
subcommands.add_parser("inspect")
|
||||
materialize_parser = subcommands.add_parser("materialize")
|
||||
materialize_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
||||
materialize_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
|
||||
apply_parser = subcommands.add_parser("apply")
|
||||
apply_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
||||
build_parser = subcommands.add_parser("build")
|
||||
build_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
||||
build_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
|
||||
smoke_parser = subcommands.add_parser("smoke")
|
||||
smoke_parser.add_argument("--binary", type=pathlib.Path, required=True)
|
||||
reproduce_parser = subcommands.add_parser("reproduce")
|
||||
reproduce_parser.add_argument("--work-dir", type=pathlib.Path, default=ROOT / "build/dgr-004-smoke")
|
||||
reproduce_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command == "inspect":
|
||||
inspect()
|
||||
elif args.command == "materialize":
|
||||
materialize(args.source_dir, args.source_repository)
|
||||
elif args.command == "apply":
|
||||
apply(args.source_dir)
|
||||
elif args.command == "build":
|
||||
build(args.source_dir, args.build_dir)
|
||||
elif args.command == "smoke":
|
||||
smoke(args.binary)
|
||||
else:
|
||||
reproduce(args.work_dir, args.source_repository)
|
||||
except DependencyError as error:
|
||||
print(f"DGR-004 dependency error: {error}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user