362 lines
15 KiB
Python
362 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Fetch, verify, build, and smoke-test DGR-027's exact llama.cpp pin.
|
|
|
|
This tool deliberately owns only a source dependency boundary. It never
|
|
downloads a model, invokes inference, or interprets generated text.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
LLAMA_DIR = ROOT / "packages/node/native/llama"
|
|
LOCK_PATH = LLAMA_DIR / "UPSTREAM_LOCK.json"
|
|
PATCH_DIR = LLAMA_DIR / "patches"
|
|
|
|
|
|
def _cmake() -> str:
|
|
"""Use an explicit override, PATH, or the active Python environment."""
|
|
configured = os.environ.get("CMAKE")
|
|
if configured:
|
|
return configured
|
|
on_path = shutil.which("cmake")
|
|
if on_path:
|
|
return on_path
|
|
sibling = pathlib.Path(sys.executable).parent / "cmake"
|
|
if sibling.is_file():
|
|
return str(sibling)
|
|
raise DependencyError("cmake is unavailable; set CMAKE or activate the project toolchain")
|
|
|
|
|
|
class DependencyError(RuntimeError):
|
|
"""A fail-closed reproducibility or source-integrity failure."""
|
|
|
|
|
|
def _run(*args: str, cwd: pathlib.Path | None = None) -> str:
|
|
try:
|
|
completed = subprocess.run(
|
|
args,
|
|
cwd=cwd,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
except FileNotFoundError as error:
|
|
raise DependencyError(f"required executable is unavailable: {args[0]}") from error
|
|
except subprocess.CalledProcessError as error:
|
|
detail = (error.stderr or error.stdout).strip()
|
|
raise DependencyError(f"command failed: {' '.join(args)}\n{detail}") from error
|
|
return completed.stdout.strip()
|
|
|
|
|
|
def _load_lock() -> dict[str, Any]:
|
|
try:
|
|
lock = json.loads(LOCK_PATH.read_text())
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
raise DependencyError(f"invalid upstream lock: {LOCK_PATH}: {error}") from error
|
|
required = {
|
|
"upstream", "commit", "commit_tree", "patched_tree", "patch_series",
|
|
"required_upstream_blobs", "patched_paths", "build", "upstream_license",
|
|
"expected_source", "retrieval",
|
|
}
|
|
missing = sorted(required - lock.keys())
|
|
if missing:
|
|
raise DependencyError(f"upstream lock is missing fields: {', '.join(missing)}")
|
|
commit_file = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip()
|
|
object_ids = [commit_file, lock["commit"], lock["commit_tree"], lock["patched_tree"]]
|
|
if lock["commit"] != commit_file or not all(
|
|
isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40}", value)
|
|
for value in object_ids
|
|
):
|
|
raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on full hexadecimal object IDs")
|
|
if lock["expected_source"] != {"git_tree": lock["commit_tree"]}:
|
|
raise DependencyError("expected_source must record the locked git tree")
|
|
retrieval = lock["retrieval"]
|
|
if retrieval != {
|
|
"method": "git-clone-detached-commit",
|
|
"workspace": "build/llama.cpp",
|
|
}:
|
|
raise DependencyError("retrieval must use the locked detached-commit build workspace")
|
|
return lock
|
|
|
|
|
|
def _patches(lock: dict[str, Any]) -> list[pathlib.Path]:
|
|
series = [line for line in (PATCH_DIR / "series").read_text().splitlines() if line]
|
|
if series != lock["patch_series"] or series != sorted(series) or not series:
|
|
raise DependencyError("patches/series is empty, unordered, or disagrees with UPSTREAM_LOCK.json")
|
|
sums: dict[str, str] = {}
|
|
for line in (PATCH_DIR / "SHA256SUMS").read_text().splitlines():
|
|
if line and not line.startswith("#"):
|
|
digest, name = line.split(maxsplit=1)
|
|
sums[name] = digest
|
|
patches: list[pathlib.Path] = []
|
|
for name in series:
|
|
patch = PATCH_DIR / name
|
|
if not patch.is_file():
|
|
raise DependencyError(f"patch listed in series is missing: {name}")
|
|
actual = hashlib.sha256(patch.read_bytes()).hexdigest()
|
|
if sums.get(name) != actual:
|
|
raise DependencyError(f"patch digest mismatch for {name}: expected {sums.get(name)}, got {actual}")
|
|
patches.append(patch)
|
|
return patches
|
|
|
|
|
|
def _git(source: pathlib.Path, *args: str) -> str:
|
|
return _run("git", "-C", str(source), *args)
|
|
|
|
|
|
def _verify_tracked_content(source: pathlib.Path, lock: dict[str, Any]) -> None:
|
|
if _git(source, "write-tree") != lock["commit_tree"]:
|
|
raise DependencyError("materialized checkout index differs from the locked tree")
|
|
filemode_trusted = _git(source, "config", "--bool", "core.filemode") == "true"
|
|
records = _git(source, "ls-files", "-s", "-z").split("\0")
|
|
paths: list[str] = []
|
|
expected: list[str] = []
|
|
for record in records:
|
|
if not record:
|
|
continue
|
|
metadata, path = record.split("\t", 1)
|
|
mode, blob, stage = metadata.split()
|
|
if stage != "0" or mode not in {"100644", "100755", "120000"}:
|
|
raise DependencyError(f"unsupported tracked entry in materialized checkout: {record!r}")
|
|
if "\n" in path:
|
|
raise DependencyError(f"newline-bearing tracked path is unsupported: {path!r}")
|
|
candidate = source / path
|
|
cursor = source
|
|
for part in pathlib.Path(path).parts[:-1]:
|
|
cursor = cursor / part
|
|
if cursor.is_symlink():
|
|
raise DependencyError(f"tracked path traverses a symlink: {path!r}")
|
|
if mode == "120000":
|
|
if not candidate.is_symlink():
|
|
raise DependencyError(f"tracked symlink type differs from the locked tree: {path!r}")
|
|
else:
|
|
if candidate.is_symlink() or not candidate.is_file():
|
|
raise DependencyError(f"tracked file type differs from the locked tree: {path!r}")
|
|
executable = bool(candidate.stat().st_mode & 0o111)
|
|
if filemode_trusted and executable != (mode == "100755"):
|
|
raise DependencyError(f"tracked executable mode differs from the locked tree: {path!r}")
|
|
paths.append(path)
|
|
expected.append(blob)
|
|
try:
|
|
completed = subprocess.run(
|
|
["git", "hash-object", "--stdin-paths"],
|
|
cwd=source,
|
|
input="".join(f"{path}\n" for path in paths),
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
except subprocess.CalledProcessError as error:
|
|
raise DependencyError(f"unable to hash tracked source content: {error.stderr.strip()}") from error
|
|
actual = completed.stdout.splitlines()
|
|
if len(actual) != len(expected) or actual != expected:
|
|
raise DependencyError("tracked source content differs from the locked tree")
|
|
|
|
|
|
def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean: bool) -> None:
|
|
if not (source / ".git").exists():
|
|
raise DependencyError(f"not a materialized git checkout: {source}")
|
|
if _git(source, "rev-parse", "HEAD") != lock["commit"]:
|
|
raise DependencyError("upstream drift: checkout HEAD does not equal the locked commit")
|
|
if _git(source, "rev-parse", "HEAD^{tree}") != lock["commit_tree"]:
|
|
raise DependencyError("upstream drift: checkout tree does not equal the locked tree")
|
|
if _git(source, "rev-parse", "--abbrev-ref", "HEAD") != "HEAD":
|
|
raise DependencyError("materialized llama.cpp checkout must have a detached HEAD")
|
|
if require_clean:
|
|
_verify_tracked_content(source, lock)
|
|
tracked = _git(source, "status", "--porcelain", "--untracked-files=no")
|
|
untracked = _git(source, "ls-files", "--others", "--exclude-standard")
|
|
ignored = _git(source, "ls-files", "--others", "--ignored", "--exclude-standard")
|
|
if tracked or untracked or ignored:
|
|
raise DependencyError("local edits or unmanifested files detected in materialized llama.cpp checkout")
|
|
for relative, expected in lock["required_upstream_blobs"].items():
|
|
actual = _git(source, "rev-parse", f"HEAD:{relative}")
|
|
if actual != expected:
|
|
raise DependencyError(
|
|
f"upstream ABI/context drift for {relative}: expected {expected}, got {actual}"
|
|
)
|
|
if not (source / "LICENSE").is_file():
|
|
raise DependencyError("upstream LICENSE is missing; refusing to drop required attribution")
|
|
|
|
|
|
def _workspace_source(workspace: pathlib.Path, lock: dict[str, Any]) -> pathlib.Path:
|
|
relative = pathlib.Path(lock["retrieval"]["workspace"])
|
|
expected = (ROOT / relative).absolute()
|
|
supplied = workspace.absolute()
|
|
if supplied != expected:
|
|
raise DependencyError(f"--workspace must equal the locked ignored build root: {expected}")
|
|
cursor = ROOT
|
|
for part in relative.parts:
|
|
cursor = cursor / part
|
|
if cursor.is_symlink():
|
|
raise DependencyError(f"locked build workspace may not traverse a symlink: {cursor}")
|
|
resolved = workspace.resolve()
|
|
try:
|
|
resolved.relative_to(ROOT.resolve())
|
|
except ValueError as error:
|
|
raise DependencyError("locked build workspace escapes the repository root") from error
|
|
source = resolved / "source"
|
|
if source.is_symlink():
|
|
raise DependencyError(f"locked source checkout may not be a symlink: {source}")
|
|
return source
|
|
|
|
|
|
def fetch(workspace: pathlib.Path) -> pathlib.Path:
|
|
"""Fetch once, or verify an exact clean cached checkout for offline reuse."""
|
|
lock = _load_lock()
|
|
_patches(lock)
|
|
source = _workspace_source(workspace, lock)
|
|
if source.exists():
|
|
_verify_source(source, lock, require_clean=True)
|
|
print(f"reused verified offline cache: {source}")
|
|
return source
|
|
source.parent.mkdir(parents=True, exist_ok=True)
|
|
_run("git", "clone", "--no-checkout", lock["upstream"], str(source))
|
|
_git(source, "checkout", "--detach", lock["commit"])
|
|
_verify_source(source, lock, require_clean=True)
|
|
print(f"fetched and verified exact source: {source}")
|
|
return source
|
|
|
|
|
|
def apply(source: pathlib.Path) -> None:
|
|
lock = _load_lock()
|
|
patches = _patches(lock)
|
|
_verify_source(source, lock, require_clean=True)
|
|
for patch in patches:
|
|
_git(source, "apply", "--check", str(patch))
|
|
_git(source, "apply", "--index", str(patch))
|
|
_verify_patched_source(source, lock)
|
|
|
|
|
|
def _verify_patched_source(source: pathlib.Path, lock: dict[str, Any]) -> None:
|
|
if _git(source, "diff", "--quiet"):
|
|
raise DependencyError("local unstaged edits detected after applying patch stack")
|
|
changed_paths = _git(source, "diff", "--cached", "--name-only").splitlines()
|
|
if changed_paths != lock["patched_paths"]:
|
|
raise DependencyError(f"patched paths drifted: expected {lock['patched_paths']}, got {changed_paths}")
|
|
if _git(source, "write-tree") != lock["patched_tree"]:
|
|
raise DependencyError("patched source tree differs from the locked patch stack")
|
|
if _git(source, "diff", "--name-only"):
|
|
raise DependencyError("local unstaged edits detected after applying patch stack")
|
|
untracked = _git(source, "ls-files", "--others", "--exclude-standard").splitlines()
|
|
if untracked:
|
|
raise DependencyError(f"untracked files detected after applying patch stack: {untracked}")
|
|
|
|
|
|
def build(source: pathlib.Path, build_dir: pathlib.Path) -> pathlib.Path:
|
|
lock = _load_lock()
|
|
_patches(lock)
|
|
_verify_source(source, lock, require_clean=False)
|
|
_verify_patched_source(source, lock)
|
|
expected_marker = source / "cmake/meshnet-patch-stack.cmake"
|
|
if not expected_marker.is_file():
|
|
raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent")
|
|
if build_dir.exists():
|
|
raise DependencyError(f"build directory already exists; use reproduce for a clean rebuild: {build_dir}")
|
|
flags = lock["build"]["configure_flags"]
|
|
cmake = _cmake()
|
|
_run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags)
|
|
for target in lock["build"]["native_targets"]:
|
|
_run(cmake, "--build", str(build_dir), "--target", target, "-j2")
|
|
metadata = {
|
|
"commit": lock["commit"],
|
|
"commit_tree": lock["commit_tree"],
|
|
"patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)},
|
|
"configure_flags": flags,
|
|
"cmake": _run(cmake, "--version").splitlines()[0],
|
|
"cxx": _run("c++", "--version").splitlines()[0],
|
|
"model_downloads": False,
|
|
"semantic_certification": False,
|
|
}
|
|
(build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
|
|
return build_dir / lock["build"]["smoke_binary"]
|
|
|
|
|
|
def smoke(binary: pathlib.Path) -> None:
|
|
if not binary.is_file():
|
|
raise DependencyError(f"native smoke binary is missing: {binary}")
|
|
smoke_config = _load_lock()["build"]
|
|
output = _run(str(binary), *smoke_config["smoke_args"])
|
|
expected = smoke_config["smoke_output_token"]
|
|
if expected not in output.lower():
|
|
raise DependencyError(f"native smoke output did not contain {expected!r}: {output}")
|
|
print(output)
|
|
|
|
|
|
def reproduce(workspace: pathlib.Path) -> None:
|
|
source = fetch(workspace)
|
|
build_dir = workspace.resolve() / "build"
|
|
if build_dir.exists():
|
|
raise DependencyError(f"build directory already exists; refusing to erase possible local edits: {build_dir}")
|
|
apply(source)
|
|
smoke(build(source, build_dir))
|
|
|
|
|
|
def inspect() -> None:
|
|
lock = _load_lock()
|
|
patches = _patches(lock)
|
|
print(json.dumps({
|
|
"commit": lock["commit"],
|
|
"commit_tree": lock["commit_tree"],
|
|
"retrieval": lock["retrieval"],
|
|
"upstream_license": lock["upstream_license"],
|
|
"patch_count": len(patches),
|
|
"patches": [patch.name for patch in patches],
|
|
"model_downloads": False,
|
|
"semantic_certification": False,
|
|
"glm_stock_limitations": lock["stock_glm_limitations"],
|
|
}, indent=2, sort_keys=True))
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
subcommands = parser.add_subparsers(dest="command", required=True)
|
|
subcommands.add_parser("inspect")
|
|
fetch_parser = subcommands.add_parser("fetch")
|
|
fetch_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
|
|
apply_parser = subcommands.add_parser("apply")
|
|
apply_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
|
build_parser = subcommands.add_parser("build")
|
|
build_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
|
build_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
|
|
smoke_parser = subcommands.add_parser("smoke")
|
|
smoke_parser.add_argument("--binary", type=pathlib.Path, required=True)
|
|
reproduce_parser = subcommands.add_parser("reproduce")
|
|
reproduce_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.command == "inspect":
|
|
inspect()
|
|
elif args.command == "fetch":
|
|
fetch(args.workspace)
|
|
elif args.command == "apply":
|
|
apply(args.source_dir)
|
|
elif args.command == "build":
|
|
build(args.source_dir, args.build_dir)
|
|
elif args.command == "smoke":
|
|
smoke(args.binary)
|
|
else:
|
|
reproduce(args.workspace)
|
|
except DependencyError as error:
|
|
print(f"DGR-027 dependency error: {error}", file=sys.stderr)
|
|
return 2
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|