popov/neuron-tai
1
0
Fork 0
Code Issues 42 Pull Requests Actions Packages Projects Releases Wiki Activity

[verified] feat: harden llama.cpp provenance workspace

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-17 16:24:46 +03:00
parent ae51526e85
commit f83cf331c3
6 changed files with 405 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
scripts/llama_cpp_dependency.py
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Materialize, verify, build, and smoke-test DGR-004's llama.cpp pin.
"""Fetch, verify, build, and smoke-test DGR-027's exact llama.cpp pin.
This tool deliberately owns only a source dependency boundary. It never
downloads a model, invokes inference, or interprets generated text.
@@ -12,6 +12,7 @@ import hashlib
import json
import os
import pathlib
import re
import shutil
import subprocess
import sys
@@ -67,14 +68,27 @@ def _load_lock() -> dict[str, Any]:
raise DependencyError(f"invalid upstream lock: {LOCK_PATH}: {error}") from error
required = {
"upstream", "commit", "commit_tree", "patched_tree", "patch_series",
"required_upstream_blobs", "patched_paths", "build",
"required_upstream_blobs", "patched_paths", "build", "upstream_license",
"expected_source", "retrieval",
}
missing = sorted(required - lock.keys())
if missing:
raise DependencyError(f"upstream lock is missing fields: {', '.join(missing)}")
commit_file = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip()
if lock["commit"] != commit_file or len(commit_file) != 40:
raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on a full commit")
object_ids = [commit_file, lock["commit"], lock["commit_tree"], lock["patched_tree"]]
if lock["commit"] != commit_file or not all(
isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40}", value)
for value in object_ids
):
raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on full hexadecimal object IDs")
if lock["expected_source"] != {"git_tree": lock["commit_tree"]}:
raise DependencyError("expected_source must record the locked git tree")
retrieval = lock["retrieval"]
if retrieval != {
"method": "git-clone-detached-commit",
"workspace": "build/llama.cpp",
}:
raise DependencyError("retrieval must use the locked detached-commit build workspace")
return lock
@@ -103,6 +117,56 @@ def _git(source: pathlib.Path, *args: str) -> str:
return _run("git", "-C", str(source), *args)
def _verify_tracked_content(source: pathlib.Path, lock: dict[str, Any]) -> None:
if _git(source, "write-tree") != lock["commit_tree"]:
raise DependencyError("materialized checkout index differs from the locked tree")
filemode_trusted = _git(source, "config", "--bool", "core.filemode") == "true"
records = _git(source, "ls-files", "-s", "-z").split("\0")
paths: list[str] = []
expected: list[str] = []
for record in records:
if not record:
continue
metadata, path = record.split("\t", 1)
mode, blob, stage = metadata.split()
if stage != "0" or mode not in {"100644", "100755", "120000"}:
raise DependencyError(f"unsupported tracked entry in materialized checkout: {record!r}")
if "\n" in path:
raise DependencyError(f"newline-bearing tracked path is unsupported: {path!r}")
candidate = source / path
cursor = source
for part in pathlib.Path(path).parts[:-1]:
cursor = cursor / part
if cursor.is_symlink():
raise DependencyError(f"tracked path traverses a symlink: {path!r}")
if mode == "120000":
if not candidate.is_symlink():
raise DependencyError(f"tracked symlink type differs from the locked tree: {path!r}")
else:
if candidate.is_symlink() or not candidate.is_file():
raise DependencyError(f"tracked file type differs from the locked tree: {path!r}")
executable = bool(candidate.stat().st_mode & 0o111)
if filemode_trusted and executable != (mode == "100755"):
raise DependencyError(f"tracked executable mode differs from the locked tree: {path!r}")
paths.append(path)
expected.append(blob)
try:
completed = subprocess.run(
["git", "hash-object", "--stdin-paths"],
cwd=source,
input="".join(f"{path}\n" for path in paths),
check=True,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as error:
raise DependencyError(f"unable to hash tracked source content: {error.stderr.strip()}") from error
actual = completed.stdout.splitlines()
if len(actual) != len(expected) or actual != expected:
raise DependencyError("tracked source content differs from the locked tree")
def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean: bool) -> None:
if not (source / ".git").exists():
raise DependencyError(f"not a materialized git checkout: {source}")
@@ -110,8 +174,15 @@ def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean:
raise DependencyError("upstream drift: checkout HEAD does not equal the locked commit")
if _git(source, "rev-parse", "HEAD^{tree}") != lock["commit_tree"]:
raise DependencyError("upstream drift: checkout tree does not equal the locked tree")
if require_clean and _git(source, "status", "--porcelain"):
raise DependencyError("local edits detected in materialized llama.cpp checkout")
if _git(source, "rev-parse", "--abbrev-ref", "HEAD") != "HEAD":
raise DependencyError("materialized llama.cpp checkout must have a detached HEAD")
if require_clean:
_verify_tracked_content(source, lock)
tracked = _git(source, "status", "--porcelain", "--untracked-files=no")
untracked = _git(source, "ls-files", "--others", "--exclude-standard")
ignored = _git(source, "ls-files", "--others", "--ignored", "--exclude-standard")
if tracked or untracked or ignored:
raise DependencyError("local edits or unmanifested files detected in materialized llama.cpp checkout")
for relative, expected in lock["required_upstream_blobs"].items():
actual = _git(source, "rev-parse", f"HEAD:{relative}")
if actual != expected:
@@ -122,15 +193,43 @@ def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean:
raise DependencyError("upstream LICENSE is missing; refusing to drop required attribution")
def materialize(source: pathlib.Path, repository: str) -> None:
def _workspace_source(workspace: pathlib.Path, lock: dict[str, Any]) -> pathlib.Path:
relative = pathlib.Path(lock["retrieval"]["workspace"])
expected = (ROOT / relative).absolute()
supplied = workspace.absolute()
if supplied != expected:
raise DependencyError(f"--workspace must equal the locked ignored build root: {expected}")
cursor = ROOT
for part in relative.parts:
cursor = cursor / part
if cursor.is_symlink():
raise DependencyError(f"locked build workspace may not traverse a symlink: {cursor}")
resolved = workspace.resolve()
try:
resolved.relative_to(ROOT.resolve())
except ValueError as error:
raise DependencyError("locked build workspace escapes the repository root") from error
source = resolved / "source"
if source.is_symlink():
raise DependencyError(f"locked source checkout may not be a symlink: {source}")
return source
def fetch(workspace: pathlib.Path) -> pathlib.Path:
"""Fetch once, or verify an exact clean cached checkout for offline reuse."""
lock = _load_lock()
_patches(lock)
source = _workspace_source(workspace, lock)
if source.exists():
raise DependencyError(f"destination already exists; refusing to reuse possibly edited source: {source}")
_verify_source(source, lock, require_clean=True)
print(f"reused verified offline cache: {source}")
return source
source.parent.mkdir(parents=True, exist_ok=True)
_run("git", "clone", "--no-checkout", repository, str(source))
_run("git", "clone", "--no-checkout", lock["upstream"], str(source))
_git(source, "checkout", "--detach", lock["commit"])
_verify_source(source, lock, require_clean=True)
print(f"fetched and verified exact source: {source}")
return source
def apply(source: pathlib.Path) -> None:
@@ -198,18 +297,11 @@ def smoke(binary: pathlib.Path) -> None:
print(output)
def reproduce(work_dir: pathlib.Path, repository: str) -> None:
resolved = work_dir.resolve()
build_root = (ROOT / "build").resolve()
if build_root not in resolved.parents:
raise DependencyError(f"--work-dir must be below {build_root}: {resolved}")
if resolved.exists():
raise DependencyError(
f"work directory already exists; refusing to erase possible local edits: {resolved}"
)
source = resolved / "source"
build_dir = resolved / "build"
materialize(source, repository)
def reproduce(workspace: pathlib.Path) -> None:
source = fetch(workspace)
build_dir = workspace.resolve() / "build"
if build_dir.exists():
raise DependencyError(f"build directory already exists; refusing to erase possible local edits: {build_dir}")
apply(source)
smoke(build(source, build_dir))
@@ -219,6 +311,9 @@ def inspect() -> None:
patches = _patches(lock)
print(json.dumps({
"commit": lock["commit"],
"commit_tree": lock["commit_tree"],
"retrieval": lock["retrieval"],
"upstream_license": lock["upstream_license"],
"patch_count": len(patches),
"patches": [patch.name for patch in patches],
"model_downloads": False,
@@ -231,9 +326,8 @@ def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
subcommands = parser.add_subparsers(dest="command", required=True)
subcommands.add_parser("inspect")
materialize_parser = subcommands.add_parser("materialize")
materialize_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
materialize_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
fetch_parser = subcommands.add_parser("fetch")
fetch_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
apply_parser = subcommands.add_parser("apply")
apply_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
build_parser = subcommands.add_parser("build")
@@ -242,14 +336,13 @@ def main() -> int:
smoke_parser = subcommands.add_parser("smoke")
smoke_parser.add_argument("--binary", type=pathlib.Path, required=True)
reproduce_parser = subcommands.add_parser("reproduce")
reproduce_parser.add_argument("--work-dir", type=pathlib.Path, default=ROOT / "build/dgr-004-smoke")
reproduce_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
reproduce_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
args = parser.parse_args()
try:
if args.command == "inspect":
inspect()
elif args.command == "materialize":
materialize(args.source_dir, args.source_repository)
elif args.command == "fetch":
fetch(args.workspace)
elif args.command == "apply":
apply(args.source_dir)
elif args.command == "build":
@@ -257,9 +350,9 @@ def main() -> int:
elif args.command == "smoke":
smoke(args.binary)
else:
reproduce(args.work_dir, args.source_repository)
reproduce(args.workspace)
except DependencyError as error:
print(f"DGR-004 dependency error: {error}", file=sys.stderr)
print(f"DGR-027 dependency error: {error}", file=sys.stderr)
return 2
return 0