[verified] feat: harden llama.cpp provenance workspace

scripts/llama_cpp_dependency.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Materialize, verify, build, and smoke-test DGR-004's llama.cpp pin.
+"""Fetch, verify, build, and smoke-test DGR-027's exact llama.cpp pin.
 
 This tool deliberately owns only a source dependency boundary.  It never
 downloads a model, invokes inference, or interprets generated text.
@@ -12,6 +12,7 @@ import hashlib
 import json
 import os
 import pathlib
+import re
 import shutil
 import subprocess
 import sys
@@ -67,14 +68,27 @@ def _load_lock() -> dict[str, Any]:
         raise DependencyError(f"invalid upstream lock: {LOCK_PATH}: {error}") from error
     required = {
         "upstream", "commit", "commit_tree", "patched_tree", "patch_series",
-        "required_upstream_blobs", "patched_paths", "build",
+        "required_upstream_blobs", "patched_paths", "build", "upstream_license",
+        "expected_source", "retrieval",
     }
     missing = sorted(required - lock.keys())
     if missing:
         raise DependencyError(f"upstream lock is missing fields: {', '.join(missing)}")
     commit_file = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip()
-    if lock["commit"] != commit_file or len(commit_file) != 40:
-        raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on a full commit")
+    object_ids = [commit_file, lock["commit"], lock["commit_tree"], lock["patched_tree"]]
+    if lock["commit"] != commit_file or not all(
+        isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40}", value)
+        for value in object_ids
+    ):
+        raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on full hexadecimal object IDs")
+    if lock["expected_source"] != {"git_tree": lock["commit_tree"]}:
+        raise DependencyError("expected_source must record the locked git tree")
+    retrieval = lock["retrieval"]
+    if retrieval != {
+        "method": "git-clone-detached-commit",
+        "workspace": "build/llama.cpp",
+    }:
+        raise DependencyError("retrieval must use the locked detached-commit build workspace")
     return lock
 
 
@@ -103,6 +117,56 @@ def _git(source: pathlib.Path, *args: str) -> str:
     return _run("git", "-C", str(source), *args)
 
 
+def _verify_tracked_content(source: pathlib.Path, lock: dict[str, Any]) -> None:
+    if _git(source, "write-tree") != lock["commit_tree"]:
+        raise DependencyError("materialized checkout index differs from the locked tree")
+    filemode_trusted = _git(source, "config", "--bool", "core.filemode") == "true"
+    records = _git(source, "ls-files", "-s", "-z").split("\0")
+    paths: list[str] = []
+    expected: list[str] = []
+    for record in records:
+        if not record:
+            continue
+        metadata, path = record.split("\t", 1)
+        mode, blob, stage = metadata.split()
+        if stage != "0" or mode not in {"100644", "100755", "120000"}:
+            raise DependencyError(f"unsupported tracked entry in materialized checkout: {record!r}")
+        if "\n" in path:
+            raise DependencyError(f"newline-bearing tracked path is unsupported: {path!r}")
+        candidate = source / path
+        cursor = source
+        for part in pathlib.Path(path).parts[:-1]:
+            cursor = cursor / part
+            if cursor.is_symlink():
+                raise DependencyError(f"tracked path traverses a symlink: {path!r}")
+        if mode == "120000":
+            if not candidate.is_symlink():
+                raise DependencyError(f"tracked symlink type differs from the locked tree: {path!r}")
+        else:
+            if candidate.is_symlink() or not candidate.is_file():
+                raise DependencyError(f"tracked file type differs from the locked tree: {path!r}")
+            executable = bool(candidate.stat().st_mode & 0o111)
+            if filemode_trusted and executable != (mode == "100755"):
+                raise DependencyError(f"tracked executable mode differs from the locked tree: {path!r}")
+        paths.append(path)
+        expected.append(blob)
+    try:
+        completed = subprocess.run(
+            ["git", "hash-object", "--stdin-paths"],
+            cwd=source,
+            input="".join(f"{path}\n" for path in paths),
+            check=True,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+    except subprocess.CalledProcessError as error:
+        raise DependencyError(f"unable to hash tracked source content: {error.stderr.strip()}") from error
+    actual = completed.stdout.splitlines()
+    if len(actual) != len(expected) or actual != expected:
+        raise DependencyError("tracked source content differs from the locked tree")
+
+
 def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean: bool) -> None:
     if not (source / ".git").exists():
         raise DependencyError(f"not a materialized git checkout: {source}")
@@ -110,8 +174,15 @@ def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean:
         raise DependencyError("upstream drift: checkout HEAD does not equal the locked commit")
     if _git(source, "rev-parse", "HEAD^{tree}") != lock["commit_tree"]:
         raise DependencyError("upstream drift: checkout tree does not equal the locked tree")
-    if require_clean and _git(source, "status", "--porcelain"):
-        raise DependencyError("local edits detected in materialized llama.cpp checkout")
+    if _git(source, "rev-parse", "--abbrev-ref", "HEAD") != "HEAD":
+        raise DependencyError("materialized llama.cpp checkout must have a detached HEAD")
+    if require_clean:
+        _verify_tracked_content(source, lock)
+        tracked = _git(source, "status", "--porcelain", "--untracked-files=no")
+        untracked = _git(source, "ls-files", "--others", "--exclude-standard")
+        ignored = _git(source, "ls-files", "--others", "--ignored", "--exclude-standard")
+        if tracked or untracked or ignored:
+            raise DependencyError("local edits or unmanifested files detected in materialized llama.cpp checkout")
     for relative, expected in lock["required_upstream_blobs"].items():
         actual = _git(source, "rev-parse", f"HEAD:{relative}")
         if actual != expected:
@@ -122,15 +193,43 @@ def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean:
         raise DependencyError("upstream LICENSE is missing; refusing to drop required attribution")
 
 
-def materialize(source: pathlib.Path, repository: str) -> None:
+def _workspace_source(workspace: pathlib.Path, lock: dict[str, Any]) -> pathlib.Path:
+    relative = pathlib.Path(lock["retrieval"]["workspace"])
+    expected = (ROOT / relative).absolute()
+    supplied = workspace.absolute()
+    if supplied != expected:
+        raise DependencyError(f"--workspace must equal the locked ignored build root: {expected}")
+    cursor = ROOT
+    for part in relative.parts:
+        cursor = cursor / part
+        if cursor.is_symlink():
+            raise DependencyError(f"locked build workspace may not traverse a symlink: {cursor}")
+    resolved = workspace.resolve()
+    try:
+        resolved.relative_to(ROOT.resolve())
+    except ValueError as error:
+        raise DependencyError("locked build workspace escapes the repository root") from error
+    source = resolved / "source"
+    if source.is_symlink():
+        raise DependencyError(f"locked source checkout may not be a symlink: {source}")
+    return source
+
+
+def fetch(workspace: pathlib.Path) -> pathlib.Path:
+    """Fetch once, or verify an exact clean cached checkout for offline reuse."""
     lock = _load_lock()
     _patches(lock)
+    source = _workspace_source(workspace, lock)
     if source.exists():
-        raise DependencyError(f"destination already exists; refusing to reuse possibly edited source: {source}")
+        _verify_source(source, lock, require_clean=True)
+        print(f"reused verified offline cache: {source}")
+        return source
     source.parent.mkdir(parents=True, exist_ok=True)
-    _run("git", "clone", "--no-checkout", repository, str(source))
+    _run("git", "clone", "--no-checkout", lock["upstream"], str(source))
     _git(source, "checkout", "--detach", lock["commit"])
     _verify_source(source, lock, require_clean=True)
+    print(f"fetched and verified exact source: {source}")
+    return source
 
 
 def apply(source: pathlib.Path) -> None:
@@ -198,18 +297,11 @@ def smoke(binary: pathlib.Path) -> None:
     print(output)
 
 
-def reproduce(work_dir: pathlib.Path, repository: str) -> None:
-    resolved = work_dir.resolve()
-    build_root = (ROOT / "build").resolve()
-    if build_root not in resolved.parents:
-        raise DependencyError(f"--work-dir must be below {build_root}: {resolved}")
-    if resolved.exists():
-        raise DependencyError(
-            f"work directory already exists; refusing to erase possible local edits: {resolved}"
-        )
-    source = resolved / "source"
-    build_dir = resolved / "build"
-    materialize(source, repository)
+def reproduce(workspace: pathlib.Path) -> None:
+    source = fetch(workspace)
+    build_dir = workspace.resolve() / "build"
+    if build_dir.exists():
+        raise DependencyError(f"build directory already exists; refusing to erase possible local edits: {build_dir}")
     apply(source)
     smoke(build(source, build_dir))
 
@@ -219,6 +311,9 @@ def inspect() -> None:
     patches = _patches(lock)
     print(json.dumps({
         "commit": lock["commit"],
+        "commit_tree": lock["commit_tree"],
+        "retrieval": lock["retrieval"],
+        "upstream_license": lock["upstream_license"],
         "patch_count": len(patches),
         "patches": [patch.name for patch in patches],
         "model_downloads": False,
@@ -231,9 +326,8 @@ def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     subcommands = parser.add_subparsers(dest="command", required=True)
     subcommands.add_parser("inspect")
-    materialize_parser = subcommands.add_parser("materialize")
-    materialize_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
-    materialize_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
+    fetch_parser = subcommands.add_parser("fetch")
+    fetch_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
     apply_parser = subcommands.add_parser("apply")
     apply_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
     build_parser = subcommands.add_parser("build")
@@ -242,14 +336,13 @@ def main() -> int:
     smoke_parser = subcommands.add_parser("smoke")
     smoke_parser.add_argument("--binary", type=pathlib.Path, required=True)
     reproduce_parser = subcommands.add_parser("reproduce")
-    reproduce_parser.add_argument("--work-dir", type=pathlib.Path, default=ROOT / "build/dgr-004-smoke")
-    reproduce_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
+    reproduce_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
     args = parser.parse_args()
     try:
         if args.command == "inspect":
             inspect()
-        elif args.command == "materialize":
-            materialize(args.source_dir, args.source_repository)
+        elif args.command == "fetch":
+            fetch(args.workspace)
         elif args.command == "apply":
             apply(args.source_dir)
         elif args.command == "build":
@@ -257,9 +350,9 @@ def main() -> int:
         elif args.command == "smoke":
             smoke(args.binary)
         else:
-            reproduce(args.work_dir, args.source_repository)
+            reproduce(args.workspace)
     except DependencyError as error:
-        print(f"DGR-004 dependency error: {error}", file=sys.stderr)
+        print(f"DGR-027 dependency error: {error}", file=sys.stderr)
         return 2
     return 0