747 lines
33 KiB
Python
747 lines
33 KiB
Python
#!/usr/bin/env python3
|
|
"""Fetch, verify, build, and smoke-test DGR-027's exact llama.cpp pin.
|
|
|
|
This tool deliberately owns only a source dependency boundary. It never
|
|
downloads a model, invokes inference, or interprets generated text.
|
|
|
|
DGR-028 adds the numbered patch-stack contract: the ordered series applies,
|
|
checks, and reverses deterministically against the exact manifest pin, each
|
|
patch's recorded upstream file/API assumptions are enforced before it is
|
|
attempted, the first incompatible patch is named on failure, and the stack is
|
|
refused if it carries license/attribution damage or Meshnet control-plane
|
|
(routing, billing, relay, authentication) code.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from typing import Any
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
LLAMA_DIR = ROOT / "packages/node/native/llama"
|
|
LOCK_PATH = LLAMA_DIR / "UPSTREAM_LOCK.json"
|
|
PATCH_DIR = LLAMA_DIR / "patches"
|
|
|
|
|
|
def _toolchain_binary(name: str, env_var: str) -> str:
|
|
"""Use an explicit override, PATH, or the active Python environment."""
|
|
configured = os.environ.get(env_var)
|
|
if configured:
|
|
return configured
|
|
on_path = shutil.which(name)
|
|
if on_path:
|
|
return on_path
|
|
sibling = pathlib.Path(sys.executable).parent / name
|
|
if sibling.is_file():
|
|
return str(sibling)
|
|
raise DependencyError(f"{name} is unavailable; set {env_var} or activate the project toolchain")
|
|
|
|
|
|
def _cmake() -> str:
|
|
return _toolchain_binary("cmake", "CMAKE")
|
|
|
|
|
|
def _ctest() -> str:
|
|
return _toolchain_binary("ctest", "CTEST")
|
|
|
|
|
|
class DependencyError(RuntimeError):
|
|
"""A fail-closed reproducibility or source-integrity failure."""
|
|
|
|
|
|
def _run(*args: str, cwd: pathlib.Path | None = None) -> str:
|
|
try:
|
|
completed = subprocess.run(
|
|
args,
|
|
cwd=cwd,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
except FileNotFoundError as error:
|
|
raise DependencyError(f"required executable is unavailable: {args[0]}") from error
|
|
except subprocess.CalledProcessError as error:
|
|
detail = (error.stderr or error.stdout).strip()
|
|
raise DependencyError(f"command failed: {' '.join(args)}\n{detail}") from error
|
|
return completed.stdout.strip()
|
|
|
|
|
|
def _git(source: pathlib.Path, *args: str) -> str:
|
|
"""Run Git against the materialized upstream checkout."""
|
|
return _run("git", "-C", str(source), *args)
|
|
|
|
|
|
def _load_lock() -> dict[str, Any]:
|
|
try:
|
|
lock = json.loads(LOCK_PATH.read_text())
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
raise DependencyError(f"invalid upstream lock: {LOCK_PATH}: {error}") from error
|
|
required = {
|
|
"upstream", "commit", "commit_tree", "patched_tree", "patch_series",
|
|
"required_upstream_blobs", "patched_paths", "build", "upstream_license",
|
|
"expected_source", "retrieval",
|
|
}
|
|
missing = sorted(required - lock.keys())
|
|
if missing:
|
|
raise DependencyError(f"upstream lock is missing fields: {', '.join(missing)}")
|
|
commit_file = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip()
|
|
object_ids = [commit_file, lock["commit"], lock["commit_tree"], lock["patched_tree"]]
|
|
if lock["commit"] != commit_file or not all(
|
|
isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40}", value)
|
|
for value in object_ids
|
|
):
|
|
raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on full hexadecimal object IDs")
|
|
if lock["expected_source"] != {"git_tree": lock["commit_tree"]}:
|
|
raise DependencyError("expected_source must record the locked git tree")
|
|
retrieval = lock["retrieval"]
|
|
if retrieval != {
|
|
"method": "git-clone-detached-commit",
|
|
"workspace": "build/llama.cpp",
|
|
}:
|
|
raise DependencyError("retrieval must use the locked detached-commit build workspace")
|
|
_verify_accelerator_presets(lock)
|
|
return lock
|
|
|
|
|
|
def _verify_accelerator_presets(lock: dict[str, Any]) -> None:
|
|
"""Each preset must isolate one backend that the CPU default leaves OFF.
|
|
|
|
This is what keeps DGR-030's presets from ever being able to change the
|
|
deterministic CPU default recorded in ``build.configure_flags``: a preset
|
|
can only exist for a flag this lock already pins OFF, and
|
|
``accelerator_configure_flags`` only ever returns a fresh list, never
|
|
mutates ``build.configure_flags`` in place.
|
|
"""
|
|
presets = lock.get("accelerator_presets", {})
|
|
if not isinstance(presets, dict):
|
|
raise DependencyError("accelerator_presets must be a JSON object")
|
|
if not presets:
|
|
return
|
|
base_flags = dict(flag[len("-D"):].split("=", 1) for flag in lock["build"]["configure_flags"])
|
|
for name, preset in presets.items():
|
|
if not isinstance(preset, dict):
|
|
raise DependencyError(f"accelerator_presets.{name} must be a JSON object")
|
|
backend_flag = preset.get("backend_flag")
|
|
if not isinstance(backend_flag, str) or not backend_flag:
|
|
raise DependencyError(f"accelerator_presets.{name} is missing backend_flag")
|
|
if base_flags.get(backend_flag) != "OFF":
|
|
raise DependencyError(
|
|
f"accelerator_presets.{name} backend flag {backend_flag} must be OFF in "
|
|
"the deterministic CPU default build.configure_flags"
|
|
)
|
|
probe = preset.get("sdk_probe")
|
|
if not isinstance(probe, dict) or not isinstance(probe.get("binary"), str) or not probe["binary"]:
|
|
raise DependencyError(f"accelerator_presets.{name} is missing an sdk_probe.binary")
|
|
|
|
|
|
def _patches(lock: dict[str, Any]) -> list[pathlib.Path]:
|
|
series = [line for line in (PATCH_DIR / "series").read_text().splitlines() if line]
|
|
if series != lock["patch_series"] or series != sorted(series) or not series:
|
|
raise DependencyError("patches/series is empty, unordered, or disagrees with UPSTREAM_LOCK.json")
|
|
sums: dict[str, str] = {}
|
|
for line in (PATCH_DIR / "SHA256SUMS").read_text().splitlines():
|
|
if line and not line.startswith("#"):
|
|
digest, name = line.split(maxsplit=1)
|
|
sums[name] = digest
|
|
patches: list[pathlib.Path] = []
|
|
for name in series:
|
|
patch = PATCH_DIR / name
|
|
if not patch.is_file():
|
|
raise DependencyError(f"patch listed in series is missing: {name}")
|
|
actual = hashlib.sha256(patch.read_bytes()).hexdigest()
|
|
if sums.get(name) != actual:
|
|
raise DependencyError(f"patch digest mismatch for {name}: expected {sums.get(name)}, got {actual}")
|
|
patches.append(patch)
|
|
return patches
|
|
|
|
|
|
def _parse_patch_files(patch: pathlib.Path) -> dict[str, tuple[str | None, str | None]]:
|
|
"""Parse one patch into {path: (before-short, after-short)}.
|
|
|
|
Short object IDs come from the patch's ``index`` lines; an all-zero side
|
|
means the file is created (``before is None``) or deleted (``after is
|
|
None``). Patch order is preserved.
|
|
"""
|
|
files: dict[str, tuple[str | None, str | None]] = {}
|
|
current: str | None = None
|
|
for line in patch.read_text().splitlines():
|
|
header = re.match(r"^diff --git a/(.+) b/(.+)$", line)
|
|
if header:
|
|
if header.group(1) != header.group(2):
|
|
raise DependencyError(f"{patch.name}: rename/copy diffs are unsupported: {line}")
|
|
current = header.group(1)
|
|
files[current] = (None, None)
|
|
continue
|
|
index = re.match(r"^index ([0-9a-f]{7,40})\.\.([0-9a-f]{7,40})(?:\s|$)", line)
|
|
if index and current is not None:
|
|
before, after = index.group(1), index.group(2)
|
|
files[current] = (
|
|
None if set(before) == {"0"} else before,
|
|
None if set(after) == {"0"} else after,
|
|
)
|
|
if not files:
|
|
raise DependencyError(f"{patch.name}: no file diffs found")
|
|
return files
|
|
|
|
|
|
_ASSUMPTIONS_DEFAULT = "patches/UPSTREAM-ASSUMPTIONS.json"
|
|
|
|
|
|
def _assumptions(lock: dict[str, Any], patches: list[pathlib.Path]) -> dict[str, Any]:
|
|
"""Load and validate the recorded upstream file/API assumptions.
|
|
|
|
The record must cover exactly the ordered series, each recorded file must
|
|
match the patch's parsed diff headers, and each recorded full object ID
|
|
must agree with the patch's abbreviated ``index`` IDs. A stale or edited
|
|
record is a fail-closed error, never a warning.
|
|
"""
|
|
configured = lock.get("patch_assumptions", _ASSUMPTIONS_DEFAULT)
|
|
if not isinstance(configured, str) or not configured:
|
|
raise DependencyError("patch_assumptions must name a manifest-relative JSON path")
|
|
relative = pathlib.Path(configured)
|
|
if relative.is_absolute() or ".." in relative.parts:
|
|
raise DependencyError("patch_assumptions must stay inside the repository manifest tree")
|
|
candidate = (LLAMA_DIR / relative).absolute()
|
|
try:
|
|
candidate.relative_to(LLAMA_DIR.absolute())
|
|
except ValueError as error:
|
|
raise DependencyError("patch_assumptions must live under packages/node/native/llama") from error
|
|
if not candidate.is_file():
|
|
raise DependencyError(f"recorded upstream assumptions are missing: {candidate}")
|
|
try:
|
|
doc = json.loads(candidate.read_text())
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
raise DependencyError(f"invalid upstream assumptions: {candidate}: {error}") from error
|
|
if not isinstance(doc, dict) or doc.get("schema_version") != 1:
|
|
raise DependencyError("upstream assumptions must declare schema_version 1")
|
|
if doc.get("upstream_commit") != lock["commit"]:
|
|
raise DependencyError("upstream assumptions disagree with the locked commit")
|
|
recorded = doc.get("patches")
|
|
if not isinstance(recorded, dict):
|
|
raise DependencyError("upstream assumptions must record a patches object")
|
|
names = [patch.name for patch in patches]
|
|
if list(recorded.keys()) != names:
|
|
raise DependencyError(
|
|
f"upstream assumptions do not cover exactly the ordered series: "
|
|
f"expected {names}, got {list(recorded.keys())}"
|
|
)
|
|
for patch in patches:
|
|
entry = recorded[patch.name]
|
|
if not isinstance(entry.get("concern"), str) or not entry["concern"]:
|
|
raise DependencyError(f"{patch.name}: assumptions are missing the scoped concern")
|
|
symbols = entry.get("api_assumptions")
|
|
if not isinstance(symbols, list) or not symbols or not all(isinstance(s, str) and s for s in symbols):
|
|
raise DependencyError(f"{patch.name}: assumptions must record upstream file/API assumptions")
|
|
files = entry.get("files")
|
|
if not isinstance(files, dict):
|
|
raise DependencyError(f"{patch.name}: assumptions must record a files object")
|
|
parsed = _parse_patch_files(patch)
|
|
if list(files.keys()) != list(parsed.keys()):
|
|
raise DependencyError(
|
|
f"{patch.name}: recorded files {list(files.keys())} disagree with the patch bytes {list(parsed.keys())}"
|
|
)
|
|
for path, (before_short, after_short) in parsed.items():
|
|
blobs = files[path]
|
|
if not isinstance(blobs, dict):
|
|
raise DependencyError(f"{patch.name}: recorded blobs for {path} must be an object")
|
|
for side, short in (("before", before_short), ("after", after_short)):
|
|
value = blobs.get(side)
|
|
if short is None:
|
|
if value is not None:
|
|
raise DependencyError(f"{patch.name}: {path} {side} must be null for a created/deleted file")
|
|
elif not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{40}", value) or not value.startswith(short):
|
|
raise DependencyError(
|
|
f"{patch.name}: recorded {side} blob for {path} does not match the patch index ID {short}"
|
|
)
|
|
return doc
|
|
|
|
|
|
# Control-plane vocabulary that must never enter the upstream patch stack:
|
|
# Meshnet routing, billing, relay, authentication, and transport semantics are
|
|
# backend-agnostic and live outside the llama.cpp fork boundary (ADR-0024).
|
|
_CONTROL_PLANE_TERMS = re.compile(
|
|
r"\b(tracker|routing|route session|grpc|billing|wallet|relay|telemetry|"
|
|
r"auth|authentication|load balanc\w*)\b",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
_LICENSE_PATH = re.compile(r"(^|/)(license|copying|notice)(\..*)?$", re.IGNORECASE)
|
|
|
|
_LICENSE_TEXT = re.compile(
|
|
r"copyright|permission is hereby granted|mit license|apache license|gnu general public",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _verify_patch_stack_boundaries(patches: list[pathlib.Path]) -> None:
|
|
"""Refuse license/attribution damage or control-plane code in the stack."""
|
|
for patch in patches:
|
|
for line in patch.read_text().splitlines():
|
|
header = re.match(r"^diff --git a/(.+) b/(.+)$", line)
|
|
if header and (_LICENSE_PATH.search(header.group(1)) or _LICENSE_PATH.search(header.group(2))):
|
|
raise DependencyError(f"{patch.name}: license/attribution files may not be patched: {line}")
|
|
if line.startswith("-") and not line.startswith("---") and _LICENSE_TEXT.search(line[1:]):
|
|
raise DependencyError(f"{patch.name}: removing license or attribution text is refused: {line}")
|
|
body = patch.read_text()
|
|
match = _CONTROL_PLANE_TERMS.search(body)
|
|
if match:
|
|
raise DependencyError(
|
|
f"{patch.name}: Meshnet control-plane term {match.group(0)!r} must not enter the patch stack"
|
|
)
|
|
|
|
|
|
def _index_blob(source: pathlib.Path, path: str) -> str | None:
|
|
"""Return the staged blob object ID for path, or None when absent."""
|
|
records = _git(source, "ls-files", "-s", "-z", "--", path).split("\0")
|
|
entries = [record for record in records if record]
|
|
if not entries:
|
|
return None
|
|
if len(entries) != 1:
|
|
raise DependencyError(f"unmerged index entry blocks patch verification: {path}")
|
|
metadata, _ = entries[0].split("\t", 1)
|
|
mode, blob, stage = metadata.split()
|
|
if stage != "0":
|
|
raise DependencyError(f"unmerged index entry blocks patch verification: {path}")
|
|
return blob
|
|
|
|
|
|
def _check_assumption_blobs(
|
|
source: pathlib.Path,
|
|
patch: pathlib.Path,
|
|
files: dict[str, Any],
|
|
side: str,
|
|
) -> None:
|
|
"""Fail on the first recorded pre-/post-image blob the index disagrees with."""
|
|
for path, blobs in files.items():
|
|
expected = blobs[side]
|
|
actual = _index_blob(source, path)
|
|
if actual != expected:
|
|
raise DependencyError(
|
|
f"first incompatible patch: {patch.name}: recorded {side} blob for {path} is "
|
|
f"{expected}, found {actual}"
|
|
)
|
|
|
|
|
|
def _verify_tracked_content(source: pathlib.Path, lock: dict[str, Any]) -> None:
|
|
if _git(source, "write-tree") != lock["commit_tree"]:
|
|
raise DependencyError("materialized checkout index differs from the locked tree")
|
|
filemode_trusted = _git(source, "config", "--bool", "core.filemode") == "true"
|
|
records = _git(source, "ls-files", "-s", "-z").split("\0")
|
|
paths: list[str] = []
|
|
expected: list[str] = []
|
|
for record in records:
|
|
if not record:
|
|
continue
|
|
metadata, path = record.split("\t", 1)
|
|
mode, blob, stage = metadata.split()
|
|
if stage != "0" or mode not in {"100644", "100755", "120000"}:
|
|
raise DependencyError(f"unsupported tracked entry in materialized checkout: {record!r}")
|
|
if "\n" in path:
|
|
raise DependencyError(f"newline-bearing tracked path is unsupported: {path!r}")
|
|
candidate = source / path
|
|
cursor = source
|
|
for part in pathlib.Path(path).parts[:-1]:
|
|
cursor = cursor / part
|
|
if cursor.is_symlink():
|
|
raise DependencyError(f"tracked path traverses a symlink: {path!r}")
|
|
if mode == "120000":
|
|
if not candidate.is_symlink():
|
|
raise DependencyError(f"tracked symlink type differs from the locked tree: {path!r}")
|
|
else:
|
|
if candidate.is_symlink() or not candidate.is_file():
|
|
raise DependencyError(f"tracked file type differs from the locked tree: {path!r}")
|
|
executable = bool(candidate.stat().st_mode & 0o111)
|
|
if filemode_trusted and executable != (mode == "100755"):
|
|
raise DependencyError(f"tracked executable mode differs from the locked tree: {path!r}")
|
|
paths.append(path)
|
|
expected.append(blob)
|
|
try:
|
|
completed = subprocess.run(
|
|
["git", "hash-object", "--stdin-paths"],
|
|
cwd=source,
|
|
input="".join(f"{path}\n" for path in paths),
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
except subprocess.CalledProcessError as error:
|
|
raise DependencyError(f"unable to hash tracked source content: {error.stderr.strip()}") from error
|
|
actual = completed.stdout.splitlines()
|
|
if len(actual) != len(expected) or actual != expected:
|
|
raise DependencyError("tracked source content differs from the locked tree")
|
|
|
|
|
|
def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean: bool) -> None:
|
|
if not (source / ".git").exists():
|
|
raise DependencyError(f"not a materialized git checkout: {source}")
|
|
if _git(source, "rev-parse", "HEAD") != lock["commit"]:
|
|
raise DependencyError("upstream drift: checkout HEAD does not equal the locked commit")
|
|
if _git(source, "rev-parse", "HEAD^{tree}") != lock["commit_tree"]:
|
|
raise DependencyError("upstream drift: checkout tree does not equal the locked tree")
|
|
if _git(source, "rev-parse", "--abbrev-ref", "HEAD") != "HEAD":
|
|
raise DependencyError("materialized llama.cpp checkout must have a detached HEAD")
|
|
if require_clean:
|
|
_verify_tracked_content(source, lock)
|
|
tracked = _git(source, "status", "--porcelain", "--untracked-files=no")
|
|
untracked = _git(source, "ls-files", "--others", "--exclude-standard")
|
|
ignored = _git(source, "ls-files", "--others", "--ignored", "--exclude-standard")
|
|
if tracked or untracked or ignored:
|
|
raise DependencyError("local edits or unmanifested files detected in materialized llama.cpp checkout")
|
|
for relative, expected in lock["required_upstream_blobs"].items():
|
|
actual = _git(source, "rev-parse", f"HEAD:{relative}")
|
|
if actual != expected:
|
|
raise DependencyError(
|
|
f"upstream ABI/context drift for {relative}: expected {expected}, got {actual}"
|
|
)
|
|
if not (source / "LICENSE").is_file():
|
|
raise DependencyError("upstream LICENSE is missing; refusing to drop required attribution")
|
|
|
|
|
|
def _workspace_source(workspace: pathlib.Path, lock: dict[str, Any]) -> pathlib.Path:
|
|
relative = pathlib.Path(lock["retrieval"]["workspace"])
|
|
expected = (ROOT / relative).absolute()
|
|
supplied = workspace.absolute()
|
|
if supplied != expected:
|
|
raise DependencyError(f"--workspace must equal the locked ignored build root: {expected}")
|
|
cursor = ROOT
|
|
for part in relative.parts:
|
|
cursor = cursor / part
|
|
if cursor.is_symlink():
|
|
raise DependencyError(f"locked build workspace may not traverse a symlink: {cursor}")
|
|
resolved = workspace.resolve()
|
|
try:
|
|
resolved.relative_to(ROOT.resolve())
|
|
except ValueError as error:
|
|
raise DependencyError("locked build workspace escapes the repository root") from error
|
|
source = resolved / "source"
|
|
if source.is_symlink():
|
|
raise DependencyError(f"locked source checkout may not be a symlink: {source}")
|
|
return source
|
|
|
|
|
|
def fetch(workspace: pathlib.Path) -> pathlib.Path:
|
|
"""Fetch once, or verify an exact clean cached checkout for offline reuse."""
|
|
lock = _load_lock()
|
|
_patches(lock)
|
|
source = _workspace_source(workspace, lock)
|
|
if source.exists():
|
|
_verify_source(source, lock, require_clean=True)
|
|
print(f"reused verified offline cache: {source}")
|
|
return source
|
|
source.parent.mkdir(parents=True, exist_ok=True)
|
|
_run("git", "clone", "--no-checkout", lock["upstream"], str(source))
|
|
_git(source, "checkout", "--detach", lock["commit"])
|
|
_verify_source(source, lock, require_clean=True)
|
|
print(f"fetched and verified exact source: {source}")
|
|
return source
|
|
|
|
|
|
def apply(source: pathlib.Path) -> None:
|
|
lock = _load_lock()
|
|
patches = _patches(lock)
|
|
assumptions = _assumptions(lock, patches)
|
|
_verify_patch_stack_boundaries(patches)
|
|
_verify_source(source, lock, require_clean=True)
|
|
for patch in patches:
|
|
files = assumptions["patches"][patch.name]["files"]
|
|
_check_assumption_blobs(source, patch, files, "before")
|
|
_git(source, "apply", "--check", str(patch))
|
|
_git(source, "apply", "--index", str(patch))
|
|
_check_assumption_blobs(source, patch, files, "after")
|
|
_verify_patched_source(source, lock)
|
|
|
|
|
|
def reverse(source: pathlib.Path) -> None:
|
|
"""Reverse the complete verified stack and recover the exact clean pin."""
|
|
lock = _load_lock()
|
|
patches = _patches(lock)
|
|
assumptions = _assumptions(lock, patches)
|
|
_verify_patch_stack_boundaries(patches)
|
|
_verify_source(source, lock, require_clean=False)
|
|
_verify_patched_source(source, lock)
|
|
for patch in reversed(patches):
|
|
files = assumptions["patches"][patch.name]["files"]
|
|
_check_assumption_blobs(source, patch, files, "after")
|
|
_git(source, "apply", "--reverse", "--check", str(patch))
|
|
_git(source, "apply", "--reverse", "--index", str(patch))
|
|
_check_assumption_blobs(source, patch, files, "before")
|
|
_verify_source(source, lock, require_clean=True)
|
|
|
|
|
|
def _verify_patched_source(source: pathlib.Path, lock: dict[str, Any]) -> None:
|
|
changed_paths = _git(source, "diff", "--cached", "--name-only").splitlines()
|
|
if changed_paths != lock["patched_paths"]:
|
|
raise DependencyError(f"patched paths drifted: expected {lock['patched_paths']}, got {changed_paths}")
|
|
if _git(source, "write-tree") != lock["patched_tree"]:
|
|
raise DependencyError("patched source tree differs from the locked patch stack")
|
|
if _git(source, "diff", "--name-only"):
|
|
raise DependencyError("local unstaged edits detected after applying patch stack")
|
|
untracked = _git(source, "ls-files", "--others", "--exclude-standard").splitlines()
|
|
if untracked:
|
|
raise DependencyError(f"untracked files detected after applying patch stack: {untracked}")
|
|
|
|
|
|
def build(source: pathlib.Path, build_dir: pathlib.Path) -> pathlib.Path:
|
|
lock = _load_lock()
|
|
_patches(lock)
|
|
_verify_source(source, lock, require_clean=False)
|
|
_verify_patched_source(source, lock)
|
|
expected_marker = source / "cmake/meshnet-patch-stack.cmake"
|
|
if not expected_marker.is_file():
|
|
raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent")
|
|
if build_dir.exists():
|
|
raise DependencyError(f"build directory already exists; use reproduce for a clean rebuild: {build_dir}")
|
|
flags = lock["build"]["configure_flags"]
|
|
cmake = _cmake()
|
|
_run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags)
|
|
for target in lock["build"]["native_targets"]:
|
|
_run(cmake, "--build", str(build_dir), "--target", target, "-j2")
|
|
metadata = {
|
|
"commit": lock["commit"],
|
|
"commit_tree": lock["commit_tree"],
|
|
"patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)},
|
|
"configure_flags": flags,
|
|
"cmake": _run(cmake, "--version").splitlines()[0],
|
|
"cxx": _run("c++", "--version").splitlines()[0],
|
|
"model_downloads": False,
|
|
"semantic_certification": False,
|
|
}
|
|
(build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
|
|
return build_dir / lock["build"]["smoke_binary"]
|
|
|
|
|
|
def smoke(binary: pathlib.Path) -> None:
|
|
if not binary.is_file():
|
|
raise DependencyError(f"native smoke binary is missing: {binary}")
|
|
smoke_config = _load_lock()["build"]
|
|
output = _run(str(binary), *smoke_config["smoke_args"])
|
|
expected = smoke_config["smoke_output_token"]
|
|
if expected not in output.lower():
|
|
raise DependencyError(f"native smoke output did not contain {expected!r}: {output}")
|
|
print(output)
|
|
|
|
|
|
def ctest_lane(build_dir: pathlib.Path) -> None:
|
|
"""Run the deterministic model-free CPU CTest lane and print its output."""
|
|
lock = _load_lock()
|
|
regex = lock["build"]["ctest_regex"]
|
|
print(_run(_ctest(), "--test-dir", str(build_dir), "-R", regex, "--output-on-failure"))
|
|
|
|
|
|
def _sdk_probe(probe: dict[str, Any]) -> str | None:
|
|
"""Resolve one accelerator lane's SDK binary, or None if it is unavailable."""
|
|
platform_only = probe.get("platform_only")
|
|
if platform_only and sys.platform != platform_only:
|
|
return None
|
|
env_var = probe.get("env_var")
|
|
if env_var:
|
|
override = os.environ.get(env_var)
|
|
if override:
|
|
return override
|
|
return shutil.which(probe["binary"])
|
|
|
|
|
|
def accelerator_status(name: str, lock: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
"""Report whether lane `name`'s SDK is present, never raising for absence.
|
|
|
|
This is the single source of truth for DGR-030's "unavailable/skipped, not
|
|
false success" contract: absence is reported as data, not swallowed and
|
|
not escalated into a build attempt.
|
|
"""
|
|
lock = lock if lock is not None else _load_lock()
|
|
presets = lock.get("accelerator_presets", {})
|
|
if name not in presets:
|
|
raise DependencyError(f"unknown accelerator lane: {name}")
|
|
probe = presets[name]["sdk_probe"]
|
|
resolved = _sdk_probe(probe)
|
|
if resolved is None:
|
|
platform_only = probe.get("platform_only")
|
|
if platform_only and sys.platform != platform_only:
|
|
reason = f"platform {sys.platform!r} is not {platform_only!r}"
|
|
else:
|
|
reason = f"{probe['binary']} is unavailable on PATH"
|
|
return {"lane": name, "available": False, "reason": reason}
|
|
return {"lane": name, "available": True, "sdk_binary": resolved}
|
|
|
|
|
|
def accelerator_configure_flags(lock: dict[str, Any], name: str) -> list[str]:
|
|
"""The CPU default's configure flags with exactly one backend flag flipped ON.
|
|
|
|
Returns a new list; `lock["build"]["configure_flags"]` (the deterministic
|
|
CPU default DGR-029 locked) is never mutated.
|
|
"""
|
|
presets = lock.get("accelerator_presets", {})
|
|
if name not in presets:
|
|
raise DependencyError(f"unknown accelerator lane: {name}")
|
|
backend_flag = presets[name]["backend_flag"]
|
|
target = f"-D{backend_flag}="
|
|
flags: list[str] = []
|
|
replaced = False
|
|
for flag in lock["build"]["configure_flags"]:
|
|
if flag.startswith(target):
|
|
flags.append(f"-D{backend_flag}=ON")
|
|
replaced = True
|
|
else:
|
|
flags.append(flag)
|
|
if not replaced:
|
|
raise DependencyError(f"accelerator lane {name} backend flag {backend_flag} is not a locked base flag")
|
|
return flags
|
|
|
|
|
|
def accelerator_build(source: pathlib.Path, name: str, build_dir: pathlib.Path) -> pathlib.Path:
|
|
"""Compile lane `name` into its own out-of-tree directory. Compile-only.
|
|
|
|
This never runs `smoke`/`ctest_lane`: exercising a binary linked against an
|
|
accelerator backend would touch real hardware, and DGR-030 keeps every
|
|
backend/model/recipe lane registered-dark (compiled, never certified)
|
|
until a separate real-hardware certification record exists.
|
|
"""
|
|
lock = _load_lock()
|
|
_patches(lock)
|
|
_verify_source(source, lock, require_clean=False)
|
|
_verify_patched_source(source, lock)
|
|
expected_marker = source / "cmake/meshnet-patch-stack.cmake"
|
|
if not expected_marker.is_file():
|
|
raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent")
|
|
if build_dir.exists():
|
|
raise DependencyError(f"accelerator build directory already exists; use a clean build dir: {build_dir}")
|
|
status = accelerator_status(name, lock)
|
|
if not status["available"]:
|
|
raise DependencyError(f"accelerator lane {name} SDK is unavailable: {status['reason']}")
|
|
flags = accelerator_configure_flags(lock, name)
|
|
cmake = _cmake()
|
|
_run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags)
|
|
for target in lock["build"]["native_targets"]:
|
|
_run(cmake, "--build", str(build_dir), "--target", target, "-j2")
|
|
metadata = {
|
|
"lane": name,
|
|
"backend_flag": lock["accelerator_presets"][name]["backend_flag"],
|
|
"commit": lock["commit"],
|
|
"commit_tree": lock["commit_tree"],
|
|
"patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)},
|
|
"configure_flags": flags,
|
|
"cmake": _run(cmake, "--version").splitlines()[0],
|
|
"cxx": _run("c++", "--version").splitlines()[0],
|
|
"sdk_binary": status["sdk_binary"],
|
|
"model_downloads": False,
|
|
"hardware_execution": False,
|
|
"hardware_certified": False,
|
|
"semantic_certification": False,
|
|
"note": (
|
|
"compiled only; no accelerator device was exercised or driven. "
|
|
"Backend/model/recipe capability remains registered-dark until a "
|
|
"separate real-hardware certification record exists (see "
|
|
"DGR-041/053/067)."
|
|
),
|
|
}
|
|
(build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
|
|
return build_dir
|
|
|
|
|
|
def verify(workspace: pathlib.Path) -> None:
|
|
"""Apply, verify, reverse, and leave the exact cached pin pristine."""
|
|
source = fetch(workspace)
|
|
apply(source)
|
|
reverse(source)
|
|
|
|
|
|
def reproduce(workspace: pathlib.Path) -> None:
|
|
"""Configure, build, smoke-test, and CTest a clean checkout, then restore the pristine cache."""
|
|
source = fetch(workspace)
|
|
build_dir = workspace.resolve() / "build"
|
|
if build_dir.exists():
|
|
raise DependencyError(f"build directory already exists; refusing to erase possible local edits: {build_dir}")
|
|
apply(source)
|
|
binary = build(source, build_dir)
|
|
smoke(binary)
|
|
ctest_lane(build_dir)
|
|
reverse(source)
|
|
|
|
|
|
def inspect() -> None:
|
|
lock = _load_lock()
|
|
patches = _patches(lock)
|
|
print(json.dumps({
|
|
"commit": lock["commit"],
|
|
"commit_tree": lock["commit_tree"],
|
|
"retrieval": lock["retrieval"],
|
|
"upstream_license": lock["upstream_license"],
|
|
"patch_count": len(patches),
|
|
"patches": [patch.name for patch in patches],
|
|
"model_downloads": False,
|
|
"semantic_certification": False,
|
|
"glm_stock_limitations": lock["stock_glm_limitations"],
|
|
}, indent=2, sort_keys=True))
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
subcommands = parser.add_subparsers(dest="command", required=True)
|
|
subcommands.add_parser("inspect")
|
|
fetch_parser = subcommands.add_parser("fetch")
|
|
fetch_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
|
|
apply_parser = subcommands.add_parser("apply")
|
|
apply_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
|
reverse_parser = subcommands.add_parser("reverse")
|
|
reverse_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
|
verify_parser = subcommands.add_parser("verify")
|
|
verify_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
|
|
build_parser = subcommands.add_parser("build")
|
|
build_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
|
build_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
|
|
smoke_parser = subcommands.add_parser("smoke")
|
|
smoke_parser.add_argument("--binary", type=pathlib.Path, required=True)
|
|
ctest_parser = subcommands.add_parser("ctest")
|
|
ctest_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
|
|
accel_status_parser = subcommands.add_parser("accelerator-status")
|
|
accel_status_parser.add_argument("--name", required=True)
|
|
accel_build_parser = subcommands.add_parser("accelerator-build")
|
|
accel_build_parser.add_argument("--name", required=True)
|
|
accel_build_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
|
accel_build_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
|
|
reproduce_parser = subcommands.add_parser("reproduce")
|
|
reproduce_parser.add_argument("--workspace", type=pathlib.Path, default=ROOT / "build/llama.cpp")
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.command == "inspect":
|
|
inspect()
|
|
elif args.command == "fetch":
|
|
fetch(args.workspace)
|
|
elif args.command == "apply":
|
|
apply(args.source_dir)
|
|
elif args.command == "reverse":
|
|
reverse(args.source_dir)
|
|
elif args.command == "verify":
|
|
verify(args.workspace)
|
|
elif args.command == "build":
|
|
build(args.source_dir, args.build_dir)
|
|
elif args.command == "smoke":
|
|
smoke(args.binary)
|
|
elif args.command == "ctest":
|
|
ctest_lane(args.build_dir)
|
|
elif args.command == "accelerator-status":
|
|
print(json.dumps(accelerator_status(args.name), indent=2, sort_keys=True))
|
|
elif args.command == "accelerator-build":
|
|
accelerator_build(args.source_dir, args.name, args.build_dir)
|
|
else:
|
|
reproduce(args.workspace)
|
|
except DependencyError as error:
|
|
print(f"DGR-027 dependency error: {error}", file=sys.stderr)
|
|
return 2
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|