Merge branch 'archived_ralph/dgr-001-performance-contract' into merge/all-branches-into-master
# Conflicts: # .claude/memory/MEMORY.md # .scratch/distributed-gguf-runtime/PRD.md # .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md # .scratch/distributed-gguf-runtime/README.md # .scratch/distributed-gguf-runtime/architecture.md # .scratch/distributed-gguf-runtime/evidence/DGR-017/README.md # .scratch/distributed-gguf-runtime/implementation-strategy.md # .scratch/distributed-gguf-runtime/issues/07-add-isolated-concurrent-local-hot-kv-state.md # .scratch/distributed-gguf-runtime/issues/13-harden-failure-cancellation-and-restart-semantics.md # .scratch/distributed-gguf-runtime/milestones.md # .scratch/distributed-gguf-runtime/prd.json # docs/issues/distributed-gguf-runtime/01-lock-the-safetensors-versus-gguf-performance-contract.md # docs/issues/distributed-gguf-runtime/02-adopt-the-versioned-grpc-shard-protocol.md # docs/issues/distributed-gguf-runtime/03-define-exact-artifact-and-runtime-recipe-identity.md # docs/issues/distributed-gguf-runtime/05-implement-dense-llama-range-aware-gguf-ownership.md # docs/issues/distributed-gguf-runtime/06-implement-architecture-defined-boundary-input-output.md
This commit is contained in:
Dobromir Popov
66
scripts/bootstrap_native_toolchain.sh
Normal file
66
scripts/bootstrap_native_toolchain.sh
Normal file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build a protobuf C++ toolchain for the native Shard protocol.
|
||||
#
|
||||
# The Python side needs nothing beyond `pip install grpcio-tools` — it bundles
|
||||
# protoc. The C++ side needs libprotobuf headers and a protoc binary, and a
|
||||
# machine that has neither (no protobuf-devel, no cmake, no system protoc) can
|
||||
# still get a working one from source with this script. It is the exact recipe
|
||||
# DGR-002 used to build and run the C++ conformance test.
|
||||
#
|
||||
# gRPC C++ is deliberately NOT built here. The conformance test only needs
|
||||
# message types, so verifying the schema does not require the whole gRPC stack.
|
||||
# The worker (DGR-008) will need gRPC C++ and should extend this script then.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/bootstrap_native_toolchain.sh [install-prefix]
|
||||
#
|
||||
# Then:
|
||||
# cmake -S packages/node/native -B build/native -DCMAKE_PREFIX_PATH=<prefix>
|
||||
# cmake --build build/native -j
|
||||
# ctest --test-dir build/native --output-on-failure
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PREFIX="${1:-${PWD}/build/native-toolchain}"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "${WORK}"' EXIT
|
||||
|
||||
# Pinned: the C++ runtime a generated stub is compiled against must be a version
|
||||
# that stub is allowed to use, so these are exact, not floating.
|
||||
PROTOBUF_VERSION="33.1"
|
||||
ABSEIL_VERSION="20250814.1"
|
||||
|
||||
command -v cmake >/dev/null || {
|
||||
echo "cmake is required (pip install cmake==4.4.0)" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "--- fetching protobuf ${PROTOBUF_VERSION} and abseil ${ABSEIL_VERSION}"
|
||||
cd "${WORK}"
|
||||
curl -sfL -o protobuf.tar.gz \
|
||||
"https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz"
|
||||
tar xzf protobuf.tar.gz
|
||||
|
||||
# The protobuf release tarball ships utf8_range but not abseil, and its default
|
||||
# CMake provider expects abseil as a submodule, so vendor it into place.
|
||||
curl -sfL -o abseil.tar.gz \
|
||||
"https://github.com/abseil/abseil-cpp/releases/download/${ABSEIL_VERSION}/abseil-cpp-${ABSEIL_VERSION}.tar.gz"
|
||||
tar xzf abseil.tar.gz
|
||||
rm -rf "protobuf-${PROTOBUF_VERSION}/third_party/abseil-cpp"
|
||||
mv "abseil-cpp-${ABSEIL_VERSION}" "protobuf-${PROTOBUF_VERSION}/third_party/abseil-cpp"
|
||||
|
||||
echo "--- building protobuf into ${PREFIX}"
|
||||
cmake -S "protobuf-${PROTOBUF_VERSION}" -B build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_INSTALL_PREFIX="${PREFIX}" \
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
|
||||
-Dprotobuf_ABSL_PROVIDER=module \
|
||||
-Dprotobuf_BUILD_TESTS=OFF \
|
||||
-Dprotobuf_BUILD_SHARED_LIBS=OFF \
|
||||
-DABSL_PROPAGATE_CXX_STD=ON
|
||||
cmake --build build -j"$(nproc)"
|
||||
cmake --install build
|
||||
|
||||
echo "--- done"
|
||||
"${PREFIX}/bin/protoc" --version
|
||||
echo "configure the protocol build with: -DCMAKE_PREFIX_PATH=${PREFIX}"
|
||||
153
scripts/gen_recipe_fingerprint_vectors.py
Normal file
153
scripts/gen_recipe_fingerprint_vectors.py
Normal file
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python
|
||||
"""Generate (or verify) the committed DGR-003 fingerprint conformance vectors.
|
||||
|
||||
The node and the tracker derive artifact, recipe and Shard-binding digests from
|
||||
*separate* implementations on purpose: an admission gate that shares code with
|
||||
the thing it admits is not an independent check. The cost of that independence
|
||||
is drift — two canonicalizers that quietly stop agreeing would not fail, they
|
||||
would silently stop forming routes, or worse, silently form wrong ones.
|
||||
|
||||
``tests/data/recipe_fingerprint_vectors.json`` is what makes drift loud. It is a
|
||||
language-neutral artifact — canonical input blocks, expected digests, and the
|
||||
serialized DGR-002 ``Fingerprint`` bytes — that the node tests, the tracker tests
|
||||
and (later) the native C++ worker all check themselves against.
|
||||
|
||||
python scripts/gen_recipe_fingerprint_vectors.py --check # CI: no drift
|
||||
python scripts/gen_recipe_fingerprint_vectors.py # rewrite vectors
|
||||
|
||||
Rewriting is a deliberate act: if this changes a digest, it changed the wire
|
||||
contract, and every node and tracker in the fleet has to agree at the same time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
_ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
sys.path[:0] = [str(_ROOT / "packages" / "node"), str(_ROOT / "packages" / "tracker")]
|
||||
|
||||
from meshnet_node.runtime_recipe import ( # noqa: E402
|
||||
ArtifactIdentity,
|
||||
DerivativeBinding,
|
||||
RuntimeRecipe,
|
||||
ShardIdentity,
|
||||
)
|
||||
from meshnet_tracker.recipe import parse_identity # noqa: E402
|
||||
|
||||
VECTORS = _ROOT / "tests" / "data" / "recipe_fingerprint_vectors.json"
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
_RECIPE = RuntimeRecipe(
|
||||
weight_quantization="Q4_K_M",
|
||||
activation_dtype="bfloat16",
|
||||
compute_dtype="float32",
|
||||
kv_dtype="q8_0",
|
||||
kv_layout="paged-v1",
|
||||
tokenizer_revision="0123456789abcdef",
|
||||
architecture_adapter="llama/range-v1",
|
||||
backend_id="llama.cpp",
|
||||
runtime_version="llama.cpp@deadbeef+meshnet.1",
|
||||
recipe_id="example-gguf",
|
||||
recipe_version="1",
|
||||
catalogue_version="2026.07.1",
|
||||
)
|
||||
|
||||
_SOURCE = "a" * 64
|
||||
_SPLIT_BYTES = "c" * 64
|
||||
_CONFIG = "b" * 64
|
||||
|
||||
|
||||
def _cases() -> list[tuple[str, str, ShardIdentity]]:
|
||||
whole = ShardIdentity(
|
||||
ArtifactIdentity(
|
||||
"example/model", "0123456789abcdef", _SOURCE, "dense-llama", _CONFIG, 8
|
||||
),
|
||||
_RECIPE,
|
||||
0,
|
||||
4,
|
||||
)
|
||||
# The same recipe on the same source, held as a split: identical route
|
||||
# fingerprint, different Shard binding. Both halves of that are contract.
|
||||
derivative = ShardIdentity(
|
||||
ArtifactIdentity(
|
||||
"example/model",
|
||||
"0123456789abcdef",
|
||||
_SPLIT_BYTES,
|
||||
"dense-llama",
|
||||
_CONFIG,
|
||||
8,
|
||||
DerivativeBinding(_SOURCE, 4, 8),
|
||||
),
|
||||
_RECIPE,
|
||||
4,
|
||||
8,
|
||||
)
|
||||
return [
|
||||
("example-v1", "An undivided artifact: content digest is the source digest.", whole),
|
||||
(
|
||||
"example-v1-derivative",
|
||||
"A split of the same source: same fingerprint, different Shard binding.",
|
||||
derivative,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def build() -> dict:
|
||||
vectors = []
|
||||
for name, description, identity in _cases():
|
||||
block = identity.to_dict()
|
||||
presented = parse_identity(block)
|
||||
|
||||
# The two implementations must already agree before this is committed.
|
||||
assert identity.fingerprint.to_dict() == presented.fingerprint_dict(), name
|
||||
assert identity.shard_binding_digest == presented.shard_binding_digest, name
|
||||
|
||||
vectors.append(
|
||||
{
|
||||
"name": name,
|
||||
"description": description,
|
||||
"identity": block,
|
||||
"fingerprint": identity.fingerprint.to_dict(),
|
||||
"shard_binding_digest": identity.shard_binding_digest,
|
||||
"fingerprint_proto_hex": identity.fingerprint.to_proto()
|
||||
.SerializeToString(deterministic=True)
|
||||
.hex(),
|
||||
}
|
||||
)
|
||||
return {"schema_version": SCHEMA_VERSION, "vectors": vectors}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="fail if the committed vectors differ from what this code derives",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
built = json.dumps(build(), indent=2, sort_keys=True) + "\n"
|
||||
if not args.check:
|
||||
VECTORS.write_text(built, encoding="utf-8")
|
||||
print(f"wrote {VECTORS.relative_to(_ROOT)}")
|
||||
return 0
|
||||
|
||||
committed = VECTORS.read_text(encoding="utf-8")
|
||||
if committed != built:
|
||||
print(
|
||||
f"{VECTORS.relative_to(_ROOT)} is stale: the identity implementation no "
|
||||
"longer derives the committed digests.\nIf that change was intended, it "
|
||||
"is a wire-contract change — rerun without --check and roll out node and "
|
||||
"tracker together.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print(f"{VECTORS.relative_to(_ROOT)} matches the identity implementation")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
125
scripts/generate_native_protocol.py
Normal file
125
scripts/generate_native_protocol.py
Normal file
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the Python Shard-protocol stubs from `shard_runtime.proto`.
|
||||
|
||||
The `.proto` file is the contract; the Python modules under
|
||||
`meshnet_node/native_protocol/generated/` are build output that happens to be
|
||||
committed. They are committed so that installing the node does not require a
|
||||
protoc toolchain, and `--check` exists so a committed stub can never silently
|
||||
drift from the schema it claims to implement.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/generate_native_protocol.py # regenerate in place
|
||||
python scripts/generate_native_protocol.py --check # fail if out of date
|
||||
|
||||
C++ stubs are *not* generated here. They are build artifacts produced by CMake
|
||||
(`packages/node/native/CMakeLists.txt`) into the build tree, because a C++ build
|
||||
already requires a toolchain and nothing is gained by committing them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
PROTO_DIR = REPO_ROOT / "packages/node/native/proto"
|
||||
PROTO_FILE = PROTO_DIR / "shard_runtime.proto"
|
||||
OUT_DIR = REPO_ROOT / "packages/node/meshnet_node/native_protocol/generated"
|
||||
|
||||
# Regenerating with a different protoc emits different gencode headers, so the
|
||||
# generator version is part of the contract and `--check` would catch a drift.
|
||||
REQUIRED_GRPCIO_TOOLS = "1.82.1"
|
||||
|
||||
_HEADER = "# Generated by scripts/generate_native_protocol.py. Do not edit.\n"
|
||||
|
||||
|
||||
def _generate(into: pathlib.Path) -> None:
|
||||
"""Run protoc, writing generated modules into `into`."""
|
||||
try:
|
||||
from grpc_tools import protoc
|
||||
except ImportError: # pragma: no cover - exercised only without the toolchain
|
||||
sys.exit(
|
||||
"grpc_tools is required to generate stubs:\n"
|
||||
f" pip install grpcio-tools=={REQUIRED_GRPCIO_TOOLS}"
|
||||
)
|
||||
|
||||
into.mkdir(parents=True, exist_ok=True)
|
||||
# grpc_tools bundles protoc and the well-known types, so generation needs no
|
||||
# system protoc and produces identical output on any machine.
|
||||
well_known = pathlib.Path(protoc.__file__).parent / "_proto"
|
||||
args = [
|
||||
"protoc",
|
||||
f"--proto_path={PROTO_DIR}",
|
||||
f"--proto_path={well_known}",
|
||||
f"--python_out={into}",
|
||||
f"--pyi_out={into}",
|
||||
f"--grpc_python_out={into}",
|
||||
str(PROTO_FILE),
|
||||
]
|
||||
if protoc.main(args) != 0:
|
||||
sys.exit("protoc failed")
|
||||
|
||||
# protoc emits `import shard_runtime_pb2` — a bare top-level import that only
|
||||
# resolves if the generated directory happens to be on sys.path. Rewrite it
|
||||
# to a relative import so the package works as an installed package.
|
||||
grpc_module = into / "shard_runtime_pb2_grpc.py"
|
||||
text = grpc_module.read_text()
|
||||
text = text.replace(
|
||||
"import shard_runtime_pb2 as shard__runtime__pb2",
|
||||
"from . import shard_runtime_pb2 as shard__runtime__pb2",
|
||||
)
|
||||
grpc_module.write_text(text)
|
||||
|
||||
(into / "__init__.py").write_text(
|
||||
_HEADER + '"""Generated protobuf/gRPC stubs for the native Shard protocol."""\n'
|
||||
)
|
||||
|
||||
|
||||
def _tracked_files(directory: pathlib.Path) -> dict[str, bytes]:
|
||||
return {
|
||||
path.name: path.read_bytes()
|
||||
for path in sorted(directory.iterdir())
|
||||
if path.is_file() and path.suffix in {".py", ".pyi"}
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="verify the committed stubs match the .proto instead of rewriting them",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.check:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
fresh = pathlib.Path(tmp) / "generated"
|
||||
_generate(fresh)
|
||||
if not OUT_DIR.is_dir():
|
||||
print(f"generated stubs are missing: {OUT_DIR}", file=sys.stderr)
|
||||
return 1
|
||||
if _tracked_files(fresh) != _tracked_files(OUT_DIR):
|
||||
print(
|
||||
"generated stubs are out of date with shard_runtime.proto.\n"
|
||||
"Run: python scripts/generate_native_protocol.py",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print("generated stubs are up to date")
|
||||
return 0
|
||||
|
||||
if OUT_DIR.exists():
|
||||
shutil.rmtree(OUT_DIR)
|
||||
_generate(OUT_DIR)
|
||||
print(f"wrote {OUT_DIR.relative_to(REPO_ROOT)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
74
scripts/generate_protocol_goldens.py
Normal file
74
scripts/generate_protocol_goldens.py
Normal file
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write the committed cross-language conformance vectors.
|
||||
|
||||
The bytes under `packages/node/native/testdata/` are the reference both the
|
||||
Python and the C++ conformance tests assert against. They are committed so the
|
||||
C++ test can run without a Python step, and `--check` exists so they can never
|
||||
drift from the schema unnoticed: if a schema edit changes the canonical
|
||||
message's encoding, `--check` fails and the change has to be acknowledged.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/generate_protocol_goldens.py # rewrite vectors
|
||||
python scripts/generate_protocol_goldens.py --check # fail if stale
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "packages/node"))
|
||||
|
||||
from meshnet_node.native_protocol import conformance # noqa: E402
|
||||
|
||||
|
||||
def _vectors() -> dict[str, bytes]:
|
||||
return {
|
||||
conformance.GOLDEN_SESSION_REQUEST: conformance.serialize(
|
||||
conformance.canonical_session_request()
|
||||
),
|
||||
conformance.GOLDEN_CAPABILITY_REPORT: conformance.serialize(
|
||||
conformance.canonical_capability_report()
|
||||
),
|
||||
conformance.GOLDEN_DECODE_STEP: conformance.serialize(
|
||||
conformance.canonical_decode_step()
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--check", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
out_dir = conformance.TESTDATA_DIR
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
stale = []
|
||||
|
||||
for name, payload in _vectors().items():
|
||||
path = out_dir / name
|
||||
if args.check:
|
||||
if not path.is_file() or path.read_bytes() != payload:
|
||||
stale.append(name)
|
||||
continue
|
||||
path.write_bytes(payload)
|
||||
print(f"wrote {path.relative_to(REPO_ROOT)} ({len(payload)} bytes)")
|
||||
|
||||
if stale:
|
||||
print(
|
||||
"conformance vectors are stale: " + ", ".join(stale) + "\n"
|
||||
"The canonical message no longer encodes to the committed bytes. If "
|
||||
"that is intended, run: python scripts/generate_protocol_goldens.py",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
if args.check:
|
||||
print("conformance vectors are up to date")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
280
scripts/glm_whole_model_preflight.py
Normal file
280
scripts/glm_whole_model_preflight.py
Normal file
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""DGR-018 whole-model preflight: storage, memory, and the ordered download plan.
|
||||
|
||||
Answers, before a single artifact byte moves, the three questions DGR-018's
|
||||
finish contract asks:
|
||||
|
||||
1. Is there one filesystem, outside every forbidden prefix (``/home``), with at
|
||||
least 250 GB free for the 216.715 GB ``UD-IQ1_S`` artifact plus headroom?
|
||||
2. Does this host have the 224 GiB runtime-accessible memory the whole-model
|
||||
oracle load needs (the experimental hard-fit floor from DGR-017)?
|
||||
3. In what order do the six shards download, and against which exact revision
|
||||
URL, size, and LFS SHA-256 is each one verified?
|
||||
|
||||
Everything is resolved from the pinned target manifest
|
||||
(``meshnet_node.glm_alpha``) — this script never contacts the network and never
|
||||
invents a size or digest. It fails closed: a missing requirement is a non-zero
|
||||
exit and an explicit reason, never a downgrade to a smaller target.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/glm_whole_model_preflight.py # scan all mounts
|
||||
python scripts/glm_whole_model_preflight.py --dest DIR # judge one destination
|
||||
python scripts/glm_whole_model_preflight.py --storage-only # download-host mode
|
||||
python scripts/glm_whole_model_preflight.py --json out.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_ROOT / "packages/node"))
|
||||
|
||||
from meshnet_node.glm_alpha.manifest import ( # noqa: E402
|
||||
GIB,
|
||||
TargetManifest,
|
||||
load_target_manifest,
|
||||
)
|
||||
|
||||
GB = 1000**3
|
||||
|
||||
# 250 GB: the 216.715 GB artifact plus resume/temp headroom. DGR-018's issue
|
||||
# text names this number; it is a storage requirement, not a tunable.
|
||||
REQUIRED_FREE_BYTES = 250 * GB
|
||||
|
||||
# DGR-017's experimental hard-fit floor for whole-model runtime-accessible
|
||||
# memory. 224 GiB fits weights + Q8_0 KV at 16k context with nothing to spare.
|
||||
REQUIRED_MEMORY_GIB = 224.0
|
||||
|
||||
# Download order, by shard index. Rationale:
|
||||
# 1 first — 9.4 MB split-metadata shard: proves revision URLs, auth-free
|
||||
# access, and tooling end-to-end for the cost of a rounding error.
|
||||
# 6 second — 19.2 GB, the smallest weight shard: proves resume + hash
|
||||
# verification at real scale before any ~49 GB transfer starts.
|
||||
# 2..5 — the four ~49 GB shards, sequentially, verify-after-each, so at
|
||||
# most one unverified partial ever exists on disk.
|
||||
DOWNLOAD_ORDER = (1, 6, 2, 3, 4, 5)
|
||||
|
||||
# Filesystems that can hold a 49 GB file and survive a resume. FAT variants are
|
||||
# excluded by omission (4 GiB file limit); network/pseudo filesystems likewise.
|
||||
_USABLE_FSTYPES = {"ext4", "ext3", "xfs", "btrfs", "ntfs", "ntfs3", "fuseblk", "exfat", "f2fs"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MountCandidate:
|
||||
mountpoint: str
|
||||
fstype: str
|
||||
total_bytes: int
|
||||
free_bytes: int
|
||||
forbidden: bool
|
||||
|
||||
@property
|
||||
def eligible(self) -> bool:
|
||||
return not self.forbidden and self.free_bytes >= REQUIRED_FREE_BYTES
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"mountpoint": self.mountpoint,
|
||||
"fstype": self.fstype,
|
||||
"total_gb": round(self.total_bytes / GB, 1),
|
||||
"free_gb": round(self.free_bytes / GB, 1),
|
||||
"free_bytes": self.free_bytes,
|
||||
"forbidden": self.forbidden,
|
||||
"eligible": self.eligible,
|
||||
}
|
||||
|
||||
|
||||
def forbidden_prefixes(manifest: TargetManifest) -> tuple[str, ...]:
|
||||
"""The storage policy pinned in the manifest itself — not a script opinion."""
|
||||
storage = manifest.raw.get("storage", {})
|
||||
return tuple(storage.get("forbidden_path_prefixes", ("/home",)))
|
||||
|
||||
|
||||
def is_forbidden(path: str, prefixes: tuple[str, ...]) -> bool:
|
||||
resolved = os.path.realpath(path)
|
||||
return any(resolved == p or resolved.startswith(p.rstrip("/") + "/") for p in prefixes)
|
||||
|
||||
|
||||
def _statvfs_free(path: str) -> tuple[int, int]:
|
||||
st = os.statvfs(path)
|
||||
return st.f_frsize * st.f_blocks, st.f_frsize * st.f_bavail
|
||||
|
||||
|
||||
def scan_mounts(
|
||||
prefixes: tuple[str, ...], mounts_file: str = "/proc/mounts"
|
||||
) -> list[MountCandidate]:
|
||||
"""Every real, writable-class filesystem on the host, deduplicated by device."""
|
||||
candidates: dict[str, MountCandidate] = {}
|
||||
with open(mounts_file, encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
fields = line.split()
|
||||
if len(fields) < 3:
|
||||
continue
|
||||
device, mountpoint, fstype = fields[0], fields[1], fields[2]
|
||||
mountpoint = mountpoint.replace("\\040", " ")
|
||||
if fstype not in _USABLE_FSTYPES or device in candidates:
|
||||
continue
|
||||
try:
|
||||
total, free = _statvfs_free(mountpoint)
|
||||
except OSError:
|
||||
continue
|
||||
candidates[device] = MountCandidate(
|
||||
mountpoint=mountpoint,
|
||||
fstype=fstype,
|
||||
total_bytes=total,
|
||||
free_bytes=free,
|
||||
forbidden=is_forbidden(mountpoint, prefixes),
|
||||
)
|
||||
return sorted(candidates.values(), key=lambda c: c.free_bytes, reverse=True)
|
||||
|
||||
|
||||
def judge_destination(dest: str, prefixes: tuple[str, ...]) -> MountCandidate:
|
||||
total, free = _statvfs_free(dest)
|
||||
return MountCandidate(
|
||||
mountpoint=dest,
|
||||
fstype="(as-given)",
|
||||
total_bytes=total,
|
||||
free_bytes=free,
|
||||
forbidden=is_forbidden(dest, prefixes),
|
||||
)
|
||||
|
||||
|
||||
def memory_total_gib(meminfo_file: str = "/proc/meminfo") -> float:
|
||||
with open(meminfo_file, encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
if line.startswith("MemTotal:"):
|
||||
return int(line.split()[1]) * 1024 / GIB
|
||||
raise RuntimeError(f"MemTotal not found in {meminfo_file}")
|
||||
|
||||
|
||||
def download_plan(manifest: TargetManifest, dest_dir: str | None) -> list[dict]:
|
||||
"""The six shards in download order, each bound to its pinned identity."""
|
||||
steps = []
|
||||
for position, index in enumerate(DOWNLOAD_ORDER, start=1):
|
||||
shard = manifest.shard(index)
|
||||
target = f"{dest_dir or '$GLM_DEST'}/{shard.path}"
|
||||
steps.append(
|
||||
{
|
||||
"step": position,
|
||||
"shard_index": shard.index,
|
||||
"path": shard.path,
|
||||
"size_bytes": shard.size_bytes,
|
||||
"size_gb": round(shard.size_bytes / GB, 3),
|
||||
"sha256": shard.sha256,
|
||||
"url": shard.url,
|
||||
"download_command": f'curl -L -C - --fail -o "{target}" "{shard.url}"',
|
||||
"verify_command": (
|
||||
f"python scripts/verify_glm_shards.py "
|
||||
f'--model-dir "{dest_dir or "$GLM_DEST"}" --shard {shard.index}'
|
||||
),
|
||||
}
|
||||
)
|
||||
return steps
|
||||
|
||||
|
||||
def build_report(*, dest: str | None, storage_only: bool) -> dict:
|
||||
manifest = load_target_manifest()
|
||||
prefixes = forbidden_prefixes(manifest)
|
||||
|
||||
if dest is not None:
|
||||
mounts = [judge_destination(dest, prefixes)]
|
||||
else:
|
||||
mounts = scan_mounts(prefixes)
|
||||
eligible = [m for m in mounts if m.eligible]
|
||||
chosen = eligible[0] if eligible else None
|
||||
|
||||
mem_gib = memory_total_gib()
|
||||
storage_pass = chosen is not None
|
||||
memory_pass = mem_gib >= REQUIRED_MEMORY_GIB
|
||||
|
||||
checks = [
|
||||
{
|
||||
"check": "storage",
|
||||
"requirement": f">= {REQUIRED_FREE_BYTES / GB:.0f} GB free on one filesystem "
|
||||
f"outside {list(prefixes)}",
|
||||
"observed": (
|
||||
f"{chosen.free_bytes / GB:.1f} GB free at {chosen.mountpoint}"
|
||||
if chosen
|
||||
else "no eligible filesystem"
|
||||
),
|
||||
"passes": storage_pass,
|
||||
},
|
||||
{
|
||||
"check": "memory",
|
||||
"requirement": f">= {REQUIRED_MEMORY_GIB:.0f} GiB runtime-accessible memory "
|
||||
"(DGR-017 experimental hard-fit floor)",
|
||||
"observed": f"{mem_gib:.1f} GiB MemTotal",
|
||||
"passes": memory_pass,
|
||||
"waived": storage_only,
|
||||
},
|
||||
]
|
||||
overall = storage_pass and (memory_pass or storage_only)
|
||||
|
||||
return {
|
||||
"generated_by": "scripts/glm_whole_model_preflight.py",
|
||||
"target": {
|
||||
"gguf_repo_id": manifest.gguf_repo_id,
|
||||
"gguf_revision": manifest.gguf_revision,
|
||||
"quantization": manifest.quantization,
|
||||
"shard_count": len(manifest.shards),
|
||||
"total_bytes": manifest.total_bytes,
|
||||
"total_gb": round(manifest.total_gb, 3),
|
||||
"manifest_sha256": manifest.digest,
|
||||
},
|
||||
"forbidden_path_prefixes": list(prefixes),
|
||||
"mounts": [m.to_dict() for m in mounts],
|
||||
"chosen_destination": chosen.to_dict() if chosen else None,
|
||||
"checks": checks,
|
||||
"download_authorized": overall,
|
||||
"storage_only": storage_only,
|
||||
"download_plan": download_plan(manifest, chosen.mountpoint if chosen else None),
|
||||
"verdict": "pass" if overall else "fail",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--dest", help="judge this destination directory instead of scanning")
|
||||
parser.add_argument(
|
||||
"--storage-only",
|
||||
action="store_true",
|
||||
help="download-host mode: require storage, report memory without gating on it",
|
||||
)
|
||||
parser.add_argument("--json", dest="json_path", help="also write the full report here")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = build_report(dest=args.dest, storage_only=args.storage_only)
|
||||
|
||||
if args.json_path:
|
||||
Path(args.json_path).write_text(
|
||||
json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
target = report["target"]
|
||||
print(f"target: {target['quantization']} {target['total_gb']} GB, "
|
||||
f"{target['shard_count']} shards @ {target['gguf_revision'][:12]}")
|
||||
for check in report["checks"]:
|
||||
status = "PASS" if check["passes"] else ("WAIVED" if check.get("waived") else "FAIL")
|
||||
print(f"[{status}] {check['check']}: need {check['requirement']}; "
|
||||
f"observed {check['observed']}")
|
||||
if report["chosen_destination"]:
|
||||
print(f"destination: {report['chosen_destination']['mountpoint']}")
|
||||
else:
|
||||
print("destination: NONE — no filesystem outside "
|
||||
f"{report['forbidden_path_prefixes']} has "
|
||||
f"{REQUIRED_FREE_BYTES / GB:.0f} GB free")
|
||||
for mount in report["mounts"]:
|
||||
why = "forbidden prefix" if mount["forbidden"] else f"{mount['free_gb']} GB free"
|
||||
print(f" - {mount['mountpoint']} ({mount['fstype']}): {why}")
|
||||
print(f"verdict: {report['verdict']}")
|
||||
return 0 if report["download_authorized"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
268
scripts/llama_cpp_dependency.py
Normal file
268
scripts/llama_cpp_dependency.py
Normal file
@@ -0,0 +1,268 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Materialize, verify, build, and smoke-test DGR-004's llama.cpp pin.
|
||||
|
||||
This tool deliberately owns only a source dependency boundary. It never
|
||||
downloads a model, invokes inference, or interprets generated text.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
LLAMA_DIR = ROOT / "packages/node/native/llama"
|
||||
LOCK_PATH = LLAMA_DIR / "UPSTREAM_LOCK.json"
|
||||
PATCH_DIR = LLAMA_DIR / "patches"
|
||||
|
||||
|
||||
def _cmake() -> str:
|
||||
"""Use an explicit override, PATH, or the active Python environment."""
|
||||
configured = os.environ.get("CMAKE")
|
||||
if configured:
|
||||
return configured
|
||||
on_path = shutil.which("cmake")
|
||||
if on_path:
|
||||
return on_path
|
||||
sibling = pathlib.Path(sys.executable).parent / "cmake"
|
||||
if sibling.is_file():
|
||||
return str(sibling)
|
||||
raise DependencyError("cmake is unavailable; set CMAKE or activate the project toolchain")
|
||||
|
||||
|
||||
class DependencyError(RuntimeError):
|
||||
"""A fail-closed reproducibility or source-integrity failure."""
|
||||
|
||||
|
||||
def _run(*args: str, cwd: pathlib.Path | None = None) -> str:
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
args,
|
||||
cwd=cwd,
|
||||
check=True,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
except FileNotFoundError as error:
|
||||
raise DependencyError(f"required executable is unavailable: {args[0]}") from error
|
||||
except subprocess.CalledProcessError as error:
|
||||
detail = (error.stderr or error.stdout).strip()
|
||||
raise DependencyError(f"command failed: {' '.join(args)}\n{detail}") from error
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
def _load_lock() -> dict[str, Any]:
|
||||
try:
|
||||
lock = json.loads(LOCK_PATH.read_text())
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise DependencyError(f"invalid upstream lock: {LOCK_PATH}: {error}") from error
|
||||
required = {
|
||||
"upstream", "commit", "commit_tree", "patched_tree", "patch_series",
|
||||
"required_upstream_blobs", "patched_paths", "build",
|
||||
}
|
||||
missing = sorted(required - lock.keys())
|
||||
if missing:
|
||||
raise DependencyError(f"upstream lock is missing fields: {', '.join(missing)}")
|
||||
commit_file = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip()
|
||||
if lock["commit"] != commit_file or len(commit_file) != 40:
|
||||
raise DependencyError("UPSTREAM_COMMIT and UPSTREAM_LOCK.json do not agree on a full commit")
|
||||
return lock
|
||||
|
||||
|
||||
def _patches(lock: dict[str, Any]) -> list[pathlib.Path]:
|
||||
series = [line for line in (PATCH_DIR / "series").read_text().splitlines() if line]
|
||||
if series != lock["patch_series"] or series != sorted(series) or not series:
|
||||
raise DependencyError("patches/series is empty, unordered, or disagrees with UPSTREAM_LOCK.json")
|
||||
sums: dict[str, str] = {}
|
||||
for line in (PATCH_DIR / "SHA256SUMS").read_text().splitlines():
|
||||
if line and not line.startswith("#"):
|
||||
digest, name = line.split(maxsplit=1)
|
||||
sums[name] = digest
|
||||
patches: list[pathlib.Path] = []
|
||||
for name in series:
|
||||
patch = PATCH_DIR / name
|
||||
if not patch.is_file():
|
||||
raise DependencyError(f"patch listed in series is missing: {name}")
|
||||
actual = hashlib.sha256(patch.read_bytes()).hexdigest()
|
||||
if sums.get(name) != actual:
|
||||
raise DependencyError(f"patch digest mismatch for {name}: expected {sums.get(name)}, got {actual}")
|
||||
patches.append(patch)
|
||||
return patches
|
||||
|
||||
|
||||
def _git(source: pathlib.Path, *args: str) -> str:
|
||||
return _run("git", "-C", str(source), *args)
|
||||
|
||||
|
||||
def _verify_source(source: pathlib.Path, lock: dict[str, Any], *, require_clean: bool) -> None:
|
||||
if not (source / ".git").exists():
|
||||
raise DependencyError(f"not a materialized git checkout: {source}")
|
||||
if _git(source, "rev-parse", "HEAD") != lock["commit"]:
|
||||
raise DependencyError("upstream drift: checkout HEAD does not equal the locked commit")
|
||||
if _git(source, "rev-parse", "HEAD^{tree}") != lock["commit_tree"]:
|
||||
raise DependencyError("upstream drift: checkout tree does not equal the locked tree")
|
||||
if require_clean and _git(source, "status", "--porcelain"):
|
||||
raise DependencyError("local edits detected in materialized llama.cpp checkout")
|
||||
for relative, expected in lock["required_upstream_blobs"].items():
|
||||
actual = _git(source, "rev-parse", f"HEAD:{relative}")
|
||||
if actual != expected:
|
||||
raise DependencyError(
|
||||
f"upstream ABI/context drift for {relative}: expected {expected}, got {actual}"
|
||||
)
|
||||
if not (source / "LICENSE").is_file():
|
||||
raise DependencyError("upstream LICENSE is missing; refusing to drop required attribution")
|
||||
|
||||
|
||||
def materialize(source: pathlib.Path, repository: str) -> None:
|
||||
lock = _load_lock()
|
||||
_patches(lock)
|
||||
if source.exists():
|
||||
raise DependencyError(f"destination already exists; refusing to reuse possibly edited source: {source}")
|
||||
source.parent.mkdir(parents=True, exist_ok=True)
|
||||
_run("git", "clone", "--no-checkout", repository, str(source))
|
||||
_git(source, "checkout", "--detach", lock["commit"])
|
||||
_verify_source(source, lock, require_clean=True)
|
||||
|
||||
|
||||
def apply(source: pathlib.Path) -> None:
|
||||
lock = _load_lock()
|
||||
patches = _patches(lock)
|
||||
_verify_source(source, lock, require_clean=True)
|
||||
for patch in patches:
|
||||
_git(source, "apply", "--check", str(patch))
|
||||
_git(source, "apply", "--index", str(patch))
|
||||
_verify_patched_source(source, lock)
|
||||
|
||||
|
||||
def _verify_patched_source(source: pathlib.Path, lock: dict[str, Any]) -> None:
|
||||
if _git(source, "diff", "--quiet"):
|
||||
raise DependencyError("local unstaged edits detected after applying patch stack")
|
||||
changed_paths = _git(source, "diff", "--cached", "--name-only").splitlines()
|
||||
if changed_paths != lock["patched_paths"]:
|
||||
raise DependencyError(f"patched paths drifted: expected {lock['patched_paths']}, got {changed_paths}")
|
||||
if _git(source, "write-tree") != lock["patched_tree"]:
|
||||
raise DependencyError("patched source tree differs from the locked patch stack")
|
||||
if _git(source, "diff", "--name-only"):
|
||||
raise DependencyError("local unstaged edits detected after applying patch stack")
|
||||
untracked = _git(source, "ls-files", "--others", "--exclude-standard").splitlines()
|
||||
if untracked:
|
||||
raise DependencyError(f"untracked files detected after applying patch stack: {untracked}")
|
||||
|
||||
|
||||
def build(source: pathlib.Path, build_dir: pathlib.Path) -> pathlib.Path:
|
||||
lock = _load_lock()
|
||||
_patches(lock)
|
||||
_verify_source(source, lock, require_clean=False)
|
||||
_verify_patched_source(source, lock)
|
||||
expected_marker = source / "cmake/meshnet-patch-stack.cmake"
|
||||
if not expected_marker.is_file():
|
||||
raise DependencyError("patch stack is not applied: Meshnet CMake marker is absent")
|
||||
if build_dir.exists():
|
||||
raise DependencyError(f"build directory already exists; use reproduce for a clean rebuild: {build_dir}")
|
||||
flags = lock["build"]["configure_flags"]
|
||||
cmake = _cmake()
|
||||
_run(cmake, "-G", lock["build"]["generator"], "-S", str(source), "-B", str(build_dir), *flags)
|
||||
for target in lock["build"]["native_targets"]:
|
||||
_run(cmake, "--build", str(build_dir), "--target", target, "-j2")
|
||||
metadata = {
|
||||
"commit": lock["commit"],
|
||||
"commit_tree": lock["commit_tree"],
|
||||
"patches": {patch.name: hashlib.sha256(patch.read_bytes()).hexdigest() for patch in _patches(lock)},
|
||||
"configure_flags": flags,
|
||||
"cmake": _run(cmake, "--version").splitlines()[0],
|
||||
"cxx": _run("c++", "--version").splitlines()[0],
|
||||
"model_downloads": False,
|
||||
"semantic_certification": False,
|
||||
}
|
||||
(build_dir / "meshnet-build-metadata.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n")
|
||||
return build_dir / lock["build"]["smoke_binary"]
|
||||
|
||||
|
||||
def smoke(binary: pathlib.Path) -> None:
|
||||
if not binary.is_file():
|
||||
raise DependencyError(f"native smoke binary is missing: {binary}")
|
||||
smoke_config = _load_lock()["build"]
|
||||
output = _run(str(binary), *smoke_config["smoke_args"])
|
||||
expected = smoke_config["smoke_output_token"]
|
||||
if expected not in output.lower():
|
||||
raise DependencyError(f"native smoke output did not contain {expected!r}: {output}")
|
||||
print(output)
|
||||
|
||||
|
||||
def reproduce(work_dir: pathlib.Path, repository: str) -> None:
|
||||
resolved = work_dir.resolve()
|
||||
build_root = (ROOT / "build").resolve()
|
||||
if build_root not in resolved.parents:
|
||||
raise DependencyError(f"--work-dir must be below {build_root}: {resolved}")
|
||||
if resolved.exists():
|
||||
raise DependencyError(
|
||||
f"work directory already exists; refusing to erase possible local edits: {resolved}"
|
||||
)
|
||||
source = resolved / "source"
|
||||
build_dir = resolved / "build"
|
||||
materialize(source, repository)
|
||||
apply(source)
|
||||
smoke(build(source, build_dir))
|
||||
|
||||
|
||||
def inspect() -> None:
|
||||
lock = _load_lock()
|
||||
patches = _patches(lock)
|
||||
print(json.dumps({
|
||||
"commit": lock["commit"],
|
||||
"patch_count": len(patches),
|
||||
"patches": [patch.name for patch in patches],
|
||||
"model_downloads": False,
|
||||
"semantic_certification": False,
|
||||
"glm_stock_limitations": lock["stock_glm_limitations"],
|
||||
}, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subcommands = parser.add_subparsers(dest="command", required=True)
|
||||
subcommands.add_parser("inspect")
|
||||
materialize_parser = subcommands.add_parser("materialize")
|
||||
materialize_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
||||
materialize_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
|
||||
apply_parser = subcommands.add_parser("apply")
|
||||
apply_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
||||
build_parser = subcommands.add_parser("build")
|
||||
build_parser.add_argument("--source-dir", type=pathlib.Path, required=True)
|
||||
build_parser.add_argument("--build-dir", type=pathlib.Path, required=True)
|
||||
smoke_parser = subcommands.add_parser("smoke")
|
||||
smoke_parser.add_argument("--binary", type=pathlib.Path, required=True)
|
||||
reproduce_parser = subcommands.add_parser("reproduce")
|
||||
reproduce_parser.add_argument("--work-dir", type=pathlib.Path, default=ROOT / "build/dgr-004-smoke")
|
||||
reproduce_parser.add_argument("--source-repository", default=_load_lock()["upstream"])
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command == "inspect":
|
||||
inspect()
|
||||
elif args.command == "materialize":
|
||||
materialize(args.source_dir, args.source_repository)
|
||||
elif args.command == "apply":
|
||||
apply(args.source_dir)
|
||||
elif args.command == "build":
|
||||
build(args.source_dir, args.build_dir)
|
||||
elif args.command == "smoke":
|
||||
smoke(args.binary)
|
||||
else:
|
||||
reproduce(args.work_dir, args.source_repository)
|
||||
except DependencyError as error:
|
||||
print(f"DGR-004 dependency error: {error}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
254
scripts/refresh_glm_target_manifest.py
Normal file
254
scripts/refresh_glm_target_manifest.py
Normal file
@@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Refresh (or check) the pinned GLM-5.2 target manifest and architecture snapshot.
|
||||
|
||||
Resolves revisions, shard sizes, and LFS SHA-256 digests from the Hugging Face
|
||||
metadata API. It never downloads a weight payload: sizes and digests come from
|
||||
``/api/models/.../paths-info``, which returns the LFS pointer metadata, and the only
|
||||
files fetched in full are the small config/tokenizer/chat-template documents whose
|
||||
bytes the snapshot hashes.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/refresh_glm_target_manifest.py --check # CI: pinned bytes still resolve?
|
||||
python scripts/refresh_glm_target_manifest.py --write # re-pin HEAD after human review
|
||||
|
||||
``--check`` validates the already locked revisions through Hugging Face's
|
||||
revision-specific API. A normal new upstream commit does not mutate the target.
|
||||
``--write`` intentionally follows moving HEAD and is therefore a reviewed target
|
||||
change, never an automatic refresh.
|
||||
|
||||
This script requires network access and is not part of the default test suite.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DATA_DIR = Path(__file__).resolve().parent.parent / "packages/node/meshnet_node/glm_alpha/data"
|
||||
MANIFEST_PATH = DATA_DIR / "target-manifest.json"
|
||||
SNAPSHOT_PATH = DATA_DIR / "architecture-snapshot.json"
|
||||
|
||||
SOURCE_REPO = "zai-org/GLM-5.2"
|
||||
GGUF_REPO = "unsloth/GLM-5.2-GGUF"
|
||||
QUANT = "UD-IQ1_S"
|
||||
FALLBACK_QUANT = "UD-IQ1_M"
|
||||
SHARD_COUNT = 6
|
||||
|
||||
SNAPSHOT_FILES = (
|
||||
"config.json",
|
||||
"chat_template.jinja",
|
||||
"generation_config.json",
|
||||
"tokenizer_config.json",
|
||||
)
|
||||
|
||||
TIMEOUT = 60
|
||||
|
||||
|
||||
def _get(url: str) -> bytes:
|
||||
with urllib.request.urlopen(url, timeout=TIMEOUT) as response: # noqa: S310 - fixed HTTPS host
|
||||
return response.read()
|
||||
|
||||
|
||||
def _get_json(url: str) -> Any:
|
||||
return json.loads(_get(url))
|
||||
|
||||
|
||||
def _post_json(url: str, payload: dict) -> Any:
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=TIMEOUT) as response: # noqa: S310
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def _shard_paths(quant: str) -> list[str]:
|
||||
return [
|
||||
f"{quant}/GLM-5.2-{quant}-{index:05d}-of-{SHARD_COUNT:05d}.gguf"
|
||||
for index in range(1, SHARD_COUNT + 1)
|
||||
]
|
||||
|
||||
|
||||
def _resolve_shards(revision: str, quant: str) -> list[dict]:
|
||||
info = _post_json(
|
||||
f"https://huggingface.co/api/models/{GGUF_REPO}/paths-info/{revision}",
|
||||
{"paths": _shard_paths(quant)},
|
||||
)
|
||||
by_path = {entry["path"]: entry for entry in info}
|
||||
shards = []
|
||||
for index, path in enumerate(_shard_paths(quant), start=1):
|
||||
entry = by_path.get(path)
|
||||
if entry is None:
|
||||
raise SystemExit(f"upstream is missing shard {path} at revision {revision}")
|
||||
lfs = entry.get("lfs") or {}
|
||||
oid = lfs.get("oid")
|
||||
if not oid:
|
||||
raise SystemExit(
|
||||
f"{path} has no LFS oid; a non-LFS shard is not the published artifact"
|
||||
)
|
||||
shards.append(
|
||||
{
|
||||
"index": index,
|
||||
"path": path,
|
||||
"size_bytes": int(entry["size"]),
|
||||
"sha256": oid,
|
||||
"url": f"https://huggingface.co/{GGUF_REPO}/resolve/{revision}/{path}",
|
||||
}
|
||||
)
|
||||
return shards
|
||||
|
||||
|
||||
def build_documents(
|
||||
*,
|
||||
source_revision: str | None = None,
|
||||
gguf_revision: str | None = None,
|
||||
) -> tuple[dict, dict]:
|
||||
"""Resolve documents at explicit pins, or at current HEAD for reviewed re-pinning."""
|
||||
|
||||
source_api = f"https://huggingface.co/api/models/{SOURCE_REPO}"
|
||||
gguf_api = f"https://huggingface.co/api/models/{GGUF_REPO}"
|
||||
if source_revision is not None:
|
||||
source_api += f"/revision/{source_revision}"
|
||||
if gguf_revision is not None:
|
||||
gguf_api += f"/revision/{gguf_revision}"
|
||||
source_info = _get_json(source_api)
|
||||
gguf_info = _get_json(gguf_api)
|
||||
source_rev = source_info["sha"]
|
||||
gguf_rev = gguf_info["sha"]
|
||||
if source_revision is not None and source_rev != source_revision:
|
||||
raise SystemExit(
|
||||
f"source revision endpoint returned {source_rev}, expected {source_revision}"
|
||||
)
|
||||
if gguf_revision is not None and gguf_rev != gguf_revision:
|
||||
raise SystemExit(f"GGUF revision endpoint returned {gguf_rev}, expected {gguf_revision}")
|
||||
|
||||
shards = _resolve_shards(gguf_rev, QUANT)
|
||||
total = sum(shard["size_bytes"] for shard in shards)
|
||||
fallback = _resolve_shards(gguf_rev, FALLBACK_QUANT)
|
||||
fallback_total = sum(shard["size_bytes"] for shard in fallback)
|
||||
|
||||
manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
|
||||
manifest["source_model"]["revision"] = source_rev
|
||||
manifest["source_model"]["last_modified"] = source_info.get("lastModified")
|
||||
manifest["source_model"]["revision_url"] = (
|
||||
f"https://huggingface.co/{SOURCE_REPO}/tree/{source_rev}"
|
||||
)
|
||||
manifest["gguf_artifact"]["revision"] = gguf_rev
|
||||
manifest["gguf_artifact"]["last_modified"] = gguf_info.get("lastModified")
|
||||
manifest["gguf_artifact"]["revision_url"] = (
|
||||
f"https://huggingface.co/{GGUF_REPO}/tree/{gguf_rev}"
|
||||
)
|
||||
manifest["gguf_artifact"]["shards"] = shards
|
||||
manifest["gguf_artifact"]["total_bytes"] = total
|
||||
manifest["gguf_artifact"]["total_gib"] = round(total / 1024**3, 3)
|
||||
manifest["gguf_artifact"]["total_gb"] = round(total / 1000**3, 3)
|
||||
manifest["diagnostic_fallback"]["total_bytes"] = fallback_total
|
||||
manifest["diagnostic_fallback"]["total_gib"] = round(fallback_total / 1024**3, 3)
|
||||
manifest["diagnostic_fallback"]["total_gb"] = round(fallback_total / 1000**3, 3)
|
||||
|
||||
snapshot = json.loads(SNAPSHOT_PATH.read_text(encoding="utf-8"))
|
||||
snapshot["source_revision"] = source_rev
|
||||
source_files = []
|
||||
config: dict[str, Any] = {}
|
||||
for name in SNAPSHOT_FILES:
|
||||
url = f"https://huggingface.co/{SOURCE_REPO}/resolve/{source_rev}/{name}"
|
||||
body = _get(url)
|
||||
source_files.append(
|
||||
{
|
||||
"path": name,
|
||||
"size_bytes": len(body),
|
||||
"sha256": hashlib.sha256(body).hexdigest(),
|
||||
"url": url,
|
||||
}
|
||||
)
|
||||
if name == "config.json":
|
||||
config = json.loads(body)
|
||||
snapshot["source_files"] = source_files
|
||||
|
||||
indexer_types = config["indexer_types"]
|
||||
arch = snapshot["architecture"]
|
||||
arch["num_hidden_layers"] = config["num_hidden_layers"]
|
||||
arch["num_nextn_predict_layers"] = config["num_nextn_predict_layers"]
|
||||
arch["total_artifact_layers"] = config["num_hidden_layers"] + config["num_nextn_predict_layers"]
|
||||
arch["hidden_size"] = config["hidden_size"]
|
||||
arch["n_routed_experts"] = config["n_routed_experts"]
|
||||
arch["num_experts_per_tok"] = config["num_experts_per_tok"]
|
||||
arch["n_shared_experts"] = config["n_shared_experts"]
|
||||
arch["index_topk"] = config["index_topk"]
|
||||
arch["index_head_dim"] = config["index_head_dim"]
|
||||
arch["kv_lora_rank"] = config["kv_lora_rank"]
|
||||
arch["qk_rope_head_dim"] = config["qk_rope_head_dim"]
|
||||
arch["mla_cached_values_per_token_per_layer"] = (
|
||||
config["kv_lora_rank"] + config["qk_rope_head_dim"]
|
||||
)
|
||||
arch["indexer_full_layers"] = sum(1 for role in indexer_types if role == "full")
|
||||
arch["indexer_shared_layers"] = sum(1 for role in indexer_types if role == "shared")
|
||||
arch["indexer_types_sha256"] = hashlib.sha256(
|
||||
json.dumps(indexer_types, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
arch["max_position_embeddings"] = config["max_position_embeddings"]
|
||||
arch["vocab_size"] = config["vocab_size"]
|
||||
arch["first_k_dense_replace"] = config["first_k_dense_replace"]
|
||||
arch["dense_layers"] = config["first_k_dense_replace"]
|
||||
arch["sparse_moe_layers"] = config["num_hidden_layers"] - config["first_k_dense_replace"]
|
||||
|
||||
return manifest, snapshot
|
||||
|
||||
|
||||
def _dump(document: dict) -> str:
|
||||
return json.dumps(document, indent=2, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--check", action="store_true", help="fail if the pins have drifted")
|
||||
group.add_argument("--write", action="store_true", help="re-pin from upstream")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.check:
|
||||
pinned = json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
|
||||
manifest, snapshot = build_documents(
|
||||
source_revision=pinned["source_model"]["revision"],
|
||||
gguf_revision=pinned["gguf_artifact"]["revision"],
|
||||
)
|
||||
else:
|
||||
# --write intentionally follows current HEAD and therefore requires review.
|
||||
manifest, snapshot = build_documents()
|
||||
|
||||
if args.write:
|
||||
MANIFEST_PATH.write_text(_dump(manifest), encoding="utf-8")
|
||||
SNAPSHOT_PATH.write_text(_dump(snapshot), encoding="utf-8")
|
||||
print(f"wrote {MANIFEST_PATH}")
|
||||
print(f"wrote {SNAPSHOT_PATH}")
|
||||
print("Re-pinning changes the alpha target. Update the alpha contract under review.")
|
||||
return 0
|
||||
|
||||
drifted = False
|
||||
for path, fresh in ((MANIFEST_PATH, manifest), (SNAPSHOT_PATH, snapshot)):
|
||||
current = json.loads(path.read_text(encoding="utf-8"))
|
||||
if current != fresh:
|
||||
drifted = True
|
||||
print(f"DRIFT: {path.name} no longer matches upstream", file=sys.stderr)
|
||||
if drifted:
|
||||
print(
|
||||
"\nPinned revision metadata no longer matches the locked files. Treat this as "
|
||||
"artifact-integrity drift; do not heal or re-pin without human review.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
print("target manifest and architecture snapshot match upstream")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
135
scripts/verify_glm_shards.py
Normal file
135
scripts/verify_glm_shards.py
Normal file
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify downloaded GLM-5.2 ``UD-IQ1_S`` shards against the pinned manifest.
|
||||
|
||||
Every shard must match the exact byte size and LFS SHA-256 that DGR-017 locked
|
||||
in ``target-manifest.json``. Size is checked first because it is free and
|
||||
catches truncated resumes; the digest is then streamed so a 49 GB file never
|
||||
loads into memory. A shard that fails is reported with what was measured — the
|
||||
file is never deleted or "healed" by this script.
|
||||
|
||||
Fail-closed: exit 0 only when every requested shard is present and matches.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/verify_glm_shards.py --model-dir /mnt/models/glm-5.2
|
||||
python scripts/verify_glm_shards.py --model-dir DIR --shard 6 # one shard
|
||||
python scripts/verify_glm_shards.py --model-dir DIR --json report.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_ROOT / "packages/node"))
|
||||
|
||||
from meshnet_node.glm_alpha.manifest import ( # noqa: E402
|
||||
Shard,
|
||||
TargetManifest,
|
||||
load_target_manifest,
|
||||
)
|
||||
|
||||
_CHUNK_BYTES = 8 * 1024 * 1024
|
||||
|
||||
|
||||
def resolve_shard_file(model_dir: Path, shard: Shard) -> Path:
|
||||
"""Accept the repository layout (``UD-IQ1_S/<name>``) or a flat directory."""
|
||||
nested = model_dir / shard.path
|
||||
if nested.exists():
|
||||
return nested
|
||||
return model_dir / Path(shard.path).name
|
||||
|
||||
|
||||
def streaming_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(_CHUNK_BYTES):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def verify_shard(model_dir: Path, shard: Shard) -> dict:
|
||||
result: dict = {
|
||||
"shard_index": shard.index,
|
||||
"path": shard.path,
|
||||
"expected_size_bytes": shard.size_bytes,
|
||||
"expected_sha256": shard.sha256,
|
||||
}
|
||||
file = resolve_shard_file(model_dir, shard)
|
||||
result["file"] = str(file)
|
||||
if not file.exists():
|
||||
result.update(status="missing")
|
||||
return result
|
||||
|
||||
size = file.stat().st_size
|
||||
result["measured_size_bytes"] = size
|
||||
if size != shard.size_bytes:
|
||||
result.update(status="size_mismatch")
|
||||
return result
|
||||
|
||||
started = time.monotonic()
|
||||
measured = streaming_sha256(file)
|
||||
result["measured_sha256"] = measured
|
||||
result["hash_seconds"] = round(time.monotonic() - started, 1)
|
||||
result.update(status="ok" if measured == shard.sha256 else "sha256_mismatch")
|
||||
return result
|
||||
|
||||
|
||||
def verify_shards(
|
||||
model_dir: Path, manifest: TargetManifest, indices: list[int] | None = None
|
||||
) -> dict:
|
||||
shards = manifest.shards if indices is None else [manifest.shard(i) for i in indices]
|
||||
results = [verify_shard(model_dir, shard) for shard in shards]
|
||||
return {
|
||||
"generated_by": "scripts/verify_glm_shards.py",
|
||||
"model_dir": str(model_dir),
|
||||
"gguf_repo_id": manifest.gguf_repo_id,
|
||||
"gguf_revision": manifest.gguf_revision,
|
||||
"quantization": manifest.quantization,
|
||||
"manifest_sha256": manifest.digest,
|
||||
"shards": results,
|
||||
"verdict": "pass" if all(r["status"] == "ok" for r in results) else "fail",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--model-dir", required=True, help="directory holding the shards")
|
||||
parser.add_argument(
|
||||
"--shard",
|
||||
type=int,
|
||||
action="append",
|
||||
dest="shards",
|
||||
help="verify only this shard index (repeatable); default: all six",
|
||||
)
|
||||
parser.add_argument("--json", dest="json_path", help="also write the full report here")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = verify_shards(Path(args.model_dir), load_target_manifest(), args.shards)
|
||||
|
||||
if args.json_path:
|
||||
Path(args.json_path).write_text(
|
||||
json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
for result in report["shards"]:
|
||||
line = f"shard {result['shard_index']}: {result['status']}"
|
||||
if result["status"] == "size_mismatch":
|
||||
line += (
|
||||
f" (expected {result['expected_size_bytes']}, "
|
||||
f"measured {result['measured_size_bytes']})"
|
||||
)
|
||||
elif result["status"] == "sha256_mismatch":
|
||||
line += f" (measured {result['measured_sha256']})"
|
||||
print(line)
|
||||
print(f"verdict: {report['verdict']}")
|
||||
return 0 if report["verdict"] == "pass" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user