diff --git a/scripts/glm_whole_model_preflight.py b/scripts/glm_whole_model_preflight.py new file mode 100644 index 0000000..1922145 --- /dev/null +++ b/scripts/glm_whole_model_preflight.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""DGR-018 whole-model preflight: storage, memory, and the ordered download plan. + +Answers, before a single artifact byte moves, the three questions DGR-018's +finish contract asks: + +1. Is there one filesystem, outside every forbidden prefix (``/home``), with at + least 250 GB free for the 216.715 GB ``UD-IQ1_S`` artifact plus headroom? +2. Does this host have the 224 GiB runtime-accessible memory the whole-model + oracle load needs (the experimental hard-fit floor from DGR-017)? +3. In what order do the six shards download, and against which exact revision + URL, size, and LFS SHA-256 is each one verified? + +Everything is resolved from the pinned target manifest +(``meshnet_node.glm_alpha``) — this script never contacts the network and never +invents a size or digest. It fails closed: a missing requirement is a non-zero +exit and an explicit reason, never a downgrade to a smaller target. + +Usage:: + + python scripts/glm_whole_model_preflight.py # scan all mounts + python scripts/glm_whole_model_preflight.py --dest DIR # judge one destination + python scripts/glm_whole_model_preflight.py --storage-only # download-host mode + python scripts/glm_whole_model_preflight.py --json out.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import dataclass +from pathlib import Path + +_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_ROOT / "packages/node")) + +from meshnet_node.glm_alpha.manifest import ( # noqa: E402 + GIB, + TargetManifest, + load_target_manifest, +) + +GB = 1000**3 + +# 250 GB: the 216.715 GB artifact plus resume/temp headroom. DGR-018's issue +# text names this number; it is a storage requirement, not a tunable. +REQUIRED_FREE_BYTES = 250 * GB + +# DGR-017's experimental hard-fit floor for whole-model runtime-accessible +# memory. 224 GiB fits weights + Q8_0 KV at 16k context with nothing to spare. +REQUIRED_MEMORY_GIB = 224.0 + +# Download order, by shard index. Rationale: +# 1 first — 9.4 MB split-metadata shard: proves revision URLs, auth-free +# access, and tooling end-to-end for the cost of a rounding error. +# 6 second — 19.2 GB, the smallest weight shard: proves resume + hash +# verification at real scale before any ~49 GB transfer starts. +# 2..5 — the four ~49 GB shards, sequentially, verify-after-each, so at +# most one unverified partial ever exists on disk. +DOWNLOAD_ORDER = (1, 6, 2, 3, 4, 5) + +# Filesystems that can hold a 49 GB file and survive a resume. FAT variants are +# excluded by omission (4 GiB file limit); network/pseudo filesystems likewise. +_USABLE_FSTYPES = {"ext4", "ext3", "xfs", "btrfs", "ntfs", "ntfs3", "fuseblk", "exfat", "f2fs"} + + +@dataclass(frozen=True) +class MountCandidate: + mountpoint: str + fstype: str + total_bytes: int + free_bytes: int + forbidden: bool + + @property + def eligible(self) -> bool: + return not self.forbidden and self.free_bytes >= REQUIRED_FREE_BYTES + + def to_dict(self) -> dict: + return { + "mountpoint": self.mountpoint, + "fstype": self.fstype, + "total_gb": round(self.total_bytes / GB, 1), + "free_gb": round(self.free_bytes / GB, 1), + "free_bytes": self.free_bytes, + "forbidden": self.forbidden, + "eligible": self.eligible, + } + + +def forbidden_prefixes(manifest: TargetManifest) -> tuple[str, ...]: + """The storage policy pinned in the manifest itself — not a script opinion.""" + storage = manifest.raw.get("storage", {}) + return tuple(storage.get("forbidden_path_prefixes", ("/home",))) + + +def is_forbidden(path: str, prefixes: tuple[str, ...]) -> bool: + resolved = os.path.realpath(path) + return any(resolved == p or resolved.startswith(p.rstrip("/") + "/") for p in prefixes) + + +def _statvfs_free(path: str) -> tuple[int, int]: + st = os.statvfs(path) + return st.f_frsize * st.f_blocks, st.f_frsize * st.f_bavail + + +def scan_mounts( + prefixes: tuple[str, ...], mounts_file: str = "/proc/mounts" +) -> list[MountCandidate]: + """Every real, writable-class filesystem on the host, deduplicated by device.""" + candidates: dict[str, MountCandidate] = {} + with open(mounts_file, encoding="utf-8") as handle: + for line in handle: + fields = line.split() + if len(fields) < 3: + continue + device, mountpoint, fstype = fields[0], fields[1], fields[2] + mountpoint = mountpoint.replace("\\040", " ") + if fstype not in _USABLE_FSTYPES or device in candidates: + continue + try: + total, free = _statvfs_free(mountpoint) + except OSError: + continue + candidates[device] = MountCandidate( + mountpoint=mountpoint, + fstype=fstype, + total_bytes=total, + free_bytes=free, + forbidden=is_forbidden(mountpoint, prefixes), + ) + return sorted(candidates.values(), key=lambda c: c.free_bytes, reverse=True) + + +def judge_destination(dest: str, prefixes: tuple[str, ...]) -> MountCandidate: + total, free = _statvfs_free(dest) + return MountCandidate( + mountpoint=dest, + fstype="(as-given)", + total_bytes=total, + free_bytes=free, + forbidden=is_forbidden(dest, prefixes), + ) + + +def memory_total_gib(meminfo_file: str = "/proc/meminfo") -> float: + with open(meminfo_file, encoding="utf-8") as handle: + for line in handle: + if line.startswith("MemTotal:"): + return int(line.split()[1]) * 1024 / GIB + raise RuntimeError(f"MemTotal not found in {meminfo_file}") + + +def download_plan(manifest: TargetManifest, dest_dir: str | None) -> list[dict]: + """The six shards in download order, each bound to its pinned identity.""" + steps = [] + for position, index in enumerate(DOWNLOAD_ORDER, start=1): + shard = manifest.shard(index) + target = f"{dest_dir or '$GLM_DEST'}/{shard.path}" + steps.append( + { + "step": position, + "shard_index": shard.index, + "path": shard.path, + "size_bytes": shard.size_bytes, + "size_gb": round(shard.size_bytes / GB, 3), + "sha256": shard.sha256, + "url": shard.url, + "download_command": f'curl -L -C - --fail -o "{target}" "{shard.url}"', + "verify_command": ( + f"python scripts/verify_glm_shards.py " + f'--model-dir "{dest_dir or "$GLM_DEST"}" --shard {shard.index}' + ), + } + ) + return steps + + +def build_report(*, dest: str | None, storage_only: bool) -> dict: + manifest = load_target_manifest() + prefixes = forbidden_prefixes(manifest) + + if dest is not None: + mounts = [judge_destination(dest, prefixes)] + else: + mounts = scan_mounts(prefixes) + eligible = [m for m in mounts if m.eligible] + chosen = eligible[0] if eligible else None + + mem_gib = memory_total_gib() + storage_pass = chosen is not None + memory_pass = mem_gib >= REQUIRED_MEMORY_GIB + + checks = [ + { + "check": "storage", + "requirement": f">= {REQUIRED_FREE_BYTES / GB:.0f} GB free on one filesystem " + f"outside {list(prefixes)}", + "observed": ( + f"{chosen.free_bytes / GB:.1f} GB free at {chosen.mountpoint}" + if chosen + else "no eligible filesystem" + ), + "passes": storage_pass, + }, + { + "check": "memory", + "requirement": f">= {REQUIRED_MEMORY_GIB:.0f} GiB runtime-accessible memory " + "(DGR-017 experimental hard-fit floor)", + "observed": f"{mem_gib:.1f} GiB MemTotal", + "passes": memory_pass, + "waived": storage_only, + }, + ] + overall = storage_pass and (memory_pass or storage_only) + + return { + "generated_by": "scripts/glm_whole_model_preflight.py", + "target": { + "gguf_repo_id": manifest.gguf_repo_id, + "gguf_revision": manifest.gguf_revision, + "quantization": manifest.quantization, + "shard_count": len(manifest.shards), + "total_bytes": manifest.total_bytes, + "total_gb": round(manifest.total_gb, 3), + "manifest_sha256": manifest.digest, + }, + "forbidden_path_prefixes": list(prefixes), + "mounts": [m.to_dict() for m in mounts], + "chosen_destination": chosen.to_dict() if chosen else None, + "checks": checks, + "download_authorized": overall, + "storage_only": storage_only, + "download_plan": download_plan(manifest, chosen.mountpoint if chosen else None), + "verdict": "pass" if overall else "fail", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dest", help="judge this destination directory instead of scanning") + parser.add_argument( + "--storage-only", + action="store_true", + help="download-host mode: require storage, report memory without gating on it", + ) + parser.add_argument("--json", dest="json_path", help="also write the full report here") + args = parser.parse_args() + + report = build_report(dest=args.dest, storage_only=args.storage_only) + + if args.json_path: + Path(args.json_path).write_text( + json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + + target = report["target"] + print(f"target: {target['quantization']} {target['total_gb']} GB, " + f"{target['shard_count']} shards @ {target['gguf_revision'][:12]}") + for check in report["checks"]: + status = "PASS" if check["passes"] else ("WAIVED" if check.get("waived") else "FAIL") + print(f"[{status}] {check['check']}: need {check['requirement']}; " + f"observed {check['observed']}") + if report["chosen_destination"]: + print(f"destination: {report['chosen_destination']['mountpoint']}") + else: + print("destination: NONE — no filesystem outside " + f"{report['forbidden_path_prefixes']} has " + f"{REQUIRED_FREE_BYTES / GB:.0f} GB free") + for mount in report["mounts"]: + why = "forbidden prefix" if mount["forbidden"] else f"{mount['free_gb']} GB free" + print(f" - {mount['mountpoint']} ({mount['fstype']}): {why}") + print(f"verdict: {report['verdict']}") + return 0 if report["download_authorized"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_glm_shards.py b/scripts/verify_glm_shards.py new file mode 100644 index 0000000..3e3a647 --- /dev/null +++ b/scripts/verify_glm_shards.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Verify downloaded GLM-5.2 ``UD-IQ1_S`` shards against the pinned manifest. + +Every shard must match the exact byte size and LFS SHA-256 that DGR-017 locked +in ``target-manifest.json``. Size is checked first because it is free and +catches truncated resumes; the digest is then streamed so a 49 GB file never +loads into memory. A shard that fails is reported with what was measured — the +file is never deleted or "healed" by this script. + +Fail-closed: exit 0 only when every requested shard is present and matches. + +Usage:: + + python scripts/verify_glm_shards.py --model-dir /mnt/models/glm-5.2 + python scripts/verify_glm_shards.py --model-dir DIR --shard 6 # one shard + python scripts/verify_glm_shards.py --model-dir DIR --json report.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from pathlib import Path + +_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_ROOT / "packages/node")) + +from meshnet_node.glm_alpha.manifest import ( # noqa: E402 + Shard, + TargetManifest, + load_target_manifest, +) + +_CHUNK_BYTES = 8 * 1024 * 1024 + + +def resolve_shard_file(model_dir: Path, shard: Shard) -> Path: + """Accept the repository layout (``UD-IQ1_S/``) or a flat directory.""" + nested = model_dir / shard.path + if nested.exists(): + return nested + return model_dir / Path(shard.path).name + + +def streaming_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(_CHUNK_BYTES): + digest.update(chunk) + return digest.hexdigest() + + +def verify_shard(model_dir: Path, shard: Shard) -> dict: + result: dict = { + "shard_index": shard.index, + "path": shard.path, + "expected_size_bytes": shard.size_bytes, + "expected_sha256": shard.sha256, + } + file = resolve_shard_file(model_dir, shard) + result["file"] = str(file) + if not file.exists(): + result.update(status="missing") + return result + + size = file.stat().st_size + result["measured_size_bytes"] = size + if size != shard.size_bytes: + result.update(status="size_mismatch") + return result + + started = time.monotonic() + measured = streaming_sha256(file) + result["measured_sha256"] = measured + result["hash_seconds"] = round(time.monotonic() - started, 1) + result.update(status="ok" if measured == shard.sha256 else "sha256_mismatch") + return result + + +def verify_shards( + model_dir: Path, manifest: TargetManifest, indices: list[int] | None = None +) -> dict: + shards = manifest.shards if indices is None else [manifest.shard(i) for i in indices] + results = [verify_shard(model_dir, shard) for shard in shards] + return { + "generated_by": "scripts/verify_glm_shards.py", + "model_dir": str(model_dir), + "gguf_repo_id": manifest.gguf_repo_id, + "gguf_revision": manifest.gguf_revision, + "quantization": manifest.quantization, + "manifest_sha256": manifest.digest, + "shards": results, + "verdict": "pass" if all(r["status"] == "ok" for r in results) else "fail", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-dir", required=True, help="directory holding the shards") + parser.add_argument( + "--shard", + type=int, + action="append", + dest="shards", + help="verify only this shard index (repeatable); default: all six", + ) + parser.add_argument("--json", dest="json_path", help="also write the full report here") + args = parser.parse_args() + + report = verify_shards(Path(args.model_dir), load_target_manifest(), args.shards) + + if args.json_path: + Path(args.json_path).write_text( + json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + + for result in report["shards"]: + line = f"shard {result['shard_index']}: {result['status']}" + if result["status"] == "size_mismatch": + line += ( + f" (expected {result['expected_size_bytes']}, " + f"measured {result['measured_size_bytes']})" + ) + elif result["status"] == "sha256_mismatch": + line += f" (measured {result['measured_sha256']})" + print(line) + print(f"verdict: {report['verdict']}") + return 0 if report["verdict"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main())