chore: preserve DGR-018 preflight scripts (postponed)
This commit is contained in:
Dobromir Popov
280
scripts/glm_whole_model_preflight.py
Normal file
280
scripts/glm_whole_model_preflight.py
Normal file
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""DGR-018 whole-model preflight: storage, memory, and the ordered download plan.
|
||||
|
||||
Answers, before a single artifact byte moves, the three questions DGR-018's
|
||||
finish contract asks:
|
||||
|
||||
1. Is there one filesystem, outside every forbidden prefix (``/home``), with at
|
||||
least 250 GB free for the 216.715 GB ``UD-IQ1_S`` artifact plus headroom?
|
||||
2. Does this host have the 224 GiB runtime-accessible memory the whole-model
|
||||
oracle load needs (the experimental hard-fit floor from DGR-017)?
|
||||
3. In what order do the six shards download, and against which exact revision
|
||||
URL, size, and LFS SHA-256 is each one verified?
|
||||
|
||||
Everything is resolved from the pinned target manifest
|
||||
(``meshnet_node.glm_alpha``) — this script never contacts the network and never
|
||||
invents a size or digest. It fails closed: a missing requirement is a non-zero
|
||||
exit and an explicit reason, never a downgrade to a smaller target.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/glm_whole_model_preflight.py # scan all mounts
|
||||
python scripts/glm_whole_model_preflight.py --dest DIR # judge one destination
|
||||
python scripts/glm_whole_model_preflight.py --storage-only # download-host mode
|
||||
python scripts/glm_whole_model_preflight.py --json out.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_ROOT / "packages/node"))
|
||||
|
||||
from meshnet_node.glm_alpha.manifest import ( # noqa: E402
|
||||
GIB,
|
||||
TargetManifest,
|
||||
load_target_manifest,
|
||||
)
|
||||
|
||||
GB = 1000**3
|
||||
|
||||
# 250 GB: the 216.715 GB artifact plus resume/temp headroom. DGR-018's issue
|
||||
# text names this number; it is a storage requirement, not a tunable.
|
||||
REQUIRED_FREE_BYTES = 250 * GB
|
||||
|
||||
# DGR-017's experimental hard-fit floor for whole-model runtime-accessible
|
||||
# memory. 224 GiB fits weights + Q8_0 KV at 16k context with nothing to spare.
|
||||
REQUIRED_MEMORY_GIB = 224.0
|
||||
|
||||
# Download order, by shard index. Rationale:
|
||||
# 1 first — 9.4 MB split-metadata shard: proves revision URLs, auth-free
|
||||
# access, and tooling end-to-end for the cost of a rounding error.
|
||||
# 6 second — 19.2 GB, the smallest weight shard: proves resume + hash
|
||||
# verification at real scale before any ~49 GB transfer starts.
|
||||
# 2..5 — the four ~49 GB shards, sequentially, verify-after-each, so at
|
||||
# most one unverified partial ever exists on disk.
|
||||
DOWNLOAD_ORDER = (1, 6, 2, 3, 4, 5)
|
||||
|
||||
# Filesystems that can hold a 49 GB file and survive a resume. FAT variants are
|
||||
# excluded by omission (4 GiB file limit); network/pseudo filesystems likewise.
|
||||
_USABLE_FSTYPES = {"ext4", "ext3", "xfs", "btrfs", "ntfs", "ntfs3", "fuseblk", "exfat", "f2fs"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MountCandidate:
|
||||
mountpoint: str
|
||||
fstype: str
|
||||
total_bytes: int
|
||||
free_bytes: int
|
||||
forbidden: bool
|
||||
|
||||
@property
|
||||
def eligible(self) -> bool:
|
||||
return not self.forbidden and self.free_bytes >= REQUIRED_FREE_BYTES
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"mountpoint": self.mountpoint,
|
||||
"fstype": self.fstype,
|
||||
"total_gb": round(self.total_bytes / GB, 1),
|
||||
"free_gb": round(self.free_bytes / GB, 1),
|
||||
"free_bytes": self.free_bytes,
|
||||
"forbidden": self.forbidden,
|
||||
"eligible": self.eligible,
|
||||
}
|
||||
|
||||
|
||||
def forbidden_prefixes(manifest: TargetManifest) -> tuple[str, ...]:
|
||||
"""The storage policy pinned in the manifest itself — not a script opinion."""
|
||||
storage = manifest.raw.get("storage", {})
|
||||
return tuple(storage.get("forbidden_path_prefixes", ("/home",)))
|
||||
|
||||
|
||||
def is_forbidden(path: str, prefixes: tuple[str, ...]) -> bool:
|
||||
resolved = os.path.realpath(path)
|
||||
return any(resolved == p or resolved.startswith(p.rstrip("/") + "/") for p in prefixes)
|
||||
|
||||
|
||||
def _statvfs_free(path: str) -> tuple[int, int]:
|
||||
st = os.statvfs(path)
|
||||
return st.f_frsize * st.f_blocks, st.f_frsize * st.f_bavail
|
||||
|
||||
|
||||
def scan_mounts(
|
||||
prefixes: tuple[str, ...], mounts_file: str = "/proc/mounts"
|
||||
) -> list[MountCandidate]:
|
||||
"""Every real, writable-class filesystem on the host, deduplicated by device."""
|
||||
candidates: dict[str, MountCandidate] = {}
|
||||
with open(mounts_file, encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
fields = line.split()
|
||||
if len(fields) < 3:
|
||||
continue
|
||||
device, mountpoint, fstype = fields[0], fields[1], fields[2]
|
||||
mountpoint = mountpoint.replace("\\040", " ")
|
||||
if fstype not in _USABLE_FSTYPES or device in candidates:
|
||||
continue
|
||||
try:
|
||||
total, free = _statvfs_free(mountpoint)
|
||||
except OSError:
|
||||
continue
|
||||
candidates[device] = MountCandidate(
|
||||
mountpoint=mountpoint,
|
||||
fstype=fstype,
|
||||
total_bytes=total,
|
||||
free_bytes=free,
|
||||
forbidden=is_forbidden(mountpoint, prefixes),
|
||||
)
|
||||
return sorted(candidates.values(), key=lambda c: c.free_bytes, reverse=True)
|
||||
|
||||
|
||||
def judge_destination(dest: str, prefixes: tuple[str, ...]) -> MountCandidate:
|
||||
total, free = _statvfs_free(dest)
|
||||
return MountCandidate(
|
||||
mountpoint=dest,
|
||||
fstype="(as-given)",
|
||||
total_bytes=total,
|
||||
free_bytes=free,
|
||||
forbidden=is_forbidden(dest, prefixes),
|
||||
)
|
||||
|
||||
|
||||
def memory_total_gib(meminfo_file: str = "/proc/meminfo") -> float:
|
||||
with open(meminfo_file, encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
if line.startswith("MemTotal:"):
|
||||
return int(line.split()[1]) * 1024 / GIB
|
||||
raise RuntimeError(f"MemTotal not found in {meminfo_file}")
|
||||
|
||||
|
||||
def download_plan(manifest: TargetManifest, dest_dir: str | None) -> list[dict]:
|
||||
"""The six shards in download order, each bound to its pinned identity."""
|
||||
steps = []
|
||||
for position, index in enumerate(DOWNLOAD_ORDER, start=1):
|
||||
shard = manifest.shard(index)
|
||||
target = f"{dest_dir or '$GLM_DEST'}/{shard.path}"
|
||||
steps.append(
|
||||
{
|
||||
"step": position,
|
||||
"shard_index": shard.index,
|
||||
"path": shard.path,
|
||||
"size_bytes": shard.size_bytes,
|
||||
"size_gb": round(shard.size_bytes / GB, 3),
|
||||
"sha256": shard.sha256,
|
||||
"url": shard.url,
|
||||
"download_command": f'curl -L -C - --fail -o "{target}" "{shard.url}"',
|
||||
"verify_command": (
|
||||
f"python scripts/verify_glm_shards.py "
|
||||
f'--model-dir "{dest_dir or "$GLM_DEST"}" --shard {shard.index}'
|
||||
),
|
||||
}
|
||||
)
|
||||
return steps
|
||||
|
||||
|
||||
def build_report(*, dest: str | None, storage_only: bool) -> dict:
|
||||
manifest = load_target_manifest()
|
||||
prefixes = forbidden_prefixes(manifest)
|
||||
|
||||
if dest is not None:
|
||||
mounts = [judge_destination(dest, prefixes)]
|
||||
else:
|
||||
mounts = scan_mounts(prefixes)
|
||||
eligible = [m for m in mounts if m.eligible]
|
||||
chosen = eligible[0] if eligible else None
|
||||
|
||||
mem_gib = memory_total_gib()
|
||||
storage_pass = chosen is not None
|
||||
memory_pass = mem_gib >= REQUIRED_MEMORY_GIB
|
||||
|
||||
checks = [
|
||||
{
|
||||
"check": "storage",
|
||||
"requirement": f">= {REQUIRED_FREE_BYTES / GB:.0f} GB free on one filesystem "
|
||||
f"outside {list(prefixes)}",
|
||||
"observed": (
|
||||
f"{chosen.free_bytes / GB:.1f} GB free at {chosen.mountpoint}"
|
||||
if chosen
|
||||
else "no eligible filesystem"
|
||||
),
|
||||
"passes": storage_pass,
|
||||
},
|
||||
{
|
||||
"check": "memory",
|
||||
"requirement": f">= {REQUIRED_MEMORY_GIB:.0f} GiB runtime-accessible memory "
|
||||
"(DGR-017 experimental hard-fit floor)",
|
||||
"observed": f"{mem_gib:.1f} GiB MemTotal",
|
||||
"passes": memory_pass,
|
||||
"waived": storage_only,
|
||||
},
|
||||
]
|
||||
overall = storage_pass and (memory_pass or storage_only)
|
||||
|
||||
return {
|
||||
"generated_by": "scripts/glm_whole_model_preflight.py",
|
||||
"target": {
|
||||
"gguf_repo_id": manifest.gguf_repo_id,
|
||||
"gguf_revision": manifest.gguf_revision,
|
||||
"quantization": manifest.quantization,
|
||||
"shard_count": len(manifest.shards),
|
||||
"total_bytes": manifest.total_bytes,
|
||||
"total_gb": round(manifest.total_gb, 3),
|
||||
"manifest_sha256": manifest.digest,
|
||||
},
|
||||
"forbidden_path_prefixes": list(prefixes),
|
||||
"mounts": [m.to_dict() for m in mounts],
|
||||
"chosen_destination": chosen.to_dict() if chosen else None,
|
||||
"checks": checks,
|
||||
"download_authorized": overall,
|
||||
"storage_only": storage_only,
|
||||
"download_plan": download_plan(manifest, chosen.mountpoint if chosen else None),
|
||||
"verdict": "pass" if overall else "fail",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--dest", help="judge this destination directory instead of scanning")
|
||||
parser.add_argument(
|
||||
"--storage-only",
|
||||
action="store_true",
|
||||
help="download-host mode: require storage, report memory without gating on it",
|
||||
)
|
||||
parser.add_argument("--json", dest="json_path", help="also write the full report here")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = build_report(dest=args.dest, storage_only=args.storage_only)
|
||||
|
||||
if args.json_path:
|
||||
Path(args.json_path).write_text(
|
||||
json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
target = report["target"]
|
||||
print(f"target: {target['quantization']} {target['total_gb']} GB, "
|
||||
f"{target['shard_count']} shards @ {target['gguf_revision'][:12]}")
|
||||
for check in report["checks"]:
|
||||
status = "PASS" if check["passes"] else ("WAIVED" if check.get("waived") else "FAIL")
|
||||
print(f"[{status}] {check['check']}: need {check['requirement']}; "
|
||||
f"observed {check['observed']}")
|
||||
if report["chosen_destination"]:
|
||||
print(f"destination: {report['chosen_destination']['mountpoint']}")
|
||||
else:
|
||||
print("destination: NONE — no filesystem outside "
|
||||
f"{report['forbidden_path_prefixes']} has "
|
||||
f"{REQUIRED_FREE_BYTES / GB:.0f} GB free")
|
||||
for mount in report["mounts"]:
|
||||
why = "forbidden prefix" if mount["forbidden"] else f"{mount['free_gb']} GB free"
|
||||
print(f" - {mount['mountpoint']} ({mount['fstype']}): {why}")
|
||||
print(f"verdict: {report['verdict']}")
|
||||
return 0 if report["download_authorized"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
135
scripts/verify_glm_shards.py
Normal file
135
scripts/verify_glm_shards.py
Normal file
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify downloaded GLM-5.2 ``UD-IQ1_S`` shards against the pinned manifest.
|
||||
|
||||
Every shard must match the exact byte size and LFS SHA-256 that DGR-017 locked
|
||||
in ``target-manifest.json``. Size is checked first because it is free and
|
||||
catches truncated resumes; the digest is then streamed so a 49 GB file never
|
||||
loads into memory. A shard that fails is reported with what was measured — the
|
||||
file is never deleted or "healed" by this script.
|
||||
|
||||
Fail-closed: exit 0 only when every requested shard is present and matches.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/verify_glm_shards.py --model-dir /mnt/models/glm-5.2
|
||||
python scripts/verify_glm_shards.py --model-dir DIR --shard 6 # one shard
|
||||
python scripts/verify_glm_shards.py --model-dir DIR --json report.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(_ROOT / "packages/node"))
|
||||
|
||||
from meshnet_node.glm_alpha.manifest import ( # noqa: E402
|
||||
Shard,
|
||||
TargetManifest,
|
||||
load_target_manifest,
|
||||
)
|
||||
|
||||
_CHUNK_BYTES = 8 * 1024 * 1024
|
||||
|
||||
|
||||
def resolve_shard_file(model_dir: Path, shard: Shard) -> Path:
|
||||
"""Accept the repository layout (``UD-IQ1_S/<name>``) or a flat directory."""
|
||||
nested = model_dir / shard.path
|
||||
if nested.exists():
|
||||
return nested
|
||||
return model_dir / Path(shard.path).name
|
||||
|
||||
|
||||
def streaming_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(_CHUNK_BYTES):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def verify_shard(model_dir: Path, shard: Shard) -> dict:
|
||||
result: dict = {
|
||||
"shard_index": shard.index,
|
||||
"path": shard.path,
|
||||
"expected_size_bytes": shard.size_bytes,
|
||||
"expected_sha256": shard.sha256,
|
||||
}
|
||||
file = resolve_shard_file(model_dir, shard)
|
||||
result["file"] = str(file)
|
||||
if not file.exists():
|
||||
result.update(status="missing")
|
||||
return result
|
||||
|
||||
size = file.stat().st_size
|
||||
result["measured_size_bytes"] = size
|
||||
if size != shard.size_bytes:
|
||||
result.update(status="size_mismatch")
|
||||
return result
|
||||
|
||||
started = time.monotonic()
|
||||
measured = streaming_sha256(file)
|
||||
result["measured_sha256"] = measured
|
||||
result["hash_seconds"] = round(time.monotonic() - started, 1)
|
||||
result.update(status="ok" if measured == shard.sha256 else "sha256_mismatch")
|
||||
return result
|
||||
|
||||
|
||||
def verify_shards(
|
||||
model_dir: Path, manifest: TargetManifest, indices: list[int] | None = None
|
||||
) -> dict:
|
||||
shards = manifest.shards if indices is None else [manifest.shard(i) for i in indices]
|
||||
results = [verify_shard(model_dir, shard) for shard in shards]
|
||||
return {
|
||||
"generated_by": "scripts/verify_glm_shards.py",
|
||||
"model_dir": str(model_dir),
|
||||
"gguf_repo_id": manifest.gguf_repo_id,
|
||||
"gguf_revision": manifest.gguf_revision,
|
||||
"quantization": manifest.quantization,
|
||||
"manifest_sha256": manifest.digest,
|
||||
"shards": results,
|
||||
"verdict": "pass" if all(r["status"] == "ok" for r in results) else "fail",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--model-dir", required=True, help="directory holding the shards")
|
||||
parser.add_argument(
|
||||
"--shard",
|
||||
type=int,
|
||||
action="append",
|
||||
dest="shards",
|
||||
help="verify only this shard index (repeatable); default: all six",
|
||||
)
|
||||
parser.add_argument("--json", dest="json_path", help="also write the full report here")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = verify_shards(Path(args.model_dir), load_target_manifest(), args.shards)
|
||||
|
||||
if args.json_path:
|
||||
Path(args.json_path).write_text(
|
||||
json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
for result in report["shards"]:
|
||||
line = f"shard {result['shard_index']}: {result['status']}"
|
||||
if result["status"] == "size_mismatch":
|
||||
line += (
|
||||
f" (expected {result['expected_size_bytes']}, "
|
||||
f"measured {result['measured_size_bytes']})"
|
||||
)
|
||||
elif result["status"] == "sha256_mismatch":
|
||||
line += f" (measured {result['measured_sha256']})"
|
||||
print(line)
|
||||
print(f"verdict: {report['verdict']}")
|
||||
return 0 if report["verdict"] == "pass" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user