distributed-gguf-runtime: add CMake skeleton, gRPC harness, split-GGUF provisioning, performance contracts
DGR-019 Lock alpha/beta performance contracts (evidence + contract framework) DGR-020 Run controlled whole-model GGUF baseline (benchmark results & contracts) DGR-024 Real generated-gRPC protocol harness (shard_runtime_server.py + tests) DGR-026 split-GGUF provisioning outside /home (provision script + manifest + tests) DGR-028 Numbered patch-stack apply & verify (llama_cpp_dependency.py + UPSTREAM_LOCK.json) DGR-029 Native CMake skeleton + deterministic CPU lane (UPSTREAM_LOCK.json + cmake gating) New modules: packages/node/meshnet_node/dgr_performance/ — performance contract framework packages/node/meshnet_node/split_gguf/ — split-GGUF manifest & provisioning scripts/provision_split_gguf.py — artifact provisioning CLI tests/test_dgr_performance_contract.py — contract validation tests tests/test_split_gguf_manifest.py — manifest tests tests/test_split_gguf_provision.py — provisioning tests tests/test_shard_runtime_harness.py — gRPC harness tests
This commit is contained in:
339
tests/test_dgr_performance_contract.py
Normal file
339
tests/test_dgr_performance_contract.py
Normal file
@@ -0,0 +1,339 @@
|
||||
"""DGR-019 — the locked alpha/beta performance contract.
|
||||
|
||||
Deterministic, offline, GPU-free, model-download-free. These tests assert
|
||||
against the *pinned* v1 contract, so they fail if a later change loosens a
|
||||
threshold, drops a lane, or removes the human-approval gate on alpha's
|
||||
useful-speed threshold without going through a new contract_id/version.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from meshnet_node.dgr_performance import (
|
||||
ALPHA_VERDICTS,
|
||||
BETA_VERDICTS,
|
||||
CONTRACT_ID,
|
||||
CONTRACT_V1_SHA256,
|
||||
NEWLY_LOCKED_LANES,
|
||||
REFERENCED_LANES,
|
||||
REQUIRED_LANES,
|
||||
AlphaBetaContract,
|
||||
DgrPerformanceContractError,
|
||||
compute_contract_digest,
|
||||
load_contract,
|
||||
parse_contract,
|
||||
seal_contract,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def contract() -> AlphaBetaContract:
|
||||
return load_contract()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def contract_doc(contract: AlphaBetaContract) -> dict:
|
||||
return contract.to_dict()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Loading and identity
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_loads_the_packaged_contract(contract: AlphaBetaContract) -> None:
|
||||
assert contract.contract_id == CONTRACT_ID == "dgr-alpha-beta-performance/v1"
|
||||
assert contract.schema_version == 1
|
||||
assert contract.contract_version == 1
|
||||
assert contract.locked_by == "DGR-019"
|
||||
assert contract.digest == CONTRACT_V1_SHA256
|
||||
|
||||
|
||||
def test_digest_matches_recomputation_from_content(contract_doc: dict) -> None:
|
||||
assert compute_contract_digest(contract_doc) == CONTRACT_V1_SHA256
|
||||
|
||||
|
||||
def test_all_four_lanes_are_present(contract: AlphaBetaContract) -> None:
|
||||
assert set(contract.lanes) == set(REQUIRED_LANES) == {
|
||||
"controlled-safetensors",
|
||||
"whole-model-gguf",
|
||||
"dense-distributed-gguf",
|
||||
"v4-flash-distributed",
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Lanes 1 & 2 reference the existing DGR-001 lock; lanes 3 & 4 are new
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_controlled_and_whole_model_lanes_reference_the_existing_dgr_001_lock(
|
||||
contract: AlphaBetaContract,
|
||||
) -> None:
|
||||
for name in REFERENCED_LANES:
|
||||
lane = contract.lane(name)
|
||||
assert lane["locked_elsewhere"] is True
|
||||
assert lane["contract_module"] == "meshnet_node.performance_contract"
|
||||
assert lane["contract_id"] == "dgr-001-controlled-whole-model-baseline-v1"
|
||||
|
||||
|
||||
def test_dgr_001_referenced_contract_id_matches_the_real_locked_module() -> None:
|
||||
from meshnet_node.performance_contract import ContractThresholds
|
||||
|
||||
# DGR-001's thresholds are immutable v1; this pins the assumption this
|
||||
# contract's reference actually still points at, without re-locking them.
|
||||
assert ContractThresholds().min_decode_speedup == 1.25
|
||||
assert ContractThresholds().max_resident_memory_ratio == 0.75
|
||||
|
||||
|
||||
def test_dense_and_v4_lanes_are_newly_locked_with_full_benchmark_plans(
|
||||
contract: AlphaBetaContract,
|
||||
) -> None:
|
||||
for name in NEWLY_LOCKED_LANES:
|
||||
lane = contract.lane(name)
|
||||
assert "locked_elsewhere" not in lane
|
||||
assert lane["prompt_ids"]
|
||||
assert lane["hardware"]
|
||||
assert lane["metrics"]
|
||||
assert lane["certification_scenarios"]["stage_count"]
|
||||
assert lane["certification_scenarios"]["quantization"]
|
||||
|
||||
|
||||
def test_dense_lane_defines_fixed_context_output_concurrency(contract: AlphaBetaContract) -> None:
|
||||
lane = contract.lane("dense-distributed-gguf")
|
||||
assert lane["context_tokens"] == 2048
|
||||
assert lane["output_tokens"] == 128
|
||||
assert list(lane["concurrency_levels"]) == [1, 4]
|
||||
|
||||
|
||||
def test_v4_lane_defines_separate_alpha_and_beta_scale(contract: AlphaBetaContract) -> None:
|
||||
lane = contract.lane("v4-flash-distributed")
|
||||
assert lane["alpha_context_tokens"] == 4096
|
||||
assert list(lane["alpha_concurrency_levels"]) == [1, 4]
|
||||
assert lane["beta_context_tokens"] == 16384
|
||||
assert list(lane["beta_concurrency_levels"]) == [1, 4, 8, 16]
|
||||
assert lane["hardware"]["mtp"].startswith("reserved and off for alpha")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Fixed prompts and sampling are shared, not per-lane free variables
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_prompt_set_is_fixed_and_referenced_by_id(contract_doc: dict) -> None:
|
||||
prompt_ids = {p["id"] for p in contract_doc["prompt_set"]["prompts"]}
|
||||
assert prompt_ids == {
|
||||
"short-instruction",
|
||||
"code-completion",
|
||||
"multi-step-reasoning",
|
||||
"long-context-fill",
|
||||
}
|
||||
for lane_name in ("dense-distributed-gguf", "v4-flash-distributed"):
|
||||
lane = contract_doc["lanes"][lane_name]
|
||||
for key in ("prompt_ids", "beta_prompt_ids"):
|
||||
if key in lane:
|
||||
assert set(lane[key]) <= prompt_ids
|
||||
|
||||
|
||||
def test_sampling_is_greedy(contract_doc: dict) -> None:
|
||||
sampling = contract_doc["sampling"]
|
||||
assert sampling["temperature"] == 0.0
|
||||
assert sampling["top_p"] == 1.0
|
||||
assert sampling["top_k"] == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Gain attribution separates quantization/model-fit from runtime/transport
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_gain_attribution_axes_are_disjoint(contract_doc: dict) -> None:
|
||||
attribution = contract_doc["gain_attribution"]
|
||||
fit_metrics = set(attribution["quantization_model_fit_metrics"])
|
||||
runtime_metrics = set(attribution["runtime_transport_batching_kernel_metrics"])
|
||||
assert fit_metrics, "quantization/model-fit axis must not be empty"
|
||||
assert runtime_metrics, "runtime/transport/batching/kernel axis must not be empty"
|
||||
assert fit_metrics.isdisjoint(runtime_metrics), (
|
||||
"a metric cannot count as both a quantization/model-fit gain and a "
|
||||
"runtime/transport/batching/kernel gain"
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Quants and stage counts are named certification scenarios only
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_certification_scenarios_are_named_labels_not_defaults(contract_doc: dict) -> None:
|
||||
scenarios = contract_doc["certification_scenarios"]
|
||||
assert scenarios["stage_count"]["names"] == ["2-4-stage", "10-plus-stage"]
|
||||
assert set(scenarios["quantization"]["names"]) == {"Q4_K_M", "Q8_0", "bf16-reference"}
|
||||
for axis in ("stage_count", "quantization"):
|
||||
assert "no product or runtime code path" in scenarios[axis]["rule"].lower()
|
||||
|
||||
|
||||
def test_no_product_module_hardcodes_the_named_stage_counts_or_quant() -> None:
|
||||
"""The contract may *name* '2-4-stage'/'10-plus-stage'/'Q4_K_M' as scenarios;
|
||||
no runtime module outside this contract package and its evidence/spec
|
||||
peers may hardcode them as product logic (e.g. `range(2, 5)` gating
|
||||
placement, or a literal default quantization string)."""
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
node_pkg = repo_root / "packages" / "node" / "meshnet_node"
|
||||
allowed_hits = {
|
||||
node_pkg / "dgr_performance" / "contract.py",
|
||||
node_pkg / "dgr_performance" / "data" / "alpha-beta-contract-v1.json",
|
||||
}
|
||||
offenders = []
|
||||
for path in node_pkg.rglob("*.py"):
|
||||
if "__pycache__" in path.parts or path in allowed_hits:
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
if "2-4-stage" in text or "10-plus-stage" in text:
|
||||
offenders.append(str(path))
|
||||
assert not offenders, f"stage-count certification labels leaked into product code: {offenders}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Alpha: correctness + human-approved useful speed
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_alpha_verdicts_are_exactly_alpha_optimize_stop(contract: AlphaBetaContract) -> None:
|
||||
assert tuple(contract.alpha["verdicts"]) == ALPHA_VERDICTS == ("alpha", "optimize", "stop")
|
||||
|
||||
|
||||
def test_alpha_requires_correctness_thresholds(contract: AlphaBetaContract) -> None:
|
||||
correctness = contract.alpha["correctness"]
|
||||
assert correctness["min_greedy_token_agreement"] == 0.9
|
||||
assert correctness["forbid_nonfinite_tensors"] is True
|
||||
assert correctness["dense_attention_fallback_satisfies_alpha"] is False
|
||||
|
||||
|
||||
def test_alpha_useful_speed_requires_human_approval(contract: AlphaBetaContract) -> None:
|
||||
useful_speed = contract.alpha["useful_speed"]
|
||||
assert useful_speed["min_decode_speedup_vs_reference_baseline"] == 1.25
|
||||
approval = useful_speed["human_approval"]
|
||||
assert approval["required"] is True
|
||||
assert approval["approved"] is False
|
||||
assert approval["approved_by"] is None
|
||||
assert approval["approved_at"] is None
|
||||
|
||||
|
||||
def test_alpha_holds_mtp_reserved_and_off(contract: AlphaBetaContract) -> None:
|
||||
mtp = contract.alpha["mtp"]
|
||||
assert mtp["reserved"] is True
|
||||
assert mtp["enabled_for_alpha"] is False
|
||||
assert mtp["ownership_contract_and_benchmark_required_before_beta"] is True
|
||||
|
||||
|
||||
def test_parse_contract_rejects_a_contract_missing_human_approval() -> None:
|
||||
doc = json.loads(
|
||||
Path("packages/node/meshnet_node/dgr_performance/data/alpha-beta-contract-v1.json")
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
mutated = copy.deepcopy(doc)
|
||||
mutated["alpha"]["useful_speed"]["human_approval"]["required"] = False
|
||||
resealed = seal_contract({k: v for k, v in mutated.items() if k != "contract_sha256"})
|
||||
with pytest.raises(DgrPerformanceContractError, match="human-approved useful-speed"):
|
||||
parse_contract(resealed, source="<mutated>")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Beta: adds concurrency, long-context, failure, sustained-throughput
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_beta_verdicts_are_exactly_beta_targeted_optimization_stop_rollback(
|
||||
contract: AlphaBetaContract,
|
||||
) -> None:
|
||||
assert tuple(contract.beta["verdicts"]) == BETA_VERDICTS == (
|
||||
"beta",
|
||||
"targeted-optimization",
|
||||
"stop-rollback",
|
||||
)
|
||||
|
||||
|
||||
def test_beta_adds_exactly_the_four_required_axes(contract: AlphaBetaContract) -> None:
|
||||
assert set(contract.beta["adds"]) == {
|
||||
"concurrency",
|
||||
"long_context",
|
||||
"failure",
|
||||
"sustained_throughput",
|
||||
}
|
||||
for axis in contract.beta["adds"]:
|
||||
assert axis in contract.beta
|
||||
|
||||
|
||||
def test_beta_long_context_matches_the_v4_lane_beta_scale(contract: AlphaBetaContract) -> None:
|
||||
assert contract.beta["long_context"]["context_tokens"] == 16384
|
||||
assert contract.beta["long_context"]["context_tokens"] == contract.lane(
|
||||
"v4-flash-distributed"
|
||||
)["beta_context_tokens"]
|
||||
|
||||
|
||||
def test_beta_failure_axis_forbids_silent_kv_migration(contract: AlphaBetaContract) -> None:
|
||||
failure = contract.beta["failure"]
|
||||
assert failure["forbid_silent_kv_migration"] is True
|
||||
assert failure["synthetic_workers_satisfy_beta"] is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Immutability: mutation after sealing is rejected, not silently trusted
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_contract_rejects_a_mutated_threshold() -> None:
|
||||
doc = json.loads(
|
||||
Path("packages/node/meshnet_node/dgr_performance/data/alpha-beta-contract-v1.json")
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
mutated = copy.deepcopy(doc)
|
||||
mutated["alpha"]["useful_speed"]["min_decode_speedup_vs_reference_baseline"] = 1.01
|
||||
with pytest.raises(DgrPerformanceContractError, match="modified since it was locked"):
|
||||
parse_contract(mutated, source="<mutated>")
|
||||
|
||||
|
||||
def test_parse_contract_rejects_a_resealed_mutation_against_the_pinned_digest() -> None:
|
||||
doc = json.loads(
|
||||
Path("packages/node/meshnet_node/dgr_performance/data/alpha-beta-contract-v1.json")
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
mutated = copy.deepcopy(doc)
|
||||
mutated["alpha"]["useful_speed"]["min_decode_speedup_vs_reference_baseline"] = 1.01
|
||||
resealed = seal_contract({k: v for k, v in mutated.items() if k != "contract_sha256"})
|
||||
with pytest.raises(DgrPerformanceContractError, match="re-sealed mutation"):
|
||||
parse_contract(resealed, source="<resealed>")
|
||||
|
||||
|
||||
def test_parse_contract_rejects_missing_digest() -> None:
|
||||
doc = json.loads(
|
||||
Path("packages/node/meshnet_node/dgr_performance/data/alpha-beta-contract-v1.json")
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
stripped = {k: v for k, v in doc.items() if k != "contract_sha256"}
|
||||
with pytest.raises(DgrPerformanceContractError, match="carries no contract_sha256"):
|
||||
parse_contract(stripped, source="<stripped>")
|
||||
|
||||
|
||||
def test_load_contract_from_explicit_path_matches_packaged_load(contract: AlphaBetaContract) -> None:
|
||||
on_disk = load_contract(
|
||||
Path("packages/node/meshnet_node/dgr_performance/data/alpha-beta-contract-v1.json")
|
||||
)
|
||||
assert on_disk.digest == contract.digest
|
||||
assert on_disk.to_dict() == contract.to_dict()
|
||||
|
||||
|
||||
def test_seal_contract_is_the_only_supported_way_to_produce_a_digest(contract_doc: dict) -> None:
|
||||
unsigned = {k: v for k, v in contract_doc.items() if k != "contract_sha256"}
|
||||
sealed = seal_contract(unsigned)
|
||||
assert sealed["contract_sha256"] == CONTRACT_V1_SHA256
|
||||
|
||||
|
||||
def test_amendment_policy_is_locked_and_nonempty(contract: AlphaBetaContract) -> None:
|
||||
assert "new contract_id" in contract.amendment_policy
|
||||
assert "human review" in contract.amendment_policy
|
||||
@@ -6,9 +6,12 @@ import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
LLAMA_DIR = ROOT / "packages/node/native/llama"
|
||||
@@ -19,6 +22,26 @@ def _sha256(path: pathlib.Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _load_dependency_module():
|
||||
spec = importlib.util.spec_from_file_location("llama_cpp_dependency_ctest", SCRIPT)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _cmake_available() -> bool:
|
||||
if shutil.which("cmake"):
|
||||
return True
|
||||
sibling = pathlib.Path(sys.executable).parent / "cmake"
|
||||
return sibling.is_file()
|
||||
|
||||
|
||||
requires_cmake = pytest.mark.skipif(
|
||||
not _cmake_available(), reason="cmake toolchain is required to build the native CTest lane"
|
||||
)
|
||||
|
||||
|
||||
def test_lock_and_patch_manifest_are_self_consistent_and_exact() -> None:
|
||||
lock = json.loads((LLAMA_DIR / "UPSTREAM_LOCK.json").read_text())
|
||||
commit = (LLAMA_DIR / "UPSTREAM_COMMIT").read_text().strip()
|
||||
@@ -255,3 +278,59 @@ def test_patch_stack_does_not_contain_meshnet_control_plane_code() -> None:
|
||||
for name in (LLAMA_DIR / "patches/series").read_text().splitlines()
|
||||
)
|
||||
assert not any(term in patch_text for term in forbidden)
|
||||
|
||||
|
||||
def test_build_config_locks_an_explicit_cpu_only_deterministic_lane() -> None:
|
||||
lock = json.loads((LLAMA_DIR / "UPSTREAM_LOCK.json").read_text())
|
||||
build = lock["build"]
|
||||
flag_values = dict(flag[len("-D"):].split("=", 1) for flag in build["configure_flags"])
|
||||
|
||||
assert flag_values["GGML_CPU"] == "ON"
|
||||
for backend in ("GGML_CUDA", "GGML_HIP", "GGML_VULKAN", "GGML_METAL", "GGML_BLAS"):
|
||||
assert flag_values[backend] == "OFF"
|
||||
assert flag_values["LLAMA_BUILD_TESTS"] == "ON"
|
||||
|
||||
assert build["ctest_regex"] == "^test-meshnet-range-ownership$"
|
||||
assert "test-meshnet-range-ownership" in build["native_targets"]
|
||||
assert pathlib.Path(build["smoke_binary"]).name in build["native_targets"]
|
||||
assert "tests/test-meshnet-range-ownership.cpp" in lock["patched_paths"]
|
||||
|
||||
|
||||
@requires_cmake
|
||||
def test_ctest_lane_raises_an_actionable_error_for_a_failing_named_test(tmp_path: pathlib.Path) -> None:
|
||||
dependency = _load_dependency_module()
|
||||
|
||||
project = tmp_path / "project"
|
||||
project.mkdir()
|
||||
(project / "CMakeLists.txt").write_text(
|
||||
"cmake_minimum_required(VERSION 3.14)\n"
|
||||
"project(ctest_lane_fixture NONE)\n"
|
||||
"enable_testing()\n"
|
||||
"add_test(NAME meshnet-fixture-pass COMMAND ${CMAKE_COMMAND} -E true)\n"
|
||||
"add_test(NAME meshnet-fixture-fail COMMAND ${CMAKE_COMMAND} -E false)\n"
|
||||
)
|
||||
build_dir = tmp_path / "build"
|
||||
subprocess.run(
|
||||
[dependency._cmake(), "-S", str(project), "-B", str(build_dir)],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
base_lock = json.loads((LLAMA_DIR / "UPSTREAM_LOCK.json").read_text())
|
||||
|
||||
def _lock_with_regex(regex: str):
|
||||
patched = dict(base_lock)
|
||||
patched["build"] = {**base_lock["build"], "ctest_regex": regex}
|
||||
return patched
|
||||
|
||||
dependency._load_lock = lambda: _lock_with_regex("^meshnet-fixture-pass$")
|
||||
dependency.ctest_lane(build_dir)
|
||||
|
||||
dependency._load_lock = lambda: _lock_with_regex("^meshnet-fixture-fail$")
|
||||
try:
|
||||
dependency.ctest_lane(build_dir)
|
||||
except dependency.DependencyError as error:
|
||||
assert "meshnet-fixture-fail" in str(error)
|
||||
else:
|
||||
raise AssertionError("a failing named CTest lane must raise DependencyError")
|
||||
|
||||
@@ -19,6 +19,7 @@ fails the test.
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
@@ -371,6 +372,20 @@ def test_shard_runtime_real_subprocess_harness():
|
||||
assert [bytes.fromhex(h) for h in relay_capture["requests"]] == relay_req_bytes
|
||||
assert direct_capture["requests"] == relay_capture["requests"]
|
||||
|
||||
# Wire-frame hashes: the server-persisted SHA-256 over the exact
|
||||
# captured request/response frame bytes must match independently
|
||||
# computed hashes over what the client sent/received, and must be
|
||||
# identical between the direct hop and the opaque relay carry.
|
||||
expected_req_sha256 = hashlib.sha256(b"".join(direct_req_bytes)).hexdigest()
|
||||
expected_resp_sha256 = hashlib.sha256(b"".join(direct_resp_bytes)).hexdigest()
|
||||
assert direct_capture["requests_sha256"] == expected_req_sha256
|
||||
assert direct_capture["responses_sha256"] == expected_resp_sha256
|
||||
assert relay_capture["requests_sha256"] == expected_req_sha256
|
||||
assert relay_capture["responses_sha256"] == expected_resp_sha256
|
||||
print(
|
||||
f"wire-frame sha256: requests={expected_req_sha256} responses={expected_resp_sha256}"
|
||||
)
|
||||
|
||||
channel.close()
|
||||
finally:
|
||||
proc.terminate()
|
||||
|
||||
216
tests/test_split_gguf_manifest.py
Normal file
216
tests/test_split_gguf_manifest.py
Normal file
@@ -0,0 +1,216 @@
|
||||
"""DGR-026 — exact split-GGUF artifact manifest.
|
||||
|
||||
Deterministic, offline, GPU-free, and download-free: every manifest here is a
|
||||
tiny in-memory fixture, never a real model artifact.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
|
||||
from meshnet_node.split_gguf.manifest import (
|
||||
SplitArtifactManifestError,
|
||||
SplitFile,
|
||||
canonical_sha256,
|
||||
parse_split_artifact_manifest,
|
||||
)
|
||||
|
||||
|
||||
def _sha(label: str) -> str:
|
||||
return hashlib.sha256(label.encode()).hexdigest()
|
||||
|
||||
|
||||
def _rev(label: str) -> str:
|
||||
# a syntactically valid 40-hex "revision" derived from a human label
|
||||
return hashlib.sha1(label.encode()).hexdigest()
|
||||
|
||||
|
||||
def _manifest_doc() -> dict:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"manifest_id": "deepseek-v4-flash-q4_k_m-2way",
|
||||
"manifest_version": "2026-07-22.1",
|
||||
"quantization": "Q4_K_M",
|
||||
"source": {
|
||||
"artifact_id": "deepseek-v4-flash",
|
||||
"repo_id": "example/deepseek-v4-flash-gguf",
|
||||
"revision": _rev("source-revision"),
|
||||
"sha256": _sha("whole-model-artifact"),
|
||||
"size_bytes": 2000,
|
||||
},
|
||||
"tokenizer": {
|
||||
"repo_id": "example/deepseek-v4-flash",
|
||||
"revision": _rev("tokenizer-revision"),
|
||||
"sha256": _sha("tokenizer-bytes"),
|
||||
},
|
||||
"total_bytes": 1200,
|
||||
"splits": [
|
||||
{
|
||||
"name": "model-00001-of-00002.gguf",
|
||||
"size_bytes": 700,
|
||||
"sha256": _sha("split-1"),
|
||||
"role": "layers-0-20",
|
||||
"url": "https://example.invalid/model-00001-of-00002.gguf",
|
||||
"shard_start": 0,
|
||||
"shard_end": 20,
|
||||
},
|
||||
{
|
||||
"name": "model-00002-of-00002.gguf",
|
||||
"size_bytes": 500,
|
||||
"sha256": _sha("split-2"),
|
||||
"role": "layers-20-43",
|
||||
"url": "https://example.invalid/model-00002-of-00002.gguf",
|
||||
"shard_start": 20,
|
||||
"shard_end": 43,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manifest_doc() -> dict:
|
||||
return _manifest_doc()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manifest(manifest_doc):
|
||||
return parse_split_artifact_manifest(manifest_doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_manifest_resolves_source_tokenizer_and_both_splits(manifest):
|
||||
assert manifest.source.artifact_id == "deepseek-v4-flash"
|
||||
assert manifest.source.sha256 == _sha("whole-model-artifact")
|
||||
assert manifest.tokenizer.repo_id == "example/deepseek-v4-flash"
|
||||
assert [s.name for s in manifest.splits] == [
|
||||
"model-00001-of-00002.gguf",
|
||||
"model-00002-of-00002.gguf",
|
||||
]
|
||||
assert manifest.split("model-00001-of-00002.gguf").role == "layers-0-20"
|
||||
assert manifest.split("model-00001-of-00002.gguf").shard_start == 0
|
||||
assert manifest.split("model-00001-of-00002.gguf").shard_end == 20
|
||||
|
||||
|
||||
def test_manifest_aggregate_bytes_are_exact_and_self_consistent(manifest):
|
||||
assert manifest.total_bytes == sum(s.size_bytes for s in manifest.splits) == 1200
|
||||
|
||||
|
||||
def test_quantization_and_topology_are_manifest_data_not_constants(manifest_doc):
|
||||
# A manifest with a different quantization label and a different split
|
||||
# count must parse just as validly — nothing in this module hardcodes
|
||||
# either.
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["quantization"] = "IQ2_XS"
|
||||
doc["manifest_id"] = "single-split-example"
|
||||
doc["total_bytes"] = 2000
|
||||
doc["splits"] = [
|
||||
{
|
||||
"name": "model-00001-of-00001.gguf",
|
||||
"size_bytes": 2000,
|
||||
"sha256": _sha("single-split"),
|
||||
"role": "whole",
|
||||
}
|
||||
]
|
||||
manifest = parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
assert manifest.quantization == "IQ2_XS"
|
||||
assert len(manifest.splits) == 1
|
||||
assert manifest.splits[0].has_range is False
|
||||
|
||||
|
||||
def test_manifest_digest_is_stable_canonical_json(manifest, manifest_doc):
|
||||
assert manifest.digest == canonical_sha256(manifest_doc)
|
||||
|
||||
|
||||
def test_split_with_only_shard_start_is_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
del doc["splits"][0]["shard_end"]
|
||||
with pytest.raises(SplitArtifactManifestError, match="both shard_start and shard_end"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_split_with_empty_range_is_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["splits"][0]["shard_end"] = doc["splits"][0]["shard_start"]
|
||||
with pytest.raises(SplitArtifactManifestError, match="shard_end"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_a_missing_split_field_is_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
del doc["splits"][0]["role"]
|
||||
with pytest.raises(SplitArtifactManifestError, match="role"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_a_duplicate_split_name_is_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["splits"][1]["name"] = doc["splits"][0]["name"]
|
||||
with pytest.raises(SplitArtifactManifestError, match="duplicate split name"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_two_splits_claiming_the_same_content_digest_are_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["splits"][1]["sha256"] = doc["splits"][0]["sha256"]
|
||||
with pytest.raises(SplitArtifactManifestError, match="repeats SHA-256"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_an_inconsistent_aggregate_byte_total_is_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["total_bytes"] = 999999
|
||||
with pytest.raises(SplitArtifactManifestError, match="does not equal the sum"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_a_split_size_edited_to_make_the_artifact_look_smaller_is_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["splits"][0]["size_bytes"] = 1
|
||||
with pytest.raises(SplitArtifactManifestError, match="does not equal the sum"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_a_truncated_sha256_is_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["splits"][0]["sha256"] = doc["splits"][0]["sha256"][:10]
|
||||
with pytest.raises(SplitArtifactManifestError, match="64-character hex SHA-256"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_a_branch_name_is_not_an_acceptable_source_revision_pin(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["source"]["revision"] = "main"
|
||||
with pytest.raises(SplitArtifactManifestError, match="40-character commit revision"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_a_branch_name_is_not_an_acceptable_tokenizer_revision_pin(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["tokenizer"]["revision"] = "main"
|
||||
with pytest.raises(SplitArtifactManifestError, match="40-character commit revision"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_an_unsupported_schema_version_is_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["schema_version"] = 2
|
||||
with pytest.raises(SplitArtifactManifestError, match="schema version"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_an_empty_splits_array_is_rejected(manifest_doc):
|
||||
doc = copy.deepcopy(manifest_doc)
|
||||
doc["splits"] = []
|
||||
with pytest.raises(SplitArtifactManifestError, match="non-empty JSON array"):
|
||||
parse_split_artifact_manifest(doc, origin="<fixture>")
|
||||
|
||||
|
||||
def test_split_file_is_a_plain_frozen_dataclass_round_trip():
|
||||
split = SplitFile(
|
||||
name="a.gguf", size_bytes=10, sha256=_sha("a"), role="whole", url="file:///a.gguf"
|
||||
)
|
||||
assert split.has_range is False
|
||||
assert split.to_dict()["name"] == "a.gguf"
|
||||
assert "shard_start" not in split.to_dict()
|
||||
249
tests/test_split_gguf_provision.py
Normal file
249
tests/test_split_gguf_provision.py
Normal file
@@ -0,0 +1,249 @@
|
||||
"""DGR-026 — resumable, hash-verifying split-GGUF provisioning to mounted-drive storage.
|
||||
|
||||
Deterministic, offline, GPU-free, and download-free: every split here is a
|
||||
tiny local fixture file; nothing is downloaded from a network.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
|
||||
from meshnet_node.split_gguf.manifest import parse_split_artifact_manifest
|
||||
from meshnet_node.split_gguf.provision import (
|
||||
SplitProvisionError,
|
||||
local_directory_fetcher,
|
||||
provision_split_artifact,
|
||||
reject_home_path,
|
||||
verify_provisioned_split_artifact,
|
||||
)
|
||||
|
||||
|
||||
def _sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _rev(label: str) -> str:
|
||||
return hashlib.sha1(label.encode()).hexdigest()
|
||||
|
||||
|
||||
SPLIT_A = b"deepseek-v4-flash split A payload bytes " * 100
|
||||
SPLIT_B = b"deepseek-v4-flash split B payload bytes, a bit longer than A " * 130
|
||||
|
||||
|
||||
def _manifest_doc() -> dict:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"manifest_id": "deepseek-v4-flash-fixture",
|
||||
"manifest_version": "test.1",
|
||||
"quantization": "Q4_K_M",
|
||||
"source": {
|
||||
"artifact_id": "deepseek-v4-flash",
|
||||
"repo_id": "example/deepseek-v4-flash-gguf",
|
||||
"revision": _rev("source"),
|
||||
"sha256": _sha256_bytes(b"whole-model"),
|
||||
"size_bytes": len(SPLIT_A) + len(SPLIT_B),
|
||||
},
|
||||
"tokenizer": {
|
||||
"repo_id": "example/deepseek-v4-flash",
|
||||
"revision": _rev("tokenizer"),
|
||||
"sha256": _sha256_bytes(b"tokenizer"),
|
||||
},
|
||||
"total_bytes": len(SPLIT_A) + len(SPLIT_B),
|
||||
"splits": [
|
||||
{
|
||||
"name": "split-a.gguf",
|
||||
"size_bytes": len(SPLIT_A),
|
||||
"sha256": _sha256_bytes(SPLIT_A),
|
||||
"role": "layers-0-20",
|
||||
"shard_start": 0,
|
||||
"shard_end": 20,
|
||||
},
|
||||
{
|
||||
"name": "split-b.gguf",
|
||||
"size_bytes": len(SPLIT_B),
|
||||
"sha256": _sha256_bytes(SPLIT_B),
|
||||
"role": "layers-20-43",
|
||||
"shard_start": 20,
|
||||
"shard_end": 43,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manifest():
|
||||
return parse_split_artifact_manifest(_manifest_doc(), origin="<fixture>")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def source_dir(tmp_path):
|
||||
d = tmp_path / "source"
|
||||
d.mkdir()
|
||||
(d / "split-a.gguf").write_bytes(SPLIT_A)
|
||||
(d / "split-b.gguf").write_bytes(SPLIT_B)
|
||||
return d
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# /home rejection
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_provisioning_refuses_a_destination_under_home(manifest, source_dir):
|
||||
with pytest.raises(SplitProvisionError, match="never /home"):
|
||||
provision_split_artifact(manifest, "/home/someuser/models", local_directory_fetcher(source_dir))
|
||||
|
||||
|
||||
def test_reject_home_path_refuses_home_itself():
|
||||
with pytest.raises(SplitProvisionError, match="never /home"):
|
||||
reject_home_path("/home")
|
||||
|
||||
|
||||
def test_reject_home_path_refuses_a_nested_home_subdirectory():
|
||||
with pytest.raises(SplitProvisionError, match="never /home"):
|
||||
reject_home_path("/home/someuser/.cache/meshnet/models")
|
||||
|
||||
|
||||
def test_reject_home_path_accepts_a_mounted_drive_path(tmp_path):
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
assert reject_home_path(dest) == dest.expanduser().resolve()
|
||||
|
||||
|
||||
def test_verify_provisioned_also_refuses_home(manifest):
|
||||
with pytest.raises(SplitProvisionError, match="never /home"):
|
||||
verify_provisioned_split_artifact(manifest, "/home/someuser/models")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Happy path + idempotent re-run
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_provisioning_fetches_and_verifies_every_split(manifest, source_dir, tmp_path):
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
result = provision_split_artifact(manifest, dest, local_directory_fetcher(source_dir))
|
||||
|
||||
assert result.dest_dir == dest.resolve()
|
||||
assert set(result.verified_splits) == {"split-a.gguf", "split-b.gguf"}
|
||||
assert (dest / "split-a.gguf").read_bytes() == SPLIT_A
|
||||
assert (dest / "split-b.gguf").read_bytes() == SPLIT_B
|
||||
assert not (dest / "split-a.gguf.partial").exists()
|
||||
assert not (dest / "split-b.gguf.partial").exists()
|
||||
|
||||
verify_provisioned_split_artifact(manifest, dest) # does not raise
|
||||
|
||||
|
||||
def test_a_second_provisioning_run_is_a_no_op_over_complete_splits(manifest, source_dir, tmp_path):
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
provision_split_artifact(manifest, dest, local_directory_fetcher(source_dir))
|
||||
|
||||
# Delete the source so a second run could not possibly re-fetch anything;
|
||||
# the already-complete, hash-correct splits must be recognized as done.
|
||||
(source_dir / "split-a.gguf").unlink()
|
||||
(source_dir / "split-b.gguf").unlink()
|
||||
|
||||
result = provision_split_artifact(manifest, dest, local_directory_fetcher(source_dir))
|
||||
assert set(result.verified_splits) == {"split-a.gguf", "split-b.gguf"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Interrupted download → resume
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_an_interrupted_partial_download_resumes_from_its_exact_byte_offset(manifest, source_dir, tmp_path):
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
dest.mkdir(parents=True)
|
||||
|
||||
# Simulate an interrupted prior attempt: split-a is half-written as a
|
||||
# `.partial` file with correct bytes so-far; split-b has not started.
|
||||
cut = len(SPLIT_A) // 2
|
||||
(dest / "split-a.gguf.partial").write_bytes(SPLIT_A[:cut])
|
||||
|
||||
calls: list[tuple[str, int]] = []
|
||||
real_fetch = local_directory_fetcher(source_dir)
|
||||
|
||||
def tracking_fetch(split, dest_path, resume_from_bytes):
|
||||
calls.append((split.name, resume_from_bytes))
|
||||
real_fetch(split, dest_path, resume_from_bytes)
|
||||
|
||||
result = provision_split_artifact(manifest, dest, tracking_fetch)
|
||||
|
||||
assert ("split-a.gguf", cut) in calls # resumed from the exact offset, not from 0
|
||||
assert ("split-b.gguf", 0) in calls
|
||||
assert (dest / "split-a.gguf").read_bytes() == SPLIT_A
|
||||
assert (dest / "split-b.gguf").read_bytes() == SPLIT_B
|
||||
assert set(result.verified_splits) == {"split-a.gguf", "split-b.gguf"}
|
||||
|
||||
|
||||
def test_a_partial_larger_than_the_manifest_size_is_discarded_and_restarted(manifest, source_dir, tmp_path):
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
dest.mkdir(parents=True)
|
||||
(dest / "split-a.gguf.partial").write_bytes(SPLIT_A + b"stray corrupt trailing bytes")
|
||||
|
||||
result = provision_split_artifact(manifest, dest, local_directory_fetcher(source_dir))
|
||||
|
||||
assert (dest / "split-a.gguf").read_bytes() == SPLIT_A
|
||||
assert set(result.verified_splits) == {"split-a.gguf", "split-b.gguf"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Missing split
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_missing_split_source_file_raises(manifest, source_dir, tmp_path):
|
||||
(source_dir / "split-b.gguf").unlink()
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
|
||||
with pytest.raises(SplitProvisionError, match="split source is missing"):
|
||||
provision_split_artifact(manifest, dest, local_directory_fetcher(source_dir))
|
||||
|
||||
|
||||
def test_verify_reports_a_split_missing_from_the_destination(manifest, source_dir, tmp_path):
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
provision_split_artifact(manifest, dest, local_directory_fetcher(source_dir))
|
||||
(dest / "split-b.gguf").unlink()
|
||||
|
||||
with pytest.raises(SplitProvisionError, match="missing split"):
|
||||
verify_provisioned_split_artifact(manifest, dest)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Hash mismatch
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_hash_mismatched_source_file_is_rejected_and_not_left_on_disk(manifest, source_dir, tmp_path):
|
||||
# Same size as the pinned split so the mismatch is caught by hash, not by
|
||||
# the incomplete-byte-count check.
|
||||
(source_dir / "split-a.gguf").write_bytes(b"x" * len(SPLIT_A))
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
|
||||
with pytest.raises(SplitProvisionError, match="hash mismatch"):
|
||||
provision_split_artifact(manifest, dest, local_directory_fetcher(source_dir))
|
||||
|
||||
assert not (dest / "split-a.gguf").exists()
|
||||
assert not (dest / "split-a.gguf.partial").exists()
|
||||
|
||||
|
||||
def test_a_destination_file_with_wrong_hash_is_not_trusted_and_is_replaced(manifest, source_dir, tmp_path):
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
dest.mkdir(parents=True)
|
||||
(dest / "split-a.gguf").write_bytes(b"x" * len(SPLIT_A)) # right size, wrong content
|
||||
|
||||
result = provision_split_artifact(manifest, dest, local_directory_fetcher(source_dir))
|
||||
|
||||
assert (dest / "split-a.gguf").read_bytes() == SPLIT_A
|
||||
assert set(result.verified_splits) == {"split-a.gguf", "split-b.gguf"}
|
||||
|
||||
|
||||
def test_verify_reports_a_hash_mismatched_destination_file(manifest, source_dir, tmp_path):
|
||||
dest = tmp_path / "mnt" / "models"
|
||||
provision_split_artifact(manifest, dest, local_directory_fetcher(source_dir))
|
||||
(dest / "split-a.gguf").write_bytes(b"corrupted after the fact" + SPLIT_A)
|
||||
|
||||
with pytest.raises(SplitProvisionError, match="mismatch"):
|
||||
verify_provisioned_split_artifact(manifest, dest)
|
||||
Reference in New Issue
Block a user