503 lines
17 KiB
Python
503 lines
17 KiB
Python
"""Tests for the model-agnostic capability report and local recipe manifest."""
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from meshnet_node import capability, recipe_manifest
|
|
from meshnet_node.capability import (
|
|
CAPABILITY_SCHEMA_VERSION,
|
|
CapabilityReport,
|
|
CapabilityReportError,
|
|
build_capability_report,
|
|
config_fingerprint,
|
|
sanitize_diagnostic,
|
|
sanitize_diagnostics,
|
|
)
|
|
from meshnet_node.recipe_manifest import (
|
|
RECIPE_SCHEMA_VERSION,
|
|
RecipeManifestError,
|
|
load_recipe_manifest,
|
|
parse_recipe_manifest,
|
|
)
|
|
|
|
# Deliberately unrelated to any vendor the network ships against: the report
|
|
# must carry whatever the operator selected, verbatim.
|
|
FIXTURE_MODEL_A = "acme-labs/Widget-9000-Instruct"
|
|
FIXTURE_MODEL_B = "some_org/tiny.model-v2_PREVIEW"
|
|
|
|
|
|
def _report(**overrides):
|
|
kwargs = dict(
|
|
model_id=FIXTURE_MODEL_A,
|
|
shard_start=0,
|
|
shard_end=7,
|
|
recipe_id="baseline",
|
|
recipe_version="1",
|
|
catalogue_version="2026.07.1",
|
|
backend_id="torch-transformers",
|
|
device="test-device",
|
|
status="passed",
|
|
duration_ms=142,
|
|
validated_at=1_760_000_000.0,
|
|
)
|
|
kwargs.update(overrides)
|
|
return build_capability_report(**kwargs)
|
|
|
|
|
|
# --- model-agnostic identity ------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("model_id", [FIXTURE_MODEL_A, FIXTURE_MODEL_B])
|
|
def test_arbitrary_model_ids_survive_a_json_round_trip_verbatim(model_id):
|
|
"Arbitrary model ids survive a json round trip verbatim\n\nTags: node, startup"
|
|
report = _report(model_id=model_id)
|
|
restored = CapabilityReport.from_json(report.to_json())
|
|
|
|
assert restored.model.model_id == model_id
|
|
assert restored.to_dict() == report.to_dict()
|
|
|
|
|
|
def test_two_arbitrary_models_stay_distinct_without_normalization():
|
|
"Two arbitrary models stay distinct without normalization\n\nTags: node, startup"
|
|
a = _report(model_id=FIXTURE_MODEL_A)
|
|
b = _report(model_id=FIXTURE_MODEL_B)
|
|
|
|
assert a.identity_key() != b.identity_key()
|
|
assert a.model.model_id != b.model.model_id
|
|
|
|
|
|
def test_capability_and_recipe_modules_have_no_model_or_kernel_branch():
|
|
"No vendor/model/kernel name may be a default or a code-path discriminator.\n\nTags: node, startup"
|
|
|
|
forbidden = re.compile(
|
|
r"qwen|triton|\bfla\b|flash[_-]?attn|flash[_-]?attention|rocm|nvidia|\bcuda\b",
|
|
re.IGNORECASE,
|
|
)
|
|
sources = [
|
|
Path(capability.__file__),
|
|
Path(recipe_manifest.__file__),
|
|
Path(recipe_manifest.__file__).with_name("recipes.json"),
|
|
]
|
|
for source in sources:
|
|
hits = forbidden.findall(source.read_text(encoding="utf-8"))
|
|
assert not hits, f"{source.name} names {hits} — recipes must stay generic data"
|
|
|
|
|
|
def test_device_is_an_opaque_label():
|
|
"Device is an opaque label\n\nTags: node, startup"
|
|
report = _report(device="some-accelerator", device_name="Vendor Accelerator XT")
|
|
restored = CapabilityReport.from_json(report.to_json())
|
|
|
|
assert restored.backend.device == "some-accelerator"
|
|
assert restored.backend.device_name == "Vendor Accelerator XT"
|
|
|
|
|
|
# --- schema and serialization -----------------------------------------------
|
|
|
|
|
|
def test_report_dict_has_the_stable_documented_key_set():
|
|
"Report dict has the stable documented key set\n\nTags: node, startup"
|
|
payload = _report(
|
|
model_config={"num_hidden_layers": 8},
|
|
revision="a1b2c3",
|
|
quantization="int8",
|
|
runtime={"torch": "2.9.0"},
|
|
diagnostics=["loaded 8 layers"],
|
|
).to_dict()
|
|
|
|
assert set(payload) == {
|
|
"schema_version",
|
|
"model",
|
|
"shard",
|
|
"recipe",
|
|
"backend",
|
|
"status",
|
|
"validated_at",
|
|
"duration_ms",
|
|
"diagnostics",
|
|
}
|
|
assert payload["schema_version"] == CAPABILITY_SCHEMA_VERSION
|
|
assert set(payload["model"]) == {"model_id", "revision", "config_fingerprint"}
|
|
assert set(payload["shard"]) == {"start", "end"}
|
|
assert set(payload["recipe"]) == {
|
|
"recipe_id",
|
|
"recipe_version",
|
|
"catalogue_version",
|
|
}
|
|
assert set(payload["backend"]) == {
|
|
"backend_id",
|
|
"device",
|
|
"device_name",
|
|
"quantization",
|
|
"runtime",
|
|
}
|
|
# JSON-serializable end to end.
|
|
assert json.loads(json.dumps(payload)) == payload
|
|
|
|
|
|
def test_identity_key_pins_model_shard_recipe_and_backend():
|
|
"Identity key pins model shard recipe and backend\n\nTags: node, startup"
|
|
base = _report()
|
|
|
|
assert base.identity_key() == (
|
|
FIXTURE_MODEL_A,
|
|
0,
|
|
7,
|
|
"baseline",
|
|
"1",
|
|
"torch-transformers",
|
|
"test-device",
|
|
)
|
|
assert _report(shard_end=8).identity_key() != base.identity_key()
|
|
assert _report(recipe_version="2").identity_key() != base.identity_key()
|
|
assert _report(device="other-device").identity_key() != base.identity_key()
|
|
|
|
|
|
def test_config_fingerprint_is_stable_under_key_order_and_detects_change():
|
|
"Config fingerprint is stable under key order and detects change\n\nTags: node, startup"
|
|
a = config_fingerprint({"num_hidden_layers": 8, "hidden_size": 512})
|
|
b = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 8})
|
|
c = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 9})
|
|
|
|
assert a == b
|
|
assert a != c
|
|
assert a.startswith("sha256:")
|
|
assert config_fingerprint(None) is None
|
|
assert config_fingerprint("sha256:deadbeef") == "sha256:deadbeef"
|
|
|
|
|
|
def test_status_and_passed_flag():
|
|
"Status and passed flag\n\nTags: node, startup"
|
|
assert _report(status="passed").passed
|
|
assert not _report(status="failed").passed
|
|
assert not _report(status="skipped").passed
|
|
|
|
|
|
# --- malformed report input -------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"overrides, expected",
|
|
[
|
|
({"model_id": ""}, "model.model_id"),
|
|
({"shard_start": -1}, "shard.start"),
|
|
({"shard_start": 5, "shard_end": 2}, "shard.end"),
|
|
({"recipe_id": ""}, "recipe.recipe_id"),
|
|
({"catalogue_version": ""}, "recipe.catalogue_version"),
|
|
({"backend_id": ""}, "backend.backend_id"),
|
|
({"device": ""}, "backend.device"),
|
|
({"status": "maybe"}, "status"),
|
|
({"duration_ms": -5}, "duration_ms"),
|
|
],
|
|
)
|
|
def test_malformed_report_fields_name_the_offending_field(overrides, expected):
|
|
"Malformed report fields name the offending field\n\nTags: node, startup"
|
|
with pytest.raises(CapabilityReportError) as exc:
|
|
_report(**overrides)
|
|
assert expected in str(exc.value)
|
|
|
|
|
|
def test_unsupported_report_schema_version_is_actionable():
|
|
"Unsupported report schema version is actionable\n\nTags: node, startup"
|
|
payload = _report().to_dict()
|
|
payload["schema_version"] = 99
|
|
|
|
with pytest.raises(CapabilityReportError) as exc:
|
|
CapabilityReport.from_dict(payload)
|
|
message = str(exc.value)
|
|
assert "99" in message
|
|
assert str(CAPABILITY_SCHEMA_VERSION) in message
|
|
|
|
|
|
def test_missing_schema_version_is_rejected():
|
|
"Missing schema version is rejected\n\nTags: node, startup"
|
|
payload = _report().to_dict()
|
|
del payload["schema_version"]
|
|
|
|
with pytest.raises(CapabilityReportError, match="schema_version"):
|
|
CapabilityReport.from_dict(payload)
|
|
|
|
|
|
def test_malformed_report_json_reports_position_not_content():
|
|
"Malformed report json reports position not content\n\nTags: node, startup"
|
|
with pytest.raises(CapabilityReportError) as exc:
|
|
CapabilityReport.from_json('{"schema_version": 1,')
|
|
assert "line 1" in str(exc.value)
|
|
|
|
|
|
def test_missing_report_section_is_named():
|
|
"Missing report section is named\n\nTags: node, startup"
|
|
payload = _report().to_dict()
|
|
payload["backend"] = "torch"
|
|
|
|
with pytest.raises(CapabilityReportError, match="backend"):
|
|
CapabilityReport.from_dict(payload)
|
|
|
|
|
|
# --- diagnostics sanitization -----------------------------------------------
|
|
|
|
|
|
def test_diagnostics_redact_secret_env_values(monkeypatch):
|
|
"Diagnostics redact secret env values\n\nTags: node, startup"
|
|
monkeypatch.setenv("MESHNET_API_TOKEN", "super-secret-value-123")
|
|
monkeypatch.setenv("MESHNET_MODEL_ID", "acme-labs/Widget-9000-Instruct")
|
|
|
|
report = _report(
|
|
status="failed",
|
|
diagnostics=["auth failed for super-secret-value-123 while fetching config"],
|
|
)
|
|
text = report.to_json()
|
|
|
|
assert "super-secret-value-123" not in text
|
|
assert capability.REDACTED in report.diagnostics[0]
|
|
# A non-secret env var is left alone — the model id stays readable.
|
|
assert "acme-labs/Widget-9000-Instruct" in text
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"raw",
|
|
[
|
|
"401 Unauthorized: hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345",
|
|
"request used Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig",
|
|
"config had api_key=abcdef0123456789",
|
|
"openai style sk-abcdefghijklmnopqrstuvwxyz012345",
|
|
],
|
|
)
|
|
def test_diagnostics_redact_credential_shaped_strings(raw):
|
|
"Diagnostics redact credential shaped strings\n\nTags: node, startup"
|
|
cleaned = sanitize_diagnostic(raw, environ={})
|
|
assert capability.REDACTED in cleaned
|
|
for secret in (
|
|
"hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345",
|
|
"eyJhbGciOiJIUzI1NiJ9.payload.sig",
|
|
"abcdef0123456789",
|
|
"sk-abcdefghijklmnopqrstuvwxyz012345",
|
|
):
|
|
assert secret not in cleaned
|
|
|
|
|
|
def test_diagnostics_strip_the_home_directory(monkeypatch, tmp_path):
|
|
"Diagnostics strip the home directory\n\nTags: node, startup"
|
|
monkeypatch.setenv("HOME", str(tmp_path))
|
|
cleaned = sanitize_diagnostic(f"missing weights at {tmp_path}/models/shard", environ={})
|
|
|
|
assert str(tmp_path) not in cleaned
|
|
assert cleaned.startswith("missing weights at ~/models/shard")
|
|
|
|
|
|
def test_diagnostics_are_bounded_in_length_and_count():
|
|
"Diagnostics are bounded in length and count\n\nTags: node, startup"
|
|
long_line = sanitize_diagnostic("x" * 2000, environ={})
|
|
assert len(long_line) <= capability.MAX_DIAGNOSTIC_CHARS
|
|
|
|
many = sanitize_diagnostics([f"line {i}" for i in range(50)], environ={})
|
|
assert len(many) == capability.MAX_DIAGNOSTICS + 1
|
|
assert "further diagnostic(s) omitted" in many[-1]
|
|
|
|
|
|
def test_diagnostics_reject_non_string_entries():
|
|
"Diagnostics reject non string entries\n\nTags: node, startup"
|
|
with pytest.raises(CapabilityReportError, match=r"diagnostics\[1\]"):
|
|
sanitize_diagnostics(["ok", 42], environ={})
|
|
|
|
with pytest.raises(CapabilityReportError, match="bare string"):
|
|
sanitize_diagnostics("one big string", environ={})
|
|
|
|
|
|
def test_deserializing_a_report_re_sanitizes_diagnostics(monkeypatch):
|
|
"Deserializing a report re sanitizes diagnostics\n\nTags: node, startup"
|
|
monkeypatch.setenv("NODE_SECRET", "leak-me-please")
|
|
payload = _report().to_dict()
|
|
payload["diagnostics"] = ["backend said leak-me-please"]
|
|
|
|
restored = CapabilityReport.from_dict(payload)
|
|
assert "leak-me-please" not in restored.to_json()
|
|
|
|
|
|
# --- recipe manifest --------------------------------------------------------
|
|
|
|
|
|
def test_packaged_manifest_loads_with_explicit_versions():
|
|
"Packaged manifest loads with explicit versions\n\nTags: node, startup"
|
|
manifest = load_recipe_manifest()
|
|
|
|
assert manifest.schema_version == RECIPE_SCHEMA_VERSION
|
|
assert manifest.catalogue_version
|
|
assert recipe_manifest.DEFAULT_RECIPE_ID in manifest.ids
|
|
for recipe in manifest.recipes:
|
|
assert recipe.id and recipe.version and recipe.backend_id
|
|
assert isinstance(recipe.params, dict)
|
|
|
|
|
|
def test_packaged_manifest_feeds_a_report():
|
|
"Packaged manifest feeds a report\n\nTags: node, startup"
|
|
manifest = load_recipe_manifest()
|
|
recipe = manifest.require(recipe_manifest.DEFAULT_RECIPE_ID)
|
|
|
|
report = _report(
|
|
recipe_id=recipe.id,
|
|
recipe_version=recipe.version,
|
|
catalogue_version=manifest.catalogue_version,
|
|
backend_id=recipe.backend_id,
|
|
)
|
|
assert report.recipe.catalogue_version == manifest.catalogue_version
|
|
|
|
|
|
def test_unknown_recipe_lists_available_ids():
|
|
"Unknown recipe lists available ids\n\nTags: node, startup"
|
|
manifest = load_recipe_manifest()
|
|
|
|
with pytest.raises(RecipeManifestError) as exc:
|
|
manifest.require("no-such-recipe")
|
|
message = str(exc.value)
|
|
assert "no-such-recipe" in message
|
|
assert recipe_manifest.DEFAULT_RECIPE_ID in message
|
|
|
|
|
|
def _write_manifest(tmp_path: Path, doc) -> Path:
|
|
path = tmp_path / "recipes.json"
|
|
path.write_text(
|
|
doc if isinstance(doc, str) else json.dumps(doc), encoding="utf-8"
|
|
)
|
|
return path
|
|
|
|
|
|
def test_valid_local_manifest_loads(tmp_path):
|
|
"Valid local manifest loads\n\nTags: node, startup"
|
|
path = _write_manifest(
|
|
tmp_path,
|
|
{
|
|
"schema_version": RECIPE_SCHEMA_VERSION,
|
|
"catalogue_version": "2099.01.0-test",
|
|
"recipes": [
|
|
{
|
|
"id": "custom",
|
|
"version": "3",
|
|
"backend_id": "some-backend",
|
|
"params": {"knob": 1},
|
|
}
|
|
],
|
|
},
|
|
)
|
|
manifest = load_recipe_manifest(path)
|
|
|
|
assert manifest.catalogue_version == "2099.01.0-test"
|
|
assert manifest.require("custom").params == {"knob": 1}
|
|
assert manifest.source == str(path)
|
|
|
|
|
|
def test_unknown_manifest_schema_version_is_actionable(tmp_path):
|
|
"Unknown manifest schema version is actionable\n\nTags: node, startup"
|
|
path = _write_manifest(
|
|
tmp_path,
|
|
{
|
|
"schema_version": 99,
|
|
"catalogue_version": "x",
|
|
"recipes": [{"id": "a", "version": "1", "backend_id": "b"}],
|
|
},
|
|
)
|
|
with pytest.raises(RecipeManifestError) as exc:
|
|
load_recipe_manifest(path)
|
|
message = str(exc.value)
|
|
assert "99" in message
|
|
assert str(RECIPE_SCHEMA_VERSION) in message
|
|
assert str(path) in message
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"doc, expected",
|
|
[
|
|
({"catalogue_version": "x", "recipes": []}, "schema_version"),
|
|
(
|
|
{"schema_version": RECIPE_SCHEMA_VERSION, "recipes": [{"id": "a"}]},
|
|
"catalogue_version",
|
|
),
|
|
(
|
|
{"schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": []},
|
|
"non-empty JSON array",
|
|
),
|
|
(
|
|
{
|
|
"schema_version": RECIPE_SCHEMA_VERSION,
|
|
"catalogue_version": "x",
|
|
"recipes": [{"version": "1", "backend_id": "b"}],
|
|
},
|
|
"recipes[0].id",
|
|
),
|
|
(
|
|
{
|
|
"schema_version": RECIPE_SCHEMA_VERSION,
|
|
"catalogue_version": "x",
|
|
"recipes": [{"id": "a", "backend_id": "b"}],
|
|
},
|
|
"recipes[a].version",
|
|
),
|
|
(
|
|
{
|
|
"schema_version": RECIPE_SCHEMA_VERSION,
|
|
"catalogue_version": "x",
|
|
"recipes": [{"id": "a", "version": "1"}],
|
|
},
|
|
"recipes[a].backend_id",
|
|
),
|
|
(
|
|
{
|
|
"schema_version": RECIPE_SCHEMA_VERSION,
|
|
"catalogue_version": "x",
|
|
"recipes": [
|
|
{"id": "a", "version": "1", "backend_id": "b", "params": [1, 2]}
|
|
],
|
|
},
|
|
"recipes[a].params",
|
|
),
|
|
(
|
|
{
|
|
"schema_version": RECIPE_SCHEMA_VERSION,
|
|
"catalogue_version": "x",
|
|
"recipes": [
|
|
{"id": "a", "version": "1", "backend_id": "b"},
|
|
{"id": "a", "version": "2", "backend_id": "b"},
|
|
],
|
|
},
|
|
"duplicate recipe id",
|
|
),
|
|
],
|
|
)
|
|
def test_malformed_manifest_names_the_offending_field(tmp_path, doc, expected):
|
|
"Malformed manifest names the offending field\n\nTags: node, startup"
|
|
path = _write_manifest(tmp_path, doc)
|
|
|
|
with pytest.raises(RecipeManifestError) as exc:
|
|
load_recipe_manifest(path)
|
|
assert expected in str(exc.value)
|
|
|
|
|
|
def test_malformed_manifest_json_reports_position_not_content(tmp_path):
|
|
"Malformed manifest json reports position not content\n\nTags: node, startup"
|
|
path = _write_manifest(tmp_path, '{"schema_version": 1, "catalogue_version":')
|
|
|
|
with pytest.raises(RecipeManifestError) as exc:
|
|
load_recipe_manifest(path)
|
|
message = str(exc.value)
|
|
assert "not valid JSON" in message
|
|
assert "line 1" in message
|
|
# The failing document body is never echoed back.
|
|
assert "catalogue_version\":" not in message
|
|
|
|
|
|
def test_missing_manifest_file_is_actionable(tmp_path):
|
|
"Missing manifest file is actionable\n\nTags: node, startup"
|
|
with pytest.raises(RecipeManifestError, match="cannot read recipe manifest"):
|
|
load_recipe_manifest(tmp_path / "nope.json")
|
|
|
|
|
|
def test_manifest_round_trips_through_its_own_dict():
|
|
"Manifest round trips through its own dict\n\nTags: node, startup"
|
|
manifest = load_recipe_manifest()
|
|
reparsed = parse_recipe_manifest(manifest.to_dict(), source="<memory>")
|
|
|
|
assert reparsed.to_dict() == manifest.to_dict()
|