popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test grouping

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-11 22:11:21 +03:00
parent c195b5ce78
commit 7d259d7c9b
9 changed files with 1424 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

474
tests/test_node_capability.py Normal file
View File

@@ -0,0 +1,474 @@
"""Tests for the model-agnostic capability report and local recipe manifest."""
import json
import re
from pathlib import Path
import pytest
from meshnet_node import capability, recipe_manifest
from meshnet_node.capability import (
CAPABILITY_SCHEMA_VERSION,
CapabilityReport,
CapabilityReportError,
build_capability_report,
config_fingerprint,
sanitize_diagnostic,
sanitize_diagnostics,
)
from meshnet_node.recipe_manifest import (
RECIPE_SCHEMA_VERSION,
RecipeManifestError,
load_recipe_manifest,
parse_recipe_manifest,
)
# Deliberately unrelated to any vendor the network ships against: the report
# must carry whatever the operator selected, verbatim.
FIXTURE_MODEL_A = "acme-labs/Widget-9000-Instruct"
FIXTURE_MODEL_B = "some_org/tiny.model-v2_PREVIEW"
def _report(**overrides):
kwargs = dict(
model_id=FIXTURE_MODEL_A,
shard_start=0,
shard_end=7,
recipe_id="baseline",
recipe_version="1",
catalogue_version="2026.07.1",
backend_id="torch-transformers",
device="test-device",
status="passed",
duration_ms=142,
validated_at=1_760_000_000.0,
)
kwargs.update(overrides)
return build_capability_report(**kwargs)
# --- model-agnostic identity ------------------------------------------------
@pytest.mark.parametrize("model_id", [FIXTURE_MODEL_A, FIXTURE_MODEL_B])
def test_arbitrary_model_ids_survive_a_json_round_trip_verbatim(model_id):
report = _report(model_id=model_id)
restored = CapabilityReport.from_json(report.to_json())
assert restored.model.model_id == model_id
assert restored.to_dict() == report.to_dict()
def test_two_arbitrary_models_stay_distinct_without_normalization():
a = _report(model_id=FIXTURE_MODEL_A)
b = _report(model_id=FIXTURE_MODEL_B)
assert a.identity_key() != b.identity_key()
assert a.model.model_id != b.model.model_id
def test_capability_and_recipe_modules_have_no_model_or_kernel_branch():
"""No vendor/model/kernel name may be a default or a code-path discriminator."""
forbidden = re.compile(
r"qwen|triton|\bfla\b|flash[_-]?attn|flash[_-]?attention|rocm|nvidia|\bcuda\b",
re.IGNORECASE,
)
sources = [
Path(capability.__file__),
Path(recipe_manifest.__file__),
Path(recipe_manifest.__file__).with_name("recipes.json"),
]
for source in sources:
hits = forbidden.findall(source.read_text(encoding="utf-8"))
assert not hits, f"{source.name} names {hits} — recipes must stay generic data"
def test_device_is_an_opaque_label():
report = _report(device="some-accelerator", device_name="Vendor Accelerator XT")
restored = CapabilityReport.from_json(report.to_json())
assert restored.backend.device == "some-accelerator"
assert restored.backend.device_name == "Vendor Accelerator XT"
# --- schema and serialization -----------------------------------------------
def test_report_dict_has_the_stable_documented_key_set():
payload = _report(
model_config={"num_hidden_layers": 8},
revision="a1b2c3",
quantization="int8",
runtime={"torch": "2.9.0"},
diagnostics=["loaded 8 layers"],
).to_dict()
assert set(payload) == {
"schema_version",
"model",
"shard",
"recipe",
"backend",
"status",
"validated_at",
"duration_ms",
"diagnostics",
}
assert payload["schema_version"] == CAPABILITY_SCHEMA_VERSION
assert set(payload["model"]) == {"model_id", "revision", "config_fingerprint"}
assert set(payload["shard"]) == {"start", "end"}
assert set(payload["recipe"]) == {
"recipe_id",
"recipe_version",
"catalogue_version",
}
assert set(payload["backend"]) == {
"backend_id",
"device",
"device_name",
"quantization",
"runtime",
}
# JSON-serializable end to end.
assert json.loads(json.dumps(payload)) == payload
def test_identity_key_pins_model_shard_recipe_and_backend():
base = _report()
assert base.identity_key() == (
FIXTURE_MODEL_A,
0,
7,
"baseline",
"1",
"torch-transformers",
"test-device",
)
assert _report(shard_end=8).identity_key() != base.identity_key()
assert _report(recipe_version="2").identity_key() != base.identity_key()
assert _report(device="other-device").identity_key() != base.identity_key()
def test_config_fingerprint_is_stable_under_key_order_and_detects_change():
a = config_fingerprint({"num_hidden_layers": 8, "hidden_size": 512})
b = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 8})
c = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 9})
assert a == b
assert a != c
assert a.startswith("sha256:")
assert config_fingerprint(None) is None
assert config_fingerprint("sha256:deadbeef") == "sha256:deadbeef"
def test_status_and_passed_flag():
assert _report(status="passed").passed
assert not _report(status="failed").passed
assert not _report(status="skipped").passed
# --- malformed report input -------------------------------------------------
@pytest.mark.parametrize(
"overrides, expected",
[
({"model_id": ""}, "model.model_id"),
({"shard_start": -1}, "shard.start"),
({"shard_start": 5, "shard_end": 2}, "shard.end"),
({"recipe_id": ""}, "recipe.recipe_id"),
({"catalogue_version": ""}, "recipe.catalogue_version"),
({"backend_id": ""}, "backend.backend_id"),
({"device": ""}, "backend.device"),
({"status": "maybe"}, "status"),
({"duration_ms": -5}, "duration_ms"),
],
)
def test_malformed_report_fields_name_the_offending_field(overrides, expected):
with pytest.raises(CapabilityReportError) as exc:
_report(**overrides)
assert expected in str(exc.value)
def test_unsupported_report_schema_version_is_actionable():
payload = _report().to_dict()
payload["schema_version"] = 99
with pytest.raises(CapabilityReportError) as exc:
CapabilityReport.from_dict(payload)
message = str(exc.value)
assert "99" in message
assert str(CAPABILITY_SCHEMA_VERSION) in message
def test_missing_schema_version_is_rejected():
payload = _report().to_dict()
del payload["schema_version"]
with pytest.raises(CapabilityReportError, match="schema_version"):
CapabilityReport.from_dict(payload)
def test_malformed_report_json_reports_position_not_content():
with pytest.raises(CapabilityReportError) as exc:
CapabilityReport.from_json('{"schema_version": 1,')
assert "line 1" in str(exc.value)
def test_missing_report_section_is_named():
payload = _report().to_dict()
payload["backend"] = "torch"
with pytest.raises(CapabilityReportError, match="backend"):
CapabilityReport.from_dict(payload)
# --- diagnostics sanitization -----------------------------------------------
def test_diagnostics_redact_secret_env_values(monkeypatch):
monkeypatch.setenv("MESHNET_API_TOKEN", "super-secret-value-123")
monkeypatch.setenv("MESHNET_MODEL_ID", "acme-labs/Widget-9000-Instruct")
report = _report(
status="failed",
diagnostics=["auth failed for super-secret-value-123 while fetching config"],
)
text = report.to_json()
assert "super-secret-value-123" not in text
assert capability.REDACTED in report.diagnostics[0]
# A non-secret env var is left alone — the model id stays readable.
assert "acme-labs/Widget-9000-Instruct" in text
@pytest.mark.parametrize(
"raw",
[
"401 Unauthorized: hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345",
"request used Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig",
"config had api_key=abcdef0123456789",
"openai style sk-abcdefghijklmnopqrstuvwxyz012345",
],
)
def test_diagnostics_redact_credential_shaped_strings(raw):
cleaned = sanitize_diagnostic(raw, environ={})
assert capability.REDACTED in cleaned
for secret in (
"hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345",
"eyJhbGciOiJIUzI1NiJ9.payload.sig",
"abcdef0123456789",
"sk-abcdefghijklmnopqrstuvwxyz012345",
):
assert secret not in cleaned
def test_diagnostics_strip_the_home_directory(monkeypatch, tmp_path):
monkeypatch.setenv("HOME", str(tmp_path))
cleaned = sanitize_diagnostic(f"missing weights at {tmp_path}/models/shard", environ={})
assert str(tmp_path) not in cleaned
assert cleaned.startswith("missing weights at ~/models/shard")
def test_diagnostics_are_bounded_in_length_and_count():
long_line = sanitize_diagnostic("x" * 2000, environ={})
assert len(long_line) <= capability.MAX_DIAGNOSTIC_CHARS
many = sanitize_diagnostics([f"line {i}" for i in range(50)], environ={})
assert len(many) == capability.MAX_DIAGNOSTICS + 1
assert "further diagnostic(s) omitted" in many[-1]
def test_diagnostics_reject_non_string_entries():
with pytest.raises(CapabilityReportError, match=r"diagnostics\[1\]"):
sanitize_diagnostics(["ok", 42], environ={})
with pytest.raises(CapabilityReportError, match="bare string"):
sanitize_diagnostics("one big string", environ={})
def test_deserializing_a_report_re_sanitizes_diagnostics(monkeypatch):
monkeypatch.setenv("NODE_SECRET", "leak-me-please")
payload = _report().to_dict()
payload["diagnostics"] = ["backend said leak-me-please"]
restored = CapabilityReport.from_dict(payload)
assert "leak-me-please" not in restored.to_json()
# --- recipe manifest --------------------------------------------------------
def test_packaged_manifest_loads_with_explicit_versions():
manifest = load_recipe_manifest()
assert manifest.schema_version == RECIPE_SCHEMA_VERSION
assert manifest.catalogue_version
assert recipe_manifest.DEFAULT_RECIPE_ID in manifest.ids
for recipe in manifest.recipes:
assert recipe.id and recipe.version and recipe.backend_id
assert isinstance(recipe.params, dict)
def test_packaged_manifest_feeds_a_report():
manifest = load_recipe_manifest()
recipe = manifest.require(recipe_manifest.DEFAULT_RECIPE_ID)
report = _report(
recipe_id=recipe.id,
recipe_version=recipe.version,
catalogue_version=manifest.catalogue_version,
backend_id=recipe.backend_id,
)
assert report.recipe.catalogue_version == manifest.catalogue_version
def test_unknown_recipe_lists_available_ids():
manifest = load_recipe_manifest()
with pytest.raises(RecipeManifestError) as exc:
manifest.require("no-such-recipe")
message = str(exc.value)
assert "no-such-recipe" in message
assert recipe_manifest.DEFAULT_RECIPE_ID in message
def _write_manifest(tmp_path: Path, doc) -> Path:
path = tmp_path / "recipes.json"
path.write_text(
doc if isinstance(doc, str) else json.dumps(doc), encoding="utf-8"
)
return path
def test_valid_local_manifest_loads(tmp_path):
path = _write_manifest(
tmp_path,
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "2099.01.0-test",
"recipes": [
{
"id": "custom",
"version": "3",
"backend_id": "some-backend",
"params": {"knob": 1},
}
],
},
)
manifest = load_recipe_manifest(path)
assert manifest.catalogue_version == "2099.01.0-test"
assert manifest.require("custom").params == {"knob": 1}
assert manifest.source == str(path)
def test_unknown_manifest_schema_version_is_actionable(tmp_path):
path = _write_manifest(
tmp_path,
{
"schema_version": 99,
"catalogue_version": "x",
"recipes": [{"id": "a", "version": "1", "backend_id": "b"}],
},
)
with pytest.raises(RecipeManifestError) as exc:
load_recipe_manifest(path)
message = str(exc.value)
assert "99" in message
assert str(RECIPE_SCHEMA_VERSION) in message
assert str(path) in message
@pytest.mark.parametrize(
"doc, expected",
[
({"catalogue_version": "x", "recipes": []}, "schema_version"),
(
{"schema_version": RECIPE_SCHEMA_VERSION, "recipes": [{"id": "a"}]},
"catalogue_version",
),
(
{"schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": []},
"non-empty JSON array",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [{"version": "1", "backend_id": "b"}],
},
"recipes[0].id",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [{"id": "a", "backend_id": "b"}],
},
"recipes[a].version",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [{"id": "a", "version": "1"}],
},
"recipes[a].backend_id",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [
{"id": "a", "version": "1", "backend_id": "b", "params": [1, 2]}
],
},
"recipes[a].params",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [
{"id": "a", "version": "1", "backend_id": "b"},
{"id": "a", "version": "2", "backend_id": "b"},
],
},
"duplicate recipe id",
),
],
)
def test_malformed_manifest_names_the_offending_field(tmp_path, doc, expected):
path = _write_manifest(tmp_path, doc)
with pytest.raises(RecipeManifestError) as exc:
load_recipe_manifest(path)
assert expected in str(exc.value)
def test_malformed_manifest_json_reports_position_not_content(tmp_path):
path = _write_manifest(tmp_path, '{"schema_version": 1, "catalogue_version":')
with pytest.raises(RecipeManifestError) as exc:
load_recipe_manifest(path)
message = str(exc.value)
assert "not valid JSON" in message
assert "line 1" in message
# The failing document body is never echoed back.
assert "catalogue_version\":" not in message
def test_missing_manifest_file_is_actionable(tmp_path):
with pytest.raises(RecipeManifestError, match="cannot read recipe manifest"):
load_recipe_manifest(tmp_path / "nope.json")
def test_manifest_round_trips_through_its_own_dict():
manifest = load_recipe_manifest()
reparsed = parse_recipe_manifest(manifest.to_dict(), source="<memory>")
assert reparsed.to_dict() == manifest.to_dict()