"""Tests for the model-agnostic capability report and local recipe manifest.""" import json import re from pathlib import Path import pytest from meshnet_node import capability, recipe_manifest from meshnet_node.capability import ( CAPABILITY_SCHEMA_VERSION, CapabilityReport, CapabilityReportError, build_capability_report, config_fingerprint, sanitize_diagnostic, sanitize_diagnostics, ) from meshnet_node.recipe_manifest import ( RECIPE_SCHEMA_VERSION, RecipeManifestError, load_recipe_manifest, parse_recipe_manifest, ) # Deliberately unrelated to any vendor the network ships against: the report # must carry whatever the operator selected, verbatim. FIXTURE_MODEL_A = "acme-labs/Widget-9000-Instruct" FIXTURE_MODEL_B = "some_org/tiny.model-v2_PREVIEW" def _report(**overrides): kwargs = dict( model_id=FIXTURE_MODEL_A, shard_start=0, shard_end=7, recipe_id="baseline", recipe_version="1", catalogue_version="2026.07.1", backend_id="torch-transformers", device="test-device", status="passed", duration_ms=142, validated_at=1_760_000_000.0, ) kwargs.update(overrides) return build_capability_report(**kwargs) # --- model-agnostic identity ------------------------------------------------ @pytest.mark.parametrize("model_id", [FIXTURE_MODEL_A, FIXTURE_MODEL_B]) def test_arbitrary_model_ids_survive_a_json_round_trip_verbatim(model_id): "Arbitrary model ids survive a json round trip verbatim\n\nTags: node, startup" report = _report(model_id=model_id) restored = CapabilityReport.from_json(report.to_json()) assert restored.model.model_id == model_id assert restored.to_dict() == report.to_dict() def test_two_arbitrary_models_stay_distinct_without_normalization(): "Two arbitrary models stay distinct without normalization\n\nTags: node, startup" a = _report(model_id=FIXTURE_MODEL_A) b = _report(model_id=FIXTURE_MODEL_B) assert a.identity_key() != b.identity_key() assert a.model.model_id != b.model.model_id def test_capability_and_recipe_modules_have_no_model_or_kernel_branch(): "No vendor/model/kernel name may be a default or a code-path discriminator.\n\nTags: node, startup" forbidden = re.compile( r"qwen|triton|\bfla\b|flash[_-]?attn|flash[_-]?attention|rocm|nvidia|\bcuda\b", re.IGNORECASE, ) sources = [ Path(capability.__file__), Path(recipe_manifest.__file__), Path(recipe_manifest.__file__).with_name("recipes.json"), ] for source in sources: hits = forbidden.findall(source.read_text(encoding="utf-8")) assert not hits, f"{source.name} names {hits} — recipes must stay generic data" def test_device_is_an_opaque_label(): "Device is an opaque label\n\nTags: node, startup" report = _report(device="some-accelerator", device_name="Vendor Accelerator XT") restored = CapabilityReport.from_json(report.to_json()) assert restored.backend.device == "some-accelerator" assert restored.backend.device_name == "Vendor Accelerator XT" # --- schema and serialization ----------------------------------------------- def test_report_dict_has_the_stable_documented_key_set(): "Report dict has the stable documented key set\n\nTags: node, startup" payload = _report( model_config={"num_hidden_layers": 8}, revision="a1b2c3", quantization="int8", runtime={"torch": "2.9.0"}, diagnostics=["loaded 8 layers"], ).to_dict() assert set(payload) == { "schema_version", "model", "shard", "recipe", "backend", "status", "validated_at", "duration_ms", "diagnostics", } assert payload["schema_version"] == CAPABILITY_SCHEMA_VERSION assert set(payload["model"]) == {"model_id", "revision", "config_fingerprint"} assert set(payload["shard"]) == {"start", "end"} assert set(payload["recipe"]) == { "recipe_id", "recipe_version", "catalogue_version", } assert set(payload["backend"]) == { "backend_id", "device", "device_name", "quantization", "runtime", } # JSON-serializable end to end. assert json.loads(json.dumps(payload)) == payload def test_identity_key_pins_model_shard_recipe_and_backend(): "Identity key pins model shard recipe and backend\n\nTags: node, startup" base = _report() assert base.identity_key() == ( FIXTURE_MODEL_A, 0, 7, "baseline", "1", "torch-transformers", "test-device", ) assert _report(shard_end=8).identity_key() != base.identity_key() assert _report(recipe_version="2").identity_key() != base.identity_key() assert _report(device="other-device").identity_key() != base.identity_key() def test_config_fingerprint_is_stable_under_key_order_and_detects_change(): "Config fingerprint is stable under key order and detects change\n\nTags: node, startup" a = config_fingerprint({"num_hidden_layers": 8, "hidden_size": 512}) b = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 8}) c = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 9}) assert a == b assert a != c assert a.startswith("sha256:") assert config_fingerprint(None) is None assert config_fingerprint("sha256:deadbeef") == "sha256:deadbeef" def test_status_and_passed_flag(): "Status and passed flag\n\nTags: node, startup" assert _report(status="passed").passed assert not _report(status="failed").passed assert not _report(status="skipped").passed # --- malformed report input ------------------------------------------------- @pytest.mark.parametrize( "overrides, expected", [ ({"model_id": ""}, "model.model_id"), ({"shard_start": -1}, "shard.start"), ({"shard_start": 5, "shard_end": 2}, "shard.end"), ({"recipe_id": ""}, "recipe.recipe_id"), ({"catalogue_version": ""}, "recipe.catalogue_version"), ({"backend_id": ""}, "backend.backend_id"), ({"device": ""}, "backend.device"), ({"status": "maybe"}, "status"), ({"duration_ms": -5}, "duration_ms"), ], ) def test_malformed_report_fields_name_the_offending_field(overrides, expected): "Malformed report fields name the offending field\n\nTags: node, startup" with pytest.raises(CapabilityReportError) as exc: _report(**overrides) assert expected in str(exc.value) def test_unsupported_report_schema_version_is_actionable(): "Unsupported report schema version is actionable\n\nTags: node, startup" payload = _report().to_dict() payload["schema_version"] = 99 with pytest.raises(CapabilityReportError) as exc: CapabilityReport.from_dict(payload) message = str(exc.value) assert "99" in message assert str(CAPABILITY_SCHEMA_VERSION) in message def test_missing_schema_version_is_rejected(): "Missing schema version is rejected\n\nTags: node, startup" payload = _report().to_dict() del payload["schema_version"] with pytest.raises(CapabilityReportError, match="schema_version"): CapabilityReport.from_dict(payload) def test_malformed_report_json_reports_position_not_content(): "Malformed report json reports position not content\n\nTags: node, startup" with pytest.raises(CapabilityReportError) as exc: CapabilityReport.from_json('{"schema_version": 1,') assert "line 1" in str(exc.value) def test_missing_report_section_is_named(): "Missing report section is named\n\nTags: node, startup" payload = _report().to_dict() payload["backend"] = "torch" with pytest.raises(CapabilityReportError, match="backend"): CapabilityReport.from_dict(payload) # --- diagnostics sanitization ----------------------------------------------- def test_diagnostics_redact_secret_env_values(monkeypatch): "Diagnostics redact secret env values\n\nTags: node, startup" monkeypatch.setenv("MESHNET_API_TOKEN", "super-secret-value-123") monkeypatch.setenv("MESHNET_MODEL_ID", "acme-labs/Widget-9000-Instruct") report = _report( status="failed", diagnostics=["auth failed for super-secret-value-123 while fetching config"], ) text = report.to_json() assert "super-secret-value-123" not in text assert capability.REDACTED in report.diagnostics[0] # A non-secret env var is left alone — the model id stays readable. assert "acme-labs/Widget-9000-Instruct" in text @pytest.mark.parametrize( "raw", [ "401 Unauthorized: hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", "request used Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", "config had api_key=abcdef0123456789", "openai style sk-abcdefghijklmnopqrstuvwxyz012345", ], ) def test_diagnostics_redact_credential_shaped_strings(raw): "Diagnostics redact credential shaped strings\n\nTags: node, startup" cleaned = sanitize_diagnostic(raw, environ={}) assert capability.REDACTED in cleaned for secret in ( "hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", "eyJhbGciOiJIUzI1NiJ9.payload.sig", "abcdef0123456789", "sk-abcdefghijklmnopqrstuvwxyz012345", ): assert secret not in cleaned def test_diagnostics_strip_the_home_directory(monkeypatch, tmp_path): "Diagnostics strip the home directory\n\nTags: node, startup" monkeypatch.setenv("HOME", str(tmp_path)) cleaned = sanitize_diagnostic(f"missing weights at {tmp_path}/models/shard", environ={}) assert str(tmp_path) not in cleaned assert cleaned.startswith("missing weights at ~/models/shard") def test_diagnostics_are_bounded_in_length_and_count(): "Diagnostics are bounded in length and count\n\nTags: node, startup" long_line = sanitize_diagnostic("x" * 2000, environ={}) assert len(long_line) <= capability.MAX_DIAGNOSTIC_CHARS many = sanitize_diagnostics([f"line {i}" for i in range(50)], environ={}) assert len(many) == capability.MAX_DIAGNOSTICS + 1 assert "further diagnostic(s) omitted" in many[-1] def test_diagnostics_reject_non_string_entries(): "Diagnostics reject non string entries\n\nTags: node, startup" with pytest.raises(CapabilityReportError, match=r"diagnostics\[1\]"): sanitize_diagnostics(["ok", 42], environ={}) with pytest.raises(CapabilityReportError, match="bare string"): sanitize_diagnostics("one big string", environ={}) def test_deserializing_a_report_re_sanitizes_diagnostics(monkeypatch): "Deserializing a report re sanitizes diagnostics\n\nTags: node, startup" monkeypatch.setenv("NODE_SECRET", "leak-me-please") payload = _report().to_dict() payload["diagnostics"] = ["backend said leak-me-please"] restored = CapabilityReport.from_dict(payload) assert "leak-me-please" not in restored.to_json() # --- recipe manifest -------------------------------------------------------- def test_packaged_manifest_loads_with_explicit_versions(): "Packaged manifest loads with explicit versions\n\nTags: node, startup" manifest = load_recipe_manifest() assert manifest.schema_version == RECIPE_SCHEMA_VERSION assert manifest.catalogue_version assert recipe_manifest.DEFAULT_RECIPE_ID in manifest.ids for recipe in manifest.recipes: assert recipe.id and recipe.version and recipe.backend_id assert isinstance(recipe.params, dict) def test_packaged_manifest_feeds_a_report(): "Packaged manifest feeds a report\n\nTags: node, startup" manifest = load_recipe_manifest() recipe = manifest.require(recipe_manifest.DEFAULT_RECIPE_ID) report = _report( recipe_id=recipe.id, recipe_version=recipe.version, catalogue_version=manifest.catalogue_version, backend_id=recipe.backend_id, ) assert report.recipe.catalogue_version == manifest.catalogue_version def test_unknown_recipe_lists_available_ids(): "Unknown recipe lists available ids\n\nTags: node, startup" manifest = load_recipe_manifest() with pytest.raises(RecipeManifestError) as exc: manifest.require("no-such-recipe") message = str(exc.value) assert "no-such-recipe" in message assert recipe_manifest.DEFAULT_RECIPE_ID in message def _write_manifest(tmp_path: Path, doc) -> Path: path = tmp_path / "recipes.json" path.write_text( doc if isinstance(doc, str) else json.dumps(doc), encoding="utf-8" ) return path def test_valid_local_manifest_loads(tmp_path): "Valid local manifest loads\n\nTags: node, startup" path = _write_manifest( tmp_path, { "schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "2099.01.0-test", "recipes": [ { "id": "custom", "version": "3", "backend_id": "some-backend", "params": {"knob": 1}, } ], }, ) manifest = load_recipe_manifest(path) assert manifest.catalogue_version == "2099.01.0-test" assert manifest.require("custom").params == {"knob": 1} assert manifest.source == str(path) def test_unknown_manifest_schema_version_is_actionable(tmp_path): "Unknown manifest schema version is actionable\n\nTags: node, startup" path = _write_manifest( tmp_path, { "schema_version": 99, "catalogue_version": "x", "recipes": [{"id": "a", "version": "1", "backend_id": "b"}], }, ) with pytest.raises(RecipeManifestError) as exc: load_recipe_manifest(path) message = str(exc.value) assert "99" in message assert str(RECIPE_SCHEMA_VERSION) in message assert str(path) in message @pytest.mark.parametrize( "doc, expected", [ ({"catalogue_version": "x", "recipes": []}, "schema_version"), ( {"schema_version": RECIPE_SCHEMA_VERSION, "recipes": [{"id": "a"}]}, "catalogue_version", ), ( {"schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": []}, "non-empty JSON array", ), ( { "schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": [{"version": "1", "backend_id": "b"}], }, "recipes[0].id", ), ( { "schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": [{"id": "a", "backend_id": "b"}], }, "recipes[a].version", ), ( { "schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": [{"id": "a", "version": "1"}], }, "recipes[a].backend_id", ), ( { "schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": [ {"id": "a", "version": "1", "backend_id": "b", "params": [1, 2]} ], }, "recipes[a].params", ), ( { "schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": [ {"id": "a", "version": "1", "backend_id": "b"}, {"id": "a", "version": "2", "backend_id": "b"}, ], }, "duplicate recipe id", ), ], ) def test_malformed_manifest_names_the_offending_field(tmp_path, doc, expected): "Malformed manifest names the offending field\n\nTags: node, startup" path = _write_manifest(tmp_path, doc) with pytest.raises(RecipeManifestError) as exc: load_recipe_manifest(path) assert expected in str(exc.value) def test_malformed_manifest_json_reports_position_not_content(tmp_path): "Malformed manifest json reports position not content\n\nTags: node, startup" path = _write_manifest(tmp_path, '{"schema_version": 1, "catalogue_version":') with pytest.raises(RecipeManifestError) as exc: load_recipe_manifest(path) message = str(exc.value) assert "not valid JSON" in message assert "line 1" in message # The failing document body is never echoed back. assert "catalogue_version\":" not in message def test_missing_manifest_file_is_actionable(tmp_path): "Missing manifest file is actionable\n\nTags: node, startup" with pytest.raises(RecipeManifestError, match="cannot read recipe manifest"): load_recipe_manifest(tmp_path / "nope.json") def test_manifest_round_trips_through_its_own_dict(): "Manifest round trips through its own dict\n\nTags: node, startup" manifest = load_recipe_manifest() reparsed = parse_recipe_manifest(manifest.to_dict(), source="") assert reparsed.to_dict() == manifest.to_dict()