test grouping
This commit is contained in:
Dobromir Popov
@@ -344,6 +344,11 @@ def test_dashboard_testing_tab_uses_dynamic_api_and_run_controls():
|
||||
# Targets are rendered from the API payload (tests + suites), not literals.
|
||||
assert "testCollection.suites" in html
|
||||
assert "testCollection.tests" in html
|
||||
assert "testCollection.test_metadata" in html
|
||||
assert "testCollection.tags" in html
|
||||
assert 'data-test-target="suite:all"' in html
|
||||
assert 'id="testing-tag-filter"' in html
|
||||
assert "Run ${esc(t.name)}" in html
|
||||
assert "renderTestTargets" in html
|
||||
|
||||
# Run buttons are per-target and delegated; disabled while a run is active.
|
||||
|
||||
474
tests/test_node_capability.py
Normal file
474
tests/test_node_capability.py
Normal file
@@ -0,0 +1,474 @@
|
||||
"""Tests for the model-agnostic capability report and local recipe manifest."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from meshnet_node import capability, recipe_manifest
|
||||
from meshnet_node.capability import (
|
||||
CAPABILITY_SCHEMA_VERSION,
|
||||
CapabilityReport,
|
||||
CapabilityReportError,
|
||||
build_capability_report,
|
||||
config_fingerprint,
|
||||
sanitize_diagnostic,
|
||||
sanitize_diagnostics,
|
||||
)
|
||||
from meshnet_node.recipe_manifest import (
|
||||
RECIPE_SCHEMA_VERSION,
|
||||
RecipeManifestError,
|
||||
load_recipe_manifest,
|
||||
parse_recipe_manifest,
|
||||
)
|
||||
|
||||
# Deliberately unrelated to any vendor the network ships against: the report
|
||||
# must carry whatever the operator selected, verbatim.
|
||||
FIXTURE_MODEL_A = "acme-labs/Widget-9000-Instruct"
|
||||
FIXTURE_MODEL_B = "some_org/tiny.model-v2_PREVIEW"
|
||||
|
||||
|
||||
def _report(**overrides):
|
||||
kwargs = dict(
|
||||
model_id=FIXTURE_MODEL_A,
|
||||
shard_start=0,
|
||||
shard_end=7,
|
||||
recipe_id="baseline",
|
||||
recipe_version="1",
|
||||
catalogue_version="2026.07.1",
|
||||
backend_id="torch-transformers",
|
||||
device="test-device",
|
||||
status="passed",
|
||||
duration_ms=142,
|
||||
validated_at=1_760_000_000.0,
|
||||
)
|
||||
kwargs.update(overrides)
|
||||
return build_capability_report(**kwargs)
|
||||
|
||||
|
||||
# --- model-agnostic identity ------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [FIXTURE_MODEL_A, FIXTURE_MODEL_B])
|
||||
def test_arbitrary_model_ids_survive_a_json_round_trip_verbatim(model_id):
|
||||
report = _report(model_id=model_id)
|
||||
restored = CapabilityReport.from_json(report.to_json())
|
||||
|
||||
assert restored.model.model_id == model_id
|
||||
assert restored.to_dict() == report.to_dict()
|
||||
|
||||
|
||||
def test_two_arbitrary_models_stay_distinct_without_normalization():
|
||||
a = _report(model_id=FIXTURE_MODEL_A)
|
||||
b = _report(model_id=FIXTURE_MODEL_B)
|
||||
|
||||
assert a.identity_key() != b.identity_key()
|
||||
assert a.model.model_id != b.model.model_id
|
||||
|
||||
|
||||
def test_capability_and_recipe_modules_have_no_model_or_kernel_branch():
|
||||
"""No vendor/model/kernel name may be a default or a code-path discriminator."""
|
||||
forbidden = re.compile(
|
||||
r"qwen|triton|\bfla\b|flash[_-]?attn|flash[_-]?attention|rocm|nvidia|\bcuda\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
sources = [
|
||||
Path(capability.__file__),
|
||||
Path(recipe_manifest.__file__),
|
||||
Path(recipe_manifest.__file__).with_name("recipes.json"),
|
||||
]
|
||||
for source in sources:
|
||||
hits = forbidden.findall(source.read_text(encoding="utf-8"))
|
||||
assert not hits, f"{source.name} names {hits} — recipes must stay generic data"
|
||||
|
||||
|
||||
def test_device_is_an_opaque_label():
|
||||
report = _report(device="some-accelerator", device_name="Vendor Accelerator XT")
|
||||
restored = CapabilityReport.from_json(report.to_json())
|
||||
|
||||
assert restored.backend.device == "some-accelerator"
|
||||
assert restored.backend.device_name == "Vendor Accelerator XT"
|
||||
|
||||
|
||||
# --- schema and serialization -----------------------------------------------
|
||||
|
||||
|
||||
def test_report_dict_has_the_stable_documented_key_set():
|
||||
payload = _report(
|
||||
model_config={"num_hidden_layers": 8},
|
||||
revision="a1b2c3",
|
||||
quantization="int8",
|
||||
runtime={"torch": "2.9.0"},
|
||||
diagnostics=["loaded 8 layers"],
|
||||
).to_dict()
|
||||
|
||||
assert set(payload) == {
|
||||
"schema_version",
|
||||
"model",
|
||||
"shard",
|
||||
"recipe",
|
||||
"backend",
|
||||
"status",
|
||||
"validated_at",
|
||||
"duration_ms",
|
||||
"diagnostics",
|
||||
}
|
||||
assert payload["schema_version"] == CAPABILITY_SCHEMA_VERSION
|
||||
assert set(payload["model"]) == {"model_id", "revision", "config_fingerprint"}
|
||||
assert set(payload["shard"]) == {"start", "end"}
|
||||
assert set(payload["recipe"]) == {
|
||||
"recipe_id",
|
||||
"recipe_version",
|
||||
"catalogue_version",
|
||||
}
|
||||
assert set(payload["backend"]) == {
|
||||
"backend_id",
|
||||
"device",
|
||||
"device_name",
|
||||
"quantization",
|
||||
"runtime",
|
||||
}
|
||||
# JSON-serializable end to end.
|
||||
assert json.loads(json.dumps(payload)) == payload
|
||||
|
||||
|
||||
def test_identity_key_pins_model_shard_recipe_and_backend():
|
||||
base = _report()
|
||||
|
||||
assert base.identity_key() == (
|
||||
FIXTURE_MODEL_A,
|
||||
0,
|
||||
7,
|
||||
"baseline",
|
||||
"1",
|
||||
"torch-transformers",
|
||||
"test-device",
|
||||
)
|
||||
assert _report(shard_end=8).identity_key() != base.identity_key()
|
||||
assert _report(recipe_version="2").identity_key() != base.identity_key()
|
||||
assert _report(device="other-device").identity_key() != base.identity_key()
|
||||
|
||||
|
||||
def test_config_fingerprint_is_stable_under_key_order_and_detects_change():
|
||||
a = config_fingerprint({"num_hidden_layers": 8, "hidden_size": 512})
|
||||
b = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 8})
|
||||
c = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 9})
|
||||
|
||||
assert a == b
|
||||
assert a != c
|
||||
assert a.startswith("sha256:")
|
||||
assert config_fingerprint(None) is None
|
||||
assert config_fingerprint("sha256:deadbeef") == "sha256:deadbeef"
|
||||
|
||||
|
||||
def test_status_and_passed_flag():
|
||||
assert _report(status="passed").passed
|
||||
assert not _report(status="failed").passed
|
||||
assert not _report(status="skipped").passed
|
||||
|
||||
|
||||
# --- malformed report input -------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, expected",
|
||||
[
|
||||
({"model_id": ""}, "model.model_id"),
|
||||
({"shard_start": -1}, "shard.start"),
|
||||
({"shard_start": 5, "shard_end": 2}, "shard.end"),
|
||||
({"recipe_id": ""}, "recipe.recipe_id"),
|
||||
({"catalogue_version": ""}, "recipe.catalogue_version"),
|
||||
({"backend_id": ""}, "backend.backend_id"),
|
||||
({"device": ""}, "backend.device"),
|
||||
({"status": "maybe"}, "status"),
|
||||
({"duration_ms": -5}, "duration_ms"),
|
||||
],
|
||||
)
|
||||
def test_malformed_report_fields_name_the_offending_field(overrides, expected):
|
||||
with pytest.raises(CapabilityReportError) as exc:
|
||||
_report(**overrides)
|
||||
assert expected in str(exc.value)
|
||||
|
||||
|
||||
def test_unsupported_report_schema_version_is_actionable():
|
||||
payload = _report().to_dict()
|
||||
payload["schema_version"] = 99
|
||||
|
||||
with pytest.raises(CapabilityReportError) as exc:
|
||||
CapabilityReport.from_dict(payload)
|
||||
message = str(exc.value)
|
||||
assert "99" in message
|
||||
assert str(CAPABILITY_SCHEMA_VERSION) in message
|
||||
|
||||
|
||||
def test_missing_schema_version_is_rejected():
|
||||
payload = _report().to_dict()
|
||||
del payload["schema_version"]
|
||||
|
||||
with pytest.raises(CapabilityReportError, match="schema_version"):
|
||||
CapabilityReport.from_dict(payload)
|
||||
|
||||
|
||||
def test_malformed_report_json_reports_position_not_content():
|
||||
with pytest.raises(CapabilityReportError) as exc:
|
||||
CapabilityReport.from_json('{"schema_version": 1,')
|
||||
assert "line 1" in str(exc.value)
|
||||
|
||||
|
||||
def test_missing_report_section_is_named():
|
||||
payload = _report().to_dict()
|
||||
payload["backend"] = "torch"
|
||||
|
||||
with pytest.raises(CapabilityReportError, match="backend"):
|
||||
CapabilityReport.from_dict(payload)
|
||||
|
||||
|
||||
# --- diagnostics sanitization -----------------------------------------------
|
||||
|
||||
|
||||
def test_diagnostics_redact_secret_env_values(monkeypatch):
|
||||
monkeypatch.setenv("MESHNET_API_TOKEN", "super-secret-value-123")
|
||||
monkeypatch.setenv("MESHNET_MODEL_ID", "acme-labs/Widget-9000-Instruct")
|
||||
|
||||
report = _report(
|
||||
status="failed",
|
||||
diagnostics=["auth failed for super-secret-value-123 while fetching config"],
|
||||
)
|
||||
text = report.to_json()
|
||||
|
||||
assert "super-secret-value-123" not in text
|
||||
assert capability.REDACTED in report.diagnostics[0]
|
||||
# A non-secret env var is left alone — the model id stays readable.
|
||||
assert "acme-labs/Widget-9000-Instruct" in text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
"401 Unauthorized: hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345",
|
||||
"request used Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig",
|
||||
"config had api_key=abcdef0123456789",
|
||||
"openai style sk-abcdefghijklmnopqrstuvwxyz012345",
|
||||
],
|
||||
)
|
||||
def test_diagnostics_redact_credential_shaped_strings(raw):
|
||||
cleaned = sanitize_diagnostic(raw, environ={})
|
||||
assert capability.REDACTED in cleaned
|
||||
for secret in (
|
||||
"hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345",
|
||||
"eyJhbGciOiJIUzI1NiJ9.payload.sig",
|
||||
"abcdef0123456789",
|
||||
"sk-abcdefghijklmnopqrstuvwxyz012345",
|
||||
):
|
||||
assert secret not in cleaned
|
||||
|
||||
|
||||
def test_diagnostics_strip_the_home_directory(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
cleaned = sanitize_diagnostic(f"missing weights at {tmp_path}/models/shard", environ={})
|
||||
|
||||
assert str(tmp_path) not in cleaned
|
||||
assert cleaned.startswith("missing weights at ~/models/shard")
|
||||
|
||||
|
||||
def test_diagnostics_are_bounded_in_length_and_count():
|
||||
long_line = sanitize_diagnostic("x" * 2000, environ={})
|
||||
assert len(long_line) <= capability.MAX_DIAGNOSTIC_CHARS
|
||||
|
||||
many = sanitize_diagnostics([f"line {i}" for i in range(50)], environ={})
|
||||
assert len(many) == capability.MAX_DIAGNOSTICS + 1
|
||||
assert "further diagnostic(s) omitted" in many[-1]
|
||||
|
||||
|
||||
def test_diagnostics_reject_non_string_entries():
|
||||
with pytest.raises(CapabilityReportError, match=r"diagnostics\[1\]"):
|
||||
sanitize_diagnostics(["ok", 42], environ={})
|
||||
|
||||
with pytest.raises(CapabilityReportError, match="bare string"):
|
||||
sanitize_diagnostics("one big string", environ={})
|
||||
|
||||
|
||||
def test_deserializing_a_report_re_sanitizes_diagnostics(monkeypatch):
|
||||
monkeypatch.setenv("NODE_SECRET", "leak-me-please")
|
||||
payload = _report().to_dict()
|
||||
payload["diagnostics"] = ["backend said leak-me-please"]
|
||||
|
||||
restored = CapabilityReport.from_dict(payload)
|
||||
assert "leak-me-please" not in restored.to_json()
|
||||
|
||||
|
||||
# --- recipe manifest --------------------------------------------------------
|
||||
|
||||
|
||||
def test_packaged_manifest_loads_with_explicit_versions():
|
||||
manifest = load_recipe_manifest()
|
||||
|
||||
assert manifest.schema_version == RECIPE_SCHEMA_VERSION
|
||||
assert manifest.catalogue_version
|
||||
assert recipe_manifest.DEFAULT_RECIPE_ID in manifest.ids
|
||||
for recipe in manifest.recipes:
|
||||
assert recipe.id and recipe.version and recipe.backend_id
|
||||
assert isinstance(recipe.params, dict)
|
||||
|
||||
|
||||
def test_packaged_manifest_feeds_a_report():
|
||||
manifest = load_recipe_manifest()
|
||||
recipe = manifest.require(recipe_manifest.DEFAULT_RECIPE_ID)
|
||||
|
||||
report = _report(
|
||||
recipe_id=recipe.id,
|
||||
recipe_version=recipe.version,
|
||||
catalogue_version=manifest.catalogue_version,
|
||||
backend_id=recipe.backend_id,
|
||||
)
|
||||
assert report.recipe.catalogue_version == manifest.catalogue_version
|
||||
|
||||
|
||||
def test_unknown_recipe_lists_available_ids():
|
||||
manifest = load_recipe_manifest()
|
||||
|
||||
with pytest.raises(RecipeManifestError) as exc:
|
||||
manifest.require("no-such-recipe")
|
||||
message = str(exc.value)
|
||||
assert "no-such-recipe" in message
|
||||
assert recipe_manifest.DEFAULT_RECIPE_ID in message
|
||||
|
||||
|
||||
def _write_manifest(tmp_path: Path, doc) -> Path:
|
||||
path = tmp_path / "recipes.json"
|
||||
path.write_text(
|
||||
doc if isinstance(doc, str) else json.dumps(doc), encoding="utf-8"
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def test_valid_local_manifest_loads(tmp_path):
|
||||
path = _write_manifest(
|
||||
tmp_path,
|
||||
{
|
||||
"schema_version": RECIPE_SCHEMA_VERSION,
|
||||
"catalogue_version": "2099.01.0-test",
|
||||
"recipes": [
|
||||
{
|
||||
"id": "custom",
|
||||
"version": "3",
|
||||
"backend_id": "some-backend",
|
||||
"params": {"knob": 1},
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
manifest = load_recipe_manifest(path)
|
||||
|
||||
assert manifest.catalogue_version == "2099.01.0-test"
|
||||
assert manifest.require("custom").params == {"knob": 1}
|
||||
assert manifest.source == str(path)
|
||||
|
||||
|
||||
def test_unknown_manifest_schema_version_is_actionable(tmp_path):
|
||||
path = _write_manifest(
|
||||
tmp_path,
|
||||
{
|
||||
"schema_version": 99,
|
||||
"catalogue_version": "x",
|
||||
"recipes": [{"id": "a", "version": "1", "backend_id": "b"}],
|
||||
},
|
||||
)
|
||||
with pytest.raises(RecipeManifestError) as exc:
|
||||
load_recipe_manifest(path)
|
||||
message = str(exc.value)
|
||||
assert "99" in message
|
||||
assert str(RECIPE_SCHEMA_VERSION) in message
|
||||
assert str(path) in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"doc, expected",
|
||||
[
|
||||
({"catalogue_version": "x", "recipes": []}, "schema_version"),
|
||||
(
|
||||
{"schema_version": RECIPE_SCHEMA_VERSION, "recipes": [{"id": "a"}]},
|
||||
"catalogue_version",
|
||||
),
|
||||
(
|
||||
{"schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": []},
|
||||
"non-empty JSON array",
|
||||
),
|
||||
(
|
||||
{
|
||||
"schema_version": RECIPE_SCHEMA_VERSION,
|
||||
"catalogue_version": "x",
|
||||
"recipes": [{"version": "1", "backend_id": "b"}],
|
||||
},
|
||||
"recipes[0].id",
|
||||
),
|
||||
(
|
||||
{
|
||||
"schema_version": RECIPE_SCHEMA_VERSION,
|
||||
"catalogue_version": "x",
|
||||
"recipes": [{"id": "a", "backend_id": "b"}],
|
||||
},
|
||||
"recipes[a].version",
|
||||
),
|
||||
(
|
||||
{
|
||||
"schema_version": RECIPE_SCHEMA_VERSION,
|
||||
"catalogue_version": "x",
|
||||
"recipes": [{"id": "a", "version": "1"}],
|
||||
},
|
||||
"recipes[a].backend_id",
|
||||
),
|
||||
(
|
||||
{
|
||||
"schema_version": RECIPE_SCHEMA_VERSION,
|
||||
"catalogue_version": "x",
|
||||
"recipes": [
|
||||
{"id": "a", "version": "1", "backend_id": "b", "params": [1, 2]}
|
||||
],
|
||||
},
|
||||
"recipes[a].params",
|
||||
),
|
||||
(
|
||||
{
|
||||
"schema_version": RECIPE_SCHEMA_VERSION,
|
||||
"catalogue_version": "x",
|
||||
"recipes": [
|
||||
{"id": "a", "version": "1", "backend_id": "b"},
|
||||
{"id": "a", "version": "2", "backend_id": "b"},
|
||||
],
|
||||
},
|
||||
"duplicate recipe id",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_malformed_manifest_names_the_offending_field(tmp_path, doc, expected):
|
||||
path = _write_manifest(tmp_path, doc)
|
||||
|
||||
with pytest.raises(RecipeManifestError) as exc:
|
||||
load_recipe_manifest(path)
|
||||
assert expected in str(exc.value)
|
||||
|
||||
|
||||
def test_malformed_manifest_json_reports_position_not_content(tmp_path):
|
||||
path = _write_manifest(tmp_path, '{"schema_version": 1, "catalogue_version":')
|
||||
|
||||
with pytest.raises(RecipeManifestError) as exc:
|
||||
load_recipe_manifest(path)
|
||||
message = str(exc.value)
|
||||
assert "not valid JSON" in message
|
||||
assert "line 1" in message
|
||||
# The failing document body is never echoed back.
|
||||
assert "catalogue_version\":" not in message
|
||||
|
||||
|
||||
def test_missing_manifest_file_is_actionable(tmp_path):
|
||||
with pytest.raises(RecipeManifestError, match="cannot read recipe manifest"):
|
||||
load_recipe_manifest(tmp_path / "nope.json")
|
||||
|
||||
|
||||
def test_manifest_round_trips_through_its_own_dict():
|
||||
manifest = load_recipe_manifest()
|
||||
reparsed = parse_recipe_manifest(manifest.to_dict(), source="<memory>")
|
||||
|
||||
assert reparsed.to_dict() == manifest.to_dict()
|
||||
@@ -34,6 +34,9 @@ def _make_repo(tmp_path: Path) -> Path:
|
||||
"import time\n\ndef test_sleeps():\n time.sleep(30)\n"
|
||||
)
|
||||
(tests / "test_smoke.py").write_text("def test_smoke():\n assert True\n")
|
||||
(tests / "test_cache.py").write_text(
|
||||
'def test_cache_roundtrip():\n """Cache roundtrip remains deterministic."""\n assert True\n'
|
||||
)
|
||||
# Real-inference stand-in: must never be collected or run by default.
|
||||
(tests / "test_real_stub.py").write_text(
|
||||
"def test_real_inference():\n raise AssertionError('env-gated')\n"
|
||||
@@ -139,6 +142,12 @@ def test_collection_lists_tests_and_excludes_real_inference(tmp_path, monkeypatc
|
||||
assert data["enabled"] is True
|
||||
assert "tests/test_quick.py::test_passes" in data["tests"]
|
||||
assert "tests/test_slow.py::test_sleeps" in data["tests"]
|
||||
metadata = {item["id"]: item for item in data["test_metadata"]}
|
||||
assert metadata["tests/test_cache.py::test_cache_roundtrip"]["description"] == (
|
||||
"Cache roundtrip remains deterministic."
|
||||
)
|
||||
assert "cache" in metadata["tests/test_cache.py::test_cache_roundtrip"]["tags"]
|
||||
assert any(tag["id"] == "tag:cache" for tag in data["tags"])
|
||||
assert not any("test_real_" in node_id for node_id in data["tests"])
|
||||
assert data["collected_at"] is not None
|
||||
suite_ids = [suite["id"] for suite in data["suites"]]
|
||||
@@ -190,6 +199,31 @@ def test_run_approved_suite_without_prior_collection(tmp_path):
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def test_run_all_and_tag_targets(tmp_path):
|
||||
tracker, port, admin, _user = _start_tracker(tmp_path)
|
||||
try:
|
||||
assert _request(port, "GET", "/v1/tests", token=admin)[0] == 200
|
||||
runner = tracker._test_runner
|
||||
assert runner is not None
|
||||
all_state = runner.start("suite:all")
|
||||
assert all_state["run"]["target"] == "suite:all"
|
||||
assert len(all_state["run"]["args"]) == len(runner._collected)
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
second_repo = tmp_path / "second"
|
||||
second_repo.mkdir()
|
||||
tracker, _port, _admin, _user = _start_tracker(second_repo)
|
||||
try:
|
||||
runner = tracker._test_runner
|
||||
assert runner is not None
|
||||
runner.collect()
|
||||
tagged = runner.start("tag:cache")
|
||||
assert tagged["run"]["args"] == ["tests/test_cache.py::test_cache_roundtrip"]
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", [
|
||||
"",
|
||||
"-x",
|
||||
|
||||
Reference in New Issue
Block a user