popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test grouping

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-11 22:11:21 +03:00
parent c195b5ce78
commit 7d259d7c9b
9 changed files with 1424 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
.scratch/node-capability-admission/prd.json
View File

@@ -1,7 +1,7 @@
{
"name": "Model-agnostic Node capability admission",
"branchName": "ralph/node-capability-admission",
"description": "Make a Node prove its selected Model Artifact, Shard, and execution recipe work before it becomes routable. Qwen3.6 is only an opt-in development fixture; the implementation and protocol are model-agnostic.",
"branchName": "ralph/node-capability-admission",
"userStories": [
{
"id": "NCA-001",
@@ -36,7 +36,9 @@
"priority": 2,
"passes": false,
"notes": "Source issue: .scratch/node-capability-admission/issues/02-doctor-real-forward.md",
"dependsOn": ["NCA-001"]
"dependsOn": [
"NCA-001"
]
},
{
"id": "NCA-003",
@@ -53,7 +55,10 @@
"priority": 3,
"passes": false,
"notes": "Source issue: .scratch/node-capability-admission/issues/03-fail-closed-startup-admission.md",
"dependsOn": ["NCA-001", "NCA-002"]
"dependsOn": [
"NCA-001",
"NCA-002"
]
},
{
"id": "NCA-004",
@@ -72,7 +77,10 @@
"priority": 4,
"passes": false,
"notes": "Source issue: .scratch/node-capability-admission/issues/04-tracker-validated-capability-routing.md",
"dependsOn": ["NCA-001", "NCA-003"]
"dependsOn": [
"NCA-001",
"NCA-003"
]
},
{
"id": "NCA-005",
@@ -89,7 +97,13 @@
"priority": 5,
"passes": false,
"notes": "Source issue: .scratch/node-capability-admission/issues/05-docs-hardware-lane-contract.md",
"dependsOn": ["NCA-002", "NCA-004"]
"dependsOn": [
"NCA-002",
"NCA-004"
]
}
]
}
],
"metadata": {
"updatedAt": "2026-07-11T19:02:57.532Z"
}
}

494
packages/node/meshnet_node/capability.py Normal file
View File

@@ -0,0 +1,494 @@
"""Model-agnostic node capability report.
A capability report is the node's local proof that one concrete combination —
model artifact, shard range, recipe, backend/device — actually executed. It is
plain versioned data: arbitrary model ids pass through verbatim, and no model,
vendor, or kernel name is a default or a code-path discriminator here.
Later stories consume this: `doctor` produces a report from a real forward
(NCA-002), startup refuses to register without a fresh passing one (NCA-003),
and the tracker routes only to admitted, matching capabilities (NCA-004).
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import time
from dataclasses import dataclass, field
from typing import Any, Mapping
# Layout of the serialized report. Bump when the JSON shape changes.
CAPABILITY_SCHEMA_VERSION = 1
STATUS_PASSED = "passed"
STATUS_FAILED = "failed"
STATUS_SKIPPED = "skipped"
VALID_STATUSES = (STATUS_PASSED, STATUS_FAILED, STATUS_SKIPPED)
# Diagnostics are operator-facing, not a log sink: keep them short and few.
MAX_DIAGNOSTIC_CHARS = 500
MAX_DIAGNOSTICS = 20
REDACTED = "[redacted]"
# An env var whose *name* contains one of these holds a secret by convention.
_SECRET_NAME_HINTS = (
"TOKEN",
"SECRET",
"PASSWORD",
"PASSWD",
"CREDENTIAL",
"APIKEY",
"API_KEY",
"PRIVATE_KEY",
"ACCESS_KEY",
)
# Below this length a value is too generic to redact without mangling prose.
_MIN_SECRET_LEN = 6
# Provider-shaped bearer credentials that can appear in a backend error string.
_CREDENTIAL_PATTERNS = (
re.compile(r"\b[A-Za-z0-9_]{2,6}_[A-Za-z0-9]{16,}\b"), # hf_…, ghp_…, sk_live_…
re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"),
re.compile(r"(?i)\bbearer\s+\S+"),
re.compile(r"(?i)\b(?:token|api[_-]?key|password|secret)\s*[=:]\s*\S+"),
)
class CapabilityReportError(ValueError):
"""Raised when report input is malformed.
Messages name the offending field and the expected shape, and carry no
caller-supplied payload beyond the field path itself.
"""
def _secret_env_values(environ: Mapping[str, str] | None = None) -> list[str]:
env = os.environ if environ is None else environ
values: list[str] = []
for name, value in env.items():
if not isinstance(value, str) or len(value) < _MIN_SECRET_LEN:
continue
upper = name.upper()
if any(hint in upper for hint in _SECRET_NAME_HINTS):
values.append(value)
# Redact longest first so a value that contains another is not partially masked.
return sorted(values, key=len, reverse=True)
def sanitize_diagnostic(
text: str,
environ: Mapping[str, str] | None = None,
) -> str:
"""Return `text` with credentials and host identity stripped, clipped to length."""
cleaned = " ".join(str(text).split())
for secret in _secret_env_values(environ):
cleaned = cleaned.replace(secret, REDACTED)
for pattern in _CREDENTIAL_PATTERNS:
cleaned = pattern.sub(REDACTED, cleaned)
home = os.path.expanduser("~")
if home and home not in ("/", ""):
cleaned = cleaned.replace(home, "~")
if len(cleaned) > MAX_DIAGNOSTIC_CHARS:
cleaned = cleaned[: MAX_DIAGNOSTIC_CHARS - 1].rstrip() + "…"
return cleaned
def sanitize_diagnostics(
diagnostics: Any,
environ: Mapping[str, str] | None = None,
) -> tuple[str, ...]:
"""Sanitize and bound a diagnostics sequence."""
if diagnostics is None:
return ()
if isinstance(diagnostics, str):
raise CapabilityReportError(
"'diagnostics' must be a list of strings, got a bare string"
)
try:
items = list(diagnostics)
except TypeError as exc:
raise CapabilityReportError(
f"'diagnostics' must be a list of strings, got {type(diagnostics).__name__}"
) from exc
out: list[str] = []
for index, item in enumerate(items[:MAX_DIAGNOSTICS]):
if not isinstance(item, str):
raise CapabilityReportError(
f"'diagnostics[{index}]' must be a string, got {type(item).__name__}"
)
cleaned = sanitize_diagnostic(item, environ)
if cleaned:
out.append(cleaned)
dropped = len(items) - MAX_DIAGNOSTICS
if dropped > 0:
out.append(f"… {dropped} further diagnostic(s) omitted")
return tuple(out)
def config_fingerprint(config: Any) -> str | None:
"""Return a stable content hash of a model config mapping.
Two nodes that loaded the same artifact revision with the same config
produce the same fingerprint; anything unserializable degrades to its
string form rather than failing the report.
"""
if config is None:
return None
if isinstance(config, str):
return config if config.startswith("sha256:") else "sha256:" + _sha256(config)
if not isinstance(config, Mapping):
raise CapabilityReportError(
f"model config must be a mapping or a fingerprint string, "
f"got {type(config).__name__}"
)
canonical = json.dumps(
config, sort_keys=True, separators=(",", ":"), default=str, ensure_ascii=False
)
return "sha256:" + _sha256(canonical)
def _sha256(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def _require_text(value: Any, field_name: str) -> str:
if not isinstance(value, str) or not value.strip():
raise CapabilityReportError(f"{field_name!r} must be a non-empty string")
return value
def _optional_text(value: Any, field_name: str) -> str | None:
if value is None:
return None
return _require_text(value, field_name)
def _require_int(value: Any, field_name: str, minimum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise CapabilityReportError(f"{field_name!r} must be an integer")
if value < minimum:
raise CapabilityReportError(f"{field_name!r} must be >= {minimum}, got {value}")
return value
@dataclass(frozen=True)
class ModelIdentity:
"""Which artifact was validated. `model_id` is opaque and preserved verbatim."""
model_id: str
revision: str | None = None
config_fingerprint: str | None = None
def __post_init__(self) -> None:
_require_text(self.model_id, "model.model_id")
_optional_text(self.revision, "model.revision")
_optional_text(self.config_fingerprint, "model.config_fingerprint")
def to_dict(self) -> dict:
return {
"model_id": self.model_id,
"revision": self.revision,
"config_fingerprint": self.config_fingerprint,
}
@classmethod
def from_dict(cls, data: Any) -> ModelIdentity:
doc = _as_mapping(data, "model")
return cls(
model_id=_require_text(doc.get("model_id"), "model.model_id"),
revision=_optional_text(doc.get("revision"), "model.revision"),
config_fingerprint=_optional_text(
doc.get("config_fingerprint"), "model.config_fingerprint"
),
)
@dataclass(frozen=True)
class ShardRange:
"""Inclusive layer range, matching the CLI and backend convention."""
start: int
end: int
def __post_init__(self) -> None:
_require_int(self.start, "shard.start", 0)
_require_int(self.end, "shard.end", 0)
if self.end < self.start:
raise CapabilityReportError(
f"'shard.end' ({self.end}) must be >= 'shard.start' ({self.start})"
)
def to_dict(self) -> dict:
return {"start": self.start, "end": self.end}
@classmethod
def from_dict(cls, data: Any) -> ShardRange:
doc = _as_mapping(data, "shard")
return cls(
start=_require_int(doc.get("start"), "shard.start", 0),
end=_require_int(doc.get("end"), "shard.end", 0),
)
@dataclass(frozen=True)
class RecipeIdentity:
"""Which recipe, from which catalogue, was exercised."""
recipe_id: str
recipe_version: str
catalogue_version: str
def __post_init__(self) -> None:
_require_text(self.recipe_id, "recipe.recipe_id")
_require_text(self.recipe_version, "recipe.recipe_version")
_require_text(self.catalogue_version, "recipe.catalogue_version")
def to_dict(self) -> dict:
return {
"recipe_id": self.recipe_id,
"recipe_version": self.recipe_version,
"catalogue_version": self.catalogue_version,
}
@classmethod
def from_dict(cls, data: Any) -> RecipeIdentity:
doc = _as_mapping(data, "recipe")
return cls(
recipe_id=_require_text(doc.get("recipe_id"), "recipe.recipe_id"),
recipe_version=_require_text(
doc.get("recipe_version"), "recipe.recipe_version"
),
catalogue_version=_require_text(
doc.get("catalogue_version"), "recipe.catalogue_version"
),
)
@dataclass(frozen=True)
class BackendIdentity:
"""Which execution stack ran it. All fields are opaque labels, never branches."""
backend_id: str
device: str
device_name: str | None = None
quantization: str | None = None
runtime: Mapping[str, str] = field(default_factory=dict)
def __post_init__(self) -> None:
_require_text(self.backend_id, "backend.backend_id")
_require_text(self.device, "backend.device")
_optional_text(self.device_name, "backend.device_name")
_optional_text(self.quantization, "backend.quantization")
for key, value in self.runtime.items():
if not isinstance(key, str) or not isinstance(value, str):
raise CapabilityReportError(
"'backend.runtime' must map string names to string versions"
)
def to_dict(self) -> dict:
return {
"backend_id": self.backend_id,
"device": self.device,
"device_name": self.device_name,
"quantization": self.quantization,
"runtime": dict(self.runtime),
}
@classmethod
def from_dict(cls, data: Any) -> BackendIdentity:
doc = _as_mapping(data, "backend")
runtime = doc.get("runtime") or {}
if not isinstance(runtime, Mapping):
raise CapabilityReportError("'backend.runtime' must be a JSON object")
return cls(
backend_id=_require_text(doc.get("backend_id"), "backend.backend_id"),
device=_require_text(doc.get("device"), "backend.device"),
device_name=_optional_text(doc.get("device_name"), "backend.device_name"),
quantization=_optional_text(
doc.get("quantization"), "backend.quantization"
),
runtime={str(k): str(v) for k, v in runtime.items()},
)
def _as_mapping(data: Any, field_name: str) -> Mapping[str, Any]:
if not isinstance(data, Mapping):
raise CapabilityReportError(
f"{field_name!r} must be a JSON object, got {type(data).__name__}"
)
return data
@dataclass(frozen=True)
class CapabilityReport:
"""One node's validated (or failed) model/shard/recipe/backend combination."""
model: ModelIdentity
shard: ShardRange
recipe: RecipeIdentity
backend: BackendIdentity
status: str
validated_at: float
duration_ms: int
diagnostics: tuple[str, ...] = ()
schema_version: int = CAPABILITY_SCHEMA_VERSION
def __post_init__(self) -> None:
if self.status not in VALID_STATUSES:
raise CapabilityReportError(
f"'status' must be one of {', '.join(VALID_STATUSES)}; got {self.status!r}"
)
if isinstance(self.validated_at, bool) or not isinstance(
self.validated_at, (int, float)
):
raise CapabilityReportError("'validated_at' must be a Unix timestamp")
if self.validated_at < 0:
raise CapabilityReportError("'validated_at' must not be negative")
_require_int(self.duration_ms, "duration_ms", 0)
_require_int(self.schema_version, "schema_version", 1)
@property
def passed(self) -> bool:
return self.status == STATUS_PASSED
def identity_key(self) -> tuple[str, int, int, str, str, str, str]:
"""The tuple a consumer must match to reuse this proof.
Startup and the tracker compare on exactly this: a report proves nothing
about a different model, shard, recipe version, or device.
"""
return (
self.model.model_id,
self.shard.start,
self.shard.end,
self.recipe.recipe_id,
self.recipe.recipe_version,
self.backend.backend_id,
self.backend.device,
)
def age_seconds(self, now: float | None = None) -> float:
return max(0.0, (time.time() if now is None else now) - self.validated_at)
def to_dict(self) -> dict:
return {
"schema_version": self.schema_version,
"model": self.model.to_dict(),
"shard": self.shard.to_dict(),
"recipe": self.recipe.to_dict(),
"backend": self.backend.to_dict(),
"status": self.status,
"validated_at": self.validated_at,
"duration_ms": self.duration_ms,
"diagnostics": list(self.diagnostics),
}
def to_json(self, indent: int | None = None) -> str:
return json.dumps(self.to_dict(), indent=indent, sort_keys=True)
@classmethod
def from_dict(cls, data: Any) -> CapabilityReport:
doc = _as_mapping(data, "report")
if "schema_version" not in doc:
raise CapabilityReportError(
"report is missing 'schema_version'; this node reads capability "
f"schema version {CAPABILITY_SCHEMA_VERSION}"
)
schema_version = _require_int(doc["schema_version"], "schema_version", 1)
if schema_version != CAPABILITY_SCHEMA_VERSION:
raise CapabilityReportError(
f"report declares capability schema version {schema_version}, but this "
f"node reads version {CAPABILITY_SCHEMA_VERSION}"
)
validated_at = doc.get("validated_at")
if isinstance(validated_at, bool) or not isinstance(
validated_at, (int, float)
):
raise CapabilityReportError("'validated_at' must be a Unix timestamp")
return cls(
schema_version=schema_version,
model=ModelIdentity.from_dict(doc.get("model")),
shard=ShardRange.from_dict(doc.get("shard")),
recipe=RecipeIdentity.from_dict(doc.get("recipe")),
backend=BackendIdentity.from_dict(doc.get("backend")),
status=_require_text(doc.get("status"), "status"),
validated_at=float(validated_at),
duration_ms=_require_int(doc.get("duration_ms"), "duration_ms", 0),
diagnostics=sanitize_diagnostics(doc.get("diagnostics")),
)
@classmethod
def from_json(cls, text: str) -> CapabilityReport:
try:
data = json.loads(text)
except json.JSONDecodeError as exc:
raise CapabilityReportError(
f"capability report is not valid JSON: {exc.msg} "
f"at line {exc.lineno} column {exc.colno}"
) from exc
return cls.from_dict(data)
def build_capability_report(
*,
model_id: str,
shard_start: int,
shard_end: int,
recipe_id: str,
recipe_version: str,
catalogue_version: str,
backend_id: str,
device: str,
status: str,
duration_ms: int,
revision: str | None = None,
model_config: Any = None,
device_name: str | None = None,
quantization: str | None = None,
runtime: Mapping[str, str] | None = None,
diagnostics: Any = None,
validated_at: float | None = None,
environ: Mapping[str, str] | None = None,
) -> CapabilityReport:
"""Assemble a report from flat validation results.
`model_config` may be the loaded config mapping (hashed into a fingerprint)
or an already-computed ``sha256:…`` string. `validated_at` defaults to now,
so callers that need determinism pass it explicitly.
"""
return CapabilityReport(
model=ModelIdentity(
model_id=model_id,
revision=revision,
config_fingerprint=config_fingerprint(model_config),
),
shard=ShardRange(start=shard_start, end=shard_end),
recipe=RecipeIdentity(
recipe_id=recipe_id,
recipe_version=recipe_version,
catalogue_version=catalogue_version,
),
backend=BackendIdentity(
backend_id=backend_id,
device=device,
device_name=device_name,
quantization=quantization,
runtime=dict(runtime or {}),
),
status=status,
validated_at=time.time() if validated_at is None else validated_at,
duration_ms=duration_ms,
diagnostics=sanitize_diagnostics(diagnostics, environ),
)

222
packages/node/meshnet_node/recipe_manifest.py Normal file
View File

@@ -0,0 +1,222 @@
"""Local, versioned recipe manifest.
A recipe is *data*: a named, versioned set of execution parameters handed to the
model backend. It carries no model- or vendor-specific code path — a recipe is
only ever valid once its own real forward has succeeded on this node
(see :mod:`meshnet_node.capability`).
The manifest ships with the node release. ``schema_version`` describes the file
layout this reader understands; ``catalogue_version`` identifies the recipe set
itself so a tracker can reason about which catalogue a node validated against.
"""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from importlib.resources import files
from pathlib import Path
from typing import Any, Mapping
# Layout of recipes.json understood by this reader. Bump when the file shape changes.
RECIPE_SCHEMA_VERSION = 1
DEFAULT_RECIPE_ID = "baseline"
_MANIFEST_RESOURCE = "recipes.json"
class RecipeManifestError(ValueError):
"""Raised when a recipe manifest is missing, malformed, or unsupported.
The message is operator-facing: it names the source and the fix, and never
echoes raw file content back (a manifest may sit next to secrets in a
misconfigured deployment).
"""
@dataclass(frozen=True)
class Recipe:
"""One named, versioned execution recipe."""
id: str
version: str
backend_id: str
description: str = ""
params: Mapping[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict:
return {
"id": self.id,
"version": self.version,
"backend_id": self.backend_id,
"description": self.description,
"params": dict(self.params),
}
@dataclass(frozen=True)
class RecipeManifest:
"""A parsed, validated recipe catalogue."""
schema_version: int
catalogue_version: str
recipes: tuple[Recipe, ...]
source: str = "<memory>"
def get(self, recipe_id: str) -> Recipe | None:
for recipe in self.recipes:
if recipe.id == recipe_id:
return recipe
return None
def require(self, recipe_id: str) -> Recipe:
"""Return the named recipe, or raise listing what this catalogue offers."""
recipe = self.get(recipe_id)
if recipe is None:
available = ", ".join(r.id for r in self.recipes) or "(none)"
raise RecipeManifestError(
f"unknown recipe {recipe_id!r} in {self.source}; "
f"available recipes: {available}"
)
return recipe
@property
def ids(self) -> tuple[str, ...]:
return tuple(r.id for r in self.recipes)
def to_dict(self) -> dict:
return {
"schema_version": self.schema_version,
"catalogue_version": self.catalogue_version,
"recipes": [r.to_dict() for r in self.recipes],
}
def _require_mapping(value: Any, what: str, source: str) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
raise RecipeManifestError(
f"{what} in {source} must be a JSON object, got {type(value).__name__}"
)
return value
def _require_text(value: Any, what: str, source: str) -> str:
if not isinstance(value, str) or not value.strip():
raise RecipeManifestError(
f"{what} in {source} must be a non-empty string"
)
return value
def _parse_recipe(raw: Any, index: int, source: str) -> Recipe:
entry = _require_mapping(raw, f"recipes[{index}]", source)
recipe_id = _require_text(entry.get("id"), f"recipes[{index}].id", source)
version = _require_text(
entry.get("version"), f"recipes[{recipe_id}].version", source
)
backend_id = _require_text(
entry.get("backend_id"), f"recipes[{recipe_id}].backend_id", source
)
params = entry.get("params", {})
if params is None:
params = {}
_require_mapping(params, f"recipes[{recipe_id}].params", source)
description = entry.get("description", "")
if not isinstance(description, str):
raise RecipeManifestError(
f"recipes[{recipe_id}].description in {source} must be a string"
)
return Recipe(
id=recipe_id,
version=version,
backend_id=backend_id,
description=description,
params=dict(params),
)
def parse_recipe_manifest(data: Any, source: str = "<memory>") -> RecipeManifest:
"""Validate an already-decoded manifest document."""
doc = _require_mapping(data, "manifest root", source)
if "schema_version" not in doc:
raise RecipeManifestError(
f"{source} is missing 'schema_version'; "
f"this node reads recipe schema version {RECIPE_SCHEMA_VERSION}"
)
schema_version = doc["schema_version"]
if not isinstance(schema_version, int) or isinstance(schema_version, bool):
raise RecipeManifestError(
f"'schema_version' in {source} must be an integer, "
f"got {type(schema_version).__name__}"
)
if schema_version != RECIPE_SCHEMA_VERSION:
raise RecipeManifestError(
f"{source} declares recipe schema version {schema_version}, "
f"but this node reads version {RECIPE_SCHEMA_VERSION}; "
"upgrade the node or use a manifest for the supported version"
)
catalogue_version = _require_text(
doc.get("catalogue_version"), "'catalogue_version'", source
)
raw_recipes = doc.get("recipes")
if not isinstance(raw_recipes, list) or not raw_recipes:
raise RecipeManifestError(
f"'recipes' in {source} must be a non-empty JSON array"
)
recipes: list[Recipe] = []
seen: set[str] = set()
for index, raw in enumerate(raw_recipes):
recipe = _parse_recipe(raw, index, source)
if recipe.id in seen:
raise RecipeManifestError(
f"duplicate recipe id {recipe.id!r} in {source}; recipe ids must be unique"
)
seen.add(recipe.id)
recipes.append(recipe)
return RecipeManifest(
schema_version=schema_version,
catalogue_version=catalogue_version,
recipes=tuple(recipes),
source=source,
)
def load_recipe_manifest(path: Path | None = None) -> RecipeManifest:
"""Load the packaged manifest, or one at ``path``.
No network access and no remote catalogue: P0 recipes ship with the node.
"""
if path is None:
source = f"packaged {_MANIFEST_RESOURCE}"
try:
raw = files("meshnet_node").joinpath(_MANIFEST_RESOURCE).read_text(
encoding="utf-8"
)
except (OSError, FileNotFoundError, ModuleNotFoundError) as exc:
raise RecipeManifestError(
f"{source} is missing from this node installation "
f"({type(exc).__name__}); reinstall the node package"
) from exc
else:
source = str(path)
try:
raw = path.read_text(encoding="utf-8")
except OSError as exc:
raise RecipeManifestError(
f"cannot read recipe manifest {source}: {exc.strerror or exc}"
) from exc
try:
data = json.loads(raw)
except json.JSONDecodeError as exc:
raise RecipeManifestError(
f"{source} is not valid JSON: {exc.msg} at line {exc.lineno} column {exc.colno}"
) from exc
return parse_recipe_manifest(data, source=source)

31
packages/node/meshnet_node/recipes.json Normal file
View File

@@ -0,0 +1,31 @@
{
"schema_version": 1,
"catalogue_version": "2026.07.1",
"recipes": [
{
"id": "baseline",
"version": "1",
"backend_id": "torch-transformers",
"description": "Backend defaults with no execution overrides.",
"params": {}
},
{
"id": "eager-attention",
"version": "1",
"backend_id": "torch-transformers",
"description": "Force the reference attention path instead of the backend's autoselected one.",
"params": {
"attn_implementation": "eager"
}
},
{
"id": "stateless",
"version": "1",
"backend_id": "torch-transformers",
"description": "Disable the incremental state cache; every step re-runs the full prefill.",
"params": {
"use_cache": false
}
}
]
}

48
packages/tracker/meshnet_tracker/dashboard.html
View File

@@ -298,8 +298,13 @@
<h2>Tests &amp; suites</h2>
<div class="testing-controls">
<input id="testing-filter" type="search" placeholder="filter tests…" oninput="renderTestTargets()" aria-label="Filter tests">
<select id="testing-tag-filter" onchange="renderTestTargets()" aria-label="filter by test tag">
<option value="">all tags</option>
</select>
<button class="small" type="button" data-test-target="suite:all">Run all tests</button>
<button class="small" type="button" onclick="reloadTestTargets()">re-collect</button>
</div>
<div id="testing-tag-actions" class="testing-tags"></div>
<div id="testing-targets" class="empty">admin login required</div>
</section>
<section data-tab="testing" data-admin-only class="wide">
@@ -1016,7 +1021,7 @@ function renderConsole(data) {
// ---- testing tab (opt-in tracker test runner, dashboard-test-runner US-002) ----
let testCollection = { tests: [], suites: [] };
let testCollection = { tests: [], test_metadata: [], suites: [], tags: [] };
let testRun = null;
let testRunStarting = false;
@@ -1122,21 +1127,39 @@ function renderTestTargets() {
const el = $("testing-targets");
if (!el) return;
const filter = ($("testing-filter")?.value || "").trim().toLowerCase();
const selectedTag = ($("testing-tag-filter")?.value || "").trim().toLowerCase();
const metadata = testCollection.test_metadata || testCollection.tests.map(id => ({ id, description: id, tags: [] }));
const suites = (testCollection.suites || [])
.map(s => ({ id: s.id, label: `${s.name} (${(s.paths || []).join(", ")})`, suite: true }));
const tests = (testCollection.tests || []).map(t => ({ id: t, label: t, suite: false }));
.map(s => ({ id: s.id, label: `${s.name} (${(s.paths || []).join(", ")})`, description: "Approved test suite", tags: ["suite"], suite: true }));
const tests = metadata.map(t => ({ ...t, label: t.id, suite: false }));
const targets = [...suites, ...tests]
.filter(t => !filter || t.label.toLowerCase().includes(filter));
.filter(t => !filter || `${t.label} ${t.description} ${(t.tags || []).join(" ")}`.toLowerCase().includes(filter))
.filter(t => !selectedTag || t.suite || (t.tags || []).includes(selectedTag));
const disabled = testRunActive() ? " disabled" : "";
const tagFilter = $("testing-tag-filter");
if (tagFilter) {
const value = tagFilter.value;
tagFilter.innerHTML = '<option value="">all tags</option>' +
(testCollection.tags || []).map(t => `<option value="${esc(t.name)}">${esc(t.name)} (${esc(t.count)})</option>`).join("");
tagFilter.value = value;
}
const tagActions = $("testing-tag-actions");
if (tagActions) {
tagActions.innerHTML = (testCollection.tags || []).map(t =>
`<button class="small" type="button" data-test-target="${esc(t.id)}"${disabled}>Run ${esc(t.name)} (${esc(t.count)})</button>`
).join("");
}
if (!targets.length) {
el.className = "empty";
el.innerHTML = filter ? "no targets match the filter" : "no tests collected";
el.innerHTML = filter || selectedTag ? "no tests match the filter" : "no tests collected";
return;
}
const disabled = testRunActive() ? " disabled" : "";
el.className = "testing-list";
el.innerHTML = targets.map(t =>
`<div class="testing-row">` +
`<span class="testing-target">${t.suite ? '<span class="pill">suite</span> ' : ""}${esc(t.label)}</span>` +
`<span class="testing-target"><strong>${t.suite ? '<span class="pill">suite</span> ' : ""}${esc(t.label)}</strong>` +
`<br><span class="dim">${esc(t.description || "")}</span>` +
`${(t.tags || []).map(tag => ` <span class="pill">${esc(tag)}</span>`).join("")}</span>` +
`<button class="small" type="button" data-test-target="${esc(t.id)}"${disabled}>Run</button>` +
`</div>`
).join("");
@@ -1183,7 +1206,12 @@ async function loadTestTargets(refresh) {
$("testing-targets").innerHTML = "test targets unavailable";
return false;
}
testCollection = { tests: result.data.tests || [], suites: result.data.suites || [] };
testCollection = {
tests: result.data.tests || [],
test_metadata: result.data.test_metadata || [],
suites: result.data.suites || [],
tags: result.data.tags || [],
};
renderTestTargets();
return true;
}
@@ -1214,9 +1242,7 @@ async function pollTestRunIfActive() {
}
function bindTestingControls() {
const list = $("testing-targets");
if (!list) return;
list.addEventListener("click", event => {
document.addEventListener("click", event => {
const button = event.target.closest("[data-test-target]");
if (!button || button.disabled) return;
event.preventDefault();

107
packages/tracker/meshnet_tracker/test_runner.py
View File

@@ -20,6 +20,7 @@ Security posture (dashboard-test-runner US-001):
from __future__ import annotations
import ast
import os
import re
import subprocess
@@ -61,6 +62,44 @@ PYTHON_ENV_VAR = "MESHNET_PYTHON"
_NODE_ID_RE = re.compile(r"^[\w./\[\]:,= @-]+$")
_MODULE_TAGS: dict[str, tuple[str, ...]] = {
"dashboard": ("dashboard", "http"),
"accounts": ("auth", "accounts", "http"),
"auth_boundary": ("auth", "security", "http"),
"billing": ("billing", "payments", "http"),
"contracts": ("contracts", "settlement"),
"settlement": ("billing", "settlement"),
"tracker": ("tracker", "routing"),
"routing": ("tracker", "routing", "http"),
"dynamic_routing": ("tracker", "routing", "performance"),
"node": ("node", "startup"),
"model": ("node", "model"),
"kv_cache": ("node", "model", "cache"),
"real_": ("real-inference", "node", "model"),
"two_node": ("integration", "inference"),
"openai": ("gateway", "sdk", "http"),
"meshnet_sdk": ("sdk", "gateway", "http"),
"gossip": ("relay", "gossip", "network"),
"relay": ("relay", "network"),
"wallet": ("wallet", "security", "auth"),
"toploc": ("audit", "calibration"),
"forfeiture": ("security", "billing"),
"fraud": ("security", "billing"),
"test_runner": ("dashboard", "test-runner"),
}
_FUNCTION_TAGS: dict[str, tuple[str, ...]] = {
"auth": ("auth", "security"),
"wallet": ("wallet", "security"),
"route": ("routing",),
"stream": ("streaming",),
"cache": ("cache",),
"performance": ("performance",),
"benchmark": ("performance",),
"persist": ("persistence",),
"gossip": ("gossip", "network"),
}
class TestRunnerError(Exception):
"""Base class; carries the HTTP status the handler should send."""
@@ -116,6 +155,28 @@ def _test_python() -> str:
return sys.executable
def _function_description(path: Path, function_name: str) -> str:
"""Return a concise source docstring, falling back to the test name."""
try:
tree = ast.parse(path.read_text())
base_name = function_name.split("[")[0]
function = next(
(node for node in ast.walk(tree)
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
and node.name == base_name),
None,
)
if function is not None:
doc = ast.get_docstring(function)
if doc:
return " ".join(doc.split())
except (OSError, SyntaxError):
pass
words = re.sub(r"([a-z])([A-Z])", r"\\1 \\2", function_name.split("[")[0])
words = words.replace("_", " ").strip()
return words[:1].upper() + words[1:] if words else "Test"
class TestRunManager:
"""Collects pytest node IDs and runs one fixed target at a time."""
@@ -133,6 +194,7 @@ class TestRunManager:
self.run_timeout = run_timeout
self._lock = threading.Lock()
self._collected: set[str] = set()
self._metadata: dict[str, dict] = {}
self._collected_at: float | None = None
self._run: dict | None = None
self._process: subprocess.Popen | None = None
@@ -193,16 +255,45 @@ class TestRunManager:
node_ids = [n for n in node_ids if not _REAL_INFERENCE_PATTERN.search(n.split("::", 1)[0])]
with self._lock:
self._collected = set(node_ids)
self._metadata = {node_id: self._test_metadata(node_id) for node_id in node_ids}
self._collected_at = time.time()
return self._collection_snapshot_locked()
def _test_metadata(self, node_id: str) -> dict:
module_path, function_name = node_id.split("::", 1)
module_name = Path(module_path).stem
tags: set[str] = set()
for key, values in _MODULE_TAGS.items():
if key in module_name:
tags.update(values)
lowered = function_name.lower()
for key, values in _FUNCTION_TAGS.items():
if key in lowered:
tags.update(values)
if not tags:
tags.add("general")
return {
"id": node_id,
"description": _function_description(self.repo_root / module_path, function_name),
"tags": sorted(tags),
}
def _collection_snapshot_locked(self) -> dict:
tag_counts: dict[str, int] = {}
for metadata in self._metadata.values():
for tag in metadata["tags"]:
tag_counts[tag] = tag_counts.get(tag, 0) + 1
return {
"tests": sorted(self._collected),
"test_metadata": [self._metadata[node_id] for node_id in sorted(self._collected)],
"suites": [
{"id": f"suite:{name}", "name": name, "paths": paths}
for name, paths in sorted(self.suites().items())
],
"tags": [
{"id": f"tag:{tag}", "name": tag, "count": count}
for tag, count in sorted(tag_counts.items())
],
"collected_at": self._collected_at,
}
@@ -216,7 +307,21 @@ class TestRunManager:
if target.startswith("-") or not _NODE_ID_RE.match(target):
raise UnknownTargetError("target must be a collected test node ID or an approved suite")
if target.startswith("suite:"):
if target == "suite:all":
with self._lock:
pytest_args = sorted(self._collected)
if not pytest_args:
raise UnknownTargetError("no collected tests are available")
elif target.startswith("tag:"):
tag = target.removeprefix("tag:").strip().lower()
with self._lock:
pytest_args = sorted(
node_id for node_id, metadata in self._metadata.items()
if tag in metadata["tags"]
)
if not pytest_args:
raise UnknownTargetError(f"unknown or empty test tag: {tag!r}")
elif target.startswith("suite:"):
suite_name = target.removeprefix("suite:")
paths = self.suites().get(suite_name)
if paths is None:

5
tests/test_dashboard.py
View File

@@ -344,6 +344,11 @@ def test_dashboard_testing_tab_uses_dynamic_api_and_run_controls():
# Targets are rendered from the API payload (tests + suites), not literals.
assert "testCollection.suites" in html
assert "testCollection.tests" in html
assert "testCollection.test_metadata" in html
assert "testCollection.tags" in html
assert 'data-test-target="suite:all"' in html
assert 'id="testing-tag-filter"' in html
assert "Run ${esc(t.name)}" in html
assert "renderTestTargets" in html
# Run buttons are per-target and delegated; disabled while a run is active.

474
tests/test_node_capability.py Normal file
View File

@@ -0,0 +1,474 @@
"""Tests for the model-agnostic capability report and local recipe manifest."""
import json
import re
from pathlib import Path
import pytest
from meshnet_node import capability, recipe_manifest
from meshnet_node.capability import (
CAPABILITY_SCHEMA_VERSION,
CapabilityReport,
CapabilityReportError,
build_capability_report,
config_fingerprint,
sanitize_diagnostic,
sanitize_diagnostics,
)
from meshnet_node.recipe_manifest import (
RECIPE_SCHEMA_VERSION,
RecipeManifestError,
load_recipe_manifest,
parse_recipe_manifest,
)
# Deliberately unrelated to any vendor the network ships against: the report
# must carry whatever the operator selected, verbatim.
FIXTURE_MODEL_A = "acme-labs/Widget-9000-Instruct"
FIXTURE_MODEL_B = "some_org/tiny.model-v2_PREVIEW"
def _report(**overrides):
kwargs = dict(
model_id=FIXTURE_MODEL_A,
shard_start=0,
shard_end=7,
recipe_id="baseline",
recipe_version="1",
catalogue_version="2026.07.1",
backend_id="torch-transformers",
device="test-device",
status="passed",
duration_ms=142,
validated_at=1_760_000_000.0,
)
kwargs.update(overrides)
return build_capability_report(**kwargs)
# --- model-agnostic identity ------------------------------------------------
@pytest.mark.parametrize("model_id", [FIXTURE_MODEL_A, FIXTURE_MODEL_B])
def test_arbitrary_model_ids_survive_a_json_round_trip_verbatim(model_id):
report = _report(model_id=model_id)
restored = CapabilityReport.from_json(report.to_json())
assert restored.model.model_id == model_id
assert restored.to_dict() == report.to_dict()
def test_two_arbitrary_models_stay_distinct_without_normalization():
a = _report(model_id=FIXTURE_MODEL_A)
b = _report(model_id=FIXTURE_MODEL_B)
assert a.identity_key() != b.identity_key()
assert a.model.model_id != b.model.model_id
def test_capability_and_recipe_modules_have_no_model_or_kernel_branch():
"""No vendor/model/kernel name may be a default or a code-path discriminator."""
forbidden = re.compile(
r"qwen|triton|\bfla\b|flash[_-]?attn|flash[_-]?attention|rocm|nvidia|\bcuda\b",
re.IGNORECASE,
)
sources = [
Path(capability.__file__),
Path(recipe_manifest.__file__),
Path(recipe_manifest.__file__).with_name("recipes.json"),
]
for source in sources:
hits = forbidden.findall(source.read_text(encoding="utf-8"))
assert not hits, f"{source.name} names {hits} — recipes must stay generic data"
def test_device_is_an_opaque_label():
report = _report(device="some-accelerator", device_name="Vendor Accelerator XT")
restored = CapabilityReport.from_json(report.to_json())
assert restored.backend.device == "some-accelerator"
assert restored.backend.device_name == "Vendor Accelerator XT"
# --- schema and serialization -----------------------------------------------
def test_report_dict_has_the_stable_documented_key_set():
payload = _report(
model_config={"num_hidden_layers": 8},
revision="a1b2c3",
quantization="int8",
runtime={"torch": "2.9.0"},
diagnostics=["loaded 8 layers"],
).to_dict()
assert set(payload) == {
"schema_version",
"model",
"shard",
"recipe",
"backend",
"status",
"validated_at",
"duration_ms",
"diagnostics",
}
assert payload["schema_version"] == CAPABILITY_SCHEMA_VERSION
assert set(payload["model"]) == {"model_id", "revision", "config_fingerprint"}
assert set(payload["shard"]) == {"start", "end"}
assert set(payload["recipe"]) == {
"recipe_id",
"recipe_version",
"catalogue_version",
}
assert set(payload["backend"]) == {
"backend_id",
"device",
"device_name",
"quantization",
"runtime",
}
# JSON-serializable end to end.
assert json.loads(json.dumps(payload)) == payload
def test_identity_key_pins_model_shard_recipe_and_backend():
base = _report()
assert base.identity_key() == (
FIXTURE_MODEL_A,
0,
7,
"baseline",
"1",
"torch-transformers",
"test-device",
)
assert _report(shard_end=8).identity_key() != base.identity_key()
assert _report(recipe_version="2").identity_key() != base.identity_key()
assert _report(device="other-device").identity_key() != base.identity_key()
def test_config_fingerprint_is_stable_under_key_order_and_detects_change():
a = config_fingerprint({"num_hidden_layers": 8, "hidden_size": 512})
b = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 8})
c = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 9})
assert a == b
assert a != c
assert a.startswith("sha256:")
assert config_fingerprint(None) is None
assert config_fingerprint("sha256:deadbeef") == "sha256:deadbeef"
def test_status_and_passed_flag():
assert _report(status="passed").passed
assert not _report(status="failed").passed
assert not _report(status="skipped").passed
# --- malformed report input -------------------------------------------------
@pytest.mark.parametrize(
"overrides, expected",
[
({"model_id": ""}, "model.model_id"),
({"shard_start": -1}, "shard.start"),
({"shard_start": 5, "shard_end": 2}, "shard.end"),
({"recipe_id": ""}, "recipe.recipe_id"),
({"catalogue_version": ""}, "recipe.catalogue_version"),
({"backend_id": ""}, "backend.backend_id"),
({"device": ""}, "backend.device"),
({"status": "maybe"}, "status"),
({"duration_ms": -5}, "duration_ms"),
],
)
def test_malformed_report_fields_name_the_offending_field(overrides, expected):
with pytest.raises(CapabilityReportError) as exc:
_report(**overrides)
assert expected in str(exc.value)
def test_unsupported_report_schema_version_is_actionable():
payload = _report().to_dict()
payload["schema_version"] = 99
with pytest.raises(CapabilityReportError) as exc:
CapabilityReport.from_dict(payload)
message = str(exc.value)
assert "99" in message
assert str(CAPABILITY_SCHEMA_VERSION) in message
def test_missing_schema_version_is_rejected():
payload = _report().to_dict()
del payload["schema_version"]
with pytest.raises(CapabilityReportError, match="schema_version"):
CapabilityReport.from_dict(payload)
def test_malformed_report_json_reports_position_not_content():
with pytest.raises(CapabilityReportError) as exc:
CapabilityReport.from_json('{"schema_version": 1,')
assert "line 1" in str(exc.value)
def test_missing_report_section_is_named():
payload = _report().to_dict()
payload["backend"] = "torch"
with pytest.raises(CapabilityReportError, match="backend"):
CapabilityReport.from_dict(payload)
# --- diagnostics sanitization -----------------------------------------------
def test_diagnostics_redact_secret_env_values(monkeypatch):
monkeypatch.setenv("MESHNET_API_TOKEN", "super-secret-value-123")
monkeypatch.setenv("MESHNET_MODEL_ID", "acme-labs/Widget-9000-Instruct")
report = _report(
status="failed",
diagnostics=["auth failed for super-secret-value-123 while fetching config"],
)
text = report.to_json()
assert "super-secret-value-123" not in text
assert capability.REDACTED in report.diagnostics[0]
# A non-secret env var is left alone — the model id stays readable.
assert "acme-labs/Widget-9000-Instruct" in text
@pytest.mark.parametrize(
"raw",
[
"401 Unauthorized: hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345",
"request used Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig",
"config had api_key=abcdef0123456789",
"openai style sk-abcdefghijklmnopqrstuvwxyz012345",
],
)
def test_diagnostics_redact_credential_shaped_strings(raw):
cleaned = sanitize_diagnostic(raw, environ={})
assert capability.REDACTED in cleaned
for secret in (
"hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345",
"eyJhbGciOiJIUzI1NiJ9.payload.sig",
"abcdef0123456789",
"sk-abcdefghijklmnopqrstuvwxyz012345",
):
assert secret not in cleaned
def test_diagnostics_strip_the_home_directory(monkeypatch, tmp_path):
monkeypatch.setenv("HOME", str(tmp_path))
cleaned = sanitize_diagnostic(f"missing weights at {tmp_path}/models/shard", environ={})
assert str(tmp_path) not in cleaned
assert cleaned.startswith("missing weights at ~/models/shard")
def test_diagnostics_are_bounded_in_length_and_count():
long_line = sanitize_diagnostic("x" * 2000, environ={})
assert len(long_line) <= capability.MAX_DIAGNOSTIC_CHARS
many = sanitize_diagnostics([f"line {i}" for i in range(50)], environ={})
assert len(many) == capability.MAX_DIAGNOSTICS + 1
assert "further diagnostic(s) omitted" in many[-1]
def test_diagnostics_reject_non_string_entries():
with pytest.raises(CapabilityReportError, match=r"diagnostics\[1\]"):
sanitize_diagnostics(["ok", 42], environ={})
with pytest.raises(CapabilityReportError, match="bare string"):
sanitize_diagnostics("one big string", environ={})
def test_deserializing_a_report_re_sanitizes_diagnostics(monkeypatch):
monkeypatch.setenv("NODE_SECRET", "leak-me-please")
payload = _report().to_dict()
payload["diagnostics"] = ["backend said leak-me-please"]
restored = CapabilityReport.from_dict(payload)
assert "leak-me-please" not in restored.to_json()
# --- recipe manifest --------------------------------------------------------
def test_packaged_manifest_loads_with_explicit_versions():
manifest = load_recipe_manifest()
assert manifest.schema_version == RECIPE_SCHEMA_VERSION
assert manifest.catalogue_version
assert recipe_manifest.DEFAULT_RECIPE_ID in manifest.ids
for recipe in manifest.recipes:
assert recipe.id and recipe.version and recipe.backend_id
assert isinstance(recipe.params, dict)
def test_packaged_manifest_feeds_a_report():
manifest = load_recipe_manifest()
recipe = manifest.require(recipe_manifest.DEFAULT_RECIPE_ID)
report = _report(
recipe_id=recipe.id,
recipe_version=recipe.version,
catalogue_version=manifest.catalogue_version,
backend_id=recipe.backend_id,
)
assert report.recipe.catalogue_version == manifest.catalogue_version
def test_unknown_recipe_lists_available_ids():
manifest = load_recipe_manifest()
with pytest.raises(RecipeManifestError) as exc:
manifest.require("no-such-recipe")
message = str(exc.value)
assert "no-such-recipe" in message
assert recipe_manifest.DEFAULT_RECIPE_ID in message
def _write_manifest(tmp_path: Path, doc) -> Path:
path = tmp_path / "recipes.json"
path.write_text(
doc if isinstance(doc, str) else json.dumps(doc), encoding="utf-8"
)
return path
def test_valid_local_manifest_loads(tmp_path):
path = _write_manifest(
tmp_path,
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "2099.01.0-test",
"recipes": [
{
"id": "custom",
"version": "3",
"backend_id": "some-backend",
"params": {"knob": 1},
}
],
},
)
manifest = load_recipe_manifest(path)
assert manifest.catalogue_version == "2099.01.0-test"
assert manifest.require("custom").params == {"knob": 1}
assert manifest.source == str(path)
def test_unknown_manifest_schema_version_is_actionable(tmp_path):
path = _write_manifest(
tmp_path,
{
"schema_version": 99,
"catalogue_version": "x",
"recipes": [{"id": "a", "version": "1", "backend_id": "b"}],
},
)
with pytest.raises(RecipeManifestError) as exc:
load_recipe_manifest(path)
message = str(exc.value)
assert "99" in message
assert str(RECIPE_SCHEMA_VERSION) in message
assert str(path) in message
@pytest.mark.parametrize(
"doc, expected",
[
({"catalogue_version": "x", "recipes": []}, "schema_version"),
(
{"schema_version": RECIPE_SCHEMA_VERSION, "recipes": [{"id": "a"}]},
"catalogue_version",
),
(
{"schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": []},
"non-empty JSON array",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [{"version": "1", "backend_id": "b"}],
},
"recipes[0].id",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [{"id": "a", "backend_id": "b"}],
},
"recipes[a].version",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [{"id": "a", "version": "1"}],
},
"recipes[a].backend_id",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [
{"id": "a", "version": "1", "backend_id": "b", "params": [1, 2]}
],
},
"recipes[a].params",
),
(
{
"schema_version": RECIPE_SCHEMA_VERSION,
"catalogue_version": "x",
"recipes": [
{"id": "a", "version": "1", "backend_id": "b"},
{"id": "a", "version": "2", "backend_id": "b"},
],
},
"duplicate recipe id",
),
],
)
def test_malformed_manifest_names_the_offending_field(tmp_path, doc, expected):
path = _write_manifest(tmp_path, doc)
with pytest.raises(RecipeManifestError) as exc:
load_recipe_manifest(path)
assert expected in str(exc.value)
def test_malformed_manifest_json_reports_position_not_content(tmp_path):
path = _write_manifest(tmp_path, '{"schema_version": 1, "catalogue_version":')
with pytest.raises(RecipeManifestError) as exc:
load_recipe_manifest(path)
message = str(exc.value)
assert "not valid JSON" in message
assert "line 1" in message
# The failing document body is never echoed back.
assert "catalogue_version\":" not in message
def test_missing_manifest_file_is_actionable(tmp_path):
with pytest.raises(RecipeManifestError, match="cannot read recipe manifest"):
load_recipe_manifest(tmp_path / "nope.json")
def test_manifest_round_trips_through_its_own_dict():
manifest = load_recipe_manifest()
reparsed = parse_recipe_manifest(manifest.to_dict(), source="<memory>")
assert reparsed.to_dict() == manifest.to_dict()

34
tests/test_tracker_test_runner.py
View File

@@ -34,6 +34,9 @@ def _make_repo(tmp_path: Path) -> Path:
"import time\n\ndef test_sleeps():\n time.sleep(30)\n"
)
(tests / "test_smoke.py").write_text("def test_smoke():\n assert True\n")
(tests / "test_cache.py").write_text(
'def test_cache_roundtrip():\n """Cache roundtrip remains deterministic."""\n assert True\n'
)
# Real-inference stand-in: must never be collected or run by default.
(tests / "test_real_stub.py").write_text(
"def test_real_inference():\n raise AssertionError('env-gated')\n"
@@ -139,6 +142,12 @@ def test_collection_lists_tests_and_excludes_real_inference(tmp_path, monkeypatc
assert data["enabled"] is True
assert "tests/test_quick.py::test_passes" in data["tests"]
assert "tests/test_slow.py::test_sleeps" in data["tests"]
metadata = {item["id"]: item for item in data["test_metadata"]}
assert metadata["tests/test_cache.py::test_cache_roundtrip"]["description"] == (
"Cache roundtrip remains deterministic."
)
assert "cache" in metadata["tests/test_cache.py::test_cache_roundtrip"]["tags"]
assert any(tag["id"] == "tag:cache" for tag in data["tags"])
assert not any("test_real_" in node_id for node_id in data["tests"])
assert data["collected_at"] is not None
suite_ids = [suite["id"] for suite in data["suites"]]
@@ -190,6 +199,31 @@ def test_run_approved_suite_without_prior_collection(tmp_path):
tracker.stop()
def test_run_all_and_tag_targets(tmp_path):
tracker, port, admin, _user = _start_tracker(tmp_path)
try:
assert _request(port, "GET", "/v1/tests", token=admin)[0] == 200
runner = tracker._test_runner
assert runner is not None
all_state = runner.start("suite:all")
assert all_state["run"]["target"] == "suite:all"
assert len(all_state["run"]["args"]) == len(runner._collected)
finally:
tracker.stop()
second_repo = tmp_path / "second"
second_repo.mkdir()
tracker, _port, _admin, _user = _start_tracker(second_repo)
try:
runner = tracker._test_runner
assert runner is not None
runner.collect()
tagged = runner.start("tag:cache")
assert tagged["run"]["args"] == ["tests/test_cache.py::test_cache_roundtrip"]
finally:
tracker.stop()
@pytest.mark.parametrize("target", [
"",
"-x",