diff --git a/.scratch/node-capability-admission/prd.json b/.scratch/node-capability-admission/prd.json index 12eb265..3406ed1 100644 --- a/.scratch/node-capability-admission/prd.json +++ b/.scratch/node-capability-admission/prd.json @@ -1,7 +1,7 @@ { "name": "Model-agnostic Node capability admission", - "branchName": "ralph/node-capability-admission", "description": "Make a Node prove its selected Model Artifact, Shard, and execution recipe work before it becomes routable. Qwen3.6 is only an opt-in development fixture; the implementation and protocol are model-agnostic.", + "branchName": "ralph/node-capability-admission", "userStories": [ { "id": "NCA-001", @@ -36,7 +36,9 @@ "priority": 2, "passes": false, "notes": "Source issue: .scratch/node-capability-admission/issues/02-doctor-real-forward.md", - "dependsOn": ["NCA-001"] + "dependsOn": [ + "NCA-001" + ] }, { "id": "NCA-003", @@ -53,7 +55,10 @@ "priority": 3, "passes": false, "notes": "Source issue: .scratch/node-capability-admission/issues/03-fail-closed-startup-admission.md", - "dependsOn": ["NCA-001", "NCA-002"] + "dependsOn": [ + "NCA-001", + "NCA-002" + ] }, { "id": "NCA-004", @@ -72,7 +77,10 @@ "priority": 4, "passes": false, "notes": "Source issue: .scratch/node-capability-admission/issues/04-tracker-validated-capability-routing.md", - "dependsOn": ["NCA-001", "NCA-003"] + "dependsOn": [ + "NCA-001", + "NCA-003" + ] }, { "id": "NCA-005", @@ -89,7 +97,13 @@ "priority": 5, "passes": false, "notes": "Source issue: .scratch/node-capability-admission/issues/05-docs-hardware-lane-contract.md", - "dependsOn": ["NCA-002", "NCA-004"] + "dependsOn": [ + "NCA-002", + "NCA-004" + ] } - ] -} + ], + "metadata": { + "updatedAt": "2026-07-11T19:02:57.532Z" + } +} \ No newline at end of file diff --git a/packages/node/meshnet_node/capability.py b/packages/node/meshnet_node/capability.py new file mode 100644 index 0000000..eac587b --- /dev/null +++ b/packages/node/meshnet_node/capability.py @@ -0,0 +1,494 @@ +"""Model-agnostic node capability report. + +A capability report is the node's local proof that one concrete combination — +model artifact, shard range, recipe, backend/device — actually executed. It is +plain versioned data: arbitrary model ids pass through verbatim, and no model, +vendor, or kernel name is a default or a code-path discriminator here. + +Later stories consume this: `doctor` produces a report from a real forward +(NCA-002), startup refuses to register without a fresh passing one (NCA-003), +and the tracker routes only to admitted, matching capabilities (NCA-004). +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import time +from dataclasses import dataclass, field +from typing import Any, Mapping + +# Layout of the serialized report. Bump when the JSON shape changes. +CAPABILITY_SCHEMA_VERSION = 1 + +STATUS_PASSED = "passed" +STATUS_FAILED = "failed" +STATUS_SKIPPED = "skipped" +VALID_STATUSES = (STATUS_PASSED, STATUS_FAILED, STATUS_SKIPPED) + +# Diagnostics are operator-facing, not a log sink: keep them short and few. +MAX_DIAGNOSTIC_CHARS = 500 +MAX_DIAGNOSTICS = 20 + +REDACTED = "[redacted]" + +# An env var whose *name* contains one of these holds a secret by convention. +_SECRET_NAME_HINTS = ( + "TOKEN", + "SECRET", + "PASSWORD", + "PASSWD", + "CREDENTIAL", + "APIKEY", + "API_KEY", + "PRIVATE_KEY", + "ACCESS_KEY", +) +# Below this length a value is too generic to redact without mangling prose. +_MIN_SECRET_LEN = 6 + +# Provider-shaped bearer credentials that can appear in a backend error string. +_CREDENTIAL_PATTERNS = ( + re.compile(r"\b[A-Za-z0-9_]{2,6}_[A-Za-z0-9]{16,}\b"), # hf_…, ghp_…, sk_live_… + re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"), + re.compile(r"(?i)\bbearer\s+\S+"), + re.compile(r"(?i)\b(?:token|api[_-]?key|password|secret)\s*[=:]\s*\S+"), +) + + +class CapabilityReportError(ValueError): + """Raised when report input is malformed. + + Messages name the offending field and the expected shape, and carry no + caller-supplied payload beyond the field path itself. + """ + + +def _secret_env_values(environ: Mapping[str, str] | None = None) -> list[str]: + env = os.environ if environ is None else environ + values: list[str] = [] + for name, value in env.items(): + if not isinstance(value, str) or len(value) < _MIN_SECRET_LEN: + continue + upper = name.upper() + if any(hint in upper for hint in _SECRET_NAME_HINTS): + values.append(value) + # Redact longest first so a value that contains another is not partially masked. + return sorted(values, key=len, reverse=True) + + +def sanitize_diagnostic( + text: str, + environ: Mapping[str, str] | None = None, +) -> str: + """Return `text` with credentials and host identity stripped, clipped to length.""" + cleaned = " ".join(str(text).split()) + + for secret in _secret_env_values(environ): + cleaned = cleaned.replace(secret, REDACTED) + + for pattern in _CREDENTIAL_PATTERNS: + cleaned = pattern.sub(REDACTED, cleaned) + + home = os.path.expanduser("~") + if home and home not in ("/", ""): + cleaned = cleaned.replace(home, "~") + + if len(cleaned) > MAX_DIAGNOSTIC_CHARS: + cleaned = cleaned[: MAX_DIAGNOSTIC_CHARS - 1].rstrip() + "…" + return cleaned + + +def sanitize_diagnostics( + diagnostics: Any, + environ: Mapping[str, str] | None = None, +) -> tuple[str, ...]: + """Sanitize and bound a diagnostics sequence.""" + if diagnostics is None: + return () + if isinstance(diagnostics, str): + raise CapabilityReportError( + "'diagnostics' must be a list of strings, got a bare string" + ) + try: + items = list(diagnostics) + except TypeError as exc: + raise CapabilityReportError( + f"'diagnostics' must be a list of strings, got {type(diagnostics).__name__}" + ) from exc + + out: list[str] = [] + for index, item in enumerate(items[:MAX_DIAGNOSTICS]): + if not isinstance(item, str): + raise CapabilityReportError( + f"'diagnostics[{index}]' must be a string, got {type(item).__name__}" + ) + cleaned = sanitize_diagnostic(item, environ) + if cleaned: + out.append(cleaned) + dropped = len(items) - MAX_DIAGNOSTICS + if dropped > 0: + out.append(f"… {dropped} further diagnostic(s) omitted") + return tuple(out) + + +def config_fingerprint(config: Any) -> str | None: + """Return a stable content hash of a model config mapping. + + Two nodes that loaded the same artifact revision with the same config + produce the same fingerprint; anything unserializable degrades to its + string form rather than failing the report. + """ + if config is None: + return None + if isinstance(config, str): + return config if config.startswith("sha256:") else "sha256:" + _sha256(config) + if not isinstance(config, Mapping): + raise CapabilityReportError( + f"model config must be a mapping or a fingerprint string, " + f"got {type(config).__name__}" + ) + canonical = json.dumps( + config, sort_keys=True, separators=(",", ":"), default=str, ensure_ascii=False + ) + return "sha256:" + _sha256(canonical) + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _require_text(value: Any, field_name: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise CapabilityReportError(f"{field_name!r} must be a non-empty string") + return value + + +def _optional_text(value: Any, field_name: str) -> str | None: + if value is None: + return None + return _require_text(value, field_name) + + +def _require_int(value: Any, field_name: str, minimum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise CapabilityReportError(f"{field_name!r} must be an integer") + if value < minimum: + raise CapabilityReportError(f"{field_name!r} must be >= {minimum}, got {value}") + return value + + +@dataclass(frozen=True) +class ModelIdentity: + """Which artifact was validated. `model_id` is opaque and preserved verbatim.""" + + model_id: str + revision: str | None = None + config_fingerprint: str | None = None + + def __post_init__(self) -> None: + _require_text(self.model_id, "model.model_id") + _optional_text(self.revision, "model.revision") + _optional_text(self.config_fingerprint, "model.config_fingerprint") + + def to_dict(self) -> dict: + return { + "model_id": self.model_id, + "revision": self.revision, + "config_fingerprint": self.config_fingerprint, + } + + @classmethod + def from_dict(cls, data: Any) -> ModelIdentity: + doc = _as_mapping(data, "model") + return cls( + model_id=_require_text(doc.get("model_id"), "model.model_id"), + revision=_optional_text(doc.get("revision"), "model.revision"), + config_fingerprint=_optional_text( + doc.get("config_fingerprint"), "model.config_fingerprint" + ), + ) + + +@dataclass(frozen=True) +class ShardRange: + """Inclusive layer range, matching the CLI and backend convention.""" + + start: int + end: int + + def __post_init__(self) -> None: + _require_int(self.start, "shard.start", 0) + _require_int(self.end, "shard.end", 0) + if self.end < self.start: + raise CapabilityReportError( + f"'shard.end' ({self.end}) must be >= 'shard.start' ({self.start})" + ) + + def to_dict(self) -> dict: + return {"start": self.start, "end": self.end} + + @classmethod + def from_dict(cls, data: Any) -> ShardRange: + doc = _as_mapping(data, "shard") + return cls( + start=_require_int(doc.get("start"), "shard.start", 0), + end=_require_int(doc.get("end"), "shard.end", 0), + ) + + +@dataclass(frozen=True) +class RecipeIdentity: + """Which recipe, from which catalogue, was exercised.""" + + recipe_id: str + recipe_version: str + catalogue_version: str + + def __post_init__(self) -> None: + _require_text(self.recipe_id, "recipe.recipe_id") + _require_text(self.recipe_version, "recipe.recipe_version") + _require_text(self.catalogue_version, "recipe.catalogue_version") + + def to_dict(self) -> dict: + return { + "recipe_id": self.recipe_id, + "recipe_version": self.recipe_version, + "catalogue_version": self.catalogue_version, + } + + @classmethod + def from_dict(cls, data: Any) -> RecipeIdentity: + doc = _as_mapping(data, "recipe") + return cls( + recipe_id=_require_text(doc.get("recipe_id"), "recipe.recipe_id"), + recipe_version=_require_text( + doc.get("recipe_version"), "recipe.recipe_version" + ), + catalogue_version=_require_text( + doc.get("catalogue_version"), "recipe.catalogue_version" + ), + ) + + +@dataclass(frozen=True) +class BackendIdentity: + """Which execution stack ran it. All fields are opaque labels, never branches.""" + + backend_id: str + device: str + device_name: str | None = None + quantization: str | None = None + runtime: Mapping[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + _require_text(self.backend_id, "backend.backend_id") + _require_text(self.device, "backend.device") + _optional_text(self.device_name, "backend.device_name") + _optional_text(self.quantization, "backend.quantization") + for key, value in self.runtime.items(): + if not isinstance(key, str) or not isinstance(value, str): + raise CapabilityReportError( + "'backend.runtime' must map string names to string versions" + ) + + def to_dict(self) -> dict: + return { + "backend_id": self.backend_id, + "device": self.device, + "device_name": self.device_name, + "quantization": self.quantization, + "runtime": dict(self.runtime), + } + + @classmethod + def from_dict(cls, data: Any) -> BackendIdentity: + doc = _as_mapping(data, "backend") + runtime = doc.get("runtime") or {} + if not isinstance(runtime, Mapping): + raise CapabilityReportError("'backend.runtime' must be a JSON object") + return cls( + backend_id=_require_text(doc.get("backend_id"), "backend.backend_id"), + device=_require_text(doc.get("device"), "backend.device"), + device_name=_optional_text(doc.get("device_name"), "backend.device_name"), + quantization=_optional_text( + doc.get("quantization"), "backend.quantization" + ), + runtime={str(k): str(v) for k, v in runtime.items()}, + ) + + +def _as_mapping(data: Any, field_name: str) -> Mapping[str, Any]: + if not isinstance(data, Mapping): + raise CapabilityReportError( + f"{field_name!r} must be a JSON object, got {type(data).__name__}" + ) + return data + + +@dataclass(frozen=True) +class CapabilityReport: + """One node's validated (or failed) model/shard/recipe/backend combination.""" + + model: ModelIdentity + shard: ShardRange + recipe: RecipeIdentity + backend: BackendIdentity + status: str + validated_at: float + duration_ms: int + diagnostics: tuple[str, ...] = () + schema_version: int = CAPABILITY_SCHEMA_VERSION + + def __post_init__(self) -> None: + if self.status not in VALID_STATUSES: + raise CapabilityReportError( + f"'status' must be one of {', '.join(VALID_STATUSES)}; got {self.status!r}" + ) + if isinstance(self.validated_at, bool) or not isinstance( + self.validated_at, (int, float) + ): + raise CapabilityReportError("'validated_at' must be a Unix timestamp") + if self.validated_at < 0: + raise CapabilityReportError("'validated_at' must not be negative") + _require_int(self.duration_ms, "duration_ms", 0) + _require_int(self.schema_version, "schema_version", 1) + + @property + def passed(self) -> bool: + return self.status == STATUS_PASSED + + def identity_key(self) -> tuple[str, int, int, str, str, str, str]: + """The tuple a consumer must match to reuse this proof. + + Startup and the tracker compare on exactly this: a report proves nothing + about a different model, shard, recipe version, or device. + """ + return ( + self.model.model_id, + self.shard.start, + self.shard.end, + self.recipe.recipe_id, + self.recipe.recipe_version, + self.backend.backend_id, + self.backend.device, + ) + + def age_seconds(self, now: float | None = None) -> float: + return max(0.0, (time.time() if now is None else now) - self.validated_at) + + def to_dict(self) -> dict: + return { + "schema_version": self.schema_version, + "model": self.model.to_dict(), + "shard": self.shard.to_dict(), + "recipe": self.recipe.to_dict(), + "backend": self.backend.to_dict(), + "status": self.status, + "validated_at": self.validated_at, + "duration_ms": self.duration_ms, + "diagnostics": list(self.diagnostics), + } + + def to_json(self, indent: int | None = None) -> str: + return json.dumps(self.to_dict(), indent=indent, sort_keys=True) + + @classmethod + def from_dict(cls, data: Any) -> CapabilityReport: + doc = _as_mapping(data, "report") + + if "schema_version" not in doc: + raise CapabilityReportError( + "report is missing 'schema_version'; this node reads capability " + f"schema version {CAPABILITY_SCHEMA_VERSION}" + ) + schema_version = _require_int(doc["schema_version"], "schema_version", 1) + if schema_version != CAPABILITY_SCHEMA_VERSION: + raise CapabilityReportError( + f"report declares capability schema version {schema_version}, but this " + f"node reads version {CAPABILITY_SCHEMA_VERSION}" + ) + + validated_at = doc.get("validated_at") + if isinstance(validated_at, bool) or not isinstance( + validated_at, (int, float) + ): + raise CapabilityReportError("'validated_at' must be a Unix timestamp") + + return cls( + schema_version=schema_version, + model=ModelIdentity.from_dict(doc.get("model")), + shard=ShardRange.from_dict(doc.get("shard")), + recipe=RecipeIdentity.from_dict(doc.get("recipe")), + backend=BackendIdentity.from_dict(doc.get("backend")), + status=_require_text(doc.get("status"), "status"), + validated_at=float(validated_at), + duration_ms=_require_int(doc.get("duration_ms"), "duration_ms", 0), + diagnostics=sanitize_diagnostics(doc.get("diagnostics")), + ) + + @classmethod + def from_json(cls, text: str) -> CapabilityReport: + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise CapabilityReportError( + f"capability report is not valid JSON: {exc.msg} " + f"at line {exc.lineno} column {exc.colno}" + ) from exc + return cls.from_dict(data) + + +def build_capability_report( + *, + model_id: str, + shard_start: int, + shard_end: int, + recipe_id: str, + recipe_version: str, + catalogue_version: str, + backend_id: str, + device: str, + status: str, + duration_ms: int, + revision: str | None = None, + model_config: Any = None, + device_name: str | None = None, + quantization: str | None = None, + runtime: Mapping[str, str] | None = None, + diagnostics: Any = None, + validated_at: float | None = None, + environ: Mapping[str, str] | None = None, +) -> CapabilityReport: + """Assemble a report from flat validation results. + + `model_config` may be the loaded config mapping (hashed into a fingerprint) + or an already-computed ``sha256:…`` string. `validated_at` defaults to now, + so callers that need determinism pass it explicitly. + """ + return CapabilityReport( + model=ModelIdentity( + model_id=model_id, + revision=revision, + config_fingerprint=config_fingerprint(model_config), + ), + shard=ShardRange(start=shard_start, end=shard_end), + recipe=RecipeIdentity( + recipe_id=recipe_id, + recipe_version=recipe_version, + catalogue_version=catalogue_version, + ), + backend=BackendIdentity( + backend_id=backend_id, + device=device, + device_name=device_name, + quantization=quantization, + runtime=dict(runtime or {}), + ), + status=status, + validated_at=time.time() if validated_at is None else validated_at, + duration_ms=duration_ms, + diagnostics=sanitize_diagnostics(diagnostics, environ), + ) diff --git a/packages/node/meshnet_node/recipe_manifest.py b/packages/node/meshnet_node/recipe_manifest.py new file mode 100644 index 0000000..f2f4a2c --- /dev/null +++ b/packages/node/meshnet_node/recipe_manifest.py @@ -0,0 +1,222 @@ +"""Local, versioned recipe manifest. + +A recipe is *data*: a named, versioned set of execution parameters handed to the +model backend. It carries no model- or vendor-specific code path — a recipe is +only ever valid once its own real forward has succeeded on this node +(see :mod:`meshnet_node.capability`). + +The manifest ships with the node release. ``schema_version`` describes the file +layout this reader understands; ``catalogue_version`` identifies the recipe set +itself so a tracker can reason about which catalogue a node validated against. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from importlib.resources import files +from pathlib import Path +from typing import Any, Mapping + +# Layout of recipes.json understood by this reader. Bump when the file shape changes. +RECIPE_SCHEMA_VERSION = 1 + +DEFAULT_RECIPE_ID = "baseline" + +_MANIFEST_RESOURCE = "recipes.json" + + +class RecipeManifestError(ValueError): + """Raised when a recipe manifest is missing, malformed, or unsupported. + + The message is operator-facing: it names the source and the fix, and never + echoes raw file content back (a manifest may sit next to secrets in a + misconfigured deployment). + """ + + +@dataclass(frozen=True) +class Recipe: + """One named, versioned execution recipe.""" + + id: str + version: str + backend_id: str + description: str = "" + params: Mapping[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict: + return { + "id": self.id, + "version": self.version, + "backend_id": self.backend_id, + "description": self.description, + "params": dict(self.params), + } + + +@dataclass(frozen=True) +class RecipeManifest: + """A parsed, validated recipe catalogue.""" + + schema_version: int + catalogue_version: str + recipes: tuple[Recipe, ...] + source: str = "" + + def get(self, recipe_id: str) -> Recipe | None: + for recipe in self.recipes: + if recipe.id == recipe_id: + return recipe + return None + + def require(self, recipe_id: str) -> Recipe: + """Return the named recipe, or raise listing what this catalogue offers.""" + recipe = self.get(recipe_id) + if recipe is None: + available = ", ".join(r.id for r in self.recipes) or "(none)" + raise RecipeManifestError( + f"unknown recipe {recipe_id!r} in {self.source}; " + f"available recipes: {available}" + ) + return recipe + + @property + def ids(self) -> tuple[str, ...]: + return tuple(r.id for r in self.recipes) + + def to_dict(self) -> dict: + return { + "schema_version": self.schema_version, + "catalogue_version": self.catalogue_version, + "recipes": [r.to_dict() for r in self.recipes], + } + + +def _require_mapping(value: Any, what: str, source: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise RecipeManifestError( + f"{what} in {source} must be a JSON object, got {type(value).__name__}" + ) + return value + + +def _require_text(value: Any, what: str, source: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise RecipeManifestError( + f"{what} in {source} must be a non-empty string" + ) + return value + + +def _parse_recipe(raw: Any, index: int, source: str) -> Recipe: + entry = _require_mapping(raw, f"recipes[{index}]", source) + recipe_id = _require_text(entry.get("id"), f"recipes[{index}].id", source) + version = _require_text( + entry.get("version"), f"recipes[{recipe_id}].version", source + ) + backend_id = _require_text( + entry.get("backend_id"), f"recipes[{recipe_id}].backend_id", source + ) + params = entry.get("params", {}) + if params is None: + params = {} + _require_mapping(params, f"recipes[{recipe_id}].params", source) + description = entry.get("description", "") + if not isinstance(description, str): + raise RecipeManifestError( + f"recipes[{recipe_id}].description in {source} must be a string" + ) + return Recipe( + id=recipe_id, + version=version, + backend_id=backend_id, + description=description, + params=dict(params), + ) + + +def parse_recipe_manifest(data: Any, source: str = "") -> RecipeManifest: + """Validate an already-decoded manifest document.""" + doc = _require_mapping(data, "manifest root", source) + + if "schema_version" not in doc: + raise RecipeManifestError( + f"{source} is missing 'schema_version'; " + f"this node reads recipe schema version {RECIPE_SCHEMA_VERSION}" + ) + schema_version = doc["schema_version"] + if not isinstance(schema_version, int) or isinstance(schema_version, bool): + raise RecipeManifestError( + f"'schema_version' in {source} must be an integer, " + f"got {type(schema_version).__name__}" + ) + if schema_version != RECIPE_SCHEMA_VERSION: + raise RecipeManifestError( + f"{source} declares recipe schema version {schema_version}, " + f"but this node reads version {RECIPE_SCHEMA_VERSION}; " + "upgrade the node or use a manifest for the supported version" + ) + + catalogue_version = _require_text( + doc.get("catalogue_version"), "'catalogue_version'", source + ) + + raw_recipes = doc.get("recipes") + if not isinstance(raw_recipes, list) or not raw_recipes: + raise RecipeManifestError( + f"'recipes' in {source} must be a non-empty JSON array" + ) + + recipes: list[Recipe] = [] + seen: set[str] = set() + for index, raw in enumerate(raw_recipes): + recipe = _parse_recipe(raw, index, source) + if recipe.id in seen: + raise RecipeManifestError( + f"duplicate recipe id {recipe.id!r} in {source}; recipe ids must be unique" + ) + seen.add(recipe.id) + recipes.append(recipe) + + return RecipeManifest( + schema_version=schema_version, + catalogue_version=catalogue_version, + recipes=tuple(recipes), + source=source, + ) + + +def load_recipe_manifest(path: Path | None = None) -> RecipeManifest: + """Load the packaged manifest, or one at ``path``. + + No network access and no remote catalogue: P0 recipes ship with the node. + """ + if path is None: + source = f"packaged {_MANIFEST_RESOURCE}" + try: + raw = files("meshnet_node").joinpath(_MANIFEST_RESOURCE).read_text( + encoding="utf-8" + ) + except (OSError, FileNotFoundError, ModuleNotFoundError) as exc: + raise RecipeManifestError( + f"{source} is missing from this node installation " + f"({type(exc).__name__}); reinstall the node package" + ) from exc + else: + source = str(path) + try: + raw = path.read_text(encoding="utf-8") + except OSError as exc: + raise RecipeManifestError( + f"cannot read recipe manifest {source}: {exc.strerror or exc}" + ) from exc + + try: + data = json.loads(raw) + except json.JSONDecodeError as exc: + raise RecipeManifestError( + f"{source} is not valid JSON: {exc.msg} at line {exc.lineno} column {exc.colno}" + ) from exc + + return parse_recipe_manifest(data, source=source) diff --git a/packages/node/meshnet_node/recipes.json b/packages/node/meshnet_node/recipes.json new file mode 100644 index 0000000..73cd29c --- /dev/null +++ b/packages/node/meshnet_node/recipes.json @@ -0,0 +1,31 @@ +{ + "schema_version": 1, + "catalogue_version": "2026.07.1", + "recipes": [ + { + "id": "baseline", + "version": "1", + "backend_id": "torch-transformers", + "description": "Backend defaults with no execution overrides.", + "params": {} + }, + { + "id": "eager-attention", + "version": "1", + "backend_id": "torch-transformers", + "description": "Force the reference attention path instead of the backend's autoselected one.", + "params": { + "attn_implementation": "eager" + } + }, + { + "id": "stateless", + "version": "1", + "backend_id": "torch-transformers", + "description": "Disable the incremental state cache; every step re-runs the full prefill.", + "params": { + "use_cache": false + } + } + ] +} diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html index 026ea64..15898bf 100644 --- a/packages/tracker/meshnet_tracker/dashboard.html +++ b/packages/tracker/meshnet_tracker/dashboard.html @@ -298,8 +298,13 @@

Tests & suites

+ +
+
admin login required
@@ -1016,7 +1021,7 @@ function renderConsole(data) { // ---- testing tab (opt-in tracker test runner, dashboard-test-runner US-002) ---- -let testCollection = { tests: [], suites: [] }; +let testCollection = { tests: [], test_metadata: [], suites: [], tags: [] }; let testRun = null; let testRunStarting = false; @@ -1122,21 +1127,39 @@ function renderTestTargets() { const el = $("testing-targets"); if (!el) return; const filter = ($("testing-filter")?.value || "").trim().toLowerCase(); + const selectedTag = ($("testing-tag-filter")?.value || "").trim().toLowerCase(); + const metadata = testCollection.test_metadata || testCollection.tests.map(id => ({ id, description: id, tags: [] })); const suites = (testCollection.suites || []) - .map(s => ({ id: s.id, label: `${s.name} (${(s.paths || []).join(", ")})`, suite: true })); - const tests = (testCollection.tests || []).map(t => ({ id: t, label: t, suite: false })); + .map(s => ({ id: s.id, label: `${s.name} (${(s.paths || []).join(", ")})`, description: "Approved test suite", tags: ["suite"], suite: true })); + const tests = metadata.map(t => ({ ...t, label: t.id, suite: false })); const targets = [...suites, ...tests] - .filter(t => !filter || t.label.toLowerCase().includes(filter)); + .filter(t => !filter || `${t.label} ${t.description} ${(t.tags || []).join(" ")}`.toLowerCase().includes(filter)) + .filter(t => !selectedTag || t.suite || (t.tags || []).includes(selectedTag)); + const disabled = testRunActive() ? " disabled" : ""; + const tagFilter = $("testing-tag-filter"); + if (tagFilter) { + const value = tagFilter.value; + tagFilter.innerHTML = '' + + (testCollection.tags || []).map(t => ``).join(""); + tagFilter.value = value; + } + const tagActions = $("testing-tag-actions"); + if (tagActions) { + tagActions.innerHTML = (testCollection.tags || []).map(t => + `` + ).join(""); + } if (!targets.length) { el.className = "empty"; - el.innerHTML = filter ? "no targets match the filter" : "no tests collected"; + el.innerHTML = filter || selectedTag ? "no tests match the filter" : "no tests collected"; return; } - const disabled = testRunActive() ? " disabled" : ""; el.className = "testing-list"; el.innerHTML = targets.map(t => `
` + - `${t.suite ? 'suite ' : ""}${esc(t.label)}` + + `${t.suite ? 'suite ' : ""}${esc(t.label)}` + + `
${esc(t.description || "")}` + + `${(t.tags || []).map(tag => ` ${esc(tag)}`).join("")}` + `` + `
` ).join(""); @@ -1183,7 +1206,12 @@ async function loadTestTargets(refresh) { $("testing-targets").innerHTML = "test targets unavailable"; return false; } - testCollection = { tests: result.data.tests || [], suites: result.data.suites || [] }; + testCollection = { + tests: result.data.tests || [], + test_metadata: result.data.test_metadata || [], + suites: result.data.suites || [], + tags: result.data.tags || [], + }; renderTestTargets(); return true; } @@ -1214,9 +1242,7 @@ async function pollTestRunIfActive() { } function bindTestingControls() { - const list = $("testing-targets"); - if (!list) return; - list.addEventListener("click", event => { + document.addEventListener("click", event => { const button = event.target.closest("[data-test-target]"); if (!button || button.disabled) return; event.preventDefault(); diff --git a/packages/tracker/meshnet_tracker/test_runner.py b/packages/tracker/meshnet_tracker/test_runner.py index fdd8b6d..170e765 100644 --- a/packages/tracker/meshnet_tracker/test_runner.py +++ b/packages/tracker/meshnet_tracker/test_runner.py @@ -20,6 +20,7 @@ Security posture (dashboard-test-runner US-001): from __future__ import annotations +import ast import os import re import subprocess @@ -61,6 +62,44 @@ PYTHON_ENV_VAR = "MESHNET_PYTHON" _NODE_ID_RE = re.compile(r"^[\w./\[\]:,= @-]+$") +_MODULE_TAGS: dict[str, tuple[str, ...]] = { + "dashboard": ("dashboard", "http"), + "accounts": ("auth", "accounts", "http"), + "auth_boundary": ("auth", "security", "http"), + "billing": ("billing", "payments", "http"), + "contracts": ("contracts", "settlement"), + "settlement": ("billing", "settlement"), + "tracker": ("tracker", "routing"), + "routing": ("tracker", "routing", "http"), + "dynamic_routing": ("tracker", "routing", "performance"), + "node": ("node", "startup"), + "model": ("node", "model"), + "kv_cache": ("node", "model", "cache"), + "real_": ("real-inference", "node", "model"), + "two_node": ("integration", "inference"), + "openai": ("gateway", "sdk", "http"), + "meshnet_sdk": ("sdk", "gateway", "http"), + "gossip": ("relay", "gossip", "network"), + "relay": ("relay", "network"), + "wallet": ("wallet", "security", "auth"), + "toploc": ("audit", "calibration"), + "forfeiture": ("security", "billing"), + "fraud": ("security", "billing"), + "test_runner": ("dashboard", "test-runner"), +} + +_FUNCTION_TAGS: dict[str, tuple[str, ...]] = { + "auth": ("auth", "security"), + "wallet": ("wallet", "security"), + "route": ("routing",), + "stream": ("streaming",), + "cache": ("cache",), + "performance": ("performance",), + "benchmark": ("performance",), + "persist": ("persistence",), + "gossip": ("gossip", "network"), +} + class TestRunnerError(Exception): """Base class; carries the HTTP status the handler should send.""" @@ -116,6 +155,28 @@ def _test_python() -> str: return sys.executable +def _function_description(path: Path, function_name: str) -> str: + """Return a concise source docstring, falling back to the test name.""" + try: + tree = ast.parse(path.read_text()) + base_name = function_name.split("[")[0] + function = next( + (node for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == base_name), + None, + ) + if function is not None: + doc = ast.get_docstring(function) + if doc: + return " ".join(doc.split()) + except (OSError, SyntaxError): + pass + words = re.sub(r"([a-z])([A-Z])", r"\\1 \\2", function_name.split("[")[0]) + words = words.replace("_", " ").strip() + return words[:1].upper() + words[1:] if words else "Test" + + class TestRunManager: """Collects pytest node IDs and runs one fixed target at a time.""" @@ -133,6 +194,7 @@ class TestRunManager: self.run_timeout = run_timeout self._lock = threading.Lock() self._collected: set[str] = set() + self._metadata: dict[str, dict] = {} self._collected_at: float | None = None self._run: dict | None = None self._process: subprocess.Popen | None = None @@ -193,16 +255,45 @@ class TestRunManager: node_ids = [n for n in node_ids if not _REAL_INFERENCE_PATTERN.search(n.split("::", 1)[0])] with self._lock: self._collected = set(node_ids) + self._metadata = {node_id: self._test_metadata(node_id) for node_id in node_ids} self._collected_at = time.time() return self._collection_snapshot_locked() + def _test_metadata(self, node_id: str) -> dict: + module_path, function_name = node_id.split("::", 1) + module_name = Path(module_path).stem + tags: set[str] = set() + for key, values in _MODULE_TAGS.items(): + if key in module_name: + tags.update(values) + lowered = function_name.lower() + for key, values in _FUNCTION_TAGS.items(): + if key in lowered: + tags.update(values) + if not tags: + tags.add("general") + return { + "id": node_id, + "description": _function_description(self.repo_root / module_path, function_name), + "tags": sorted(tags), + } + def _collection_snapshot_locked(self) -> dict: + tag_counts: dict[str, int] = {} + for metadata in self._metadata.values(): + for tag in metadata["tags"]: + tag_counts[tag] = tag_counts.get(tag, 0) + 1 return { "tests": sorted(self._collected), + "test_metadata": [self._metadata[node_id] for node_id in sorted(self._collected)], "suites": [ {"id": f"suite:{name}", "name": name, "paths": paths} for name, paths in sorted(self.suites().items()) ], + "tags": [ + {"id": f"tag:{tag}", "name": tag, "count": count} + for tag, count in sorted(tag_counts.items()) + ], "collected_at": self._collected_at, } @@ -216,7 +307,21 @@ class TestRunManager: if target.startswith("-") or not _NODE_ID_RE.match(target): raise UnknownTargetError("target must be a collected test node ID or an approved suite") - if target.startswith("suite:"): + if target == "suite:all": + with self._lock: + pytest_args = sorted(self._collected) + if not pytest_args: + raise UnknownTargetError("no collected tests are available") + elif target.startswith("tag:"): + tag = target.removeprefix("tag:").strip().lower() + with self._lock: + pytest_args = sorted( + node_id for node_id, metadata in self._metadata.items() + if tag in metadata["tags"] + ) + if not pytest_args: + raise UnknownTargetError(f"unknown or empty test tag: {tag!r}") + elif target.startswith("suite:"): suite_name = target.removeprefix("suite:") paths = self.suites().get(suite_name) if paths is None: diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index a7ae450..e053547 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -344,6 +344,11 @@ def test_dashboard_testing_tab_uses_dynamic_api_and_run_controls(): # Targets are rendered from the API payload (tests + suites), not literals. assert "testCollection.suites" in html assert "testCollection.tests" in html + assert "testCollection.test_metadata" in html + assert "testCollection.tags" in html + assert 'data-test-target="suite:all"' in html + assert 'id="testing-tag-filter"' in html + assert "Run ${esc(t.name)}" in html assert "renderTestTargets" in html # Run buttons are per-target and delegated; disabled while a run is active. diff --git a/tests/test_node_capability.py b/tests/test_node_capability.py new file mode 100644 index 0000000..2f90be7 --- /dev/null +++ b/tests/test_node_capability.py @@ -0,0 +1,474 @@ +"""Tests for the model-agnostic capability report and local recipe manifest.""" + +import json +import re +from pathlib import Path + +import pytest + +from meshnet_node import capability, recipe_manifest +from meshnet_node.capability import ( + CAPABILITY_SCHEMA_VERSION, + CapabilityReport, + CapabilityReportError, + build_capability_report, + config_fingerprint, + sanitize_diagnostic, + sanitize_diagnostics, +) +from meshnet_node.recipe_manifest import ( + RECIPE_SCHEMA_VERSION, + RecipeManifestError, + load_recipe_manifest, + parse_recipe_manifest, +) + +# Deliberately unrelated to any vendor the network ships against: the report +# must carry whatever the operator selected, verbatim. +FIXTURE_MODEL_A = "acme-labs/Widget-9000-Instruct" +FIXTURE_MODEL_B = "some_org/tiny.model-v2_PREVIEW" + + +def _report(**overrides): + kwargs = dict( + model_id=FIXTURE_MODEL_A, + shard_start=0, + shard_end=7, + recipe_id="baseline", + recipe_version="1", + catalogue_version="2026.07.1", + backend_id="torch-transformers", + device="test-device", + status="passed", + duration_ms=142, + validated_at=1_760_000_000.0, + ) + kwargs.update(overrides) + return build_capability_report(**kwargs) + + +# --- model-agnostic identity ------------------------------------------------ + + +@pytest.mark.parametrize("model_id", [FIXTURE_MODEL_A, FIXTURE_MODEL_B]) +def test_arbitrary_model_ids_survive_a_json_round_trip_verbatim(model_id): + report = _report(model_id=model_id) + restored = CapabilityReport.from_json(report.to_json()) + + assert restored.model.model_id == model_id + assert restored.to_dict() == report.to_dict() + + +def test_two_arbitrary_models_stay_distinct_without_normalization(): + a = _report(model_id=FIXTURE_MODEL_A) + b = _report(model_id=FIXTURE_MODEL_B) + + assert a.identity_key() != b.identity_key() + assert a.model.model_id != b.model.model_id + + +def test_capability_and_recipe_modules_have_no_model_or_kernel_branch(): + """No vendor/model/kernel name may be a default or a code-path discriminator.""" + forbidden = re.compile( + r"qwen|triton|\bfla\b|flash[_-]?attn|flash[_-]?attention|rocm|nvidia|\bcuda\b", + re.IGNORECASE, + ) + sources = [ + Path(capability.__file__), + Path(recipe_manifest.__file__), + Path(recipe_manifest.__file__).with_name("recipes.json"), + ] + for source in sources: + hits = forbidden.findall(source.read_text(encoding="utf-8")) + assert not hits, f"{source.name} names {hits} — recipes must stay generic data" + + +def test_device_is_an_opaque_label(): + report = _report(device="some-accelerator", device_name="Vendor Accelerator XT") + restored = CapabilityReport.from_json(report.to_json()) + + assert restored.backend.device == "some-accelerator" + assert restored.backend.device_name == "Vendor Accelerator XT" + + +# --- schema and serialization ----------------------------------------------- + + +def test_report_dict_has_the_stable_documented_key_set(): + payload = _report( + model_config={"num_hidden_layers": 8}, + revision="a1b2c3", + quantization="int8", + runtime={"torch": "2.9.0"}, + diagnostics=["loaded 8 layers"], + ).to_dict() + + assert set(payload) == { + "schema_version", + "model", + "shard", + "recipe", + "backend", + "status", + "validated_at", + "duration_ms", + "diagnostics", + } + assert payload["schema_version"] == CAPABILITY_SCHEMA_VERSION + assert set(payload["model"]) == {"model_id", "revision", "config_fingerprint"} + assert set(payload["shard"]) == {"start", "end"} + assert set(payload["recipe"]) == { + "recipe_id", + "recipe_version", + "catalogue_version", + } + assert set(payload["backend"]) == { + "backend_id", + "device", + "device_name", + "quantization", + "runtime", + } + # JSON-serializable end to end. + assert json.loads(json.dumps(payload)) == payload + + +def test_identity_key_pins_model_shard_recipe_and_backend(): + base = _report() + + assert base.identity_key() == ( + FIXTURE_MODEL_A, + 0, + 7, + "baseline", + "1", + "torch-transformers", + "test-device", + ) + assert _report(shard_end=8).identity_key() != base.identity_key() + assert _report(recipe_version="2").identity_key() != base.identity_key() + assert _report(device="other-device").identity_key() != base.identity_key() + + +def test_config_fingerprint_is_stable_under_key_order_and_detects_change(): + a = config_fingerprint({"num_hidden_layers": 8, "hidden_size": 512}) + b = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 8}) + c = config_fingerprint({"hidden_size": 512, "num_hidden_layers": 9}) + + assert a == b + assert a != c + assert a.startswith("sha256:") + assert config_fingerprint(None) is None + assert config_fingerprint("sha256:deadbeef") == "sha256:deadbeef" + + +def test_status_and_passed_flag(): + assert _report(status="passed").passed + assert not _report(status="failed").passed + assert not _report(status="skipped").passed + + +# --- malformed report input ------------------------------------------------- + + +@pytest.mark.parametrize( + "overrides, expected", + [ + ({"model_id": ""}, "model.model_id"), + ({"shard_start": -1}, "shard.start"), + ({"shard_start": 5, "shard_end": 2}, "shard.end"), + ({"recipe_id": ""}, "recipe.recipe_id"), + ({"catalogue_version": ""}, "recipe.catalogue_version"), + ({"backend_id": ""}, "backend.backend_id"), + ({"device": ""}, "backend.device"), + ({"status": "maybe"}, "status"), + ({"duration_ms": -5}, "duration_ms"), + ], +) +def test_malformed_report_fields_name_the_offending_field(overrides, expected): + with pytest.raises(CapabilityReportError) as exc: + _report(**overrides) + assert expected in str(exc.value) + + +def test_unsupported_report_schema_version_is_actionable(): + payload = _report().to_dict() + payload["schema_version"] = 99 + + with pytest.raises(CapabilityReportError) as exc: + CapabilityReport.from_dict(payload) + message = str(exc.value) + assert "99" in message + assert str(CAPABILITY_SCHEMA_VERSION) in message + + +def test_missing_schema_version_is_rejected(): + payload = _report().to_dict() + del payload["schema_version"] + + with pytest.raises(CapabilityReportError, match="schema_version"): + CapabilityReport.from_dict(payload) + + +def test_malformed_report_json_reports_position_not_content(): + with pytest.raises(CapabilityReportError) as exc: + CapabilityReport.from_json('{"schema_version": 1,') + assert "line 1" in str(exc.value) + + +def test_missing_report_section_is_named(): + payload = _report().to_dict() + payload["backend"] = "torch" + + with pytest.raises(CapabilityReportError, match="backend"): + CapabilityReport.from_dict(payload) + + +# --- diagnostics sanitization ----------------------------------------------- + + +def test_diagnostics_redact_secret_env_values(monkeypatch): + monkeypatch.setenv("MESHNET_API_TOKEN", "super-secret-value-123") + monkeypatch.setenv("MESHNET_MODEL_ID", "acme-labs/Widget-9000-Instruct") + + report = _report( + status="failed", + diagnostics=["auth failed for super-secret-value-123 while fetching config"], + ) + text = report.to_json() + + assert "super-secret-value-123" not in text + assert capability.REDACTED in report.diagnostics[0] + # A non-secret env var is left alone — the model id stays readable. + assert "acme-labs/Widget-9000-Instruct" in text + + +@pytest.mark.parametrize( + "raw", + [ + "401 Unauthorized: hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", + "request used Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig", + "config had api_key=abcdef0123456789", + "openai style sk-abcdefghijklmnopqrstuvwxyz012345", + ], +) +def test_diagnostics_redact_credential_shaped_strings(raw): + cleaned = sanitize_diagnostic(raw, environ={}) + assert capability.REDACTED in cleaned + for secret in ( + "hf_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", + "eyJhbGciOiJIUzI1NiJ9.payload.sig", + "abcdef0123456789", + "sk-abcdefghijklmnopqrstuvwxyz012345", + ): + assert secret not in cleaned + + +def test_diagnostics_strip_the_home_directory(monkeypatch, tmp_path): + monkeypatch.setenv("HOME", str(tmp_path)) + cleaned = sanitize_diagnostic(f"missing weights at {tmp_path}/models/shard", environ={}) + + assert str(tmp_path) not in cleaned + assert cleaned.startswith("missing weights at ~/models/shard") + + +def test_diagnostics_are_bounded_in_length_and_count(): + long_line = sanitize_diagnostic("x" * 2000, environ={}) + assert len(long_line) <= capability.MAX_DIAGNOSTIC_CHARS + + many = sanitize_diagnostics([f"line {i}" for i in range(50)], environ={}) + assert len(many) == capability.MAX_DIAGNOSTICS + 1 + assert "further diagnostic(s) omitted" in many[-1] + + +def test_diagnostics_reject_non_string_entries(): + with pytest.raises(CapabilityReportError, match=r"diagnostics\[1\]"): + sanitize_diagnostics(["ok", 42], environ={}) + + with pytest.raises(CapabilityReportError, match="bare string"): + sanitize_diagnostics("one big string", environ={}) + + +def test_deserializing_a_report_re_sanitizes_diagnostics(monkeypatch): + monkeypatch.setenv("NODE_SECRET", "leak-me-please") + payload = _report().to_dict() + payload["diagnostics"] = ["backend said leak-me-please"] + + restored = CapabilityReport.from_dict(payload) + assert "leak-me-please" not in restored.to_json() + + +# --- recipe manifest -------------------------------------------------------- + + +def test_packaged_manifest_loads_with_explicit_versions(): + manifest = load_recipe_manifest() + + assert manifest.schema_version == RECIPE_SCHEMA_VERSION + assert manifest.catalogue_version + assert recipe_manifest.DEFAULT_RECIPE_ID in manifest.ids + for recipe in manifest.recipes: + assert recipe.id and recipe.version and recipe.backend_id + assert isinstance(recipe.params, dict) + + +def test_packaged_manifest_feeds_a_report(): + manifest = load_recipe_manifest() + recipe = manifest.require(recipe_manifest.DEFAULT_RECIPE_ID) + + report = _report( + recipe_id=recipe.id, + recipe_version=recipe.version, + catalogue_version=manifest.catalogue_version, + backend_id=recipe.backend_id, + ) + assert report.recipe.catalogue_version == manifest.catalogue_version + + +def test_unknown_recipe_lists_available_ids(): + manifest = load_recipe_manifest() + + with pytest.raises(RecipeManifestError) as exc: + manifest.require("no-such-recipe") + message = str(exc.value) + assert "no-such-recipe" in message + assert recipe_manifest.DEFAULT_RECIPE_ID in message + + +def _write_manifest(tmp_path: Path, doc) -> Path: + path = tmp_path / "recipes.json" + path.write_text( + doc if isinstance(doc, str) else json.dumps(doc), encoding="utf-8" + ) + return path + + +def test_valid_local_manifest_loads(tmp_path): + path = _write_manifest( + tmp_path, + { + "schema_version": RECIPE_SCHEMA_VERSION, + "catalogue_version": "2099.01.0-test", + "recipes": [ + { + "id": "custom", + "version": "3", + "backend_id": "some-backend", + "params": {"knob": 1}, + } + ], + }, + ) + manifest = load_recipe_manifest(path) + + assert manifest.catalogue_version == "2099.01.0-test" + assert manifest.require("custom").params == {"knob": 1} + assert manifest.source == str(path) + + +def test_unknown_manifest_schema_version_is_actionable(tmp_path): + path = _write_manifest( + tmp_path, + { + "schema_version": 99, + "catalogue_version": "x", + "recipes": [{"id": "a", "version": "1", "backend_id": "b"}], + }, + ) + with pytest.raises(RecipeManifestError) as exc: + load_recipe_manifest(path) + message = str(exc.value) + assert "99" in message + assert str(RECIPE_SCHEMA_VERSION) in message + assert str(path) in message + + +@pytest.mark.parametrize( + "doc, expected", + [ + ({"catalogue_version": "x", "recipes": []}, "schema_version"), + ( + {"schema_version": RECIPE_SCHEMA_VERSION, "recipes": [{"id": "a"}]}, + "catalogue_version", + ), + ( + {"schema_version": RECIPE_SCHEMA_VERSION, "catalogue_version": "x", "recipes": []}, + "non-empty JSON array", + ), + ( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "catalogue_version": "x", + "recipes": [{"version": "1", "backend_id": "b"}], + }, + "recipes[0].id", + ), + ( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "catalogue_version": "x", + "recipes": [{"id": "a", "backend_id": "b"}], + }, + "recipes[a].version", + ), + ( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "catalogue_version": "x", + "recipes": [{"id": "a", "version": "1"}], + }, + "recipes[a].backend_id", + ), + ( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "catalogue_version": "x", + "recipes": [ + {"id": "a", "version": "1", "backend_id": "b", "params": [1, 2]} + ], + }, + "recipes[a].params", + ), + ( + { + "schema_version": RECIPE_SCHEMA_VERSION, + "catalogue_version": "x", + "recipes": [ + {"id": "a", "version": "1", "backend_id": "b"}, + {"id": "a", "version": "2", "backend_id": "b"}, + ], + }, + "duplicate recipe id", + ), + ], +) +def test_malformed_manifest_names_the_offending_field(tmp_path, doc, expected): + path = _write_manifest(tmp_path, doc) + + with pytest.raises(RecipeManifestError) as exc: + load_recipe_manifest(path) + assert expected in str(exc.value) + + +def test_malformed_manifest_json_reports_position_not_content(tmp_path): + path = _write_manifest(tmp_path, '{"schema_version": 1, "catalogue_version":') + + with pytest.raises(RecipeManifestError) as exc: + load_recipe_manifest(path) + message = str(exc.value) + assert "not valid JSON" in message + assert "line 1" in message + # The failing document body is never echoed back. + assert "catalogue_version\":" not in message + + +def test_missing_manifest_file_is_actionable(tmp_path): + with pytest.raises(RecipeManifestError, match="cannot read recipe manifest"): + load_recipe_manifest(tmp_path / "nope.json") + + +def test_manifest_round_trips_through_its_own_dict(): + manifest = load_recipe_manifest() + reparsed = parse_recipe_manifest(manifest.to_dict(), source="") + + assert reparsed.to_dict() == manifest.to_dict() diff --git a/tests/test_tracker_test_runner.py b/tests/test_tracker_test_runner.py index 5198b0e..db19c62 100644 --- a/tests/test_tracker_test_runner.py +++ b/tests/test_tracker_test_runner.py @@ -34,6 +34,9 @@ def _make_repo(tmp_path: Path) -> Path: "import time\n\ndef test_sleeps():\n time.sleep(30)\n" ) (tests / "test_smoke.py").write_text("def test_smoke():\n assert True\n") + (tests / "test_cache.py").write_text( + 'def test_cache_roundtrip():\n """Cache roundtrip remains deterministic."""\n assert True\n' + ) # Real-inference stand-in: must never be collected or run by default. (tests / "test_real_stub.py").write_text( "def test_real_inference():\n raise AssertionError('env-gated')\n" @@ -139,6 +142,12 @@ def test_collection_lists_tests_and_excludes_real_inference(tmp_path, monkeypatc assert data["enabled"] is True assert "tests/test_quick.py::test_passes" in data["tests"] assert "tests/test_slow.py::test_sleeps" in data["tests"] + metadata = {item["id"]: item for item in data["test_metadata"]} + assert metadata["tests/test_cache.py::test_cache_roundtrip"]["description"] == ( + "Cache roundtrip remains deterministic." + ) + assert "cache" in metadata["tests/test_cache.py::test_cache_roundtrip"]["tags"] + assert any(tag["id"] == "tag:cache" for tag in data["tags"]) assert not any("test_real_" in node_id for node_id in data["tests"]) assert data["collected_at"] is not None suite_ids = [suite["id"] for suite in data["suites"]] @@ -190,6 +199,31 @@ def test_run_approved_suite_without_prior_collection(tmp_path): tracker.stop() +def test_run_all_and_tag_targets(tmp_path): + tracker, port, admin, _user = _start_tracker(tmp_path) + try: + assert _request(port, "GET", "/v1/tests", token=admin)[0] == 200 + runner = tracker._test_runner + assert runner is not None + all_state = runner.start("suite:all") + assert all_state["run"]["target"] == "suite:all" + assert len(all_state["run"]["args"]) == len(runner._collected) + finally: + tracker.stop() + + second_repo = tmp_path / "second" + second_repo.mkdir() + tracker, _port, _admin, _user = _start_tracker(second_repo) + try: + runner = tracker._test_runner + assert runner is not None + runner.collect() + tagged = runner.start("tag:cache") + assert tagged["run"]["args"] == ["tests/test_cache.py::test_cache_roundtrip"] + finally: + tracker.stop() + + @pytest.mark.parametrize("target", [ "", "-x",