story: DGR-043 Expose GGUF compatibility and measured cost inputs to existing routing
This commit is contained in:
@@ -365,6 +365,62 @@ class ExecutionCapacity:
|
||||
return cls(**values)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoutingMeasurements:
|
||||
"""Optional backend-neutral observations for existing tracker routing.
|
||||
|
||||
These are measurements, rather than policy: the tracker continues to own
|
||||
admission, route formation, load balancing, and certification. Keeping
|
||||
this block optional makes it additive for existing Transformers reports.
|
||||
"""
|
||||
|
||||
tokens_per_second: float | None = None
|
||||
queue_depth: int | None = None
|
||||
seam_latency_ms: float | None = None
|
||||
healthy: bool | None = None
|
||||
reliability: float | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in ("tokens_per_second", "seam_latency_ms"):
|
||||
value = getattr(self, name)
|
||||
if value is not None and (
|
||||
isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0
|
||||
):
|
||||
raise CapabilityReportError(f"routing.{name} must be a non-negative number")
|
||||
if self.tokens_per_second == 0:
|
||||
raise CapabilityReportError("routing.tokens_per_second must be positive when present")
|
||||
if self.queue_depth is not None:
|
||||
_require_int(self.queue_depth, "routing.queue_depth", 0)
|
||||
if self.healthy is not None and not isinstance(self.healthy, bool):
|
||||
raise CapabilityReportError("routing.healthy must be a boolean")
|
||||
if self.reliability is not None and (
|
||||
isinstance(self.reliability, bool)
|
||||
or not isinstance(self.reliability, (int, float))
|
||||
or not 0.0 <= self.reliability <= 1.0
|
||||
):
|
||||
raise CapabilityReportError("routing.reliability must be a number from 0 to 1")
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"tokens_per_second": self.tokens_per_second,
|
||||
"queue_depth": self.queue_depth,
|
||||
"seam_latency_ms": self.seam_latency_ms,
|
||||
"healthy": self.healthy,
|
||||
"reliability": self.reliability,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Any) -> RoutingMeasurements:
|
||||
doc = _as_mapping(data, "routing")
|
||||
return cls(
|
||||
tokens_per_second=doc.get("tokens_per_second"),
|
||||
queue_depth=doc.get("queue_depth"),
|
||||
seam_latency_ms=doc.get("seam_latency_ms"),
|
||||
healthy=doc.get("healthy"),
|
||||
reliability=doc.get("reliability"),
|
||||
)
|
||||
|
||||
|
||||
def _as_mapping(data: Any, field_name: str) -> Mapping[str, Any]:
|
||||
if not isinstance(data, Mapping):
|
||||
raise CapabilityReportError(
|
||||
@@ -397,6 +453,7 @@ class CapabilityReport:
|
||||
schema_version: int = CAPABILITY_SCHEMA_VERSION
|
||||
identity: ShardIdentity | None = None
|
||||
capacity: ExecutionCapacity | None = None
|
||||
routing: RoutingMeasurements | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.status not in VALID_STATUSES:
|
||||
@@ -456,6 +513,8 @@ class CapabilityReport:
|
||||
doc["identity"] = self.identity.to_dict()
|
||||
if self.capacity is not None:
|
||||
doc["capacity"] = self.capacity.to_dict()
|
||||
if self.routing is not None:
|
||||
doc["routing"] = self.routing.to_dict()
|
||||
return doc
|
||||
|
||||
def to_json(self, indent: int | None = None) -> str:
|
||||
@@ -500,6 +559,9 @@ class CapabilityReport:
|
||||
capacity=(
|
||||
None if doc.get("capacity") is None else ExecutionCapacity.from_dict(doc["capacity"])
|
||||
),
|
||||
routing=(
|
||||
None if doc.get("routing") is None else RoutingMeasurements.from_dict(doc["routing"])
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -536,6 +598,7 @@ def build_capability_report(
|
||||
environ: Mapping[str, str] | None = None,
|
||||
identity: ShardIdentity | None = None,
|
||||
capacity: ExecutionCapacity | None = None,
|
||||
routing: RoutingMeasurements | None = None,
|
||||
) -> CapabilityReport:
|
||||
"""Assemble a report from flat validation results.
|
||||
|
||||
@@ -569,4 +632,5 @@ def build_capability_report(
|
||||
diagnostics=sanitize_diagnostics(diagnostics, environ),
|
||||
identity=identity,
|
||||
capacity=capacity,
|
||||
routing=routing,
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from .capability import ExecutionCapacity, build_capability_report
|
||||
from .capability import ExecutionCapacity, RoutingMeasurements, build_capability_report
|
||||
from .native_worker_supervisor import NativeWorkerProbe, NativeWorkerSpec, NativeWorkerSupervisor
|
||||
from .runtime_recipe import ShardIdentity
|
||||
|
||||
@@ -33,6 +33,7 @@ class NativeShardRegistration:
|
||||
device: str
|
||||
capacity: ExecutionCapacity
|
||||
duration_ms: int = 0
|
||||
routing: RoutingMeasurements | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.endpoint:
|
||||
@@ -104,8 +105,9 @@ class NativeShardRegistration:
|
||||
duration_ms=self.duration_ms,
|
||||
identity=self.identity,
|
||||
capacity=self.capacity,
|
||||
routing=self.routing,
|
||||
)
|
||||
return {
|
||||
payload = {
|
||||
"endpoint": self.endpoint,
|
||||
"model": self.model_id.rsplit("/", 1)[-1],
|
||||
"hf_repo": self.model_id,
|
||||
@@ -118,6 +120,15 @@ class NativeShardRegistration:
|
||||
"ram_bytes": self.capacity.memory_capacity_bytes or 0,
|
||||
"max_loaded_shards": 1,
|
||||
}
|
||||
# These are the tracker’s established dynamic scoring inputs. The
|
||||
# exact same optional report can be sent by any backend; no native
|
||||
# route or balancing branch is introduced here.
|
||||
if self.routing is not None:
|
||||
if self.routing.tokens_per_second is not None:
|
||||
payload["benchmark_tokens_per_sec"] = self.routing.tokens_per_second
|
||||
if self.routing.queue_depth is not None:
|
||||
payload["queue_depth"] = self.routing.queue_depth
|
||||
return payload
|
||||
|
||||
|
||||
RegistrationSender = Callable[[dict[str, Any]], None]
|
||||
|
||||
@@ -193,6 +193,11 @@ class CapabilityState:
|
||||
memory_capacity_bytes: int | None = None
|
||||
kv_capacity_tokens: int | None = None
|
||||
max_concurrent_sessions: int | None = None
|
||||
measured_tokens_per_second: float | None = None
|
||||
reported_queue_depth: int | None = None
|
||||
seam_latency_ms: float | None = None
|
||||
healthy: bool | None = None
|
||||
reliability: float | None = None
|
||||
|
||||
@property
|
||||
def proven(self) -> bool:
|
||||
@@ -239,6 +244,11 @@ class CapabilityState:
|
||||
"memory_capacity_bytes": self.memory_capacity_bytes,
|
||||
"kv_capacity_tokens": self.kv_capacity_tokens,
|
||||
"max_concurrent_sessions": self.max_concurrent_sessions,
|
||||
"measured_tokens_per_second": self.measured_tokens_per_second,
|
||||
"reported_queue_depth": self.reported_queue_depth,
|
||||
"seam_latency_ms": self.seam_latency_ms,
|
||||
"healthy": self.healthy,
|
||||
"reliability": self.reliability,
|
||||
}
|
||||
|
||||
|
||||
@@ -500,6 +510,9 @@ def _parse_report(doc: Mapping[str, Any]) -> dict:
|
||||
capacity = doc.get("capacity")
|
||||
if capacity is not None:
|
||||
capacity = _object(capacity, "capacity")
|
||||
routing = doc.get("routing")
|
||||
if routing is not None:
|
||||
routing = _object(routing, "routing")
|
||||
|
||||
return {
|
||||
"model_id": _text(model.get("model_id"), "model.model_id"),
|
||||
@@ -530,6 +543,24 @@ def _parse_report(doc: Mapping[str, Any]) -> dict:
|
||||
None if capacity is None else capacity.get("max_concurrent_sessions"),
|
||||
"capacity.max_concurrent_sessions",
|
||||
),
|
||||
"measured_tokens_per_second": _optional_positive_float(
|
||||
None if routing is None else routing.get("tokens_per_second"),
|
||||
"routing.tokens_per_second",
|
||||
),
|
||||
"reported_queue_depth": _optional_nonnegative_int(
|
||||
None if routing is None else routing.get("queue_depth"),
|
||||
"routing.queue_depth",
|
||||
),
|
||||
"seam_latency_ms": _optional_nonnegative_float(
|
||||
None if routing is None else routing.get("seam_latency_ms"),
|
||||
"routing.seam_latency_ms",
|
||||
),
|
||||
"healthy": _optional_bool(
|
||||
None if routing is None else routing.get("healthy"), "routing.healthy"
|
||||
),
|
||||
"reliability": _optional_unit_float(
|
||||
None if routing is None else routing.get("reliability"), "routing.reliability"
|
||||
),
|
||||
"_status": _text(doc.get("status"), "status"),
|
||||
}
|
||||
|
||||
@@ -566,6 +597,45 @@ def _optional_positive_int(value: Any, field_name: str) -> int | None:
|
||||
return value
|
||||
|
||||
|
||||
def _optional_nonnegative_int(value: Any, field_name: str) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
raise _ReportError(f"{field_name!r} must be a non-negative integer")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_positive_float(value: Any, field_name: str) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
|
||||
raise _ReportError(f"{field_name!r} must be a positive number")
|
||||
return float(value)
|
||||
|
||||
|
||||
def _optional_nonnegative_float(value: Any, field_name: str) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
|
||||
raise _ReportError(f"{field_name!r} must be a non-negative number")
|
||||
return float(value)
|
||||
|
||||
|
||||
def _optional_bool(value: Any, field_name: str) -> bool | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, bool):
|
||||
raise _ReportError(f"{field_name!r} must be a boolean")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_unit_float(value: Any, field_name: str) -> float | None:
|
||||
parsed = _optional_nonnegative_float(value, field_name)
|
||||
if parsed is not None and parsed > 1:
|
||||
raise _ReportError(f"{field_name!r} must be a number from 0 to 1")
|
||||
return parsed
|
||||
|
||||
|
||||
def _maybe_int(value: Any) -> int | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
return None
|
||||
|
||||
@@ -4684,6 +4684,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
friendly_name=friendly_name,
|
||||
capability=capability,
|
||||
)
|
||||
# A report may seed the same load/throughput inputs that legacy nodes
|
||||
# supply through registration and heartbeats. The optional block is
|
||||
# backend-neutral; routing still applies its usual queue adjustment.
|
||||
if capability.reported_queue_depth is not None:
|
||||
entry.queue_depth = capability.reported_queue_depth
|
||||
with server.lock:
|
||||
self._purge_expired_nodes()
|
||||
# Dedup: replace the same node id or the same endpoint+model assignment.
|
||||
|
||||
Reference in New Issue
Block a user