story: DGR-043 Expose GGUF compatibility and measured cost inputs to existing routing
This commit is contained in:
@@ -12,7 +12,7 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from .capability import ExecutionCapacity, build_capability_report
|
||||
from .capability import ExecutionCapacity, RoutingMeasurements, build_capability_report
|
||||
from .native_worker_supervisor import NativeWorkerProbe, NativeWorkerSpec, NativeWorkerSupervisor
|
||||
from .runtime_recipe import ShardIdentity
|
||||
|
||||
@@ -33,6 +33,7 @@ class NativeShardRegistration:
|
||||
device: str
|
||||
capacity: ExecutionCapacity
|
||||
duration_ms: int = 0
|
||||
routing: RoutingMeasurements | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.endpoint:
|
||||
@@ -104,8 +105,9 @@ class NativeShardRegistration:
|
||||
duration_ms=self.duration_ms,
|
||||
identity=self.identity,
|
||||
capacity=self.capacity,
|
||||
routing=self.routing,
|
||||
)
|
||||
return {
|
||||
payload = {
|
||||
"endpoint": self.endpoint,
|
||||
"model": self.model_id.rsplit("/", 1)[-1],
|
||||
"hf_repo": self.model_id,
|
||||
@@ -118,6 +120,15 @@ class NativeShardRegistration:
|
||||
"ram_bytes": self.capacity.memory_capacity_bytes or 0,
|
||||
"max_loaded_shards": 1,
|
||||
}
|
||||
# These are the tracker’s established dynamic scoring inputs. The
|
||||
# exact same optional report can be sent by any backend; no native
|
||||
# route or balancing branch is introduced here.
|
||||
if self.routing is not None:
|
||||
if self.routing.tokens_per_second is not None:
|
||||
payload["benchmark_tokens_per_sec"] = self.routing.tokens_per_second
|
||||
if self.routing.queue_depth is not None:
|
||||
payload["queue_depth"] = self.routing.queue_depth
|
||||
return payload
|
||||
|
||||
|
||||
RegistrationSender = Callable[[dict[str, Any]], None]
|
||||
|
||||
Reference in New Issue
Block a user