qol improvements

This commit is contained in:
Dobromir Popov
2026-06-30 19:27:46 +02:00
parent d8a723a4c7
commit f1e4870124
4 changed files with 317 additions and 7 deletions

View File

@@ -56,6 +56,27 @@ DEFAULT_QUANTIZATIONS = ["bfloat16"]
DEFAULT_BENCHMARK_TOKENS_PER_SEC = 1.0
def _model_aliases(model: str | None) -> set[str]:
"""Return stable lookup aliases for a model repo or display name."""
if not model:
return set()
aliases = {model}
short = model.rsplit("/", 1)[-1]
aliases.add(short)
lowered = short.lower()
aliases.add(lowered)
if lowered.endswith("-instruct"):
aliases.add(lowered.removesuffix("-instruct"))
return aliases
def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
requested = _model_aliases(model)
if not requested:
return False
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
class _RollingCounter:
"""Circular-bucket request counter.
@@ -549,7 +570,7 @@ def _nodes_and_bounds_for_model(
nodes = [
node for node in server.registry.values()
if (node.hf_repo == model or node.model == model)
if _node_matches_model(node, model)
and node.shard_start is not None
and node.shard_end is not None
and node.num_layers is not None
@@ -1028,7 +1049,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._purge_expired_nodes()
candidates = [
n for n in server.registry.values()
if n.tracker_mode and (n.model == model or n.hf_repo == model)
if n.tracker_mode and _node_matches_model(n, model)
]
if not candidates:
@@ -1036,7 +1057,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
with server.lock:
candidates = [
n for n in server.registry.values()
if n.shard_start == 0 and (n.model == model or n.hf_repo == model)
if n.shard_start == 0 and _node_matches_model(n, model)
]
if not candidates:
@@ -1064,7 +1085,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
else:
all_nodes = [
n for n in server.registry.values()
if (n.hf_repo == route_model or n.model == route_model)
if _node_matches_model(n, route_model)
and n.shard_start is not None and n.num_layers is not None
]
rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
@@ -1745,7 +1766,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
# HF model routing: match by hf_repo (full) or model short name.
alive = [
node for node in server.registry.values()
if (node.hf_repo == model or node.model == model)
if _node_matches_model(node, model)
and node.shard_start is not None
and node.shard_end is not None
and node.num_layers is not None