qol improvements
This commit is contained in:
@@ -56,6 +56,27 @@ DEFAULT_QUANTIZATIONS = ["bfloat16"]
|
||||
DEFAULT_BENCHMARK_TOKENS_PER_SEC = 1.0
|
||||
|
||||
|
||||
def _model_aliases(model: str | None) -> set[str]:
|
||||
"""Return stable lookup aliases for a model repo or display name."""
|
||||
if not model:
|
||||
return set()
|
||||
aliases = {model}
|
||||
short = model.rsplit("/", 1)[-1]
|
||||
aliases.add(short)
|
||||
lowered = short.lower()
|
||||
aliases.add(lowered)
|
||||
if lowered.endswith("-instruct"):
|
||||
aliases.add(lowered.removesuffix("-instruct"))
|
||||
return aliases
|
||||
|
||||
|
||||
def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
|
||||
requested = _model_aliases(model)
|
||||
if not requested:
|
||||
return False
|
||||
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
|
||||
|
||||
|
||||
class _RollingCounter:
|
||||
"""Circular-bucket request counter.
|
||||
|
||||
@@ -549,7 +570,7 @@ def _nodes_and_bounds_for_model(
|
||||
|
||||
nodes = [
|
||||
node for node in server.registry.values()
|
||||
if (node.hf_repo == model or node.model == model)
|
||||
if _node_matches_model(node, model)
|
||||
and node.shard_start is not None
|
||||
and node.shard_end is not None
|
||||
and node.num_layers is not None
|
||||
@@ -1028,7 +1049,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
self._purge_expired_nodes()
|
||||
candidates = [
|
||||
n for n in server.registry.values()
|
||||
if n.tracker_mode and (n.model == model or n.hf_repo == model)
|
||||
if n.tracker_mode and _node_matches_model(n, model)
|
||||
]
|
||||
|
||||
if not candidates:
|
||||
@@ -1036,7 +1057,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
with server.lock:
|
||||
candidates = [
|
||||
n for n in server.registry.values()
|
||||
if n.shard_start == 0 and (n.model == model or n.hf_repo == model)
|
||||
if n.shard_start == 0 and _node_matches_model(n, model)
|
||||
]
|
||||
|
||||
if not candidates:
|
||||
@@ -1064,7 +1085,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
else:
|
||||
all_nodes = [
|
||||
n for n in server.registry.values()
|
||||
if (n.hf_repo == route_model or n.model == route_model)
|
||||
if _node_matches_model(n, route_model)
|
||||
and n.shard_start is not None and n.num_layers is not None
|
||||
]
|
||||
rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
|
||||
@@ -1745,7 +1766,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
# HF model routing: match by hf_repo (full) or model short name.
|
||||
alive = [
|
||||
node for node in server.registry.values()
|
||||
if (node.hf_repo == model or node.model == model)
|
||||
if _node_matches_model(node, model)
|
||||
and node.shard_start is not None
|
||||
and node.shard_end is not None
|
||||
and node.num_layers is not None
|
||||
|
||||
Reference in New Issue
Block a user