loading more

This commit is contained in:
Dobromir Popov
2026-07-15 12:54:51 +02:00
parent 22f28bd69a
commit 2852b1f80b
4 changed files with 41 additions and 11 deletions

View File

@@ -1535,7 +1535,9 @@ def _request_model_load_locked(
return None
def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
def _force_model_load_locked(
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
) -> dict | None:
"""Replace the fastest ready assignment after an explicit admin eviction."""
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
if preset is None or not preset.get("hf_repo"):
@@ -1544,8 +1546,11 @@ def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> di
# An explicit admin eviction is permitted to recover a stuck/loading node
# and to use the preset default precision. It must only avoid a node that
# already has another assignment in flight.
candidates = [node for node in server.registry.values()
if node.pending_new_assignment is None]
candidates = [
node for node in server.registry.values()
if node.pending_new_assignment is None
and (node_id is None or node.node_id == node_id)
]
if not candidates:
return None
node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec)
@@ -4917,8 +4922,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
with server.lock:
self._purge_expired_nodes()
assignment = _request_model_load_locked(server, model, node_id)
if assignment is None and node_id is None and body.get("force") is True:
assignment = _force_model_load_locked(server, model)
if assignment is None and body.get("force") is True:
assignment = _force_model_load_locked(server, model, node_id)
if assignment is None:
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
return