loading more
This commit is contained in:
@@ -1535,7 +1535,9 @@ def _request_model_load_locked(
|
||||
return None
|
||||
|
||||
|
||||
def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
|
||||
def _force_model_load_locked(
|
||||
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
|
||||
) -> dict | None:
|
||||
"""Replace the fastest ready assignment after an explicit admin eviction."""
|
||||
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
|
||||
if preset is None or not preset.get("hf_repo"):
|
||||
@@ -1544,8 +1546,11 @@ def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> di
|
||||
# An explicit admin eviction is permitted to recover a stuck/loading node
|
||||
# and to use the preset default precision. It must only avoid a node that
|
||||
# already has another assignment in flight.
|
||||
candidates = [node for node in server.registry.values()
|
||||
if node.pending_new_assignment is None]
|
||||
candidates = [
|
||||
node for node in server.registry.values()
|
||||
if node.pending_new_assignment is None
|
||||
and (node_id is None or node.node_id == node_id)
|
||||
]
|
||||
if not candidates:
|
||||
return None
|
||||
node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec)
|
||||
@@ -4917,8 +4922,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
with server.lock:
|
||||
self._purge_expired_nodes()
|
||||
assignment = _request_model_load_locked(server, model, node_id)
|
||||
if assignment is None and node_id is None and body.get("force") is True:
|
||||
assignment = _force_model_load_locked(server, model)
|
||||
if assignment is None and body.get("force") is True:
|
||||
assignment = _force_model_load_locked(server, model, node_id)
|
||||
if assignment is None:
|
||||
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user