loading more
This commit is contained in:
@@ -332,6 +332,7 @@
|
|||||||
<div id="model-placement-dialog-title"></div>
|
<div id="model-placement-dialog-title"></div>
|
||||||
<label for="model-placement-node">Node</label>
|
<label for="model-placement-node">Node</label>
|
||||||
<select id="model-placement-node"></select>
|
<select id="model-placement-node"></select>
|
||||||
|
<label id="model-placement-replace" style="display:none"><input type="checkbox" id="model-placement-replace-confirm"> Unload the currently loaded model before loading this one</label>
|
||||||
<div class="placement-dialog-actions"><button value="cancel">Cancel</button><button value="confirm">Confirm</button></div>
|
<div class="placement-dialog-actions"><button value="cancel">Cancel</button><button value="confirm">Confirm</button></div>
|
||||||
</form>
|
</form>
|
||||||
</dialog>
|
</dialog>
|
||||||
@@ -1812,8 +1813,8 @@ async function requestSelectedModelLoad() {
|
|||||||
$("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
|
$("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function requestAdminModelLoad(model, nodeId) {
|
async function requestAdminModelLoad(model, nodeId, replacing) {
|
||||||
const result = await apiCall("/v1/models/load", "POST", { model, node_id: nodeId, force: true });
|
const result = await apiCall("/v1/models/load", "POST", { model, node_id: nodeId, force: replacing });
|
||||||
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model load request failed", true);
|
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model load request failed", true);
|
||||||
const assignment = result.data.assignment || {};
|
const assignment = result.data.assignment || {};
|
||||||
showAdminModelPlacementStatus(`Load queued on ${short(assignment.node_id || "node")} for ${model}.`);
|
showAdminModelPlacementStatus(`Load queued on ${short(assignment.node_id || "node")} for ${model}.`);
|
||||||
@@ -1887,14 +1888,31 @@ $("admin-model-placement").addEventListener("click", event => {
|
|||||||
function chooseModelPlacementNode(action, model) {
|
function chooseModelPlacementNode(action, model) {
|
||||||
const dialog = $("model-placement-dialog");
|
const dialog = $("model-placement-dialog");
|
||||||
const select = $("model-placement-node");
|
const select = $("model-placement-node");
|
||||||
const aliases = new Set([model].filter(Boolean));
|
const targetAlias = modelAliasKey(model);
|
||||||
const nodes = (lastNetworkMap?.nodes || []).filter(node => action === "load" || aliases.has(node.model) || aliases.has(node.hf_repo));
|
const nodes = (lastNetworkMap?.nodes || []).filter(node => action === "load" ||
|
||||||
|
modelAliasKey(node.model) === targetAlias || modelAliasKey(node.hf_repo) === targetAlias);
|
||||||
if (!nodes.length) return showAdminModelPlacementStatus(`No node can ${action} ${model}.`, true);
|
if (!nodes.length) return showAdminModelPlacementStatus(`No node can ${action} ${model}.`, true);
|
||||||
$("model-placement-dialog-title").textContent = `${action === "load" ? "Load" : "Release"} ${model} on a node`;
|
$("model-placement-dialog-title").textContent = `${action === "load" ? "Load" : "Release"} ${model} on a node`;
|
||||||
select.innerHTML = nodes.map(node => `<option value="${esc(node.node_id)}">${esc(short(node.friendly_name || node.node_id, 20))} — ${esc(node.hf_repo || node.model || "unassigned")}</option>`).join("");
|
select.innerHTML = nodes.map(node => `<option value="${esc(node.node_id)}">${esc(short(node.friendly_name || node.node_id, 20))} — ${esc(node.hf_repo || node.model || "unassigned")}</option>`).join("");
|
||||||
|
const replace = $("model-placement-replace");
|
||||||
|
const replaceConfirm = $("model-placement-replace-confirm");
|
||||||
|
const selectedNode = () => nodes.find(node => node.node_id === select.value);
|
||||||
|
const updateReplacementWarning = () => {
|
||||||
|
const node = selectedNode();
|
||||||
|
const occupied = action === "load" && node && node.shard_start != null && node.shard_end != null &&
|
||||||
|
modelAliasKey(node.hf_repo || node.model) !== targetAlias;
|
||||||
|
replace.style.display = occupied ? "" : "none";
|
||||||
|
replaceConfirm.checked = false;
|
||||||
|
};
|
||||||
|
select.onchange = updateReplacementWarning;
|
||||||
|
updateReplacementWarning();
|
||||||
dialog.onclose = () => {
|
dialog.onclose = () => {
|
||||||
if (dialog.returnValue !== "confirm") return;
|
if (dialog.returnValue !== "confirm") return;
|
||||||
if (action === "load") void requestAdminModelLoad(model, select.value);
|
const replacing = replace.style.display !== "none";
|
||||||
|
if (replacing && !replaceConfirm.checked) {
|
||||||
|
return showAdminModelPlacementStatus("Confirm unloading the existing model before replacing it.", true);
|
||||||
|
}
|
||||||
|
if (action === "load") void requestAdminModelLoad(model, select.value, replacing);
|
||||||
else void releaseAdminModel(model, select.value);
|
else void releaseAdminModel(model, select.value);
|
||||||
};
|
};
|
||||||
dialog.showModal();
|
dialog.showModal();
|
||||||
|
|||||||
@@ -1535,7 +1535,9 @@ def _request_model_load_locked(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
|
def _force_model_load_locked(
|
||||||
|
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
|
||||||
|
) -> dict | None:
|
||||||
"""Replace the fastest ready assignment after an explicit admin eviction."""
|
"""Replace the fastest ready assignment after an explicit admin eviction."""
|
||||||
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
|
||||||
if preset is None or not preset.get("hf_repo"):
|
if preset is None or not preset.get("hf_repo"):
|
||||||
@@ -1544,8 +1546,11 @@ def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> di
|
|||||||
# An explicit admin eviction is permitted to recover a stuck/loading node
|
# An explicit admin eviction is permitted to recover a stuck/loading node
|
||||||
# and to use the preset default precision. It must only avoid a node that
|
# and to use the preset default precision. It must only avoid a node that
|
||||||
# already has another assignment in flight.
|
# already has another assignment in flight.
|
||||||
candidates = [node for node in server.registry.values()
|
candidates = [
|
||||||
if node.pending_new_assignment is None]
|
node for node in server.registry.values()
|
||||||
|
if node.pending_new_assignment is None
|
||||||
|
and (node_id is None or node.node_id == node_id)
|
||||||
|
]
|
||||||
if not candidates:
|
if not candidates:
|
||||||
return None
|
return None
|
||||||
node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec)
|
node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec)
|
||||||
@@ -4917,8 +4922,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
with server.lock:
|
with server.lock:
|
||||||
self._purge_expired_nodes()
|
self._purge_expired_nodes()
|
||||||
assignment = _request_model_load_locked(server, model, node_id)
|
assignment = _request_model_load_locked(server, model, node_id)
|
||||||
if assignment is None and node_id is None and body.get("force") is True:
|
if assignment is None and body.get("force") is True:
|
||||||
assignment = _force_model_load_locked(server, model)
|
assignment = _force_model_load_locked(server, model, node_id)
|
||||||
if assignment is None:
|
if assignment is None:
|
||||||
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
|
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -126,6 +126,9 @@ def test_dashboard_exposes_admin_model_inventory_and_release_controls():
|
|||||||
assert 'id="model-placement-dialog"' in html
|
assert 'id="model-placement-dialog"' in html
|
||||||
assert "chooseModelPlacementNode" in html
|
assert "chooseModelPlacementNode" in html
|
||||||
assert "node_id: nodeId" in html
|
assert "node_id: nodeId" in html
|
||||||
|
assert "modelAliasKey(node.model)" in html
|
||||||
|
assert 'id="model-placement-replace"' in html
|
||||||
|
assert "force: replacing" in html
|
||||||
|
|
||||||
|
|
||||||
def test_network_map_includes_node_friendly_name():
|
def test_network_map_includes_node_friendly_name():
|
||||||
|
|||||||
@@ -370,7 +370,11 @@ def test_admin_can_replace_a_served_model_and_release_it():
|
|||||||
headers = {"Content-Type": "application/json", "Authorization": "Bearer test-admin"}
|
headers = {"Content-Type": "application/json", "Authorization": "Bearer test-admin"}
|
||||||
load = urllib.request.Request(
|
load = urllib.request.Request(
|
||||||
f"http://127.0.0.1:{port}/v1/models/load",
|
f"http://127.0.0.1:{port}/v1/models/load",
|
||||||
data=json.dumps({"model": "qwen2.5-0.5b-instruct", "force": True}).encode(),
|
data=json.dumps({
|
||||||
|
"model": "qwen2.5-0.5b-instruct",
|
||||||
|
"node_id": node["node_id"],
|
||||||
|
"force": True,
|
||||||
|
}).encode(),
|
||||||
headers=headers, method="POST")
|
headers=headers, method="POST")
|
||||||
with urllib.request.urlopen(load) as response:
|
with urllib.request.urlopen(load) as response:
|
||||||
assert json.loads(response.read())["assignment"]["node_id"] == node["node_id"]
|
assert json.loads(response.read())["assignment"]["node_id"] == node["node_id"]
|
||||||
|
|||||||
Reference in New Issue
Block a user