fix model load/unload
This commit is contained in:
@@ -431,6 +431,7 @@ def _start_heartbeat(
|
||||
which is logged for now (hot-reload implemented in US-026).
|
||||
"""
|
||||
_start_time = start_time or time.monotonic()
|
||||
completed_directives: list[dict] = []
|
||||
|
||||
def _current_requests_snapshot() -> list[dict]:
|
||||
if node_ref is None:
|
||||
@@ -455,6 +456,8 @@ def _start_heartbeat(
|
||||
current_requests = _current_requests_snapshot()
|
||||
if current_requests:
|
||||
stats["current_requests"] = current_requests
|
||||
if completed_directives:
|
||||
stats["completed_directives"] = list(completed_directives)
|
||||
return stats
|
||||
|
||||
def _sleep_interval() -> float:
|
||||
@@ -512,21 +515,26 @@ def _start_heartbeat(
|
||||
except Exception as exc:
|
||||
print(f" [node] WARNING: additional model registration failed: {exc}", flush=True)
|
||||
|
||||
def _apply_directives(directives: list[dict]) -> None:
|
||||
def _apply_directives(directives: list[dict]) -> dict | None:
|
||||
if not directives:
|
||||
return
|
||||
return None
|
||||
if node_ref is None or not hasattr(node_ref, "apply_tracker_directives"):
|
||||
print(f" [node] tracker directives received: {directives}", flush=True)
|
||||
return
|
||||
return None
|
||||
try:
|
||||
applied = node_ref.apply_tracker_directives(directives)
|
||||
except Exception as exc:
|
||||
print(f" [node] WARNING: failed to apply tracker directives: {exc}", flush=True)
|
||||
return
|
||||
return None
|
||||
if applied:
|
||||
completed_directives.append(dict(applied))
|
||||
if applied.get("action") == "ADD_SHARD":
|
||||
_register_additional_assignment(applied)
|
||||
return
|
||||
return applied
|
||||
if applied.get("action") == "DROP_SHARD":
|
||||
# A release has no replacement range. It is not a failed
|
||||
# heartbeat and must not re-register the released assignment.
|
||||
return applied
|
||||
model_id = applied.get("model", register_payload.get("hf_repo") or register_payload.get("model"))
|
||||
register_payload["model"] = str(model_id).split("/")[-1]
|
||||
register_payload["hf_repo"] = model_id
|
||||
@@ -534,6 +542,7 @@ def _start_heartbeat(
|
||||
register_payload["shard_end"] = applied["shard_end"]
|
||||
register_payload["quantization"] = applied.get("quantization", register_payload.get("quantization"))
|
||||
register_payload["tracker_mode"] = bool(applied.get("tracker_mode", False))
|
||||
return applied
|
||||
|
||||
def _loop() -> None:
|
||||
nonlocal node_id
|
||||
@@ -561,7 +570,10 @@ def _start_heartbeat(
|
||||
continue
|
||||
|
||||
try:
|
||||
resp = _post_json(hb_url, _get_stats())
|
||||
heartbeat = _get_stats()
|
||||
resp = _post_json(hb_url, heartbeat)
|
||||
if heartbeat.get("completed_directives"):
|
||||
completed_directives.clear()
|
||||
_apply_directives(resp.get("directives", []))
|
||||
new_asgn = resp.get("new_assignment")
|
||||
if new_asgn:
|
||||
|
||||
@@ -3276,7 +3276,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
node.hf_repo or node.model
|
||||
for node in alive
|
||||
if node.model is not None
|
||||
and node.model not in server.model_presets
|
||||
# The same model can be registered under its HF repository while
|
||||
# the catalogue exposes its short preset id. Do not emit a second
|
||||
# repo-keyed entry when either node identifier resolves to a preset.
|
||||
and _resolve_model_preset(
|
||||
server.model_presets, node.hf_repo or node.model,
|
||||
)[1] is None
|
||||
and node.shard_start is not None
|
||||
and node.shard_end is not None
|
||||
and node.num_layers is not None
|
||||
@@ -4820,6 +4825,20 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
entry.uptime_seconds = float(body["uptime_seconds"])
|
||||
if "status" in body and body["status"] in ("ready", "loading"):
|
||||
entry.status = body["status"]
|
||||
completed_directives = body.get("completed_directives", [])
|
||||
if isinstance(completed_directives, list):
|
||||
for directive in completed_directives:
|
||||
if not isinstance(directive, dict) or directive.get("action") != "DROP_SHARD":
|
||||
continue
|
||||
# A node has confirmed the release. Stop advertising its
|
||||
# old route immediately so the dashboard and routing state
|
||||
# agree with the runtime.
|
||||
entry.model = "stub-model"
|
||||
entry.hf_repo = None
|
||||
entry.shard_start = None
|
||||
entry.shard_end = None
|
||||
entry.tracker_mode = False
|
||||
entry.status = "ready"
|
||||
if "friendly_name" in body:
|
||||
try:
|
||||
entry.friendly_name = _normalize_friendly_name(body.get("friendly_name"))
|
||||
|
||||
Reference in New Issue
Block a user