ram pool map

This commit is contained in:
Dobromir Popov
2026-07-07 18:35:36 +03:00
parent 1299a6bb1c
commit e9a094b620
6 changed files with 253 additions and 54 deletions

View File

@@ -944,6 +944,20 @@ def _pool_summary(nodes: list[_NodeEntry], preset: dict | None = None) -> dict:
}
def _repo_demand_rpm(
server: "_TrackerHTTPServer",
repo: str,
local_rpms: dict[str, dict],
) -> float:
"""Look up recent request rate by hf_repo or preset alias."""
if repo in local_rpms:
return float(local_rpms[repo].get("rpm_last_hour", 0.0) or 0.0)
resolved_name, _preset = _resolve_model_preset(server.model_presets, repo)
if resolved_name and resolved_name in local_rpms:
return float(local_rpms[resolved_name].get("rpm_last_hour", 0.0) or 0.0)
return 0.0
def _preset_for_node(server: "_TrackerHTTPServer", node: _NodeEntry) -> dict | None:
if node.hf_repo:
_resolved, preset = _resolve_model_preset(server.model_presets, node.hf_repo)
@@ -1392,20 +1406,6 @@ def _assign_redundant_managed_nodes(
_emit_shard_change_directives(node, model, previous_range, preset)
def _repo_demand_rpm(
server: "_TrackerHTTPServer",
repo: str,
local_rpms: dict[str, dict],
) -> float:
"""Look up recent request rate by hf_repo or preset alias."""
if repo in local_rpms:
return float(local_rpms[repo].get("rpm_last_hour", 0.0) or 0.0)
resolved_name, _preset = _resolve_model_preset(server.model_presets, repo)
if resolved_name and resolved_name in local_rpms:
return float(local_rpms[resolved_name].get("rpm_last_hour", 0.0) or 0.0)
return 0.0
def _relay_http_request_frames(
relay_addr: str,
path: str,
@@ -2587,6 +2587,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
with server.lock:
self._purge_expired_nodes()
nodes = list(server.registry.values())
memory_pool = _memory_pool_map(server)
def capacity_for(node: _NodeEntry) -> dict:
preset = None
@@ -2611,6 +2612,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(200, {
"relay_url": server.relay_url,
"pool": _pool_summary(nodes),
"memory_pool": memory_pool,
"recommended_models": [
{
"id": name,
@@ -3538,10 +3540,16 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
)
with server.lock:
self._purge_expired_nodes()
# Dedup: if this endpoint is already registered, remove the old entry first.
# Dedup: replace the same node id or the same endpoint+model assignment.
endpoint_key = entry.endpoint.rstrip("/")
model_key = entry.hf_repo or entry.model
stale_ids = [
eid for eid, e in server.registry.items()
if e.endpoint == entry.endpoint.rstrip("/")
if eid == node_id
or (
e.endpoint.rstrip("/") == endpoint_key
and (e.hf_repo or e.model) == model_key
)
]
stale_entries: list[tuple[str, _NodeEntry]] = []
for eid in stale_ids:
@@ -5436,6 +5444,7 @@ class TrackerServer:
with self._lock:
_purge_expired_nodes_locked(server)
_rebalance_all_locked(server)
_scale_demanded_models_locked(server)
def _push_to_peers(self, path: str, body: bytes) -> bool:
"""POST a hive-signed payload to every cluster peer; True if all succeeded."""