feat: DGR-017 - Lock the GLM-5.2 Max target and alpha contract
This commit is contained in:
Dobromir Popov
522
packages/node/meshnet_node/glm_alpha/planner.py
Normal file
522
packages/node/meshnet_node/glm_alpha/planner.py
Normal file
@@ -0,0 +1,522 @@
|
||||
"""Deterministic memory, KV, and network planner for the GLM-5.2 Max alpha route.
|
||||
|
||||
Everything here is arithmetic over the exact pinned artifact bytes and the exact
|
||||
pinned architecture. There is no measurement, no probing, and no heuristic tuned
|
||||
to a result — the planner is written *before* the target runs so that a later
|
||||
story cannot discover a topology that "works" and then rationalise it.
|
||||
|
||||
Three ideas do the real work.
|
||||
|
||||
**Unified memory is one pool.** On an integrated-GPU machine the "VRAM" the driver
|
||||
reports is carved out of the same physical DRAM the OS is already counting. Adding
|
||||
them produces a node that appears to hold twice what it holds, and the failure mode
|
||||
is not a clean admission rejection — it is an OOM or a swap-thrash halfway through
|
||||
a 200 GiB load. :class:`NodeMemory` therefore refuses to be constructed from an
|
||||
additive claim about one shared pool.
|
||||
|
||||
**The reserve is not optional headroom.** Weights plus KV are not the whole
|
||||
resident cost: backend workspaces, quantization scratch, the graph plan, the
|
||||
process, and the OS all live outside them, and the largest of those scale with the
|
||||
backend rather than with the shard. Alpha reserves ``max(20% of physically usable
|
||||
memory, 8 GiB)`` per node, and the *remainder* is the placement budget.
|
||||
|
||||
**Equal layer counts are not equal bytes.** Embeddings and the output head are
|
||||
endpoint-only; three layers are dense and 75 are MoE; shared experts, indexer
|
||||
tensors, and quant block alignment all skew the per-node share. Until DGR-018/019
|
||||
report measured per-tensor placement, the planner carries an explicit
|
||||
:data:`PLACEMENT_IMBALANCE_FACTOR` and reports the arithmetic minimum and the
|
||||
recommended count as two separate numbers. The arithmetic minimum is a fit probe;
|
||||
it is admissible only with exact measured placement evidence behind it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
from .manifest import GIB, ArchitectureSnapshot, TargetManifest
|
||||
|
||||
# Q8_0 stores 32 int8 quants plus one fp16 scale per block: 34 bytes / 32 values.
|
||||
Q8_0_BYTES_PER_VALUE = 34 / 32
|
||||
F16_BYTES_PER_VALUE = 2.0
|
||||
|
||||
KV_DTYPES: dict[str, float] = {
|
||||
"Q8_0": Q8_0_BYTES_PER_VALUE,
|
||||
"F16": F16_BYTES_PER_VALUE,
|
||||
}
|
||||
|
||||
# The alpha KV configuration, locked by the roadmap.
|
||||
ALPHA_KV_DTYPE = "Q8_0"
|
||||
ALPHA_CONTEXT_TOKENS = 16384
|
||||
ALPHA_CONCURRENCY = 1
|
||||
|
||||
# The reserve every node holds outside its weight-plus-KV placement budget.
|
||||
RESERVE_FRACTION = 0.20
|
||||
RESERVE_FLOOR_GIB = 8.0
|
||||
|
||||
# The aggregate runtime-accessible memory at which the artifact *just* fits.
|
||||
# This is an experimental hard-fit floor, not an operational envelope: it has no
|
||||
# room for a backend that allocates more scratch than another, and none for the
|
||||
# imbalance below.
|
||||
AGGREGATE_HARD_FIT_FLOOR_GIB = 224.0
|
||||
|
||||
# How much more than an equal share the worst-placed node is expected to hold.
|
||||
# 1.10 is the roadmap's recommended-topology column expressed as arithmetic: it
|
||||
# reproduces 10 / 6 / 5 / 3 / 3 nodes for the 32 / 48 / 64 / 96 / 128 GiB tiers.
|
||||
# DGR-019 must replace it with measured per-tensor placement.
|
||||
PLACEMENT_IMBALANCE_FACTOR = 1.10
|
||||
|
||||
# Alpha network floor. A link rate is a bandwidth claim, never a speed claim.
|
||||
MIN_LINK_RATE_GBPS = 2.5
|
||||
RECOMMENDED_LINK_RATE_GBPS = 10.0
|
||||
|
||||
BF16_BYTES = 2
|
||||
DSA_SIDEBAND_INT32_BYTES = 4
|
||||
|
||||
IndexerLayout = Literal["optimized", "conservative"]
|
||||
|
||||
|
||||
class ResourcePlanError(ValueError):
|
||||
"""Raised when a node or route cannot be accounted for honestly."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NodeMemory:
|
||||
"""One node's physically usable memory, counted once.
|
||||
|
||||
``physical_usable_gib`` is what the node can actually place bytes into after
|
||||
firmware and fixed carve-outs — not the marketing capacity, and not a sum of
|
||||
two views of the same DRAM.
|
||||
"""
|
||||
|
||||
name: str
|
||||
physical_usable_gib: float
|
||||
unified: bool
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.name, str) or not self.name.strip():
|
||||
raise ResourcePlanError("node name must be a non-empty physical-host identity")
|
||||
if (
|
||||
isinstance(self.physical_usable_gib, bool)
|
||||
or not isinstance(self.physical_usable_gib, (int, float))
|
||||
or not math.isfinite(self.physical_usable_gib)
|
||||
or self.physical_usable_gib <= 0
|
||||
):
|
||||
raise ResourcePlanError(
|
||||
f"node {self.name!r} must declare finite positive usable memory"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_host(
|
||||
cls,
|
||||
name: str,
|
||||
*,
|
||||
system_ram_gib: float,
|
||||
gpu_memory_gib: float = 0.0,
|
||||
unified: bool,
|
||||
) -> "NodeMemory":
|
||||
"""Build a node from a host's reported RAM and GPU memory.
|
||||
|
||||
On a unified machine the GPU memory *is* system RAM, so it is counted once
|
||||
and never added. Passing a non-zero ``gpu_memory_gib`` alongside
|
||||
``unified=True`` is the double-count this project has already decided is a
|
||||
bug (RALPH-CONTEXT runtime decision 16), so it is rejected rather than
|
||||
silently discarded: a caller who believes an integrated GPU adds memory has
|
||||
a wrong model of the machine, and quietly ignoring the argument would let
|
||||
that belief survive.
|
||||
"""
|
||||
if not isinstance(unified, bool):
|
||||
raise ResourcePlanError(f"node {name!r} unified flag must be boolean")
|
||||
for value, label, allow_zero in (
|
||||
(system_ram_gib, "system RAM", False),
|
||||
(gpu_memory_gib, "GPU memory", True),
|
||||
):
|
||||
if (
|
||||
isinstance(value, bool)
|
||||
or not isinstance(value, (int, float))
|
||||
or not math.isfinite(value)
|
||||
or value < 0
|
||||
or (not allow_zero and value == 0)
|
||||
):
|
||||
qualifier = "finite non-negative" if allow_zero else "finite positive"
|
||||
raise ResourcePlanError(f"node {name!r} must declare {qualifier} {label}")
|
||||
if unified:
|
||||
if gpu_memory_gib:
|
||||
raise ResourcePlanError(
|
||||
f"node {name!r} declares unified memory and {gpu_memory_gib} GiB of "
|
||||
"separate GPU memory. Integrated-GPU memory is carved out of the same "
|
||||
"physical DRAM as system RAM; adding them double-counts one pool. "
|
||||
"Pass unified=True with system_ram_gib only."
|
||||
)
|
||||
usable = system_ram_gib
|
||||
else:
|
||||
usable = system_ram_gib + gpu_memory_gib
|
||||
return cls(name=name, physical_usable_gib=usable, unified=unified)
|
||||
|
||||
@property
|
||||
def reserve_gib(self) -> float:
|
||||
"""``max(20% of physically usable memory, 8 GiB)``."""
|
||||
return max(RESERVE_FRACTION * self.physical_usable_gib, RESERVE_FLOOR_GIB)
|
||||
|
||||
@property
|
||||
def placement_budget_gib(self) -> float:
|
||||
"""What remains for weights plus KV after the reserve."""
|
||||
return self.physical_usable_gib - self.reserve_gib
|
||||
|
||||
|
||||
def kv_bytes(
|
||||
snapshot: ArchitectureSnapshot,
|
||||
*,
|
||||
context_tokens: int = ALPHA_CONTEXT_TOKENS,
|
||||
concurrency: int = ALPHA_CONCURRENCY,
|
||||
dtype: str = ALPHA_KV_DTYPE,
|
||||
indexer_layout: IndexerLayout = "conservative",
|
||||
include_indexer: bool = True,
|
||||
) -> int:
|
||||
"""Bytes of MLA (and DSA indexer) KV cache for the whole model.
|
||||
|
||||
``indexer_layout`` is the honest part. Correct DSA only needs indexer keys for
|
||||
the Full producer layers, but the current experimental implementation may
|
||||
allocate them across every backbone layer. Alpha budgets ``conservative``
|
||||
(all 78) so that a route admitted by this planner cannot be surprised by the
|
||||
implementation it actually gets.
|
||||
"""
|
||||
if (
|
||||
not isinstance(context_tokens, int)
|
||||
or isinstance(context_tokens, bool)
|
||||
or context_tokens <= 0
|
||||
or not isinstance(concurrency, int)
|
||||
or isinstance(concurrency, bool)
|
||||
or concurrency <= 0
|
||||
):
|
||||
raise ResourcePlanError("context_tokens and concurrency must be positive integers")
|
||||
if dtype not in KV_DTYPES:
|
||||
raise ResourcePlanError(
|
||||
f"unsupported KV dtype {dtype!r}; alpha locks {ALPHA_KV_DTYPE} "
|
||||
f"(known: {', '.join(sorted(KV_DTYPES))})"
|
||||
)
|
||||
bytes_per_value = KV_DTYPES[dtype]
|
||||
|
||||
layers = int(snapshot["num_hidden_layers"])
|
||||
mla_values = int(snapshot["mla_cached_values_per_token_per_layer"])
|
||||
total_values = mla_values * layers
|
||||
|
||||
if include_indexer:
|
||||
if indexer_layout == "optimized":
|
||||
indexer_layers = int(snapshot["indexer_full_layers"])
|
||||
elif indexer_layout == "conservative":
|
||||
indexer_layers = layers
|
||||
else: # pragma: no cover - Literal keeps this unreachable from typed callers
|
||||
raise ResourcePlanError(f"unknown indexer_layout {indexer_layout!r}")
|
||||
total_values += int(snapshot["index_head_dim"]) * indexer_layers
|
||||
|
||||
return int(total_values * context_tokens * concurrency * bytes_per_value)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TopologyPlan:
|
||||
"""The node count a homogeneous tier needs, and how it was reached."""
|
||||
|
||||
physical_usable_gib: float
|
||||
reserve_gib: float
|
||||
placement_budget_gib: float
|
||||
weight_gib: float
|
||||
kv_gib: float
|
||||
total_placement_gib: float
|
||||
arithmetic_minimum_nodes: int
|
||||
recommended_nodes: int
|
||||
imbalance_factor: float
|
||||
|
||||
@property
|
||||
def is_arithmetic_minimum_topology(self) -> bool:
|
||||
"""True when the recommendation offers no imbalance headroom at all."""
|
||||
return self.recommended_nodes == self.arithmetic_minimum_nodes
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"physical_usable_gib": round(self.physical_usable_gib, 3),
|
||||
"reserve_gib": round(self.reserve_gib, 3),
|
||||
"placement_budget_gib": round(self.placement_budget_gib, 3),
|
||||
"weight_gib": round(self.weight_gib, 3),
|
||||
"kv_gib": round(self.kv_gib, 3),
|
||||
"total_placement_gib": round(self.total_placement_gib, 3),
|
||||
"arithmetic_minimum_nodes": self.arithmetic_minimum_nodes,
|
||||
"recommended_nodes": self.recommended_nodes,
|
||||
"imbalance_factor": self.imbalance_factor,
|
||||
}
|
||||
|
||||
|
||||
def plan_topology(
|
||||
manifest: TargetManifest,
|
||||
snapshot: ArchitectureSnapshot,
|
||||
*,
|
||||
physical_usable_gib: float,
|
||||
context_tokens: int = ALPHA_CONTEXT_TOKENS,
|
||||
concurrency: int = ALPHA_CONCURRENCY,
|
||||
kv_dtype: str = ALPHA_KV_DTYPE,
|
||||
indexer_layout: IndexerLayout = "conservative",
|
||||
imbalance_factor: float = PLACEMENT_IMBALANCE_FACTOR,
|
||||
) -> TopologyPlan:
|
||||
"""Minimum and recommended node count for a homogeneous tier of this size."""
|
||||
if (
|
||||
isinstance(imbalance_factor, bool)
|
||||
or not isinstance(imbalance_factor, (int, float))
|
||||
or not math.isfinite(imbalance_factor)
|
||||
or imbalance_factor < 1.0
|
||||
):
|
||||
raise ResourcePlanError(
|
||||
"imbalance_factor must be finite and at least 1.0; a lower value would "
|
||||
"assume the worst-placed node holds less than an equal share"
|
||||
)
|
||||
|
||||
node = NodeMemory(
|
||||
name=f"{physical_usable_gib:g}GiB-tier",
|
||||
physical_usable_gib=physical_usable_gib,
|
||||
unified=False,
|
||||
)
|
||||
budget = node.placement_budget_gib
|
||||
if budget <= 0:
|
||||
raise ResourcePlanError(
|
||||
f"a {physical_usable_gib:g} GiB node has no placement budget after its "
|
||||
f"{node.reserve_gib:.1f} GiB reserve"
|
||||
)
|
||||
|
||||
weight_gib = manifest.total_bytes / GIB
|
||||
kv_gib = (
|
||||
kv_bytes(
|
||||
snapshot,
|
||||
context_tokens=context_tokens,
|
||||
concurrency=concurrency,
|
||||
dtype=kv_dtype,
|
||||
indexer_layout=indexer_layout,
|
||||
)
|
||||
/ GIB
|
||||
)
|
||||
total = weight_gib + kv_gib
|
||||
|
||||
return TopologyPlan(
|
||||
physical_usable_gib=physical_usable_gib,
|
||||
reserve_gib=node.reserve_gib,
|
||||
placement_budget_gib=budget,
|
||||
weight_gib=weight_gib,
|
||||
kv_gib=kv_gib,
|
||||
total_placement_gib=total,
|
||||
arithmetic_minimum_nodes=math.ceil(total / budget),
|
||||
recommended_nodes=math.ceil(total * imbalance_factor / budget),
|
||||
imbalance_factor=imbalance_factor,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RouteFit:
|
||||
"""Whether a concrete, possibly heterogeneous set of nodes can hold the target."""
|
||||
|
||||
node_count: int
|
||||
aggregate_usable_gib: float
|
||||
aggregate_placement_budget_gib: float
|
||||
required_placement_gib: float
|
||||
fits: bool
|
||||
meets_hard_fit_floor: bool
|
||||
no_single_node_can_admit_target: bool
|
||||
headroom_gib: float
|
||||
reasons: tuple[str, ...]
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"node_count": self.node_count,
|
||||
"aggregate_usable_gib": round(self.aggregate_usable_gib, 3),
|
||||
"aggregate_placement_budget_gib": round(self.aggregate_placement_budget_gib, 3),
|
||||
"required_placement_gib": round(self.required_placement_gib, 3),
|
||||
"fits": self.fits,
|
||||
"meets_hard_fit_floor": self.meets_hard_fit_floor,
|
||||
"no_single_node_can_admit_target": self.no_single_node_can_admit_target,
|
||||
"headroom_gib": round(self.headroom_gib, 3),
|
||||
"reasons": list(self.reasons),
|
||||
}
|
||||
|
||||
|
||||
def plan_route(
|
||||
manifest: TargetManifest,
|
||||
snapshot: ArchitectureSnapshot,
|
||||
nodes: list[NodeMemory],
|
||||
*,
|
||||
context_tokens: int = ALPHA_CONTEXT_TOKENS,
|
||||
concurrency: int = ALPHA_CONCURRENCY,
|
||||
kv_dtype: str = ALPHA_KV_DTYPE,
|
||||
indexer_layout: IndexerLayout = "conservative",
|
||||
) -> RouteFit:
|
||||
"""Evaluate a concrete route. Every node's memory is already counted once."""
|
||||
if len(nodes) < 2:
|
||||
raise ResourcePlanError(
|
||||
"the alpha target is distributed by definition; a route needs at least two "
|
||||
"physical nodes"
|
||||
)
|
||||
names = [node.name for node in nodes]
|
||||
if len(set(names)) != len(names):
|
||||
raise ResourcePlanError(
|
||||
"duplicate node names in the route; one physical machine counted twice is "
|
||||
"the same double-count as adding integrated-GPU memory to system RAM"
|
||||
)
|
||||
|
||||
weight_gib = manifest.total_bytes / GIB
|
||||
kv_gib = (
|
||||
kv_bytes(
|
||||
snapshot,
|
||||
context_tokens=context_tokens,
|
||||
concurrency=concurrency,
|
||||
dtype=kv_dtype,
|
||||
indexer_layout=indexer_layout,
|
||||
)
|
||||
/ GIB
|
||||
)
|
||||
required = weight_gib + kv_gib
|
||||
|
||||
aggregate_usable = sum(node.physical_usable_gib for node in nodes)
|
||||
aggregate_budget = sum(node.placement_budget_gib for node in nodes)
|
||||
fits = aggregate_budget >= required
|
||||
largest_budget = max(node.placement_budget_gib for node in nodes)
|
||||
no_single_node = largest_budget < required
|
||||
|
||||
reasons: list[str] = []
|
||||
if not fits:
|
||||
reasons.append(
|
||||
f"aggregate placement budget {aggregate_budget:.1f} GiB is below the "
|
||||
f"{required:.1f} GiB the target needs after each node's reserve"
|
||||
)
|
||||
if not no_single_node:
|
||||
reasons.append(
|
||||
"at least one node could admit the complete target alone; that is a "
|
||||
"single-host run, not distributed alpha"
|
||||
)
|
||||
if aggregate_usable < AGGREGATE_HARD_FIT_FLOOR_GIB:
|
||||
reasons.append(
|
||||
f"aggregate usable memory {aggregate_usable:.1f} GiB is below the "
|
||||
f"{AGGREGATE_HARD_FIT_FLOOR_GIB:g} GiB experimental hard-fit floor"
|
||||
)
|
||||
|
||||
return RouteFit(
|
||||
node_count=len(nodes),
|
||||
aggregate_usable_gib=aggregate_usable,
|
||||
aggregate_placement_budget_gib=aggregate_budget,
|
||||
required_placement_gib=required,
|
||||
fits=fits,
|
||||
meets_hard_fit_floor=aggregate_usable >= AGGREGATE_HARD_FIT_FLOOR_GIB,
|
||||
no_single_node_can_admit_target=no_single_node,
|
||||
headroom_gib=aggregate_budget - required,
|
||||
reasons=tuple(reasons),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SeamPlan:
|
||||
"""Bytes and latency across the activation seams of a route.
|
||||
|
||||
Bandwidth and latency are reported separately on purpose. Decode moves almost
|
||||
nothing — 12 KiB per token per seam — so a faster link barely helps it. What
|
||||
decode pays is *serial*: every generated token crosses every seam in order, so
|
||||
the cost that matters is ``seams x per-hop latency``. A route that claims to be
|
||||
fast because it is on 10 GbE has confused the two.
|
||||
"""
|
||||
|
||||
node_count: int
|
||||
seam_count: int
|
||||
hidden_size: int
|
||||
bytes_per_token_per_seam: int
|
||||
prefill_bytes_per_seam: int
|
||||
decode_bytes_per_seam_per_token: int
|
||||
dsa_sideband_bytes_per_query: int
|
||||
link_rate_gbps: float
|
||||
meets_alpha_minimum: bool
|
||||
is_recommended_link: bool
|
||||
decode_serialization_ms_per_token: float
|
||||
decode_latency_ms_per_token: float
|
||||
decode_bandwidth_share_ms_per_token: float
|
||||
prefill_serialization_ms: float
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"node_count": self.node_count,
|
||||
"seam_count": self.seam_count,
|
||||
"hidden_size": self.hidden_size,
|
||||
"bytes_per_token_per_seam": self.bytes_per_token_per_seam,
|
||||
"prefill_bytes_per_seam": self.prefill_bytes_per_seam,
|
||||
"decode_bytes_per_seam_per_token": self.decode_bytes_per_seam_per_token,
|
||||
"dsa_sideband_bytes_per_query": self.dsa_sideband_bytes_per_query,
|
||||
"link_rate_gbps": self.link_rate_gbps,
|
||||
"meets_alpha_minimum": self.meets_alpha_minimum,
|
||||
"is_recommended_link": self.is_recommended_link,
|
||||
"decode_serialization_ms_per_token": round(self.decode_serialization_ms_per_token, 4),
|
||||
"decode_latency_ms_per_token": round(self.decode_latency_ms_per_token, 4),
|
||||
"decode_bandwidth_share_ms_per_token": round(
|
||||
self.decode_bandwidth_share_ms_per_token, 4
|
||||
),
|
||||
"prefill_serialization_ms": round(self.prefill_serialization_ms, 3),
|
||||
}
|
||||
|
||||
|
||||
def plan_seams(
|
||||
snapshot: ArchitectureSnapshot,
|
||||
*,
|
||||
node_count: int,
|
||||
context_tokens: int = ALPHA_CONTEXT_TOKENS,
|
||||
link_rate_gbps: float = MIN_LINK_RATE_GBPS,
|
||||
per_hop_latency_ms: float = 0.5,
|
||||
) -> SeamPlan:
|
||||
"""Model seam bytes, wire serialization, and serial per-hop latency separately."""
|
||||
if not isinstance(node_count, int) or isinstance(node_count, bool) or node_count < 2:
|
||||
raise ResourcePlanError("a seam exists only between two nodes")
|
||||
if not isinstance(context_tokens, int) or isinstance(context_tokens, bool) or context_tokens <= 0:
|
||||
raise ResourcePlanError("context_tokens must be a positive integer")
|
||||
if (
|
||||
isinstance(link_rate_gbps, bool)
|
||||
or not isinstance(link_rate_gbps, (int, float))
|
||||
or not math.isfinite(link_rate_gbps)
|
||||
or link_rate_gbps <= 0
|
||||
):
|
||||
raise ResourcePlanError("link_rate_gbps must be finite and positive")
|
||||
if (
|
||||
isinstance(per_hop_latency_ms, bool)
|
||||
or not isinstance(per_hop_latency_ms, (int, float))
|
||||
or not math.isfinite(per_hop_latency_ms)
|
||||
or per_hop_latency_ms < 0
|
||||
):
|
||||
raise ResourcePlanError("per_hop_latency_ms must be finite and non-negative")
|
||||
|
||||
hidden = int(snapshot["hidden_size"])
|
||||
bytes_per_token = hidden * BF16_BYTES
|
||||
seams = node_count - 1
|
||||
|
||||
bits_per_ms = link_rate_gbps * 1e9 / 1e3
|
||||
decode_serialization_ms = (bytes_per_token * 8) / bits_per_ms
|
||||
prefill_serialization_ms = (bytes_per_token * context_tokens * 8) / bits_per_ms
|
||||
|
||||
return SeamPlan(
|
||||
node_count=node_count,
|
||||
seam_count=seams,
|
||||
hidden_size=hidden,
|
||||
bytes_per_token_per_seam=bytes_per_token,
|
||||
prefill_bytes_per_seam=bytes_per_token * context_tokens,
|
||||
decode_bytes_per_seam_per_token=bytes_per_token,
|
||||
dsa_sideband_bytes_per_query=int(snapshot["index_topk"]) * DSA_SIDEBAND_INT32_BYTES,
|
||||
link_rate_gbps=link_rate_gbps,
|
||||
meets_alpha_minimum=link_rate_gbps >= MIN_LINK_RATE_GBPS,
|
||||
is_recommended_link=link_rate_gbps >= RECOMMENDED_LINK_RATE_GBPS,
|
||||
decode_serialization_ms_per_token=decode_serialization_ms * seams,
|
||||
decode_latency_ms_per_token=per_hop_latency_ms * seams,
|
||||
decode_bandwidth_share_ms_per_token=decode_serialization_ms * seams,
|
||||
prefill_serialization_ms=prefill_serialization_ms * seams,
|
||||
)
|
||||
|
||||
|
||||
ALPHA_TIERS_GIB: tuple[float, ...] = (32.0, 48.0, 64.0, 96.0, 128.0)
|
||||
|
||||
|
||||
def plan_all_tiers(
|
||||
manifest: TargetManifest, snapshot: ArchitectureSnapshot
|
||||
) -> dict[str, TopologyPlan]:
|
||||
"""The alpha tier table, recomputed from the pinned artifact and architecture."""
|
||||
return {
|
||||
f"{tier:g}": plan_topology(manifest, snapshot, physical_usable_gib=tier)
|
||||
for tier in ALPHA_TIERS_GIB
|
||||
}
|
||||
Reference in New Issue
Block a user