feat: harden node placement and partial model loading

This commit is contained in:
Dobromir Popov
2026-07-13 21:58:08 +02:00
parent a6bcc69288
commit 5d87e81bc9
21 changed files with 497 additions and 55 deletions

View File

@@ -899,12 +899,41 @@ def _load_partial_model_from_snapshot(
dtype=dtype,
)
for module in _active_modules_for_shard(model, shard_start, shard_end):
if hasattr(module, "to"):
module.to(device)
_finalize_active_shard_modules_on_device(model, shard_start, shard_end, device)
return model
def _finalize_active_shard_modules_on_device(
model: Any, shard_start: int, shard_end: int, device: Any
) -> None:
"""Place active shard modules on device without copying unmaterialized meta weights."""
for module in _active_modules_for_shard(model, shard_start, shard_end):
parameters = getattr(module, "parameters", None)
if not callable(parameters):
if hasattr(module, "to"):
module.to(device)
continue
params = list(parameters(recurse=True))
buffers_fn = getattr(module, "buffers", None)
buffers = list(buffers_fn(recurse=True)) if callable(buffers_fn) else []
tensors = params + buffers
if not tensors:
if hasattr(module, "to"):
module.to(device)
continue
if all(tensor.device.type == "meta" for tensor in tensors):
to_empty = getattr(module, "to_empty", None)
if callable(to_empty):
to_empty(device)
continue
if all(tensor.device.type != "meta" for tensor in tensors):
if hasattr(module, "to"):
module.to(device)
continue
# Partially materialized: set_module_tensor_to_device already placed loaded
# weights on the target device; leave remaining meta parameters untouched.
def _model_load_plan(
auto_config: Any,
model_id: str,