feat: harden node placement and partial model loading
This commit is contained in:
@@ -899,12 +899,41 @@ def _load_partial_model_from_snapshot(
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
for module in _active_modules_for_shard(model, shard_start, shard_end):
|
||||
if hasattr(module, "to"):
|
||||
module.to(device)
|
||||
_finalize_active_shard_modules_on_device(model, shard_start, shard_end, device)
|
||||
return model
|
||||
|
||||
|
||||
def _finalize_active_shard_modules_on_device(
|
||||
model: Any, shard_start: int, shard_end: int, device: Any
|
||||
) -> None:
|
||||
"""Place active shard modules on device without copying unmaterialized meta weights."""
|
||||
for module in _active_modules_for_shard(model, shard_start, shard_end):
|
||||
parameters = getattr(module, "parameters", None)
|
||||
if not callable(parameters):
|
||||
if hasattr(module, "to"):
|
||||
module.to(device)
|
||||
continue
|
||||
params = list(parameters(recurse=True))
|
||||
buffers_fn = getattr(module, "buffers", None)
|
||||
buffers = list(buffers_fn(recurse=True)) if callable(buffers_fn) else []
|
||||
tensors = params + buffers
|
||||
if not tensors:
|
||||
if hasattr(module, "to"):
|
||||
module.to(device)
|
||||
continue
|
||||
if all(tensor.device.type == "meta" for tensor in tensors):
|
||||
to_empty = getattr(module, "to_empty", None)
|
||||
if callable(to_empty):
|
||||
to_empty(device)
|
||||
continue
|
||||
if all(tensor.device.type != "meta" for tensor in tensors):
|
||||
if hasattr(module, "to"):
|
||||
module.to(device)
|
||||
continue
|
||||
# Partially materialized: set_module_tensor_to_device already placed loaded
|
||||
# weights on the target device; leave remaining meta parameters untouched.
|
||||
|
||||
|
||||
def _model_load_plan(
|
||||
auto_config: Any,
|
||||
model_id: str,
|
||||
|
||||
Reference in New Issue
Block a user