Load recommended model metadata from JSON
This commit is contained in:
@@ -78,7 +78,8 @@ def test_tracker_exposes_registered_model_metadata():
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
kimi = next(model for model in models["data"] if model["id"] == "unsloth/Kimi-K2.7-Code")
|
||||
kimi = next(model for model in models["data"] if model["id"] == "kimi-k2.7")
|
||||
assert kimi["hf_repo"] == "unsloth/Kimi-K2.7-Code"
|
||||
assert kimi["metadata"]["total_parameters"] == "1T"
|
||||
assert kimi["metadata"]["activated_parameters"] == "32B"
|
||||
assert kimi["metadata"]["num_layers"] == 61
|
||||
@@ -87,6 +88,113 @@ def test_tracker_exposes_registered_model_metadata():
|
||||
assert registered["model_metadata"]["context_length"] == 256000
|
||||
|
||||
|
||||
def test_tracker_lists_recommended_kimi_before_nodes_register():
|
||||
tracker = TrackerServer()
|
||||
port = tracker.start()
|
||||
url = f"http://127.0.0.1:{port}"
|
||||
try:
|
||||
models = _get_json(f"{url}/v1/models")
|
||||
network_map = _get_json(f"{url}/v1/network/map")
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
kimi = next(model for model in models["data"] if model["id"] == "kimi-k2.7")
|
||||
assert kimi["hf_repo"] == "unsloth/Kimi-K2.7-Code"
|
||||
assert "Kimi-K2.7-Code" in kimi["aliases"]
|
||||
assert kimi["metadata"]["download_size_gb"] == 595
|
||||
assert kimi["deployment"]["native_quantization"] == "int4"
|
||||
assert kimi["deployment"]["deployable"] is False
|
||||
assert network_map["pool"]["node_count"] == 0
|
||||
assert network_map["recommended_models"][0]["id"] == "kimi-k2.7"
|
||||
|
||||
|
||||
def test_network_map_exposes_pool_size_and_speed_summary():
|
||||
tracker = TrackerServer()
|
||||
port = tracker.start()
|
||||
url = f"http://127.0.0.1:{port}"
|
||||
try:
|
||||
_post_json(
|
||||
f"{url}/v1/nodes/register",
|
||||
{
|
||||
"endpoint": "http://127.0.0.1:7101",
|
||||
"model": "inventory-a",
|
||||
"vram_bytes": 10_000,
|
||||
"ram_bytes": 20_000,
|
||||
"benchmark_tokens_per_sec": 4.0,
|
||||
"hardware_profile": {},
|
||||
},
|
||||
)
|
||||
_post_json(
|
||||
f"{url}/v1/nodes/register",
|
||||
{
|
||||
"endpoint": "http://127.0.0.1:7102",
|
||||
"model": "inventory-b",
|
||||
"vram_bytes": 0,
|
||||
"ram_bytes": 30_000,
|
||||
"benchmark_tokens_per_sec": 6.0,
|
||||
"hardware_profile": {},
|
||||
},
|
||||
)
|
||||
|
||||
network_map = _get_json(f"{url}/v1/network/map")
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
pool = network_map["pool"]
|
||||
assert pool["node_count"] == 2
|
||||
assert pool["total_vram_bytes"] == 10_000
|
||||
assert pool["total_ram_bytes"] == 50_000
|
||||
assert pool["total_memory_budget_bytes"] == 40_000
|
||||
assert pool["total_benchmark_tokens_per_sec"] == 10.0
|
||||
assert pool["total_effective_throughput"] == 10.0
|
||||
|
||||
|
||||
def test_recommended_kimi_becomes_deployable_when_pool_is_large_enough():
|
||||
tracker = TrackerServer()
|
||||
port = tracker.start()
|
||||
url = f"http://127.0.0.1:{port}"
|
||||
gib = 1024 * 1024 * 1024
|
||||
try:
|
||||
for idx in range(2):
|
||||
_post_json(
|
||||
f"{url}/v1/nodes/register",
|
||||
{
|
||||
"endpoint": f"http://127.0.0.1:{7200 + idx}",
|
||||
"model": f"inventory-{idx}",
|
||||
"vram_bytes": 0,
|
||||
"ram_bytes": 400 * gib,
|
||||
"benchmark_tokens_per_sec": 5.0,
|
||||
"hardware_profile": {},
|
||||
},
|
||||
)
|
||||
|
||||
models = _get_json(f"{url}/v1/models")
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
kimi = next(model for model in models["data"] if model["id"] == "kimi-k2.7")
|
||||
assert kimi["deployment"]["deployable"] is True
|
||||
assert kimi["deployment"]["missing_effective_memory_bytes"] == 0
|
||||
|
||||
|
||||
def test_network_assign_can_start_recommended_kimi_by_short_name():
|
||||
tracker = TrackerServer()
|
||||
port = tracker.start()
|
||||
url = f"http://127.0.0.1:{port}"
|
||||
try:
|
||||
assignment = _get_json(
|
||||
f"{url}/v1/network/assign?device=cpu&ram_mb=204800&hf_repo=kimi-k2.7"
|
||||
)
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
assert assignment["model"] == "kimi-k2.7"
|
||||
assert assignment["hf_repo"] == "unsloth/Kimi-K2.7-Code"
|
||||
assert assignment["num_layers"] == 61
|
||||
assert assignment["shard_start"] == 0
|
||||
assert 0 <= assignment["shard_end"] < 60
|
||||
|
||||
|
||||
def test_tracker_serves_health_while_proxy_request_is_in_flight():
|
||||
"""Long inference proxy requests must not block heartbeats/health checks."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user