feat(us-016): auto-detect shard range from model config

Layer count is now fetched from the curated catalog (zero network calls
for known models) or via AutoConfig.from_pretrained() (~1 KB config.json
only) when model_id is given without --shard-start/--shard-end.

- model_catalog: add detect_num_layers(), two small Qwen models at top
- startup: _detect_num_layers() helper; shard range auto-derived
- wizard: show detected layer count for custom HF repos
- tests: 3 new tests for auto-shard; fix catalog-order assumptions

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Dobromir Popov
2026-06-29 18:27:50 +03:00
parent a2258d3df4
commit 080d49b2c2
4 changed files with 135 additions and 12 deletions

View File

@@ -42,6 +42,24 @@ class ModelPreset:
CURATED_MODELS: list[ModelPreset] = [
ModelPreset(
name="Qwen2.5-0.5B-Instruct",
hf_repo="Qwen/Qwen2.5-0.5B-Instruct",
num_layers=24,
vram_nf4=0.4,
vram_int8=0.6,
vram_bf16=1.0,
description="Smallest no-gating model — great for testing, ~1 GB",
),
ModelPreset(
name="Qwen2.5-1.5B-Instruct",
hf_repo="Qwen/Qwen2.5-1.5B-Instruct",
num_layers=28,
vram_nf4=1.0,
vram_int8=1.8,
vram_bf16=3.2,
description="Fast no-gating model — good quality, ~3 GB",
),
ModelPreset(
name="Llama-3-70B-Instruct",
hf_repo="meta-llama/Meta-Llama-3-70B-Instruct",
@@ -72,7 +90,7 @@ CURATED_MODELS: list[ModelPreset] = [
ModelPreset(
name="Llama-3-8B-Instruct",
hf_repo="meta-llama/Meta-Llama-3-8B-Instruct",
num_layers=32,
num_layers=32, # gated repo — requires HF login
vram_nf4=4.5,
vram_int8=8.5,
vram_bf16=16.0,
@@ -108,6 +126,20 @@ CURATED_MODELS: list[ModelPreset] = [
]
def detect_num_layers(hf_repo: str) -> int | None:
"""Return num_hidden_layers from HuggingFace config.json (downloads ~1 KB only)."""
# Check curated list first (no network call)
for m in CURATED_MODELS:
if m.hf_repo == hf_repo:
return m.num_layers
try:
from transformers import AutoConfig # type: ignore[import]
cfg = AutoConfig.from_pretrained(hf_repo)
return int(cfg.num_hidden_layers)
except Exception:
return None
def browse_hf_hub(top_n: int = 20) -> list[dict]:
"""Fetch top downloaded text-generation models from HuggingFace Hub."""
try: