"""Curated list of models supported by the network with VRAM requirements.""" from __future__ import annotations from dataclasses import dataclass @dataclass class ModelPreset: name: str hf_repo: str num_layers: int # VRAM in GB at each quantization level (None = too large to quantize this way) vram_nf4: float vram_int8: float vram_bf16: float description: str def vram_for_quant(self, quant: str) -> float: """Return VRAM requirement in GB for the given quantization.""" q = quant.lower().replace("bfloat16", "bf16") if q == "nf4": return self.vram_nf4 if q in ("int8", "int8"): return self.vram_int8 if q in ("bf16", "bfloat16"): return self.vram_bf16 raise ValueError(f"unknown quantization: {quant!r}") def fits_vram(self, available_gb: float, quant: str) -> bool: return self.vram_for_quant(quant) <= available_gb def recommended_quant(self, available_gb: float) -> str | None: """Return the highest-quality quantization that fits available VRAM, or None.""" if self.vram_bf16 <= available_gb: return "bf16" if self.vram_int8 <= available_gb: return "int8" if self.vram_nf4 <= available_gb: return "nf4" return None CURATED_MODELS: list[ModelPreset] = [ ModelPreset( name="Llama-3-70B-Instruct", hf_repo="meta-llama/Meta-Llama-3-70B-Instruct", num_layers=80, vram_nf4=18.0, vram_int8=40.0, vram_bf16=140.0, description="Meta's flagship 70B instruction model", ), ModelPreset( name="Qwen2.5-72B-Instruct", hf_repo="Qwen/Qwen2.5-72B-Instruct", num_layers=80, vram_nf4=19.0, vram_int8=41.0, vram_bf16=145.0, description="Alibaba's 72B multilingual instruction model", ), ModelPreset( name="Mixtral-8x7B-Instruct", hf_repo="mistralai/Mixtral-8x7B-Instruct-v0.1", num_layers=32, vram_nf4=7.0, vram_int8=14.0, vram_bf16=27.0, description="Mistral's sparse MoE — fast and efficient", ), ModelPreset( name="Llama-3-8B-Instruct", hf_repo="meta-llama/Meta-Llama-3-8B-Instruct", num_layers=32, vram_nf4=4.5, vram_int8=8.5, vram_bf16=16.0, description="Meta's compact 8B model — good for low-VRAM nodes", ), ModelPreset( name="Phi-3-medium-128k", hf_repo="microsoft/Phi-3-medium-128k-instruct", num_layers=40, vram_nf4=4.0, vram_int8=8.0, vram_bf16=15.0, description="Microsoft's efficient 14B model with 128k context", ), ModelPreset( name="Gemma-2-27B-IT", hf_repo="google/gemma-2-27b-it", num_layers=46, vram_nf4=10.0, vram_int8=20.0, vram_bf16=54.0, description="Google's 27B instruction-tuned model", ), ModelPreset( name="DeepSeek-V2-Lite-Chat", hf_repo="deepseek-ai/DeepSeek-V2-Lite-Chat", num_layers=27, vram_nf4=5.0, vram_int8=9.0, vram_bf16=16.0, description="DeepSeek's efficient MoE — strong coding + reasoning", ), ] def browse_hf_hub(top_n: int = 20) -> list[dict]: """Fetch top downloaded text-generation models from HuggingFace Hub.""" try: from huggingface_hub import list_models # type: ignore[import] models = list( list_models( pipeline_tag="text-generation", library="transformers", sort="downloads", direction=-1, limit=top_n, ) ) return [ { "repo": m.id, "downloads": getattr(m, "downloads", 0) or 0, } for m in models ] except Exception as exc: raise RuntimeError(f"HuggingFace Hub lookup failed: {exc}") from exc