feat: checkpoint distributed gguf runtime stories
This commit is contained in:
@@ -71,6 +71,74 @@ class BenchmarkLane:
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BenchmarkWorkload:
|
||||
"""Identical request shape both recipes must run so speed stays comparable.
|
||||
|
||||
Pinning prompts, context lengths, output lengths, and sampling policy in the
|
||||
versioned contract is what makes the safetensors-versus-GGUF numbers a
|
||||
controlled comparison instead of two differently-configured runs.
|
||||
"""
|
||||
|
||||
prompts: tuple[str, ...]
|
||||
context_lengths: tuple[int, ...]
|
||||
output_lengths: tuple[int, ...]
|
||||
sampling_policy: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"prompts": list(self.prompts),
|
||||
"context_lengths": list(self.context_lengths),
|
||||
"output_lengths": list(self.output_lengths),
|
||||
"sampling_policy": self.sampling_policy,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QualityPolicy:
|
||||
"""Correctness/quality lane kept separate from the performance/fit lanes.
|
||||
|
||||
BF16 safetensors and Q2_K GGUF are not numerically equivalent, so quality is
|
||||
measured as its own lane (output drift against the BF16 reference under a
|
||||
documented tolerance) rather than assumed away by the speed/fit comparison.
|
||||
"""
|
||||
|
||||
statement: str
|
||||
reference_lane_runtime: str
|
||||
measured_lane_runtime: str
|
||||
max_output_drift: float
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"statement": self.statement,
|
||||
"reference_lane_runtime": self.reference_lane_runtime,
|
||||
"measured_lane_runtime": self.measured_lane_runtime,
|
||||
"max_output_drift": self.max_output_drift,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReleaseGate:
|
||||
"""Versioned thresholds later release gates (DGR-014) consume unchanged.
|
||||
|
||||
Thresholds live in the contract, not in code, so the release gate cannot be
|
||||
weakened after seeing implementation results.
|
||||
"""
|
||||
|
||||
min_decode_speedup: float
|
||||
max_artifact_bytes_ratio: float
|
||||
max_memory_bytes_ratio: float
|
||||
max_quality_drift: float
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"min_decode_speedup": self.min_decode_speedup,
|
||||
"max_artifact_bytes_ratio": self.max_artifact_bytes_ratio,
|
||||
"max_memory_bytes_ratio": self.max_memory_bytes_ratio,
|
||||
"max_quality_drift": self.max_quality_drift,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PerformanceContract:
|
||||
"""Machine-readable contract for the DGR-001 benchmark story."""
|
||||
|
||||
Reference in New Issue
Block a user