fix: DGR-033 repair native worker protocol per cross-review BLOCK

Address the Codex GPT-5.5 review of the standalone fake C++ gRPC Shard
worker. Four root protocol defects fixed:

- Fail closed before SessionOpen: a per-session `opened` flag gates
  chunk/decode so no activation bypasses lifecycle, cancellation, epoch
  or flow-control state (terminal ERROR_CODE_INTERNAL), even when an
  out-of-band Cancel created placeholder state.
- Strict flow-control negotiation: NegotiateFlow takes the strictest of
  peer-vs-worker bounds (mirrors codec.negotiate_flow_control) and the
  negotiated per-session max_chunk_bytes is enforced on every bundle
  instead of trusting the peer proposal.
- In-stream ReleaseSignal now erases session state immediately.
- SessionOpen rejects incompatible schema, fingerprint, and shard-range
  identity and reports the worker's own served fingerprint rather than
  echoing the caller.

Adds 9 regression tests (worker suite 18 -> 27). Real gates on the
rebuilt pinned-gRPC binary: cmake build exit 0; ctest 2/2; worker
pytest 27 passed; harness+protocol 63 passed; compileall 0; diff --check
clean; ldd/nm show 0 llama/ggml linkage. DGR-033 passes -> true.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dobromir Popov
2026-07-26 22:57:03 +03:00
parent c073826374
commit 7473bb7e44
6 changed files with 399 additions and 70 deletions

View File

@@ -139,24 +139,43 @@ def _crc32c(payload: bytes) -> bytes:
return zlib.crc32(payload).to_bytes(4, "big")
def _open(*, route_session_id="rs-1", route_epoch=7, credits_granted=16) -> pb.SessionRequest:
_WORKER_FINGERPRINT = dict(
model_artifact_digest="sha256:native-test-artifact",
runtime_recipe_digest="sha256:native-test-recipe",
recipe_id="native-test",
recipe_version="1",
catalogue_version="1",
)
def _open(
*,
route_session_id="rs-1",
route_epoch=7,
credits_granted=16,
max_inflight_chunks=16,
max_chunk_bytes=4 * 1024 * 1024,
schema_version=pb.SCHEMA_VERSION_1,
fingerprint=None,
shard_range=None,
) -> pb.SessionRequest:
fp = pb.Fingerprint(**_WORKER_FINGERPRINT) if fingerprint is None else fingerprint
sr = (
pb.ShardRange(start_layer=0, end_layer=32, effective_start_layer=0)
if shard_range is None
else shard_range
)
return pb.SessionRequest(
open=pb.SessionOpen(
schema_version=pb.SCHEMA_VERSION_1,
schema_version=schema_version,
route_session_id=route_session_id,
route_epoch=route_epoch,
fingerprint=pb.Fingerprint(
model_artifact_digest="sha256:native-test-artifact",
runtime_recipe_digest="sha256:native-test-recipe",
recipe_id="native-test",
recipe_version="1",
catalogue_version="1",
),
shard_range=pb.ShardRange(start_layer=0, end_layer=32, effective_start_layer=0),
fingerprint=fp,
shard_range=sr,
proposed_flow_control=pb.FlowControl(
credits_granted=credits_granted,
max_inflight_chunks=16,
max_chunk_bytes=4 * 1024 * 1024,
max_inflight_chunks=max_inflight_chunks,
max_chunk_bytes=max_chunk_bytes,
max_prefill_chunk_tokens=512,
),
accepted_compression=[pb.COMPRESSION_NONE],
@@ -434,9 +453,9 @@ def test_out_of_band_cancel_rpc_races_ahead_of_open(worker):
def test_release_rpc_is_idempotent(worker):
stub = worker.stub()
# Open a session so state exists, then release it out of band twice.
worker.session([_open(route_session_id="rs-rel"), _release()])
# (release signal in-stream does not erase state; the unary Release RPC does)
# Open a session WITHOUT an in-stream release so state persists on the
# servicer, then drop it out of band twice.
worker.session([_open(route_session_id="rs-rel")])
first = stub.Release(pb.ReleaseRequest(schema_version=pb.SCHEMA_VERSION_1, route_session_id="rs-rel", route_epoch=7))
second = stub.Release(pb.ReleaseRequest(schema_version=pb.SCHEMA_VERSION_1, route_session_id="rs-rel", route_epoch=7))
assert first.released is True
@@ -496,3 +515,122 @@ def test_direct_and_opaque_relay_yield_identical_responses(worker):
assert len(direct_resp) == len(relay_resp) == 3
for i, (d, r) in enumerate(zip(direct_resp, relay_resp)):
assert d == r, f"response #{i} differs between direct and opaque relay"
# --- fail-closed before SessionOpen ----------------------------------------
def test_chunk_before_open_is_rejected(worker):
# An activation with no preceding SessionOpen must fail closed and end the
# stream: no work may bypass the lifecycle handshake.
responses = worker.session([_chunk("w-noopen", b"payload", step=1)])
assert len(responses) == 1
assert responses[0].WhichOneof("kind") == "status"
assert responses[0].status.error.code == pb.ERROR_CODE_INTERNAL
assert responses[0].status.terminal is True
assert "SessionOpen" in responses[0].status.error.detail
def test_decode_before_open_is_rejected(worker):
responses = worker.session([_decode("w-noopen", b"payload", step=1, position=0)])
assert len(responses) == 1
assert responses[0].WhichOneof("kind") == "status"
assert responses[0].status.error.code == pb.ERROR_CODE_INTERNAL
assert responses[0].status.terminal is True
# --- flow-control negotiation with strict worker bounds --------------------
def test_flow_control_proposal_is_clamped_to_worker_bounds(worker):
# A peer proposing a window far above the worker limits must be clamped to
# the worker own ceilings, never granted the inflated proposal.
responses = worker.session(
[_open(credits_granted=9999, max_inflight_chunks=9999, max_chunk_bytes=1073741824)]
)
fc = responses[0].accepted.flow_control
assert fc.max_inflight_chunks == 16
assert fc.credits_granted == 16
assert fc.max_chunk_bytes == 4 * 1024 * 1024
def test_negotiated_max_chunk_bytes_caps_peer_proposal():
# Worker ceiling is 64 bytes; the peer proposes 4 MiB. The negotiated per
# session ceiling is the stricter 64, so a 128-byte tensor is refused even
# though the peer allowed it — the worker never adopts the peer proposal.
w = _Worker(extra_env={"MESHNET_MAX_CHUNK_BYTES": "64"})
try:
big = b"x" * 128
responses = w.session(
[_open(max_chunk_bytes=4 * 1024 * 1024), _chunk("w-big", big, step=1, total_bytes=128)]
)
assert responses[0].accepted.flow_control.max_chunk_bytes == 64
assert responses[1].status.error.code == pb.ERROR_CODE_RESOURCE_EXHAUSTED
assert "max_chunk_bytes" in responses[1].status.error.detail
finally:
if w.proc.poll() is None:
w.close()
# --- in-stream release erases session state --------------------------------
def test_in_stream_release_erases_session_state(worker):
stub = worker.stub()
resp = worker.session(
[
_open(route_session_id="rs-erase"),
pb.SessionRequest(
release=pb.ReleaseSignal(route_session_id="rs-erase", route_epoch=7, work_id="w-final")
),
]
)
assert resp[-1].status.terminal is True
# The state is already gone: an out-of-band Release finds nothing to drop.
after = stub.Release(
pb.ReleaseRequest(schema_version=pb.SCHEMA_VERSION_1, route_session_id="rs-erase", route_epoch=7)
)
assert after.released is False
# --- SessionOpen identity validation ---------------------------------------
def test_incompatible_schema_is_rejected_at_open(worker):
responses = worker.session([_open(schema_version=pb.SCHEMA_VERSION_UNSPECIFIED)])
assert responses[0].WhichOneof("kind") == "status"
assert responses[0].status.error.code == pb.ERROR_CODE_SCHEMA_UNSUPPORTED
assert responses[0].status.terminal is True
def test_incompatible_fingerprint_is_rejected_at_open(worker):
bad_fp = pb.Fingerprint(
model_artifact_digest="sha256:some-other-model",
runtime_recipe_digest="sha256:native-test-recipe",
recipe_id="native-test",
recipe_version="1",
catalogue_version="1",
)
responses = worker.session([_open(fingerprint=bad_fp)])
assert responses[0].WhichOneof("kind") == "status"
assert responses[0].status.error.code == pb.ERROR_CODE_FINGERPRINT_MISMATCH
assert responses[0].status.terminal is True
def test_shard_range_mismatch_is_rejected_at_open(worker):
responses = worker.session(
[_open(shard_range=pb.ShardRange(start_layer=0, end_layer=64, effective_start_layer=0))]
)
assert responses[0].WhichOneof("kind") == "status"
assert responses[0].status.error.code == pb.ERROR_CODE_SHARD_RANGE_MISMATCH
assert responses[0].status.terminal is True
def test_session_accepted_reports_worker_fingerprint_not_caller(worker):
# The caller asserts no fingerprint; SessionAccepted must carry the worker
# OWN served identity, not a copy of the caller (empty) fingerprint.
responses = worker.session([_open(fingerprint=pb.Fingerprint())])
assert responses[0].WhichOneof("kind") == "accepted"
accepted = responses[0].accepted
assert accepted.fingerprint.model_artifact_digest == "sha256:native-test-artifact"
assert accepted.fingerprint.runtime_recipe_digest == "sha256:native-test-recipe"