fix: DGR-033 repair native worker protocol per cross-review BLOCK
Address the Codex GPT-5.5 review of the standalone fake C++ gRPC Shard worker. Four root protocol defects fixed: - Fail closed before SessionOpen: a per-session `opened` flag gates chunk/decode so no activation bypasses lifecycle, cancellation, epoch or flow-control state (terminal ERROR_CODE_INTERNAL), even when an out-of-band Cancel created placeholder state. - Strict flow-control negotiation: NegotiateFlow takes the strictest of peer-vs-worker bounds (mirrors codec.negotiate_flow_control) and the negotiated per-session max_chunk_bytes is enforced on every bundle instead of trusting the peer proposal. - In-stream ReleaseSignal now erases session state immediately. - SessionOpen rejects incompatible schema, fingerprint, and shard-range identity and reports the worker's own served fingerprint rather than echoing the caller. Adds 9 regression tests (worker suite 18 -> 27). Real gates on the rebuilt pinned-gRPC binary: cmake build exit 0; ctest 2/2; worker pytest 27 passed; harness+protocol 63 passed; compileall 0; diff --check clean; ldd/nm show 0 llama/ggml linkage. DGR-033 passes -> true. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -139,24 +139,43 @@ def _crc32c(payload: bytes) -> bytes:
|
||||
return zlib.crc32(payload).to_bytes(4, "big")
|
||||
|
||||
|
||||
def _open(*, route_session_id="rs-1", route_epoch=7, credits_granted=16) -> pb.SessionRequest:
|
||||
_WORKER_FINGERPRINT = dict(
|
||||
model_artifact_digest="sha256:native-test-artifact",
|
||||
runtime_recipe_digest="sha256:native-test-recipe",
|
||||
recipe_id="native-test",
|
||||
recipe_version="1",
|
||||
catalogue_version="1",
|
||||
)
|
||||
|
||||
|
||||
def _open(
|
||||
*,
|
||||
route_session_id="rs-1",
|
||||
route_epoch=7,
|
||||
credits_granted=16,
|
||||
max_inflight_chunks=16,
|
||||
max_chunk_bytes=4 * 1024 * 1024,
|
||||
schema_version=pb.SCHEMA_VERSION_1,
|
||||
fingerprint=None,
|
||||
shard_range=None,
|
||||
) -> pb.SessionRequest:
|
||||
fp = pb.Fingerprint(**_WORKER_FINGERPRINT) if fingerprint is None else fingerprint
|
||||
sr = (
|
||||
pb.ShardRange(start_layer=0, end_layer=32, effective_start_layer=0)
|
||||
if shard_range is None
|
||||
else shard_range
|
||||
)
|
||||
return pb.SessionRequest(
|
||||
open=pb.SessionOpen(
|
||||
schema_version=pb.SCHEMA_VERSION_1,
|
||||
schema_version=schema_version,
|
||||
route_session_id=route_session_id,
|
||||
route_epoch=route_epoch,
|
||||
fingerprint=pb.Fingerprint(
|
||||
model_artifact_digest="sha256:native-test-artifact",
|
||||
runtime_recipe_digest="sha256:native-test-recipe",
|
||||
recipe_id="native-test",
|
||||
recipe_version="1",
|
||||
catalogue_version="1",
|
||||
),
|
||||
shard_range=pb.ShardRange(start_layer=0, end_layer=32, effective_start_layer=0),
|
||||
fingerprint=fp,
|
||||
shard_range=sr,
|
||||
proposed_flow_control=pb.FlowControl(
|
||||
credits_granted=credits_granted,
|
||||
max_inflight_chunks=16,
|
||||
max_chunk_bytes=4 * 1024 * 1024,
|
||||
max_inflight_chunks=max_inflight_chunks,
|
||||
max_chunk_bytes=max_chunk_bytes,
|
||||
max_prefill_chunk_tokens=512,
|
||||
),
|
||||
accepted_compression=[pb.COMPRESSION_NONE],
|
||||
@@ -434,9 +453,9 @@ def test_out_of_band_cancel_rpc_races_ahead_of_open(worker):
|
||||
|
||||
def test_release_rpc_is_idempotent(worker):
|
||||
stub = worker.stub()
|
||||
# Open a session so state exists, then release it out of band twice.
|
||||
worker.session([_open(route_session_id="rs-rel"), _release()])
|
||||
# (release signal in-stream does not erase state; the unary Release RPC does)
|
||||
# Open a session WITHOUT an in-stream release so state persists on the
|
||||
# servicer, then drop it out of band twice.
|
||||
worker.session([_open(route_session_id="rs-rel")])
|
||||
first = stub.Release(pb.ReleaseRequest(schema_version=pb.SCHEMA_VERSION_1, route_session_id="rs-rel", route_epoch=7))
|
||||
second = stub.Release(pb.ReleaseRequest(schema_version=pb.SCHEMA_VERSION_1, route_session_id="rs-rel", route_epoch=7))
|
||||
assert first.released is True
|
||||
@@ -496,3 +515,122 @@ def test_direct_and_opaque_relay_yield_identical_responses(worker):
|
||||
assert len(direct_resp) == len(relay_resp) == 3
|
||||
for i, (d, r) in enumerate(zip(direct_resp, relay_resp)):
|
||||
assert d == r, f"response #{i} differs between direct and opaque relay"
|
||||
|
||||
|
||||
# --- fail-closed before SessionOpen ----------------------------------------
|
||||
|
||||
|
||||
def test_chunk_before_open_is_rejected(worker):
|
||||
# An activation with no preceding SessionOpen must fail closed and end the
|
||||
# stream: no work may bypass the lifecycle handshake.
|
||||
responses = worker.session([_chunk("w-noopen", b"payload", step=1)])
|
||||
assert len(responses) == 1
|
||||
assert responses[0].WhichOneof("kind") == "status"
|
||||
assert responses[0].status.error.code == pb.ERROR_CODE_INTERNAL
|
||||
assert responses[0].status.terminal is True
|
||||
assert "SessionOpen" in responses[0].status.error.detail
|
||||
|
||||
|
||||
def test_decode_before_open_is_rejected(worker):
|
||||
responses = worker.session([_decode("w-noopen", b"payload", step=1, position=0)])
|
||||
assert len(responses) == 1
|
||||
assert responses[0].WhichOneof("kind") == "status"
|
||||
assert responses[0].status.error.code == pb.ERROR_CODE_INTERNAL
|
||||
assert responses[0].status.terminal is True
|
||||
|
||||
|
||||
# --- flow-control negotiation with strict worker bounds --------------------
|
||||
|
||||
|
||||
def test_flow_control_proposal_is_clamped_to_worker_bounds(worker):
|
||||
# A peer proposing a window far above the worker limits must be clamped to
|
||||
# the worker own ceilings, never granted the inflated proposal.
|
||||
responses = worker.session(
|
||||
[_open(credits_granted=9999, max_inflight_chunks=9999, max_chunk_bytes=1073741824)]
|
||||
)
|
||||
fc = responses[0].accepted.flow_control
|
||||
assert fc.max_inflight_chunks == 16
|
||||
assert fc.credits_granted == 16
|
||||
assert fc.max_chunk_bytes == 4 * 1024 * 1024
|
||||
|
||||
|
||||
def test_negotiated_max_chunk_bytes_caps_peer_proposal():
|
||||
# Worker ceiling is 64 bytes; the peer proposes 4 MiB. The negotiated per
|
||||
# session ceiling is the stricter 64, so a 128-byte tensor is refused even
|
||||
# though the peer allowed it — the worker never adopts the peer proposal.
|
||||
w = _Worker(extra_env={"MESHNET_MAX_CHUNK_BYTES": "64"})
|
||||
try:
|
||||
big = b"x" * 128
|
||||
responses = w.session(
|
||||
[_open(max_chunk_bytes=4 * 1024 * 1024), _chunk("w-big", big, step=1, total_bytes=128)]
|
||||
)
|
||||
assert responses[0].accepted.flow_control.max_chunk_bytes == 64
|
||||
assert responses[1].status.error.code == pb.ERROR_CODE_RESOURCE_EXHAUSTED
|
||||
assert "max_chunk_bytes" in responses[1].status.error.detail
|
||||
finally:
|
||||
if w.proc.poll() is None:
|
||||
w.close()
|
||||
|
||||
|
||||
# --- in-stream release erases session state --------------------------------
|
||||
|
||||
|
||||
def test_in_stream_release_erases_session_state(worker):
|
||||
stub = worker.stub()
|
||||
resp = worker.session(
|
||||
[
|
||||
_open(route_session_id="rs-erase"),
|
||||
pb.SessionRequest(
|
||||
release=pb.ReleaseSignal(route_session_id="rs-erase", route_epoch=7, work_id="w-final")
|
||||
),
|
||||
]
|
||||
)
|
||||
assert resp[-1].status.terminal is True
|
||||
# The state is already gone: an out-of-band Release finds nothing to drop.
|
||||
after = stub.Release(
|
||||
pb.ReleaseRequest(schema_version=pb.SCHEMA_VERSION_1, route_session_id="rs-erase", route_epoch=7)
|
||||
)
|
||||
assert after.released is False
|
||||
|
||||
|
||||
# --- SessionOpen identity validation ---------------------------------------
|
||||
|
||||
|
||||
def test_incompatible_schema_is_rejected_at_open(worker):
|
||||
responses = worker.session([_open(schema_version=pb.SCHEMA_VERSION_UNSPECIFIED)])
|
||||
assert responses[0].WhichOneof("kind") == "status"
|
||||
assert responses[0].status.error.code == pb.ERROR_CODE_SCHEMA_UNSUPPORTED
|
||||
assert responses[0].status.terminal is True
|
||||
|
||||
|
||||
def test_incompatible_fingerprint_is_rejected_at_open(worker):
|
||||
bad_fp = pb.Fingerprint(
|
||||
model_artifact_digest="sha256:some-other-model",
|
||||
runtime_recipe_digest="sha256:native-test-recipe",
|
||||
recipe_id="native-test",
|
||||
recipe_version="1",
|
||||
catalogue_version="1",
|
||||
)
|
||||
responses = worker.session([_open(fingerprint=bad_fp)])
|
||||
assert responses[0].WhichOneof("kind") == "status"
|
||||
assert responses[0].status.error.code == pb.ERROR_CODE_FINGERPRINT_MISMATCH
|
||||
assert responses[0].status.terminal is True
|
||||
|
||||
|
||||
def test_shard_range_mismatch_is_rejected_at_open(worker):
|
||||
responses = worker.session(
|
||||
[_open(shard_range=pb.ShardRange(start_layer=0, end_layer=64, effective_start_layer=0))]
|
||||
)
|
||||
assert responses[0].WhichOneof("kind") == "status"
|
||||
assert responses[0].status.error.code == pb.ERROR_CODE_SHARD_RANGE_MISMATCH
|
||||
assert responses[0].status.terminal is True
|
||||
|
||||
|
||||
def test_session_accepted_reports_worker_fingerprint_not_caller(worker):
|
||||
# The caller asserts no fingerprint; SessionAccepted must carry the worker
|
||||
# OWN served identity, not a copy of the caller (empty) fingerprint.
|
||||
responses = worker.session([_open(fingerprint=pb.Fingerprint())])
|
||||
assert responses[0].WhichOneof("kind") == "accepted"
|
||||
accepted = responses[0].accepted
|
||||
assert accepted.fingerprint.model_artifact_digest == "sha256:native-test-artifact"
|
||||
assert accepted.fingerprint.runtime_recipe_digest == "sha256:native-test-recipe"
|
||||
|
||||
Reference in New Issue
Block a user