relay preparing for public internet

tests/test_kv_cache_distributed.py

@@ -260,6 +260,49 @@ def test_eos_token_id_stops_generation():
     assert len(tail_backend.calls) == 2
 
 
+def test_stateless_fallback_stops_at_eos_token_id():
+    """When kv caching is off, EOS must still stop generation by token id —
+    EOS decodes to "" (skip_special_tokens) so the text check never fires."""
+
+    class _StatelessHead(_CachedHeadBackend):
+        supports_kv_cache = False
+
+    head_backend = _StatelessHead()
+    tail_backend = _CachedTailBackend([(" a", 1), ("", 99), (" never", 3)])
+    head = TorchNodeServer(backend=head_backend, tracker_mode=True)
+    tail = TorchNodeServer(backend=tail_backend)
+    head_port = head.start()
+    tail_port = tail.start()
+    try:
+        content = _chat_once(head_port, tail_port, max_tokens=8)
+    finally:
+        head.stop()
+        tail.stop()
+
+    assert content == " a"
+    # Stops after the EOS step instead of burning steps until max_tokens.
+    assert len(tail_backend.calls) == 2
+    assert head_backend.decode_calls == []
+
+
+def test_decode_forward_logging_is_rate_limited():
+    """Shard nodes log a per-session decode summary, not one line per token."""
+    tail_backend = _CachedTailBackend([])
+    tail = TorchNodeServer(backend=tail_backend)
+    tail.start()
+    try:
+        srv = tail._server
+        assert srv.note_decode_step("s1", now=0.0) == 1
+        assert srv.note_decode_step("s1", now=1.0) is None
+        assert srv.note_decode_step("s1", now=4.9) is None
+        assert srv.note_decode_step("s1", now=5.5) == 4
+        assert srv.note_decode_step("s1", now=6.0) is None
+        # Sessions are throttled independently.
+        assert srv.note_decode_step("s2", now=6.0) == 1
+    finally:
+        tail.stop()
+
+
 def test_downstream_cache_miss_falls_back_to_full_reprefill():
     head_backend = _CachedHeadBackend()
     # Call 1 (the first decode) raises KVCacheMiss → node answers 409 →