146 lines
4.7 KiB
Python
146 lines
4.7 KiB
Python
"""US-007 integration tests: optimistic fraud detection and slashing."""
|
|
|
|
import json
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from meshnet_contracts import LocalSolanaContracts
|
|
from meshnet_gateway.server import GatewayServer
|
|
from meshnet_node.server import StubNodeServer
|
|
from meshnet_tracker.server import TrackerServer
|
|
from meshnet_validator import ValidatorProcess
|
|
|
|
|
|
def _post_json(url: str, payload: dict, headers: dict | None = None) -> dict:
|
|
req = urllib.request.Request(
|
|
url,
|
|
data=json.dumps(payload).encode(),
|
|
headers={"Content-Type": "application/json", **(headers or {})},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(req) as resp:
|
|
return json.loads(resp.read())
|
|
|
|
|
|
def _register_node(
|
|
tracker_url: str,
|
|
endpoint: str,
|
|
shard_start: int,
|
|
shard_end: int,
|
|
wallet_address: str,
|
|
) -> None:
|
|
_post_json(
|
|
f"{tracker_url}/v1/nodes/register",
|
|
{
|
|
"endpoint": endpoint,
|
|
"shard_start": shard_start,
|
|
"shard_end": shard_end,
|
|
"hardware_profile": {},
|
|
"wallet_address": wallet_address,
|
|
"score": 1.0,
|
|
},
|
|
)
|
|
|
|
|
|
def _send_completion(gateway_url: str, prompt: str) -> str:
|
|
body = _post_json(
|
|
f"{gateway_url}/v1/chat/completions",
|
|
{
|
|
"model": "stub-model",
|
|
"messages": [{"role": "user", "content": prompt}],
|
|
},
|
|
{"Authorization": "Bearer fraud-test-key"},
|
|
)
|
|
return body["choices"][0]["message"]["content"]
|
|
|
|
|
|
def test_bad_node_is_slashed_and_excluded_from_gateway_routes(capsys):
|
|
"""A bad final shard is slashed by the validator and then excluded by routing."""
|
|
contracts = LocalSolanaContracts()
|
|
contracts.registry.submit_stake("wallet-good", 500)
|
|
contracts.registry.submit_stake("wallet-bad", 500)
|
|
tracker = TrackerServer(contracts=contracts)
|
|
tracker_port = tracker.start()
|
|
tracker_url = f"http://127.0.0.1:{tracker_port}"
|
|
|
|
good_node = StubNodeServer(shard_start=0, shard_end=15, is_last_shard=False)
|
|
bad_node = StubNodeServer(
|
|
shard_start=16,
|
|
shard_end=31,
|
|
is_last_shard=True,
|
|
response_prefix="fraudulent response to:",
|
|
)
|
|
reference_node = StubNodeServer(shard_start=0, shard_end=31, is_last_shard=True)
|
|
good_port = good_node.start()
|
|
bad_port = bad_node.start()
|
|
reference_port = reference_node.start()
|
|
|
|
_register_node(tracker_url, f"http://127.0.0.1:{good_port}", 0, 15, "wallet-good")
|
|
_register_node(tracker_url, f"http://127.0.0.1:{bad_port}", 16, 31, "wallet-bad")
|
|
|
|
gateway = GatewayServer(tracker_url=tracker_url, contracts=contracts)
|
|
gateway_port = gateway.start()
|
|
gateway_url = f"http://127.0.0.1:{gateway_port}"
|
|
validator = ValidatorProcess(
|
|
contracts=contracts,
|
|
reference_node_url=f"http://127.0.0.1:{reference_port}",
|
|
sample_rate=1.0,
|
|
slash_amount=125,
|
|
strike_threshold=1,
|
|
random_seed=7,
|
|
)
|
|
|
|
try:
|
|
for i in range(20):
|
|
assert _send_completion(gateway_url, f"prompt {i}").startswith("fraudulent response to:")
|
|
|
|
receipts = validator.validate_once()
|
|
|
|
bad_wallet = contracts.registry.get_wallet("wallet-bad")
|
|
assert receipts
|
|
assert bad_wallet.stake_balance == 375
|
|
assert bad_wallet.strike_count == 1
|
|
assert bad_wallet.banned is True
|
|
assert "WARNING: node wallet-bad slashed" in capsys.readouterr().out
|
|
|
|
try:
|
|
_send_completion(gateway_url, "after slash")
|
|
raise AssertionError("Expected no route after bad wallet is excluded")
|
|
except urllib.error.HTTPError as exc:
|
|
assert exc.code == 503
|
|
finally:
|
|
gateway.stop()
|
|
reference_node.stop()
|
|
bad_node.stop()
|
|
good_node.stop()
|
|
tracker.stop()
|
|
|
|
|
|
def test_validator_sampling_rate_is_configurable():
|
|
"""The validator only reruns requests selected by its configured sample rate."""
|
|
contracts = LocalSolanaContracts()
|
|
|
|
class InProcessReferenceValidator(ValidatorProcess):
|
|
def _run_reference(self, messages: list[dict]) -> str:
|
|
return f"stub response to: {messages[-1]['content']}"
|
|
|
|
validator = InProcessReferenceValidator(
|
|
contracts=contracts,
|
|
reference_node_url="http://reference-node.invalid",
|
|
sample_rate=0.05,
|
|
random_seed=1,
|
|
)
|
|
|
|
for i in range(1_000):
|
|
contracts.validation.record_completed_inference(
|
|
session_id=f"session-{i}",
|
|
model="stub-model",
|
|
messages=[{"role": "user", "content": f"prompt {i}"}],
|
|
observed_output=f"stub response to: prompt {i}",
|
|
route_nodes=[{"wallet_address": "wallet-good"}],
|
|
)
|
|
|
|
validator.validate_once()
|
|
|
|
assert 30 <= validator.sampled_count <= 70
|