diff --git a/QUICKSTART.md b/QUICKSTART.md index 7e91671..371fb18 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -120,6 +120,10 @@ $env:HF_HOME = "D:\DEV\models" --port 8005 ``` +The example above starts the Windows node as the second half of a split Qwen +model. To run the full model on Windows instead, omit `--shard-start` and +`--shard-end`. + `--host 0.0.0.0` binds the node to all Windows interfaces. `--advertise-host` is what the tracker gives to other nodes, so it must be the Windows LAN IP that the tracker and peer nodes can actually reach. @@ -139,10 +143,10 @@ lines like: curl http://192.168.0.42:8005/v1/health ``` -If that endpoint returns 404, that is okay: it still proves the TCP connection -reached the node process. If it times out or connection-refuses, check the -Windows Firewall rule, `--host 0.0.0.0`, the selected LAN IP, and that the node is -still running. +If that endpoint returns 404 or 501, that is okay: it still proves the TCP +connection reached the node process. If it times out or connection-refuses, check +the Windows Firewall rule, `--host 0.0.0.0`, the selected LAN IP, and that the +node is still running. ### Public tracker + WSS relay diff --git a/tests/test_mining_cli.py b/tests/test_mining_cli.py index e26331f..f52f498 100644 --- a/tests/test_mining_cli.py +++ b/tests/test_mining_cli.py @@ -452,6 +452,7 @@ def test_default_cli_passes_advertise_host(monkeypatch): "meshnet-node", "--tracker", "http://192.168.0.179:8081", "--advertise-host", "192.168.0.42", + "--debug", "--no-tui", ]) @@ -465,3 +466,4 @@ def test_default_cli_passes_advertise_host(monkeypatch): assert captured["tracker_url"] == "http://192.168.0.179:8081" assert captured["advertise_host"] == "192.168.0.42" + assert captured["debug"] is True diff --git a/tests/test_real_model_backend.py b/tests/test_real_model_backend.py index 2757670..de3b740 100644 --- a/tests/test_real_model_backend.py +++ b/tests/test_real_model_backend.py @@ -81,6 +81,38 @@ class _FakeFullBackend(_FakeBackend): return 1 +class _FakeChatTokenizer: + eos_token = "" + + def apply_chat_template(self, messages, add_generation_prompt=True, tokenize=False): + assert add_generation_prompt is True + assert tokenize is False + return "debug prompt" + + +class _FakePipelineHeadBackend(_FakeBackend): + tokenizer = _FakeChatTokenizer() + + def encode_prompt(self, prompt: str) -> TensorPayload: + assert prompt == "debug prompt" + return TensorPayload( + body=b"\x00" * (1 * 6 * 8 * 2), + shape=[1, 6, 8], + attention_mask_header=None, + position_ids_header=None, + ) + + +class _FakePipelineTailBackend(_FakeTailBackend): + def __init__(self) -> None: + self.start_layers: list[int | None] = [] + + def forward_bytes(self, body, shape, attention_mask_header, position_ids_header, start_layer=None): + self.start_layers.append(start_layer) + assert len(body) == 1 * 6 * 8 * 2 + return " token" + + def test_quantization_flag_validation(): assert validate_quantization("bfloat16") == "bfloat16" assert validate_quantization("int8") == "int8" @@ -198,6 +230,75 @@ def test_full_model_chat_completion_uses_generation_not_single_token_decode(): node.stop() +def test_pipeline_hop_logs_are_suppressed_without_debug(capsys): + tail_backend = _FakePipelineTailBackend() + head = TorchNodeServer(backend=_FakePipelineHeadBackend(), tracker_mode=True) + tail = TorchNodeServer(backend=tail_backend) + head_port = head.start() + tail_port = tail.start() + try: + payload = json.dumps({ + "model": "fake-model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 1, + }).encode() + req = urllib.request.Request( + f"http://127.0.0.1:{head_port}/v1/chat/completions", + data=payload, + headers={ + "Content-Type": "application/json", + "X-Meshnet-Route": json.dumps([ + {"endpoint": f"http://127.0.0.1:{tail_port}", "start_layer": 22}, + ]), + }, + method="POST", + ) + with urllib.request.urlopen(req, timeout=5) as resp: + body = json.loads(resp.read()) + finally: + head.stop() + tail.stop() + + out = capsys.readouterr().out + assert body["choices"][0]["message"]["content"] == " token" + assert tail_backend.start_layers == [22] + assert "pipeline hop 0:" not in out + assert "pipeline hop 0 returned text" not in out + + +def test_pipeline_hop_logs_are_enabled_with_debug(capsys): + head = TorchNodeServer(backend=_FakePipelineHeadBackend(), tracker_mode=True, debug=True) + tail = TorchNodeServer(backend=_FakePipelineTailBackend()) + head_port = head.start() + tail_port = tail.start() + try: + payload = json.dumps({ + "model": "fake-model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 1, + }).encode() + req = urllib.request.Request( + f"http://127.0.0.1:{head_port}/v1/chat/completions", + data=payload, + headers={ + "Content-Type": "application/json", + "X-Meshnet-Route": json.dumps([ + {"endpoint": f"http://127.0.0.1:{tail_port}", "start_layer": 22}, + ]), + }, + method="POST", + ) + with urllib.request.urlopen(req, timeout=5) as resp: + json.loads(resp.read()) + finally: + head.stop() + tail.stop() + + out = capsys.readouterr().out + assert f" [node] pipeline hop 0: http://127.0.0.1:{tail_port} start_layer=22" in out + assert " [node] pipeline hop 0 returned text=' token'" in out + + def test_int_tensor_header_serializes_torch_tensors(): torch = pytest.importorskip("torch")