dbf856f497
- Tracker: add GET /v1/tracker-nodes/<model> returning nodes registered with tracker_mode=true whose shard_start matches the model's first layer - Node: StubNodeServer and TorchNodeServer accept tracker_mode/tracker_url; when tracker_mode=True (or auto-detected via shard_start==0 for Torch), /v1/chat/completions is served alongside /forward - TorchNodeServer: full pipeline implementation — encode_prompt → route selection via tracker → binary forward through remaining hops → decode - Gateway: _handle_chat_completions checks _get_tracker_nodes() first and proxies round-robin to tracker-nodes; falls back to existing direct pipeline when none found (preserves all US-005 backward compat) - CLI: --tracker-mode and --tracker-url flags added to meshnet-node start - Test: two stub tracker-nodes + two mid-shard nodes for gpt2; 10 requests; round-robin 5/5 split verified; all OpenAI-format responses validated - All 78 tests pass Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
85 lines
2.6 KiB
Python
85 lines
2.6 KiB
Python
"""meshnet-node CLI entry point."""
|
|
|
|
import argparse
|
|
import sys
|
|
import time
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
prog="meshnet-node",
|
|
description="Distributed Inference Network node client",
|
|
)
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
start_cmd = subparsers.add_parser("start", help="Start the node server")
|
|
start_cmd.add_argument(
|
|
"--tracker", default="http://localhost:8080", help="Tracker URL"
|
|
)
|
|
start_cmd.add_argument("--port", type=int, default=7000, help="Port to listen on")
|
|
start_cmd.add_argument(
|
|
"--model", default="stub-model", help="Model preset to request from tracker"
|
|
)
|
|
start_cmd.add_argument(
|
|
"--model-id",
|
|
help="HuggingFace model id for the real PyTorch backend",
|
|
)
|
|
start_cmd.add_argument("--shard-start", type=int, help="First layer index for an explicit shard")
|
|
start_cmd.add_argument("--shard-end", type=int, help="Exclusive layer end index for an explicit shard")
|
|
start_cmd.add_argument(
|
|
"--quantization",
|
|
choices=["bfloat16", "int8", "nf4"],
|
|
default="bfloat16",
|
|
help="Weight quantization for the real PyTorch backend",
|
|
)
|
|
start_cmd.add_argument(
|
|
"--host", default="0.0.0.0", help="Interface to bind to"
|
|
)
|
|
start_cmd.add_argument(
|
|
"--advertise-host",
|
|
help="Reachable host/IP to advertise to the tracker (defaults to FQDN when binding 0.0.0.0)",
|
|
)
|
|
start_cmd.add_argument(
|
|
"--tracker-mode",
|
|
action="store_true",
|
|
help="Enable client-facing /v1/chat/completions (auto-enabled when shard-start=0)",
|
|
)
|
|
start_cmd.add_argument(
|
|
"--tracker-url",
|
|
default=None,
|
|
help="Tracker URL for route selection (used in tracker mode)",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.command == "start":
|
|
from meshnet_node.startup import run_startup
|
|
|
|
try:
|
|
node = run_startup(
|
|
tracker_url=args.tracker,
|
|
port=args.port,
|
|
model=args.model,
|
|
model_id=args.model_id,
|
|
shard_start=args.shard_start,
|
|
shard_end=args.shard_end,
|
|
quantization=args.quantization,
|
|
host=args.host,
|
|
advertise_host=args.advertise_host,
|
|
)
|
|
except Exception as exc:
|
|
print(f"ERROR: {exc}", file=sys.stderr, flush=True)
|
|
sys.exit(1)
|
|
try:
|
|
while True:
|
|
time.sleep(1)
|
|
except KeyboardInterrupt:
|
|
node.stop()
|
|
sys.exit(0)
|
|
else:
|
|
parser.print_help()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|