inference working
This commit is contained in:
Dobromir Popov
@@ -1,83 +1,257 @@
|
||||
"""meshnet-node CLI entry point."""
|
||||
"""meshnet-node CLI entry point — mining-style UX."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _run_node(cfg: dict) -> None:
|
||||
"""Start the node and hand off to the live dashboard. Blocks until Ctrl-C."""
|
||||
from .startup import run_startup
|
||||
from .dashboard import run_dashboard
|
||||
|
||||
start_time = time.monotonic()
|
||||
try:
|
||||
node = run_startup(
|
||||
tracker_url=cfg["tracker_url"],
|
||||
port=cfg.get("port", 7000),
|
||||
model=cfg.get("model_name") or "stub-model",
|
||||
model_id=cfg.get("model_hf_repo") or None,
|
||||
shard_start=cfg.get("shard_start"),
|
||||
shard_end=cfg.get("shard_end"),
|
||||
quantization=cfg.get("quantization", "bfloat16").replace("bf16", "bfloat16"),
|
||||
wallet_path=Path(cfg["wallet_path"]) if cfg.get("wallet_path") else None,
|
||||
cache_dir=Path(cfg["download_dir"]) if cfg.get("download_dir") else None,
|
||||
host=cfg.get("host", "0.0.0.0"),
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"\nERROR: {exc}", file=sys.stderr, flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
run_dashboard(node, cfg, start_time)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
node.stop()
|
||||
req = getattr(node, "chat_completion_count", 0)
|
||||
elapsed = time.monotonic() - start_time
|
||||
h, rem = divmod(int(elapsed), 3600)
|
||||
m, s = divmod(rem, 60)
|
||||
print(
|
||||
f"\nmeshnet-node stopped. "
|
||||
f"Served {req} requests in {h:02d}:{m:02d}:{s:02d}.",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def _cmd_default(args) -> int:
|
||||
"""No subcommand: wizard if no config, else start with saved config."""
|
||||
from .config import load_config, save_config, merge_cli_overrides
|
||||
from .wizard import run_wizard
|
||||
|
||||
cfg = load_config()
|
||||
if cfg is None or args.reset_config:
|
||||
if args.reset_config and cfg is not None:
|
||||
print("Resetting config — re-running setup wizard.\n")
|
||||
try:
|
||||
cfg = run_wizard()
|
||||
except KeyboardInterrupt:
|
||||
print("\nSetup cancelled.")
|
||||
return 1
|
||||
save_config(cfg)
|
||||
print(f"\nConfig saved to ~/.config/meshnet/config.json\n")
|
||||
|
||||
# Apply CLI overrides on top of saved config
|
||||
overrides: dict = {}
|
||||
if args.model:
|
||||
overrides["model_hf_repo"] = args.model
|
||||
overrides["model_name"] = args.model.split("/")[-1]
|
||||
if args.quantization:
|
||||
overrides["quantization"] = args.quantization
|
||||
if args.download_dir:
|
||||
overrides["download_dir"] = args.download_dir
|
||||
if args.tracker:
|
||||
overrides["tracker_url"] = args.tracker
|
||||
if args.wallet:
|
||||
overrides["wallet_path"] = args.wallet
|
||||
if args.shard_start is not None:
|
||||
overrides["shard_start"] = args.shard_start
|
||||
if args.shard_end is not None:
|
||||
overrides["shard_end"] = args.shard_end
|
||||
if args.port is not None:
|
||||
overrides["port"] = args.port
|
||||
if args.host:
|
||||
overrides["host"] = args.host
|
||||
|
||||
if overrides:
|
||||
cfg = merge_cli_overrides(cfg, **overrides)
|
||||
|
||||
_run_node(cfg)
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_models(args) -> int:
|
||||
"""List curated models (with optional HF Hub browse)."""
|
||||
from .wizard import print_models_table, _browse_hf_interactive
|
||||
|
||||
if args.browse:
|
||||
from .model_catalog import browse_hf_hub
|
||||
print("Fetching HuggingFace Hub top models...\n")
|
||||
try:
|
||||
models = browse_hf_hub(top_n=20)
|
||||
print(f"{'#':<4} {'Repo':<60} {'Downloads':>12}")
|
||||
print(f"{'─'*4} {'─'*60} {'─'*12}")
|
||||
for i, m in enumerate(models, 1):
|
||||
dl = m["downloads"]
|
||||
dl_str = (
|
||||
f"{dl/1e6:.1f}M" if dl >= 1_000_000
|
||||
else f"{dl/1e3:.0f}k" if dl >= 1000
|
||||
else str(dl)
|
||||
)
|
||||
print(f"{i:<4} {m['repo']:<60} {dl_str:>12}")
|
||||
except RuntimeError as exc:
|
||||
print(f"Error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
else:
|
||||
print_models_table()
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_config(args) -> int:
|
||||
"""Print current config."""
|
||||
import json
|
||||
from .config import load_config, config_path
|
||||
|
||||
cfg = load_config()
|
||||
if cfg is None:
|
||||
print("No config file found. Run `meshnet-node` to start setup.")
|
||||
return 1
|
||||
print(f"Config: {config_path()}")
|
||||
print(json.dumps(cfg, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_start(args) -> int:
|
||||
"""Legacy `start` subcommand — preserves backward compatibility with existing tests."""
|
||||
from .config import load_config, DEFAULTS
|
||||
|
||||
# Build a transient config from flags (don't write to disk)
|
||||
cfg = dict(DEFAULTS)
|
||||
cfg["tracker_url"] = args.tracker
|
||||
cfg["port"] = args.port
|
||||
cfg["model_name"] = args.model
|
||||
cfg["quantization"] = args.quantization
|
||||
cfg["host"] = args.host
|
||||
if args.model_id:
|
||||
cfg["model_hf_repo"] = args.model_id
|
||||
if args.shard_start is not None:
|
||||
cfg["shard_start"] = args.shard_start
|
||||
if args.shard_end is not None:
|
||||
cfg["shard_end"] = args.shard_end
|
||||
if args.wallet:
|
||||
cfg["wallet_path"] = args.wallet
|
||||
if args.download_dir:
|
||||
cfg["download_dir"] = args.download_dir
|
||||
|
||||
# Legacy start: just run without the dashboard (keep original blocking loop)
|
||||
from .startup import run_startup
|
||||
|
||||
try:
|
||||
node = run_startup(
|
||||
tracker_url=cfg["tracker_url"],
|
||||
port=cfg["port"],
|
||||
model=cfg["model_name"],
|
||||
model_id=cfg.get("model_hf_repo"),
|
||||
shard_start=cfg.get("shard_start"),
|
||||
shard_end=cfg.get("shard_end"),
|
||||
quantization=cfg["quantization"].replace("bf16", "bfloat16"),
|
||||
wallet_path=Path(cfg["wallet_path"]) if cfg.get("wallet_path") else None,
|
||||
cache_dir=Path(cfg["download_dir"]) if cfg.get("download_dir") else None,
|
||||
host=cfg["host"],
|
||||
advertise_host=getattr(args, "advertise_host", None),
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr, flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
node.stop()
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="meshnet-node",
|
||||
description="Distributed Inference Network node client",
|
||||
description="Distributed AI Inference — Node Client",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=(
|
||||
"Run with no arguments to start the setup wizard.\n"
|
||||
"After first setup, `meshnet-node` starts using your saved config.\n\n"
|
||||
"Subcommands:\n"
|
||||
" models List supported models\n"
|
||||
" models --browse Browse HuggingFace Hub\n"
|
||||
" config Show current config\n"
|
||||
),
|
||||
)
|
||||
|
||||
# Flags that apply to the no-subcommand (default) path
|
||||
parser.add_argument("--model", metavar="HF_REPO", help="HuggingFace repo ID to serve")
|
||||
parser.add_argument("--quantization", "-q", choices=["bf16", "int8", "nf4", "bfloat16"],
|
||||
help="Quantization level")
|
||||
parser.add_argument("--download-dir", metavar="PATH", help="Model download directory")
|
||||
parser.add_argument("--tracker", metavar="URL", help="Tracker URL")
|
||||
parser.add_argument("--wallet", metavar="PATH", help="Wallet file path")
|
||||
parser.add_argument("--shard-start", type=int, metavar="N", help="Pin shard start layer")
|
||||
parser.add_argument("--shard-end", type=int, metavar="N", help="Pin shard end layer")
|
||||
parser.add_argument("--port", type=int, metavar="N", help="Port to listen on")
|
||||
parser.add_argument("--host", metavar="ADDR", help="Interface to bind (default 0.0.0.0)")
|
||||
parser.add_argument("--no-tui", action="store_true", help="Plain-text output (no rich dashboard)")
|
||||
parser.add_argument("--compact", action="store_true", help="Single-line status output")
|
||||
parser.add_argument("--reset-config", action="store_true", help="Re-run wizard even if config exists")
|
||||
|
||||
subparsers = parser.add_subparsers(dest="command")
|
||||
|
||||
start_cmd = subparsers.add_parser("start", help="Start the node server")
|
||||
start_cmd.add_argument(
|
||||
"--tracker", default="http://localhost:8080", help="Tracker URL"
|
||||
)
|
||||
start_cmd.add_argument("--port", type=int, default=7000, help="Port to listen on")
|
||||
start_cmd.add_argument(
|
||||
"--model", default="stub-model", help="Model preset to request from tracker"
|
||||
)
|
||||
start_cmd.add_argument(
|
||||
"--model-id",
|
||||
help="HuggingFace model id for the real PyTorch backend",
|
||||
)
|
||||
start_cmd.add_argument("--shard-start", type=int, help="First layer index for an explicit shard")
|
||||
start_cmd.add_argument("--shard-end", type=int, help="Exclusive layer end index for an explicit shard")
|
||||
start_cmd.add_argument(
|
||||
"--quantization",
|
||||
choices=["bfloat16", "int8", "nf4"],
|
||||
default="bfloat16",
|
||||
help="Weight quantization for the real PyTorch backend",
|
||||
)
|
||||
start_cmd.add_argument(
|
||||
"--host", default="0.0.0.0", help="Interface to bind to"
|
||||
)
|
||||
start_cmd.add_argument(
|
||||
"--advertise-host",
|
||||
help="Reachable host/IP to advertise to the tracker (defaults to FQDN when binding 0.0.0.0)",
|
||||
)
|
||||
start_cmd.add_argument(
|
||||
"--tracker-mode",
|
||||
action="store_true",
|
||||
help="Enable client-facing /v1/chat/completions (auto-enabled when shard-start=0)",
|
||||
)
|
||||
start_cmd.add_argument(
|
||||
"--tracker-url",
|
||||
default=None,
|
||||
help="Tracker URL for route selection (used in tracker mode)",
|
||||
)
|
||||
# models subcommand
|
||||
models_cmd = subparsers.add_parser("models", help="List supported models")
|
||||
models_cmd.add_argument("--browse", action="store_true", help="Browse HuggingFace Hub top-20")
|
||||
|
||||
# config subcommand
|
||||
subparsers.add_parser("config", help="Show current saved config")
|
||||
|
||||
# start subcommand (legacy / backward-compat)
|
||||
start_cmd = subparsers.add_parser("start", help="Start node (legacy flags)")
|
||||
start_cmd.add_argument("--tracker", default="http://localhost:8080")
|
||||
start_cmd.add_argument("--port", type=int, default=7000)
|
||||
start_cmd.add_argument("--model", default="stub-model")
|
||||
start_cmd.add_argument("--model-id", help="HuggingFace repo ID")
|
||||
start_cmd.add_argument("--shard-start", type=int)
|
||||
start_cmd.add_argument("--shard-end", type=int)
|
||||
start_cmd.add_argument("--quantization", choices=["bfloat16", "int8", "nf4", "bf16"], default="bfloat16")
|
||||
start_cmd.add_argument("--host", default="0.0.0.0")
|
||||
start_cmd.add_argument("--advertise-host")
|
||||
start_cmd.add_argument("--tracker-mode", action="store_true")
|
||||
start_cmd.add_argument("--tracker-url", default=None)
|
||||
start_cmd.add_argument("--wallet")
|
||||
start_cmd.add_argument("--download-dir")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == "start":
|
||||
from meshnet_node.startup import run_startup
|
||||
|
||||
try:
|
||||
node = run_startup(
|
||||
tracker_url=args.tracker,
|
||||
port=args.port,
|
||||
model=args.model,
|
||||
model_id=args.model_id,
|
||||
shard_start=args.shard_start,
|
||||
shard_end=args.shard_end,
|
||||
quantization=args.quantization,
|
||||
host=args.host,
|
||||
advertise_host=args.advertise_host,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr, flush=True)
|
||||
sys.exit(1)
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
node.stop()
|
||||
sys.exit(0)
|
||||
if args.command == "models":
|
||||
sys.exit(_cmd_models(args))
|
||||
elif args.command == "config":
|
||||
sys.exit(_cmd_config(args))
|
||||
elif args.command == "start":
|
||||
sys.exit(_cmd_start(args))
|
||||
else:
|
||||
parser.print_help()
|
||||
# Default: wizard or start with saved config
|
||||
sys.exit(_cmd_default(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user