less verbose node hop if not in debug mode
This commit is contained in:
@@ -30,6 +30,7 @@ def _run_node(cfg: dict) -> None:
|
|||||||
advertise_host=cfg.get("advertise_host"),
|
advertise_host=cfg.get("advertise_host"),
|
||||||
route_timeout=float(cfg.get("route_timeout", 30.0)),
|
route_timeout=float(cfg.get("route_timeout", 30.0)),
|
||||||
vram_mb_override=cfg.get("vram_mb_override"),
|
vram_mb_override=cfg.get("vram_mb_override"),
|
||||||
|
debug=bool(cfg.get("debug", False)),
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
print(f"\nERROR: {exc}", file=sys.stderr, flush=True)
|
print(f"\nERROR: {exc}", file=sys.stderr, flush=True)
|
||||||
@@ -109,6 +110,8 @@ def _cmd_default(args) -> int:
|
|||||||
overrides["route_timeout"] = args.route_timeout
|
overrides["route_timeout"] = args.route_timeout
|
||||||
if getattr(args, "memory", None) is not None:
|
if getattr(args, "memory", None) is not None:
|
||||||
overrides["vram_mb_override"] = args.memory
|
overrides["vram_mb_override"] = args.memory
|
||||||
|
if args.debug:
|
||||||
|
overrides["debug"] = True
|
||||||
|
|
||||||
if overrides:
|
if overrides:
|
||||||
cfg = merge_cli_overrides(cfg, **overrides)
|
cfg = merge_cli_overrides(cfg, **overrides)
|
||||||
@@ -202,6 +205,7 @@ def _cmd_start(args) -> int:
|
|||||||
advertise_host=getattr(args, "advertise_host", None),
|
advertise_host=getattr(args, "advertise_host", None),
|
||||||
route_timeout=getattr(args, "route_timeout", 30.0),
|
route_timeout=getattr(args, "route_timeout", 30.0),
|
||||||
vram_mb_override=getattr(args, "memory", None),
|
vram_mb_override=getattr(args, "memory", None),
|
||||||
|
debug=getattr(args, "debug", False),
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
print(f"ERROR: {exc}", file=sys.stderr, flush=True)
|
print(f"ERROR: {exc}", file=sys.stderr, flush=True)
|
||||||
@@ -246,6 +250,7 @@ def main() -> None:
|
|||||||
help="Seconds to wait for tracker route lookup (default 30)")
|
help="Seconds to wait for tracker route lookup (default 30)")
|
||||||
parser.add_argument("--memory", type=int, metavar="MB", default=None,
|
parser.add_argument("--memory", type=int, metavar="MB", default=None,
|
||||||
help="Override autodetected VRAM/RAM budget in MB used for shard assignment")
|
help="Override autodetected VRAM/RAM budget in MB used for shard assignment")
|
||||||
|
parser.add_argument("--debug", action="store_true", help="Enable verbose node debug logging")
|
||||||
parser.add_argument("--no-tui", action="store_true", help="Plain-text output (no rich dashboard)")
|
parser.add_argument("--no-tui", action="store_true", help="Plain-text output (no rich dashboard)")
|
||||||
parser.add_argument("--compact", action="store_true", help="Single-line status output")
|
parser.add_argument("--compact", action="store_true", help="Single-line status output")
|
||||||
parser.add_argument("--reset-config", action="store_true", help="Re-run wizard even if config exists")
|
parser.add_argument("--reset-config", action="store_true", help="Re-run wizard even if config exists")
|
||||||
@@ -278,6 +283,7 @@ def main() -> None:
|
|||||||
help="Seconds to wait for tracker route lookup (default 30)")
|
help="Seconds to wait for tracker route lookup (default 30)")
|
||||||
start_cmd.add_argument("--memory", type=int, default=None, metavar="MB",
|
start_cmd.add_argument("--memory", type=int, default=None, metavar="MB",
|
||||||
help="Override autodetected VRAM/RAM budget in MB used for shard assignment")
|
help="Override autodetected VRAM/RAM budget in MB used for shard assignment")
|
||||||
|
start_cmd.add_argument("--debug", action="store_true", help="Enable verbose node debug logging")
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ DEFAULTS = {
|
|||||||
"shard_end": None,
|
"shard_end": None,
|
||||||
"port": 7000,
|
"port": 7000,
|
||||||
"host": "0.0.0.0",
|
"host": "0.0.0.0",
|
||||||
|
"debug": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -223,6 +223,7 @@ def run_startup(
|
|||||||
contracts: Any | None = None,
|
contracts: Any | None = None,
|
||||||
route_timeout: float = 30.0,
|
route_timeout: float = 30.0,
|
||||||
vram_mb_override: int | None = None,
|
vram_mb_override: int | None = None,
|
||||||
|
debug: bool = False,
|
||||||
) -> StubNodeServer | TorchNodeServer:
|
) -> StubNodeServer | TorchNodeServer:
|
||||||
"""Execute the full startup sequence and return a running node server.
|
"""Execute the full startup sequence and return a running node server.
|
||||||
|
|
||||||
@@ -320,6 +321,7 @@ def run_startup(
|
|||||||
quantization=quantization,
|
quantization=quantization,
|
||||||
tracker_url=tracker_url,
|
tracker_url=tracker_url,
|
||||||
route_timeout=route_timeout,
|
route_timeout=route_timeout,
|
||||||
|
debug=debug,
|
||||||
)
|
)
|
||||||
_node_start_time = time.monotonic()
|
_node_start_time = time.monotonic()
|
||||||
actual_port = node.start()
|
actual_port = node.start()
|
||||||
@@ -414,6 +416,7 @@ def run_startup(
|
|||||||
quantization=quantization,
|
quantization=quantization,
|
||||||
tracker_url=tracker_url,
|
tracker_url=tracker_url,
|
||||||
route_timeout=route_timeout,
|
route_timeout=route_timeout,
|
||||||
|
debug=debug,
|
||||||
)
|
)
|
||||||
_node_start_time = time.monotonic()
|
_node_start_time = time.monotonic()
|
||||||
actual_port = node.start()
|
actual_port = node.start()
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ class _TorchHTTPServer(http.server.HTTPServer):
|
|||||||
tracker_mode: bool = False,
|
tracker_mode: bool = False,
|
||||||
tracker_url: str | None = None,
|
tracker_url: str | None = None,
|
||||||
route_timeout: float = 30.0,
|
route_timeout: float = 30.0,
|
||||||
|
debug: bool = False,
|
||||||
):
|
):
|
||||||
super().__init__(addr, handler)
|
super().__init__(addr, handler)
|
||||||
self.backend = backend
|
self.backend = backend
|
||||||
@@ -46,6 +47,7 @@ class _TorchHTTPServer(http.server.HTTPServer):
|
|||||||
self.tracker_mode = tracker_mode
|
self.tracker_mode = tracker_mode
|
||||||
self.tracker_url = tracker_url
|
self.tracker_url = tracker_url
|
||||||
self.route_timeout = route_timeout
|
self.route_timeout = route_timeout
|
||||||
|
self.debug = debug
|
||||||
self.total_requests: int = 0
|
self.total_requests: int = 0
|
||||||
self.failed_requests: int = 0
|
self.failed_requests: int = 0
|
||||||
self.queue_depth: int = 0
|
self.queue_depth: int = 0
|
||||||
@@ -406,7 +408,8 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
current_pos = pos_ids
|
current_pos = pos_ids
|
||||||
|
|
||||||
for hop_index, (node_url, start_layer) in enumerate(route):
|
for hop_index, (node_url, start_layer) in enumerate(route):
|
||||||
print(f" [node] pipeline hop {hop_index}: {node_url} start_layer={start_layer}", flush=True)
|
if server.debug:
|
||||||
|
print(f" [node] pipeline hop {hop_index}: {node_url} start_layer={start_layer}", flush=True)
|
||||||
headers: dict[str, str] = {
|
headers: dict[str, str] = {
|
||||||
"Content-Type": "application/octet-stream",
|
"Content-Type": "application/octet-stream",
|
||||||
"X-Meshnet-Wire": _WIRE_VERSION,
|
"X-Meshnet-Wire": _WIRE_VERSION,
|
||||||
@@ -440,7 +443,8 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
try:
|
try:
|
||||||
data = json.loads(resp_body)
|
data = json.loads(resp_body)
|
||||||
text = str(data.get("text", ""))
|
text = str(data.get("text", ""))
|
||||||
print(f" [node] pipeline hop {hop_index} returned text={text!r}", flush=True)
|
if server.debug:
|
||||||
|
print(f" [node] pipeline hop {hop_index} returned text={text!r}", flush=True)
|
||||||
return text
|
return text
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
return resp_body.decode("utf-8", errors="replace")
|
return resp_body.decode("utf-8", errors="replace")
|
||||||
@@ -611,6 +615,7 @@ class TorchNodeServer:
|
|||||||
tracker_mode: bool | None = None,
|
tracker_mode: bool | None = None,
|
||||||
tracker_url: str | None = None,
|
tracker_url: str | None = None,
|
||||||
route_timeout: float = 30.0,
|
route_timeout: float = 30.0,
|
||||||
|
debug: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._host = host
|
self._host = host
|
||||||
self._requested_port = port
|
self._requested_port = port
|
||||||
@@ -624,6 +629,7 @@ class TorchNodeServer:
|
|||||||
self._tracker_mode = tracker_mode if tracker_mode is not None else (shard_start == 0)
|
self._tracker_mode = tracker_mode if tracker_mode is not None else (shard_start == 0)
|
||||||
self._tracker_url = tracker_url
|
self._tracker_url = tracker_url
|
||||||
self._route_timeout = route_timeout
|
self._route_timeout = route_timeout
|
||||||
|
self._debug = debug
|
||||||
self._server: _TorchHTTPServer | None = None
|
self._server: _TorchHTTPServer | None = None
|
||||||
self._thread: threading.Thread | None = None
|
self._thread: threading.Thread | None = None
|
||||||
self.port: int | None = None
|
self.port: int | None = None
|
||||||
@@ -666,6 +672,7 @@ class TorchNodeServer:
|
|||||||
self._tracker_mode,
|
self._tracker_mode,
|
||||||
self._tracker_url,
|
self._tracker_url,
|
||||||
self._route_timeout,
|
self._route_timeout,
|
||||||
|
self._debug,
|
||||||
)
|
)
|
||||||
self.port = self._server.server_address[1]
|
self.port = self._server.server_address[1]
|
||||||
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
|
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
|
||||||
|
|||||||
Reference in New Issue
Block a user