From 4f007aeef9a55f9a6804f2c42a7291b0282ae16e Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Mon, 6 Jul 2026 19:27:37 +0300 Subject: [PATCH] DL fix again --- .../46-tracker-env-and-first-node-autojoin.md | 63 ++++++++++++++++++ packages/node/meshnet_node/downloader.py | 3 +- packages/tracker/meshnet_tracker/cli.py | 65 ++++++++++++++----- packages/tracker/meshnet_tracker/server.py | 31 +++++++-- 4 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 docs/issues/46-tracker-env-and-first-node-autojoin.md diff --git a/docs/issues/46-tracker-env-and-first-node-autojoin.md b/docs/issues/46-tracker-env-and-first-node-autojoin.md new file mode 100644 index 0000000..b49e504 --- /dev/null +++ b/docs/issues/46-tracker-env-and-first-node-autojoin.md @@ -0,0 +1,63 @@ +# US-046 — Tracker .env awareness + first-node auto-join bootstrap + +Status: in progress +Priority: High (blocks the US-044 two-machine test; auto-join dead on fresh trackers) + +## Context + +Reported 2026-07-06: a node auto-joining a fresh tracker prints +`(auto-join unavailable: HTTP Error 503)` and then cannot download the model +from the tracker. Two independent causes: + +1. **Bootstrap chicken-and-egg.** `_handle_network_assign` handles an empty + registry by falling back to the first *deployable* recommended preset — but + deployability is computed from the registered pool only + (`_deployment_summary(all_nodes, preset)`), which is empty, so nothing is + ever deployable and the first node always gets 503. The caller's own + `vram_mb` / `ram_mb` (already sent in the query) are ignored. +2. **Tracker ignores `.env`.** `meshnet-node` loads `.env` (cwd) and + `~/.config/meshnet/secrets.env` at startup; `meshnet-tracker` does not. The + repo `.env` sets `MESHNET_DOWNLOAD_DIR=/run/media/popov/DATA/llm/safetensor/models`, + but the tracker only reads `--models-dir` / `MESHNET_MODELS_DIR`, so + `models_dir` stays unset → `/v1/model-files/download` returns + 404 "tracker model-file source is not enabled" and assignments carry no + `model_sources`. +3. **Symlink tars (found while verifying).** HF cache snapshots are symlink + farms into `blobs/`; both the tracker's `/v1/model-files/download` and the + node's `write_shard_archive` tarred the symlinks themselves + (`tarfile` default `dereference=False`), so receivers extracted dangling + links instead of weights. + +## Fix + +1. In the empty-registry branch of `_handle_network_assign`, synthesize a + candidate `_NodeEntry` from the caller's `vram_mb`/`ram_mb` query params and + include it in the pool used for the deployability gate (and the reported + `deployment` summary), so a recommended preset that fits *pool + caller* + bootstraps the network. +2. Tracker CLI loads env defaults the same way the node CLI does + (`.env` in cwd, then `~/.config/meshnet/secrets.env`, never overriding + already-set env vars). +3. `TrackerServer` models-dir resolution falls back + `--models-dir` → `MESHNET_MODELS_DIR` → `MESHNET_DOWNLOAD_DIR` (the node + store and tracker source are the same directory on a box running both). +4. Archive with `dereference=True` in both tar writers so model file contents + ship instead of snapshot symlinks. + +## Acceptance criteria + +- [x] Fresh tracker (empty registry) + caller with enough memory for a + recommended preset → `/v1/network/assign` returns 200 with that preset, + `gap_found=true`, and `model_sources` populated when the tracker holds a + local snapshot. (Verified live: 128 GB caller got qwen3.6-35b-a3b 0–39 + with tracker `model_sources`.) +- [x] Fresh tracker + caller too small for any recommended preset → still 503. + (Verified live with a 4 GB caller.) +- [x] `meshnet-tracker start` in a directory with `.env` setting + `MESHNET_DOWNLOAD_DIR` serves `/v1/model-files/download` from that dir with + no extra flags. (Verified live; tar entries are regular files, not + symlinks.) +- [x] Explicit `--models-dir` and `MESHNET_MODELS_DIR` still take precedence, + in that order. +- [x] `python -m pytest` passes from repo root (two known env-dependent + failures occur only while a live meshnet-node holds port 7000). diff --git a/packages/node/meshnet_node/downloader.py b/packages/node/meshnet_node/downloader.py index 5f44cc4..c57958d 100644 --- a/packages/node/meshnet_node/downloader.py +++ b/packages/node/meshnet_node/downloader.py @@ -37,7 +37,8 @@ def compute_shard_checksum(shard_dir: Path) -> str: def write_shard_archive(shard_dir: Path, out_file: Any) -> None: """Write a tar archive for *shard_dir* to a binary file-like object.""" - with tarfile.open(fileobj=out_file, mode="w|") as archive: + # dereference: HF cache snapshots are symlinks into blobs/ — ship contents. + with tarfile.open(fileobj=out_file, mode="w|", dereference=True) as archive: for path in sorted(p for p in shard_dir.rglob("*") if p.is_file()): archive.add(path, arcname=path.relative_to(shard_dir).as_posix()) diff --git a/packages/tracker/meshnet_tracker/cli.py b/packages/tracker/meshnet_tracker/cli.py index e14ba30..e6d7bdd 100644 --- a/packages/tracker/meshnet_tracker/cli.py +++ b/packages/tracker/meshnet_tracker/cli.py @@ -1,8 +1,10 @@ """meshnet-tracker CLI entry point.""" import argparse +import os import sys import time +from pathlib import Path from .accounts import DEFAULT_ACCOUNTS_DB_PATH from .billing import DEFAULT_BILLING_DB_PATH @@ -17,7 +19,40 @@ from .server import ( DEFAULT_REGISTRY_DB_PATH = "meshnet_registry.sqlite3" +def _load_env_file(path: Path) -> None: + """Load simple KEY=VALUE pairs from an env file without overriding env vars.""" + if not path.exists(): + return + try: + lines = path.read_text().splitlines() + except OSError: + return + for line in lines: + text = line.strip() + if not text or text.startswith("#"): + continue + if text.startswith("export "): + text = text[len("export "):].strip() + if "=" not in text: + continue + key, value = text.split("=", 1) + key = key.strip() + if not key or key in os.environ: + continue + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + os.environ[key] = value + + +def _load_env_defaults() -> None: + """Load local and user-level tracker env defaults before parsing arguments.""" + _load_env_file(Path.cwd() / ".env") + _load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env") + + def main() -> None: + _load_env_defaults() common = argparse.ArgumentParser(add_help=False) common.add_argument("--host", default="0.0.0.0", help="Host interface to listen on") common.add_argument("--port", type=int, default=8080, help="Port to listen on") @@ -214,18 +249,18 @@ def main() -> None: "enables GET /v1/pricing/hf/history)" ), ) - common.add_argument( - "--hf-pricing-refresh-interval", - type=float, - default=86400.0, - help="Seconds between dynamic pricing refresh passes (default: daily)", - ) - common.add_argument( - "--models-dir", - default=None, - metavar="PATH", - help="Local HuggingFace snapshot root advertised as tracker model-file source (default: MESHNET_MODELS_DIR)", - ) + common.add_argument( + "--hf-pricing-refresh-interval", + type=float, + default=86400.0, + help="Seconds between dynamic pricing refresh passes (default: daily)", + ) + common.add_argument( + "--models-dir", + default=None, + metavar="PATH", + help="Local HuggingFace snapshot root advertised as tracker model-file source (default: MESHNET_MODELS_DIR)", + ) parser = argparse.ArgumentParser( prog="meshnet-tracker", @@ -282,9 +317,9 @@ def main() -> None: args.hf_pricing_log_db or (DEFAULT_HF_PRICING_LOG_DB_PATH if args.enable_hf_pricing else None) ), - hf_pricing_refresh_interval=args.hf_pricing_refresh_interval, - models_dir=args.models_dir, - ) + hf_pricing_refresh_interval=args.hf_pricing_refresh_interval, + models_dir=args.models_dir, + ) port = server.start() print(f"meshnet-tracker listening on http://{args.host}:{port}", flush=True) try: diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index acd07e4..fbd724a 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -3879,7 +3879,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): self.send_header("Content-Type", "application/x-tar") self.send_header("X-Meshnet-Model-Source", "tracker") self.end_headers() - with tarfile.open(fileobj=self.wfile, mode="w|") as archive: + # dereference: HF cache snapshots are symlinks into blobs/ — ship contents. + with tarfile.open(fileobj=self.wfile, mode="w|", dereference=True) as archive: for rel in rel_files: archive.add(snapshot_dir / rel, arcname=rel) @@ -3924,6 +3925,22 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): ] if not hf_nodes: + # The caller is not registered yet — count its memory toward + # deployability, or the first node can never bootstrap the network. + caller = _NodeEntry( + node_id="assign-candidate", + endpoint="", + shard_start=None, + shard_end=None, + model=None, + shard_checksum=None, + hardware_profile={}, + wallet_address=None, + score=1.0, + vram_bytes=vram_mb * 1024 * 1024, + ram_bytes=ram_mb * 1024 * 1024, + ) + pool_with_caller = all_nodes + [caller] resolved_name = None preset = None if filter_repo: @@ -3932,7 +3949,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): deployable = [ (name, preset) for name, preset in server.model_presets.items() - if preset.get("recommended") and _deployment_summary(all_nodes, preset)["deployable"] + if preset.get("recommended") and _deployment_summary(pool_with_caller, preset)["deployable"] ] if deployable: resolved_name, preset = deployable[0] @@ -3951,7 +3968,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): "num_layers": total_l, "gap_found": True, "price_per_token": 0.0, - "deployment": _deployment_summary(all_nodes, preset), + "deployment": _deployment_summary(pool_with_caller, preset), "model_sources": self._model_sources( str(resolved_name), preset, @@ -4340,7 +4357,13 @@ class TrackerServer: self._enable_hf_pricing = enable_hf_pricing self._hf_pricing_refresh_interval = hf_pricing_refresh_interval self._hf_pricing_fetch_html = hf_pricing_fetch_html - raw_models_dir = models_dir if models_dir is not None else os.environ.get("MESHNET_MODELS_DIR") + # MESHNET_DOWNLOAD_DIR is the node-side model store; on a box running + # both tracker and node it doubles as the tracker's snapshot source. + raw_models_dir = ( + models_dir + if models_dir is not None + else os.environ.get("MESHNET_MODELS_DIR") or os.environ.get("MESHNET_DOWNLOAD_DIR") + ) self._models_dir = Path(raw_models_dir).expanduser() if raw_models_dir else None self._hf_pricing_stop = threading.Event() self._hf_pricing_thread: threading.Thread | None = None