popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DL fix again

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-06 19:27:37 +03:00
parent b615acf582
commit 4f007aeef9
4 changed files with 142 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
packages/tracker/meshnet_tracker/cli.py
View File

@@ -1,8 +1,10 @@
"""meshnet-tracker CLI entry point."""
import argparse
import os
import sys
import time
from pathlib import Path
from .accounts import DEFAULT_ACCOUNTS_DB_PATH
from .billing import DEFAULT_BILLING_DB_PATH
@@ -17,7 +19,40 @@ from .server import (
DEFAULT_REGISTRY_DB_PATH = "meshnet_registry.sqlite3"
def _load_env_file(path: Path) -> None:
"""Load simple KEY=VALUE pairs from an env file without overriding env vars."""
if not path.exists():
return
try:
lines = path.read_text().splitlines()
except OSError:
return
for line in lines:
text = line.strip()
if not text or text.startswith("#"):
continue
if text.startswith("export "):
text = text[len("export "):].strip()
if "=" not in text:
continue
key, value = text.split("=", 1)
key = key.strip()
if not key or key in os.environ:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
value = value[1:-1]
os.environ[key] = value
def _load_env_defaults() -> None:
"""Load local and user-level tracker env defaults before parsing arguments."""
_load_env_file(Path.cwd() / ".env")
_load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env")
def main() -> None:
_load_env_defaults()
common = argparse.ArgumentParser(add_help=False)
common.add_argument("--host", default="0.0.0.0", help="Host interface to listen on")
common.add_argument("--port", type=int, default=8080, help="Port to listen on")
@@ -214,18 +249,18 @@ def main() -> None:
"enables GET /v1/pricing/hf/history)"
),
)
common.add_argument(
"--hf-pricing-refresh-interval",
type=float,
default=86400.0,
help="Seconds between dynamic pricing refresh passes (default: daily)",
)
common.add_argument(
"--models-dir",
default=None,
metavar="PATH",
help="Local HuggingFace snapshot root advertised as tracker model-file source (default: MESHNET_MODELS_DIR)",
)
common.add_argument(
"--hf-pricing-refresh-interval",
type=float,
default=86400.0,
help="Seconds between dynamic pricing refresh passes (default: daily)",
)
common.add_argument(
"--models-dir",
default=None,
metavar="PATH",
help="Local HuggingFace snapshot root advertised as tracker model-file source (default: MESHNET_MODELS_DIR)",
)
parser = argparse.ArgumentParser(
prog="meshnet-tracker",
@@ -282,9 +317,9 @@ def main() -> None:
args.hf_pricing_log_db
or (DEFAULT_HF_PRICING_LOG_DB_PATH if args.enable_hf_pricing else None)
),
hf_pricing_refresh_interval=args.hf_pricing_refresh_interval,
models_dir=args.models_dir,
)
hf_pricing_refresh_interval=args.hf_pricing_refresh_interval,
models_dir=args.models_dir,
)
port = server.start()
print(f"meshnet-tracker listening on http://{args.host}:{port}", flush=True)
try:

31
packages/tracker/meshnet_tracker/server.py
View File

@@ -3879,7 +3879,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Type", "application/x-tar")
self.send_header("X-Meshnet-Model-Source", "tracker")
self.end_headers()
with tarfile.open(fileobj=self.wfile, mode="w|") as archive:
# dereference: HF cache snapshots are symlinks into blobs/ — ship contents.
with tarfile.open(fileobj=self.wfile, mode="w|", dereference=True) as archive:
for rel in rel_files:
archive.add(snapshot_dir / rel, arcname=rel)
@@ -3924,6 +3925,22 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
]
if not hf_nodes:
# The caller is not registered yet — count its memory toward
# deployability, or the first node can never bootstrap the network.
caller = _NodeEntry(
node_id="assign-candidate",
endpoint="",
shard_start=None,
shard_end=None,
model=None,
shard_checksum=None,
hardware_profile={},
wallet_address=None,
score=1.0,
vram_bytes=vram_mb * 1024 * 1024,
ram_bytes=ram_mb * 1024 * 1024,
)
pool_with_caller = all_nodes + [caller]
resolved_name = None
preset = None
if filter_repo:
@@ -3932,7 +3949,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
deployable = [
(name, preset)
for name, preset in server.model_presets.items()
if preset.get("recommended") and _deployment_summary(all_nodes, preset)["deployable"]
if preset.get("recommended") and _deployment_summary(pool_with_caller, preset)["deployable"]
]
if deployable:
resolved_name, preset = deployable[0]
@@ -3951,7 +3968,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"num_layers": total_l,
"gap_found": True,
"price_per_token": 0.0,
"deployment": _deployment_summary(all_nodes, preset),
"deployment": _deployment_summary(pool_with_caller, preset),
"model_sources": self._model_sources(
str(resolved_name),
preset,
@@ -4340,7 +4357,13 @@ class TrackerServer:
self._enable_hf_pricing = enable_hf_pricing
self._hf_pricing_refresh_interval = hf_pricing_refresh_interval
self._hf_pricing_fetch_html = hf_pricing_fetch_html
raw_models_dir = models_dir if models_dir is not None else os.environ.get("MESHNET_MODELS_DIR")
# MESHNET_DOWNLOAD_DIR is the node-side model store; on a box running
# both tracker and node it doubles as the tracker's snapshot source.
raw_models_dir = (
models_dir
if models_dir is not None
else os.environ.get("MESHNET_MODELS_DIR") or os.environ.get("MESHNET_DOWNLOAD_DIR")
)
self._models_dir = Path(raw_models_dir).expanduser() if raw_models_dir else None
self._hf_pricing_stop = threading.Event()
self._hf_pricing_thread: threading.Thread | None = None