feat: distributed GGUF shard load integration test with TinyLlama 1.1B

2026-07-14 13:01:51 +03:00
parent ec36290863
commit 31065c0e12
2 changed files with 301 additions and 0 deletions
--- a/packages/node/native/llama/meshnet-range-loader.cpp
+++ b/packages/node/native/llama/meshnet-range-loader.cpp
@@ -0,0 +1,68 @@
+/**
+ * meshnet-range-loader — CLI wrapper that loads a GGUF shard range
+ * and prints the meshnet range report as JSON.
+ *
+ * Usage: meshnet-range-loader <gguf-path> <start-layer> <end-layer>
+ *
+ * Build: g++ -std=c++17 -I<source>/include -L<build>/bin
+ *        meshnet-range-loader.cpp -lllama -o meshnet-range-loader
+ *        LD_LIBRARY_PATH=<build>/bin ./meshnet-range-loader ...
+ */
+
+#include "llama.h"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+int main(int argc, char **argv) {
+    if (argc != 4) {
+        fprintf(stderr, "Usage: %s <gguf-path> <start-layer> <end-layer>\n", argv[0]);
+        return 1;
+    }
+
+    const char *path = argv[1];
+    int start = std::atoi(argv[2]);
+    int end = std::atoi(argv[3]);
+
+    llama_backend_init();
+
+    llama_model_params params = llama_model_default_params();
+    params.meshnet_owned_layer_start = start;
+    params.meshnet_owned_layer_end = end;
+
+    llama_model *model = llama_model_load_from_file(path, params);
+    if (!model) {
+        fprintf(stderr, "ERROR model_load returned nullptr\n");
+        llama_backend_free();
+        return 1;
+    }
+
+    llama_meshnet_range_report report;
+    bool ok = llama_model_meshnet_range_report(model, &report);
+
+    int n_layer = llama_model_n_layer(model);
+    int n_embd = llama_model_n_embd(model);
+    uint64_t size = llama_model_size(model);
+
+    if (ok) {
+        fprintf(stdout,
+            "{"
+            "\"ok\":true,"
+            "\"start_layer\":%d,\"end_layer\":%d,"
+            "\"mapped_bytes\":%lu,\"resident_bytes\":%lu,"
+            "\"model_n_layer\":%d,\"model_n_embd\":%d,\"model_size\":%lu"
+            "}\n",
+            report.start_layer, report.end_layer,
+            (unsigned long)report.mapped_bytes,
+            (unsigned long)report.resident_bytes,
+            n_layer, n_embd, (unsigned long)size
+        );
+    } else {
+        fprintf(stderr, "ERROR range report not available\n");
+    }
+
+    llama_model_free(model);
+    llama_backend_free();
+    return ok ? 0 : 1;
+}