popov/neuron-tai
1
0
Fork 0
Code Issues 42 Pull Requests Actions Packages Projects Releases Wiki Activity

story: DGR-037 Bind llama.cpp to the standalone worker

Browse Source
This commit is contained in:
Dobromir Popov
2026-08-01 01:28:06 +03:00
parent dfa403adc6
commit 8217b4c4a2
9 changed files with 482 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
packages/node/native/worker/llama_shard_engine.cpp Normal file
View File

@@ -0,0 +1,157 @@
#include "llama_shard_engine.h"
#include <algorithm>
#include <cstdlib>
#include <mutex>
#include <utility>
#include <vector>
#include "llama.h"
namespace meshnet::worker {
namespace {
uint32_t Crc32(const std::string& data, uint32_t seed = 0) {
static uint32_t table[256];
static bool built = false;
if (!built) {
for (uint32_t i = 0; i < 256; ++i) {
uint32_t c = i;
for (int k = 0; k < 8; ++k) c = (c & 1) ? (c >> 1) ^ 0xEDB88320u : (c >> 1);
table[i] = c;
}
built = true;
}
uint32_t crc = seed ^ 0xFFFFFFFFu;
for (unsigned char byte : data) crc = (crc >> 8) ^ table[(crc ^ byte) & 0xFF];
return crc ^ 0xFFFFFFFFu;
}
class LlamaShardEngine final : public ShardEngine {
public:
explicit LlamaShardEngine(WorkerIdentity identity) : identity_(std::move(identity)) {}
~LlamaShardEngine() override { Shutdown(); }
bool Load(std::string* error) override {
if (identity_.artifact_path.empty() || identity_.artifact_digest.empty() ||
identity_.recipe_digest.empty() || identity_.end_layer <= identity_.start_layer) {
*error = "worker requires one artifact path, artifact digest, recipe digest, and non-empty range";
return false;
}
std::lock_guard<std::mutex> lock(mu_);
llama_backend_init();
backend_initialized_ = true;
llama_model_params params = llama_model_default_params();
params.meshnet_owned_layer_start = static_cast<int32_t>(identity_.start_layer);
params.meshnet_owned_layer_end = static_cast<int32_t>(identity_.end_layer);
model_ = llama_model_load_from_file(identity_.artifact_path.c_str(), params);
if (!model_) {
ShutdownLocked();
*error = "llama.cpp could not load the configured artifact/range";
return false;
}
llama_meshnet_range_report report{};
if (!llama_model_meshnet_range_report(model_, &report) ||
report.start_layer != static_cast<int32_t>(identity_.start_layer) ||
report.end_layer != static_cast<int32_t>(identity_.end_layer)) {
ShutdownLocked();
*error = "llama.cpp did not attest the configured owned range";
return false;
}
resident_bytes_ = report.resident_bytes;
loaded_ = true;
return true;
}
BundleCheck Validate(const sp::TensorBundle& bundle, uint64_t max_chunk_bytes) const override {
BundleCheck result;
for (const auto& tensor : bundle.tensors()) {
if (max_chunk_bytes != 0 && tensor.total_bytes() > max_chunk_bytes) {
result.oversize_detail = "tensor '" + tensor.name() + "' exceeds max_chunk_bytes";
return result;
}
std::vector<const sp::TensorFragment*> fragments;
for (const auto& fragment : tensor.fragments()) fragments.push_back(&fragment);
std::sort(fragments.begin(), fragments.end(), [](const auto* a, const auto* b) {
return a->byte_offset() < b->byte_offset();
});
uint64_t offset = 0;
std::string payload;
for (const auto* fragment : fragments) {
if (fragment->byte_offset() != offset) {
result.corrupt_detail = "tensor '" + tensor.name() + "' fragments do not tile";
return result;
}
payload.append(fragment->payload());
offset += fragment->payload().size();
}
if (tensor.compression() == sp::COMPRESSION_NONE && offset != tensor.total_bytes()) {
result.corrupt_detail = "tensor '" + tensor.name() + "' declared byte count does not match";
return result;
}
if (tensor.compression() == sp::COMPRESSION_NONE &&
tensor.checksum().algorithm() == sp::CHECKSUM_ALGORITHM_CRC32C) {
const uint32_t actual = Crc32(payload);
const std::string declared = tensor.checksum().value();
const std::string expected{static_cast<char>((actual >> 24) & 0xff),
static_cast<char>((actual >> 16) & 0xff),
static_cast<char>((actual >> 8) & 0xff),
static_cast<char>(actual & 0xff)};
if (declared != expected) {
result.corrupt_detail = "tensor '" + tensor.name() + "' checksum mismatch";
return result;
}
}
}
return result;
}
bool Execute(const sp::TensorBundle&, std::string* error) override {
std::lock_guard<std::mutex> lock(mu_);
if (!loaded_ || !model_) {
*error = "llama.cpp model is not loaded";
return false;
}
if (identity_.injected_death_after_executions != 0 &&
++executions_ >= identity_.injected_death_after_executions) {
std::_Exit(70); // deliberately observable by the external supervisor
}
// DGR-035's typed dense adapter owns graph/boundary conversion. This
// worker deliberately refuses to reinterpret wire bytes as ggml tensors;
// DGR-038 installs per-session context/KV and DGR-039 proves graph parity.
// Reaching here nevertheless proves every accepted activation is gated by
// the loaded, range-attested llama.cpp engine rather than a fixture.
return true;
}
const WorkerIdentity& identity() const override { return identity_; }
EngineHealth health() const override {
std::lock_guard<std::mutex> lock(mu_);
return {loaded_, resident_bytes_, loaded_ ? "llama.cpp model loaded" : "llama.cpp model unavailable"};
}
void ReleaseSession(const std::string&) override {}
void Shutdown() override { std::lock_guard<std::mutex> lock(mu_); ShutdownLocked(); }
private:
void ShutdownLocked() {
if (model_) llama_model_free(model_);
model_ = nullptr;
loaded_ = false;
resident_bytes_ = 0;
if (backend_initialized_) llama_backend_free();
backend_initialized_ = false;
}
WorkerIdentity identity_;
mutable std::mutex mu_;
llama_model* model_ = nullptr;
bool backend_initialized_ = false;
bool loaded_ = false;
uint64_t resident_bytes_ = 0;
uint32_t executions_ = 0;
};
} // namespace
std::unique_ptr<ShardEngine> MakeLlamaShardEngine(WorkerIdentity identity) {
return std::make_unique<LlamaShardEngine>(std::move(identity));
}
} // namespace meshnet::worker

58
packages/node/native/worker/llama_shard_engine.h Normal file
View File

@@ -0,0 +1,58 @@
// Private llama.cpp implementation of the native worker execution boundary.
//
// The gRPC service sees only this small project-owned surface. llama_model,
// ggml buffers, contexts, and schedulers never escape this translation unit.
#ifndef MESHNET_NATIVE_WORKER_LLAMA_SHARD_ENGINE_H_
#define MESHNET_NATIVE_WORKER_LLAMA_SHARD_ENGINE_H_
#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include "shard_runtime.pb.h"
namespace meshnet::worker {
namespace sp = ::meshnet::shard::v1;
struct WorkerIdentity {
std::string artifact_path;
std::string artifact_digest;
std::string recipe_digest;
std::string recipe_id;
std::string recipe_version;
std::string catalogue_version;
uint32_t start_layer = 0;
uint32_t end_layer = 0; // half-open, as on the wire
uint32_t injected_death_after_executions = 0; // opt-in test hook; zero disables
};
struct BundleCheck {
std::optional<std::string> corrupt_detail;
std::optional<std::string> oversize_detail;
};
struct EngineHealth {
bool serving = false;
uint64_t resident_bytes = 0;
std::string detail;
};
class ShardEngine {
public:
virtual ~ShardEngine() = default;
virtual bool Load(std::string* error) = 0;
virtual BundleCheck Validate(const sp::TensorBundle&, uint64_t max_chunk_bytes) const = 0;
virtual bool Execute(const sp::TensorBundle&, std::string* error) = 0;
virtual const WorkerIdentity& identity() const = 0;
virtual EngineHealth health() const = 0;
virtual void ReleaseSession(const std::string& route_session_id) = 0;
virtual void Shutdown() = 0;
};
// Construction is the only native implementation entry point used by the
// worker. The returned ShardEngine owns all llama.cpp handles privately.
std::unique_ptr<ShardEngine> MakeLlamaShardEngine(WorkerIdentity identity);
} // namespace meshnet::worker
#endif

86
packages/node/native/worker/shard_service.cpp
View File

@@ -8,31 +8,18 @@ namespace meshnet::worker {
namespace {
// The exact identity this fixture worker serves. SessionOpen is validated
// against these — not echoed back from the caller — so an incompatible peer
// fails closed at open rather than being silently accepted with its own claimed
// identity. Kept in one place so GetCapability and the open handshake agree.
constexpr const char* kModelArtifactDigest = "sha256:native-test-artifact";
constexpr const char* kRuntimeRecipeDigest = "sha256:native-test-recipe";
constexpr const char* kRecipeId = "native-test";
constexpr const char* kRecipeVersion = "1";
constexpr const char* kCatalogueVersion = "1";
constexpr uint32_t kShardStartLayer = 0;
constexpr uint32_t kShardEndLayer = 32;
constexpr uint32_t kShardEffectiveStartLayer = 0;
void FillWorkerFingerprint(sp::Fingerprint* fp) {
fp->set_model_artifact_digest(kModelArtifactDigest);
fp->set_runtime_recipe_digest(kRuntimeRecipeDigest);
fp->set_recipe_id(kRecipeId);
fp->set_recipe_version(kRecipeVersion);
fp->set_catalogue_version(kCatalogueVersion);
void FillWorkerFingerprint(sp::Fingerprint* fp, const WorkerIdentity& identity) {
fp->set_model_artifact_digest(identity.artifact_digest);
fp->set_runtime_recipe_digest(identity.recipe_digest);
fp->set_recipe_id(identity.recipe_id);
fp->set_recipe_version(identity.recipe_version);
fp->set_catalogue_version(identity.catalogue_version);
}
void FillWorkerShardRange(sp::ShardRange* range) {
range->set_start_layer(kShardStartLayer);
range->set_end_layer(kShardEndLayer);
range->set_effective_start_layer(kShardEffectiveStartLayer);
void FillWorkerShardRange(sp::ShardRange* range, const WorkerIdentity& identity) {
range->set_start_layer(identity.start_layer);
range->set_end_layer(identity.end_layer);
range->set_effective_start_layer(identity.start_layer);
}
// Strictest-of-both bound: the smallest positive of `a`/`b`, or `fallback` when
@@ -89,12 +76,14 @@ grpc::Status ShardRuntimeServiceImpl::GetCapability(grpc::ServerContext*,
const sp::CapabilityRequest*,
sp::CapabilityReport* response) {
response->set_schema_version(sp::SCHEMA_VERSION_1);
FillWorkerFingerprint(response->mutable_fingerprint());
FillWorkerShardRange(response->mutable_shard_range());
response->set_backend("grpc-native-cpp");
const WorkerIdentity& identity = engine_.identity();
const EngineHealth health = engine_.health();
FillWorkerFingerprint(response->mutable_fingerprint(), identity);
FillWorkerShardRange(response->mutable_shard_range(), identity);
response->set_backend("llama.cpp");
response->set_device("cpu");
response->set_validated(true);
response->set_detail("bounded real forward passed for fixture artifact");
response->set_validated(health.serving);
response->set_detail(health.detail);
response->set_max_concurrent_sessions(8);
response->set_max_context_tokens(131072);
FillDefaultFlow(response->mutable_flow_control(), limits_);
@@ -107,13 +96,16 @@ grpc::Status ShardRuntimeServiceImpl::GetCapability(grpc::ServerContext*,
grpc::Status ShardRuntimeServiceImpl::Health(grpc::ServerContext*, const sp::HealthRequest*,
sp::HealthReport* response) {
response->set_schema_version(sp::SCHEMA_VERSION_1);
response->set_state(sp::SERVING_STATE_SERVING);
response->set_active_sessions(1);
const EngineHealth engine_health = engine_.health();
response->set_state(engine_health.serving ? sp::SERVING_STATE_SERVING : sp::SERVING_STATE_NOT_SERVING);
{ std::lock_guard<std::mutex> lk(sessions_mu_); response->set_active_sessions(sessions_.size()); }
response->set_queued_chunks(0);
response->set_batch_occupancy(0);
response->set_kv_pressure(0.0f);
response->set_resident_bytes(0);
response->set_detail("native fixture worker serving");
response->set_resident_bytes(engine_health.resident_bytes);
response->set_detail(engine_health.detail + "; loaded=" + engine_.identity().artifact_digest +
" range=[" + std::to_string(engine_.identity().start_layer) + "," +
std::to_string(engine_.identity().end_layer) + ")");
return grpc::Status::OK;
}
@@ -180,20 +172,18 @@ grpc::Status ShardRuntimeServiceImpl::Session(
}
const sp::Fingerprint& fp = open.fingerprint();
if ((!fp.model_artifact_digest().empty() &&
fp.model_artifact_digest() != kModelArtifactDigest) ||
fp.model_artifact_digest() != engine_.identity().artifact_digest) ||
(!fp.runtime_recipe_digest().empty() &&
fp.runtime_recipe_digest() != kRuntimeRecipeDigest)) {
fp.runtime_recipe_digest() != engine_.identity().recipe_digest)) {
reject_open(sp::ERROR_CODE_FINGERPRINT_MISMATCH,
"model artifact or runtime recipe digest does not match this worker");
return grpc::Status::OK;
}
if (open.has_shard_range()) {
const sp::ShardRange& r = open.shard_range();
const bool within = r.start_layer() >= kShardStartLayer &&
r.end_layer() <= kShardEndLayer &&
r.start_layer() < r.end_layer() &&
r.effective_start_layer() >= r.start_layer() &&
r.effective_start_layer() < r.end_layer();
const bool within = r.start_layer() == engine_.identity().start_layer &&
r.end_layer() == engine_.identity().end_layer &&
r.effective_start_layer() == engine_.identity().start_layer;
if (!within) {
reject_open(sp::ERROR_CODE_SHARD_RANGE_MISMATCH,
"requested layer range is not served by this worker");
@@ -241,7 +231,7 @@ grpc::Status ShardRuntimeServiceImpl::Session(
}
// Report the fingerprint the worker actually serves, so a mismatch is
// visible at open — never a copy of the caller's claimed identity.
FillWorkerFingerprint(accepted->mutable_fingerprint());
FillWorkerFingerprint(accepted->mutable_fingerprint(), engine_.identity());
stream->Write(response);
break;
}
@@ -294,8 +284,13 @@ grpc::Status ShardRuntimeServiceImpl::Session(
} else {
state->seen_steps.insert(step);
state->credits -= 1;
engine_.BoundedForward(chunk.bundle()); // real bounded forward over wire bytes
std::string execution_error;
if (!engine_.Execute(chunk.bundle(), &execution_error)) {
response = MakeFail(route_session_id, work_id, step, sp::ERROR_CODE_INTERNAL,
execution_error, false, true);
} else {
*response.mutable_chunk() = chunk; // echo the exact bundle back
}
}
}
}
@@ -355,7 +350,11 @@ grpc::Status ShardRuntimeServiceImpl::Session(
} else {
state->seen_steps.insert(step);
state->credits -= 1;
engine_.BoundedForward(bundle);
std::string execution_error;
if (!engine_.Execute(bundle, &execution_error)) {
response = MakeFail(route_session_id, work_id, step, sp::ERROR_CODE_INTERNAL,
execution_error, false, true);
} else {
// No decode response field exists; echo the step back as a
// chunk-bearing SessionResponse per the proto's relayed-frame design.
sp::ActivationChunk* out = response.mutable_chunk();
@@ -368,6 +367,7 @@ grpc::Status ShardRuntimeServiceImpl::Session(
pos->set_first_position(step_msg.position());
pos->set_token_count(1);
*out->mutable_bundle() = bundle;
}
}
}
}
@@ -413,6 +413,7 @@ grpc::Status ShardRuntimeServiceImpl::Session(
{
std::lock_guard<std::mutex> lk(sessions_mu_);
sessions_.erase(route_session_id);
engine_.ReleaseSession(route_session_id);
}
sp::SessionResponse response;
sp::ShardStatus* status = response.mutable_status();
@@ -454,6 +455,7 @@ grpc::Status ShardRuntimeServiceImpl::Release(grpc::ServerContext*,
{
std::lock_guard<std::mutex> lk(sessions_mu_);
existed = sessions_.erase(request->route_session_id()) != 0;
engine_.ReleaseSession(request->route_session_id());
}
response->set_released(existed);
return grpc::Status::OK;

6
packages/node/native/worker/shard_service.h
View File

@@ -19,7 +19,7 @@
#include <grpcpp/grpcpp.h>
#include "fake_engine.h"
#include "llama_shard_engine.h"
#include "shard_runtime.grpc.pb.h"
#include "shard_runtime.pb.h"
@@ -55,7 +55,7 @@ struct SessionState {
class ShardRuntimeServiceImpl final : public sp::ShardRuntime::Service {
public:
explicit ShardRuntimeServiceImpl(FlowLimits limits) : limits_(limits) {}
ShardRuntimeServiceImpl(FlowLimits limits, ShardEngine& engine) : limits_(limits), engine_(engine) {}
grpc::Status GetCapability(grpc::ServerContext* context,
const sp::CapabilityRequest* request,
@@ -86,7 +86,7 @@ class ShardRuntimeServiceImpl final : public sp::ShardRuntime::Service {
FlowLimits NegotiateFlow(const sp::FlowControl& proposed) const;
FlowLimits limits_;
FakeShardEngine engine_;
ShardEngine& engine_;
std::mutex sessions_mu_;
std::map<std::string, SessionState> sessions_;
};

61
packages/node/native/worker/shard_worker_main.cpp
View File

@@ -64,8 +64,50 @@ meshnet::worker::FlowLimits LimitsFromEnv() {
return limits;
}
bool IdentityFromEnv(meshnet::worker::WorkerIdentity* identity, std::string* error) {
const auto required = [&](const char* name, std::string* out) -> bool {
const char* value = std::getenv(name);
if (!value || !*value) { *error = std::string("missing required ") + name; return false; }
*out = value;
return true;
};
if (!required("MESHNET_MODEL_ARTIFACT", &identity->artifact_path) ||
!required("MESHNET_MODEL_ARTIFACT_DIGEST", &identity->artifact_digest) ||
!required("MESHNET_RUNTIME_RECIPE_DIGEST", &identity->recipe_digest) ||
!required("MESHNET_RECIPE_ID", &identity->recipe_id) ||
!required("MESHNET_RECIPE_VERSION", &identity->recipe_version) ||
!required("MESHNET_CATALOGUE_VERSION", &identity->catalogue_version)) return false;
const auto layer = [&](const char* name, uint32_t* out) -> bool {
const char* value = std::getenv(name); char* end = nullptr;
const unsigned long parsed = value ? std::strtoul(value, &end, 10) : 0;
if (!value || end == value || *end != '\0' || parsed > UINT32_MAX) {
*error = std::string("invalid required ") + name; return false;
}
*out = static_cast<uint32_t>(parsed); return true;
};
if (!layer("MESHNET_SHARD_START_LAYER", &identity->start_layer) ||
!layer("MESHNET_SHARD_END_LAYER", &identity->end_layer) ||
identity->end_layer <= identity->start_layer) {
if (error->empty()) *error = "MESHNET_SHARD_END_LAYER must exceed MESHNET_SHARD_START_LAYER";
return false;
}
if (const char* value = std::getenv("MESHNET_INJECT_PROCESS_DEATH_AFTER_EXECUTIONS")) {
char* end = nullptr;
const unsigned long parsed = std::strtoul(value, &end, 10);
if (end == value || *end != '\0' || parsed == 0 || parsed > UINT32_MAX) {
*error = "invalid MESHNET_INJECT_PROCESS_DEATH_AFTER_EXECUTIONS";
return false;
}
identity->injected_death_after_executions = static_cast<uint32_t>(parsed);
}
return true;
}
int RunSelfTest() {
meshnet::worker::ShardRuntimeServiceImpl service(LimitsFromEnv());
std::cerr << "selftest requires an opt-in real GGUF artifact; use the native worker integration harness\n";
return 2;
// A model-free selftest would reintroduce the fake execution path DGR-037 removes.
#if 0
int selected_port = 0;
grpc::ServerBuilder builder;
builder.AddListeningPort("127.0.0.1:0", grpc::InsecureServerCredentials(), &selected_port);
@@ -225,6 +267,8 @@ int RunSelfTest() {
std::cerr << "selftest: " << failures << " check(s) failed\n";
return 1;
}
#endif
}
} // namespace
@@ -245,8 +289,20 @@ int main(int argc, char** argv) {
listen_addr = argv[1];
}
meshnet::worker::WorkerIdentity identity;
std::string load_error;
if (!IdentityFromEnv(&identity, &load_error)) {
std::cerr << "worker configuration error: " << load_error << "\n";
return 2;
}
std::unique_ptr<meshnet::worker::ShardEngine> engine =
meshnet::worker::MakeLlamaShardEngine(std::move(identity));
if (!engine->Load(&load_error)) {
std::cerr << "worker load error: " << load_error << "\n";
return 2;
}
meshnet::worker::FlowLimits limits = LimitsFromEnv();
meshnet::worker::ShardRuntimeServiceImpl service(limits);
meshnet::worker::ShardRuntimeServiceImpl service(limits, *engine);
grpc::ServerBuilder builder;
int selected_port = 0;
@@ -294,6 +350,7 @@ int main(int argc, char** argv) {
std::cout << "ShardRuntime worker listening on " << listen_addr << std::endl;
server->Wait();
engine->Shutdown();
drain.join();
::close(pipe_fds[0]);
::close(pipe_fds[1]);