popov/neuron-tai
1
0
Fork 0
Code Issues 42 Pull Requests Actions Packages Projects Releases Wiki Activity

story: DGR-037 Bind llama.cpp to the standalone worker

Browse Source
This commit is contained in:
Dobromir Popov
2026-08-01 01:28:06 +03:00
parent dfa403adc6
commit 8217b4c4a2
9 changed files with 482 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
packages/node/native/worker/shard_service.cpp
View File

@@ -8,31 +8,18 @@ namespace meshnet::worker {
namespace {
// The exact identity this fixture worker serves. SessionOpen is validated
// against these — not echoed back from the caller — so an incompatible peer
// fails closed at open rather than being silently accepted with its own claimed
// identity. Kept in one place so GetCapability and the open handshake agree.
constexpr const char* kModelArtifactDigest = "sha256:native-test-artifact";
constexpr const char* kRuntimeRecipeDigest = "sha256:native-test-recipe";
constexpr const char* kRecipeId = "native-test";
constexpr const char* kRecipeVersion = "1";
constexpr const char* kCatalogueVersion = "1";
constexpr uint32_t kShardStartLayer = 0;
constexpr uint32_t kShardEndLayer = 32;
constexpr uint32_t kShardEffectiveStartLayer = 0;
void FillWorkerFingerprint(sp::Fingerprint* fp) {
fp->set_model_artifact_digest(kModelArtifactDigest);
fp->set_runtime_recipe_digest(kRuntimeRecipeDigest);
fp->set_recipe_id(kRecipeId);
fp->set_recipe_version(kRecipeVersion);
fp->set_catalogue_version(kCatalogueVersion);
void FillWorkerFingerprint(sp::Fingerprint* fp, const WorkerIdentity& identity) {
fp->set_model_artifact_digest(identity.artifact_digest);
fp->set_runtime_recipe_digest(identity.recipe_digest);
fp->set_recipe_id(identity.recipe_id);
fp->set_recipe_version(identity.recipe_version);
fp->set_catalogue_version(identity.catalogue_version);
}
void FillWorkerShardRange(sp::ShardRange* range) {
range->set_start_layer(kShardStartLayer);
range->set_end_layer(kShardEndLayer);
range->set_effective_start_layer(kShardEffectiveStartLayer);
void FillWorkerShardRange(sp::ShardRange* range, const WorkerIdentity& identity) {
range->set_start_layer(identity.start_layer);
range->set_end_layer(identity.end_layer);
range->set_effective_start_layer(identity.start_layer);
}
// Strictest-of-both bound: the smallest positive of `a`/`b`, or `fallback` when
@@ -89,12 +76,14 @@ grpc::Status ShardRuntimeServiceImpl::GetCapability(grpc::ServerContext*,
const sp::CapabilityRequest*,
sp::CapabilityReport* response) {
response->set_schema_version(sp::SCHEMA_VERSION_1);
FillWorkerFingerprint(response->mutable_fingerprint());
FillWorkerShardRange(response->mutable_shard_range());
response->set_backend("grpc-native-cpp");
const WorkerIdentity& identity = engine_.identity();
const EngineHealth health = engine_.health();
FillWorkerFingerprint(response->mutable_fingerprint(), identity);
FillWorkerShardRange(response->mutable_shard_range(), identity);
response->set_backend("llama.cpp");
response->set_device("cpu");
response->set_validated(true);
response->set_detail("bounded real forward passed for fixture artifact");
response->set_validated(health.serving);
response->set_detail(health.detail);
response->set_max_concurrent_sessions(8);
response->set_max_context_tokens(131072);
FillDefaultFlow(response->mutable_flow_control(), limits_);
@@ -107,13 +96,16 @@ grpc::Status ShardRuntimeServiceImpl::GetCapability(grpc::ServerContext*,
grpc::Status ShardRuntimeServiceImpl::Health(grpc::ServerContext*, const sp::HealthRequest*,
sp::HealthReport* response) {
response->set_schema_version(sp::SCHEMA_VERSION_1);
response->set_state(sp::SERVING_STATE_SERVING);
response->set_active_sessions(1);
const EngineHealth engine_health = engine_.health();
response->set_state(engine_health.serving ? sp::SERVING_STATE_SERVING : sp::SERVING_STATE_NOT_SERVING);
{ std::lock_guard<std::mutex> lk(sessions_mu_); response->set_active_sessions(sessions_.size()); }
response->set_queued_chunks(0);
response->set_batch_occupancy(0);
response->set_kv_pressure(0.0f);
response->set_resident_bytes(0);
response->set_detail("native fixture worker serving");
response->set_resident_bytes(engine_health.resident_bytes);
response->set_detail(engine_health.detail + "; loaded=" + engine_.identity().artifact_digest +
" range=[" + std::to_string(engine_.identity().start_layer) + "," +
std::to_string(engine_.identity().end_layer) + ")");
return grpc::Status::OK;
}
@@ -180,20 +172,18 @@ grpc::Status ShardRuntimeServiceImpl::Session(
}
const sp::Fingerprint& fp = open.fingerprint();
if ((!fp.model_artifact_digest().empty() &&
fp.model_artifact_digest() != kModelArtifactDigest) ||
fp.model_artifact_digest() != engine_.identity().artifact_digest) ||
(!fp.runtime_recipe_digest().empty() &&
fp.runtime_recipe_digest() != kRuntimeRecipeDigest)) {
fp.runtime_recipe_digest() != engine_.identity().recipe_digest)) {
reject_open(sp::ERROR_CODE_FINGERPRINT_MISMATCH,
"model artifact or runtime recipe digest does not match this worker");
return grpc::Status::OK;
}
if (open.has_shard_range()) {
const sp::ShardRange& r = open.shard_range();
const bool within = r.start_layer() >= kShardStartLayer &&
r.end_layer() <= kShardEndLayer &&
r.start_layer() < r.end_layer() &&
r.effective_start_layer() >= r.start_layer() &&
r.effective_start_layer() < r.end_layer();
const bool within = r.start_layer() == engine_.identity().start_layer &&
r.end_layer() == engine_.identity().end_layer &&
r.effective_start_layer() == engine_.identity().start_layer;
if (!within) {
reject_open(sp::ERROR_CODE_SHARD_RANGE_MISMATCH,
"requested layer range is not served by this worker");
@@ -241,7 +231,7 @@ grpc::Status ShardRuntimeServiceImpl::Session(
}
// Report the fingerprint the worker actually serves, so a mismatch is
// visible at open — never a copy of the caller's claimed identity.
FillWorkerFingerprint(accepted->mutable_fingerprint());
FillWorkerFingerprint(accepted->mutable_fingerprint(), engine_.identity());
stream->Write(response);
break;
}
@@ -294,8 +284,13 @@ grpc::Status ShardRuntimeServiceImpl::Session(
} else {
state->seen_steps.insert(step);
state->credits -= 1;
engine_.BoundedForward(chunk.bundle()); // real bounded forward over wire bytes
std::string execution_error;
if (!engine_.Execute(chunk.bundle(), &execution_error)) {
response = MakeFail(route_session_id, work_id, step, sp::ERROR_CODE_INTERNAL,
execution_error, false, true);
} else {
*response.mutable_chunk() = chunk; // echo the exact bundle back
}
}
}
}
@@ -355,7 +350,11 @@ grpc::Status ShardRuntimeServiceImpl::Session(
} else {
state->seen_steps.insert(step);
state->credits -= 1;
engine_.BoundedForward(bundle);
std::string execution_error;
if (!engine_.Execute(bundle, &execution_error)) {
response = MakeFail(route_session_id, work_id, step, sp::ERROR_CODE_INTERNAL,
execution_error, false, true);
} else {
// No decode response field exists; echo the step back as a
// chunk-bearing SessionResponse per the proto's relayed-frame design.
sp::ActivationChunk* out = response.mutable_chunk();
@@ -368,6 +367,7 @@ grpc::Status ShardRuntimeServiceImpl::Session(
pos->set_first_position(step_msg.position());
pos->set_token_count(1);
*out->mutable_bundle() = bundle;
}
}
}
}
@@ -413,6 +413,7 @@ grpc::Status ShardRuntimeServiceImpl::Session(
{
std::lock_guard<std::mutex> lk(sessions_mu_);
sessions_.erase(route_session_id);
engine_.ReleaseSession(route_session_id);
}
sp::SessionResponse response;
sp::ShardStatus* status = response.mutable_status();
@@ -454,6 +455,7 @@ grpc::Status ShardRuntimeServiceImpl::Release(grpc::ServerContext*,
{
std::lock_guard<std::mutex> lk(sessions_mu_);
existed = sessions_.erase(request->route_session_id()) != 0;
engine_.ReleaseSession(request->route_session_id());
}
response->set_released(existed);
return grpc::Status::OK;