86 lines
2.8 KiB
C++
86 lines
2.8 KiB
C++
// Private llama.cpp implementation of the native worker execution boundary.
|
|
//
|
|
// The gRPC service sees only this small project-owned surface. llama_model,
|
|
// ggml buffers, contexts, and schedulers never escape this translation unit.
|
|
#ifndef MESHNET_NATIVE_WORKER_LLAMA_SHARD_ENGINE_H_
|
|
#define MESHNET_NATIVE_WORKER_LLAMA_SHARD_ENGINE_H_
|
|
|
|
#include <cstdint>
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <string>
|
|
|
|
#include "shard_runtime.pb.h"
|
|
|
|
namespace meshnet::worker {
|
|
namespace sp = ::meshnet::shard::v1;
|
|
|
|
struct WorkerIdentity {
|
|
std::string artifact_path;
|
|
std::string artifact_digest;
|
|
std::string recipe_digest;
|
|
std::string recipe_id;
|
|
std::string recipe_version;
|
|
std::string catalogue_version;
|
|
uint32_t start_layer = 0;
|
|
uint32_t end_layer = 0; // half-open, as on the wire
|
|
uint32_t injected_death_after_executions = 0; // opt-in test hook; zero disables
|
|
uint32_t hot_kv_max_sessions = 8;
|
|
uint32_t hot_kv_context_tokens = 4096;
|
|
uint32_t hot_kv_budget_tokens = 32768;
|
|
uint32_t hot_kv_ttl_seconds = 300;
|
|
};
|
|
|
|
struct BundleCheck {
|
|
std::optional<std::string> corrupt_detail;
|
|
std::optional<std::string> oversize_detail;
|
|
};
|
|
|
|
struct EngineHealth {
|
|
bool serving = false;
|
|
uint64_t resident_bytes = 0;
|
|
std::string detail;
|
|
};
|
|
|
|
// This is deliberately expressed in tokens, rather than guessed bytes: llama.cpp
|
|
// owns the actual K/V layout for the loaded range and backend. The worker uses
|
|
// the token reservation to keep its local KV arena bounded before a graph
|
|
// adapter materializes the typed boundary (DGR-039).
|
|
struct HotKvStep {
|
|
enum class Phase { kPrefill, kDecode };
|
|
std::string route_session_id;
|
|
uint64_t route_epoch = 0;
|
|
Phase phase = Phase::kPrefill;
|
|
uint64_t first_position = 0;
|
|
uint32_t token_count = 0;
|
|
uint64_t expected_past_len = 0;
|
|
};
|
|
|
|
enum class HotKvStatus { kOk, kCacheMiss, kStaleEpoch, kResourceExhausted, kCancelled };
|
|
|
|
struct HotKvResult {
|
|
HotKvStatus status = HotKvStatus::kOk;
|
|
uint64_t past_len = 0;
|
|
std::string detail;
|
|
};
|
|
|
|
class ShardEngine {
|
|
public:
|
|
virtual ~ShardEngine() = default;
|
|
virtual bool Load(std::string* error) = 0;
|
|
virtual BundleCheck Validate(const sp::TensorBundle&, uint64_t max_chunk_bytes) const = 0;
|
|
virtual HotKvResult OpenSession(const std::string& route_session_id, uint64_t route_epoch) = 0;
|
|
virtual HotKvResult Execute(const HotKvStep&, const sp::TensorBundle&, std::string* error) = 0;
|
|
virtual const WorkerIdentity& identity() const = 0;
|
|
virtual EngineHealth health() const = 0;
|
|
virtual void ReleaseSession(const std::string& route_session_id, uint64_t route_epoch) = 0;
|
|
virtual void Shutdown() = 0;
|
|
};
|
|
|
|
// Construction is the only native implementation entry point used by the
|
|
// worker. The returned ShardEngine owns all llama.cpp handles privately.
|
|
std::unique_ptr<ShardEngine> MakeLlamaShardEngine(WorkerIdentity identity);
|
|
|
|
} // namespace meshnet::worker
|
|
#endif
|