59 lines
1.8 KiB
C++
59 lines
1.8 KiB
C++
// Private llama.cpp implementation of the native worker execution boundary.
|
|
//
|
|
// The gRPC service sees only this small project-owned surface. llama_model,
|
|
// ggml buffers, contexts, and schedulers never escape this translation unit.
|
|
#ifndef MESHNET_NATIVE_WORKER_LLAMA_SHARD_ENGINE_H_
|
|
#define MESHNET_NATIVE_WORKER_LLAMA_SHARD_ENGINE_H_
|
|
|
|
#include <cstdint>
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <string>
|
|
|
|
#include "shard_runtime.pb.h"
|
|
|
|
namespace meshnet::worker {
|
|
namespace sp = ::meshnet::shard::v1;
|
|
|
|
struct WorkerIdentity {
|
|
std::string artifact_path;
|
|
std::string artifact_digest;
|
|
std::string recipe_digest;
|
|
std::string recipe_id;
|
|
std::string recipe_version;
|
|
std::string catalogue_version;
|
|
uint32_t start_layer = 0;
|
|
uint32_t end_layer = 0; // half-open, as on the wire
|
|
uint32_t injected_death_after_executions = 0; // opt-in test hook; zero disables
|
|
};
|
|
|
|
struct BundleCheck {
|
|
std::optional<std::string> corrupt_detail;
|
|
std::optional<std::string> oversize_detail;
|
|
};
|
|
|
|
struct EngineHealth {
|
|
bool serving = false;
|
|
uint64_t resident_bytes = 0;
|
|
std::string detail;
|
|
};
|
|
|
|
class ShardEngine {
|
|
public:
|
|
virtual ~ShardEngine() = default;
|
|
virtual bool Load(std::string* error) = 0;
|
|
virtual BundleCheck Validate(const sp::TensorBundle&, uint64_t max_chunk_bytes) const = 0;
|
|
virtual bool Execute(const sp::TensorBundle&, std::string* error) = 0;
|
|
virtual const WorkerIdentity& identity() const = 0;
|
|
virtual EngineHealth health() const = 0;
|
|
virtual void ReleaseSession(const std::string& route_session_id) = 0;
|
|
virtual void Shutdown() = 0;
|
|
};
|
|
|
|
// Construction is the only native implementation entry point used by the
|
|
// worker. The returned ShardEngine owns all llama.cpp handles privately.
|
|
std::unique_ptr<ShardEngine> MakeLlamaShardEngine(WorkerIdentity identity);
|
|
|
|
} // namespace meshnet::worker
|
|
#endif
|