feat: DGR-002 - Adopt the versioned gRPC Shard protocol

scripts/bootstrap_native_toolchain.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Build a protobuf C++ toolchain for the native Shard protocol.
+#
+# The Python side needs nothing beyond `pip install grpcio-tools` — it bundles
+# protoc. The C++ side needs libprotobuf headers and a protoc binary, and a
+# machine that has neither (no protobuf-devel, no cmake, no system protoc) can
+# still get a working one from source with this script. It is the exact recipe
+# DGR-002 used to build and run the C++ conformance test.
+#
+# gRPC C++ is deliberately NOT built here. The conformance test only needs
+# message types, so verifying the schema does not require the whole gRPC stack.
+# The worker (DGR-008) will need gRPC C++ and should extend this script then.
+#
+# Usage:
+#   scripts/bootstrap_native_toolchain.sh [install-prefix]
+#
+# Then:
+#   cmake -S packages/node/native -B build/native -DCMAKE_PREFIX_PATH=<prefix>
+#   cmake --build build/native -j
+#   ctest --test-dir build/native --output-on-failure
+
+set -euo pipefail
+
+PREFIX="${1:-${PWD}/build/native-toolchain}"
+WORK="$(mktemp -d)"
+trap 'rm -rf "${WORK}"' EXIT
+
+# Pinned: the C++ runtime a generated stub is compiled against must be a version
+# that stub is allowed to use, so these are exact, not floating.
+PROTOBUF_VERSION="33.1"
+ABSEIL_VERSION="20250814.1"
+
+command -v cmake >/dev/null || {
+  echo "cmake is required (pip install cmake==4.4.0)" >&2
+  exit 1
+}
+
+echo "--- fetching protobuf ${PROTOBUF_VERSION} and abseil ${ABSEIL_VERSION}"
+cd "${WORK}"
+curl -sfL -o protobuf.tar.gz \
+  "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-${PROTOBUF_VERSION}.tar.gz"
+tar xzf protobuf.tar.gz
+
+# The protobuf release tarball ships utf8_range but not abseil, and its default
+# CMake provider expects abseil as a submodule, so vendor it into place.
+curl -sfL -o abseil.tar.gz \
+  "https://github.com/abseil/abseil-cpp/releases/download/${ABSEIL_VERSION}/abseil-cpp-${ABSEIL_VERSION}.tar.gz"
+tar xzf abseil.tar.gz
+rm -rf "protobuf-${PROTOBUF_VERSION}/third_party/abseil-cpp"
+mv "abseil-cpp-${ABSEIL_VERSION}" "protobuf-${PROTOBUF_VERSION}/third_party/abseil-cpp"
+
+echo "--- building protobuf into ${PREFIX}"
+cmake -S "protobuf-${PROTOBUF_VERSION}" -B build \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX="${PREFIX}" \
+  -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+  -Dprotobuf_ABSL_PROVIDER=module \
+  -Dprotobuf_BUILD_TESTS=OFF \
+  -Dprotobuf_BUILD_SHARED_LIBS=OFF \
+  -DABSL_PROPAGATE_CXX_STD=ON
+cmake --build build -j"$(nproc)"
+cmake --install build
+
+echo "--- done"
+"${PREFIX}/bin/protoc" --version
+echo "configure the protocol build with: -DCMAKE_PREFIX_PATH=${PREFIX}"

scripts/generate_native_protocol.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Generate the Python Shard-protocol stubs from `shard_runtime.proto`.
+
+The `.proto` file is the contract; the Python modules under
+`meshnet_node/native_protocol/generated/` are build output that happens to be
+committed. They are committed so that installing the node does not require a
+protoc toolchain, and `--check` exists so a committed stub can never silently
+drift from the schema it claims to implement.
+
+Usage::
+
+    python scripts/generate_native_protocol.py           # regenerate in place
+    python scripts/generate_native_protocol.py --check   # fail if out of date
+
+C++ stubs are *not* generated here. They are build artifacts produced by CMake
+(`packages/node/native/CMakeLists.txt`) into the build tree, because a C++ build
+already requires a toolchain and nothing is gained by committing them.
+"""
+
+from __future__ import annotations
+
+import argparse
+import pathlib
+import shutil
+import subprocess
+import sys
+import tempfile
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
+PROTO_DIR = REPO_ROOT / "packages/node/native/proto"
+PROTO_FILE = PROTO_DIR / "shard_runtime.proto"
+OUT_DIR = REPO_ROOT / "packages/node/meshnet_node/native_protocol/generated"
+
+# Regenerating with a different protoc emits different gencode headers, so the
+# generator version is part of the contract and `--check` would catch a drift.
+REQUIRED_GRPCIO_TOOLS = "1.82.1"
+
+_HEADER = "# Generated by scripts/generate_native_protocol.py. Do not edit.\n"
+
+
+def _generate(into: pathlib.Path) -> None:
+    """Run protoc, writing generated modules into `into`."""
+    try:
+        from grpc_tools import protoc
+    except ImportError:  # pragma: no cover - exercised only without the toolchain
+        sys.exit(
+            "grpc_tools is required to generate stubs:\n"
+            f"    pip install grpcio-tools=={REQUIRED_GRPCIO_TOOLS}"
+        )
+
+    into.mkdir(parents=True, exist_ok=True)
+    # grpc_tools bundles protoc and the well-known types, so generation needs no
+    # system protoc and produces identical output on any machine.
+    well_known = pathlib.Path(protoc.__file__).parent / "_proto"
+    args = [
+        "protoc",
+        f"--proto_path={PROTO_DIR}",
+        f"--proto_path={well_known}",
+        f"--python_out={into}",
+        f"--pyi_out={into}",
+        f"--grpc_python_out={into}",
+        str(PROTO_FILE),
+    ]
+    if protoc.main(args) != 0:
+        sys.exit("protoc failed")
+
+    # protoc emits `import shard_runtime_pb2` — a bare top-level import that only
+    # resolves if the generated directory happens to be on sys.path. Rewrite it
+    # to a relative import so the package works as an installed package.
+    grpc_module = into / "shard_runtime_pb2_grpc.py"
+    text = grpc_module.read_text()
+    text = text.replace(
+        "import shard_runtime_pb2 as shard__runtime__pb2",
+        "from . import shard_runtime_pb2 as shard__runtime__pb2",
+    )
+    grpc_module.write_text(text)
+
+    (into / "__init__.py").write_text(
+        _HEADER + '"""Generated protobuf/gRPC stubs for the native Shard protocol."""\n'
+    )
+
+
+def _tracked_files(directory: pathlib.Path) -> dict[str, bytes]:
+    return {
+        path.name: path.read_bytes()
+        for path in sorted(directory.iterdir())
+        if path.is_file() and path.suffix in {".py", ".pyi"}
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="verify the committed stubs match the .proto instead of rewriting them",
+    )
+    args = parser.parse_args()
+
+    if args.check:
+        with tempfile.TemporaryDirectory() as tmp:
+            fresh = pathlib.Path(tmp) / "generated"
+            _generate(fresh)
+            if not OUT_DIR.is_dir():
+                print(f"generated stubs are missing: {OUT_DIR}", file=sys.stderr)
+                return 1
+            if _tracked_files(fresh) != _tracked_files(OUT_DIR):
+                print(
+                    "generated stubs are out of date with shard_runtime.proto.\n"
+                    "Run: python scripts/generate_native_protocol.py",
+                    file=sys.stderr,
+                )
+                return 1
+        print("generated stubs are up to date")
+        return 0
+
+    if OUT_DIR.exists():
+        shutil.rmtree(OUT_DIR)
+    _generate(OUT_DIR)
+    print(f"wrote {OUT_DIR.relative_to(REPO_ROOT)}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/generate_protocol_goldens.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Write the committed cross-language conformance vectors.
+
+The bytes under `packages/node/native/testdata/` are the reference both the
+Python and the C++ conformance tests assert against. They are committed so the
+C++ test can run without a Python step, and `--check` exists so they can never
+drift from the schema unnoticed: if a schema edit changes the canonical
+message's encoding, `--check` fails and the change has to be acknowledged.
+
+Usage::
+
+    python scripts/generate_protocol_goldens.py           # rewrite vectors
+    python scripts/generate_protocol_goldens.py --check    # fail if stale
+"""
+
+from __future__ import annotations
+
+import argparse
+import pathlib
+import sys
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "packages/node"))
+
+from meshnet_node.native_protocol import conformance  # noqa: E402
+
+
+def _vectors() -> dict[str, bytes]:
+    return {
+        conformance.GOLDEN_SESSION_REQUEST: conformance.serialize(
+            conformance.canonical_session_request()
+        ),
+        conformance.GOLDEN_CAPABILITY_REPORT: conformance.serialize(
+            conformance.canonical_capability_report()
+        ),
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true")
+    args = parser.parse_args()
+
+    out_dir = conformance.TESTDATA_DIR
+    out_dir.mkdir(parents=True, exist_ok=True)
+    stale = []
+
+    for name, payload in _vectors().items():
+        path = out_dir / name
+        if args.check:
+            if not path.is_file() or path.read_bytes() != payload:
+                stale.append(name)
+            continue
+        path.write_bytes(payload)
+        print(f"wrote {path.relative_to(REPO_ROOT)} ({len(payload)} bytes)")
+
+    if stale:
+        print(
+            "conformance vectors are stale: " + ", ".join(stale) + "\n"
+            "The canonical message no longer encodes to the committed bytes. If "
+            "that is intended, run: python scripts/generate_protocol_goldens.py",
+            file=sys.stderr,
+        )
+        return 1
+    if args.check:
+        print("conformance vectors are up to date")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())