feat: support zstd compression in PBJ GRPC (#770)

anthony-swirldslabs · web-flow · commit c38d91b58163 · 2026-04-01T09:52:48.000-07:00
Signed-off-by: Anthony Petrov &lt;anthony@swirldslabs.com&gt;
diff --git a/README.md b/README.md
@@ -39,6 +39,7 @@ striving for.
     * ### [**Grpc Helidon** `pbj-grpc-helidon`](pbj-core/pbj-grpc-helidon/README.md)
     * ### [**Grpc Helidon Config** `pbj-grpc-helidon-config](pbj-core/pbj-grpc-helidon-config/README.md)
     * ### [**Grpc Client Helidon** `pbj-grpc-client-helidon`](pbj-core/pbj-grpc-client-helidon/README.md)
+    * ### [**Grpc Common Code** `pbj-grpc-common`](pbj-core/pbj-grpc-client-helidon/README.md)
   * ### [**Integration Tests** `pbj-integration-tests`](pbj-integration-tests/README.md) 
 
 ## Build Libraries
@@ -56,7 +57,5 @@ protobuf features and insure all the generated code is tested for hiero node use
 PBJ is a long term project with many goals. Here are some of the long term goals:
   * Support all Protobuf Features
   * Support new versions of Protobuf as possible
-  * Generate GRPC Services
-  * Built in GRPC-Web support
   * Auto mapping GRPC APIs to JSON REST APIs
   * JSON REST performance as good as GRPC
diff --git a/pbj-core/gradle/aggregation/build.gradle.kts b/pbj-core/gradle/aggregation/build.gradle.kts
@@ -2,4 +2,5 @@
 dependencies {
     published(project(":pbj-grpc-helidon"))
     published(project(":pbj-grpc-client-helidon"))
+    published(project(":pbj-grpc-common"))
 }
diff --git a/pbj-core/hiero-dependency-versions/build.gradle.kts b/pbj-core/hiero-dependency-versions/build.gradle.kts
@@ -55,7 +55,6 @@ dependencies.constraints {
         excludes.add("org.antlr:antlr4")
         excludes.add("com.google.protobuf:protoc")
         excludes.add("io.grpc:protoc-gen-grpc-java")
-        excludes.add("com.github.luben:zstd-jni")
         excludes.add("io.helidon.logging:helidon-logging-jul")
     }
 
diff --git a/pbj-core/pbj-grpc-client-helidon/src/main/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcClient.java b/pbj-core/pbj-grpc-client-helidon/src/main/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcClient.java
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 package com.hedera.pbj.grpc.client.helidon;
 
+import com.hedera.pbj.grpc.common.compression.ZstdGrpcTransformer;
 import com.hedera.pbj.runtime.Codec;
 import com.hedera.pbj.runtime.grpc.GrpcCall;
 import com.hedera.pbj.runtime.grpc.GrpcClient;
@@ -30,6 +31,10 @@
  * A PBJ GRPC client that uses the Helidon WebClient and its HTTP2 client implementation to call remote GRPC services.
  */
 public final class PbjGrpcClient implements GrpcClient, AutoCloseable {
+    static {
+        new ZstdGrpcTransformer().register("zstd");
+    }
+
     private final WebClient webClient;
     private final PbjGrpcClientConfig config;
 
diff --git a/pbj-core/pbj-grpc-client-helidon/src/main/java/module-info.java b/pbj-core/pbj-grpc-client-helidon/src/main/java/module-info.java
@@ -4,6 +4,7 @@
     requires transitive com.hedera.pbj.runtime;
     requires transitive io.helidon.common.tls;
     requires transitive io.helidon.webclient.api;
+    requires com.hedera.pbj.grpc.common;
     requires io.helidon.builder.api;
     requires io.helidon.common.buffers;
     requires io.helidon.common.socket;
diff --git a/pbj-core/pbj-grpc-common/README.md b/pbj-core/pbj-grpc-common/README.md
@@ -0,0 +1,9 @@
+# PBJ GRPC Helidon
+
+This project produces a module with a common code used by both PBJ gRPC client and server
+that may not be suitable for the generic and lightweight `pbj-runtime` module due to extra dependencies.
+Examples of such code include:
+- common classes that depend on Helidon libraries,
+- code with external, potentially heavy dependencies, such as compressor/decompressor implementations
+
+It produces `pbj-grpc-common-VERSION.jar`
diff --git a/pbj-core/pbj-grpc-common/build.gradle.kts b/pbj-core/pbj-grpc-common/build.gradle.kts
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: Apache-2.0
+plugins { id("org.hiero.gradle.module.library") }
+
+description = "A library with common code used by both PBJ gRPC client and server"
+
+testModuleInfo {
+    requires("org.junit.jupiter.api")
+    requiresStatic("java.annotation")
+}
diff --git a/pbj-core/pbj-grpc-common/src/main/java/com/hedera/pbj/grpc/common/compression/ZstdGrpcTransformer.java b/pbj-core/pbj-grpc-common/src/main/java/com/hedera/pbj/grpc/common/compression/ZstdGrpcTransformer.java
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
-package com.hedera.pbj.integration.jmh.grpc;
+package com.hedera.pbj.grpc.common.compression;
 
 import com.github.luben.zstd.ZstdInputStream;
 import com.github.luben.zstd.ZstdOutputStream;
diff --git a/pbj-core/pbj-grpc-common/src/main/java/com/hedera/pbj/grpc/common/compression/package-info.java b/pbj-core/pbj-grpc-common/src/main/java/com/hedera/pbj/grpc/common/compression/package-info.java
@@ -0,0 +1,5 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * An implementation of additional compressors and decompressors for the PBJ gRPC client and server.
+ */
+package com.hedera.pbj.grpc.common.compression;
diff --git a/pbj-core/pbj-grpc-common/src/main/java/module-info.java b/pbj-core/pbj-grpc-common/src/main/java/module-info.java
@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: Apache-2.0
+/** PBJ gRPC client implementation. */
+module com.hedera.pbj.grpc.common {
+    requires transitive com.hedera.pbj.runtime;
+    requires com.github.luben.zstd_jni;
+
+    exports com.hedera.pbj.grpc.common.compression;
+}
diff --git a/pbj-core/pbj-grpc-common/src/test/java/com/hedera/pbj/grpc/common/compression/ZstdGrpcTransformerTest.java b/pbj-core/pbj-grpc-common/src/test/java/com/hedera/pbj/grpc/common/compression/ZstdGrpcTransformerTest.java
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: Apache-2.0
+package com.hedera.pbj.grpc.common.compression;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import com.hedera.pbj.runtime.grpc.GrpcCompression;
+import com.hedera.pbj.runtime.io.buffer.Bytes;
+import java.util.Arrays;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class ZstdGrpcTransformerTest {
+    // A helper assertion that also prints entire arrays in addition to the default first mismatching index only
+    public static void assertArrayEquals(byte[] expected, byte[] actual) {
+        Assertions.assertArrayEquals(
+                expected,
+                actual,
+                () -> "Expected:\n" + Arrays.toString(expected) + "\nbut got:\n" + Arrays.toString(actual) + "\n");
+    }
+
+    @Test
+    void testRegistration() {
+        final String name = "ZstdGrpcTransformerTestRegistrationTest".toLowerCase();
+
+        assertFalse(GrpcCompression.getCompressorNames().contains(name));
+        assertFalse(GrpcCompression.getDecompressorNames().contains(name));
+
+        final ZstdGrpcTransformer zstdGrpcTransformer = new ZstdGrpcTransformer();
+        zstdGrpcTransformer.register(name);
+
+        assertEquals(zstdGrpcTransformer, GrpcCompression.getCompressor(name));
+        assertEquals(zstdGrpcTransformer, GrpcCompression.getDecompressor(name));
+    }
+
+    @Test
+    void testTransformation() {
+        final byte[] data = new byte[] {55, -37, 3, 0, 87, 57, 17};
+        final byte[] expectedCompressedData =
+                new byte[] {40, -75, 47, -3, 0, 88, 56, 0, 0, 55, -37, 3, 0, 87, 57, 17, 1, 0, 0};
+
+        final ZstdGrpcTransformer zstdGrpcTransformer = new ZstdGrpcTransformer();
+
+        final Bytes compressedData = zstdGrpcTransformer.compress(Bytes.wrap(data));
+        assertArrayEquals(expectedCompressedData, compressedData.toByteArray());
+
+        final Bytes decompressedData = zstdGrpcTransformer.decompress(compressedData);
+        assertArrayEquals(data, decompressedData.toByteArray());
+    }
+}
diff --git a/pbj-core/pbj-grpc-helidon/src/main/java/com/hedera/pbj/grpc/helidon/PbjProtocolHandler.java b/pbj-core/pbj-grpc-helidon/src/main/java/com/hedera/pbj/grpc/helidon/PbjProtocolHandler.java
@@ -13,6 +13,7 @@
 import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNull;
 
+import com.hedera.pbj.grpc.common.compression.ZstdGrpcTransformer;
 import com.hedera.pbj.grpc.helidon.config.PbjConfig;
 import com.hedera.pbj.runtime.grpc.GrpcCompression;
 import com.hedera.pbj.runtime.grpc.GrpcException;
@@ -64,6 +65,10 @@
  * created for each new connection, and each connection is made to a specific method endpoint.
  */
 final class PbjProtocolHandler implements Http2SubProtocolSelector.SubProtocolHandler {
+    static {
+        new ZstdGrpcTransformer().register("zstd");
+    }
+
     private static final System.Logger LOGGER = System.getLogger(PbjProtocolHandler.class.getName());
 
     private static final HeaderName GRPC_ENCODING = HeaderNames.createFromLowercase("grpc-encoding");
diff --git a/pbj-core/pbj-grpc-helidon/src/main/java/module-info.java b/pbj-core/pbj-grpc-helidon/src/main/java/module-info.java
@@ -18,6 +18,7 @@
     requires transitive io.helidon.config;
     requires transitive io.helidon.webserver.http2;
     requires transitive io.helidon.webserver;
+    requires com.hedera.pbj.grpc.common;
     requires io.helidon.common.buffers;
     requires io.helidon.common.media.type;
     requires io.helidon.common.socket;
diff --git a/pbj-core/pbj-grpc-helidon/src/test/java/com/hedera/pbj/grpc/helidon/PbjProtocolHandlerTest.java b/pbj-core/pbj-grpc-helidon/src/test/java/com/hedera/pbj/grpc/helidon/PbjProtocolHandlerTest.java
@@ -317,7 +317,7 @@ service, new PbjGrpcServiceConfig("identity", Set.of()), ServiceInterfaceStub.ME
      *
      * @param encoding the encoding to test with.
      */
-    @ValueSource(strings = {"compress", "deflate", "br", "zstd", "gzip, deflate;q=0.5"})
+    @ValueSource(strings = {"compress", "deflate", "br", "rar", "gzip, deflate;q=0.5"})
     @ParameterizedTest
     void unsupportedGrpcEncodings(String encoding) {
         final var h = WritableHeaders.create();
diff --git a/pbj-core/pbj-grpc-helidon/src/test/java/com/hedera/pbj/grpc/helidon/PbjTest.java b/pbj-core/pbj-grpc-helidon/src/test/java/com/hedera/pbj/grpc/helidon/PbjTest.java
@@ -266,7 +266,7 @@ void acceptEncodingExcludesAllSupportedEncodings() {
             try (var response = CLIENT.post()
                     .contentType(APPLICATION_GRPC_PROTO)
                     .path(SAY_HELLO_PATH)
-                    .header(HeaderNames.create("grpc-accept-encoding"), "deflate, zstd")
+                    .header(HeaderNames.create("grpc-accept-encoding"), "deflate, rar")
                     .submit(messageBytes(SIMPLE_REQUEST))) {
 
                 assertThat(response.status().code()).isEqualTo(200);
@@ -339,7 +339,7 @@ class CompressionTests {
          * unsupported compression schemes.
          */
         @ParameterizedTest
-        @ValueSource(strings = {"zstd", "deflate", "random"})
+        @ValueSource(strings = {"rar", "deflate", "random"})
         void compressionNotSupported(final String grpcEncoding) {
             try (var response = CLIENT.post()
                     .contentType(APPLICATION_GRPC_PROTO)
diff --git a/pbj-integration-tests/build.gradle.kts b/pbj-integration-tests/build.gradle.kts
@@ -53,10 +53,10 @@ testModuleInfo {
 
 jmhModuleInfo {
     requires("com.hedera.pbj.runtime")
+    requires("com.hedera.pbj.grpc.common")
     requires("com.google.protobuf.util")
     requires("io.helidon.common")
     requires("io.helidon.webserver")
-    requires("com.github.luben.zstd_jni")
     runtimeOnly("io.helidon.logging.jul")
 }
 
@@ -72,7 +72,7 @@ configurations.testRuntimeClasspath {
 
 // IMPROVE: Test code should not have a direct dependency to 'com.hedera.pbj.compiler'
 dependencies {
-    jmhImplementation("com.github.luben:zstd-jni")
+    jmhImplementation("com.hedera.pbj:pbj-grpc-common")
     testImplementation("com.hedera.pbj:pbj-compiler") { isTransitive = false }
 }
 
diff --git a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/grpc/PbjGrpcBench.java b/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/grpc/PbjGrpcBench.java
@@ -59,8 +59,6 @@ public class PbjGrpcBench {
     private static final int INVOCATIONS = 2_000;
 
     static {
-        new ZstdGrpcTransformer().register("zstd");
-
         // 1Gbps network:
         NetworkLatencySimulator.simulate(1_000, false);
     }
diff --git a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/grpc/block/TestBlockGrpcBench.java b/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/grpc/block/TestBlockGrpcBench.java
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: Apache-2.0
 package com.hedera.pbj.integration.jmh.grpc.block;
 
+import com.hedera.pbj.grpc.common.compression.ZstdGrpcTransformer;
 import com.hedera.pbj.grpc.helidon.PbjGrpcServiceConfig;
 import com.hedera.pbj.integration.grpc.GrpcTestUtils;
 import com.hedera.pbj.integration.grpc.PortsAllocator;
 import com.hedera.pbj.integration.jmh.grpc.NetworkLatencySimulator;
 import com.hedera.pbj.integration.jmh.grpc.PbjGrpcBench;
-import com.hedera.pbj.integration.jmh.grpc.ZstdGrpcTransformer;
 import com.hedera.pbj.runtime.Codec;
 import com.hedera.pbj.runtime.grpc.GrpcClient;
 import com.hedera.pbj.runtime.grpc.Pipeline;
@@ -61,7 +61,7 @@ public class TestBlockGrpcBench {
 
     static {
         new ZstdGrpcTransformer(-5).register("zstd-5");
-        new ZstdGrpcTransformer(3).register("zstd"); // the default level
+        // new ZstdGrpcTransformer(3).register("zstd"); // supported by PBJ by default
         new ZstdGrpcTransformer(10).register("zstd10");
 
         // 1Gbps network:

Original file line number	Diff line number	Diff line change
`@@ -2,4 +2,5 @@`
`2`	`2`	`dependencies {`
`3`	`3`	`published(project(":pbj-grpc-helidon"))`
`4`	`4`	`published(project(":pbj-grpc-client-helidon"))`
	`5`	`+ published(project(":pbj-grpc-common"))`
`5`	`6`	`}`
Original file line number	Diff line number	Diff line change
`@@ -55,7 +55,6 @@ dependencies.constraints {`
`55`	`55`	`excludes.add("org.antlr:antlr4")`
`56`	`56`	`excludes.add("com.google.protobuf:protoc")`
`57`	`57`	`excludes.add("io.grpc:protoc-gen-grpc-java")`
`58`		`- excludes.add("com.github.luben:zstd-jni")`
`59`	`58`	`excludes.add("io.helidon.logging:helidon-logging-jul")`
`60`	`59`	`}`
`61`	`60`