[Backport 3.x] gRPC support for opensearch-java

.ci/opensearch/Dockerfile

@@ -6,4 +6,8 @@ ARG opensearch_yml=$opensearch_path/config/opensearch.yml
 
 ARG SECURE_INTEGRATION
 ENV OPENSEARCH_INITIAL_ADMIN_PASSWORD=0_aD^min_0
+
+# Copy custom opensearch.yml with gRPC transport configuration
+COPY opensearch.yml $opensearch_yml
+
 RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then echo "plugins.security.disabled: true" >> $opensearch_yml; fi

.ci/opensearch/docker-compose.yml

@@ -14,4 +14,5 @@ services:
       - bootstrap.memory_lock=true
     ports:
       - "9200:9200"
+      - "9400:9400"
     user: opensearch

.ci/opensearch/opensearch.yml

@@ -1,2 +1,6 @@
 cluster.name: "docker-cluster"
 network.host: 0.0.0.0
+
+# gRPC transport configuration
+aux.transport.types: [transport-grpc]
+aux.transport.transport-grpc.port: '9400'

.github/workflows/test-integration.yml

@@ -30,6 +30,8 @@ jobs:
           - { opensearch_version: 3.0.0, java: 21, os: ubuntu-latest }
           - { opensearch_version: 3.2.0, java: 21, os: ubuntu-latest }
           - { opensearch_version: 3.2.0, java: 25, os: ubuntu-latest }
+          - { opensearch_version: 3.5.0, java: 21, os: ubuntu-latest }
+          - { opensearch_version: 3.5.0, java: 25, os: ubuntu-latest }
     steps:
       - name: Checkout Java Client
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

CHANGELOG.md

@@ -6,6 +6,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Run Java client integration tests with a Testcontainers-managed OpenSearch instance by default ([#2033](https://github.com/opensearch-project/opensearch-java/pull/2033))
 - Added `equals()` and `hashCode()` implementations to `FieldValue` ([#1998](https://github.com/opensearch-project/opensearch-java/pull/1998))
 - Add document lifecycle guide and runnable sample ([#2017](https://github.com/opensearch-project/opensearch-java/pull/2017))
+- Add transparent gRPC transport with HybridTransport (bulk over gRPC, REST fallback), translation layer, TLS, basic auth, AWS SigV4, and JWT support ([#2062](https://github.com/opensearch-project/opensearch-java/pull/2062))
 
 ### Dependencies
 - Bump `org.apache.httpcomponents.client5:httpclient5` from 5.6 to 5.6.1 ([#1967](https://github.com/opensearch-project/opensearch-java/pull/1967))

java-client-grpc/build.gradle.kts

@@ -0,0 +1,152 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+plugins {
+    java
+    `java-library`
+    `maven-publish`
+    id("opensearch-java.spotless-conventions")
+}
+
+repositories {
+    mavenLocal()
+    maven(url = "https://ci.opensearch.org/ci/dbc/snapshots/maven/")
+    mavenCentral()
+}
+
+java {
+    targetCompatibility = JavaVersion.VERSION_1_8
+    sourceCompatibility = JavaVersion.VERSION_1_8
+
+    withJavadocJar()
+    withSourcesJar()
+}
+
+val opensearchVersion = "3.5.0"
+val grpcVersion = "1.68.0"
+val protobufVersion = "3.25.5"
+val opensearchProtobufVersion = "1.2.0"
+
+dependencies {
+    // Depend on java-client core
+    api(project(":java-client"))
+
+    // gRPC runtime
+    api("io.grpc", "grpc-api", grpcVersion)
+    api("io.grpc", "grpc-stub", grpcVersion)
+    api("io.grpc", "grpc-protobuf", grpcVersion)
+    api("io.grpc", "grpc-netty-shaded", grpcVersion)
+    api("com.google.protobuf", "protobuf-java", protobufVersion)
+
+    // OpenSearch Protobufs (compiled protobuf Java classes)
+    api("org.opensearch", "protobufs", opensearchProtobufVersion)
+
+    // For AwsSdk2 SigV4 support (optional, compile-only)
+    compileOnly("software.amazon.awssdk", "sdk-core", "[2.21,3.0)")
+    compileOnly("software.amazon.awssdk", "auth", "[2.21,3.0)")
+    compileOnly("software.amazon.awssdk", "http-auth-aws", "[2.21,3.0)")
+
+    // Test dependencies
+    testImplementation("io.grpc", "grpc-testing", grpcVersion)
+    testImplementation("junit", "junit", "4.13.2")
+    testImplementation("org.opensearch.client", "opensearch-rest-client", opensearchVersion)
+    testImplementation("software.amazon.awssdk", "sdk-core", "[2.21,3.0)")
+    testImplementation("software.amazon.awssdk", "auth", "[2.21,3.0)")
+    testImplementation("software.amazon.awssdk", "http-auth-aws", "[2.21,3.0)")
+}
+
+tasks.test {
+    systemProperty("tests.security.manager", "false")
+}
+
+val unitTest = tasks.register<Test>("unitTest") {
+    filter {
+        excludeTestsMatching("org.opensearch.client.opensearch.integTest.*")
+    }
+    systemProperty("tests.security.manager", "false")
+}
+
+val integrationTest = tasks.register<Test>("integrationTest") {
+    filter {
+        includeTestsMatching("org.opensearch.client.opensearch.integTest.*")
+    }
+    systemProperty("tests.security.manager", "false")
+    systemProperty("https", System.getProperty("https", "false"))
+    systemProperty("user", System.getProperty("user", "admin"))
+    systemProperty("password", System.getProperty("password", "admin"))
+    systemProperty("tests.opensearch.testcontainers.enabled",
+        System.getProperty("tests.opensearch.testcontainers.enabled", "true"))
+    systemProperty("tests.opensearch.version",
+        System.getProperty("tests.opensearch.version", opensearchVersion))
+}
+
+// Integration tests require Java 21+ and live in src/test/java11
+val runtimeJavaVersion = (System.getProperty("runtime.java")?.toInt())?.let(JavaVersion::toVersion) ?: JavaVersion.current()
+if (runtimeJavaVersion >= JavaVersion.VERSION_21) {
+    val java21: SourceSet = sourceSets.create("java21") {
+        java {
+            compileClasspath += sourceSets.main.get().output + sourceSets.test.get().output
+            runtimeClasspath += sourceSets.main.get().output + sourceSets.test.get().output
+            srcDir("src/test/java11")
+        }
+    }
+
+    configurations[java21.implementationConfigurationName].extendsFrom(configurations.testImplementation.get())
+    configurations[java21.runtimeOnlyConfigurationName].extendsFrom(configurations.testRuntimeOnly.get())
+
+    dependencies {
+        "java21Implementation"("org.opensearch.test", "framework", opensearchVersion) {
+            exclude(group = "org.hamcrest")
+        }
+        "java21Implementation"("org.opensearch:opensearch-testcontainers:4.1.0")
+        "java21Implementation"("org.testcontainers:testcontainers:2.0.5")
+    }
+
+    tasks.named<JavaCompile>("compileJava21Java") {
+        targetCompatibility = JavaVersion.VERSION_21.toString()
+        sourceCompatibility = JavaVersion.VERSION_21.toString()
+    }
+
+    tasks.named<JavaCompile>("compileTestJava") {
+        targetCompatibility = JavaVersion.VERSION_21.toString()
+        sourceCompatibility = JavaVersion.VERSION_21.toString()
+    }
+
+    integrationTest.configure {
+        testClassesDirs += java21.output.classesDirs
+        classpath = sourceSets["java21"].runtimeClasspath
+    }
+}
+
+tasks.withType<Jar> {
+    manifest {
+        attributes["Implementation-Title"] = "OpenSearch Java Client - gRPC Transport"
+        attributes["Implementation-Vendor"] = "OpenSearch"
+        attributes["Implementation-URL"] = "https://github.com/opensearch-project/opensearch-java/"
+    }
+
+    metaInf {
+        from("../LICENSE.txt")
+        from("../NOTICE.txt")
+    }
+}
+
+publishing {
+    publications {
+        create<MavenPublication>("publishMaven") {
+            from(components["java"])
+            pom {
+                name.set("OpenSearch Java Client - gRPC Transport")
+                packaging = "jar"
+                artifactId = "opensearch-java-grpc"
+                description.set("gRPC transport for the OpenSearch Java Client.")
+                url.set("https://github.com/opensearch-project/opensearch-java/")
+            }
+        }
+    }
+}

java-client-grpc/src/main/java/org/opensearch/client/transport/grpc/AwsGrpcTransport.java

@@ -0,0 +1,156 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.client.transport.grpc;
+
+import io.grpc.ManagedChannel;
+import javax.annotation.Nullable;
+import org.opensearch.client.json.JsonpMapper;
+import org.opensearch.client.transport.TransportOptions;
+
+/**
+ * AWS-specific gRPC transport that adds SigV4 signing to the gRPC channel.
+ * <p>
+ * This follows the same pattern as {@code AwsSdk2Transport} vs
+ * {@code ApacheHttpClient5Transport} — separating general transport from
+ * AWS-specific authentication.
+ * <p>
+ * Usage:
+ * <pre>{@code
+ * var grpcTransport = AwsGrpcTransport.builder("domain.us-east-1.es.amazonaws.com", 9400)
+ *     .jsonpMapper(new JacksonJsonpMapper())
+ *     .tls(GrpcTlsConfig.builder().build())
+ *     .sigV4(GrpcSigV4Config.builder()
+ *         .region(Region.US_EAST_1)
+ *         .service("es")
+ *         .credentialsProvider(DefaultCredentialsProvider.create())
+ *         .build())
+ *     .build();
+ * }</pre>
+ */
+public class AwsGrpcTransport extends GrpcTransport {
+
+    private final GrpcSigV4Interceptor sigV4Interceptor;
+
+    AwsGrpcTransport(
+        ManagedChannel channel,
+        JsonpMapper jsonpMapper,
+        GrpcTransportOptions grpcOptions,
+        @Nullable TransportOptions transportOptions,
+        GrpcSigV4Interceptor sigV4Interceptor
+    ) {
+        super(channel, jsonpMapper, grpcOptions, transportOptions);
+        this.sigV4Interceptor = sigV4Interceptor;
+    }
+
+    @Override
+    protected void preProcessBulk(org.opensearch.protobufs.BulkRequest protoRequest) {
+        if (sigV4Interceptor != null) {
+            String payloadHash = GrpcSigV4Interceptor.computePayloadHash(protoRequest.toByteArray());
+            sigV4Interceptor.setPayloadHash(payloadHash);
+        }
+    }
+
+    /**
+     * Creates a builder for AwsGrpcTransport.
+     *
+     * @param host the gRPC server hostname
+     * @param port the gRPC server port (default: 9400)
+     */
+    public static Builder awsBuilder(String host, int port) {
+        return new Builder(host, port);
+    }
+
+    public static final class Builder {
+        private final String host;
+        private final int port;
+        private JsonpMapper jsonpMapper;
+        private GrpcTransportOptions grpcOptions = GrpcTransportOptions.defaults();
+        private TransportOptions transportOptions;
+        private GrpcTlsConfig tlsConfig;
+        private GrpcSigV4Config sigV4Config;
+        private ManagedChannel channel;
+
+        Builder(String host, int port) {
+            this.host = host;
+            this.port = port;
+        }
+
+        public Builder jsonpMapper(JsonpMapper mapper) {
+            this.jsonpMapper = mapper;
+            return this;
+        }
+
+        public Builder grpcOptions(GrpcTransportOptions options) {
+            this.grpcOptions = options;
+            return this;
+        }
+
+        public Builder transportOptions(TransportOptions options) {
+            this.transportOptions = options;
+            return this;
+        }
+
+        /**
+         * Configures TLS for the gRPC channel. Required for SigV4.
+         */
+        public Builder tls(GrpcTlsConfig tlsConfig) {
+            this.tlsConfig = tlsConfig;
+            return this;
+        }
+
+        /**
+         * Configures AWS SigV4 signing for the gRPC channel.
+         * TLS is required when using SigV4.
+         *
+         * @param sigV4Config the SigV4 configuration (region, service, credentials)
+         */
+        public Builder sigV4(GrpcSigV4Config sigV4Config) {
+            this.sigV4Config = sigV4Config;
+            return this;
+        }
+
+        /**
+         * Inject a pre-built channel (primarily for testing).
+         */
+        public Builder channel(ManagedChannel channel) {
+            this.channel = channel;
+            return this;
+        }
+
+        public AwsGrpcTransport build() {
+            if (jsonpMapper == null) {
+                throw new IllegalArgumentException("jsonpMapper is required");
+            }
+            if (sigV4Config == null) {
+                throw new IllegalArgumentException("sigV4 config is required for AwsGrpcTransport. Use GrpcTransport for non-AWS usage.");
+            }
+            if (tlsConfig == null) {
+                throw new IllegalStateException("TLS is required when using SigV4 signing. Configure TLS with .tls() before .sigV4().");
+            }
+
+            ManagedChannel ch = this.channel;
+            GrpcSigV4Interceptor sigV4InterceptorRef = null;
+
+            if (ch == null) {
+                java.util.List<io.grpc.ClientInterceptor> interceptors = new java.util.ArrayList<>();
+
+                sigV4InterceptorRef = new GrpcSigV4Interceptor(sigV4Config, host);
+                interceptors.add(sigV4InterceptorRef);
+
+                try {
+                    ch = GrpcChannelFactory.createChannel(host, port, tlsConfig, grpcOptions, interceptors);
+                } catch (java.io.IOException e) {
+                    throw new IllegalStateException("Failed to create gRPC channel: " + e.getMessage(), e);
+                }
+            }
+
+            return new AwsGrpcTransport(ch, jsonpMapper, grpcOptions, transportOptions, sigV4InterceptorRef);
+        }
+    }
+}