From ab113e111bd8562eb7dadad1cd9acf308a271c68 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sat, 18 Jul 2026 21:34:42 +0800 Subject: [PATCH 1/2] [SPARK-58202][K8S] Skip driver containerPort declaration when the port is 0. Signed-off-by: zhengchenyu --- .../k8s/features/BasicDriverFeatureStep.scala | 34 +++++++---------- .../BasicDriverFeatureStepSuite.scala | 38 +++++++++++++++++++ 2 files changed, 52 insertions(+), 20 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala index 579e5baeffdac..3201ab48e86e3 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala @@ -100,30 +100,24 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf) val driverUIPort = SparkUI.getUIPort(conf.sparkConf) val driverSparkConnectServerPort = conf.sparkConf.getInt(CONNECT_GRPC_BINDING_PORT, DEFAULT_SPARK_CONNECT_SERVER_PORT) + // 0 is invalid as kubernetes containerPort request, we shall leave it unmounted. + val driverContainerPorts = Seq( + DRIVER_PORT_NAME -> driverPort, + BLOCK_MANAGER_PORT_NAME -> driverBlockManagerPort, + UI_PORT_NAME -> driverUIPort, + SPARK_CONNECT_SERVER_PORT_NAME -> driverSparkConnectServerPort + ).collect { case (name, port) if port != 0 => + new ContainerPortBuilder() + .withName(name) + .withContainerPort(port) + .withProtocol("TCP") + .build() + } val driverContainer = new ContainerBuilder(pod.container) .withName(Option(pod.container.getName).getOrElse(DEFAULT_DRIVER_CONTAINER_NAME)) .withImage(driverContainerImage) .withImagePullPolicy(conf.imagePullPolicy) - .addNewPort() - .withName(DRIVER_PORT_NAME) - .withContainerPort(driverPort) - .withProtocol("TCP") - .endPort() - .addNewPort() - .withName(BLOCK_MANAGER_PORT_NAME) - .withContainerPort(driverBlockManagerPort) - .withProtocol("TCP") - .endPort() - .addNewPort() - .withName(UI_PORT_NAME) - .withContainerPort(driverUIPort) - .withProtocol("TCP") - .endPort() - .addNewPort() - .withName(SPARK_CONNECT_SERVER_PORT_NAME) - .withContainerPort(driverSparkConnectServerPort) - .withProtocol("TCP") - .endPort() + .addAllToPorts(driverContainerPorts.asJava) .addNewEnv() .withName(ENV_SPARK_USER) .withValue(Utils.getCurrentUserName()) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala index 70e2239205aef..fce22a28c6d0f 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala @@ -475,6 +475,44 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite { assert(amountAndFormat(limits("memory")) === "5500Mi") } + test("SPARK-58202: containerPort entries are skipped when the corresponding Spark port is 0") { + val sparkConf = new SparkConf() + .set(CONTAINER_IMAGE, "spark-driver:latest") + .set(KUBERNETES_DRIVER_POD_NAME, "spark-driver-pod") + .set(DRIVER_PORT, 0) + .set(DRIVER_BLOCK_MANAGER_PORT, 0) + .set(UI_PORT, 0) + .set(CONNECT_GRPC_BINDING_PORT, "0") + val kubernetesConf: KubernetesDriverConf = KubernetesTestConf.createDriverConf( + sparkConf = sparkConf, + environment = DRIVER_ENVS, + annotations = DRIVER_ANNOTATIONS) + + val featureStep = new BasicDriverFeatureStep(kubernetesConf) + val configuredPod = featureStep.configurePod(SparkPod.initialPod()) + assert(configuredPod.container.getPorts.isEmpty) + } + + test("SPARK-58202: containerPort entries include only ports with non-zero values") { + val sparkConf = new SparkConf() + .set(CONTAINER_IMAGE, "spark-driver:latest") + .set(KUBERNETES_DRIVER_POD_NAME, "spark-driver-pod") + .set(DRIVER_PORT, 9000) + .set(DRIVER_BLOCK_MANAGER_PORT, 0) + .set(UI_PORT, 4040) + .set(CONNECT_GRPC_BINDING_PORT, "0") + val kubernetesConf: KubernetesDriverConf = KubernetesTestConf.createDriverConf( + sparkConf = sparkConf, + environment = DRIVER_ENVS, + annotations = DRIVER_ANNOTATIONS) + + val featureStep = new BasicDriverFeatureStep(kubernetesConf) + val configuredPod = featureStep.configurePod(SparkPod.initialPod()) + val portsByName = configuredPod.container.getPorts.asScala + .map(cp => cp.getName -> cp.getContainerPort).toMap + assert(portsByName === Map(DRIVER_PORT_NAME -> 9000, UI_PORT_NAME -> 4040)) + } + def containerPort(name: String, portNumber: Int): ContainerPort = new ContainerPortBuilder() From 0d5274ed9a0e25dbdf9e60ca35e2f7b8bb512e63 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Sat, 18 Jul 2026 21:47:12 +0800 Subject: [PATCH 2/2] trigger ci Signed-off-by: zhengchenyu