[AURON #2392] Support Spark 4.2

.github/workflows/build-amd64-releases.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1]
+        sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1, spark-4.2]
         scalaver: [ 2.12, 2.13 ]
         javaver: [ 8, 21 ]
         auronver: [9.0.0-SNAPSHOT]
@@ -69,17 +69,24 @@ jobs:
             scalaver: '2.12'
           - sparkver: spark-4.1
             scalaver: '2.12'
+          - sparkver: spark-4.2
+            scalaver: '2.12'
           - sparkver: spark-4.0
             javaver: '8'
           - sparkver: spark-4.1
             javaver: '8'
+          - sparkver: spark-4.2
+            javaver: '8'
           # Spark 4.x JDK 21 builds are handled by explicit rockylinux8 include entries below.
           - sparkver: spark-4.0
             scalaver: '2.13'
             javaver: '21'
           - sparkver: spark-4.1
             scalaver: '2.13'
             javaver: '21'
+          - sparkver: spark-4.2
+            scalaver: '2.13'
+            javaver: '21'
         include:
           # Spark 4.x uses rockylinux8 image (with JDK 21)
           - sparkver: spark-4.0
@@ -94,6 +101,12 @@ jobs:
             auronver: 9.0.0-SNAPSHOT
             runner: ubuntu-24.04
             image: rockylinux8
+          - sparkver: spark-4.2
+            scalaver: '2.13'
+            javaver: '21'
+            auronver: 9.0.0-SNAPSHOT
+            runner: ubuntu-24.04
+            image: rockylinux8
 
     steps:
       - uses: actions/checkout@v7

.github/workflows/build-arm-releases.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1]
+        sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1, spark-4.2]
         auronver: [9.0.0-SNAPSHOT]
         scalaver: [2.12, 2.13]
         javaver: [ 8, 21 ]
@@ -69,10 +69,14 @@ jobs:
             scalaver: '2.12'
           - sparkver: spark-4.1
             scalaver: '2.12'
+          - sparkver: spark-4.2
+            scalaver: '2.12'
           - sparkver: spark-4.0
             javaver: '8'
           - sparkver: spark-4.1
             javaver: '8'
+          - sparkver: spark-4.2
+            javaver: '8'
 
     steps:
       - uses: actions/checkout@v7

.github/workflows/build-macos-releases.yml

@@ -37,7 +37,7 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1]
+        sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1, spark-4.2]
         auronver: [9.0.0-SNAPSHOT]
         scalaver: [2.12, 2.13]
         javaver: [8, 21]
@@ -69,10 +69,14 @@ jobs:
             scalaver: '2.12'
           - sparkver: spark-4.1
             scalaver: '2.12'
+          - sparkver: spark-4.2
+            scalaver: '2.12'
           - sparkver: spark-4.0
             javaver: '8'
           - sparkver: spark-4.1
             javaver: '8'
+          - sparkver: spark-4.2
+            javaver: '8'
 
     steps:
       - uses: actions/checkout@v7

.github/workflows/tpcds-reusable.yml

@@ -226,7 +226,7 @@ jobs:
         if: steps.cache-spark-bin.outputs.cache-hit != 'true'
         run: |
           SPARK_PATH="spark/spark-${{ steps.get-dependency-version.outputs.sparkversion }}"
-          if [[ ${{ inputs.scalaver }} = "2.13" && "${{ inputs.sparkver }}" != "spark-4.0" && "${{ inputs.sparkver }}" != "spark-4.1" ]]; then
+          if [[ ${{ inputs.scalaver }} = "2.13" && "${{ inputs.sparkver }}" != "spark-4.0" && "${{ inputs.sparkver }}" != "spark-4.1" && "${{ inputs.sparkver }}" != "spark-4.2" ]]; then
             SPARK_FILE="spark-${{ steps.get-dependency-version.outputs.sparkversion }}-bin-${{ inputs.hadoop-profile }}-scala${{ inputs.scalaver }}.tgz"
           else
             SPARK_FILE="spark-${{ steps.get-dependency-version.outputs.sparkversion }}-bin-${{ inputs.hadoop-profile }}.tgz"

.github/workflows/tpcds.yml

@@ -112,3 +112,13 @@ jobs:
       scalaver: '2.13'
       hadoop-profile: 'hadoop3'
       sparktests: 'true'
+
+  test-spark-42-jdk21-scala-2-13:
+    name: Test spark-4.2 JDK21 Scala-2.13
+    uses: ./.github/workflows/tpcds-reusable.yml
+    with:
+      sparkver: spark-4.2
+      javaver: '21'
+      scalaver: '2.13'
+      hadoop-profile: 'hadoop3'
+      sparktests: 'true'

auron-build.sh

@@ -30,7 +30,7 @@
 #   Define constants for supported component versions
 # -----------------------------------------------------------------------------
 SUPPORTED_OS_IMAGES=("centos7" "ubuntu24" "rockylinux8" "debian11" "azurelinux3")
-SUPPORTED_SPARK_VERSIONS=("3.0" "3.1" "3.2" "3.3" "3.4" "3.5" "4.0" "4.1")
+SUPPORTED_SPARK_VERSIONS=("3.0" "3.1" "3.2" "3.3" "3.4" "3.5" "4.0" "4.1" "4.2")
 SUPPORTED_SCALA_VERSIONS=("2.12" "2.13")
 SUPPORTED_CELEBORN_VERSIONS=("0.5" "0.6")
 # Currently only one supported version, but kept plural for consistency

auron-spark-tests/common/src/test/scala/org/apache/spark/sql/SparkExpressionTestsBase.scala

@@ -386,7 +386,7 @@ trait SparkExpressionTestsBase
     Column(expression)
   }
 
-  @sparkver("4.0 / 4.1")
+  @sparkver("4.0 / 4.1 / 4.2")
   private def columnFromExpression(expression: Expression): Column = {
     new Column(org.apache.spark.sql.classic.ExpressionColumnNode(expression))
   }
@@ -398,7 +398,7 @@ trait SparkExpressionTestsBase
     _spark.internalCreateDataFrame(rows, schema)
   }
 
-  @sparkver("4.0 / 4.1")
+  @sparkver("4.0 / 4.1 / 4.2")
   private def internalCreateDataFrame(
       rows: org.apache.spark.rdd.RDD[InternalRow],
       schema: StructType): org.apache.spark.sql.classic.DataFrame = {

auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronAllExecutionsPage.scala

@@ -32,7 +32,7 @@ private[ui] class AuronAllExecutionsPage(parent: AuronSQLTab) extends WebUIPage(
     UIUtils.headerSparkPage(request, "Auron", buildInfoSummary(sqlStore.buildInfo()), parent)
   }
 
-  @sparkver("4.0 / 4.1")
+  @sparkver("4.0 / 4.1 / 4.2")
   override def render(request: jakarta.servlet.http.HttpServletRequest): Seq[Node] = {
     UIUtils.headerSparkPage(request, "Auron", buildInfoSummary(sqlStore.buildInfo()), parent)
   }

pom.xml

@@ -1154,6 +1154,64 @@
       </build>
     </profile>
 
+    <profile>
+      <id>spark-4.2</id>
+      <properties>
+        <shimName>spark-4.2</shimName>
+        <scalaTestVersion>3.2.9</scalaTestVersion>
+        <sparkVersion>4.2.0</sparkVersion>
+        <parquetVersion>1.17.0</parquetVersion>
+        <shortSparkVersion>4.2</shortSparkVersion>
+        <nettyVersion>4.2.13.Final</nettyVersion>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <version>${maven-enforcer-plugin.version}</version>
+            <executions>
+              <execution>
+                <id>spark42-enforce-java-scala-version</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <!-- Spark 4.1 requires JDK 17+ and Scala 2.13.x -->
+                    <requireJavaVersion>
+                      <version>[17,)</version>
+                      <message>Spark 4.2 requires JDK 17 or higher. Current: ${java.version}</message>
+                    </requireJavaVersion>
+                    <requireProperty>
+                      <property>scalaLongVersion</property>
+                      <regex>2\.13\.\d+</regex>
+                      <regexMessage>Spark 4.2 requires Scala 2.13.x. Current: ${scalaLongVersion}</regexMessage>
+                    </requireProperty>
+                  </rules>
+                </configuration>
+              </execution>
+              <execution>
+                <id>spark42-disallow-iceberg</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireProperty>
+                      <property>icebergEnabled</property>
+                      <regex>false</regex>
+                      <regexMessage>Spark 4.2 is not supported for Iceberg. Use Spark 3.4-4.0 or disable icebergEnabled.</regexMessage>
+                    </requireProperty>
+                  </rules>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
     <profile>
       <id>jdk-8</id>
       <activation>

spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/InterceptedValidateSparkPlan.scala

@@ -25,7 +25,7 @@ import org.apache.auron.sparkver
 
 object InterceptedValidateSparkPlan extends Logging {
 
-  @sparkver("3.2 / 3.3 / 3.4 / 3.5 / 4.0 / 4.1")
+  @sparkver("3.2 / 3.3 / 3.4 / 3.5 / 4.0 / 4.1 / 4.2")
   def validate(plan: SparkPlan): Unit = {
     import org.apache.spark.sql.execution.adaptive.BroadcastQueryStageExec
     import org.apache.spark.sql.execution.auron.plan.NativeRenameColumnsBase
@@ -79,7 +79,7 @@ object InterceptedValidateSparkPlan extends Logging {
     throw new UnsupportedOperationException("validate is not supported in spark 3.0.3 or 3.1.3")
   }
 
-  @sparkver("3.2 / 3.3 / 3.4 / 3.5 / 4.0 / 4.1")
+  @sparkver("3.2 / 3.3 / 3.4 / 3.5 / 4.0 / 4.1 / 4.2")
   private def errorOnInvalidBroadcastQueryStage(plan: SparkPlan): Unit = {
     import org.apache.spark.sql.execution.adaptive.InvalidAQEPlanException
     throw InvalidAQEPlanException("Invalid broadcast query stage", plan)