chore: Update Java distribution and enhance Spark configurations

- Changed Java distribution from Oracle to Temurin in build and check workflows for improved compatibility.
- Added "spark.executor.processTreeMetrics.enabled" configuration to multiple files to enhance Spark metrics handling.
- Ensured consistency across application and test configurations by aligning the settings in Constants.scala, application.conf, and application-integration.conf.

Author: pflooky

.github/workflows/build.yml

@@ -35,7 +35,7 @@ jobs:
           java-version: '17'
           java-package: jdk
           architecture: x64
-          distribution: oracle
+          distribution: temurin
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:

.github/workflows/check.yml

@@ -15,7 +15,7 @@ jobs:
           java-version: '17'
           java-package: jdk
           architecture: x64
-          distribution: oracle
+          distribution: temurin
       - name: Gradle build with cache
         uses: burrunan/gradle-cache-action@v1
         with:

api/src/main/scala/io/github/datacatering/datacaterer/api/model/Constants.scala

@@ -289,7 +289,8 @@ object Constants {
     "spark.hadoop.fs.hdfs.impl" -> "org.apache.hadoop.hdfs.DistributedFileSystem",
     "spark.hadoop.fs.file.impl" -> "com.globalmentor.apache.hadoop.fs.BareLocalFileSystem",
     "spark.sql.extensions" -> "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
-    "spark.metrics.executorMetricsSource.enabled" -> "false"
+    "spark.metrics.executorMetricsSource.enabled" -> "false",
+    "spark.executor.processTreeMetrics.enabled" -> "false"
   )
 
   //jdbc defaults

app/src/test/resources/application-integration.conf

@@ -110,7 +110,10 @@ runtime {
         "spark.hadoop.fs.s3a.bucket.all.committer.magic.enabled" = "true",
         "spark.hadoop.fs.hdfs.impl" = "org.apache.hadoop.hdfs.DistributedFileSystem",
         "spark.hadoop.fs.file.impl" = "com.globalmentor.apache.hadoop.fs.BareLocalFileSystem",
-        "spark.sql.extensions" = "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
+        "spark.sql.extensions" = "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+        "spark.metrics.executorMetricsSource.enabled" = "false",
+        "spark.ui.enabled" = "false",
+        "spark.executor.processTreeMetrics.enabled" = "false"
     }
 }
 

app/src/test/resources/application.conf

@@ -41,7 +41,9 @@ runtime{
         "spark.hadoop.fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
         "spark.hadoop.fs.file.impl": "com.globalmentor.apache.hadoop.fs.BareLocalFileSystem",
         "spark.sql.extensions": "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
-        "spark.metrics.executorMetricsSource.enabled": "false"
+        "spark.metrics.executorMetricsSource.enabled": "false",
+        "spark.ui.enabled": "false",
+        "spark.executor.processTreeMetrics.enabled": "false"
     }
 }
 

app/src/test/scala/io/github/datacatering/datacaterer/core/util/SparkSuite.scala

@@ -12,7 +12,7 @@ trait SparkSuite extends AnyFunSuite with BeforeAndAfterAll with BeforeAndAfterE
       .appName("spark tests")
       .config("spark.sql.legacy.allowUntypedScalaUDF", "true")
       .config("spark.sql.shuffle.partitions", "2")
-//      .config("spark.ui.enabled", "false")
+      .config("spark.ui.enabled", "false")
       .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,io.delta.sql.DeltaSparkSessionExtension")
       .config("spark.metrics.executorMetricsSource.enabled", "false")
       .getOrCreate()