chore: Add spark.metrics.executorMetricsSource.enabled configuration

- Updated configuration files to include "spark.metrics.executorMetricsSource.enabled" set to "false" for improved metrics handling.
- Ensured consistency across application and test configurations by aligning the settings in Constants.scala and application.conf.
- Modified SparkSuite to reflect the new configuration, enhancing test environment setup.

Author: pflooky

api/src/main/scala/io/github/datacatering/datacaterer/api/model/Constants.scala

@@ -288,7 +288,8 @@ object Constants {
     "spark.hadoop.fs.s3a.bucket.all.committer.magic.enabled" -> "true",
     "spark.hadoop.fs.hdfs.impl" -> "org.apache.hadoop.hdfs.DistributedFileSystem",
     "spark.hadoop.fs.file.impl" -> "com.globalmentor.apache.hadoop.fs.BareLocalFileSystem",
-    "spark.sql.extensions" -> "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
+    "spark.sql.extensions" -> "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+    "spark.metrics.executorMetricsSource.enabled" -> "false"
   )
 
   //jdbc defaults

app/src/test/resources/application.conf

@@ -40,7 +40,8 @@ runtime{
         "spark.hadoop.fs.s3a.bucket.all.committer.magic.enabled": "true",
         "spark.hadoop.fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
         "spark.hadoop.fs.file.impl": "com.globalmentor.apache.hadoop.fs.BareLocalFileSystem",
-        "spark.sql.extensions": "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
+        "spark.sql.extensions": "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+        "spark.metrics.executorMetricsSource.enabled": "false"
     }
 }
 

app/src/test/scala/io/github/datacatering/datacaterer/core/util/SparkSuite.scala

@@ -14,6 +14,7 @@ trait SparkSuite extends AnyFunSuite with BeforeAndAfterAll with BeforeAndAfterE
       .config("spark.sql.shuffle.partitions", "2")
 //      .config("spark.ui.enabled", "false")
       .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,io.delta.sql.DeltaSparkSessionExtension")
+      .config("spark.metrics.executorMetricsSource.enabled", "false")
       .getOrCreate()
   }