docs: Update version to 0.19.1, enhance foreign key features, and imp… (#130)

* docs: Update version to 0.19.1, enhance foreign key features, and improve documentation

- Bump version in various files to 0.19.1.
- Introduce cardinality and nullability controls for foreign key relationships.
- Add new documentation for foreign key enhancements and update existing links.
- Improve logging levels for better debugging.
- Update Dockerfile and example configurations to reflect the new version.
- Add feature catalog for standardized documentation of all features.

* chore: Add spark.metrics.executorMetricsSource.enabled configuration

- Updated configuration files to include "spark.metrics.executorMetricsSource.enabled" set to "false" for improved metrics handling.
- Ensured consistency across application and test configurations by aligning the settings in Constants.scala and application.conf.
- Modified SparkSuite to reflect the new configuration, enhancing test environment setup.

* chore: Update Java distribution and enhance Spark configurations

- Changed Java distribution from Oracle to Temurin in build and check workflows for improved compatibility.
- Added "spark.executor.processTreeMetrics.enabled" configuration to multiple files to enhance Spark metrics handling.
- Ensured consistency across application and test configurations by aligning the settings in Constants.scala, application.conf, and application-integration.conf.

Author: pflooky

.github/workflows/build.yml

@@ -35,7 +35,7 @@ jobs:
           java-version: '17'
           java-package: jdk
           architecture: x64
-          distribution: oracle
+          distribution: temurin
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:

.github/workflows/check.yml

@@ -15,7 +15,7 @@ jobs:
           java-version: '17'
           java-package: jdk
           architecture: x64
-          distribution: oracle
+          distribution: temurin
       - name: Gradle build with cache
         uses: burrunan/gradle-cache-action@v1
         with:

.gitignore

@@ -21,6 +21,7 @@ site
 # Python/virtualenvs used by docs
 .venv
 .python-version
+__pycache__
 
 app/docs
 app/out

README.md

@@ -76,7 +76,7 @@ Check results at `docker/data/custom/report/index.html`.
 ### UI
 
 ```shell
-docker run -d -p 9898:9898 -e DEPLOY_MODE=standalone --name datacaterer datacatering/data-caterer:0.19.0
+docker run -d -p 9898:9898 -e DEPLOY_MODE=standalone --name datacaterer datacatering/data-caterer:0.19.1
 ```
 
 Open [http://localhost:9898](http://localhost:9898).

api/src/main/scala/io/github/datacatering/datacaterer/api/model/Constants.scala

@@ -288,7 +288,9 @@ object Constants {
     "spark.hadoop.fs.s3a.bucket.all.committer.magic.enabled" -> "true",
     "spark.hadoop.fs.hdfs.impl" -> "org.apache.hadoop.hdfs.DistributedFileSystem",
     "spark.hadoop.fs.file.impl" -> "com.globalmentor.apache.hadoop.fs.BareLocalFileSystem",
-    "spark.sql.extensions" -> "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
+    "spark.sql.extensions" -> "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+    "spark.metrics.executorMetricsSource.enabled" -> "false",
+    "spark.executor.processTreeMetrics.enabled" -> "false"
   )
 
   //jdbc defaults

app/build.gradle.kts

@@ -543,6 +543,7 @@ tasks.shadowJar {
     val newTransformer = com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer()
     newTransformer.resource = "reference.conf"
     transformers.add(newTransformer)
+    mergeServiceFiles()
 }
 
 // Configure Scoverage only when it's applied (configuration cache disabled)

app/src/main/scala/io/github/datacatering/datacaterer/core/plan/ForeignKeyUniquenessProcessor.scala

@@ -40,7 +40,7 @@ class ForeignKeyUniquenessProcessor(val dataCatererConfiguration: DataCatererCon
                       validations: List[ValidationConfiguration]
                     ): (Plan, List[Task], List[ValidationConfiguration]) = {
 
-    LOGGER.info("ForeignKeyUniquenessProcessor starting...")
+    LOGGER.debug("ForeignKeyUniquenessProcessor starting...")
 
     // Extract foreign keys from plan's sink options
     val foreignKeys = plan.sinkOptions.map(_.foreignKeys).getOrElse(List())

app/src/main/scala/io/github/datacatering/datacaterer/core/plan/PlanProcessor.scala

@@ -1,7 +1,7 @@
 package io.github.datacatering.datacaterer.core.plan
 
 import io.github.datacatering.datacaterer.api.PlanRun
-import io.github.datacatering.datacaterer.api.model.Constants.{DATA_CATERER_INTERFACE_JAVA, DATA_CATERER_INTERFACE_SCALA, DATA_CATERER_INTERFACE_YAML, PLAN_CLASS, PLAN_STAGE_EXTRACT_METADATA, PLAN_STAGE_PARSE_PLAN}
+import io.github.datacatering.datacaterer.api.model.Constants.{DATA_CATERER_INTERFACE_JAVA, DATA_CATERER_INTERFACE_SCALA, DATA_CATERER_INTERFACE_YAML, DEFAULT_STEP_TYPE, FORMAT, PLAN_CLASS, PLAN_STAGE_EXTRACT_METADATA, PLAN_STAGE_PARSE_PLAN}
 import io.github.datacatering.datacaterer.api.model.{DataCatererConfiguration, Plan, Task, ValidationConfiguration}
 import io.github.datacatering.datacaterer.core.activity.{PlanRunPostPlanProcessor, PlanRunPrePlanProcessor}
 import io.github.datacatering.datacaterer.core.config.ConfigParser
@@ -118,7 +118,7 @@ object PlanProcessor {
         basePlan, baseTasks, baseValidations, dataCatererConfiguration, resolvedInterface
       )
 
-      LOGGER.info(s"After pre-processors: num-tasks=${finalTasks.size}")
+      LOGGER.info(s"After pre-processors: num-tasks=${finalTasks.size}, task-names=${finalTasks.map(_.name).mkString(", ")}")
 
       // Step 4: Generate data with the final modified plan/tasks
       val dataGeneratorProcessor = new DataGeneratorProcessor(dataCatererConfiguration)
@@ -358,7 +358,13 @@ class YamlPlanRun(
 
     // Merge connection config into each step's options (connection config as base, step options override)
     val stepsWithConnectionConfig = task.steps.map(step => {
-      step.copy(options = connectionConfig ++ step.options)
+      val mergedOptions = connectionConfig ++ step.options
+      val optionsWithFormat = if (!mergedOptions.contains(FORMAT) && step.`type` != DEFAULT_STEP_TYPE) {
+        mergedOptions + (FORMAT -> step.`type`)
+      } else {
+        mergedOptions
+      }
+      step.copy(options = optionsWithFormat)
     })
 
     task.copy(steps = stepsWithConnectionConfig)
@@ -386,7 +392,13 @@ class UnifiedPlanRun(
     val connectionConfig = dataCatererConfig.connectionConfigByName.getOrElse(dataSourceName, Map())
 
     val stepsWithConnectionConfig = task.steps.map(step => {
-      step.copy(options = connectionConfig ++ step.options)
+      val mergedOptions = connectionConfig ++ step.options
+      val optionsWithFormat = if (!mergedOptions.contains(FORMAT) && step.`type` != DEFAULT_STEP_TYPE) {
+        mergedOptions + (FORMAT -> step.`type`)
+      } else {
+        mergedOptions
+      }
+      step.copy(options = optionsWithFormat)
     })
 
     task.copy(steps = stepsWithConnectionConfig)

app/src/test/resources/application-integration.conf

@@ -110,7 +110,10 @@ runtime {
         "spark.hadoop.fs.s3a.bucket.all.committer.magic.enabled" = "true",
         "spark.hadoop.fs.hdfs.impl" = "org.apache.hadoop.hdfs.DistributedFileSystem",
         "spark.hadoop.fs.file.impl" = "com.globalmentor.apache.hadoop.fs.BareLocalFileSystem",
-        "spark.sql.extensions" = "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
+        "spark.sql.extensions" = "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+        "spark.metrics.executorMetricsSource.enabled" = "false",
+        "spark.ui.enabled" = "false",
+        "spark.executor.processTreeMetrics.enabled" = "false"
     }
 }
 

app/src/test/resources/application.conf

@@ -40,7 +40,10 @@ runtime{
         "spark.hadoop.fs.s3a.bucket.all.committer.magic.enabled": "true",
         "spark.hadoop.fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
         "spark.hadoop.fs.file.impl": "com.globalmentor.apache.hadoop.fs.BareLocalFileSystem",
-        "spark.sql.extensions": "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"
+        "spark.sql.extensions": "io.delta.sql.DeltaSparkSessionExtension,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+        "spark.metrics.executorMetricsSource.enabled": "false",
+        "spark.ui.enabled": "false",
+        "spark.executor.processTreeMetrics.enabled": "false"
     }
 }