Update version to 0.17.0, enhance YAML support with a new YamlBuilder API, improve metadata processing, and introduce comprehensive end-to-end testing infrastructure. Add new examples and documentation for YAML configurations, along with updates to existing sample plans and tasks.

Author: pflooky

.gitignore

@@ -32,7 +32,9 @@ app/src/test/resources/sample/documentation
 app/src/test/resources/sample/java
 app/src/test/resources/sample/report
 app/src/test/resources/sample/plan-gen
-app/src/test/resources/sample/e2e
+app/src/test/resources/sample/e2e/generated
+app/src/test/resources/sample/e2e/report
+app/src/test/resources/sample/e2e/data
 
 api/out
 api/src/test/resources/sample/documentation

README.md

@@ -38,7 +38,7 @@ and deep dive into issues [from the generated report](https://data.catering/late
 
 1. Docker
    ```shell
-   docker run -d -i -p 9898:9898 -e DEPLOY_MODE=standalone --name datacaterer datacatering/data-caterer:0.16.11
+   docker run -d -i -p 9898:9898 -e DEPLOY_MODE=standalone --name datacaterer datacatering/data-caterer:0.17.0
    ```
    [Open localhost:9898](http://localhost:9898).
 1. [Run Scala/Java examples](#run-scalajava-examples)

api/src/main/java/io/github/datacatering/datacaterer/javaapi/api/PlanRun.java

@@ -124,6 +124,18 @@ public GeneratorBuilder generator() {
         return new GeneratorBuilder();
     }
 
+    /**
+     * Creates a WeightedValue for weighted field values. Useful for oneOfWeightedJava field method.
+     *
+     * @param value  The value
+     * @param weight The weight associated with the value
+     * @return A WeightedValue containing the value and weight
+     */
+    public static WeightedValue weightedValue(Object value, double weight) {
+        return new WeightedValue(value, weight);
+    }
+
+
     /**
      * Creates a CountBuilder instance.
      *
@@ -819,6 +831,21 @@ public void execute(
         execute(plan(), configuration(), Collections.emptyList(), connectionTaskBuilder, connectionTaskBuilders);
     }
 
+    /**
+     * Executes the data catering plan with the provided list of tasks.
+     *
+     * @param connectionTaskBuilders A list of ConnectionTaskBuilder objects representing tasks to be executed.
+     */
+    public void execute(List<ConnectionTaskBuilder<?>> connectionTaskBuilders) {
+        if (connectionTaskBuilders.isEmpty()) {
+            throw new IllegalArgumentException("At least one ConnectionTaskBuilder must be provided");
+        }
+        ConnectionTaskBuilder<?> first = connectionTaskBuilders.get(0);
+        ConnectionTaskBuilder<?>[] rest = connectionTaskBuilders.subList(1, connectionTaskBuilders.size())
+                .toArray(new ConnectionTaskBuilder<?>[0]);
+        execute(first, rest);
+    }
+
 
     /**
      * Executes the data catering plan with the provided configurations.
@@ -835,6 +862,25 @@ public void execute(
         execute(plan(), configurationBuilder, Collections.emptyList(), connectionTaskBuilder, connectionTaskBuilders);
     }
 
+    /**
+     * Executes the data catering plan with the provided configuration and list of tasks.
+     *
+     * @param configurationBuilder   The DataCatererConfigurationBuilder object representing the configuration for the plan execution.
+     * @param connectionTaskBuilders A list of ConnectionTaskBuilder objects representing tasks to be executed.
+     */
+    public void execute(
+            DataCatererConfigurationBuilder configurationBuilder,
+            List<ConnectionTaskBuilder<?>> connectionTaskBuilders
+    ) {
+        if (connectionTaskBuilders.isEmpty()) {
+            throw new IllegalArgumentException("At least one ConnectionTaskBuilder must be provided");
+        }
+        ConnectionTaskBuilder<?> first = connectionTaskBuilders.get(0);
+        ConnectionTaskBuilder<?>[] rest = connectionTaskBuilders.subList(1, connectionTaskBuilders.size())
+                .toArray(new ConnectionTaskBuilder<?>[0]);
+        execute(configurationBuilder, first, rest);
+    }
+
     /**
      * Executes the data catering plan with the provided configurations.
      *
@@ -852,6 +898,27 @@ public void execute(
         execute(planBuilder, configurationBuilder, Collections.emptyList(), connectionTaskBuilder, connectionTaskBuilders);
     }
 
+    /**
+     * Executes the data catering plan with the provided plan, configuration and list of tasks.
+     *
+     * @param planBuilder            The PlanBuilder object representing the plan to be executed.
+     * @param configurationBuilder   The DataCatererConfigurationBuilder object representing the configuration for the plan execution.
+     * @param connectionTaskBuilders A list of ConnectionTaskBuilder objects representing tasks to be executed.
+     */
+    public void execute(
+            PlanBuilder planBuilder,
+            DataCatererConfigurationBuilder configurationBuilder,
+            List<ConnectionTaskBuilder<?>> connectionTaskBuilders
+    ) {
+        if (connectionTaskBuilders.isEmpty()) {
+            throw new IllegalArgumentException("At least one ConnectionTaskBuilder must be provided");
+        }
+        ConnectionTaskBuilder<?> first = connectionTaskBuilders.get(0);
+        ConnectionTaskBuilder<?>[] rest = connectionTaskBuilders.subList(1, connectionTaskBuilders.size())
+                .toArray(new ConnectionTaskBuilder<?>[0]);
+        execute(planBuilder, configurationBuilder, first, rest);
+    }
+
     /**
      * Executes the data catering plan with the provided configurations.
      *

api/src/main/java/io/github/datacatering/datacaterer/javaapi/api/WeightedValue.java

@@ -0,0 +1,49 @@
+package io.github.datacatering.datacaterer.javaapi.api;
+
+/**
+ * A simple container for value-weight pairs used in Java API for oneOfWeighted field method.
+ */
+public class WeightedValue {
+    private final Object value;
+    private final double weight;
+
+    /**
+     * Constructs a new WeightedValue.
+     * 
+     * @param value  The value
+     * @param weight The weight associated with the value
+     */
+    public WeightedValue(Object value, double weight) {
+        this.value = value;
+        this.weight = weight;
+    }
+
+    /**
+     * Gets the value.
+     * 
+     * @return The value
+     */
+    public Object value() {
+        return value;
+    }
+
+    /**
+     * Gets the weight.
+     * 
+     * @return The weight
+     */
+    public double weight() {
+        return weight;
+    }
+
+    /**
+     * Static factory method for easier creation.
+     * 
+     * @param value  The value
+     * @param weight The weight associated with the value
+     * @return A new WeightedValue instance
+     */
+    public static WeightedValue of(Object value, double weight) {
+        return new WeightedValue(value, weight);
+    }
+}

api/src/main/scala/io/github/datacatering/datacaterer/api/PlanRun.scala

@@ -641,6 +641,26 @@ trait PlanRun {
     execute(configuration, connectionTaskBuilder, connectionTaskBuilders: _*)
   }
 
+  /**
+   * Execute with non-default configurations for a set of tasks
+   *
+   * @param baseConfiguration      Runtime configurations
+   * @param connectionTaskBuilders Connection and task builders
+   */
+  def execute(
+               baseConfiguration: DataCatererConfigurationBuilder,
+               connectionTaskBuilders: List[ConnectionTaskBuilder[_]]
+             ): Unit = {
+    if (connectionTaskBuilders.isEmpty) {
+      throw new IllegalArgumentException("At least one ConnectionTaskBuilder must be provided")
+    }
+    if (connectionTaskBuilders.tail.isEmpty) {
+      execute(plan, baseConfiguration, List(), connectionTaskBuilders.head)
+    } else {
+      execute(plan, baseConfiguration, List(), connectionTaskBuilders.head, connectionTaskBuilders.tail: _*)
+    }
+  }
+
   /**
    * Execute with non-default configurations for a set of tasks
    *
@@ -673,6 +693,28 @@ trait PlanRun {
     execute(planBuilder, baseConfiguration, List(), connectionTaskBuilder, connectionTaskBuilders: _*)
   }
 
+  /**
+   * Execute with non-default configurations with validations and tasks
+   *
+   * @param planBuilder            Plan to set high level task configurations
+   * @param baseConfiguration      Runtime configurations
+   * @param connectionTaskBuilders Connection and task builders
+   */
+  def execute(
+               planBuilder: PlanBuilder,
+               baseConfiguration: DataCatererConfigurationBuilder,
+               connectionTaskBuilders: List[ConnectionTaskBuilder[_]]
+             ): Unit = {
+    if (connectionTaskBuilders.isEmpty) {
+      throw new IllegalArgumentException("At least one ConnectionTaskBuilder must be provided")
+    }
+    if (connectionTaskBuilders.tail.isEmpty) {
+      execute(planBuilder, baseConfiguration, List(), connectionTaskBuilders.head)
+    } else {
+      execute(planBuilder, baseConfiguration, List(), connectionTaskBuilders.head, connectionTaskBuilders.tail: _*)
+    }
+  }
+
   /**
    * Execute with non-default configurations with validations and tasks. Validations have to be enabled before running
    * (see [[DataCatererConfigurationBuilder.enableValidation()]].

api/src/main/scala/io/github/datacatering/datacaterer/api/TaskBuilder.scala

@@ -8,6 +8,7 @@ import io.github.datacatering.datacaterer.api.model.Constants._
 import io.github.datacatering.datacaterer.api.model.{ArrayType, Count, DataType, DoubleType, Field, HeaderType, PerFieldCount, Step, StringType, Task, TaskSummary}
 
 import scala.annotation.varargs
+import scala.collection.JavaConverters._
 
 /**
  * Builds a `TaskSummary` with optional `Task` information.
@@ -147,17 +148,25 @@ case class TaskBuilder(task: Task = Task()) {
   @varargs def steps(steps: StepBuilder*): TaskBuilder = this.modify(_.task.steps)(_ ++ steps.map(_.step))
 
   /**
-   * Create a TaskBuilder that loads base configuration from a YAML file.
-   * This allows referencing existing YAML task definitions while still being able to override
-   * specific configurations using the builder pattern.
+   * Create a TaskBuilder that references configuration from a YAML file.
+   * This stores YAML metadata information that will be resolved during execution.
+   * The actual YAML loading will happen during execution when the task is processed.
    *
-   * @param yamlConfig Configuration specifying which YAML file to load
-   * @return TaskBuilder with YAML task as base configuration
+   * Note: The YAML configuration is stored as a placeholder step with the YAML metadata
+   * in its options. The actual YAML task will be loaded and merged during plan execution.
+   *
+   * @param yamlConfig Configuration specifying which YAML file to reference
+   * @return TaskBuilder with YAML reference metadata stored in a placeholder step
    */
   def fromYaml(yamlConfig: YamlConfig): TaskBuilder = {
-    // Add special marker to indicate this task should load from YAML
-    // The actual YAML loading will happen during execution when the task is processed
-    this.modify(_.task.name).setTo(s"${task.name}_yaml_${yamlConfig.taskFile.getOrElse("unknown").hashCode}")
+    // Store YAML config as a placeholder step's options
+    // This will be resolved during execution by the metadata loading pipeline
+    val yamlOptions = yamlConfig.toOptionsMap
+    val placeholderStep = Step(
+      name = "yaml_placeholder",
+      options = yamlOptions
+    )
+    this.modify(_.task.steps)(_ ++ List(placeholderStep))
   }
 }
 
@@ -775,6 +784,38 @@ case class FieldBuilder(field: Field = Field()) {
   def oneOfWeighted(values: List[(Any, Double)]): List[FieldBuilder] =
     if (values.nonEmpty) oneOfWeighted(values.head, values.tail: _*) else List(this)
 
+  /**
+   * Java-friendly overload for oneOfWeighted method using WeightedValue objects.
+   * Sets the field to use weighted random values from the provided weighted values.
+   * Returns an array to work seamlessly with Java varargs in fields() method.
+   *
+   * @param value  The first value-weight pair (required)
+   * @param values Additional value-weight pairs
+   * @return An array of FieldBuilder instances (the weighted field and a weight field)
+   */
+  @varargs def oneOfWeightedJava(value: io.github.datacatering.datacaterer.javaapi.api.WeightedValue, values: io.github.datacatering.datacaterer.javaapi.api.WeightedValue*): Array[FieldBuilder] = {
+    val scalaValues = values.map(wv => (wv.value, wv.weight))
+    oneOfWeighted((value.value, value.weight), scalaValues: _*).toArray
+  }
+
+  /**
+   * Java-friendly overload for oneOfWeighted method using a List of WeightedValue.
+   * Sets the field to use weighted random values from the provided weighted values.
+   * Returns an array to work seamlessly with Java varargs in fields() method.
+   *
+   * @param values A Java List of WeightedValue objects containing value-weight pairs
+   * @return An array of FieldBuilder instances (the weighted field and a weight field)
+   */
+  def oneOfWeightedJava(values: java.util.List[io.github.datacatering.datacaterer.javaapi.api.WeightedValue]): Array[FieldBuilder] = {
+    if (values.isEmpty) {
+      Array(this)
+    } else {
+      val first = values.get(0)
+      val rest = values.subList(1, values.size()).asScala
+      oneOfWeightedJava(first, rest: _*)
+    }
+  }
+
   /**
    * Sets the options for the field generator.
    *

api/src/main/scala/io/github/datacatering/datacaterer/api/YamlBuilder.scala

@@ -28,55 +28,77 @@ case class YamlBuilder(yamlConfig: YamlConfig = YamlConfig()) {
   }
 
   /**
-   * Load from a YAML task file. This creates a task builder that references the existing YAML task
-   * and allows overriding specific configurations.
+   * Load from a YAML task file. Creates a task builder that references the existing YAML task.
+   * 
+   * WARNING: If the YAML task file contains multiple tasks or a task with multiple steps, 
+   * you may encounter schema ambiguity. Consider using stepByFile() if you need a specific step.
    *
    * @param taskFile Path to the YAML task file
    * @return TaskBuilder with YAML task as base
    */
-  def task(taskFile: String): TaskBuilder = {
+  def taskByFile(taskFile: String): TaskBuilder = {
     val updatedConfig = this.modify(_.yamlConfig.taskFile).setTo(Some(taskFile))
     TaskBuilder().fromYaml(updatedConfig.yamlConfig)
   }
 
   /**
-   * Load from a YAML task file with specific task name filter.
+   * Load from a YAML task by name. Creates a task builder that references the existing YAML task
+   * identified by the task name.
+   * 
+   * WARNING: If the specified task has multiple steps, you may encounter schema ambiguity.
+   * Consider using stepByName() if you need a specific step.
    *
-   * @param taskFile Path to the YAML task file
-   * @param taskName Name of the specific task to use from the YAML file
+   * @param taskName Name of the specific task to use
    * @return TaskBuilder with filtered YAML task as base
    */
-  def task(taskFile: String, taskName: String): TaskBuilder = {
-    val updatedConfig = this.modify(_.yamlConfig.taskFile).setTo(Some(taskFile))
-      .modify(_.yamlConfig.taskName).setTo(Some(taskName))
+  def taskByName(taskName: String): TaskBuilder = {
+    val updatedConfig = this.modify(_.yamlConfig.taskName).setTo(Some(taskName))
     TaskBuilder().fromYaml(updatedConfig.yamlConfig)
   }
 
   /**
-   * Load from a YAML task file with specific task and step name filters.
+   * Load a specific step from a YAML task file. Creates a task builder that references a specific step
+   * within a YAML task file.
    *
    * @param taskFile Path to the YAML task file
-   * @param taskName Name of the specific task to use from the YAML file
    * @param stepName Name of the specific step to use from the task
-   * @return TaskBuilder with filtered YAML task as base
+   * @return TaskBuilder with filtered YAML step as base
    */
-  def task(taskFile: String, taskName: String, stepName: String): TaskBuilder = {
+  def stepByFile(taskFile: String, stepName: String): TaskBuilder = {
     val updatedConfig = this.modify(_.yamlConfig.taskFile).setTo(Some(taskFile))
-      .modify(_.yamlConfig.taskName).setTo(Some(taskName))
       .modify(_.yamlConfig.stepName).setTo(Some(stepName))
     TaskBuilder().fromYaml(updatedConfig.yamlConfig)
   }
 
   /**
-   * Java API - Load from a YAML task file with specific task name filter.
+   * Load a specific step from a YAML task by name. Creates a task builder that references a specific step
+   * within a named YAML task.
+   *
+   * @param taskName Name of the specific task to use
+   * @param stepName Name of the specific step to use from the task
+   * @return TaskBuilder with filtered YAML step as base
    */
-  def taskJava(taskFile: String, taskName: String): TaskBuilder = task(taskFile, taskName)
+  def stepByName(taskName: String, stepName: String): TaskBuilder = {
+    val updatedConfig = this.modify(_.yamlConfig.taskName).setTo(Some(taskName))
+      .modify(_.yamlConfig.stepName).setTo(Some(stepName))
+    TaskBuilder().fromYaml(updatedConfig.yamlConfig)
+  }
 
   /**
-   * Java API - Load from a YAML task file with specific task and step name filters.
+   * Load a specific step from a YAML task file with task name filter. Creates a task builder that references 
+   * a specific step within a specific task from a YAML file.
+   *
+   * @param taskFile Path to the YAML task file
+   * @param taskName Name of the specific task to use from the YAML file
+   * @param stepName Name of the specific step to use from the task
+   * @return TaskBuilder with filtered YAML step as base
    */
-  def taskJava(taskFile: String, taskName: String, stepName: String): TaskBuilder = 
-    task(taskFile, taskName, stepName)
+  def stepByFileAndName(taskFile: String, taskName: String, stepName: String): TaskBuilder = {
+    val updatedConfig = this.modify(_.yamlConfig.taskFile).setTo(Some(taskFile))
+      .modify(_.yamlConfig.taskName).setTo(Some(taskName))
+      .modify(_.yamlConfig.stepName).setTo(Some(stepName))
+    TaskBuilder().fromYaml(updatedConfig.yamlConfig)
+  }
 }
 
 /**