Fix PR comments

Author: pflooky

app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/PlanRepository.scala

@@ -410,9 +410,14 @@ object PlanRepository {
     }
   }
 
+  private def createSparkSession(): SparkSession = {
+    new SparkProvider(DEFAULT_MASTER, DEFAULT_RUNTIME_CONFIG).getSparkSession
+  }
+
   private def generateFromTaskFile(request: TaskFileSampleRequest): SampleResponse = {
     LOGGER.debug(s"Generating sample from task file: ${request.taskYamlPath}, step: ${request.stepName}")
     try {
+      implicit val sparkSession: SparkSession = createSparkSession()
       FastSampleGenerator.generateFromTaskFile(request)
     } catch {
       case ex: Throwable =>
@@ -428,6 +433,7 @@ object PlanRepository {
   private def generateFromTaskYaml(request: TaskYamlSampleRequest): SampleResponse = {
     LOGGER.debug(s"Generating sample from task YAML content, step: ${request.stepName}")
     try {
+      implicit val sparkSession: SparkSession = createSparkSession()
       FastSampleGenerator.generateFromTaskYaml(request)
     } catch {
       case ex: Throwable =>
@@ -443,6 +449,7 @@ object PlanRepository {
   private def generateFromSchema(request: SchemaSampleRequest): SampleResponse = {
     LOGGER.debug(s"Generating sample from inline fields: ${request.fields.size} fields")
     try {
+      implicit val sparkSession: SparkSession = createSparkSession()
       FastSampleGenerator.generateFromSchema(request)
     } catch {
       case ex: Throwable =>
@@ -459,7 +466,7 @@ object PlanRepository {
     LOGGER.debug("Starting up Spark")
     setUiRunning
     try {
-      implicit val sparkSession: SparkSession = new SparkProvider(DEFAULT_MASTER, DEFAULT_RUNTIME_CONFIG).getSparkSession
+      implicit val sparkSession: SparkSession = createSparkSession()
       //run some dummy query
       sparkSession.sql("SELECT 1").collect()
 

app/src/main/scala/io/github/datacatering/datacaterer/core/ui/sample/FastSampleGenerator.scala

@@ -17,9 +17,8 @@ object FastSampleGenerator {
 
   private val LOGGER = Logger.getLogger(getClass.getName)
   private val MAX_SAMPLE_SIZE = 100
-  private implicit val sparkSession: SparkSession = new SparkProvider(DEFAULT_MASTER, DEFAULT_RUNTIME_CONFIG).getSparkSession
 
-  def generateFromTaskFile(request: TaskFileSampleRequest): SampleResponse = {
+  def generateFromTaskFile(request: TaskFileSampleRequest)(implicit sparkSession: SparkSession): SampleResponse = {
     LOGGER.info(s"Generating sample from task file: ${request.taskYamlPath}, step: ${request.stepName}")
 
     Try {
@@ -47,7 +46,7 @@ object FastSampleGenerator {
     }
   }
 
-  def generateFromSchema(request: SchemaSampleRequest): SampleResponse = {
+  def generateFromSchema(request: SchemaSampleRequest)(implicit sparkSession: SparkSession): SampleResponse = {
     LOGGER.info(s"Generating sample from inline fields: ${request.fields.size} fields")
 
     Try {
@@ -60,7 +59,7 @@ object FastSampleGenerator {
     }
   }
 
-  def generateFromTaskYaml(request: TaskYamlSampleRequest): SampleResponse = {
+  def generateFromTaskYaml(request: TaskYamlSampleRequest)(implicit sparkSession: SparkSession): SampleResponse = {
     LOGGER.info(s"Generating sample from task YAML content, step: ${request.stepName}")
 
     Try {
@@ -89,7 +88,7 @@ object FastSampleGenerator {
   }
 
 
-  private def generateSample(fields: List[Field], sampleSize: Int, fastMode: Boolean): SampleResponse = {
+  private def generateSample(fields: List[Field], sampleSize: Int, fastMode: Boolean)(implicit sparkSession: SparkSession): SampleResponse = {
     val startTime = System.currentTimeMillis()
     val executionId = generateId()
 

app/src/test/resources/sample/task/file/csv-customer-task.yaml

@@ -14,7 +14,7 @@ steps:
       - name: "product_name"
         options:
           regex: "product_[0-9]{8}"
-  - name: "products"
+  - name: "customers"
     type: "csv"
     options:
       path: "/tmp/custom/csv/multi-relationship/customers"
@@ -31,7 +31,7 @@ steps:
       - name: "last_name"
         options:
           expression: "#{Name.lastName}"
-  - name: "products"
+  - name: "accounts"
     type: "csv"
     options:
       path: "/tmp/custom/csv/multi-relationship/accounts"
@@ -61,7 +61,7 @@ steps:
       - name: "country_code"
         options:
           expression: "#{Address.countryCode}"
-  - name: "products"
+  - name: "customer_access"
     type: "csv"
     options:
       path: "/tmp/custom/csv/multi-relationship/customer-access"