Bugfix/jdbc UI (#123)

* fix(ui): missing dbtable in connection options for jdbc connection

* feat(build): build for different os architectures and fix link for downloads

Author: pflooky

.github/workflows/build.yml

@@ -2,7 +2,8 @@ name: Build docker images
 
 on:
   push:
-    tags: ["*"]
+    branches:
+      - main
 
 jobs:
   build:
@@ -41,7 +42,16 @@ jobs:
 
   osx:
     needs: build
-    runs-on: [macos-latest]
+    strategy:
+      matrix:
+        include:
+          - runner: macos-13  # Intel x64
+            arch: x64
+            arch_name: x86_64
+          - runner: macos-14  # Apple Silicon arm64
+            arch: aarch64
+            arch_name: aarch64
+    runs-on: ${{ matrix.runner }}
 
     steps:
       - uses: actions/checkout@v4
@@ -53,7 +63,7 @@ jobs:
         with:
           java-version: '21'
           java-package: jdk
-          architecture: x64
+          architecture: ${{ matrix.arch }}
           distribution: oracle
       - name: Download fat jar
         uses: actions/download-artifact@v4
@@ -62,11 +72,13 @@ jobs:
           path: app/build/libs/
       - name: Package jar as dmg installer
         run: 'jpackage --main-jar data-caterer.jar "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-mac.cfg"'
+      - name: Rename DMG with version and architecture
+        run: mv DataCaterer-*.dmg DataCaterer-${{ env.APP_VERSION }}-macos-${{ matrix.arch_name }}.dmg
       - name: Upload dmg
         uses: actions/upload-artifact@v4
         with:
-          name: data-caterer-mac
-          path: "DataCaterer-1.0.0.dmg"
+          name: data-caterer-macos-${{ matrix.arch_name }}
+          path: "DataCaterer-${{ env.APP_VERSION }}-macos-${{ matrix.arch_name }}.dmg"
           overwrite: true
 
   windows:
@@ -92,16 +104,27 @@ jobs:
           path: app/build/libs/
       - name: Package jar as exe
         run: 'jpackage --main-jar data-caterer.jar "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-windows.cfg"'
+      - name: Rename EXE with version and architecture
+        run: mv DataCaterer-*.exe DataCaterer-$env:APP_VERSION-windows-x86_64.exe
       - name: Upload installer
         uses: actions/upload-artifact@v4
         with:
-          name: data-caterer-windows
-          path: "DataCaterer-1.0.0.exe"
+          name: data-caterer-windows-x86_64
+          path: "DataCaterer-${{ env.APP_VERSION }}-windows-x86_64.exe"
           overwrite: true
 
   linux:
     needs: build
-    runs-on: [ubuntu-latest]
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            arch: x64
+            arch_name: amd64
+          - runner: ubuntu-latest
+            arch: aarch64
+            arch_name: arm64
+    runs-on: ${{ matrix.runner }}
 
     steps:
       - uses: actions/checkout@v4
@@ -113,7 +136,7 @@ jobs:
         with:
           java-version: '21'
           java-package: jdk
-          architecture: x64
+          architecture: ${{ matrix.arch }}
           distribution: oracle
       - name: Download fat jar
         uses: actions/download-artifact@v4
@@ -122,11 +145,11 @@ jobs:
           path: app/build/libs/
       - name: Package jar as debian package
         run: 'jpackage --main-jar data-caterer.jar "@misc/jpackage/jpackage.cfg" "@misc/jpackage/jpackage-linux.cfg"'
-      - name: List directory
-        run: ls -lart
+      - name: Rename DEB with architecture
+        run: mv datacaterer_*_*.deb datacaterer_${{ env.APP_VERSION }}_${{ matrix.arch_name }}.deb
       - name: Upload deb
         uses: actions/upload-artifact@v4
         with:
-          name: data-caterer-linux
-          path: "datacaterer_1.0.0_amd64.deb"
+          name: data-caterer-linux-${{ matrix.arch_name }}
+          path: "datacaterer_${{ env.APP_VERSION }}_${{ matrix.arch_name }}.deb"
           overwrite: true

README.md

@@ -38,7 +38,7 @@ and deep dive into issues [from the generated report](https://data.catering/late
 
 1. Docker
    ```shell
-   docker run -d -i -p 9898:9898 -e DEPLOY_MODE=standalone --name datacaterer datacatering/data-caterer:0.17.2
+   docker run -d -i -p 9898:9898 -e DEPLOY_MODE=standalone --name datacaterer datacatering/data-caterer:0.17.3
    ```
    [Open localhost:9898](http://localhost:9898).
 1. [Run Scala/Java examples](#run-scalajava-examples)
@@ -47,14 +47,20 @@ and deep dive into issues [from the generated report](https://data.catering/late
    cd data-caterer-example && ./run.sh
    #check results under docker/sample/report/index.html folder
    ```
-1. [UI App: Mac download](https://nightly.link/data-catering/data-caterer/workflows/build/main/data-caterer-mac.zip)
-1. [UI App: Windows download](https://nightly.link/data-catering/data-caterer/workflows/build/main/data-caterer-windows.zip)
-   1. After downloading, go to 'Downloads' folder and 'Extract All' from data-caterer-windows
-   1. Double-click 'DataCaterer-1.0.0' to install Data Caterer
-   1. Click on 'More info' then at the bottom, click 'Run anyway'
-   1. Go to '/Program Files/DataCaterer' folder and run DataCaterer application
-   1. If your browser doesn't open, go to [http://localhost:9898](http://localhost:9898) in your preferred browser
-1. [UI App: Linux download](https://nightly.link/data-catering/data-caterer/workflows/build/main/data-caterer-linux.zip)
+1. UI App Downloads (Nightly builds from `main` branch)
+   - **macOS**:
+     - [Intel (x86_64)](https://nightly.link/data-catering/data-caterer/workflows/build/main/data-caterer-macos-x86_64.zip)
+     - [Apple Silicon (M1/M2/M3)](https://nightly.link/data-catering/data-caterer/workflows/build/main/data-caterer-macos-aarch64.zip)
+   - **Windows**:
+     - [x64](https://nightly.link/data-catering/data-caterer/workflows/build/main/data-caterer-windows-x86_64.zip)
+       1. After downloading, go to 'Downloads' folder and 'Extract All' from data-caterer-windows-x86_64
+       1. Double-click the installer to install Data Caterer
+       1. Click on 'More info' then at the bottom, click 'Run anyway'
+       1. Go to '/Program Files/DataCaterer' folder and run DataCaterer application
+       1. If your browser doesn't open, go to [http://localhost:9898](http://localhost:9898) in your preferred browser
+   - **Linux**:
+     - [amd64](https://nightly.link/data-catering/data-caterer/workflows/build/main/data-caterer-linux-amd64.zip)
+     - [arm64](https://nightly.link/data-catering/data-caterer/workflows/build/main/data-caterer-linux-arm64.zip)
 
 [Follow quick start instructions from here if you want more details](https://data.catering/latest/get-started/quick-start/).
 

app/src/main/scala/io/github/datacatering/datacaterer/core/ui/mapper/StepOptionsMapper.scala

@@ -0,0 +1,103 @@
+package io.github.datacatering.datacaterer.core.ui.mapper
+
+import io.github.datacatering.datacaterer.api.model.Constants._
+import org.apache.log4j.Logger
+
+/**
+ * Maps UI step options to internal format required by data sources
+ * 
+ * The UI may send options in a user-friendly format (e.g., separate schema and table fields)
+ * that need to be converted to the format expected by the underlying data source connectors
+ */
+object StepOptionsMapper {
+
+  private val LOGGER = Logger.getLogger(getClass.getName)
+
+  /**
+   * Convert UI step options to the format expected by the data source
+   * 
+   * @param options Step options from the UI
+   * @param format Data source format (jdbc, cassandra, etc.)
+   * @return Converted options
+   */
+  def mapStepOptions(options: Map[String, String], format: String): Map[String, String] = {
+    format.toLowerCase match {
+      case JDBC => mapJdbcOptions(options)
+      case CASSANDRA => mapCassandraOptions(options)
+      case ICEBERG => mapIcebergOptions(options)
+      case BIGQUERY => mapBigQueryOptions(options)
+      case _ => options // No mapping needed for other formats
+    }
+  }
+
+  /**
+   * Map JDBC options: convert schema + table to dbtable
+   * 
+   * JDBC connections require 'dbtable' in format 'schema.table' or just 'table'
+   * The UI sends these as separate 'schema' and 'table' fields
+   */
+  private def mapJdbcOptions(options: Map[String, String]): Map[String, String] = {
+    val hasSchema = options.contains("schema")
+    val hasTable = options.contains("table")
+    val hasDbTable = options.contains(JDBC_TABLE)
+    
+    if (!hasDbTable && hasSchema && hasTable) {
+      // Convert schema + table to dbtable
+      val schema = options("schema")
+      val table = options("table")
+      val dbtable = s"$schema.$table"
+      LOGGER.debug(s"Converting JDBC schema+table to dbtable: schema=$schema, table=$table, dbtable=$dbtable")
+      options - "schema" - "table" + (JDBC_TABLE -> dbtable)
+    } else if (!hasDbTable && hasTable) {
+      // Only table provided, rename to dbtable
+      LOGGER.debug(s"Converting JDBC table to dbtable: table=${options("table")}")
+      options - "table" + (JDBC_TABLE -> options("table"))
+    } else {
+      // dbtable already exists or neither schema nor table provided
+      options
+    }
+  }
+
+  /**
+   * Map Cassandra options: ensure keyspace and table are properly set
+   * 
+   * Cassandra requires separate 'keyspace' and 'table' options
+   * This is already the format the UI sends, so no conversion needed currently
+   */
+  private def mapCassandraOptions(options: Map[String, String]): Map[String, String] = {
+    // Cassandra already uses keyspace and table separately
+    // No conversion needed, but we validate they exist
+    if (options.contains(CASSANDRA_KEYSPACE) && options.contains(CASSANDRA_TABLE)) {
+      LOGGER.debug(s"Cassandra options validated: keyspace=${options(CASSANDRA_KEYSPACE)}, table=${options(CASSANDRA_TABLE)}")
+    }
+    options
+  }
+
+  /**
+   * Map Iceberg options: ensure table is properly formatted
+   * 
+   * Iceberg requires 'table' in format 'database.table'
+   * The UI sends this as a single 'table' field
+   */
+  private def mapIcebergOptions(options: Map[String, String]): Map[String, String] = {
+    // Iceberg uses 'table' field which should already be in database.table format from UI
+    if (options.contains(TABLE)) {
+      LOGGER.debug(s"Iceberg table option: table=${options(TABLE)}")
+    }
+    options
+  }
+
+  /**
+   * Map BigQuery options: ensure table is properly formatted
+   * 
+   * BigQuery requires 'table' in format 'project.dataset.table'
+   * The UI sends this as a single 'table' field
+   */
+  private def mapBigQueryOptions(options: Map[String, String]): Map[String, String] = {
+    // BigQuery uses 'table' field which should already be in project.dataset.table format from UI
+    if (options.contains(TABLE)) {
+      LOGGER.debug(s"BigQuery table option: table=${options(TABLE)}")
+    }
+    options
+  }
+}

app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/PlanRepository.scala

@@ -11,6 +11,7 @@ import io.github.datacatering.datacaterer.core.parser.PlanParser
 import io.github.datacatering.datacaterer.core.plan.{PlanProcessor, YamlPlanRun}
 import io.github.datacatering.datacaterer.core.ui.config.UiConfiguration.INSTALL_DIRECTORY
 import io.github.datacatering.datacaterer.core.ui.mapper.ConfigurationMapper.configurationMapping
+import io.github.datacatering.datacaterer.core.ui.mapper.StepOptionsMapper
 import io.github.datacatering.datacaterer.core.ui.model.{ConfigurationRequest, Connection, EnhancedPlanRunRequest, PlanRunExecution, PlanRunRequest, PlanRunRequests, SampleResponse, SchemaSampleRequest}
 import io.github.datacatering.datacaterer.core.ui.plan.PlanResponseHandler.{KO, OK, Response}
 import io.github.datacatering.datacaterer.core.ui.resource.SparkSessionManager
@@ -195,7 +196,7 @@ object PlanRepository {
       .map(t => t.name -> t.dataSourceName)
       .toMap
 
-    val dataSourceConnectionInfo = getConnectionDetails(taskToDataSourceMap)
+    val dataSourceConnectionInfo = getConnectionDetails(taskToDataSourceMap, baseDirectory)
       .map(c => {
         val additionalConfig = c.`type` match {
           case POSTGRES => Map(FORMAT -> JDBC, DRIVER -> POSTGRES_DRIVER)
@@ -207,8 +208,8 @@ object PlanRepository {
       .toMap
 
     //find tasks and validation using data source connection
-    val updatedValidation = validationWithConnectionInfo(parsedRequest, dataSourceConnectionInfo)
-    val updatedTasks = tasksWithConnectionInfo(parsedRequest, taskToDataSourceMap, dataSourceConnectionInfo)
+    val updatedValidation = validationWithConnectionInfo(parsedRequest, dataSourceConnectionInfo, baseDirectory)
+    val updatedTasks = tasksWithConnectionInfo(parsedRequest, taskToDataSourceMap, dataSourceConnectionInfo, baseDirectory)
     val updatedConfiguration = parsedRequest.configuration
       .map(c => configurationMapping(c, baseDirectory))
       .getOrElse(DataCatererConfigurationBuilder())
@@ -218,14 +219,15 @@ object PlanRepository {
 
   private def validationWithConnectionInfo(
                                             parsedRequest: PlanRunRequest,
-                                            dataSourceConnectionInfo: Map[String, Map[String, String]]
+                                            dataSourceConnectionInfo: Map[String, Map[String, String]],
+                                            baseDirectory: String
                                           ): List[ValidationConfiguration] = {
     parsedRequest.validation.map(yamlV => {
       val updatedDataSources = yamlV.dataSources.map(ds => {
         val dataSourceName = ds._1
         val connectionInfo = dataSourceConnectionInfo(dataSourceName)
         val updatedValidationOptions = ds._2.map(yamlDs => {
-          val metadataOpts = getMetadataSourceInfo(dataSourceConnectionInfo, yamlDs.options)
+          val metadataOpts = getMetadataSourceInfo(dataSourceConnectionInfo, yamlDs.options, baseDirectory)
           val allOpts = yamlDs.options ++ connectionInfo ++ metadataOpts
           val listValidationBuilders = yamlDs.validations.map {
             case yamlUpstreamDataSourceValidation: YamlUpstreamDataSourceValidation =>
@@ -245,7 +247,8 @@ object PlanRepository {
   private def tasksWithConnectionInfo(
                                        parsedRequest: PlanRunRequest,
                                        taskToDataSourceMap: Map[String, String],
-                                       dataSourceConnectionInfo: Map[String, Map[String, String]]
+                                       dataSourceConnectionInfo: Map[String, Map[String, String]],
+                                       baseDirectory: String
                                      ): List[Task] = {
     val updatedTasks = parsedRequest.tasks.map(s => {
       val taskName = s.name
@@ -254,23 +257,29 @@ object PlanRepository {
       }
       val dataSourceName = taskToDataSourceMap(taskName)
       val connectionInfo = dataSourceConnectionInfo(dataSourceName)
-      val metadataOpts = getMetadataSourceInfo(dataSourceConnectionInfo, s.options)
-      val updatedStep = s.copy(options = s.options ++ connectionInfo ++ metadataOpts)
+      val metadataOpts = getMetadataSourceInfo(dataSourceConnectionInfo, s.options, baseDirectory)
+      
+      // Map UI options to data source format using StepOptionsMapper
+      val format = connectionInfo.getOrElse(FORMAT, "")
+
+      val baseOptions = s.options ++ connectionInfo ++ metadataOpts
+      val mappedOptions = StepOptionsMapper.mapStepOptions(baseOptions, format)
+      val updatedStep = s.copy(options = mappedOptions)
       Task(taskName, List(updatedStep))
     })
     updatedTasks
   }
 
-  private def getMetadataSourceInfo(dataSourceConnectionInfo: Map[String, Map[String, String]], options: Map[String, String]): Map[String, String] = {
+  private def getMetadataSourceInfo(dataSourceConnectionInfo: Map[String, Map[String, String]], options: Map[String, String], baseDirectory: String): Map[String, String] = {
     if (options.contains(METADATA_SOURCE_NAME)) {
-      ConnectionService.getMetadataSourceInfo(options(METADATA_SOURCE_NAME), dataSourceConnectionInfo)
+      ConnectionService.getMetadataSourceInfo(options(METADATA_SOURCE_NAME), dataSourceConnectionInfo, baseDirectory)
     } else {
       Map()
     }
   }
 
-  private def getConnectionDetails(taskToDataSourceMap: Map[String, String]): List[Connection] = {
-    ConnectionService.getConnections(taskToDataSourceMap.values.toList, masking = false)
+  private def getConnectionDetails(taskToDataSourceMap: Map[String, String], baseDirectory: String): List[Connection] = {
+    ConnectionService.getConnections(taskToDataSourceMap.values.toList, masking = false, baseDirectory)
   }
 
   private def savePlanRunExecution(