Merge pull request #100 from data-catering/feature/clean-docs

Update to Gradle 9.0.0, add gradle configurations to speed up build a…

Author: pflooky

CLAUDE.md

@@ -20,6 +20,9 @@ The project uses Gradle with Kotlin DSL and follows a multi-module structure:
 # Build the entire project
 ./gradlew build
 
+# Build project without fat JAR tasks
+./gradlew clean :app:build -x :app:shadowJar -x :app:distTar -x :app:distZip
+
 # Build individual modules
 ./gradlew :app:build
 ./gradlew :api:build

api/build.gradle.kts

@@ -1,5 +1,3 @@
-import org.scoverage.ScoverageExtension
-
 /*
  * This file was generated by the Gradle 'init' task.
  *
@@ -20,8 +18,17 @@ plugins {
     `maven-publish`
     signing
 
-    id("org.scoverage") version "8.0.3"
-    id("com.gradleup.shadow") version "8.3.6"
+    alias(libs.plugins.scoverage) apply false
+    alias(libs.plugins.shadow)
+}
+
+// Check if configuration cache is enabled
+val isConfigCacheEnabled = providers.gradleProperty("org.gradle.configuration-cache")
+    .map { it.toBoolean() }.getOrElse(false)
+
+// Conditionally apply Scoverage plugin when configuration cache is disabled
+if (!isConfigCacheEnabled) {
+    apply(plugin = "org.scoverage")
 }
 
 repositories {
@@ -43,11 +50,11 @@ configurations {
 }
 
 dependencies {
-    compileOnly("org.scala-lang:scala-library:$scalaSpecificVersion")
-    compileOnly("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.1")
-    compileOnly("com.fasterxml.jackson.module:jackson-module-scala_$scalaVersion:2.17.1")
+    compileOnly(libs.scala.library)
+    compileOnly(libs.jackson.dataformat.yaml)
+    compileOnly(libs.jackson.module.scala)
 
-    api("com.softwaremill.quicklens:quicklens_$scalaVersion:1.9.7") {
+    api(libs.quicklens) {
         exclude(group = "org.scala-lang")
     }
 }
@@ -58,20 +65,25 @@ testing {
         val test by getting(JvmTestSuite::class) {
             dependencies {
                 // Use Scalatest for testing our library
-                implementation("org.scalatest:scalatest_$scalaVersion:3.2.19")
-                implementation("org.scalatestplus:junit-5-11_$scalaVersion:3.2.19.0")
-                implementation("org.scalamock:scalamock_$scalaVersion:5.2.0")
+                implementation(libs.scalatest)
+                implementation(libs.scalatestplus.junit)
+                implementation(libs.scalamock)
 
                 // Need scala-xml at test runtime
-                runtimeOnly("org.scala-lang.modules:scala-xml_$scalaVersion:1.3.1")
+                runtimeOnly(libs.scala.xml)
             }
         }
     }
 }
 
 tasks.test {
     jvmArgs("-Djava.security.manager=allow", "-Djdk.module.illegalAccess=deny", "--add-opens=java.base/java.lang=ALL-UNNAMED", "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", "--add-opens=java.base/java.io=ALL-UNNAMED", "--add-opens=java.base/java.net=ALL-UNNAMED", "--add-opens=java.base/java.nio=ALL-UNNAMED", "--add-opens=java.base/java.util=ALL-UNNAMED", "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", "--add-opens=java.base/sun.security.action=ALL-UNNAMED", "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED")
-    finalizedBy(tasks.reportScoverage)
+    
+    // Only finalize with Scoverage report when configuration cache is disabled
+    if (!isConfigCacheEnabled) {
+        finalizedBy(tasks.named("reportScoverage"))
+    }
+    
     useJUnitPlatform {
         includeEngines("scalatest")
         testLogging {
@@ -120,13 +132,16 @@ tasks.jar {
 tasks.shadowJar {
     archiveBaseName.set("datacaterer")
     archiveAppendix.set("api")
-    archiveVersion.set(project.version.toString())
+    archiveVersion.set(version.toString())
     archiveClassifier.set("")
     isZip64 = true
 }
 
-configure<ScoverageExtension> {
-    scoverageScalaVersion.set("2.12.15")
+// Configure Scoverage only when it's applied (configuration cache disabled)
+if (!isConfigCacheEnabled) {
+    configure<org.scoverage.ScoverageExtension> {
+        scoverageScalaVersion.set("2.12.15")
+    }
 }
 
 publishing {
@@ -145,7 +160,7 @@ publishing {
             artifact(tasks.shadowJar)
             artifact(tasks["sourcesJar"])
             artifact(tasks["javadocJar"])
-            groupId = project.properties["groupId"].toString()
+            groupId = group.toString()
             artifactId = apiArtifactId
 
             pom {

api/src/main/scala/io/github/datacatering/datacaterer/api/TaskBuilder.scala

@@ -972,7 +972,7 @@ case class FieldBuilder(field: Field = Field()) {
     * Has no effect for non-JSON sinks or non-top-level contexts.
     */
   def unwrapTopLevelArray(enable: Boolean): FieldBuilder =
-    this.modify(_.field.options).setTo(getGenBuilder.unwrapTopLevel(enable).options)
+    this.modify(_.field.options).setTo(getGenBuilder.unwrapTopLevelArray(enable).options)
 
   /**
    * Sets the primary key flag for the current field.
@@ -1547,8 +1547,8 @@ case class GeneratorBuilder(options: Map[String, Any] = Map()) {
   /**
     * Instruct JSON sink to unwrap the top-level field if it is a single array field.
     */
-  def unwrapTopLevel(enable: Boolean): GeneratorBuilder =
-    this.modify(_.options)(_ ++ Map(UNWRAP_TOP_LEVEL -> enable.toString))
+  def unwrapTopLevelArray(enable: Boolean): GeneratorBuilder =
+    this.modify(_.options)(_ ++ Map(UNWRAP_TOP_LEVEL_ARRAY -> enable.toString))
 
   /**
    * Field is a primary key of the data source.

api/src/main/scala/io/github/datacatering/datacaterer/api/connection/ConnectionBuilder.scala

@@ -105,6 +105,16 @@ trait ConnectionTaskBuilder[T] {
     this
   }
 
+  /**
+   * Set the number of records to generate per second. Only used for real time data sources such as JMS, Kafka and HTTP
+   * @param rowsPerSecond The number of records per second to generate
+   * @return The connection task builder
+   */
+  def rowsPerSecond(rowsPerSecond: Int): ConnectionTaskBuilder[T] = {
+    this.step = Some(getStep.rowsPerSecond(rowsPerSecond))
+    this
+  }
+
   /**
    * Generate all possible combinations of the fields. Fields with `oneOf` will be used to generate all possible combinations.
    *

api/src/main/scala/io/github/datacatering/datacaterer/api/model/Constants.scala

@@ -61,9 +61,7 @@ object Constants {
   lazy val DATA_CONTRACT_FILE = "dataContractFile"
   lazy val DATA_CONTRACT_SCHEMA = "dataContractSchema"
   lazy val ROWS_PER_SECOND = "rowsPerSecond"
-  // When applied on a top-level array field in a JSON task, instructs the sink to output a bare JSON array
-  // instead of an object { fieldName: [...] }.
-  lazy val UNWRAP_TOP_LEVEL = "unwrapTopLevel"
+  lazy val UNWRAP_TOP_LEVEL_ARRAY = "unwrapTopLevelArray"
   lazy val HUDI_TABLE_NAME = "hoodie.table.name"
   lazy val ICEBERG_CATALOG_TYPE = "catalogType"
   lazy val ICEBERG_CATALOG_URI = "catalogUri"