Update to 0.15.3, add in kafka relationships example

Author: pflooky

.gitignore

@@ -1,6 +1,7 @@
 .gradle
 **/build/
 !src/**/build/
+.DS_Store
 
 # Ignore Gradle GUI config
 gradle-app.setting

docker/data/custom/application.conf

@@ -210,7 +210,7 @@ jms {
 
 kafka {
     kafkaAccount {
-        kafka.bootstrap.servers = "kafkaserver:29092"
+        kafka.bootstrap.servers = "kafka:29092"
         kafka.bootstrap.servers = ${?KAFKA_BOOTSTRAP_SERVERS}
     }
 }

docker/data/custom/plan/csv-multiple-relationships.yaml

@@ -1,4 +1,4 @@
 name: "csv_multiple_relationships_plan"
 tasks:
-  - name: "csv_customer_files"
+  - name: "csv_file"
     dataSourceName: "csv"

docker/data/custom/plan/kafka.yaml

@@ -1,6 +1,6 @@
 name: "kafka_example_plan"
 description: "Create account data in Kafka"
 tasks:
-  - name: "json_account_kafka"
+  - name: "relationships_kafka"
     dataSourceName: "kafkaAccount"
     enabled: true

docker/data/custom/task/file/csv/csv-test-task.yaml

@@ -0,0 +1,42 @@
+name: "csv_test_files"
+steps:
+  - name: "customers"
+    type: "csv"
+    options:
+      path: "/opt/app/custom/csv/test/customers"
+      header: true
+      saveMode: "Overwrite"
+    count:
+      records: 1
+    fields:
+      - name: "id"
+        type: "long"
+        options:
+          incremental: 1
+      - name: "customer_id"
+        options:
+          uuid: ""
+          incremental: 1
+      - name: "first_name"
+        options:
+          expression: "#{Name.firstName}"
+      - name: "last_name"
+        options:
+          expression: "#{Name.lastName}"
+  - name: "accounts"
+    type: "csv"
+    options:
+      path: "/opt/app/custom/csv/test/accounts"
+      header: true
+      saveMode: "Overwrite"
+    count:
+      records: 1
+    fields:
+      - name: "id"
+        type: "long"
+        options:
+          incremental: 1
+      - name: "customer_id"
+        options:
+          uuid: ""
+          incremental: 1

docker/data/custom/task/kafka/kafka-relationships-task.yaml

@@ -0,0 +1,59 @@
+name: "relationships_kafka"
+steps:
+  - name: "customer"
+    type: "kafka"
+    count:
+      records: 2
+    options:
+      topic: "customer-topic"
+    fields:
+      - name: "key"
+        type: "string"
+        options:
+          sql: "body.customer_id"
+      - name: "messageBody"
+        type: struct
+        fields:
+          - name: "customer_id"
+            options:
+              uuid: ""
+              incremental: 1
+          - name: "first_name"
+            options:
+              expression: "#{Name.firstName}"
+          - name: "last_name"
+            options:
+              expression: "#{Name.lastName}"
+  - name: "customer_product"
+    type: "kafka"
+    count:
+      records: 2
+      perField:
+        count: 2
+        fieldNames:
+          - "body.customer_id"
+    options:
+      topic: "customer-topic"
+    fields:
+      - name: "key"
+        type: "string"
+        options:
+          sql: "body.customer_id"
+      - name: "customer_product_id_int"
+        type: "int"
+        options:
+          min: 10
+          max: 12
+          omit: true
+      - name: "messageBody"
+        type: struct
+        fields:
+          - name: "customer_id"
+            options:
+              uuid: ""
+              incremental: 1
+              isPrimaryKey: true
+          - name: "customer_product_id"
+            options:
+              uuid: "customer_product_id_int"
+              isPrimaryKey: true

docker/data/kafka/setup_kafka.sh

@@ -8,4 +8,7 @@ kafka-topics --bootstrap-server $host:29092 --create --if-not-exists --topic acc
 echo 'Successfully created the following topics:'
 kafka-topics --bootstrap-server $host:29092 --list
 
-#kafka-topics --delete --topic account-topic --bootstrap-server localhost:9092
+#kafka-topics --delete --topic account-topic --bootstrap-server localhost:9092
+#kafka-topics --bootstrap-server localhost:9092 --create --topic customer-product-topic --replication-factor 1 --partitions 1
+#kafka-console-consumer --bootstrap-server localhost:9092 --topic customer-product-topic --from-beginning
+#kafka-console-consumer --bootstrap-server localhost:9092 --topic customer-topic --from-beginning

gradle.properties

@@ -8,5 +8,5 @@ version=0.1.0
 
 scalaVersion=2.12
 scalaSpecificVersion=2.12.19
-dataCatererVersion=0.15.2
+dataCatererVersion=0.15.3
 sparkMajorVersion=3.5