initial commit

Author: sholokhov

.gitignore

@@ -0,0 +1,29 @@
+.idea
+
+### Go template
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env
+

LICENSE

@@ -0,0 +1,36 @@
+version: 0.1
+validations:
+  - dataset: clickhouse-prod-euw:ads.my_table_1
+    checks:
+      - id: row number between 0 and 100
+        severity: warn
+        type: row_count
+        params:
+            filter: "timestamp > '2023-01-01'"
+            min: 0
+            max: 100
+
+      - id: "no null values"
+        description: "Check if table_1 has the correct number of columns"
+        severity: error
+        type: custom
+        params:
+          query: SELECT COUNT(*) FROM table_1 WHERE column_name IS NULL
+          expected: 0
+
+      - id: "no duplicates"
+        description: "Check if table_1 has no duplicates"
+        severity: error
+        type: no_duplicates
+        params:
+          columns: ["column_1", "column_2"]
+          filter: "timestamp > '2023-01-01'"
+
+  - dataset: pgsql-staging@[public.table_1, public.table_2]
+    checks:
+      - id: "row count between 0 and 100"
+        severity: warn
+        type: row_count
+        params:
+          min: 0
+          max: 1000

README.md

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	"dbq/internal"
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+func NewCheckCommand(app internal.DbqApp) *cobra.Command {
+	var checksFile string
+	var dataSource string
+
+	cmd := &cobra.Command{
+		Use:   "check",
+		Short: "A brief description of your command",
+		Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			fmt.Println("Reading checks from " + checksFile)
+			if dataSource != "" {
+				fmt.Println("Data source is not empty: " + dataSource)
+			}
+			app.GetDbqConfig()
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource")
+	cmd.Flags().StringVarP(&checksFile, "checks", "c", "", "Validation checks")
+	_ = cmd.MarkFlagRequired("checks")
+
+	return cmd
+}

checks.yaml

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"dbq/internal"
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+func NewImportCommand(app internal.DbqApp) *cobra.Command {
+	var dataSource string
+	var filter string
+	var updateCfg bool
+
+	cmd := &cobra.Command{
+		Use:   "import",
+		Short: "A brief description of your command",
+		Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			datasets, err := app.ImportDatasets(dataSource, filter)
+			if err != nil {
+				fmt.Println("Failed to fetch datasets: " + err.Error())
+				return nil
+			}
+
+			for _, dataset := range datasets {
+				fmt.Println("Imported dataset: ", dataset)
+			}
+
+			if updateCfg {
+				fmt.Println("Updating dbq config...")
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource")
+	cmd.Flags().StringVarP(&filter, "filter", "f", "", "Filter")
+	cmd.Flags().BoolVarP(&updateCfg, "update-checks", "u", false, "Update checks config in place")
+
+	return cmd
+}

cmd/check.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"dbq/internal"
+	"fmt"
+	"github.com/spf13/cobra"
+)
+
+func NewPingCommand(app internal.DbqApp) *cobra.Command {
+	var dataSource string
+
+	cmd := &cobra.Command{
+		Use:   "ping",
+		Short: "A brief description of your command",
+		Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			err := app.PingDataSource(dataSource)
+			if err != nil {
+				fmt.Printf("Connection failed: %s\n", err.Error())
+			} else {
+				fmt.Println("Connection works")
+			}
+		},
+	}
+
+	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource")
+	//_ = cmd.MarkFlagRequired("dataSource")
+
+	return cmd
+}

cmd/import.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"dbq/internal"
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+func NewProfileCommand(app internal.DbqApp) *cobra.Command {
+	var dataSource string
+	var dataSet string
+
+	cmd := &cobra.Command{
+		Use:   "profile",
+		Short: "A brief description of your command",
+		Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			fmt.Printf("profiling %s in %s\n", dataSet, dataSource)
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource")
+	_ = cmd.MarkFlagRequired("datasource")
+
+	cmd.Flags().StringVarP(&dataSet, "dataset", "s", "", "Dataset")
+
+	return cmd
+}

cmd/ping.go

@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"dbq/internal"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "dbq",
+	Short: "A brief description of your application",
+	Long: `A longer description that spans multiple lines and likely contains
+examples and usage of using your application. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+}
+
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func AddCommands(app internal.DbqApp) {
+	rootCmd.AddCommand(NewPingCommand(app))
+	rootCmd.AddCommand(NewImportCommand(app))
+	rootCmd.AddCommand(NewCheckCommand(app))
+	rootCmd.AddCommand(NewProfileCommand(app))
+	rootCmd.AddCommand(NewVersionCommand())
+}
+
+func init() {
+	// todo: workaround for bootstrap config flag & unsupported flag issue
+	var dbqConfigFile string
+	rootCmd.PersistentFlags().StringVar(&dbqConfigFile, "config", "", "config file (default is $HOME/.dbq.yaml or ./dbq.yaml)")
+}

cmd/profile.go

@@ -0,0 +1,24 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+)
+
+func NewVersionCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "version",
+		Short: "A brief description of your command",
+		Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Println("DataBridge Quality Core: 0.0.1")
+		},
+	}
+
+	return cmd
+}