code clean up

Author: sholokhov

checks.yaml

@@ -1,13 +1,16 @@
 version: "1"
 validations:
-  # todo: support arrays of data sets [ ... ]
-  - dataset: ch-local@nyc_taxi.trips_small
+  - dataset: ch-local@[nyc_taxi.trips_small]
     where: "pickup_datetime > '2014-01-01'"
     checks:
       - id: row_count > 0
         description: "data is present" # optional
         severity: error # optional (error, warn, info), default "error"
 
+      - id: row_count between 100 and 30000
+        description: "data is not too big"
+        severity: error
+
       - id: null_count(pickup_ntaname) == 0
         description: "no nulls in column" # optional
         severity: error
@@ -16,7 +19,6 @@ validations:
         description: "min check"
         severity: error
 
-      # todo: support "between X and Y"
       - id: stddevPop(trip_distance) < 100_000
         description: "check stddev value"
         severity: error
@@ -51,34 +53,4 @@ validations:
 # stddev(col) between 1 and 100_000_000
 # custom
 
-# AI anomaly detection
-
-# empties / blanks
-# invalid count with filter
-# duplicates
-#  avg_length
-#  cross
-#  distribution
-#  duplicate_count
-#  duplicate_percent
-#  failed rows
-#  freshness
-#  group by
-#  group evolution
-#  invalid_count
-#  invalid_percent
-#  max_length
-#  min_length
-#  missing_count
-#  missing_percent
-#  percentile
-#  reconciliation
-#  reference
-#  schema
-#  schema evolution
-#  stddev_pop
-#  stddev_samp
-#  user-defined
-#  variance
-#  var_pop
-#  var_samp
+# AI anomaly detection

cmd/import.go

@@ -43,9 +43,11 @@ This command is useful for quickly onboarding data from external systems, allowi
 		},
 	}
 
-	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource")
-	cmd.Flags().StringVarP(&filter, "filter", "f", "", "Filter")
-	cmd.Flags().BoolVarP(&updateCfg, "update-checks", "u", false, "Update checks config in place")
+	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource from which datasets will be imported")
+	_ = cmd.MarkFlagRequired("datasource") // todo: support import from all
+
+	cmd.Flags().StringVarP(&filter, "filter", "f", "", "Filter applied for dataset selection")
+	cmd.Flags().BoolVarP(&updateCfg, "update-checks", "u", false, "Update checks config file in place")
 
 	return cmd
 }

cmd/profile.go

@@ -39,7 +39,7 @@ and helps in making better decisions about data processing and analysis.
 			}
 
 			for _, curDataSet := range dataSetsToProfile {
-				metrics, err := app.ProfileDataSourceById(dataSource, curDataSet)
+				metrics, err := app.ProfileDataset(dataSource, curDataSet)
 				if err != nil {
 					log.Printf("Failed to profile %s: %s\n", curDataSet, err)
 				} else {

dbq.yaml

@@ -9,6 +9,7 @@ datasources:
         password: changeme
         database: default
       datasets:
+        - nyc_taxi.trips_big
         - nyc_taxi.trips_small
     - id: pgsql
       type: postgres

internal/app.go

@@ -1,7 +1,6 @@
 package internal
 
 import (
-	"errors"
 	"fmt"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -13,7 +12,7 @@ import (
 type DbqApp interface {
 	PingDataSource(srcId string) (string, error)
 	ImportDatasets(srcId string, filter string) ([]string, error)
-	ProfileDataSourceById(srcId string, dataset string) (*TableMetrics, error)
+	ProfileDataset(srcId string, dataset string) (*TableMetrics, error)
 	GetDbqConfig() *DbqConfig
 	SaveDbqConfig() error
 	FindDataSourceById(srcId string) *DataSource
@@ -26,9 +25,9 @@ type DbqAppImpl struct {
 }
 
 func NewDbqApp(dbqConfigPath string) DbqApp {
-	dbqConfig := initConfig(dbqConfigPath)
+	dbqConfig, dbqConfigUsedPath := initConfig(dbqConfigPath)
 	return &DbqAppImpl{
-		dbqConfigPath: dbqConfigPath,
+		dbqConfigPath: dbqConfigUsedPath,
 		dbqConfig:     dbqConfig,
 	}
 }
@@ -56,16 +55,16 @@ func (app *DbqAppImpl) ImportDatasets(srcId string, filter string) ([]string, er
 		return []string{}, err
 	}
 
-	return cnn.ImportDataSets(filter)
+	return cnn.ImportDatasets(filter)
 }
 
-func (app *DbqAppImpl) ProfileDataSourceById(srcId string, dataset string) (*TableMetrics, error) {
+func (app *DbqAppImpl) ProfileDataset(srcId string, dataset string) (*TableMetrics, error) {
 	var dataSource = app.FindDataSourceById(srcId)
 	cnn, err := getDbqConnector(*dataSource)
 	if err != nil {
 		return nil, err
 	}
-	return cnn.ProfileDataSet(dataset)
+	return cnn.ProfileDataset(dataset)
 }
 
 func (app *DbqAppImpl) GetDbqConfig() *DbqConfig {
@@ -103,7 +102,7 @@ func (app *DbqAppImpl) RunCheck(check *Check, dataSource *DataSource, dataSet st
 	return cnn.RunCheck(check, dataSet, defaultWhere)
 }
 
-func initConfig(dbqConfigPath string) *DbqConfig {
+func initConfig(dbqConfigPath string) (*DbqConfig, string) {
 	v := viper.New()
 
 	if dbqConfigPath != "" {
@@ -126,7 +125,7 @@ func initConfig(dbqConfigPath string) *DbqConfig {
 		cobra.CheckErr(err)
 	}
 
-	return &dbqConfig
+	return &dbqConfig, v.ConfigFileUsed()
 }
 
 func getDbqConnector(ds DataSource) (DbqConnector, error) {
@@ -135,6 +134,6 @@ func getDbqConnector(ds DataSource) (DbqConnector, error) {
 	case "clickhouse":
 		return NewClickhouseDbqConnector(ds)
 	default:
-		return nil, errors.New(fmt.Sprintf("Data source type '%s' is not supported.", dsType))
+		return nil, fmt.Errorf("data source type '%s' is not supported", dsType)
 	}
 }