code clean up

Author: sholokhov

cmd/check.go

@@ -49,6 +49,7 @@ By automating these checks, you can proactively identify and address data qualit
 							log.Printf("Failed to run check: %s", err.Error())
 						}
 						// todo: act on check result
+						// if check.Severity {...}
 					}
 				}
 			}
@@ -57,7 +58,7 @@ By automating these checks, you can proactively identify and address data qualit
 		},
 	}
 
-	cmd.Flags().StringVarP(&checksFile, "checks", "c", "", "Validation checks")
+	cmd.Flags().StringVarP(&checksFile, "checks", "c", "", "Path to data quality checks file")
 	_ = cmd.MarkFlagRequired("checks")
 
 	return cmd

cmd/import.go

@@ -20,32 +20,43 @@ func NewImportCommand(app internal.DbqApp) *cobra.Command {
 This command is useful for quickly onboarding data from external systems, allowing you to easily access and work with already existing data.
 `,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			datasets, err := app.ImportDatasets(dataSource, filter)
-			if err != nil {
-				log.Println("Failed to fetch datasets: " + err.Error())
-				return nil
+			var importFromSources []string
+			if dataSource != "" {
+				importFromSources = append(importFromSources, dataSource)
+			} else {
+				for _, ds := range app.GetDbqConfig().DataSources {
+					importFromSources = append(importFromSources, ds.ID)
+				}
 			}
 
-			log.Printf("Found %d datasets to import: %v\n", len(datasets), datasets)
-			if updateCfg {
+			for _, curDataSource := range importFromSources {
+				datasets, err := app.ImportDatasets(curDataSource, filter)
+				if err != nil {
+					log.Println("Failed to fetch datasets: " + err.Error())
+					return nil
+				}
+
+				log.Printf("Found %d datasets in %s to import: %v\n", len(datasets), curDataSource, datasets)
+
 				ds := app.FindDataSourceById(dataSource)
 				if ds != nil {
 					ds.Datasets = datasets
-					err := app.SaveDbqConfig()
-					if err != nil {
-						return err
-					}
-					log.Println("dbq config has been updated")
 				}
 			}
 
+			if updateCfg {
+				err := app.SaveDbqConfig()
+				if err != nil {
+					return err
+				}
+				log.Println("dbq config has been updated")
+			}
+
 			return nil
 		},
 	}
 
 	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource from which datasets will be imported")
-	_ = cmd.MarkFlagRequired("datasource") // todo: support import from all
-
 	cmd.Flags().StringVarP(&filter, "filter", "f", "", "Filter applied for dataset selection")
 	cmd.Flags().BoolVarP(&updateCfg, "update-checks", "u", false, "Update checks config file in place")
 

cmd/ping.go

@@ -36,7 +36,7 @@ This is useful for quickly determining if the data source is online and respondi
 		},
 	}
 
-	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource")
+	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource to ping")
 
 	return cmd
 }

cmd/profile.go

@@ -47,22 +47,21 @@ and helps in making better decisions about data processing and analysis.
 				}
 			}
 
+			// todo: introduce output format flag
 			jsonData, err := json.Marshal(profileResults)
 			if err != nil {
 				log.Fatalf("Failed to marshal metrics to JSON: %v", err)
 			}
-
-			// todo: handle empty tables
 			log.Println(string(jsonData))
 
 			return nil
 		},
 	}
 
-	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource")
+	cmd.Flags().StringVarP(&dataSource, "datasource", "d", "", "Datasource in which datasets will be profiled")
 	_ = cmd.MarkFlagRequired("datasource")
 
-	cmd.Flags().StringVarP(&dataSet, "dataset", "s", "", "Dataset")
+	cmd.Flags().StringVarP(&dataSet, "dataset", "s", "", "Dataset within specified data source")
 
 	return cmd
 }

cmd/root.go

@@ -28,7 +28,7 @@ func AddCommands(app internal.DbqApp) {
 }
 
 func init() {
-	// todo: workaround for bootstrap config flag & unsupported flag issue
+	// workaround for bootstrap config flag & unsupported flag issue
 	var dbqConfigFile string
 	rootCmd.PersistentFlags().StringVar(&dbqConfigFile, "config", "", "config file (default is $HOME/.dbq.yaml or ./dbq.yaml)")
 }

internal/clickhouse.go

@@ -103,8 +103,6 @@ func (c *ClickhouseDbqConnector) ProfileDataset(dataset string) (*TableMetrics,
 
 	log.Printf("Calculating metrics for table: %s", dataset)
 
-	// ProfileDataSet todo: optimize/batch queries where possible
-
 	// Total Row Count
 	log.Printf("Fetching total row count...")
 	err := c.cnn.QueryRow(ctx, fmt.Sprintf("SELECT count() FROM %s", dataset)).Scan(&metrics.TotalRows)

readme.md

@@ -12,9 +12,9 @@ It is designed to be easy to use and integrate into your existing workflow.
 
 ## 0.1
 - [x] basic structure
-- [ ] define checks cfg v1
-- [ ] checks cfg parser v1
-- [ ] complete clickhouse support
+- [x] define checks cfg v1
+- [x] checks cfg parser v1
+- [x] complete clickhouse support
   - [x] ping
   - [x] import datasets
   - [x] profile dataset
@@ -23,25 +23,25 @@ It is designed to be easy to use and integrate into your existing workflow.
     - [x] count of nulls and blanks
     - [x] most frequent value in column
     - [x] JSON export
-  - [ ] run checks
-- [ ] implement support for custom sql check 
-- [ ] implement aliases for common checks based on raw sql check
-- [ ] basic cross validation (dataset is defined)
+  - [x] run checks
+- [x] implement support for custom sql check 
+- [x] implement aliases for common checks based on raw sql check
 - [x] fix cmd descriptions
-- [ ] review todos
+- [x] review todos
+- [ ] basic cross validation (dataset is defined)
 - [ ] review logs
 - [ ] review crashes (wrong arguments)
 - [ ] default values (e.g. severity)
 - [ ] improve output
 - [ ] quiet/verbose mode for logs
+- [ ] docs
 
-## 0.2
+## 0.x
 - config validation
 - add postgres support
 - CLI for adding more checks
-- schema changes checks
 - AirFlow integration (operator)
-
+- output format flag
 
 ---