logger improvements

Author: sholokhov

cmd/check.go

@@ -18,13 +18,18 @@ import (
 	"fmt"
 	"github.com/DataBridgeTech/dbqcore"
 	"github.com/DataBridgeTech/dbqctl/internal"
-	"log"
+	"log/slog"
 	"os"
 	"strings"
 
 	"github.com/spf13/cobra"
 )
 
+type FailedCheckDetails struct {
+	ID  string
+	Err error
+}
+
 func NewCheckCommand(app internal.DbqCliApp) *cobra.Command {
 	var checksFile string
 
@@ -37,7 +42,8 @@ which outlines the rules and constraints that the data within the dataset should
 By automating these checks, you can proactively identify and address data quality issues, ensuring that your datasets meet the required standards for analysis and decision-making.
 `,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			log.Printf("Reading checks configuration file: %s \n", checksFile)
+			slog.Debug("Reading checks configuration file",
+				"checks_config_path", checksFile)
 
 			checksCfg, err := dbqcore.LoadChecksConfig(checksFile)
 			if err != nil {
@@ -56,24 +62,35 @@ By automating these checks, you can proactively identify and address data qualit
 					return fmt.Errorf("specified data source not found in dbq configuration: %s", dataSourceId)
 				}
 
-				for dsIdx, dataset := range datasets {
-					log.Printf("[%d/%d] Running quality checks for: %s", dsIdx+1, len(datasets), dataset)
-					for cIdx, check := range rule.Checks {
-						pass, _, err := app.RunCheck(&check, dataSource, dataset, rule.Where)
+				var failedChecks []FailedCheckDetails
+				for _, dataset := range datasets {
+					fmt.Printf("running %d quality checks for '%s'\n", len(rule.Checks), dataset)
+					for _, check := range rule.Checks {
+						pass, _, _ := app.RunCheck(&check, dataSource, dataset, rule.Where)
+						fmt.Printf("  check %s ('%s') ... %s\n", check.Description, check.ID, getCheckResultLabel(pass))
+
 						if err != nil {
-							log.Printf("Failed to run check: %s", err.Error())
+							failedChecks = append(failedChecks, FailedCheckDetails{ID: check.ID, Err: err})
 						}
 
-						log.Printf("  [%d/%d] '%s': %s", cIdx+1, len(rule.Checks), check.ID, getCheckResultLabel(pass))
 						if !pass && strGetOrDefault(string(check.OnFail), dbqcore.OnFailActionError) == dbqcore.OnFailActionError {
 							exitCode = 1
 						}
 					}
 				}
+
+				if len(failedChecks) != 0 {
+					for _, result := range failedChecks {
+						fmt.Println()
+						fmt.Printf("--- %s ---\n", result.ID)
+						fmt.Printf("error: %s\n", result.Err)
+					}
+				}
 			}
 
 			if exitCode != 0 {
-				log.Printf("One or more checks with on_fail = 'error' action have failed, exiting.")
+				// todo: print detailed report
+				// fmt.Printf("\ncheck result: FAILED. 1 passed; 1 failed; \n")
 				os.Exit(exitCode)
 			}
 
@@ -124,9 +141,9 @@ func parseDatasetString(input string) (datasource string, datasets []string, err
 
 func getCheckResultLabel(passed bool) string {
 	if passed {
-		return "passed"
+		return "ok"
 	} else {
-		return "failed"
+		return "FAILED"
 	}
 }
 

cmd/import.go

@@ -15,10 +15,10 @@
 package cmd
 
 import (
-	"log"
-
+	"fmt"
 	"github.com/DataBridgeTech/dbqctl/internal"
 	"github.com/spf13/cobra"
+	"log"
 )
 
 func NewImportCommand(app internal.DbqCliApp) *cobra.Command {
@@ -50,7 +50,7 @@ This command is useful for quickly onboarding data from external systems, allowi
 					return nil
 				}
 
-				log.Printf("Found %d datasets in %s to import: %v\n", len(datasets), curDataSource, datasets)
+				fmt.Printf("found %d datasets in %s to import: %v\n", len(datasets), curDataSource, datasets)
 
 				ds := app.FindDataSourceById(dataSource)
 				if ds != nil {
@@ -63,7 +63,7 @@ This command is useful for quickly onboarding data from external systems, allowi
 				if err != nil {
 					return err
 				}
-				log.Println("dbq config has been updated")
+				fmt.Println("dbq config has been updated")
 			}
 
 			return nil

cmd/ping.go

@@ -15,8 +15,7 @@
 package cmd
 
 import (
-	"log"
-
+	"fmt"
 	"github.com/DataBridgeTech/dbqctl/internal"
 	"github.com/spf13/cobra"
 )
@@ -40,12 +39,12 @@ This is useful for quickly determining if the data source is online and respondi
 			}
 
 			for _, curDataSource := range sourcesToPing {
-				log.Printf("Pinging data source: %s...\n", curDataSource)
+				fmt.Printf("Conneting to data source: %s...\n", curDataSource)
 				info, err := app.PingDataSource(curDataSource)
 				if err != nil {
-					log.Printf("Connection failed: %s\n", err.Error())
+					fmt.Printf("Connection failed: %s\n", err.Error())
 				} else {
-					log.Printf("Connected: %s\n", info)
+					fmt.Printf("Connected: %s\n", info)
 				}
 			}
 		},

cmd/profile.go

@@ -16,8 +16,7 @@ package cmd
 
 import (
 	"encoding/json"
-	"log"
-
+	"fmt"
 	"github.com/DataBridgeTech/dbqcore"
 	"github.com/DataBridgeTech/dbqctl/internal"
 	"github.com/spf13/cobra"
@@ -58,7 +57,7 @@ and helps in making better decisions about data processing and analysis.
 			for _, curDataSet := range dataSetsToProfile {
 				metrics, err := app.ProfileDataset(dataSource, curDataSet, sample)
 				if err != nil {
-					log.Printf("Failed to profile %s: %s\n", curDataSet, err)
+					fmt.Printf("Failed to profile %s: %s\n", curDataSet, err)
 				} else {
 					profileResults.Profiles[curDataSet] = metrics
 				}
@@ -67,9 +66,10 @@ and helps in making better decisions about data processing and analysis.
 			// todo: introduce output format flag
 			jsonData, err := json.Marshal(profileResults)
 			if err != nil {
-				log.Fatalf("Failed to marshal metrics to JSON: %v", err)
+				fmt.Println("failed to marshal metrics to JSON")
+				panic(err)
 			}
-			log.Println(string(jsonData))
+			fmt.Println(string(jsonData))
 
 			return nil
 		},