support multiple datasets in checks.yaml

Author: sholokhov

checks.yaml

@@ -1,6 +1,6 @@
 version: "1"
 validations:
-  - dataset: ch-local@[nyc_taxi.trips_small]
+  - dataset: ch-local@[nyc_taxi.trips_small, nyc_taxi.trips_big]
     where: "pickup_datetime > '2014-01-01'"
     checks:
       - id: row_count > 0

cmd/check.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"dbq/internal"
+	"fmt"
 	"log"
 	"strings"
 
@@ -24,25 +25,31 @@ By automating these checks, you can proactively identify and address data qualit
 
 			checksCfg, err := internal.LoadChecksConfig(checksFile)
 			if err != nil {
-				log.Printf("Failed to read checks configuration: %s", err.Error())
+				return fmt.Errorf("error while loading checks configuration file: %w", err)
 			}
 
-			for i, ruleSet := range checksCfg.Validations {
-				log.Printf("Running check for %s [%d/%d]", ruleSet.Dataset, i+1, len(checksCfg.Validations))
+			for i, rule := range checksCfg.Validations {
+				log.Printf("Running check for %s [%d/%d]", rule.Dataset, i+1, len(checksCfg.Validations))
 
-				// todo: validation
-				parts := strings.Split(ruleSet.Dataset, "@")
-				dataSourceId := parts[0]
-				dataSet := parts[1] // todo: parse list
+				dataSourceId, datasets, err := parseDatasetString(rule.Dataset)
+				if err != nil {
+					return fmt.Errorf("error while parsing dataset property: %w", err)
+				}
 
 				dataSource := app.FindDataSourceById(dataSourceId)
+				if dataSource == nil {
+					return fmt.Errorf("specified data source not found in dbq configuration: %s", dataSourceId)
+				}
 
-				for _, check := range ruleSet.Checks {
-					_, err := app.RunCheck(&check, dataSource, dataSet, ruleSet.Where)
-					if err != nil {
-						log.Printf("Failed to run check: %s", err.Error())
+				for dsIdx, dataset := range datasets {
+					log.Printf("  [%d/%d] Running checks for: %s", dsIdx+1, len(datasets), dataset)
+					for _, check := range rule.Checks {
+						_, err := app.RunCheck(&check, dataSource, dataset, rule.Where)
+						if err != nil {
+							log.Printf("Failed to run check: %s", err.Error())
+						}
+						// todo: act on check result
 					}
-					// todo: act on check result
 				}
 			}
 
@@ -55,3 +62,38 @@ By automating these checks, you can proactively identify and address data qualit
 
 	return cmd
 }
+
+func parseDatasetString(input string) (datasource string, datasets []string, err error) {
+	atIndex := strings.Index(input, "@")
+	if atIndex == -1 {
+		return "", nil, fmt.Errorf("invalid dataset string format: %s", input)
+	}
+
+	datasource = strings.TrimSpace(input[:atIndex])
+	if datasource == "" {
+		return "", nil, fmt.Errorf("datasource part cannot be empty: %s", input)
+	}
+
+	datasetPart := strings.TrimSpace(input[atIndex+1:])
+	if !strings.HasPrefix(datasetPart, "[") || !strings.HasSuffix(datasetPart, "]") {
+		return "", nil, fmt.Errorf("invalid dataset format (expected '[dataset1, dataset2,...]'): %s", input)
+	}
+
+	// slice off '[' and ']'
+	datasetsContent := datasetPart[1 : len(datasetPart)-1]
+	trimmedContent := strings.TrimSpace(datasetsContent)
+	if trimmedContent == "" {
+		return "", nil, fmt.Errorf("dataset part can't be empty: %s", input)
+	}
+
+	rawDatasets := strings.Split(datasetsContent, ",")
+	datasets = make([]string, 0, len(rawDatasets))
+	for _, ds := range rawDatasets {
+		cleanedDS := strings.TrimSpace(ds)
+		if cleanedDS != "" {
+			datasets = append(datasets, cleanedDS)
+		}
+	}
+
+	return datasource, datasets, nil
+}

readme.md

@@ -56,3 +56,6 @@ It is designed to be easy to use and integrate into your existing workflow.
 ```bash
 docker run -d -p 18123:8123 -p19000:9000 -e CLICKHOUSE_PASSWORD=changeme --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
 ```
+
+# Supported Datasources
+- Clickhouse