basic docs skeleton

Author: sholokhov

LICENSE

@@ -5,54 +5,34 @@ validations:
     checks:
       - id: row_count > 0
         description: "data is present" # optional
-        severity: error # optional (error, warn, info), default "error"
+        on_fail: error # optional (error, warn), default "error"
 
       - id: row_count between 100 and 30000
         description: "data is not too big"
-        severity: error
+        on_fail: error
 
       - id: null_count(pickup_ntaname) == 0
         description: "no nulls in column" # optional
-        severity: error
+        on_fail: error
 
       - id: min(pickup_datetime) < now() - interval 3 day
         description: "min check"
-        severity: error
+        on_fail: error
 
       - id: stddevPop(trip_distance) < 100_000
         description: "check stddev value"
-        severity: error
+        on_fail: error
 
       - id: sum(fare_amount) <= 10_000_000
         description: "sum of value"
-        severity: error
+        on_fail: error
 
       - id: countIf(trip_id == 1) == 1
         description: "check trip id"
-        severity: warn
+        on_fail: warn
 
       - id: raw_query
         description: "some raw query description here"
-        severity: error
+        on_fail: error
         query: |
-          select countIf(trip_distance == 0) > 0 from {{table}} where 1=1
-
-#  - dataset: pgsql-staging@[public.table_1, public.table_2]
-#    checks:
-#      - id: row_count > 0
-#        severity: warn
-#        type:
-
-
-# v1 supported functions:
-# ---
-# row_count > 10
-# null_count(col) == 0
-# avg(col) <= 24.2
-# max(col) < 1000
-# min(col) == 0
-# sum(col) > 0
-# stddev(col) between 1 and 100_000_000
-# custom
-
-# AI anomaly detection
+          select countIf(trip_distance == 0) > 0 from {{table}} where 1=1

checks.yaml

@@ -29,7 +29,7 @@ By automating these checks, you can proactively identify and address data qualit
 				return fmt.Errorf("error while loading checks configuration file: %w", err)
 			}
 
-			shouldFail := false
+			exitCode := 0
 			for _, rule := range checksCfg.Validations {
 				dataSourceId, datasets, err := parseDatasetString(rule.Dataset)
 				if err != nil {
@@ -50,16 +50,16 @@ By automating these checks, you can proactively identify and address data qualit
 						}
 
 						log.Printf("  [%d/%d] '%s': %s", cIdx+1, len(rule.Checks), check.ID, getCheckResultLabel(pass))
-						if !pass && check.Severity == "error" {
-							shouldFail = true
+						if !pass && internal.IdOrDefault(string(check.OnFail), internal.OnFailActionError) == "error" {
+							exitCode = 1
 						}
 					}
 				}
 			}
 
-			if shouldFail {
-				log.Printf("One or more checks with 'error' severity have failed, exiting...")
-				os.Exit(1)
+			if exitCode != 0 {
+				log.Printf("One or more checks with on_fail = 'error' action have failed, exiting.")
+				os.Exit(exitCode)
 			}
 
 			return nil

cmd/check.go

@@ -13,7 +13,7 @@ func NewProfileCommand(app internal.DbqApp) *cobra.Command {
 
 	cmd := &cobra.Command{
 		Use:   "profile",
-		Short: "Collects dataset's information and generates column statistics",
+		Short: "Collects dataset`s information and generates column statistics",
 		Long: `The 'profile' command connects to the specified data source and analyzes a given dataset. It gathers essential information about the table, such as the total number of rows. 
 Additionally, for each column within the table, it calculates and reports various statistical metrics. These metrics may include the minimum value, maximum value, the count of null or missing values, the data type, 
 and other relevant statistics depending on the data type and the capabilities of the underlying data source.

cmd/profile.go

@@ -5,12 +5,16 @@ import (
 	"github.com/spf13/cobra"
 )
 
+const (
+	DbqVersion = "v0.0.3"
+)
+
 func NewVersionCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "version",
 		Short: "Prints dbq version",
 		Run: func(cmd *cobra.Command, args []string) {
-			fmt.Println("DataBridge Quality Core: 0.0.1")
+			fmt.Printf("DataBridge Quality Core: %s\n", DbqVersion)
 		},
 	}
 

cmd/version.go

@@ -5,6 +5,13 @@ import (
 	"os"
 )
 
+type OnFailAction string
+
+const (
+	OnFailActionError   = "error"
+	OnFailActionWarning = "warn"
+)
+
 type ChecksConfig struct {
 	Version     string       `yaml:"version"`
 	Validations []Validation `yaml:"validations"`
@@ -17,10 +24,10 @@ type Validation struct {
 }
 
 type Check struct {
-	ID          string `yaml:"id"`
-	Description string `yaml:"description,omitempty"` // Optional
-	Severity    string `yaml:"severity,omitempty"`    // Optional (error, warn, info)
-	Query       string `yaml:"query,omitempty"`       // Optional raw query
+	ID          string       `yaml:"id"`
+	Description string       `yaml:"description,omitempty"` // optional
+	OnFail      OnFailAction `yaml:"on_fail,omitempty"`     // optional (error, warn)
+	Query       string       `yaml:"query,omitempty"`       // optional raw query
 }
 
 func LoadChecksConfig(fileName string) (*ChecksConfig, error) {

internal/checks_config.go

@@ -0,0 +1,8 @@
+package internal
+
+func IdOrDefault(original string, defaultVal string) string {
+	if original == "" {
+		return defaultVal
+	}
+	return original
+}

internal/utils.go

@@ -1,65 +1,89 @@
 # DataBridge Quality Core
 
-dbq is a data quality tool that provides a set of tools to validate and test data in your data pipeline. 
-It is designed to be easy to use and integrate into your existing workflow.
+`dbq` is a free, open-source data quality CLI checker that provides a set of tools to profile, validate and test data in your data warehouse or databases. 
+It is designed to be flexible, fast, easy to use and integrate seamlessly into your existing workflow.
 
-## Help
-- dqb ping cnn-id
-- dbq import cnn-id --filter "reporting.*" --cfg checks.yaml --update-cfg
-- dbq check --cfg checks.yaml
-- dbq --config /Users/artem/code/dbq/dbq.yaml import 
-- dbq profile --datasource cnn-id --dataset table_name
-
-## 0.1
-- [x] basic structure
-- [x] define checks cfg v1
-- [x] checks cfg parser v1
-- [x] complete clickhouse support
-  - [x] ping
-  - [x] import datasets
-  - [x] profile dataset
-    - [x] rows in table
-    - [x] min, max, avg, stddev for numeric columns
-    - [x] count of nulls and blanks
-    - [x] most frequent value in column
-    - [x] JSON export
-  - [x] run checks
-- [x] implement support for custom sql check 
-- [x] implement aliases for common checks based on raw sql check
-- [x] fix cmd descriptions
-- [x] review todos
-- [x] improve output
-- [ ] basic cross validation (dataset is defined)
-- [ ] review logs
-- [ ] review crashes (wrong arguments)
-- [ ] default values (e.g. severity)
-- [ ] quiet/verbose mode for logs
-- [ ] docs
-
-## 0.x
-- config validation
-- add postgres support
-- CLI for adding more checks
-- AirFlow integration (operator)
-- output format flag
+## Features
+
+data profiling
 
+v1 supported checks
 ---
+row_count > 10
+null_count(col) == 0
+avg(col) <= 24.2
+max(col) < 1000
+min(col) == 0
+sum(col) > 0
+stddevPop(col) between 1 and 100_000_000
+custom
+
+## Supported databases
+- [ClickHouse](https://clickhouse.com/)
 
-## Checks config specification
-- raw_query(query = "...")
-- row_count
-- null_count(col)
-- <aggr_function> <op> <rest>
+## Usage
 
-### clickhouse 
+### Installation
+
+Download the latest binaries from [GitHub Releases](https://github.com/DataBridge-Tech/dbq/releases).
+
+### Configuration
+
+Create dbq configuration file (default lookup directory is $HOME/.dbq.yaml or ./dbq.yaml). Alternatively,
+you can specify configuration during the launch via `--config` parameter:
 
 ```bash
-docker run -d -p 18123:8123 -p19000:9000 -e CLICKHOUSE_PASSWORD=changeme --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
+dbq --config /path/to/dbq.yaml import
 ```
 
-# Supported Datasources
-- Clickhouse
+```yaml
+# dbq.yaml
+version: "1"
+datasources:
+    - id: clickhouse
+      type: clickhouse
+      configuration:
+        host: 0.0.0.0:19000
+        port: 19000
+        username: default
+        password: changeme
+        database: default
+      datasets:
+        - nyc_taxi.trips_small
+```
+
+### Checks example
+
+### Commands
 
-# dbq configuration
+```bash
+$ dbq help
+
+dbq is a CLI tool for profiling data and running quality checks across various data sources
+
+Usage:
+  dbq [command]
+
+Available Commands:
+  check       Runs data quality checks defined in a configuration file against a datasource
+  completion  Generate the autocompletion script for the specified shell
+  help        Help about any command
+  import      Connects to a data source and imports all available tables as datasets
+  ping        Checks if the data source is reachable
+  profile     Collects dataset`s information and generates column statistics
+  version     Prints dbq version
+
+Flags:
+      --config string   config file (default is $HOME/.dbq.yaml or ./dbq.yaml)
+  -h, --help            help for dbq
+
+Use "dbq [command] --help" for more information about a command.
 
-# checks configuration
+```
+
+### Quick start
+- dqb ping cnn-id
+- dbq import cnn-id --filter "reporting.*" --cfg checks.yaml --update-cfg
+- dbq check --cfg checks.yaml
+- dbq --config /Users/artem/code/dbq/dbq.yaml import 
+- dbq profile --datasource cnn-id --dataset table_name