support filter in import cmd

Author: sholokhov

internal/app.go

@@ -25,31 +25,10 @@ type DbqAppImpl struct {
 }
 
 func NewDbqApp(dbqConfigPath string) DbqApp {
-	v := viper.New()
-
-	if dbqConfigPath != "" {
-		v.SetConfigFile(dbqConfigPath)
-	} else {
-		home, err := os.UserHomeDir()
-		cobra.CheckErr(err)
-		v.AddConfigPath(home)
-		v.SetConfigType("yaml")
-		v.SetConfigName(".dbq.yaml")
-	}
-
-	v.AutomaticEnv()
-	if err := v.ReadInConfig(); err != nil {
-		cobra.CheckErr(err)
-	}
-
-	var dbqConfig DbqConfig
-	if err := v.Unmarshal(&dbqConfig); err != nil {
-		cobra.CheckErr(err)
-	}
-
+	dbqConfig := initConfig(dbqConfigPath)
 	return &DbqAppImpl{
 		dbqConfigPath: dbqConfigPath,
-		dbqConfig:     &dbqConfig,
+		dbqConfig:     dbqConfig,
 	}
 }
 
@@ -77,7 +56,7 @@ func (app *DbqAppImpl) ImportDatasets(srcId string, filter string) ([]string, er
 		return []string{}, err
 	}
 
-	return cnn.ImportDatasets()
+	return cnn.ImportDatasets(filter)
 }
 
 func (app *DbqAppImpl) GetDbqConfig() *DbqConfig {
@@ -107,6 +86,32 @@ func (app *DbqAppImpl) FindDataSourceById(srcId string) *DataSource {
 	return nil
 }
 
+func initConfig(dbqConfigPath string) *DbqConfig {
+	v := viper.New()
+
+	if dbqConfigPath != "" {
+		v.SetConfigFile(dbqConfigPath)
+	} else {
+		home, err := os.UserHomeDir()
+		cobra.CheckErr(err)
+		v.AddConfigPath(home)
+		v.SetConfigType("yaml")
+		v.SetConfigName(".dbq.yaml")
+	}
+
+	v.AutomaticEnv()
+	if err := v.ReadInConfig(); err != nil {
+		cobra.CheckErr(err)
+	}
+
+	var dbqConfig DbqConfig
+	if err := v.Unmarshal(&dbqConfig); err != nil {
+		cobra.CheckErr(err)
+	}
+
+	return &dbqConfig
+}
+
 func getDbqConnector(ds DataSource) (DbqConnector, error) {
 	dsType := strings.ToLower(ds.Type)
 	switch dsType {

internal/clickhouse.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"strings"
 )
 
 type ClickhouseDbqConnector struct {
@@ -33,21 +34,29 @@ func (c *ClickhouseDbqConnector) Ping() error {
 	return c.cnn.Ping(context.Background())
 }
 
-func (c *ClickhouseDbqConnector) ImportDatasets() ([]string, error) {
+func (c *ClickhouseDbqConnector) ImportDatasets(filter string) ([]string, error) {
 	if c.cnn == nil {
 		return nil, fmt.Errorf("database connection is not initialized")
 	}
 
-	// Query to select database and table names from the system.tables table.
-	// We exclude the 'system' database as it usually contains internal tables.
-	// You might want to exclude 'INFORMATION_SCHEMA' or others depending on your needs.
-	query := `SELECT database, name FROM system.tables WHERE database NOT IN ('system', 'information_schema', 'INFORMATION_SCHEMA') ORDER BY database, name;`
+	var args []interface{}
+	query := `
+        SELECT database, name
+        FROM system.tables
+        WHERE database NOT IN ('system', 'information_schema', 'INFORMATION_SCHEMA')`
 
-	rows, err := c.cnn.Query(context.Background(), query)
+	filter = strings.TrimSpace(filter)
+	if filter != "" {
+		query += ` AND name LIKE ?`
+		args = append(args, "%"+filter+"%")
+	}
+	query += ` ORDER BY database, name;`
+
+	rows, err := c.cnn.Query(context.Background(), query, args...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to query system.tables: %w", err)
 	}
-	defer rows.Close() // todo: needed?
+	defer rows.Close()
 
 	var datasets []string
 	for rows.Next() {
@@ -64,31 +73,3 @@ func (c *ClickhouseDbqConnector) ImportDatasets() ([]string, error) {
 
 	return datasets, nil
 }
-
-//func hehe_click() {
-//	conn, err := connect()
-//	if err != nil {
-//		panic(err)
-//	}
-//
-//	ctx := context.Background()
-//	rows, err := conn.Query(ctx, "SELECT name, toString(uuid) as uuid_str FROM system.tables LIMIT 5")
-//	if err != nil {
-//		log.Fatal(err)
-//	}
-//
-//	for rows.Next() {
-//		var (
-//			name, uuid string
-//		)
-//		if err := rows.Scan(
-//			&name,
-//			&uuid,
-//		); err != nil {
-//			log.Fatal(err)
-//		}
-//		log.Printf("name: %s, uuid: %s",
-//			name, uuid)
-//	}
-//
-//}

internal/dbq_connector.go

@@ -2,5 +2,5 @@ package internal
 
 type DbqConnector interface {
 	Ping() error
-	ImportDatasets() ([]string, error)
+	ImportDatasets(filter string) ([]string, error)
 }