Fixed SQL scrips and added trim

Author: marcosqlbi

scripts/build_data/build_all.cmd

@@ -1,14 +1,17 @@
 @ECHO OFF
 ECHO START
 
+CALL :buildandzip  csv-10k    
 CALL :buildandzip  csv-100k    
 CALL :buildandzip  csv-1m
 CALL :buildandzip  csv-10m
 CALL :buildandzip  csv-100m      -v500m
+CALL :buildandzip  delta-10k  
 CALL :buildandzip  delta-100k  
 CALL :buildandzip  delta-1m
 CALL :buildandzip  delta-10m
 CALL :buildandzip  delta-100m    -v500m
+CALL :buildandzip  parquet-10k 
 CALL :buildandzip  parquet-100k 
 CALL :buildandzip  parquet-1m
 CALL :buildandzip  parquet-10m

scripts/build_data/build_single.cmd

@@ -9,6 +9,9 @@ IF "%1" == "" (
 
 ECHO OPTION:: %id%
 
+IF "%id%" == "csv-10k"       CALL :do_build  %id%  CSV         10000      0.05  2015-01-01  10  2021-05-18  2024-04-20  ""
+IF "%id%" == "delta-10k"     CALL :do_build  %id%  DELTATABLE  10000      0.05  2015-01-01  10  2021-05-18  2024-04-20  2000
+IF "%id%" == "parquet-10k"   CALL :do_build  %id%  PARQUET     10000      0.05  2015-01-01  10  2021-05-18  2024-04-20  ""
 IF "%id%" == "csv-100k"      CALL :do_build  %id%  CSV         100000     0.05  2015-01-01  10  2014-05-18  2024-04-20  ""
 IF "%id%" == "delta-100k"    CALL :do_build  %id%  DELTATABLE  100000     0.05  2015-01-01  10  2014-05-18  2024-04-20  20000
 IF "%id%" == "parquet-100k"  CALL :do_build  %id%  PARQUET     100000     0.05  2015-01-01  10  2014-05-18  2024-04-20  ""

scripts/sql/SQLBI_ALL_DB.cmd

@@ -1,18 +1,19 @@
 @ECHO OFF
 CLS
 
-SET SqlServerName=.\K22
+SET SqlServerName=Demo
 
 @ECHO.
 ECHO SqlServerName: %SqlServerName%
 @ECHO.
 PAUSE
 
 
-CALL :BUILDBATABASE  "csv-100k"  "Contoso V2 100k"  "ContosoV2100k.bak"
-CALL :BUILDBATABASE  "csv-1m"    "Contoso V2 1M"    "ContosoV21M.bak"    
-REM CALL :BUILDBATABASE  "csv-10m"   "Contoso V2 10M"   "ContosoV210M.bak"   
-REM CALL :BUILDBATABASE  "csv-100m"  "Contoso V2 100M"  "ContosoV2100M.bak"  "-v500m"
+CALL :BUILDBATABASE  "csv-10k"   "Contoso V2 10k"   "ContosoV210k.bak"   "Trim" 
+CALL :BUILDBATABASE  "csv-100k"  "Contoso V2 100k"  "ContosoV2100k.bak"  "Trim" 
+CALL :BUILDBATABASE  "csv-1m"    "Contoso V2 1M"    "ContosoV21M.bak"    "Trim" 
+CALL :BUILDBATABASE  "csv-10m"   "Contoso V2 10M"   "ContosoV210M.bak"   "Trim" 
+CALL :BUILDBATABASE  "csv-100m"  "Contoso V2 100M"  "ContosoV2100M.bak"  "Trim"  "-v500m"
 
 ECHO.
 ECHO ######### The end #########
@@ -25,7 +26,7 @@ GOTO :EOF
 
 :BUILDBATABASE
 ECHO.
-ECHO BUILDBATABASE params: - data: %~1  db-name: %~2  backup-file: %~3  7zip-param: %~4
+ECHO BUILDBATABASE params: - data: %~1  db-name: %~2  backup-file: %~3  Trim: %~4  7zip-param: %~5
 
 SET DatabaseName=%~2
 
@@ -46,13 +47,13 @@ COPY ..\build_data\out\%~1\*.csv inputcsv
 
 ECHO --- Fill database
 ECHO --------------------------------------------------------------------------------
-CALL Sql_ImportData.cmd both
+CALL Sql_ImportData.cmd both %~4
 
 ECHO --- Backup database
 ECHO --------------------------------------------------------------------------------
 SQLCMD -S "%SqlServerName%" -d "%DatabaseName%" -Q "select '$(varBackupFile)'; select '$(varDatabaseName)'; BACKUP DATABASE [$(varDatabaseName)] TO  DISK = '$(varBackupFile)'  WITH  COPY_ONLY, NOFORMAT, INIT, NAME = N'ContosoDGV2 full database backup', SKIP, NOREWIND, NOUNLOAD, STATS = 10;" -v varBackupFile="%CD%\dump\fullbackup.bak" -v varDatabaseName="%DatabaseName%"
 RENAME dump\fullbackup.bak  %~3
-..\build_data\bin\7za.exe  a  dump\%~3.7z  dump\%~3  %~4
+..\build_data\bin\7za.exe  a  dump\%~3.7z  dump\%~3  %~5
 
 ECHO.
 EXIT /B 0

scripts/sql/SQLBI_CreateSqlDatabases.ps1

@@ -17,14 +17,13 @@ param(
 
 
 # Include the list of rows/database name for the database to generate
-# Use 'TrimCustomers' to remove Customers that have no transactions in Sales
 $databases = @()
 $databases += [System.Tuple]::Create( 'Contoso V2 10K' )
 $databases += [System.Tuple]::Create( 'Contoso V2 100K' )
 $databases += [System.Tuple]::Create( 'Contoso V2 1M' )
 # $databases += [System.Tuple]::Create( 'Contoso V2 10M DimRatio' )
 $databases += [System.Tuple]::Create( 'Contoso V2 10M' )
-# $databases += [System.Tuple]::Create( 'Contoso V2 100M' )
+$databases += [System.Tuple]::Create( 'Contoso V2 100M' )
 # $databases += [System.Tuple]::Create( 'Contoso V2 1G' )
 
 

scripts/sql/Sql_ImportData.cmd

@@ -1,8 +1,8 @@
 @ECHO OFF
 ECHO.
 
-IF "%1" == ""  SET SqlServerName=(LocalDb)\MSSQLLocalDB
-IF "%1" == ""  SET DatabaseName=ContosoDGV2Test
+IF "%1" == ""  SET SqlServerName=Demo
+IF "%1" == ""  SET DatabaseName=ContosoV2
 
 
 ECHO.
@@ -12,10 +12,19 @@ IF "%1" == "" (
 	SET RUNMODE=%1
 )
 
+IF "%2" == "" (
+    ECHO No Trim
+) ELSE (
+	SET TRIMTABLES=%2
+)
+
+
 @ECHO.
 ECHO RunMode:       %RUNMODE%
 ECHO SqlServerName: %SqlServerName%
 ECHO DatabaseName : %DatabaseName%
+ECHO TrimTables   : %TRIMTABLES%
+
 @ECHO.
 IF "%1" == "" ( PAUSE )
 
@@ -33,6 +42,10 @@ IF "%RUNMODE%" == "orders" sqlcmd -S "%SqlServerName%" -d "%DatabaseName%"  -i I
 IF "%RUNMODE%" == "both"   sqlcmd -S "%SqlServerName%" -d "%DatabaseName%"  -i ImportDataSales.sql    -i ImportDataOrders.sql  -v varCD="%CD%"
 
 
+ECHO 
+ECHO
+IF "%TRIMTABLES%" =="Trim" sqlcmd -S "%SqlServerName%" -d "%DatabaseName%"  -i TrimTables.sql -v varCD="%CD%"
+
 ECHO.
 ECHO ### The end ###
 ECHO.

scripts/sql/TrimTables.sql

@@ -0,0 +1,7 @@
+DECLARE @LOGLINE NVARCHAR(10) = Char(10) + Char(10) + '#### '
+
+PRINT @LOGLINE + 'Trim Customer'
+DELETE FROM [Data].Customer 
+WHERE CustomerKey NOT IN (SELECT DISTINCT CustomerKey FROM [Data].Sales )
+AND CustomerKey NOT IN (SELECT DISTINCT CustomerKey FROM [Data].Orders )
+GO