misc: Consolidate demo into a single quick-start call.

Author: diogodk

README.md

@@ -168,11 +168,7 @@ Verify that you can login to the UI with the `TESTGEN_USERNAME` and `TESTGEN_PAS
 The [Data Observability quickstart](https://docs.datakitchen.io/articles/open-source-data-observability/data-observability-overview) walks you through DataOps Data Quality TestGen capabilities to demonstrate how it covers critical use cases for data and analytic teams.
 
 ```shell
-testgen quick-start --delete-target-db
-testgen run-profile --table-group-id 0ea85e17-acbe-47fe-8394-9970725ad37d
-testgen run-test-generation --table-group-id 0ea85e17-acbe-47fe-8394-9970725ad37d
-testgen run-tests --project-key DEFAULT --test-suite-key default-suite-1
-testgen quick-start --simulate-fast-forward
+testgen quick-start
 ```
 
 In the TestGen UI, you will see that new data profiling and test results have been generated.

docs/local_development.md

@@ -93,11 +93,7 @@ testgen setup-system-db --yes
 
 Seed the demo data.
 ```shell
-testgen quick-start --delete-target-db
-testgen run-profile --table-group-id 0ea85e17-acbe-47fe-8394-9970725ad37d
-testgen run-test-generation --table-group-id 0ea85e17-acbe-47fe-8394-9970725ad37d
-testgen run-tests --project-key DEFAULT --test-suite-key default-suite-1
-testgen quick-start --simulate-fast-forward
+testgen quick-start
 ```
 
 ### Run the Application

testgen/__main__.py

@@ -117,10 +117,9 @@ def cli(ctx: Context, verbose: bool):
 @click.option(
     "-tg",
     "--table-group-id",
-    required=False,
+    required=True,
     type=click.STRING,
     help="The identifier for the table group used during a profile run. Use a table_group_id shown in list-table-groups.",
-    default=None,
 )
 def run_profile(configuration: Configuration, table_group_id: str):
     click.echo(f"run-profile with table_group_id: {table_group_id}")
@@ -136,16 +135,15 @@ def run_profile(configuration: Configuration, table_group_id: str):
     "-tg",
     "--table-group-id",
     help="The identifier for the table group used during a profile run. Use a table_group_id shown in list-table-groups.",
-    required=False,
+    required=True,
     type=click.STRING,
-    default=None,
 )
 @click.option(
     "-ts",
     "--test-suite-key",
     help="The identifier for a test suite. Use a test_suite_key shown in list-test-suites.",
-    required=False,
-    default=settings.DEFAULT_TEST_SUITE_KEY,
+    required=True,
+    type=click.STRING,
 )
 @click.option(
     "-gs",
@@ -339,27 +337,6 @@ def list_test_runs(configuration: Configuration, project_key: str, test_suite_ke
 
 
 @cli.command("quick-start", help="Use to generate sample target database, for demo purposes.")
-@click.option(
-    "--delete-target-db",
-    help="Will delete the current target database, if it exists",
-    is_flag=True,
-    default=False,
-)
-@click.option(
-    "--iteration",
-    "-i",
-    default=0,
-    required=False,
-    help="The monthly data increment snapshot. Can be 0, 1, 2 or 3. 0 is the initial data.",
-)
-@click.option(
-    "--simulate-fast-forward",
-    "-s",
-    default=False,
-    is_flag=True,
-    required=False,
-    help="For demo purposes, simulates that some time pass by and the target data is changing. This will call the iterations in order.",
-)
 @click.option(
     "--observability-api-url",
     help="Observability API url to be able to export TestGen data to Observability using the command 'export-observability'",
@@ -375,11 +352,10 @@ def list_test_runs(configuration: Configuration, project_key: str, test_suite_ke
     default="",
 )
 @pass_configuration
+@click.pass_context
 def quick_start(
+    ctx: Context,
     configuration: Configuration,
-    delete_target_db: bool,
-    iteration: int,
-    simulate_fast_forward: bool,
     observability_api_url: str,
     observability_api_key: str,
 ):
@@ -388,19 +364,32 @@ def quick_start(
     if observability_api_key:
         settings.OBSERVABILITY_API_KEY = observability_api_key
 
-    # Check if this is an increment or the initial state
-    if iteration == 0 and not simulate_fast_forward:
-        click.echo("quick-start command")
-        run_quick_start(delete_target_db)
+    click.echo("quick-start command")
+    run_quick_start(delete_target_db=True)
+
+    click.echo("loading initial data")
+    run_quick_start_increment(0)
+    minutes_offset = -30*24*60*3
+    table_group_id="0ea85e17-acbe-47fe-8394-9970725ad37d"
 
-    if not simulate_fast_forward:
+    click.echo(f"run-profile with table_group_id: {table_group_id}")
+    spinner = None
+    if not configuration.verbose:
+        spinner = MoonSpinner("Processing ... ")
+    message = run_profiling_queries(table_group_id, spinner=spinner, minutes_offset=minutes_offset)
+    click.echo("\n" + message)
+
+    LOG.info(f"run-test-generation with table_group_id: {table_group_id} test_suite: {settings.DEFAULT_TEST_SUITE_KEY}")
+    message = run_test_gen_queries(table_group_id, settings.DEFAULT_TEST_SUITE_KEY)
+    click.echo("\n" + message)
+
+    run_execution_steps(settings.PROJECT_KEY, settings.DEFAULT_TEST_SUITE_KEY, minutes_offset=minutes_offset)
+
+    for iteration in range(1, 4):
+        click.echo(f"Running iteration: {iteration} / 3")
+        minutes_offset = -30*24*60 * (3-iteration)
         run_quick_start_increment(iteration)
-    else:
-        for iteration in range(1, 4):
-            click.echo(f"Running iteration: {iteration} / 3")
-            minutes_offset = 2 * iteration
-            run_quick_start_increment(iteration)
-            run_execution_steps(settings.PROJECT_KEY, settings.DEFAULT_TEST_SUITE_KEY, minutes_offset=minutes_offset)
+        run_execution_steps(settings.PROJECT_KEY, settings.DEFAULT_TEST_SUITE_KEY, minutes_offset=minutes_offset)
 
     click.echo("Quick start has successfully finished.")
 

testgen/commands/queries/profiling_query.py

@@ -51,16 +51,18 @@ class CProfilingSQL:
     contingency_columns = ""
 
     exception_message = ""
+    minutes_offset = 0
 
     _data_chars_sql: CRefreshDataCharsSQL = None
     _rollup_scores_sql: CRollupScoresSQL = None
 
-    def __init__(self, strProjectCode, flavor):
+    def __init__(self, strProjectCode, flavor, minutes_offset=0):
         self.flavor = flavor
         self.project_code = strProjectCode
         # Defaults
-        self.run_date = date_service.get_now_as_string()
-        self.today = date_service.get_now_as_string()
+        self.run_date = date_service.get_now_as_string_with_offset(minutes_offset)
+        self.today = date_service.get_now_as_string_with_offset(minutes_offset)
+        self.minutes_offset = minutes_offset
 
     def _get_data_chars_sql(self) -> CRefreshDataCharsSQL:
         if not self._data_chars_sql:
@@ -102,7 +104,7 @@ def _get_params(self) -> dict:
             "PROFILE_ID_COLUMN_MASK": self.profile_id_column_mask,
             "PROFILE_SK_COLUMN_MASK": self.profile_sk_column_mask,
             "START_TIME": self.today,
-            "NOW_TIMESTAMP": date_service.get_now_as_string(),
+            "NOW_TIMESTAMP": date_service.get_now_as_string_with_offset(minutes_offset=self.minutes_offset),
             "EXCEPTION_MESSAGE": self.exception_message,
             "SAMPLING_TABLE": self.sampling_table,
             "SAMPLE_SIZE": int(self.parm_sample_size),

testgen/commands/run_profiling_bridge.py

@@ -221,7 +221,7 @@ def run_profiling_in_background(table_group_id):
 
 
 @with_database_session
-def run_profiling_queries(table_group_id: str, username: str | None = None, spinner: Spinner | None = None):
+def run_profiling_queries(table_group_id: str, username: str | None = None, spinner: Spinner | None = None, minutes_offset: int = 0):
     if table_group_id is None:
         raise ValueError("Table Group ID was not specified")
 
@@ -240,7 +240,7 @@ def run_profiling_queries(table_group_id: str, username: str | None = None, spin
     params = get_profiling_params(table_group_id)
 
     LOG.info("CurrentStep: Initializing Query Generator")
-    clsProfiling = CProfilingSQL(params["project_code"], connection.sql_flavor)
+    clsProfiling = CProfilingSQL(params["project_code"], connection.sql_flavor, minutes_offset=minutes_offset)
 
     # Set General Parms
     clsProfiling.table_groups_id = table_group_id

testgen/common/date_service.py

@@ -17,8 +17,7 @@ def parse_now(value: str) -> datetime:
 
 def get_now_as_string_with_offset(minutes_offset):
     ret = datetime.utcnow()
-    if minutes_offset > 0:
-        ret = ret + timedelta(minutes=minutes_offset)
+    ret = ret + timedelta(minutes=minutes_offset)
     return ret.strftime("%Y-%m-%d %H:%M:%S")