Merge pull request #43 from DataKitchen/release/4.1.3

Release/4.1.3

Author: datakitchen-devops

.gitignore

@@ -127,3 +127,6 @@ cython_debug/
 **/jsconfig.json
 .pytest_cache/
 .ruff_cache/
+
+testgen/ui/components/frontend/js/plugins.js
+testgen/ui/components/frontend/js/plugin_pages/

deploy/testgen.dockerfile

@@ -20,7 +20,7 @@ RUN addgroup -S testgen && adduser -S testgen -G testgen
 
 # Streamlit has to be able to write to these dirs
 RUN mkdir /var/lib/testgen
-RUN chown -R testgen:testgen /var/lib/testgen /dk/lib/python3.12/site-packages/streamlit/static
+RUN chown -R testgen:testgen /var/lib/testgen /dk/lib/python3.12/site-packages/streamlit/static /dk/lib/python3.12/site-packages/testgen/ui/components/frontend
 
 ENV TESTGEN_VERSION=${TESTGEN_VERSION}
 ENV TESTGEN_DOCKER_HUB_REPO=${TESTGEN_DOCKER_HUB_REPO}

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataops-testgen"
-version = "4.1.2"
+version = "4.1.3"
 description = "DataKitchen's Data Quality DataOps TestGen"
 authors = [
     { "name" = "DataKitchen, Inc.", "email" = "info@datakitchen.io" },

testgen/common/mixpanel_service.py

@@ -45,8 +45,10 @@ def _hash_value(self, value: bytes | str, digest_size: int = 8) -> str:
     @safe_method
     def send_event(self, event_name, **properties):
         properties.setdefault("instance_id", self.instance_id)
+        properties.setdefault("edition", settings.DOCKER_HUB_REPOSITORY)
         properties.setdefault("version", settings.VERSION)
         properties.setdefault("distinct_id", self.distinct_id)
+        properties.setdefault("username", session.username)
 
         track_payload = {
             "event": event_name,

testgen/template/dbsetup/030_initialize_new_schema_structure.sql

@@ -65,6 +65,7 @@ CREATE TABLE connections (
       CONSTRAINT connections_connection_id_pk
          PRIMARY KEY,
    sql_flavor             VARCHAR(30),
+   sql_flavor_code        VARCHAR(30),
    project_host           VARCHAR(250),
    project_port           VARCHAR(5),
    project_user           VARCHAR(50),

testgen/template/dbsetup/040_populate_new_schema_project.sql

@@ -9,11 +9,12 @@ SELECT '{PROJECT_CODE}' as project_code,
        '{OBSERVABILITY_API_URL}' as observability_api_url;
 
 INSERT INTO connections
-(project_code, sql_flavor,
+(project_code, sql_flavor, sql_flavor_code,
  project_host, project_port, project_user, project_db,
  connection_name, project_pw_encrypted, http_path, max_threads, max_query_chars)
 SELECT '{PROJECT_CODE}'                       as project_code,
        '{SQL_FLAVOR}'                         as sql_flavor,
+       '{SQL_FLAVOR}'                         as sql_flavor_code,
        '{PROJECT_HOST}'                       as project_host,
        '{PROJECT_PORT}'                       as project_port,
        '{PROJECT_USER}'                       as project_user,

testgen/template/dbupgrade/0140_incremental_upgrade.sql

@@ -0,0 +1,3 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+ALTER TABLE connections ADD COLUMN sql_flavor_code VARCHAR(30) DEFAULT NULL;

testgen/ui/bootstrap.py

@@ -1,14 +1,9 @@
 import dataclasses
-import importlib
-import inspect
 import logging
 
-import streamlit as st
-
 from testgen import settings
 from testgen.commands.run_upgrade_db_config import get_schema_revision
 from testgen.common import configure_logging, version_service
-from testgen.ui.assets import get_asset_path
 from testgen.ui.navigation.menu import Menu, Version
 from testgen.ui.navigation.page import Page
 from testgen.ui.navigation.router import Router
@@ -53,19 +48,8 @@
 LOG = logging.getLogger("testgen")
 
 
-class Logo:
-    image_path: str = get_asset_path("dk_logo.svg")
-    icon_path: str = get_asset_path("dk_icon.svg")
-
-    def render(self):
-        st.logo(
-            image=self.image_path,
-            icon_image=self.icon_path,
-        )
-
-
 class Application(singleton.Singleton):
-    def __init__(self, logo: Logo, router: Router, menu: Menu, logger: logging.Logger) -> None:
+    def __init__(self, logo: plugins.Logo, router: Router, menu: Menu, logger: logging.Logger) -> None:
         self.logo = logo
         self.router = router
         self.menu = menu
@@ -87,20 +71,30 @@ def run(log_level: int = logging.INFO) -> Application:
     pages = [*BUILTIN_PAGES]
     installed_plugins = plugins.discover()
 
+    if not settings.IS_DEBUG:
+        """
+        This cleanup is called so that TestGen can remove uninstalled
+        plugins without having to be reinstalled.
+
+        The check for DEBUG mode is because multithreading for Streamlit
+        fragments loads before the plugins can be re-loaded.
+        """
+        plugins.cleanup()
+
     configure_logging(level=log_level)
-    logo_class = Logo
+    logo_class = plugins.Logo
 
     for plugin in installed_plugins:
-        module = importlib.import_module(plugin.package)
-        for property_name in dir(module):
-            if (
-                (maybe_class := getattr(module, property_name, None))
-                and inspect.isclass(maybe_class)
-            ):
-                if issubclass(maybe_class, Page):
-                    pages.append(maybe_class)
-                elif issubclass(maybe_class, Logo):
-                    logo_class = maybe_class
+        spec = plugin.load()
+
+        if spec.page:
+            pages.append(spec.page)
+
+        if spec.logo:
+            logo_class = spec.logo
+
+        if spec.component:
+            spec.component.provide()
 
     return Application(
         logo=logo_class(),

testgen/ui/components/frontend/css/shared.css

@@ -426,6 +426,38 @@ body {
     margin-left: 40px;
 }
 
+.p-0 {
+    padding: 0;
+}
+
+.p-1 {
+    padding: 4px;
+}
+
+.p-2 {
+    padding: 8px;
+}
+
+.p-3 {
+    padding: 12px;
+}
+
+.p-4 {
+    padding: 16px;
+}
+
+.p-5 {
+    padding: 24px;
+}
+
+.p-6 {
+    padding: 32px;
+}
+
+.p-7 {
+    padding: 40px;
+}
+
 .pt-0 {
     padding-top: 0;
 }

testgen/ui/components/frontend/js/components/alert.js

@@ -1,38 +1,41 @@
 /**
- * @typedef Alert
- * @type {object}
- * @property {string} value
- * @property {string} color
- * @property {string} label
- * 
  * @typedef Properties
  * @type {object}
  * @property {string?} icon
- * @property {'info'|'success'|'error'} type
- * @property {string?} message
+ * @property {number?} timeout
+ * @property {boolean?} closeable
+ * @property {'info'|'success'|'warn'|'error'} type
  */
 import van from '../van.min.js';
-import { getValue, loadStylesheet } from '../utils.js';
+import { getValue, loadStylesheet, getRandomId } from '../utils.js';
 import { Icon } from './icon.js';
+import { Button } from './button.js';
 
 const { div } = van.tags;
 const alertTypeColors = {
     info: {backgroundColor: 'rgba(28, 131, 225, 0.1)', color: 'rgb(0, 66, 128)'},
     success: {backgroundColor: 'rgba(33, 195, 84, 0.1)', color: 'rgb(23, 114, 51)'},
+    warn: {backgroundColor: 'rgba(255, 227, 18, 0.2)', color: 'rgb(255, 255, 194)'},
     error: {backgroundColor: 'rgba(255, 43, 43, 0.09)', color: 'rgb(125, 53, 59)'},
 };
 
 const Alert = (/** @type Properties */ props, /** @type Array<HTMLElement> */ ...children) => {
     loadStylesheet('alert', stylesheet);
 
+    const elementId = getValue(props.id) ?? 'tg-alert-' + getRandomId();
+    const close = () => {
+        document.getElementById(elementId)?.remove();
+    };
+    const timeout = getValue(props.timeout);
+    if (timeout && timeout > 0) {
+        setTimeout(close, timeout);
+    }
+
     return div(
         {
             ...props,
-            class: () => (getValue(props.class) ?? '') + ` tg-alert flex-row`,
-            style: () => {
-                const colors = alertTypeColors[getValue(props.type)];
-                return `color: ${colors.color}; background-color: ${colors.backgroundColor};`;
-            },
+            id: elementId,
+            class: () => `tg-alert flex-row ${getValue(props.class) ?? ''} tg-alert-${getValue(props.type)}`,
             role: 'alert',
         },
         () => {
@@ -43,6 +46,19 @@ const Alert = (/** @type Properties */ props, /** @type Array<HTMLElement> */ ..
             {class: 'flex-column'},
             ...children,
         ),
+        () => {
+            const isCloseable = getValue(props.closeable) ?? false;
+            if (!isCloseable) {
+                return '';
+            }
+
+            const colors = alertTypeColors[getValue(props.type)];
+            return Button({
+                type: 'icon',
+                icon: 'close',
+                style: `margin-left: auto; color: ${colors.color};`,
+            });
+        },
     );
 };
 
@@ -54,6 +70,34 @@ stylesheet.replace(`
     font-size: 16px;
     line-height: 24px;
 }
+
+.tg-alert-info {
+    background-color: rgba(28, 131, 225, 0.1);
+    color: rgb(0, 66, 128);
+}
+
+.tg-alert-success {
+    background-color: rgba(33, 195, 84, 0.1);
+    color: rgb(23, 114, 51);
+}
+
+.tg-alert-error {
+    background-color: rgba(255, 43, 43, 0.09);
+    color: rgb(125, 53, 59);
+}
+
+.tg-alert-warn {
+    background-color: rgba(255, 227, 18, 0.1);
+    color: rgb(146, 108, 5);
+}
+
+@media (prefers-color-scheme: dark) {
+    .tg-alert-warn {
+        background-color: rgba(255, 227, 18, 0.2);
+        color: rgb(255, 255, 194);
+    }
+}
+
 .tg-alert > .tg-icon {
     color: inherit !important;
 }