feat(database): bundle DataFlow with both Database source channels

.github/workflows/build-version.yml

@@ -17,6 +17,7 @@
   contents: read
 
 env:
+  DATAFLOW_IMAGE: ghcr.io/${{ github.repository }}/dataflow
   RUNTIME_IMAGE: ghcr.io/${{ github.repository }}/database-frontend
   CLUSTER_IMAGE: ghcr.io/${{ github.repository }}/database-cluster
   SEALOS_VERSION: v5.1.2-rc5
@@ -58,6 +59,12 @@
           helm lint charts/database-frontend
           helm template dbprovider-frontend charts/database-frontend --namespace dbprovider-frontend > /dev/null
           python3 ../../../tests/test_deployment_compatibility.py
+      - name: Validate DataFlow and combined installer
+        run: |
+          python3 scripts/test-bundle.py
+          bash -n deploy/install.sh scripts/prepare-bundle.sh dataflow/deploy/install.sh
+          helm lint dataflow/deploy/charts/dataflow -f dataflow/deploy/charts/dataflow/dataflow-values.yaml
+          helm template dataflow dataflow/deploy/charts/dataflow --namespace dataflow-system -f dataflow/deploy/charts/dataflow/dataflow-values.yaml > /dev/null
 
   runtime:
     needs: metadata
@@ -96,6 +103,19 @@
           tags: ${{ env.RUNTIME_IMAGE }}:${{ needs.metadata.outputs.tag }}-${{ matrix.arch }}
           cache-from: type=gha,scope=database-${{ inputs.source }}-${{ matrix.arch }}
           cache-to: type=gha,scope=database-${{ inputs.source }}-${{ matrix.arch }},mode=max
+      - name: Build DataFlow runtime (${{ matrix.arch }})
+        uses: docker/build-push-action@v6
+        with:
+          context: ./dataflow
+          file: ./dataflow/core/Dockerfile
+          build-args: |
+            VERSION=${{ needs.metadata.outputs.tag }}
+            PLATFORM=sealos
+          platforms: linux/${{ matrix.arch }}
+          push: ${{ inputs.publish }}
+          tags: ${{ env.DATAFLOW_IMAGE }}:${{ needs.metadata.outputs.tag }}-${{ matrix.arch }}
+          cache-from: type=gha,scope=dataflow-${{ matrix.arch }}
+          cache-to: type=gha,scope=dataflow-${{ matrix.arch }},mode=max
 
   runtime-release:
     needs: [metadata, runtime]
@@ -118,6 +138,9 @@
           set -euo pipefail
           docker buildx imagetools create -t "${RUNTIME_IMAGE}:${TAG}" \
             "${RUNTIME_IMAGE}:${TAG}-amd64" "${RUNTIME_IMAGE}:${TAG}-arm64"
+          docker buildx imagetools create -t "${DATAFLOW_IMAGE}:${TAG}" \
+            "${DATAFLOW_IMAGE}:${TAG}-amd64" "${DATAFLOW_IMAGE}:${TAG}-arm64"
+          docker buildx imagetools inspect "${DATAFLOW_IMAGE}:${TAG}"
           docker buildx imagetools inspect "${RUNTIME_IMAGE}:${TAG}"
 
   cluster:
@@ -155,15 +178,15 @@
           GH_USER: ${{ github.actor }}
         run: sudo sealos login -u "$GH_USER" -p "$GH_TOKEN" ghcr.io
       - name: Build, push and archive cluster image
-        working-directory: ${{ inputs.source }}/database/deploy
         env:
           TAG: ${{ needs.metadata.outputs.tag }}
           ARCH: ${{ matrix.arch }}
+          SOURCE: ${{ inputs.source }}
         run: |
           set -euo pipefail
-          export RUNTIME_REF="${RUNTIME_IMAGE}:${TAG}"
-          # Change only the chart's runtime image, preserving all other defaults.
-          ruby -ryaml -e 'p = "charts/database-frontend/values.yaml"; d = YAML.load_file(p); d["image"] = ENV.fetch("RUNTIME_REF"); File.write(p, YAML.dump(d))'
+          bundle_dir="${RUNNER_TEMP}/database-bundle-${SOURCE}-${ARCH}"
+          bash scripts/prepare-bundle.sh "$SOURCE" "$bundle_dir" "$RUNTIME_IMAGE" "$DATAFLOW_IMAGE" "$TAG"
+          cd "$bundle_dir"
           sudo sealos registry save --registry-dir=registry --arch "$ARCH" .
           ref="${CLUSTER_IMAGE}:${TAG}-${ARCH}"
           sudo sealos build -t "$ref" --platform "linux/${ARCH}" -f Kubefile .
@@ -215,6 +238,7 @@
           SOURCE: ${{ inputs.source }}
         run: |
           set -euo pipefail
+          docker buildx imagetools create -t "${DATAFLOW_IMAGE}:${SOURCE}" "${DATAFLOW_IMAGE}:${TAG}"
           docker buildx imagetools create -t "${RUNTIME_IMAGE}:${SOURCE}" "${RUNTIME_IMAGE}:${TAG}"
           docker buildx imagetools create -t "${CLUSTER_IMAGE}:${SOURCE}" "${CLUSTER_IMAGE}:${TAG}"
       - uses: actions/download-artifact@v4

.gitleaksignore

@@ -0,0 +1,29 @@
+# Public upstream test TLS fixtures and the client-side PostHog ingestion key.
+# Verified byte-for-byte against sealos-apps/dataflow commit
+# a6fbbf53906d2ac9f4a0817af7f50030f90c86d3; no operational credentials.
+# Exact file/rule/line fingerprints keep other findings visible.
+dataflow/core/src/env/env.go:generic-api-key:92
+dataflow/core/src/plugins/ssl/tls_test.go:private-key:58
+dataflow/dev/certs/ca/clickhouse/ca-key.pem:private-key:1
+dataflow/dev/certs/ca/elasticsearch/ca-key.pem:private-key:1
+dataflow/dev/certs/ca/mariadb/ca-key.pem:private-key:1
+dataflow/dev/certs/ca/mysql/ca-key.pem:private-key:1
+dataflow/dev/certs/ca/mongodb/ca-key.pem:private-key:1
+dataflow/dev/certs/ca/postgres/ca-key.pem:private-key:1
+dataflow/dev/certs/ca/redis/ca-key.pem:private-key:1
+dataflow/dev/certs/client/clickhouse/client-key.pem:private-key:1
+dataflow/dev/certs/client/elasticsearch/client-key.pem:private-key:1
+dataflow/dev/certs/client/mariadb/client-key.pem:private-key:1
+dataflow/dev/certs/client/mysql/client-key.pem:private-key:1
+dataflow/dev/certs/client/postgres/client-key.pem:private-key:1
+dataflow/dev/certs/client/mongodb/client-key.pem:private-key:1
+dataflow/dev/certs/server/clickhouse/server-key.pem:private-key:1
+dataflow/dev/certs/client/redis/client-key.pem:private-key:1
+dataflow/dev/certs/server/elasticsearch/server-key.pem:private-key:1
+dataflow/dev/certs/server/mariadb/server-key.pem:private-key:1
+dataflow/dev/certs/server/mongodb/mongodb.pem:private-key:25
+dataflow/dev/certs/server/redis/server-key.pem:private-key:1
+dataflow/dev/certs/server/mysql/server-key.pem:private-key:1
+dataflow/dev/certs/server/postgres/server-key.pem:private-key:1
+dataflow/dev/certs/server/redis/redis.pem:private-key:25
+dataflow/dev/certs/server/mongodb/server-key.pem:private-key:1

README.md

@@ -13,6 +13,11 @@ source directories, not the branch that triggers CI.
 | Push a `v*` tag | Both workspaces | Yes | Yes |
 | Manual run | Both workspaces | `publish_images` or `upload_oss` | `upload_oss` |
 
+DataFlow is imported once under `dataflow/` (provenance in
+[dataflow/IMPORT.md](dataflow/IMPORT.md)). Both source channels build the same
+DataFlow source alongside their respective Database frontend. Their cluster
+images install both services using [deploy/install.sh](deploy/install.sh).
+
 Every runtime image and cluster image supports `linux/amd64` and `linux/arm64`.
 Runtime builds use each workspace's root Dockerfile with `name=database` and
 `path=database`. Cluster images include the Helm chart, entrypoint
@@ -21,6 +26,7 @@ and the runtime image saved by Sealos for the selected architecture.
 Images are published to:
 
 - `ghcr.io/sealos-apps/database/database-frontend:<tag>`
+- `ghcr.io/sealos-apps/database/dataflow:<tag>`
 - `ghcr.io/sealos-apps/database/database-cluster:<tag>`
 
 Tags are `<source>-<sha7>` for branch/manual builds and
@@ -191,3 +197,72 @@ a cluster or write database data:
 node --test tests/test_metrics_sdk.cjs
 python3 tests/test_deployment_compatibility.py
 ```
+
+## Combined Database + DataFlow installation
+
+The cluster package stages the selected Database chart and the shared DataFlow
+chart under its root `charts/` directory so Sealos can discover both runtime
+images. The installer subdirectories link to this shared chart directory. Runtime references are pinned to the same source/version tag. Install
+keeps two independent Helm releases and Deployments:
+
+| Component | Default release | Default namespace | Persisted user values |
+| --- | --- | --- | --- |
+| Database | `dbprovider-frontend` | `dbprovider-frontend` | main: `/root/.sealos/cloud/values/core/dbprovider-frontend-values.yaml`; release-v5.1: `/root/.sealos/cloud/values/apps/database/dbprovider-values.yaml` |
+| DataFlow | `dataflow` | `dataflow-system` | `/root/.sealos/cloud/values/apps/dataflow/dataflow-values.yaml` |
+
+Database retains its existing `RELEASE_NAME`, `RELEASE_NAMESPACE`, `HELM_OPTS`
+and `HELM_OPTIONS` controls. DataFlow uses `DATAFLOW_RELEASE_NAME`,
+`DATAFLOW_NAMESPACE`, `DATAFLOW_HELM_OPTS` and optionally
+`DATAFLOW_USER_VALUES_DIR`. Both use the Sealos cloud configuration; a
+`SEALOS_CLOUD_DOMAIN` override applies to both. DataFlow installs first. A
+failure stops installation before Database is upgraded. The two Helm upgrades
+are not a transaction: if Database fails afterward, DataFlow remains upgraded.
+
+See [deploy/env.template](deploy/env.template) for defaults, purposes and the
+shell environment override method; the installer does not load it automatically.
+
+DataFlow keeps the hidden `app-system/dataflow` Desktop App and internal
+`dataflow.sealos.io/health-url` annotation. Existing Database “Manage data”
+buttons continue opening `system-dataflow` with resource name, database type,
+language and theme. No React dependency merge or new cross-service API is needed.
+
+### Upgrading an existing installation
+
+Use the existing DataFlow release and namespace, and preserve the user values
+file. If that file is absent but the release exists, the installer recovers
+its effective Helm values before upgrading. This preserves external metadata
+DSNs, custom resource names, storage settings and custom runtime Secrets.
+The installer resolves the configured runtime Secret from the chart and reuses
+its session encryption key; it stops if an existing release's key or values
+cannot be read. Fresh installations generate a key once. Recovered files use
+restricted permissions because Helm values may contain credentials.
+
+The combined package supplies versioned image pins even when saved values
+contain older image references. `DATAFLOW_HELM_OPTS` remains an explicit override
+for DataFlow; Database always uses the bundled runtime image. Existing custom
+DataFlow App names must match Database's `app-system/dataflow` discovery contract.
+
+Keep the DataFlow release, PostgreSQL resource/PVC and Secret in place. Do not
+uninstall the old DataFlow release: it is now upgraded by the combined package,
+not replaced by a new release. Existing charts can delete their metadata cluster
+on uninstall. No SQL/data-copy migration is performed by this source migration.
+Rolling back requires rolling back the two Helm releases separately, with the
+same storage and encryption key; application schema compatibility must be checked
+before downgrading DataFlow.
+
+### Local validation
+
+```bash
+python3 scripts/test-bundle.py
+bash scripts/prepare-bundle.sh main /tmp/database-bundle \
+  ghcr.io/sealos-apps/database/database-frontend \
+  ghcr.io/sealos-apps/database/dataflow main-<sha7>
+```
+
+The output directory must not exist. The tests use fake Helm/kubectl for upgrade
+checks and real Helm rendering for both source packages; they never access a
+cluster or write to a database. CI runs these checks and builds both runtime
+architectures. Actual installation may initialize/upgrade DataFlow metadata;
+local source/build checks do not prove live storage, authentication or browser
+acceptance. Live deployment requires a chosen cluster and a separately authorized
+metadata modification if initialization or schema migration is needed.

dataflow/.codegraph/.gitignore

@@ -0,0 +1,5 @@
+# CodeGraph data files — local to each machine, not for committing.
+# Ignore everything in .codegraph/ except this file itself, so transient
+# files (the database, daemon.pid, sockets, logs) never show up in git.
+*
+!.gitignore

dataflow/.dockerignore

@@ -0,0 +1,16 @@
+# Ignore everything by default
+*
+
+# Core Dockerfile needs: dataflow/ and core/
+!dataflow/
+!dataflow/**
+!core/
+!core/**
+
+# Exclude test/dev artifacts within allowed directories
+**/node_modules
+dataflow/build
+dataflow/docs/superpowers
+dataflow/public/monaco-editor
+dataflow/.env
+core/tmp

dataflow/.github/scripts/install.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if command -v sealos >/dev/null 2>&1; then
+  sealos version
+  exit 0
+fi
+
+tmp_dir="$(mktemp -d)"
+trap 'rm -rf "${tmp_dir}"' EXIT
+
+arch="$(uname -m)"
+case "${arch}" in
+  x86_64|amd64)
+    sealos_arch="amd64"
+    ;;
+  aarch64|arm64)
+    sealos_arch="arm64"
+    ;;
+  *)
+    echo "Unsupported architecture: ${arch}" >&2
+    exit 1
+    ;;
+esac
+
+cd "${tmp_dir}"
+until curl -sSfLo sealos.tar.gz "https://github.com/labring/sealos/releases/download/v5.1.0-beta3/sealos_5.1.0-beta3_linux_${sealos_arch}.tar.gz"; do
+  sleep 3
+done
+
+tar -zxf sealos.tar.gz sealos
+chmod +x sealos
+mv sealos /usr/bin/sealos
+sealos version

dataflow/.github/workflows/pr-docker-build.yml

@@ -0,0 +1,139 @@
+name: PR Docker Build
+
+permissions:
+  contents: read
+
+concurrency:
+  group: pr-docker-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches:
+      - main
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+env:
+  DOCKERFILE: core/Dockerfile
+  LOCAL_RUNTIME_IMAGE: dataflow-pr-runtime
+  LOCAL_SEALOS_IMAGE: dataflow-cluster-pr
+  LOCAL_REGISTRY: 127.0.0.1:5000/dataflow-pr
+
+jobs:
+  docker:
+    name: Build Docker and Sealos Image (${{ matrix.arch }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-24.04
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Generate image tag
+        id: meta
+        run: |
+          set -euo pipefail
+          SHORT_SHA="$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7)"
+          PR_TAG="pr-${{ github.event.pull_request.number }}-${SHORT_SHA}"
+          echo "short_sha=${SHORT_SHA}" >> "${GITHUB_OUTPUT}"
+          echo "pr_tag=${PR_TAG}" >> "${GITHUB_OUTPUT}"
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ${{ env.DOCKERFILE }}
+          push: false
+          load: true
+          platforms: linux/${{ matrix.arch }}
+          tags: ${{ env.LOCAL_RUNTIME_IMAGE }}:${{ steps.meta.outputs.pr_tag }}-${{ matrix.arch }}
+          build-args: |
+            VERSION=${{ steps.meta.outputs.pr_tag }}
+            TARGETARCH=${{ matrix.arch }}
+            PLATFORM=pr
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.event.pull_request.head.sha }}
+            org.opencontainers.image.version=${{ steps.meta.outputs.pr_tag }}
+            org.opencontainers.image.architecture=${{ matrix.arch }}
+            runnumber=${{ github.run_id }}
+          cache-from: type=gha,scope=dataflow-pr-runtime-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=dataflow-pr-runtime-${{ matrix.arch }}
+
+      - name: Start temporary local registry
+        run: |
+          set -euo pipefail
+          docker rm -f dataflow-pr-registry >/dev/null 2>&1 || true
+          docker run -d -p 5000:5000 --name dataflow-pr-registry registry:2
+
+      - name: Push runtime image to temporary local registry
+        run: |
+          set -euo pipefail
+          RUNTIME_TAG="${{ steps.meta.outputs.pr_tag }}"
+          RUNTIME_LOCAL_IMAGE="${LOCAL_RUNTIME_IMAGE}:${RUNTIME_TAG}-${{ matrix.arch }}"
+          REGISTRY_IMAGE="${LOCAL_REGISTRY}/runtime:${RUNTIME_TAG}"
+          docker tag "${RUNTIME_LOCAL_IMAGE}" "${REGISTRY_IMAGE}"
+          docker push "${REGISTRY_IMAGE}"
+
+      - name: Install cache images tools
+        run: sudo bash ./.github/scripts/install.sh
+
+      - name: Cache images for sealos
+        working-directory: deploy
+        run: |
+          set -euo pipefail
+          RUNTIME_TAG="${{ steps.meta.outputs.pr_tag }}"
+          sed -i "/^image:/,/^[^[:space:]]/ s#^\([[:space:]]*repository:[[:space:]]*\).*#\1${LOCAL_REGISTRY}/runtime#" charts/dataflow/values.yaml
+          sed -i "/^image:/,/^[^[:space:]]/ s#^\([[:space:]]*tag:[[:space:]]*\).*#\1${RUNTIME_TAG}#" charts/dataflow/values.yaml
+          sudo sealos registry save --registry-dir=registry_${{ matrix.arch }} --arch ${{ matrix.arch }} .
+
+      - name: Build sealos image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./deploy
+          file: ./deploy/Kubefile
+          push: false
+          load: true
+          platforms: linux/${{ matrix.arch }}
+          tags: ${{ env.LOCAL_SEALOS_IMAGE }}:${{ steps.meta.outputs.pr_tag }}-${{ matrix.arch }}
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.event.pull_request.head.sha }}
+            org.opencontainers.image.version=${{ steps.meta.outputs.pr_tag }}
+            org.opencontainers.image.architecture=${{ matrix.arch }}
+            runnumber=${{ github.run_id }}
+          cache-from: type=gha,scope=dataflow-pr-sealos-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=dataflow-pr-sealos-${{ matrix.arch }}
+
+      - name: Save sealos image tar
+        run: |
+          set -euo pipefail
+          mkdir -p artifacts
+          TAR_NAME="dataflow-cluster-pr-${{ github.event.pull_request.number }}-${{ steps.meta.outputs.short_sha }}-${{ matrix.arch }}.tar"
+          docker save "${LOCAL_SEALOS_IMAGE}:${{ steps.meta.outputs.pr_tag }}-${{ matrix.arch }}" -o "artifacts/${TAR_NAME}"
+          gzip -f "artifacts/${TAR_NAME}"
+
+      - name: Upload sealos image tar
+        uses: actions/upload-artifact@v4
+        with:
+          name: dataflow-cluster-pr-${{ github.event.pull_request.number }}-${{ steps.meta.outputs.short_sha }}-${{ matrix.arch }}
+          path: artifacts/*.tar.gz
+          if-no-files-found: error
+          retention-days: 14
+
+      - name: Cleanup temporary local registry
+        if: always()
+        run: docker rm -f dataflow-pr-registry >/dev/null 2>&1 || true