.gitlab-ci.yml

default:
  image: node:20.18-bookworm-slim@sha256:ffc11dbf16dd0abcbb7b837410601b4d5592db2d03741e13a4a5336ab74d7ccb

stages:
  - cleanup
  - build
  - test
  - end 2 end
  - publish
  - deploy
  - provision
  - performance
  - backup

variables:
  BACKUPS_LOCATION: hetzner-s3:appsemble/backups
  PROD_BACKUP_NAME: appsemble_prod_backup

workflow:
  rules:
    # By default, run jobs for every merge request.
    - if: $CI_MERGE_REQUEST_ID
    # By default, run jobs for every commit on main.
    - if: $CI_COMMIT_BRANCH == 'main'
    # By default, run jobs for every commit on staging.
    - if: $CI_COMMIT_BRANCH == 'staging'
    # By default, run jobs for every tag
    - if: $CI_COMMIT_TAG

###################################################################################################
#  Job Templates                                                                                  #
###################################################################################################

# A preset for running Docker in Docker.
.docker:
  interruptible: true
  services:
    - docker:dind@sha256:1e9d444274b295261e49f96635f3d56a615cb3ae2021204160f8ac3cdf8f3b58
  image: docker@sha256:1e9d444274b295261e49f96635f3d56a615cb3ae2021204160f8ac3cdf8f3b58
  dependencies: []
  before_script:
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY

# A preconfigured environment for using npm.
.npm:
  interruptible: true
  dependencies: [npm_ci]
  needs:
    - job: npm_ci
      artifacts: true

# A template for creating a minor or patch release
.release:
  extends: .npm
  stage: publish
  rules:
    - if: $CI_COMMIT_BRANCH == 'main'
      when: manual
  allow_failure: true
  script:
    - apt-get update
    - apt-get install --yes git gnupg
    - gpg --import "$GPG_PRIVATE_KEY"
    - |
      if [ -n "$IDENTIFIER" ]; then
        npm run scripts -- release "$INCREMENT" --identifier "$IDENTIFIER"
      else
        npm run scripts -- release "$INCREMENT"
      fi
    - npm install
    - VERSION="$(npm --silent run appsemble -- --version)"
    - |
      MESSAGE="$VERSION

      $(npm --silent run scripts get-release-notes)"
    - git config user.email bot@appsemble.com
    - git config user.name Appsemble
    - git add .
    - git commit --message "$MESSAGE" --cleanup whitespace --gpg-sign
    - git tag "$VERSION" --message "$MESSAGE" --cleanup whitespace --sign
    - git push "https://appsemble-bot:$GITLAB_ACCESS_TOKEN@gitlab.com/appsemble/appsemble" HEAD:main
    - |
      if [ $? -eq 0 ]; then
        echo "Code push successful. Pushing tags..."
        git push "https://appsemble-bot:$GITLAB_ACCESS_TOKEN@gitlab.com/appsemble/appsemble" --tags
      else
        echo "Push to main branch failed. Tags will not be pushed."
      fi

###################################################################################################
#  Build Stage                                                                                    #
###################################################################################################

npm_ci:
  stage: build
  cache:
    key:
      files:
        - package-lock.json
    paths:
      - node_modules/
      - '**/node_modules'
      - /root/.npm/
  artifacts:
    paths:
      - node_modules/
      - '**/node_modules'
    expire_in: 1 day
  script:
    - npm ci

# Build the Docker image.
build docker image:
  extends: .docker
  stage: build
  cache:
    key: '$CI_COMMIT_REF_NAME'
    paths:
      - /root/.npm/
  script:
    # https://docs.gitlab.com/ee/ci/docker/docker_layer_caching.html
    # We're pulling latest if no previous builds exist because we want the greatest possibility of cache hits for build layers.
    - CACHE_IMAGE="$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
    # We wouldn't like anything funny to happen to us because of cache
    - if [ "$CI_COMMIT_REF_NAME" == "main" ] || [ "$CI_COMMIT_REF_NAME" == "staging" ]; then USE_CACHE="0"; else USE_CACHE="1"; fi
    - if ! docker pull "$CACHE_IMAGE"; then CACHE_IMAGE="$CI_REGISTRY_IMAGE:latest"; docker pull "$CACHE_IMAGE" || true; fi
    - docker build
      -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
      --cache-from "$CACHE_IMAGE"
      --build-arg "BUILDKIT_INLINE_CACHE=$USE_CACHE"
      --build-arg "version=$CI_COMMIT_REF_NAME"
      --build-arg "date=$CI_JOB_STARTED_AT" .
    - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"

# Build the npm packages that should be published.
pack:
  extends: .npm
  stage: build
  script:
    - mkdir build/
    - npm --pack-destination build --workspace @appsemble/types pack
    - npm --pack-destination build --workspace @appsemble/sdk pack
    - npm --pack-destination build --workspace @appsemble/lang-sdk pack
    - npm --pack-destination build --workspace @appsemble/preact pack
    - npm --pack-destination build --workspace @appsemble/utils pack
    - npm --pack-destination build --workspace @appsemble/node-utils pack
    - npm --pack-destination build --workspace @appsemble/webpack-config pack
    - npm --pack-destination build --workspace @appsemble/cli pack
    - npm --pack-destination build --workspace @appsemble/tsconfig pack
    - npm --pack-destination build --workspace @appsemble/e2e pack
    - npm --pack-destination build --workspace appsemble pack
    - npm --pack-destination build --workspace create-appsemble pack
    - npm run scripts -- validate-packaged-exports build/*
    - find build -mindepth 1 -maxdepth 1 -type d -exec rm -r {} +
  artifacts:
    name: npm packages
    expose_as: packages
    paths:
      - build/

# Package the Helm chart
helm package:
  image: dtzar/helm-kubectl:3@sha256:569fbcea3ad1bdb26c953dc76e24591e9ac06abaf577e9f9167a0517e7669425
  stage: build
  needs: []
  script:
    - cp LICENSE.md config/charts/appsemble/
    - config/bin/helm-package.sh --dependency-update --destination public config/charts/appsemble/
  artifacts:
    name: Helm chart
    expose_as: Helm chart
    paths:
      - public/

###################################################################################################
#  Test Stage                                                                                     #
###################################################################################################

# Spell check using cspell
cspell:
  extends: .npm
  stage: test
  script:
    - npx -- cspell

# Lint JavaScript code using ESLint.
eslint:
  extends: .npm
  stage: test
  script:
    - npx -- eslint --format gitlab .
  artifacts:
    reports:
      codequality: gl-codequality.json

# Lint Helm charts.
helm lint:
  image: dtzar/helm-kubectl:3@sha256:569fbcea3ad1bdb26c953dc76e24591e9ac06abaf577e9f9167a0517e7669425
  stage: test
  needs: []
  script:
    - helm lint config/charts/*

# Lint shell scripts
shellcheck:
  image: koalaman/shellcheck-alpine@sha256:71e5a217c94204e164da3a89cc83ac0212dc06549f37af61b298273036bbab6f
  stage: test
  needs: []
  script:
    - printf "Evaluating:\n$(find . -name '*.sh')"
    - find . -name '*.sh' | xargs shellcheck

# Verify app messages are in sync with the app definition.
i18n:
  extends: .npm
  stage: test
  script:
    - npm run appsemble -- app extract-messages --verify nl apps/*

# Check formatting using prettier.
prettier:
  extends: .npm
  stage: test
  script:
    - npx -- prettier .

# Lint Markdown using remark.
remark lint:
  extends: .npm
  stage: test
  script:
    - npx -- remark --frail --no-stdout .

# Lint CSS using stylelint.
stylelint:
  extends: .npm
  stage: test
  script:
    - npx -- stylelint .

# Run unittests using NodeJS.
test node:
  extends: .npm
  services:
    - postgres:15@sha256:b8e11f8a8b383e19589a086a78b10f9ca69a39d6c9dcdd9342a8697544e8b3a0
    - name: minio/minio:latest@sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e
      alias: minio
      command: ['server', '/data']
      variables:
        MINIO_ROOT_USER: admin
        MINIO_ROOT_PASSWORD: password
  variables:
    POSTGRES_DB: testAppsemble
    POSTGRES_USER: admin
    POSTGRES_PASSWORD: password
    DATABASE_URL: 'postgres://admin:password@postgres:5432/testAppsemble'
    S3_HOST: 'minio'
    S3_PORT: 9000
  stage: test
  script:
    - npm test -- --no-file-parallelism --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
  parallel: 9
  artifacts:
    paths:
      - coverage
    reports:
      junit: junit.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage/cobertura-coverage.xml

# Check type validity for our TypeScript files.
tsc:
  extends: .npm
  stage: test
  script:
    - npx --workspaces tsc

# Check type validity for our TypeScript files.
validate:
  extends: .npm
  stage: test
  script:
    - npm run scripts -- validate

# Check migrations are in sync with Sequelize models
check migrations:
  extends: .npm
  stage: test
  interruptible: true
  services:
    - postgres:15@sha256:b8e11f8a8b383e19589a086a78b10f9ca69a39d6c9dcdd9342a8697544e8b3a0
  variables:
    FORCE_COLOR: '1' # Required for Chalk.js
    POSTGRES_DB: testAppsemble
    POSTGRES_USER: admin
    POSTGRES_PASSWORD: password
    DATABASE_URL: 'postgres://admin:password@postgres:5432/testAppsemble'
  script:
    - npm run appsemble -- check-migrations

# Check down migrations
check down migrations:
  extends: .npm
  stage: test
  interruptible: true
  services:
    - postgres:15@sha256:b8e11f8a8b383e19589a086a78b10f9ca69a39d6c9dcdd9342a8697544e8b3a0
  variables:
    FORCE_COLOR: '1' # Required for Chalk.js
    POSTGRES_DB: testAppsemble
    POSTGRES_USER: admin
    POSTGRES_PASSWORD: password
    DATABASE_URL: 'postgres://admin:password@postgres:5432/testAppsemble'
  script:
    - npm run appsemble -- check-down-migrations

# Fuzz test migrations for anomalous up and down migration failures
fuzz migrations:
  extends: .npm
  stage: test
  interruptible: true
  services:
    - postgres:15@sha256:b8e11f8a8b383e19589a086a78b10f9ca69a39d6c9dcdd9342a8697544e8b3a0
  variables:
    POSTGRES_DB: testAppsemble
    POSTGRES_USER: admin
    POSTGRES_PASSWORD: password
    DATABASE_URL: 'postgres://admin:password@postgres:5432/testAppsemble'
  script:
    - npm run appsemble -- fuzz-migrations

include:
  # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml
  # https://docs.gitlab.com/ee/user/application_security/container_scanning
  # Uses https://github.com/aquasecurity/trivy
  # The 'latest' template is used to enable merge-request scanning
  # The generated report is used for https://gitlab.com/appsemble/security-policy-management/-/blob/main/.gitlab/security-policies/policy.yml
  - template: Jobs/Container-Scanning.latest.gitlab-ci.yml

container_scanning:
  needs:
    - build docker image
  variables:
    GIT_STRATEGY: fetch
    CS_DISABLE_LANGUAGE_VULNERABILITY_SCAN: false
    CS_IMAGE: '$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG'

validate-snippets:
  stage: test
  interruptible: true
  needs:
    - build docker image
  rules:
    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'appsemble/appsemble'
    - if: $CI_COMMIT_TAG
  variables:
    FF_NETWORK_PER_BUILD: 'true'
    DATABASE_HOST: database
    DATABASE_PORT: 5432
    DATABASE_NAME: appsemble
    DATABASE_USER: admin
    DATABASE_PASSWORD: password
  services:
    - name: postgres:15@sha256:b8e11f8a8b383e19589a086a78b10f9ca69a39d6c9dcdd9342a8697544e8b3a0
      alias: database
      variables:
        POSTGRES_DB: appsemble
        POSTGRES_USER: admin
        POSTGRES_PASSWORD: password

    - name: minio/minio:latest@sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e
      alias: minio
      entrypoint: ['sh', '-c', 'minio server /data']
      variables:
        MINIO_ROOT_USER: admin
        MINIO_ROOT_PASSWORD: password

    - name: '$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG'
      alias: appsemble
      variables:
        SECRET: secret
        AES_SECRET: secret
        HOST: 'http://appsemble:9999'
        REMOTE: 'https://appsemble.app'
        DATABASE_HOST: database
        DATABASE_PORT: 5432
        DATABASE_NAME: appsemble
        DATABASE_USER: admin
        DATABASE_PASSWORD: password
        S3_HOST: minio
        S3_SECURE: false
        S3_ACCESS_KEY: admin
        S3_SECRET_KEY: password
        migrateTo: next
  script:
    - npm ci
    - npm run scripts seed-account --
      --name $BOT_ACCOUNT_NAME
      --email $BOT_ACCOUNT_EMAIL
      --password $BOT_ACCOUNT_PASSWORD
      --timezone Europe/Amsterdam
      --clientCredentials $APPSEMBLE_CLIENT_CREDENTIALS
    - npm run appsemble -- config set remote 'http://appsemble:9999'
    - npm run appsemble -- organization create appsemble
      --description "The open source low-code app building platform"
      --email support@appsemble.com
      --icon packages/server/assets/appsemble.png
      --name Appsemble
      --website https://appsemble.com
    - npm run appsemble -- block publish 'blocks/*'
    - npm run scripts -- validate-docs --organization appsemble --remote 'http://appsemble:9999' -vv

###################################################################################################
#  End 2 End Stage                                                                               #
###################################################################################################

e2e:
  stage: end 2 end
  image: mcr.microsoft.com/playwright:v1.51.1-jammy@sha256:79da45705a7c3f147c435ac6d0beeddf2e132f53263cb27bed90beafbb2e552b
  interruptible: true
  needs:
    - build docker image
  rules:
    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'appsemble/appsemble'
    - if: $CI_COMMIT_TAG
  variables:
    FF_NETWORK_PER_BUILD: 'true'
    DATABASE_HOST: database
    DATABASE_PORT: 5432
    DATABASE_NAME: appsemble
    DATABASE_USER: admin
    DATABASE_PASSWORD: password
    E2E_HOST: 'http://appsemble:9999'
  services:
    - name: postgres:15@sha256:b8e11f8a8b383e19589a086a78b10f9ca69a39d6c9dcdd9342a8697544e8b3a0
      alias: database
      variables:
        POSTGRES_DB: appsemble
        POSTGRES_USER: admin
        POSTGRES_PASSWORD: password

    - name: minio/minio:latest@sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e
      alias: minio
      entrypoint: ['sh', '-c', 'minio server /data']
      variables:
        MINIO_ROOT_USER: admin
        MINIO_ROOT_PASSWORD: password

    - name: '$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG'
      alias: appsemble
      variables:
        SECRET: secret
        AES_SECRET: secret
        HOST: 'http://appsemble:9999'
        REMOTE: 'https://appsemble.app'
        DATABASE_HOST: database
        DATABASE_PORT: 5432
        DATABASE_NAME: appsemble
        DATABASE_USER: admin
        DATABASE_PASSWORD: password
        S3_HOST: minio
        S3_SECURE: false
        S3_ACCESS_KEY: admin
        S3_SECRET_KEY: password
        migrateTo: next
        E2E: true
  before_script:
    - apt-get update && apt-get install -y curl dnsutils
    - |
      echo "Aliasing app domains to appsemble..."
      APPSEMBLE_IP=$(getent hosts appsemble | awk '{ print $1 }')
      for APP_NAME in ./apps/*/; do
        APP_NAME=$(basename "$APP_NAME")
        CUSTOM_ALIAS="${APP_NAME}.appsemble.appsemble"
        echo "$APPSEMBLE_IP $CUSTOM_ALIAS" >> /etc/hosts
        echo "Added alias: $CUSTOM_ALIAS -> $APPSEMBLE_IP"
      done
  script:
    - npm ci
    - npm run scripts seed-account --
      --name $BOT_ACCOUNT_NAME
      --email $BOT_ACCOUNT_EMAIL
      --password $BOT_ACCOUNT_PASSWORD
      --timezone Europe/Amsterdam
      --clientCredentials $APPSEMBLE_CLIENT_CREDENTIALS
    - npm run appsemble -- config set context e2e
    - npm run appsemble -- config set remote 'http://appsemble:9999'
    - npm run appsemble -- organization create appsemble
      --description "The open source low-code app building platform"
      --email support@appsemble.com
      --icon packages/server/assets/appsemble.png
      --name Appsemble
      --website https://appsemble.com
    - npm run appsemble -- block publish 'blocks/*'
    - npm run appsemble -- app publish 'apps/empty' --modify-context
    - npm run appsemble -- app publish 'apps/holidays' --modify-context
    - npm run appsemble -- app publish 'apps/notes' --modify-context
    - npm run appsemble -- app publish 'apps/person' --modify-context
    - npm run appsemble -- app publish 'apps/survey' --modify-context
    - npm run appsemble -- app publish 'apps/unlittered' --modify-context
    - npm --workspace @appsemble/types run prepack
    - npm --workspace @appsemble/lang-sdk run prepack
    - npm --workspace @appsemble/e2e run e2e -- --max-failures 6
  artifacts:
    expose_as: Playwright videos
    when: always
    paths:
      - packages/e2e/test-results/
    reports:
      junit: packages/e2e/results.xml

# Apps under SLA are supported through Appsemble core changes
e2e apps obligatory:
  allow_failure: false
  stage: end 2 end
  needs:
    - build docker image
  rules:
    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'appsemble/appsemble'
    - if: $CI_COMMIT_TAG
  inherit:
    variables: false
  variables:
    APPSEMBLE_IMAGE: '$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG'
  parallel:
    matrix:
      - PROJECT:
          - 'appsemble/apps/sittard-geleen'
          - 'appsemble/apps/restaurants'
          - 'appsemble/apps/minjus'
          - 'appsemble/apps/sounding-bodies'
  trigger:
    project: $PROJECT
    strategy: depend
    branch: main

# Allow failure as we don't support these anymore
e2e apps manual:
  when: manual
  allow_failure: true
  stage: end 2 end
  needs:
    - build docker image
  rules:
    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'appsemble/appsemble'
    - if: $CI_COMMIT_TAG
  inherit:
    variables: false
  variables:
    APPSEMBLE_IMAGE: '$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG'
  parallel:
    matrix:
      - PROJECT:
          - 'appsemble/apps/template'
          - 'appsemble/apps/eindhoven'
          - 'appsemble/apps/amersfoort'
          - 'appsemble/apps/tbvwonen'
  trigger:
    project: $PROJECT
    strategy: depend
    branch: main

###################################################################################################
#  Publish Stage                                                                                  #
###################################################################################################

twitter:
  extends: .npm
  stage: publish
  # This is new and mainly used to update nice-to-have metadata.
  # Allow failure to prevent blocking of important jobs.
  allow_failure: true
  rules:
    - if: $CI_COMMIT_TAG
  script:
    - npm run scripts -- twitter

# Update metadata on Docker Hub
docker metadata:
  extends: .npm
  stage: publish
  # This is new and mainly used to update nice-to-have metadata.
  # Allow failure to prevent blocking of important jobs.
  allow_failure: true
  rules:
    - if: $CI_COMMIT_TAG
  script:
    - npm run scripts -- docker-metadata

# Create a GitHub release
github release:
  extends: .npm
  stage: publish
  # This is new and mainly used to update nice-to-have metadata.
  # Allow failure to prevent blocking of important jobs.
  allow_failure: true
  rules:
    - if: $CI_COMMIT_TAG
  script:
    - npm run scripts -- github-release

# Create a GitLab release
gitlab release:
  extends: .npm
  stage: publish
  # This is new and mainly used to update nice-to-have metadata.
  # Allow failure to prevent blocking of important jobs.
  allow_failure: true
  rules:
    - if: $CI_COMMIT_TAG
  script:
    - npm run scripts -- gitlab-release

publish docker:
  extends: .docker
  stage: publish
  needs:
    - build docker image
  variables:
    GIT_STRATEGY: none
  rules:
    - if: $CI_COMMIT_TAG
  script:
    - echo $DOCKER_HUB_PASSWORD | docker login -u $DOCKER_HUB_USERNAME --password-stdin
    - docker pull "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
    - docker tag "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" "$CI_REGISTRY_IMAGE:latest"
    - docker tag "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG"
    - docker tag "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" "appsemble/appsemble:latest"
    - docker tag "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" "appsemble/appsemble:$CI_COMMIT_TAG"
    - docker push "$CI_REGISTRY_IMAGE:latest"
    - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG"
    - docker push "appsemble/appsemble:latest"
    - docker push "appsemble/appsemble:$CI_COMMIT_TAG"

# Publish the Helm chart to https://charts.appsemble.com
publish helm:
  image: dtzar/helm-kubectl:3@sha256:569fbcea3ad1bdb26c953dc76e24591e9ac06abaf577e9f9167a0517e7669425
  needs:
    - helm package
  stage: publish
  variables:
    GIT_STRATEGY: none
  rules:
    - if: $CI_COMMIT_TAG
  script:
    - apk add git gnupg
    - gpg --import "$GPG_PRIVATE_KEY"
    - git init
    - git remote add origin "https://appsemble-bot:$GITLAB_ACCESS_TOKEN@gitlab.com/appsemble/charts.git"
    - git fetch --depth 1
    - git checkout --track origin/main
    - git config commit.gpgSign true
    - git config user.email bot@appsemble.com
    - git config user.name Appsemble
    - git add public
    - git commit --message "Add Appsemble Helm chart $CI_COMMIT_TAG"
    - git push origin main
    - helm repo add appsemble https://charts.appsemble.com
    - | # Wait until the chart has been published downstream
      while ! helm show all 'appsemble/appsemble' --version "$CI_COMMIT_TAG" 2> /dev/null; do
        sleep 3
        echo retrying in 3 seconds…
        helm repo update > /dev/null
      done

# Publish npm packages to npmjs.org.
publish npm:
  stage: publish
  needs:
    - job: pack
      artifacts: true
  variables:
    GIT_STRATEGY: none
  rules:
    - if: $CI_COMMIT_TAG
  id_tokens:
    SIGSTORE_ID_TOKEN:
      aud: sigstore
  script:
    - npm config set //registry.npmjs.org/:_authToken "$NPM_TOKEN"
    - |
      if echo "$CI_RUNNED_TAGS" | grep -q "self-hosted"; then
        PROVENANCE_FLAG=""
        echo "Detected self-hosted tag, skipping provenance"
      else
        PROVENANCE_FLAG="--provenance"
        echo "No self-hosted tag detected, enabling provenance"
      fi
      echo "Using provenance flag: '$PROVENANCE_FLAG'"
      find ./build -name "*.tgz" -print0 | xargs -0 -I{} sh -c 'npm publish "{}" --access public '"$PROVENANCE_FLAG"''

# Perform various post release actions.
sentry release:
  needs: []
  image:
    name: getsentry/sentry-cli:2@sha256:2a60d341709d8ae040b842556c66f98443cc6d59101cab751f9811f3a1df9362
    entrypoint: [/bin/sh, -c]
  stage: publish
  rules:
    - if: $CI_COMMIT_TAG
  script:
    - sentry-cli releases new --finalize "$CI_COMMIT_TAG"
    - sentry-cli releases set-commits --auto "$CI_COMMIT_TAG"

# Create a test pre-release
release test:
  extends: .release
  variables:
    INCREMENT: prerelease
    IDENTIFIER: test

# Create a patch release
release patch:
  extends: .release
  variables:
    INCREMENT: patch

# Create a minor release
release minor:
  extends: .release
  variables:
    INCREMENT: minor

###################################################################################################
#  Deploy Stage                                                                                   #
###################################################################################################

# Update lockfile and fix formatting where possible
update dependencies:
  stage: deploy
  variables:
    GIT_STRATEGY: none
  rules:
    - if: $CI_PIPELINE_SOURCE == 'schedule' && $UPDATE_PROJECT_DEPS == 'true'
  cache:
    policy: push
    key:
      files:
        - package-lock.json
    paths:
      - node_modules
      - '**/node_modules'
  script:
    - apt update
    - apt install --yes git gnupg
    - gpg --import "$GPG_PRIVATE_KEY"
    - git init
    - git remote add origin "https://appsemble-bot:$GITLAB_ACCESS_TOKEN@gitlab.com/appsemble/appsemble.git"
    - git fetch --depth 1
    - git checkout -B update-lockfile origin/main
    - git config commit.gpgSign true
    - git config user.email bot@appsemble.com
    - git config user.name Appsemble
    - rm package-lock.json
    - npm install --ignore-scripts
    - npx -- eslint --fix . || true
    - npx -- remark . -o || true
    - npx -- stylelint --fix . || true
    - npx -- prettier --write . || true
    - git commit --all --message 'Update package-lock.json'
    - git push --force origin update-lockfile
      -o merge_request.create
      -o merge_request.label=Chore
      -o merge_request.merge_when_pipeline_succeeds
      -o merge_request.remove_source_branch

# Deploy the Docker image for a branch to a review environment.
review:
  image: dtzar/helm-kubectl:3.16.0@sha256:3d0876a8a4159fddfb381fcccf23c774bcfcd4b4ffdbb80c1a5c10c1eb23de54
  stage: deploy
  needs:
    - build docker image
    - job: helm package
      artifacts: true
  rules:
    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'appsemble/appsemble'
  environment:
    name: review/$CI_MERGE_REQUEST_IID
    url: https://$CI_MERGE_REQUEST_IID.appsemble.review
    on_stop: stop review
    auto_stop_in: 1 day
  variables:
    GIT_STRATEGY: none
  script:
    - kubectl config use-context "$CI_PROJECT_PATH:development"
    - kubectl config set-context --current --namespace review
    # This is the only way to make sure the review environment can be overwritten
    - helm delete "review-$CI_MERGE_REQUEST_IID" || true
    - kubectl delete pvc --selector "app.kubernetes.io/instance=review-$CI_MERGE_REQUEST_IID" || true
    - kubectl delete namespace "companion-containers-review-$CI_MERGE_REQUEST_IID" || true
    - kubectl delete secret review-$CI_MERGE_REQUEST_IID-mailpit-tls || true
    - apk add --no-cache openssl
    - openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout mailpit.key -out mailpit.crt -subj "/CN=review-${CI_MERGE_REQUEST_IID}-mailpit-smtp" -addext "subjectAltName=DNS:review-${CI_MERGE_REQUEST_IID}-mailpit-smtp"
    - kubectl create secret tls review-$CI_MERGE_REQUEST_IID-mailpit-tls --cert=mailpit.crt --key=mailpit.key --dry-run=client -o yaml | kubectl apply -f -
    - rm mailpit.key mailpit.crt
    # default bitnami postgresql PVC size is 8Gi, but that much should not be needed for a review environment
    - helm install "review-$CI_MERGE_REQUEST_IID" ./public/appsemble-*.tgz
      --debug
      --atomic
      --set "fullnameOverride=review-$CI_MERGE_REQUEST_IID"
      --set "gitlab.app=$CI_PROJECT_PATH_SLUG"
      --set "gitlab.env=$CI_ENVIRONMENT_SLUG"
      --set "image.pullPolicy=Always"
      --set "image.repository=$CI_REGISTRY_IMAGE"
      --set "image.tag=$CI_COMMIT_REF_SLUG"
      --set "ingress.clusterIssuer=acme-issuer-dev"
      --set "ingress.enabled=true"
      --set "ingress.host=${CI_ENVIRONMENT_URL/https:\/\//}"
      --set "ingress.tls.secretName=review-$CI_MERGE_REQUEST_IID-tls"
      --set "ingress.tls.wildcardSecretName=review-$CI_MERGE_REQUEST_IID-tls-wildcard"
      --set "migrateTo=next"
      --set "global.postgresql.auth.existingSecret=postgresql-secret"
      --set "postgresql.primary.persistence.size=1Gi"
      --set "provision.organization.id=appsemble"
      --set "provision.user.organizationRole=Maintainer"
      --set "provision.user.clientCredentials=$APPSEMBLE_CLIENT_CREDENTIALS"
      --set "provision.user.email=$BOT_ACCOUNT_EMAIL"
      --set "provision.user.name=$BOT_ACCOUNT_NAME"
      --set "provision.user.password=$BOT_ACCOUNT_PASSWORD"
      --set "provision.user.timezone=Europe/Amsterdam"
      --set "postgresql.fullnameOverride=review-$CI_MERGE_REQUEST_IID-postgresql"
      --set "proxy=true"
      --set "remote=https://appsemble.app"
      --set "sentry.allowedDomains={*.appsemble.review}"
      --set "sentry.environment=review-$CI_MERGE_REQUEST_IID"
      --set "sentry.secret=sentry"
      --set "mailpit.mailpit.smtp.tls.secretName=review-$CI_MERGE_REQUEST_IID-mailpit-tls"
      --set "minio.persistence.size=1Gi"
      --set "minio.fullnameOverride=review-$CI_MERGE_REQUEST_IID-minio"
      --set "minio.ingress.hostname=console.minio.${CI_ENVIRONMENT_URL/https:\/\//}"
      --set "minio.ingress.annotations.cert-manager\\.io/cluster-issuer=acme-issuer-dev"
      --set "minio.apiIngress.hostname=api.minio.${CI_ENVIRONMENT_URL/https:\/\//}"
      --set "minio.apiIngress.annotations.cert-manager\\.io/cluster-issuer=acme-issuer-dev"
      --set "minio.apiIngress.annotations.nginx\\.ingress\\.kubernetes\\.io/proxy-body-size=256m"

# Stop a review environment.
stop review:
  image: dtzar/helm-kubectl:3@sha256:569fbcea3ad1bdb26c953dc76e24591e9ac06abaf577e9f9167a0517e7669425
  dependencies: []
  stage: deploy
  rules:
    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'appsemble/appsemble'
      when: manual
  allow_failure: true
  environment:
    name: review/$CI_MERGE_REQUEST_IID
    action: stop
  variables:
    GIT_STRATEGY: none
  script:
    - kubectl config use-context "$CI_PROJECT_PATH:development"
    - kubectl config set-context --current --namespace review
    - helm delete "review-$CI_MERGE_REQUEST_IID"
    - kubectl delete pvc --selector "app.kubernetes.io/instance=review-$CI_MERGE_REQUEST_IID"
    - kubectl delete namespace "companion-containers-review-$CI_MERGE_REQUEST_IID" || true
    - kubectl get secrets --no-headers -o custom-columns=:metadata.name | grep "^review-$CI_MERGE_REQUEST_IID" | xargs -I{} kubectl delete secret {} || true

# Deploy the Docker image for main to the staging environment.
staging:
  image: dtzar/helm-kubectl:3@sha256:569fbcea3ad1bdb26c953dc76e24591e9ac06abaf577e9f9167a0517e7669425
  stage: deploy
  needs:
    - build docker image
    - job: helm package
      artifacts: true
  rules:
    - if: $CI_COMMIT_BRANCH == 'staging'
  environment:
    name: staging
    url: https://staging.appsemble.review
  variables:
    GIT_STRATEGY: none
  script:
    - kubectl config use-context "$CI_PROJECT_PATH:development"
    - kubectl config set-context --current --namespace staging
    # This is the only way to make sure staging can be overwritten
    - helm delete staging || true
    - kubectl delete pvc --selector "app.kubernetes.io/instance=staging" || true
    - kubectl delete namespace "companion-containers-staging" || true
    - kubectl delete secret staging-mailpit-tls || true
    - apk add --no-cache openssl
    - openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout mailpit.key -out mailpit.crt -subj "/CN=staging-appsemble-mailpit-smtp" -addext "subjectAltName=DNS:staging-appsemble-mailpit-smtp"
    - kubectl create secret tls staging-mailpit-tls --cert=mailpit.crt --key=mailpit.key --dry-run=client -o yaml | kubectl apply -f -
    - rm mailpit.key mailpit.crt
    - helm install staging ./public/appsemble-*.tgz
      --atomic
      --timeout 15m0s
      --set "gitlab.app=$CI_PROJECT_PATH_SLUG"
      --set "gitlab.env=$CI_ENVIRONMENT_SLUG"
      --set "global.postgresql.auth.existingSecret=postgresql-secret"
      --set "image.pullPolicy=Always"
      --set "image.repository=$CI_REGISTRY_IMAGE"
      --set "image.tag=staging"
      --set "ingress.clusterIssuer=acme-issuer-dev"
      --set "ingress.enabled=true"
      --set "ingress.host=staging.appsemble.review"
      --set "ingress.tls.secretName=staging-tls"
      --set "ingress.tls.wildcardSecretName=staging-tls-wildcard"
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/proxy-body-size=1G"
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/force-ssl-redirect=\"true\""
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/ssl-redirect=\"true\""
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/hsts=\"true\""
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/hsts-include-subdomains=\"true\""
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/hsts-max-age=\"31536000\""
      --set "migrateTo=next"
      --set "oauthSecret=oauth2"
      --set "postgresql.fullnameOverride=staging-postgresql"
      --set "provision.organization.id=appsemble"
      --set "provision.user.organizationRole=Maintainer"
      --set "provision.user.clientCredentials=$APPSEMBLE_CLIENT_CREDENTIALS"
      --set "provision.user.email=$BOT_ACCOUNT_EMAIL"
      --set "provision.user.name=$BOT_ACCOUNT_NAME"
      --set "provision.user.password=$BOT_ACCOUNT_PASSWORD"
      --set "provision.user.timezone=Europe/Amsterdam"
      --set "importData.remoteStorage.url=$S3_REMOTE_STORAGE_URL"
      --set "importData.remoteStorage.existingAccessKeySecret=s3-access-key-secret"
      --set "importData.remoteStorage.region=$S3_REGION"
      --set "importData.remoteStorage.location=$BACKUPS_LOCATION"
      --set "importData.remoteStorage.filename=$PROD_BACKUP_NAME"
      --set "migrateDefinitions.save=true"
      --set "proxy=true"
      --set "remote=https://appsemble.app"
      --set "sentry.allowedDomains={*.appsemble.review}"
      --set "sentry.environment=staging"
      --set "sentry.secret=sentry"
      --set "quotas.appEmail.enabled=true"
      --set "quotas.appEmail.dailyLimit=10"
      --set "quotas.appEmail.alertOrganizationOwner=true"
      --set "mailpit.mailpit.smtp.tls.secretName=staging-mailpit-tls"
      --set "mailpit.fullnameOverride=staging-appsemble-mailpit"
      --set "minio.fullnameOverride=staging-minio"
      --set "minio.ingress.hostname=console.minio.staging.appsemble.review"
      --set "minio.ingress.annotations.cert-manager\\.io/cluster-issuer=acme-issuer-dev"
      --set "minio.apiIngress.hostname=api.minio.staging.appsemble.review"
      --set "minio.apiIngress.annotations.cert-manager\\.io/cluster-issuer=acme-issuer-dev"
      --set "minio.apiIngress.annotations.nginx\\.ingress\\.kubernetes\\.io/proxy-body-size=256m"

production:
  image: dtzar/helm-kubectl:3@sha256:569fbcea3ad1bdb26c953dc76e24591e9ac06abaf577e9f9167a0517e7669425
  stage: deploy
  needs:
    - publish docker
    - publish helm
  rules:
    - if: $CI_COMMIT_TAG
  environment:
    name: production
    url: https://appsemble.app
  variables:
    GIT_STRATEGY: none
  script:
    - kubectl config use-context "$CI_PROJECT_PATH:production"
    - kubectl config set-context --current --namespace appsemble
    - helm repo add appsemble https://charts.appsemble.com
    # Prevent a finished job without chart actually serving.
    - |
      retries=0
      found=0
      while [ $retries -lt 100 ] && [ $found -ne 1 ]; do
        helm repo update
        if helm search repo appsemble/appsemble --version "$CI_COMMIT_TAG" --fail-on-no-result; then
          echo "Chart found!"
          found=1
        else
          echo "Chart version $CI_COMMIT_TAG not found, retrying in 5 seconds..."
          sleep 5
          retries=$((retries + 1))
        fi
      done
      if [ $found -ne 1 ]; then
        echo "Maximum number of attempts reached. Chart not found."
        exit 1
      fi
    - kubectl delete secret appsemble-mailpit-tls || true
    - apk add --no-cache openssl
    - openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout mailpit.key -out mailpit.crt -subj "/CN=appsemble-mailpit-smtp" -addext "subjectAltName=DNS:appsemble-mailpit-smtp"
    - kubectl create secret tls appsemble-mailpit-tls --cert=mailpit.crt --key=mailpit.key --dry-run=client -o yaml | kubectl apply -f -
    - rm mailpit.key mailpit.crt
    - kubectl rollout restart deployment appsemble-mailpit || true
    - helm upgrade appsemble appsemble/appsemble
      --debug
      --timeout 30m
      --version "$CI_COMMIT_TAG"
      --set "gitlab.app=$CI_PROJECT_PATH_SLUG"
      --set "gitlab.env=$CI_ENVIRONMENT_SLUG"
      --set "global.postgresql.auth.existingSecret=$POSTGRESQL_SECRET"
      --set "global.postgresql.auth.database=$POSTGRESQL_DATABASE"
      --set "global.postgresql.auth.username=$POSTGRESQL_USERNAME"
      --set "global.postgresql.service.ports.postgresql=$POSTGRESQL_PORT"
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/proxy-body-size=1G"
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/force-ssl-redirect=\"true\""
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/ssl-redirect=\"true\""
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/hsts=\"true\""
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/hsts-include-subdomains=\"true\""
      --set "ingress.annotations.nginx\.ingress\.kubernetes\.io/hsts-max-age=\"31536000\""
      --set "ingress.clusterIssuer=acme-issuer-prod"
      --set "ingress.enabled=true"
      --set "ingress.host=appsemble.app"
      --set "ingress.tls.secretName=appsemble-tls"
      --set "ingress.tls.wildcardSecretName=appsemble-tls-wildcard"
      --set "oauthSecret=oauth2"
      --set "postgresql.fullnameOverride=$POSTGRESQL_HOST"
      --set "postgresql.enabled=false"
      --set 'postgresSSL=true'
      --set "migrateDefinitions.save=true"
      --set "proxy=true"
      --set "sentry.allowedDomains={*.appsemble.app, appsemble.app}"
      --set "sentry.environment=production"
      --set "sentry.secret=sentry"
      --set "quotas.appEmail.enabled=true"
      --set "quotas.appEmail.dailyLimit=10"
      --set "quotas.appEmail.alertOrganizationOwner=true"
      --set "mailpit.mailpit.smtp.tls.secretName=appsemble-mailpit-tls"
      --set "minio.ingress.hostname=console.minio.appsemble.app"
      --set "minio.ingress.annotations.cert-manager\\.io/cluster-issuer=acme-issuer-prod"
      --set "minio.apiIngress.hostname=api.minio.appsemble.app"
      --set "minio.apiIngress.annotations.cert-manager\\.io/cluster-issuer=acme-issuer-prod"
      --set "minio.apiIngress.annotations.nginx\\.ingress\\.kubernetes\\.io/proxy-body-size=256m"
      --wait

###################################################################################################
#  Provision Stage                                                                                #
###################################################################################################

# Provision the review environment with the blocks that were built in the build stage.
provision review:
  extends: .npm
  stage: provision
  rules:
    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'appsemble/appsemble'
      when: manual
      allow_failure: true
  variables:
    INCREMENT: patch
    IDENTIFIER: ''
  needs:
    - review
    - npm_ci
  script:
    - npm run scripts -- release "$INCREMENT" ${IDENTIFIER:+--identifier "$IDENTIFIER"}
    - npm run appsemble -- config set context review
    - npm run appsemble -- config set remote "https://$CI_MERGE_REQUEST_IID.appsemble.review"
    - npm run scripts -- wait-for-api
    - npm run appsemble -- block create --organization appsemble --template vanilla --name template-vanilla --path blocks
    - npm run appsemble -- block create --organization appsemble --template preact --name template-preact --path blocks
    - npm run appsemble -- block create --organization appsemble --template mini-jsx --name template-mini-jsx --path blocks
    - npm run appsemble -- block publish 'blocks/*'
    - npm run appsemble -- app publish 'apps/*' --resources --assets --assets-clonable --modify-context

# Provision the staging environment with the blocks that were built in the build stage.
provision staging:
  extends: .npm
  stage: provision
  rules:
    - if: $CI_COMMIT_BRANCH == 'staging'
      when: manual
      allow_failure: true
  variables:
    INCREMENT: patch
    IDENTIFIER: ''
  needs:
    - staging
    - npm_ci
  script:
    - npm run scripts -- release "$INCREMENT" ${IDENTIFIER:+--identifier "$IDENTIFIER"}
    - npm run appsemble -- config set context staging
    - npm run appsemble -- config set remote https://staging.appsemble.review
    - npm run appsemble -- -vv block publish blocks/*
    - apt-get update
    - apt-get install --yes git gnupg yq
    - ./scripts/publish-or-update-apps.sh staging

# Provision the production environment with the blocks that were built in the build stage.
provision production:
  extends: .npm
  stage: provision
  rules:
    - if: $CI_COMMIT_TAG
  needs:
    - production
    - npm_ci
  environment:
    name: production
    action: prepare
  script:
    - npm run appsemble -- config set context production
    - npm run appsemble -- config set remote https://appsemble.app
    - npm run appsemble -- config set ignore-conflict true
    - npm run appsemble -- -vv block publish blocks/*
    - apt-get update
    - apt-get install --yes git gnupg yq
    - ./scripts/publish-or-update-apps.sh --push-to-main production

###################################################################################################
#  Performance Stage                                                                               #
###################################################################################################

#
# --- Disabled for now ---
#
# lighthouse:
#   interruptible: true
#   stage: performance
#   extends: .npm
#   image: $CI_REGISTRY/appsemble/docker-node-chrome
#   needs:
#     - npm_ci
#     - job: provision review
#       optional: true
#     - job: provision staging
#       optional: true
#     - job: provision production
#       optional: true
#   rules:
#     - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH == 'appsemble/appsemble'
#     - if: $CI_COMMIT_TAG
#   script:
#     - npm run scripts -- wait-for-ssl
#     - npx -- lhci collect
#     - npx -- lhci assert
#   artifacts:
#     expose_as: Lighthouse reports
#     when: always
#     paths:
#       - .lighthouseci

###################################################################################################
#  Backup Stage                                                                                   #
###################################################################################################

backup production data:
  stage: backup
  image: postgres:15-alpine@sha256:5402d0a13eab398c7c38f1b90af081d7f9e5977606ed869cecdb661403f6586a
  tags:
    - sole-occupant
  rules:
    - if: $CI_COMMIT_BRANCH == 'main'
      when: manual
  allow_failure: true
  variables:
    GIT_STRATEGY: none
  script:
    - apk add --no-cache rclone
    - rclone config create hetzner-s3 s3 url $S3_REMOTE_STORAGE_URL vendor other access_key_id $S3_ACCESS_KEY_ID endpoint $S3_REMOTE_STORAGE_URL secret_access_key $S3_SECRET_ACCESS_KEY region $S3_REGION provider Other
    - LATEST_TARGET="${PROD_BACKUP_NAME}.sql.gz"
    - VERSIONED_TARGET="${PROD_BACKUP_NAME}_$(date '+%Y%m%d%H%M').sql.gz"
    - pg_dump "$PRODUCTION_DB_URL" | gzip > "$VERSIONED_TARGET"
    - rclone copy -v -I "$VERSIONED_TARGET" "$BACKUPS_LOCATION"
    - rclone copyto -v -I "$VERSIONED_TARGET" "$BACKUPS_LOCATION/$LATEST_TARGET"

###################################################################################################
#  Cleanup Stage                                                                                  #
###################################################################################################

# Stop and cleanup all review environments.
cleanup review environments:
  image: dtzar/helm-kubectl:3@sha256:569fbcea3ad1bdb26c953dc76e24591e9ac06abaf577e9f9167a0517e7669425
  dependencies: []
  stage: cleanup
  rules:
    - if: $CI_PIPELINE_SOURCE == 'schedule' && $CLEANUP_REVIEW_ENVIRONMENTS == 'true'
    - if: $CI_COMMIT_BRANCH == 'main'
      when: manual
  allow_failure: true
  variables:
    GIT_STRATEGY: none
  script:
    - kubectl config use-context "$CI_PROJECT_PATH:development"
    - kubectl config set-context --current --namespace review
    # Delete all review managed ingresses instead of relying on the pre-delete hooks
    - kubectl get ingress --selector app.kubernetes.io/managed-by -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\n"}{end}' | grep '^.* review-' | awk '{print "kubectl delete ingress " $1}' | bash || true
    # Delete all review managed secrets instead of relying on the pre-delete hooks
    - kubectl get secret --selector app.kubernetes.io/managed-by -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.app\.kubernetes\.io/managed-by}{"\n"}{end}' | grep '^.* review-' | awk '{print "kubectl delete secret " $1}' | bash || true
    # Delete all helm review releases without hooks
    - helm list --short | grep '^review-' | xargs -I {} helm delete {} --no-hooks || true
    # Delete all remaining review kubernetes resources with instance selector
    - kubectl get all --selector app.kubernetes.io/instance -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.app\.kubernetes\.io/instance}{" "}{.kind}{"\n"}{end}' | grep '^.* review-' | awk '{kind=tolower($3); print "kubectl delete " kind " " $1;}' | bash || true
    # Delete all persistent volume claims from review environments
    - kubectl get pvc --selector app.kubernetes.io/instance -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.app\.kubernetes\.io/instance}{"\n"}{end}' | grep '^.* review-' | awk '{print "kubectl delete pvc " $1}' | bash || true
    # Delete all companion container namespaces for review environments
    - kubectl get namespaces --no-headers -o custom-columns=:metadata.name | grep '^companion-containers-review-' | xargs -I{} kubectl delete namespace {} || true
    # Delete all ingresses for review environments
    - kubectl get ingress --no-headers -o custom-columns=:metadata.name | grep '^review-' | xargs -I{} kubectl delete ingress {} || true
    # Delete all secrets for review environments
    - kubectl get secrets --no-headers -o custom-columns=:metadata.name | grep '^review-' | xargs -I{} kubectl delete secret {} || true