.pipeline.yml

variables:
  FF_SCRIPT_SECTIONS: "true"
  GIT_CLONE_PATH: "${CI_BUILDS_DIR}/${CI_PROJECT_NAME}/${CI_JOB_ID}"
  CI_DEBUG_SERVICES: "false"

include:
  - project: "devops/gitlab/ci-templates/ruby"
    ref: "sans-dind"
    file:
      - ".rspec.yml"
      - ".rubocop.yml"
  - project: "devops/gitlab/ci-templates/docker"
    ref: "1.0.0"
    file:
      - ".amend_manifests.yml"
      - ".build_and_push_docker_images.yml"
      - ".copy_docker_images.yml"
      - ".remove_tmp_registry.yml"
  - project: "devops/gitlab/ci-templates/sast"
    ref: "master"
    file:
      - ".trivy_container_scanning.yml"
  - component: gitlab.library.upenn.edu/devops/gitlab/components/deploy/deploy@~latest
    inputs:
      deploy_environment: "staging"
      deploy_environment_url: "https://find-staging.library.upenn.edu"
      deploy_host: ${DEPLOY_HOST}
      job_suffix: "staging"
      valid_principals: ${VALID_PRINCIPALS}
      vault_auth_role: ${JWT_ROLE}
      vault_ssh_client_role: ${VAULT_SSH_CLIENT_ROLE}
      image_tag: ${CI_COMMIT_SHORT_SHA}
  - component: gitlab.library.upenn.edu/devops/gitlab/components/deploy/deploy@~latest
    inputs:
      deploy_environment: "production"
      deploy_environment_url: "https://find.library.upenn.edu"
      deploy_host: ${DEPLOY_HOST}
      job_suffix: "production"
      valid_principals: ${VALID_PRINCIPALS}
      vault_auth_role: ${JWT_ROLE}
      vault_ssh_client_role: ${VAULT_SSH_CLIENT_ROLE}
      image_tag: ${CI_COMMIT_SHORT_SHA}
  - component: gitlab.library.upenn.edu/devops/gitlab/components/general/vault_jwt_auth@~latest
    inputs:
      vault_auth_role: ${JWT_ROLE}
  - component: gitlab.library.upenn.edu/devops/gitlab/components/general/install_hashicorp_vault@~latest

stages:
  - build_and_push:tmp_base_image
  - build_and_push:tmp_dev_image
  - amend_tmp_base_dev_images
  - test
  - build_and_push:tmp_prod_image
  - amend_prod_image
  - sast
  - copy_docker_images
  - remove_tmp_registry
  - deploy

workflow:
  rules:
  - when: always

# -- Start - build and push base image
build_and_push_base_app_image:
  stage: build_and_push:tmp_base_image
  extends:
    - .build_and_push_docker_images
    - .vault_jwt_auth
  before_script:
    - !reference [.install_hashicorp_vault, before_script]
    - !reference [.vault_jwt_auth, before_script]
    - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
    - export SIDEKIQ_PRO_CREDENTIALS="$(vault kv get -field=credentials ${VAULT_SIDEKIQ_ENDPOINT})"
  parallel:
    matrix:
      - RUNNER: build-arm
        DOCKER_BUILD_PLATFORM: "linux/arm64"
      - RUNNER: build
        DOCKER_BUILD_PLATFORM: "linux/amd64"
  variables:
    DOCKER_BUILD_CONTEXT: "rails_app/"
    DOCKER_BUILD_SECRETS: "--secret id=sidekiq_pro_credentials,env=SIDEKIQ_PRO_CREDENTIALS"
    DOCKER_TARGET: "base"
    DOCKERFILE_PATH: "rails_app/"
    JWT_ROLE: "sidekiq_pro_ro"
    REGISTRY_REPOSITORY: "${CI_PIPELINE_ID}"
  tags:
    - ${RUNNER}
# -- End - build and push base image

# -- Start - build, push temp dev image
build_and_push_dev_app_image:
  stage: build_and_push:tmp_dev_image
  extends:
    - .build_and_push_docker_images
  parallel:
    matrix:
      - RUNNER: build-arm
        DOCKER_BUILD_PLATFORM: "linux/arm64"
      - RUNNER: build
        DOCKER_BUILD_PLATFORM: "linux/amd64"
  variables:
    DOCKER_ADDITIONAL_CACHE_TARGETS: "base"
    DOCKER_BUILD_ARGS: "--build-arg RAILS_ENV=development"
    DOCKER_BUILD_CONTEXT: "rails_app/"
    DOCKER_TARGET: "development"
    DOCKERFILE_PATH: "rails_app/"
    REGISTRY_REPOSITORY: "${CI_PIPELINE_ID}"
  needs:
    - build_and_push_base_app_image
  tags:
    - ${RUNNER}

amend_tmp_base_dev_images:
  stage: amend_tmp_base_dev_images
  extends:
    - .amend_manifests
  needs:
    - build_and_push_base_app_image
    - build_and_push_dev_app_image
  variables:
    REGISTRY_REPOSITORY: "${CI_PIPELINE_ID}"
  tags:
    - build
# -- End - build, push temp dev image

# -- Start - run unit tests in dev image
rspec_app_unit_test:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/${CI_PIPELINE_ID}:${CI_COMMIT_SHORT_SHA}-development
  services:
    - name: postgres:16.12
      variables:
        POSTGRES_DB: "find_test"
        POSTGRES_PASSWORD: "rspec_test"
        POSTGRES_USER: "rspec_test"
    - name: bitnami/solr:9.3.0
      alias: solr
      entrypoint: [""]
      command: [
        "/bin/bash",
        "-c",
        "./opt/bitnami/scripts/solr/entrypoint.sh /opt/bitnami/scripts/solr/run.sh & \n
        for ((i = 0; i <= 9; i += 1)); do \n
          curl --silent 'http://solr:8983/api/' && break; \n
          echo 'Waiting for solr api to be ready'; \n
          sleep '10s'; \n
        done \n
        until [ -s \"${CI_PROJECT_DIR}/rails_app/solr/conf\" ]; do sleep 1; echo 'Waiting for solr conf to exist'; done \n
        .${ZK_SCRIPT_LOCATION}/zkcli.sh -zkhost 127.0.0.1:9983 -cmd upconfig -confname ${SOLR_TEST_CONFIGNAME} -confdir ${CI_PROJECT_DIR}/rails_app/solr/conf && \n
        ./opt/bitnami/solr/bin/solr create_collection -c ${SOLR_TEST_COLLECTION} -n ${SOLR_TEST_CONFIGNAME} \n
        wait"
      ]
      variables:
        SOLR_CLOUD_BOOTSTRAP: "yes"
        SOLR_ENABLE_AUTHENTICATION: "yes"
        SOLR_HOST: "solr"
        SOLR_JAVA_MEM: "-Xms4g -Xmx4g"
        SOLR_MODE: "solrcloud"
        SOLR_OPTS: "-Dsolr.disableConfigSetsCreateAuthChecks=true"
        SOLR_TEST_COLLECTION: "find-test"
        SOLR_TEST_CONFIGNAME: "find"
        ZK_CREATE_CHROOT: "true"
        ZK_SCRIPT_LOCATION: "/opt/bitnami/solr/server/scripts/cloud-scripts"
  extends:
    - .rspec
  variables:
    APP_URL: "find.test.com"
    CI: "true"
    DATABASE_HOST: "postgres"
    DATABASE_NAME: "find_test"
    DATABASE_PASSWORD: "rspec_test"
    DATABASE_USER: "rspec_test"
    FF_NETWORK_PER_BUILD: "true"
    RAILS_ENV: "test"
    RAILS_SYSTEM_TESTING_SCREENSHOT_HTML: "1"
    SOLR_URL: "http://solr:8983"
    SOLR_TEST_URL: http://solr:8983/solr/find-test
  before_script:
    - mkdir -p /tmp/indexing && cd /tmp/indexing
    - git clone -n --depth=1 --filter=tree:0 https://gitlab.library.upenn.edu/dld/catalog/catalog-indexing.git .
    - git sparse-checkout set --no-cone rails_app/solr/conf
    - git checkout
    - rm -fr ${CI_PROJECT_DIR}/rails_app/solr/*
    - mv /tmp/indexing/rails_app/solr/conf ${CI_PROJECT_DIR}/rails_app/solr
  script:
    - >
      for ((i = 0; i <= 9; i += 1)); do
        curl --silent "http://solr:8983/api/" "-H" "'Content-type:application/json'" && break
        echo "Waiting for solr api to be ready"
        sleep "10s"
      done
    - until [ -s "${CI_PROJECT_DIR}/rails_app/solr/conf" ]; do sleep 1; echo 'Waiting for solr conf to exist'; done
    - cd ${CI_PROJECT_DIR}/rails_app/
    - bundle add rspec_junit_formatter
    - bundle exec rake db:migrate
    - bundle exec rspec --format progress --format RspecJunitFormatter --out rspec.xml --profile 10 --tag ~type:system
  needs:
    - amend_tmp_base_dev_images
  coverage: '/\(\d+.\d+\%\) covered/'
  artifacts:
    paths:
      - rails_app/tmp/coverage/index.html
      - rails_app/tmp/capybara
      - rails_app/rspec.xml
    expire_in: 1 week
    reports:
      junit: rails_app/rspec.xml
  rules:
    - if: $DISABLE_RSPEC_TEST == "true"
      when: never
    - exists:
        - rails_app/spec/**.rb
        - rails_app/spec/**/**.rb
  tags:
    - build

# -- integration tests for dev image
rspec_app_integration_test:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/${CI_PIPELINE_ID}:${CI_COMMIT_SHORT_SHA}-development
  services:
    - name: ghcr.io/browserless/chromium:v2.38.4
      alias: chrome
      variables:
        PORT: 3333
        CONCURRENT: 1
        TIMEOUT: 1200000 # 20min should cover the system test run period
    - name: postgres:16.12
      variables:
        POSTGRES_DB: "find_test"
        POSTGRES_PASSWORD: "rspec_test"
        POSTGRES_USER: "rspec_test"
    - name: bitnami/solr:9.3.0
      alias: solr
      entrypoint: [""]
      command: [
        "/bin/bash",
        "-c",
        "./opt/bitnami/scripts/solr/entrypoint.sh /opt/bitnami/scripts/solr/run.sh & \n
        for ((i = 0; i <= 9; i += 1)); do \n
          curl --silent 'http://solr:8983/api/' && break; \n
          echo 'Waiting for solr api to be ready'; \n
          sleep '10s'; \n
        done \n
        until [ -s \"${CI_PROJECT_DIR}/rails_app/solr/conf\" ]; do sleep 1; echo 'Waiting for solr conf to exist'; done \n
        .${ZK_SCRIPT_LOCATION}/zkcli.sh -zkhost 127.0.0.1:9983 -cmd upconfig -confname ${SOLR_TEST_CONFIGNAME} -confdir ${CI_PROJECT_DIR}/rails_app/solr/conf && \n
        ./opt/bitnami/solr/bin/solr create_collection -c ${SOLR_TEST_COLLECTION} -n ${SOLR_TEST_CONFIGNAME} \n
        wait"
      ]
      variables:
        SOLR_CLOUD_BOOTSTRAP: "yes"
        SOLR_ENABLE_AUTHENTICATION: "yes"
        SOLR_HOST: "solr"
        SOLR_JAVA_MEM: "-Xms4g -Xmx4g"
        SOLR_MODE: "solrcloud"
        SOLR_OPTS: "-Dsolr.disableConfigSetsCreateAuthChecks=true"
        SOLR_TEST_COLLECTION: "find-test"
        SOLR_TEST_CONFIGNAME: "find"
        ZK_CREATE_CHROOT: "true"
        ZK_SCRIPT_LOCATION: "/opt/bitnami/solr/server/scripts/cloud-scripts"
  extends:
    - .rspec
  variables:
    APP_URL: "find.test.com"
    CI: "true"
    CHROME_URL: "ws://chrome:3333"
    DATABASE_HOST: "postgres"
    DATABASE_NAME: "find_test"
    DATABASE_PASSWORD: "rspec_test"
    DATABASE_USER: "rspec_test"
    FF_NETWORK_PER_BUILD: "true"
    RAILS_ENV: "test"
    RAILS_SYSTEM_TESTING_SCREENSHOT_HTML: "1"
    SOLR_URL: "http://solr:8983"
    SOLR_TEST_URL: http://solr:8983/solr/find-test
  before_script:
    - mkdir -p /tmp/indexing && cd /tmp/indexing
    - git clone -n --depth=1 --filter=tree:0 https://gitlab.library.upenn.edu/dld/catalog/catalog-indexing.git .
    - git sparse-checkout set --no-cone rails_app/solr/conf
    - git checkout
    - rm -fr ${CI_PROJECT_DIR}/rails_app/solr/*
    - mv /tmp/indexing/rails_app/solr/conf ${CI_PROJECT_DIR}/rails_app/solr
  script:
    - >
      for ((i = 0; i <= 9; i += 1)); do
        curl --silent "http://solr:8983/api/" "-H" "'Content-type:application/json'" && break
        echo "Waiting for solr api to be ready"
        sleep "10s"
      done
    - until [ -s "${CI_PROJECT_DIR}/rails_app/solr/conf" ]; do sleep 1; echo 'Waiting for solr conf to exist'; done
    - cd ${CI_PROJECT_DIR}/rails_app/
    - bundle add rspec_junit_formatter
    - bundle exec rake db:migrate
    - bundle exec rspec --format progress --format RspecJunitFormatter --out rspec.xml --profile 10 --tag type:system
  needs:
    - amend_tmp_base_dev_images
  coverage: '/\(\d+.\d+\%\) covered/'
  artifacts:
    paths:
      - rails_app/tmp/coverage/index.html
      - rails_app/tmp/capybara
      - rails_app/rspec.xml
    expire_in: 1 week
    reports:
      junit: rails_app/rspec.xml
  rules:
    - if: $DISABLE_RSPEC_TEST == "true"
      when: never
    - exists:
        - rails_app/spec/**.rb
        - rails_app/spec/**/**.rb
  allow_failure: true
  tags:
    - build

rubocop_app_test:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/${CI_PIPELINE_ID}:${CI_COMMIT_SHORT_SHA}-development
  needs:
    - amend_tmp_base_dev_images
  extends:
    - .rubocop
  before_script:
    - cd ${CI_PROJECT_DIR}/rails_app/
  rules:
    - if: $DISABLE_RUBOCOP == "true"
      when: never
    - exists:
        - rails_app/.rubocop.yml
  tags:
    - build

reek_app_test:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/${CI_PIPELINE_ID}:${CI_COMMIT_SHORT_SHA}-development
  needs:
    - amend_tmp_base_dev_images
  before_script:
    cd ${CI_PROJECT_DIR}/rails_app/
  script:
    bundle exec reek app
  rules:
    - if: $DISABLE_REEK == "true"
      when: never
    - exists:
        - rails_app/.reek.yml
  allow_failure: false
  tags:
    - build
# -- End - test dev image

# -- Start - build and push temp prod image
build_and_push_prod_app_image:
  stage: build_and_push:tmp_prod_image
  extends:
    - .build_and_push_docker_images
    - .vault_jwt_auth
  parallel:
    matrix:
      - RUNNER: build-arm
        DOCKER_BUILD_PLATFORM: "linux/arm64"
      - RUNNER: build
        DOCKER_BUILD_PLATFORM: "linux/amd64"
  before_script:
    - !reference [.install_hashicorp_vault, before_script]
    - !reference [.vault_jwt_auth, before_script]
    - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
    - export SIDEKIQ_PRO_CREDENTIALS="$(vault kv get -field=credentials ${VAULT_SIDEKIQ_ENDPOINT})"
  variables:
    DOCKER_ADDITIONAL_CACHE_TARGETS: "base development"
    DOCKER_BUILD_ARGS: "--build-arg RAILS_ENV=production"
    DOCKER_BUILD_CONTEXT: "rails_app/"
    DOCKER_BUILD_SECRETS: "--secret id=sidekiq_pro_credentials,env=SIDEKIQ_PRO_CREDENTIALS"
    DOCKER_TARGET: "production"
    DOCKERFILE_PATH: "rails_app/"
    JWT_ROLE: "sidekiq_pro_ro"
    REGISTRY_REPOSITORY: "${CI_PIPELINE_ID}"
  needs:
    - rspec_app_unit_test
    - rubocop_app_test
  tags:
    - ${RUNNER}

amend_prod_image:
  stage: amend_prod_image
  extends:
    - .amend_manifests
  needs:
    - build_and_push_prod_app_image
  variables:
    DOCKER_TARGET: "production"
    REGISTRY_REPOSITORY: "${CI_PIPELINE_ID}"
    TAG_WITH_DOCKER_TARGET: "false"
  tags:
    - build
# -- End - build and push temp prod image

# -- Start - SAST images
trivy_app_container_scanning:
  stage: sast
  extends:
    - .trivy_container_scanning
  before_script:
    - apk add --no-cache docker
    - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
  after_script:
    - docker logout ${CI_REGISTRY}
  needs:
    - amend_prod_image
  variables:
    CI_IMAGE_NAME: ${CI_REGISTRY_IMAGE}/${CI_PIPELINE_ID}:${CI_COMMIT_SHORT_SHA}
    TRIVY_VERSION: 0.68.1
  allow_failure: true
  tags:
    - build
# -- End - SAST images

# -- Start - Copy app docker images
copy_app_docker_images:
  stage: copy_docker_images
  variables:
    ORIGINAL_REGISTRY_REPOSITORY: "${CI_PIPELINE_ID}"
  needs:
    - trivy_app_container_scanning
  extends:
    - .copy_docker_images
  tags:
    - build
# -- End - Copy app docker images

# -- Start - remove tmp registry repository
remove_tmp_registry:
  stage: remove_tmp_registry
  extends:
    - .remove_tmp_registry
  rules:
    - when: always
  needs:
    - copy_app_docker_images
  allow_failure: true
  tags:
    - build
# -- End - remove tmp registry repository

# Deploy staging
deploy_staging:
  needs:
    - copy_app_docker_images
  only:
    - main

#Deploy production
deploy_production:
  needs:
    - copy_app_docker_images
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+(\.[a-zA-Z0-9]+)?$/
#-- End - deployment