Move settings from Bash scripts to YAML config files

Gemfile

@@ -2,6 +2,7 @@ source "https://rubygems.org"
 
 ruby file: ".ruby-version"
 
+gem "activesupport"
 gem "aws-sdk-s3", "~> 1.226"
 gem "capybara"
 gem "config"

Gemfile.lock

@@ -230,6 +230,7 @@ PLATFORMS
   x86_64-linux-musl
 
 DEPENDENCIES
+  activesupport
   aws-sdk-s3 (~> 1.226)
   capybara
   config

README.md

@@ -186,4 +186,4 @@ The tests expect an editor user exist with an Auth0 database connection configur
 
 The user should belong to an active group, called "End to end tests", as a group admin to allow publishing a form.
 
-The login details should be stored in AWS parameter store. See bin/load_env_vars.sh for configuring the enviroment varibles required.
+The login details should be stored in AWS parameter store. See bin/secrets.sh for configuring the enviroment varibles required.

bin/dockerfile_test.sh

@@ -30,27 +30,17 @@ if [[ -z "$IMAGE_TO_TEST" ]]; then
   docker build -t "$IMAGE_TO_TEST" .
 fi
 
-# Map legacy environment variables to settings
-export SETTINGS__FORMS_ADMIN__URL="${SETTINGS__FORMS_ADMIN__URL:-$FORMS_ADMIN_URL}"
-export SETTINGS__FORMS_ADMIN__AUTH__USERNAME="${SETTINGS__FORMS_ADMIN__AUTH__USERNAME:-$AUTH0_EMAIL_USERNAME}"
-export SETTINGS__FORMS_ADMIN__AUTH__PASSWORD="${SETTINGS__FORMS_ADMIN__AUTH__PASSWORD:-$AUTH0_USER_PASSWORD}"
-export SETTINGS__FORMS_PRODUCT_PAGE__URL="${SETTINGS__FORMS_PRODUCT_PAGE__URL:-$PRODUCT_PAGES_URL}"
-export SETTINGS__FORMS_RUNNER__URL="${SETTINGS__FORMS_RUNNER__URL:-$FORMS_RUNNER_URL}"
-
-if [ -z "$SETTINGS__FORMS_ADMIN__URL" ] || \
-   [ -z "$SETTINGS__FORMS_ADMIN__AUTH__USERNAME" ] || \
+if [ -z "$SETTINGS__FORMS_ADMIN__AUTH__USERNAME" ] || \
    [ -z "$SETTINGS__FORMS_ADMIN__AUTH__PASSWORD" ] || \
-   [ -z "$SETTINGS__FORMS_PRODUCT_PAGE__URL" ] || \
    [ -z "$SETTINGS__GOVUK_NOTIFY__API_KEY" ]; then
-  echo "Loading env vars from parameter store"
-  source $SCRIPT_DIR/load_env_vars.sh
-  set_e2e_env_vars 'dev'
-  set_smoke_test_env_vars 'dev'
+  echo "Loading secrets from parameter store"
+  source $SCRIPT_DIR/secrets.sh
+  export_secrets 'dev'
 fi
 
 echo 'Running the tests against dev environment'
 
-env | grep -e AWS_ -e SETTINGS__ > ./env.list
+env | grep -e AWS_ -e SETTINGS__ | cut -d = -f 1 > ./env.list
 
 docker run --env-file ./env.list --rm \
   -e SMOKE_TEST_FORM_URL \

bin/end_to_end.sh

@@ -21,8 +21,8 @@ gds aws forms-dev-readonly -- $0 dev
   exit 0
 fi
 
-source ./load_env_vars.sh
-set_e2e_env_vars "$environment"
+source ./secrets.sh
+export_secrets "$environment"
 
 cd ..
 bundle install

bin/secrets.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+function get_param() {
+  path="$1"
+
+  aws ssm get-parameter \
+    --with-decrypt \
+    --name "$path" \
+    --output text \
+    --query 'Parameter.Value'
+}
+
+function export_secrets() {
+  local environment="$1"
+  if [ -z "$environment" ]; then
+    echo "usage 'set_env_vars dev|staging|production'"
+    exit 1
+  fi
+
+  if ! aws sts get-caller-identity > /dev/null 2>&1; then
+    echo "Could not access AWS resources; make sure you've assumed a role in the AWS account you're targeting." >&2
+    exit 1
+  fi
+
+  export SETTINGS__FORMS_ENV="$environment"
+
+  export SETTINGS__FORMS_ADMIN__AUTH__USERNAME="$(get_param /${environment}/automated-tests/e2e/auth0/email-username)"
+  export SETTINGS__FORMS_ADMIN__AUTH__PASSWORD="$(get_param /${environment}/automated-tests/e2e/auth0/auth0-user-password)"
+  export SETTINGS__GOVUK_NOTIFY__API_KEY="$(get_param /${environment}/automated-tests/e2e/notify/api-key)"
+  export SETTINGS__GOVUK_ONE_LOGIN__USER_EMAIL="$(get_param /${environment}/automated-tests/e2e/one-login/user-email)"
+  export SETTINGS__GOVUK_ONE_LOGIN__USER_PASSWORD="$(get_param /${environment}/automated-tests/e2e/one-login/user-password)"
+  export SETTINGS__GOVUK_ONE_LOGIN__USER_OTP_SECRET_KEY="$(get_param /${environment}/automated-tests/e2e/one-login/user-otp-secret-key)"
+  export SETTINGS__SUBMISSION_STATUS_API__SECRET="$(get_param /${environment}/automated-tests/e2e/runner/submission_status_api_shared_secret)"
+}

bin/smoke_tests.sh

@@ -13,9 +13,6 @@ Usage: $0 <dev|staging|production>
   exit 0
 fi
 
-source ./load_env_vars.sh
-set_smoke_test_env_vars "$1"
-
 cd ..
 bundle install
 

config/environment.rb

@@ -3,4 +3,5 @@
 require_relative "initializers/config"
 
 env = ENV["SETTINGS__FORMS_ENV"]
+env = "aws_#{env}" unless env == "local"
 Config.load_and_set_settings(Config.setting_files(__dir__, env))

config/settings/aws_dev.yml

@@ -0,0 +1,17 @@
+form_ids:
+  smoke_test: 11120
+  s3: 12457
+
+forms_admin:
+  url: https://admin.dev.forms.service.gov.uk
+
+forms_product_page:
+  url: https://www.dev.forms.service.gov.uk
+
+forms_runner_url:
+  url: https://submit.dev.forms.service.gov.uk
+
+aws:
+  s3_submission_bucket_name: govuk-forms-submissions-to-s3-test
+  s3_submission_iam_role_arn: arn:aws:iam::498160065950:role/govuk-s3-end-to-end-test-dev
+  email_receiver_s3_bucket_name: govuk-forms-dev-test-emails

config/settings/aws_production.yml

@@ -0,0 +1,17 @@
+form_ids:
+  smoke_test: 2570
+  s3: 5086
+
+forms_admin:
+  url: https://admin.forms.service.gov.uk
+
+forms_product_page:
+  url: https://www.forms.service.gov.uk
+
+forms_runner_url:
+  url: https://submit.forms.service.gov.uk
+
+aws:
+  s3_submission_bucket_name: govuk-forms-submissions-to-s3-test
+  s3_submission_iam_role_arn: arn:aws:iam::443944947292:role/govuk-s3-end-to-end-test-production
+  email_receiver_s3_bucket_name: govuk-forms-production-test-emails

config/settings/aws_staging.yml

@@ -0,0 +1,17 @@
+form_ids:
+  smoke_test: 12148
+  s3: 13657
+
+forms_admin:
+  url: https://admin.staging.forms.service.gov.uk
+
+forms_product_page:
+  url: https://www.staging.forms.service.gov.uk
+
+forms_runner_url:
+  url: https://submit.staging.forms.service.gov.uk
+
+aws:
+  s3_submission_bucket_name: govuk-forms-submissions-to-s3-test
+  s3_submission_iam_role_arn: arn:aws:iam::972536609845:role/govuk-s3-end-to-end-test-staging
+  email_receiver_s3_bucket_name: govuk-forms-staging-test-emails

spec/smoke_tests/smoke_test_runner_spec.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
+require "active_support"
+require "active_support/core_ext/object/blank"
+
 feature "Runner Smoke Test", type: :feature do
   let(:smoke_test_form_url) do
-    # TODO: Update this once we're confident no one is setting $SMOKE_TEST_FORM_URL
-    # https://trello.com/c/tIYmMZ3e/3457-remove-backwards-compatibility-for-legacy-e2e-test-env-vars
-    if Settings.form_ids.smoke_test
-      "#{Settings.forms_runner.url}/form/#{Settings.form_ids.smoke_test}"
-    else
-      ENV["SMOKE_TEST_FORM_URL"] { raise "Settings.form_ids.smoke_test is not set" }
-    end
+    raise "Settings.form_runner.url is not set" unless Settings.forms_runner.url.present?
+    raise "Settings.form_ids.smoke_test is not set" unless Settings.form_ids.smoke_test.present?
+
+    "#{Settings.forms_runner.url}/form/#{Settings.form_ids.smoke_test}"
   end
 
   before do

spec/spec_helper.rb

@@ -1,6 +1,6 @@
 require "debug"
 
-ENV["SETTINGS__FORMS_ENV"] ||= "development"
+ENV["SETTINGS__FORMS_ENV"] ||= "local"
 
 require_relative "../config/environment"
 

spec/support/aws_helpers.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "active_support"
+require "active_support/core_ext/object/blank"
 require "aws-sdk-s3"
 
 module AwsHelpers
@@ -86,9 +88,7 @@ def s3_client
   end
 
   def assume_role
-    @role_arn = Settings.aws.s3_submission_iam_role_arn
-
-    raise "Settings.aws.s3_submission_iam_role_arn is not set" if @role_arn.nil? || @role_arn.empty?
+    @role_arn = Settings.aws.s3_submission_iam_role_arn.presence or raise "Settings.aws.s3_submission_iam_role_arn is not set"
 
     role_session_name = "forms-e2e"
     Aws::AssumeRoleCredentials.new(
@@ -99,13 +99,7 @@ def assume_role
   end
 
   def get_submissions_bucket
-    # TODO: Update this once we're confident no one is setting $AWS_S3_BUCKET
-    # https://trello.com/c/tIYmMZ3e/3457-remove-backwards-compatibility-for-legacy-e2e-test-env-vars
-    bucket = Settings.aws.s3_submission_bucket_name || Settings.aws.file_upload_s3_bucket_name || ENV["AWS_S3_BUCKET"]
-
-    raise "Settings.aws.s3_submission_bucket_name is not set" if bucket.nil? || bucket.empty?
-
-    bucket
+    Settings.aws.s3_submission_bucket_name.presence or raise "Settings.aws.s3_submission_bucket_name is not set"
   end
 
   def find_submission_key(submission_reference, bucket, form_id)