Skip to content

Commit e768623

Browse files
davidcheungdavidcheung
authored
many fixes (#187)
* removing empty deployment from db creation we could move this back to deployable backend if we see the need this causes an error while spinning up user-auth db because project namespace is not created * fix reference to undeclared variables * allow notification secrets to partially exist * fixup! allow notification secrets to partially exist * fix user-auth domain name ended up with extra dot * fix user-auth db-endpoint failure due to ext-svc * fixup! fix user-auth db-endpoint failure due to ext-svc * fixup! fixup! fix user-auth db-endpoint failure due to ext-svc * elasticsearch seems to only allow https
1 parent 2845982 commit e768623

8 files changed

Lines changed: 41 additions & 108 deletions

File tree

‎templates/kubernetes/terraform/environments/prod/main.tf‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ locals {
2020
region = "<% index .Params `region` %>"
2121
account_id = "<% index .Params `accountId` %>"
2222
domain_name = "<% index .Params `productionHostRoot` %>"
23+
environment = "prod"
2324
file_uploads = <% if eq (index .Params `fileUploads`) "yes" %>true<% else %>false<% end %>
25+
random_seed = "<% index .Params `randomSeed` %>"
2426
}
2527

2628
provider "aws" {
@@ -37,7 +39,7 @@ module "kubernetes" {
3739
project = local.project
3840
region = local.region
3941
allowed_account_ids = [local.account_id]
40-
random_seed = "<% index .Params `randomSeed` %>"
42+
random_seed = local.random_seed
4143
cf_signing_enabled = local.file_uploads
4244

4345
# Authenticate with the EKS cluster via the cluster id
@@ -79,9 +81,9 @@ module "kubernetes" {
7981
{
8082
name = local.project
8183
auth_namespace = "user-auth"
82-
frontend_service_domain = "<% index .Params `productionFrontendSubdomain` %>.${local.domain_name}"
83-
backend_service_domain = "<% index .Params `productionBackendSubdomain` %>.${local.domain_name}"
84-
whitelisted_return_urls = ["https://<% index .Params `productionFrontendSubdomain` %>.${local.domain_name}"]
84+
frontend_service_domain = "<% index .Params `productionFrontendSubdomain` %>${local.domain_name}"
85+
backend_service_domain = "<% index .Params `productionBackendSubdomain` %>${local.domain_name}"
86+
whitelisted_return_urls = ["https://<% index .Params `productionFrontendSubdomain` %>${local.domain_name}"]
8587
jwks_secret_name = "${local.project}-${local.environment}-oathkeeper-jwks-${local.random_seed}"
8688
# This domain or address must be verified by the mail provider (Sendgrid, SES, etc.)
8789
user_auth_mail_from_address = "noreply@${local.domain_name}"
@@ -93,6 +95,8 @@ module "kubernetes" {
9395
## If you need to add another user-auth instance you will have to create another set of these resources
9496
]<% end %>
9597
notification_service_enabled = <%if eq (index .Params `notificationServiceEnabled`) "yes" %>true<% else %>false<% end %>
98+
notification_service_sendgrid_enabled = <%if ne (index .Params `sendgridApiKey`) "" %>true<% else %>false<% end %>
99+
notification_service_slack_enabled = <%if ne (index .Params `notificationServiceSlackApiKey`) "" %>true<% else %>false<% end %>
96100
notification_service_highly_available = true
97101

98102
cache_store = "<% index .Params `cacheStore` %>"

‎templates/kubernetes/terraform/environments/stage/main.tf‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ locals {
2020
region = "<% index .Params `region` %>"
2121
account_id = "<% index .Params `accountId` %>"
2222
domain_name = "<% index .Params `stagingHostRoot` %>"
23+
environment = "stage"
2324
file_uploads = <% if eq (index .Params `fileUploads`) "yes" %>true<% else %>false<% end %>
25+
random_seed = "<% index .Params `randomSeed` %>"
2426
}
2527

2628
provider "aws" {
@@ -36,7 +38,7 @@ module "kubernetes" {
3638
project = local.project
3739
region = local.region
3840
allowed_account_ids = [local.account_id]
39-
random_seed = "<% index .Params `randomSeed` %>"
41+
random_seed = local.random_seed
4042
cf_signing_enabled = local.file_uploads
4143

4244
# Authenticate with the EKS cluster via the cluster id
@@ -78,9 +80,9 @@ module "kubernetes" {
7880
{
7981
name = local.project
8082
auth_namespace = "user-auth"
81-
frontend_service_domain = "<% index .Params `stagingFrontendSubdomain` %>.${local.domain_name}"
82-
backend_service_domain = "<% index .Params `stagingBackendSubdomain` %>.${local.domain_name}"
83-
whitelisted_return_urls = ["https://<% index .Params `stagingFrontendSubdomain` %>.${local.domain_name}"]
83+
frontend_service_domain = "<% index .Params `stagingFrontendSubdomain` %>${local.domain_name}"
84+
backend_service_domain = "<% index .Params `stagingBackendSubdomain` %>${local.domain_name}"
85+
whitelisted_return_urls = ["https://<% index .Params `stagingFrontendSubdomain` %>${local.domain_name}"]
8486
jwks_secret_name = "${local.project}-${local.environment}-oathkeeper-jwks-${local.random_seed}"
8587
# This domain or address must be verified by the mail provider (Sendgrid, SES, etc.)
8688
user_auth_mail_from_address = "noreply@${local.domain_name}"
@@ -92,6 +94,8 @@ module "kubernetes" {
9294
## If you need to add another user-auth instance you will have to create another set of these resources
9395
]<% end %>
9496
notification_service_enabled = <%if eq (index .Params `notificationServiceEnabled`) "yes" %>true<% else %>false<% end %>
97+
notification_service_sendgrid_enabled = <%if ne (index .Params `sendgridApiKey`) "" %>true<% else %>false<% end %>
98+
notification_service_slack_enabled = <%if ne (index .Params `notificationServiceSlackApiKey`) "" %>true<% else %>false<% end %>
9599
notification_service_highly_available = false
96100

97101
cache_store = "<% index .Params `cacheStore` %>"

‎templates/kubernetes/terraform/modules/kubernetes/notification_service.tf‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,20 @@ locals {
55
}
66

77
data "aws_secretsmanager_secret" "sendgrid_api_key" {
8-
count = var.notification_service_enabled ? 1 : 0
8+
count = var.notification_service_enabled && var.notification_service_sendgrid_enabled ? 1 : 0
99
name = local.sendgrid_api_key_secret_name
1010
}
1111
data "aws_secretsmanager_secret_version" "sendgrid_api_key" {
12-
count = var.notification_service_enabled ? 1 : 0
12+
count = var.notification_service_enabled && var.notification_service_sendgrid_enabled ? 1 : 0
1313
secret_id = data.aws_secretsmanager_secret.sendgrid_api_key[0].id
1414
}
1515

1616
data "aws_secretsmanager_secret" "slack_api_key" {
17-
count = var.notification_service_enabled ? 1 : 0
17+
count = var.notification_service_enabled && var.notification_service_slack_enabled ? 1 : 0
1818
name = local.slack_api_key_secret_name
1919
}
2020
data "aws_secretsmanager_secret_version" "slack_api_key" {
21-
count = var.notification_service_enabled ? 1 : 0
21+
count = var.notification_service_enabled && var.notification_service_slack_enabled ? 1 : 0
2222
secret_id = data.aws_secretsmanager_secret.slack_api_key[0].id
2323
}
2424

@@ -65,11 +65,11 @@ resource "helm_release" "notification_service" {
6565
# These will become secrets provided as env vars
6666
set_sensitive {
6767
name = "application.sendgridApiKey"
68-
value = data.aws_secretsmanager_secret_version.sendgrid_api_key[0].secret_string
68+
value = var.notification_service_enabled && var.notification_service_sendgrid_enabled ? data.aws_secretsmanager_secret_version.sendgrid_api_key[0].secret_string : ""
6969
}
7070

7171
set_sensitive {
7272
name = "application.slackApiKey"
73-
value = data.aws_secretsmanager_secret_version.slack_api_key[0].secret_string
73+
value = var.notification_service_enabled && var.notification_service_slack_enabled ? data.aws_secretsmanager_secret_version.slack_api_key[0].secret_string : ""
7474
}
7575
}

‎templates/kubernetes/terraform/modules/kubernetes/variables.tf‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,18 @@ variable "notification_service_enabled" {
107107
default = false
108108
}
109109

110+
variable "notification_service_slack_enabled" {
111+
description = "If enabled, will inject slack_api_key env-vars from secret manager to notification service"
112+
type = bool
113+
default = false
114+
}
115+
116+
variable "notification_service_sendgrid_enabled" {
117+
description = "If enabled, will inject sendgrid_api_key env-vars from secret manager to notification service"
118+
type = bool
119+
default = false
120+
}
121+
110122
variable "notification_service_highly_available" {
111123
description = "If enabled, will make sure a minimum of 2 pods are running and use a horizontal pod autoscaler to make scale the number of pods based on CPU. Recommended for Production."
112124
type = bool

‎templates/scripts/create-db-user.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ echo "Some environment variables (REGION, SEED, PROJECT_NAME, ENVIRONMENT, NAMES
3434
DOCKER_IMAGE_TAG=commitdev/zero-k8s-utilities:0.0.3
3535

3636
# database info preparation
37-
DB_ENDPOINT=database.${PROJECT_NAME}
37+
# this script will run both before and after make-apply-k8s, therefore the database service is not always available
38+
DB_ENDPOINT=$(aws rds describe-db-instances --region=$REGION --db-instance-identifier "${PROJECT_NAME}-${ENVIRONMENT}" --query "DBInstances[0].Endpoint.Address" | jq -r '.')
3839
DB_NAME_LIST=$(echo ${DATABASE_NAME} | tr -dc 'A-Za-z0-9 ') # used by job
3940
DB_NAME=$(echo ${DB_NAME_LIST} | cut -d" " -f1) # used by db-pod
4041
DB_TYPE=${DATABASE_TYPE}

‎templates/scripts/db-ops/job-create-db-mysql.yml.tpl‎

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -60,47 +60,3 @@ spec:
6060
secretName: db-create-users
6161
restartPolicy: Never
6262
backoffLimit: 1
63-
64-
---
65-
apiVersion: apps/v1
66-
kind: Deployment
67-
metadata:
68-
name: db-pod
69-
namespace: $PROJECT_NAME
70-
spec:
71-
# this is purposely left at 0 so it can be enabled for troubleshooting purposes
72-
replicas: 0
73-
selector:
74-
matchLabels:
75-
app: db-pod
76-
template:
77-
metadata:
78-
labels:
79-
app: db-pod
80-
spec:
81-
automountServiceAccountToken: false
82-
containers:
83-
- command:
84-
- sh
85-
args:
86-
- "-c"
87-
# long running task so the pod doesn't exit with 0
88-
- tail -f /dev/null
89-
image: $DOCKER_IMAGE_TAG
90-
imagePullPolicy: Always
91-
name: db-pod
92-
env:
93-
- name: DB_ENDPOINT
94-
value: $DB_ENDPOINT
95-
- name: DB_NAME
96-
value: $DB_NAME
97-
- name: DB_USERNAME
98-
valueFrom:
99-
secretKeyRef:
100-
name: $SECRET_NAME
101-
key: DATABASE_USERNAME
102-
- name: DB_PASSWORD
103-
valueFrom:
104-
secretKeyRef:
105-
name: $SECRET_NAME
106-
key: DATABASE_PASSWORD

‎templates/scripts/db-ops/job-create-db-postgres.yml.tpl‎

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -62,47 +62,3 @@ spec:
6262
secretName: db-create-users
6363
restartPolicy: Never
6464
backoffLimit: 1
65-
66-
---
67-
apiVersion: apps/v1
68-
kind: Deployment
69-
metadata:
70-
name: db-pod
71-
namespace: $PROJECT_NAME
72-
spec:
73-
# this is purposely left at 0 so it can be enabled for troubleshooting purposes
74-
replicas: 0
75-
selector:
76-
matchLabels:
77-
app: db-pod
78-
template:
79-
metadata:
80-
labels:
81-
app: db-pod
82-
spec:
83-
automountServiceAccountToken: false
84-
containers:
85-
- command:
86-
- sh
87-
args:
88-
- "-c"
89-
# long running task so the pod doesn't exit with 0
90-
- tail -f /dev/null
91-
image: $DOCKER_IMAGE_TAG
92-
imagePullPolicy: Always
93-
name: db-pod
94-
env:
95-
- name: DB_ENDPOINT
96-
value: $DB_ENDPOINT
97-
- name: DB_NAME
98-
value: $DB_NAME
99-
- name: DB_USERNAME
100-
valueFrom:
101-
secretKeyRef:
102-
name: $SECRET_NAME
103-
key: DATABASE_USERNAME
104-
- name: DB_PASSWORD
105-
valueFrom:
106-
secretKeyRef:
107-
name: $SECRET_NAME
108-
key: DATABASE_PASSWORD

‎templates/scripts/files/elasticsearch-setup.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,19 @@ set -e
66
echo "Executing Elasticsearch queries to configure the ${ENVIRONMENT} environment"
77

88
# Create the index pattern
9-
curl -X POST "http://${ES_ENDPOINT}/_plugin/kibana/api/saved_objects/index-pattern" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' \
9+
curl -X POST "https://${ES_ENDPOINT}/_plugin/kibana/api/saved_objects/index-pattern" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' \
1010
-d'{"attributes":{"title":"fluentd-*","timeFieldName":"@timestamp","fields":"[]"}}'
1111

1212
if [ "${ENVIRONMENT}" = "stage" ]; then
1313
# Create the policy
14-
curl -X PUT "http://${ES_ENDPOINT}/_opendistro/_ism/policies/hot_cold_delete_workflow?pretty" -H 'Content-Type: application/json' -d@/elasticsearch-index-policy-stage.json
14+
curl -X PUT "https://${ES_ENDPOINT}/_opendistro/_ism/policies/hot_cold_delete_workflow?pretty" -H 'Content-Type: application/json' -d@/elasticsearch-index-policy-stage.json
1515
# Make indices use the policy
16-
curl -X PUT "http://${ES_ENDPOINT}/_template/fluentd_template?pretty" -H 'Content-Type: application/json' \
16+
curl -X PUT "https://${ES_ENDPOINT}/_template/fluentd_template?pretty" -H 'Content-Type: application/json' \
1717
-d'{ "index_patterns": ["fluentd-*"], "settings": { "number_of_shards": 2, "number_of_replicas": 1, "opendistro.index_state_management.policy_id": "hot_cold_delete_workflow" }}'
1818
else
1919
# Create the policy
20-
curl -X PUT "http://${ES_ENDPOINT}/_opendistro/_ism/policies/hot_warm_cold_delete_workflow?pretty" -H 'Content-Type: application/json' -d@/elasticsearch-index-policy-prod.json
20+
curl -X PUT "https://${ES_ENDPOINT}/_opendistro/_ism/policies/hot_warm_cold_delete_workflow?pretty" -H 'Content-Type: application/json' -d@/elasticsearch-index-policy-prod.json
2121
# Make indices use the policy
22-
curl -X PUT "http://${ES_ENDPOINT}/_template/fluentd_template?pretty" -H 'Content-Type: application/json' \
22+
curl -X PUT "https://${ES_ENDPOINT}/_template/fluentd_template?pretty" -H 'Content-Type: application/json' \
2323
-d'{ "index_patterns": ["fluentd-*"], "settings": { "number_of_shards": 2, "number_of_replicas": 2, "opendistro.index_state_management.policy_id": "hot_warm_cold_delete_workflow" }}'
2424
fi

0 commit comments

Comments
 (0)