Skip to content

Commit 6d464de

Browse files
joecoralljoecorall
committed
block metadata server from docker ns and non-root
and manage keys for internal services and the compose app
1 parent b2e8966 commit 6d464de

13 files changed

Lines changed: 270 additions & 25 deletions

File tree

‎README.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,13 @@ Deploy a docker compose project to a Google Cloud Compute Instance.
3535
| [google_project_iam_member.gce-suspend](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
3636
| [google_project_iam_member.log](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
3737
| [google_project_iam_member.stackdriver](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
38+
| [google_service_account.app](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
3839
| [google_service_account.cloud-compose](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
40+
| [google_service_account.internal-services](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
3941
| [google_service_account.ppb](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
42+
| [google_service_account_iam_member.app-keys](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource |
4043
| [google_service_account_iam_member.gsa-user](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource |
44+
| [google_service_account_iam_member.internal-services-keys](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource |
4145
| [google_service_account_iam_member.token-creator](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource |
4246
| [cloudinit_config.ci](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source |
4347
| [google_project_iam_custom_role.gce-start](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project_iam_custom_role) | data source |
@@ -68,9 +72,10 @@ Deploy a docker compose project to a Google Cloud Compute Instance.
6872

6973
| Name | Description |
7074
|------|-------------|
71-
| <a name="output_gsaEmail"></a> [gsaEmail](#output\_gsaEmail) | n/a |
72-
| <a name="output_gsaId"></a> [gsaId](#output\_gsaId) | n/a |
75+
| <a name="output_appGsa"></a> [appGsa](#output\_appGsa) | The Google Service Account the app can leverage to auth to other Google services |
76+
| <a name="output_instanceGsa"></a> [instanceGsa](#output\_instanceGsa) | The Google Service Account the compute instance runs as |
7377
| <a name="output_instance_id"></a> [instance\_id](#output\_instance\_id) | n/a |
7478
| <a name="output_name"></a> [name](#output\_name) | n/a |
79+
| <a name="output_serviceGsa"></a> [serviceGsa](#output\_serviceGsa) | The Google Service Account internal services that manage the VM runs as |
7580
| <a name="output_zone"></a> [zone](#output\_zone) | n/a |
7681
<!-- END_TF_DOCS -->

‎main.tf‎

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -63,13 +63,6 @@ resource "google_service_account_iam_member" "token-creator" {
6363
member = "serviceAccount:${google_service_account.cloud-compose.email}"
6464
}
6565

66-
# push metrics to GCP
67-
resource "google_project_iam_member" "stackdriver" {
68-
project = var.project_id
69-
role = "roles/monitoring.metricWriter"
70-
member = "serviceAccount:${google_service_account.cloud-compose.email}"
71-
}
72-
7366
# push logs to GCP
7467
resource "google_project_iam_member" "log" {
7568
project = var.project_id
@@ -171,10 +164,50 @@ data "google_project_iam_custom_role" "gce-suspend" {
171164
project = var.project_id
172165
role_id = "suspendVM"
173166
}
167+
168+
169+
# =============================================================================
170+
# LIBOPS ADMIN SERVICES IDENTITY
171+
# =============================================================================
172+
173+
resource "google_service_account" "internal-services" {
174+
account_id = format("internal-services-%s", var.name)
175+
project = var.project_id
176+
}
177+
178+
resource "google_service_account_iam_member" "internal-services-keys" {
179+
service_account_id = google_service_account.internal-services.id
180+
role = "roles/iam.serviceAccountKeyAdmin"
181+
member = "serviceAccount:${google_service_account.cloud-compose.email}"
182+
}
183+
184+
# push metrics to GCP
185+
resource "google_project_iam_member" "stackdriver" {
186+
project = var.project_id
187+
role = "roles/monitoring.metricWriter"
188+
member = "serviceAccount:${google_service_account.internal-services.email}"
189+
}
190+
191+
# suspend the GCP instance
174192
resource "google_project_iam_member" "gce-suspend" {
175193
project = var.project_id
176194
role = data.google_project_iam_custom_role.gce-suspend.name
177-
member = "serviceAccount:${google_service_account.cloud-compose.email}"
195+
member = "serviceAccount:${google_service_account.internal-services.email}"
196+
}
197+
198+
# =============================================================================
199+
# DOCKER COMPOSE APP IDENTITY
200+
# =============================================================================
201+
202+
resource "google_service_account" "app" {
203+
account_id = var.name
204+
project = var.project_id
205+
}
206+
207+
resource "google_service_account_iam_member" "app-keys" {
208+
service_account_id = google_service_account.app.id
209+
role = "roles/iam.serviceAccountKeyAdmin"
210+
member = "serviceAccount:${google_service_account.cloud-compose.email}"
178211
}
179212

180213
# =============================================================================

‎outputs.tf‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,30 @@ output "instance_id" {
1010
value = google_compute_instance.cloud-compose.instance_id
1111
}
1212

13-
output "gsaEmail" {
14-
value = google_service_account.cloud-compose.email
13+
14+
output "instanceGsa" {
15+
value = {
16+
email : google_service_account.cloud-compose.email,
17+
id : google_service_account.cloud-compose.id,
18+
name : google_service_account.cloud-compose.name,
19+
}
20+
description = "The Google Service Account the compute instance runs as"
21+
}
22+
23+
output "serviceGsa" {
24+
value = {
25+
email : google_service_account.internal-services.email,
26+
id : google_service_account.internal-services.id,
27+
name : google_service_account.internal-services.name,
28+
}
29+
description = "The Google Service Account internal services that manage the VM runs as"
1530
}
1631

17-
output "gsaId" {
18-
value = google_service_account.cloud-compose.id
32+
output "appGsa" {
33+
value = {
34+
email : google_service_account.app.email,
35+
id : google_service_account.app.id,
36+
name : google_service_account.app.name,
37+
}
38+
description = "The Google Service Account the app can leverage to auth to other Google services"
1939
}

‎rootfs/etc/libops/docker-compose.yaml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ networks:
22
internal:
33
services:
44
libops-lightsout:
5-
image: ghcr.io/libops/lightsout:main@sha256:b61680d4c2ad03f9cfe1f47341eec128df8782a453bcfd708068a6d5c8877a02
5+
image: ghcr.io/libops/lightsout:main@sha256:4d38ac380c86514d72efbcce0af9ea8ffd7ff86e97abea3d0b3c946cf204eaae
66
ports:
77
- "8808:8808"
88
networks:
@@ -13,9 +13,11 @@ services:
1313
GCP_PROJECT: ${GCP_PROJECT}
1414
GCP_ZONE: ${GCP_ZONE}
1515
GCP_INSTANCE_NAME: ${GCP_INSTANCE_NAME}
16+
GOOGLE_APPLICATION_CREDENTIALS: /app/GOOGLE_APPLICATION_CREDENTIALS
1617
volumes:
1718
- /var/run/docker.sock:/var/run/docker.sock:ro
1819
- /mnt/disks/data:/mnt/disks/data
20+
- ./GOOGLE_APPLICATION_CREDENTIALS:/app/GOOGLE_APPLICATION_CREDENTIALS:r
1921
libops-cadvisor:
2022
image: ghcr.io/google/cadvisor:v0.53.0@sha256:c3770bd6fc6c6a9cb2b47143e6b3cc3fdd9d20a8453dffbb7e09a145e7e0c4e4
2123
command:
@@ -45,5 +47,8 @@ services:
4547
GCP_ZONE: ${GCP_ZONE}
4648
GCP_INSTANCE_NAME: ${GCP_INSTANCE_NAME}
4749
CADVISOR_HOST: "libops-cadvisor:8080"
50+
GOOGLE_APPLICATION_CREDENTIALS: /app/GOOGLE_APPLICATION_CREDENTIALS
51+
volumes:
52+
- ./GOOGLE_APPLICATION_CREDENTIALS:/app/GOOGLE_APPLICATION_CREDENTIALS:r
4853
depends_on:
4954
- libops-cadvisor

‎rootfs/etc/systemd/system/cloud-compose.service‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ StartLimitBurst=3
88
[Service]
99
Environment="HOME=/home/cloud-compose"
1010
WorkingDirectory=/mnt/disks/data/compose
11+
ExecStartPre=-/bin/bash /home/cloud-compose/rotate-keys-app.sh
1112
ExecStart=/mnt/disks/data/up
1213
ExecStop=/mnt/disks/data/down
1314
Restart=on-failure

‎rootfs/etc/systemd/system/internal-services.service‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@ StartLimitBurst=3
77

88
[Service]
99
Environment="HOME=/home/cloud-compose"
10-
WorkingDirectory=/etc/libops
10+
WorkingDirectory=/mnt/disks/data/libops
11+
ExecStartPre=/bin/bash /home/cloud-compose/rotate-keys-internal.sh
12+
ExecStartPre=/usr/bin/test -f /mnt/disks/data/libops/GOOGLE_APPLICATION_CREDENTIALS
1113
ExecStart=/usr/bin/docker compose up
1214
ExecStop=/usr/bin/docker compose down
1315
Restart=on-failure

‎rootfs/home/cloud-compose/cron.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@ set -eou pipefail
44

55
echo "Running daily cron"
66

7-
export DIR=/etc/libops
8-
bash /home/cloud-compose/rollout.sh
7+
pushd /home/cloud-compose
8+
9+
bash rotate-keys-internal.sh
10+
bash rotate-keys-app.sh
911

1012
/usr/bin/docker system prune -af
13+
14+
popd

‎rootfs/home/cloud-compose/host-init.sh‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@ cleanup() {
77
popd
88
}
99

10+
# make sure out internal services dir exists
11+
DIR=/mnt/disks/data/libops
12+
if [ ! -d "$DIR" ]; then
13+
mkdir "$DIR"
14+
fi
15+
cp /etc/libops/docker-compose.yaml /mnt/disks/data/libops
16+
1017
pushd /home/cloud-compose
1118

1219
trap cleanup EXIT
@@ -22,7 +29,11 @@ curl -sf \
2229
echo "HOME=/home/cloud-compose" > env.tmp
2330
for K in $(jq -r '.instance.attributes | keys | .[]' tmp.attr | grep -E '(DOCKER|GCP|LIBOPS)'); do
2431
V=$(jq -r .instance.attributes."$K" tmp.attr)
25-
echo "$K=\"$V\"" >> env.tmp
32+
if [[ "$V" =~ [[:space:]\$\`\\\"\'\(\)\{\}\[\]\|\&\;\<\>\*\?] ]]; then
33+
echo "$K=\"$V\"" >> env.tmp
34+
else
35+
echo "$K=$V" >> env.tmp
36+
fi
2637
done
2738

2839
{
@@ -31,12 +42,13 @@ done
3142
} >> env.tmp
3243

3344
# shellcheck disable=SC1091
34-
source env.tmp
45+
. ./env.tmp
46+
3547
echo "SITE_DOCKER_REGISTRY=us-docker.pkg.dev/${GCP_PROJECT}/private" >> env.tmp
3648

3749
if ! diff <(md5sum env.tmp) <(md5sum env); then
3850
mv env.tmp env
39-
cp env /etc/libops/.env
51+
cp env /mnt/disks/data/compose/.env
4052
if [ -d /mnt/disks/data/compose ]; then
4153
cp env /mnt/disks/data/compose/.env
4254
fi

‎rootfs/home/cloud-compose/init-app.sh‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,9 @@
33
set -eou pipefail
44

55
# shellcheck disable=SC1091
6-
source /home/cloud-compose/env
7-
export HOME
6+
. /home/cloud-compose/env
87

9-
DIR=${DIR:-/mnt/disks/data/compose}
8+
export DIR=${DIR:-/mnt/disks/data/compose}
109

1110
if [ ! -d "$DIR" ]; then
1211
git clone -b "$DOCKER_COMPOSE_BRANCH" "$DOCKER_COMPOSE_REPO" "$DIR"
@@ -20,7 +19,6 @@ git pull origin "$DOCKER_COMPOSE_BRANCH" || echo "Unable to git pull"
2019
# run the docker compose init command if it exists
2120
/mnt/disks/data/init
2221

23-
export DIR
2422
bash /home/cloud-compose/rollout.sh
2523

2624
chgrp developers /mnt/disks/data/compose

‎rootfs/home/cloud-compose/rotate-keys-app.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
3+
set -eou pipefail
4+
5+
pushd /home/cloud-compose
6+
7+
# shellcheck disable=SC1091
8+
. ./env
9+
10+
if [ -d /mnt/disks/data/compose/secrets ]; then
11+
exit 0
12+
fi
13+
14+
bash rotate-keys.sh \
15+
"$GCP_INSTANCE_NAME@$GCP_PROJECT.iam.gserviceaccount.com" \
16+
"$GCP_PROJECT" \
17+
/mnt/disks/data/compose/secrets/GOOGLE_APPLICATION_CREDENTIALS
18+
19+
popd

0 commit comments

Comments
 (0)