Overlay prod snapshots on non-prod sites (#2)

Author: joecorall

README.md

@@ -16,12 +16,13 @@ Deploy a docker compose project to a Google Cloud Compute Instance.
 |------|---------|
 | <a name="provider_cloudinit"></a> [cloudinit](#provider\_cloudinit) | 2.3.7 |
 | <a name="provider_google"></a> [google](#provider\_google) | 7.12.0 |
+| <a name="provider_time"></a> [time](#provider\_time) | n/a |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_ppb"></a> [ppb](#module\_ppb) | git::https://github.com/libops/terraform-cloudrun-v2 | 0.4.0 |
+| <a name="module_ppb"></a> [ppb](#module\_ppb) | git::https://github.com/libops/terraform-cloudrun-v2 | 0.5.0 |
 
 ## Resources
 
@@ -30,7 +31,13 @@ Deploy a docker compose project to a Google Cloud Compute Instance.
 | [google_artifact_registry_repository_iam_member.private-policy-cloud-compose](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/artifact_registry_repository_iam_member) | resource |
 | [google_compute_disk.boot](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource |
 | [google_compute_disk.data](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource |
+| [google_compute_disk.docker-volumes](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource |
+| [google_compute_disk.overlay_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource |
+| [google_compute_disk_resource_policy_attachment.daily_snapshot](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk_resource_policy_attachment) | resource |
+| [google_compute_disk_resource_policy_attachment.weekly_snapshot](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk_resource_policy_attachment) | resource |
 | [google_compute_instance.cloud-compose](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance) | resource |
+| [google_compute_resource_policy.daily_snapshot](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy) | resource |
+| [google_compute_resource_policy.weekly_snapshot](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy) | resource |
 | [google_project_iam_member.gce-start](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.gce-suspend](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.log](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
@@ -43,7 +50,9 @@ Deploy a docker compose project to a Google Cloud Compute Instance.
 | [google_service_account_iam_member.gsa-user](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource |
 | [google_service_account_iam_member.internal-services-keys](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource |
 | [google_service_account_iam_member.token-creator](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account_iam_member) | resource |
+| [time_static.snapshot_time_static](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/static) | resource |
 | [cloudinit_config.ci](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source |
+| [google_compute_snapshot.latest_prod](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_snapshot) | data source |
 | [google_project_iam_custom_role.gce-start](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project_iam_custom_role) | data source |
 | [google_project_iam_custom_role.gce-suspend](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project_iam_custom_role) | data source |
 
@@ -58,24 +67,24 @@ Deploy a docker compose project to a Google Cloud Compute Instance.
 | <a name="input_allowed_ips"></a> [allowed\_ips](#input\_allowed\_ips) | CIDR IP Addresses allowed to turn on this site's GCP instance | `list(string)` | `[]` | no |
 | <a name="input_allowed_ssh_ipv4"></a> [allowed\_ssh\_ipv4](#input\_allowed\_ssh\_ipv4) | CIDR IPv4 Addresses allowed to to SSH into this site's GCP instance | `list(string)` | `[]` | no |
 | <a name="input_allowed_ssh_ipv6"></a> [allowed\_ssh\_ipv6](#input\_allowed\_ssh\_ipv6) | CIDR IPv6 Addresses allowed to SSH into this site's GCP instance | `list(string)` | `[]` | no |
-| <a name="input_disk_size_gb"></a> [disk\_size\_gb](#input\_disk\_size\_gb) | Data disk size in GB | `number` | `25` | no |
+| <a name="input_disk_size_gb"></a> [disk\_size\_gb](#input\_disk\_size\_gb) | Data disk size in GB | `number` | `50` | no |
 | <a name="input_docker_compose_branch"></a> [docker\_compose\_branch](#input\_docker\_compose\_branch) | git branch to checkout for var.docker\_compose\_repo | `string` | `"main"` | no |
 | <a name="input_docker_compose_down"></a> [docker\_compose\_down](#input\_docker\_compose\_down) | Command to stop the docker compose project | `string` | `"docker compose down"` | no |
 | <a name="input_docker_compose_init"></a> [docker\_compose\_init](#input\_docker\_compose\_init) | After cloning the docker compose git repo, any initialization that needs to happen before the docker compose project can start | `string` | `""` | no |
 | <a name="input_docker_compose_up"></a> [docker\_compose\_up](#input\_docker\_compose\_up) | Command to start the docker compose project | `string` | `"docker compose up --remove-orphans"` | no |
-| <a name="input_machine_type"></a> [machine\_type](#input\_machine\_type) | VM machine type | `string` | `"e2-medium"` | no |
+| <a name="input_machine_type"></a> [machine\_type](#input\_machine\_type) | VM machine type (General-purpose series that support Hyperdisk Balanced | `string` | `"n4-standard-2"` | no |
 | <a name="input_os"></a> [os](#input\_os) | The host OS to install on the GCP instance | `string` | `"cos-125-19216-104-25"` | no |
-| <a name="input_region"></a> [region](#input\_region) | GCP region for resources | `string` | `"us-central1"` | no |
-| <a name="input_zone"></a> [zone](#input\_zone) | GCP zone for resources | `string` | `"us-central1-f"` | no |
+| <a name="input_overlay_source_instance"></a> [overlay\_source\_instance](#input\_overlay\_source\_instance) | Name of production instance to get latest snapshot from (e.g., 'ojs-production'). Terraform will automatically use the most recent snapshot from this instance's data disk. Leave empty for production environments. | `string` | `""` | no |
+| <a name="input_region"></a> [region](#input\_region) | GCP region for resources | `string` | `"us-east5"` | no |
+| <a name="input_run_snapshots"></a> [run\_snapshots](#input\_run\_snapshots) | Enable daily snapshots of the data disk (recommended for production). Last seven days of snapshots are available. Also weekly snapshots for past year. | `bool` | `false` | no |
+| <a name="input_volume_names"></a> [volume\_names](#input\_volume\_names) | List of docker volumes to overlay from production snapshot (e.g., ['compose\_ojs-public']). Production data is mounted read-only as lower layer, staging writes go to upper layer. | `list(string)` | `[]` | no |
+| <a name="input_zone"></a> [zone](#input\_zone) | GCP zone for resources | `string` | `"us-east5-b"` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_appGsa"></a> [appGsa](#output\_appGsa) | The Google Service Account the app can leverage to auth to other Google services |
-| <a name="output_instanceGsa"></a> [instanceGsa](#output\_instanceGsa) | The Google Service Account the compute instance runs as |
-| <a name="output_instance_id"></a> [instance\_id](#output\_instance\_id) | n/a |
-| <a name="output_name"></a> [name](#output\_name) | n/a |
+| <a name="output_instance"></a> [instance](#output\_instance) | The Google Compute instance ID, name, zone, data disk, GSA for the instance. |
 | <a name="output_serviceGsa"></a> [serviceGsa](#output\_serviceGsa) | The Google Service Account internal services that manage the VM runs as |
-| <a name="output_zone"></a> [zone](#output\_zone) | n/a |
 <!-- END_TF_DOCS -->

examples/ojs/README.md

@@ -0,0 +1,20 @@
+# OJS
+
+Deploy OJS to Google Cloud
+
+## Usage
+
+Create the production VM
+
+```
+terraform init
+terraform apply -target=module.production
+```
+
+The staging VM then relies on a snapshot of the public files docker volume. This is to allow staging to mirror production.
+
+So need wait until snapshot schedule executes (~1h) OR get a snapshot of the production docker volume disk immediately. then the rest of the terraform runs can just execute as normal
+
+```
+terraform apply
+```

examples/ojs/main.tf

@@ -0,0 +1,39 @@
+resource "random_shuffle" "zone" {
+  input        = var.region == "us-central1" ? ["a", "b", "c", "f"] : ["a", "b", "c"]
+  result_count = 1
+}
+
+module "production" {
+  source = "git::https://github.com/libops/cloud-compose?ref=0.0.1"
+
+  name                = "ojs-production"
+  project_id          = var.project_id
+  project_number      = var.project_number
+  docker_compose_repo = var.docker_compose_repo
+  docker_compose_init = var.docker_compose_init
+  region              = var.region
+  zone                = format("%s-%s", var.region, random_shuffle.zone.result[0])
+  run_snapshots       = true
+  allowed_ips         = var.allowed_ips
+}
+
+module "staging" {
+  source = "git::https://github.com/libops/cloud-compose?ref=0.0.1"
+
+  name                = "ojs-staging"
+  project_id          = var.project_id
+  project_number      = var.project_number
+  docker_compose_repo = var.docker_compose_repo
+  docker_compose_init = var.docker_compose_init
+  region              = var.region
+  zone                = format("%s-%s", var.region, random_shuffle.zone.result[0])
+  disk_size_gb        = 20
+  allowed_ips         = var.allowed_ips
+
+  # make production public files available in staging
+  overlay_source_instance = "ojs-production"
+  volume_names = [
+    "compose_ojs-public",
+    "compose_ojs-files"
+  ]
+}

examples/ojs/terraform.tfvars

@@ -0,0 +1,2 @@
+docker_compose_repo   = "https://github.com/libops/ojs"
+docker_compose_init   = "docker compose up init"

examples/ojs/variables.tf

@@ -0,0 +1,32 @@
+variable "project_id" {
+  description = "The GCP project ID"
+  type        = string
+}
+
+variable "project_number" {
+  type        = string
+  description = "The GCP project number"
+}
+
+variable "region" {
+  description = "GCP region for resources"
+  type        = string
+  default     = "us-central1"
+}
+
+variable "docker_compose_repo" {
+  type        = string
+  description = "git repo to checkout that contains a docker compose project"
+}
+
+variable "docker_compose_init" {
+  type        = string
+  default     = ""
+  description = "After cloning the docker compose git repo, any initialization that needs to happen before the docker compose project can start"
+}
+
+variable "allowed_ips" {
+  type        = list(string)
+  default     = []
+  description = "CIDR IP Addresses allowed to turn on this site's GCP instance"
+}

main.tf

@@ -14,6 +14,8 @@ provider "google" {
   region  = var.region
 }
 
+resource "time_static" "snapshot_time_static" {}
+
 locals {
   rootFs = "${path.module}/rootfs"
   write_files_content = join("\n", [
@@ -55,11 +57,20 @@ EOT
         DOCKER_COMPOSE_REPO="${var.docker_compose_repo}"
         DOCKER_COMPOSE_BRANCH="${var.docker_compose_branch}"
 EOT
+  use_overlay      = length(var.volume_names) > 0
+  prod_disk_name   = var.overlay_source_instance != "" ? format("%s-data-disk", var.overlay_source_instance) : ""
+  prod_disk_url    = var.overlay_source_instance != "" ? format("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/disks/%s-docker-volumes", var.project_id, var.zone, var.overlay_source_instance) : ""
   cloud_init_yaml = templatefile("${path.module}/templates/cloud-init.yml", {
     WRITE_FILES_CONTENT    = local.write_files_content,
     DOCKER_COMPOSE_SCRIPTS = local.docker_compose_scripts,
     ENV_FILE_CONTENT       = local.env_file_content,
+    USE_OVERLAY            = local.use_overlay,
+    DOCKER_VOLUME_OVERLAYS = var.volume_names,
   })
+
+  # have prod snapshot begin ten minutes after the initial run
+  # so non-prod environments can have a snapshot disk to overlay
+  snapshot_start_time = formatdate("h:00", time_static.snapshot_time_static.rfc3339)
 }
 
 data "cloudinit_config" "ci" {
@@ -70,7 +81,7 @@ data "cloudinit_config" "ci" {
 }
 
 resource "google_service_account" "cloud-compose" {
-  account_id = format("cloud-compose-%s", var.name)
+  account_id = format("vm-%s", var.name)
   project    = var.project_id
 }
 
@@ -109,7 +120,7 @@ resource "google_compute_disk" "boot" {
   project                   = var.project_id
   type                      = "hyperdisk-balanced"
   zone                      = var.zone
-  size                      = 20
+  size                      = 15
   image                     = "projects/cos-cloud/global/images/${var.os}"
   physical_block_size_bytes = 4096
 }
@@ -119,11 +130,121 @@ resource "google_compute_disk" "data" {
   project                   = var.project_id
   type                      = "hyperdisk-balanced"
   zone                      = var.zone
+  size                      = 20
+  image                     = "debian-13-trixie-v20251111"
+  physical_block_size_bytes = 4096
+}
+
+resource "google_compute_disk" "docker-volumes" {
+  name                      = format("%s-docker-volumes", var.name)
+  project                   = var.project_id
+  type                      = "hyperdisk-balanced"
+  zone                      = var.zone
   size                      = var.disk_size_gb
   image                     = "debian-13-trixie-v20251111"
   physical_block_size_bytes = 4096
 }
 
+
+# Daily snapshot schedule for production docker volume disk
+resource "google_compute_resource_policy" "daily_snapshot" {
+  count   = var.run_snapshots ? 1 : 0
+  name    = format("%s-daily-snapshot", var.name)
+  project = var.project_id
+  region  = var.region
+
+  snapshot_schedule_policy {
+    schedule {
+      daily_schedule {
+        days_in_cycle = 1
+        start_time    = local.snapshot_start_time
+      }
+    }
+
+    retention_policy {
+      max_retention_days    = 7
+      on_source_disk_delete = "KEEP_AUTO_SNAPSHOTS"
+    }
+
+    snapshot_properties {
+      labels = {
+        managed_by = "terraform"
+        instance   = var.name
+      }
+      storage_locations = [var.region]
+      guest_flush       = false
+    }
+  }
+}
+resource "google_compute_disk_resource_policy_attachment" "daily_snapshot" {
+  count   = var.run_snapshots ? 1 : 0
+  name    = google_compute_resource_policy.daily_snapshot[0].name
+  disk    = google_compute_disk.docker-volumes.name
+  project = var.project_id
+  zone    = var.zone
+}
+
+resource "google_compute_resource_policy" "weekly_snapshot" {
+  count   = var.run_snapshots ? 1 : 0
+  name    = format("%s-weekly-snapshot", var.name)
+  project = var.project_id
+  region  = var.region
+
+  snapshot_schedule_policy {
+    schedule {
+      weekly_schedule {
+        day_of_weeks {
+          day        = "SUNDAY"
+          start_time = "01:00"
+        }
+      }
+    }
+
+    retention_policy {
+      max_retention_days    = 365
+      on_source_disk_delete = "KEEP_AUTO_SNAPSHOTS"
+    }
+
+    snapshot_properties {
+      storage_locations = [var.region]
+      guest_flush       = false
+    }
+  }
+}
+
+resource "google_compute_disk_resource_policy_attachment" "weekly_snapshot" {
+  count   = var.run_snapshots ? 1 : 0
+  name    = google_compute_resource_policy.weekly_snapshot[0].name
+  disk    = google_compute_disk.docker-volumes.name
+  project = var.project_id
+  zone    = var.zone
+}
+
+# Get the latest snapshot from production instance's data disk
+data "google_compute_snapshot" "latest_prod" {
+  count   = local.use_overlay ? 1 : 0
+  project = var.project_id
+
+  # Filter to snapshots of the production data disk, get most recent
+  most_recent = true
+  filter      = "sourceDisk eq ${local.prod_disk_url}"
+}
+
+# Restore production snapshot to a staging-specific disk for overlays
+resource "google_compute_disk" "overlay_disk" {
+  count                     = local.use_overlay ? 1 : 0
+  name                      = format("%s-overlay-disk", var.name)
+  project                   = var.project_id
+  type                      = "hyperdisk-balanced"
+  zone                      = var.zone
+  snapshot                  = data.google_compute_snapshot.latest_prod[0].self_link
+  physical_block_size_bytes = 4096
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
 resource "google_compute_instance" "cloud-compose" {
   name                      = var.name
   project                   = var.project_id
@@ -142,6 +263,21 @@ resource "google_compute_instance" "cloud-compose" {
     device_name = "data"
     source      = google_compute_disk.data.self_link
   }
+  attached_disk {
+    device_name = "docker-volumes"
+    source      = google_compute_disk.docker-volumes.self_link
+  }
+
+  dynamic "attached_disk" {
+    for_each = local.use_overlay ? [1] : []
+    content {
+      device_name = "prod-volumes"
+      source      = google_compute_disk.overlay_disk[0].self_link
+      # hyperdisk needs to be attached rw
+      # even though we're setting this as lowerdir read only
+      mode = "READ_WRITE"
+    }
+  }
 
   metadata = {
     google-logging-enabled       = "true"
@@ -182,6 +318,10 @@ resource "google_compute_instance" "cloud-compose" {
     enable_secure_boot          = "true"
     enable_vtpm                 = "true"
   }
+
+  lifecycle {
+    create_before_destroy = false
+  }
 }
 
 # machine needs to be able to suspend itself
@@ -196,7 +336,7 @@ data "google_project_iam_custom_role" "gce-suspend" {
 # =============================================================================
 
 resource "google_service_account" "internal-services" {
-  account_id = format("internal-services-%s", var.name)
+  account_id = format("internal-%s", var.name)
   project    = var.project_id
 }
 
@@ -286,7 +426,7 @@ resource "google_service_account" "ppb" {
 }
 
 module "ppb" {
-  source = "git::https://github.com/libops/terraform-cloudrun-v2?ref=0.4.0"
+  source = "git::https://github.com/libops/terraform-cloudrun-v2?ref=0.5.0"
 
   name              = var.name
   project           = var.project_id