helm(cronjobs): schedule mongo backup/restore jobs on non-preemptible nodes

GKE preemptible VMs can be reclaimed by Google mid-run, causing long-running
mongo jobs (sync, backup, restore) to be killed with no error — the Job then
reports Failed because the Pod disappeared.

Adding nodeAffinity (requiredDuringScheduling, cloud.google.com/gke-preemptible
DoesNotExist) to all four mongo workloads so they are never placed on preemptible
nodes:
- cronjob/sync-mongo-production-data
- cronjob/mongo-backup
- cronjob/mongo-backup-extra
- jobs/mongo-restore

The nodeAffinity is values-driven (backup.mongo.nodeAffinity, restore.nodeAffinity,
cronJobs.syncMongoProductionData.nodeAffinity) so it can be overridden per
environment (set to {} to opt out).

Made-with: Cursor

Author: yodem

helm-chart/sefaria/templates/cronjob/mongo-backup-extra.yaml

@@ -27,6 +27,10 @@ spec:
                       values:
                       - mongo
                   topologyKey: kubernetes.io/hostname
+            {{- if not (empty .Values.backup.mongo.nodeAffinity) }}
+            nodeAffinity:
+              {{- toYaml .Values.backup.mongo.nodeAffinity | nindent 14 }}
+            {{- end }}
           tolerations:
             - key: schedule-on-database-vm
               operator: "Equal"

helm-chart/sefaria/templates/cronjob/mongo-backup.yaml

@@ -27,6 +27,10 @@ spec:
                       values:
                       - mongo
                   topologyKey: kubernetes.io/hostname
+            {{- if not (empty .Values.backup.mongo.nodeAffinity) }}
+            nodeAffinity:
+              {{- toYaml .Values.backup.mongo.nodeAffinity | nindent 14 }}
+            {{- end }}
           tolerations:
             - key: schedule-on-database-vm
               operator: "Equal"

helm-chart/sefaria/templates/jobs/mongo-restore.yaml

@@ -17,6 +17,11 @@ spec:
   template:
     spec:
       serviceAccount: {{ .Values.restore.serviceAccount }}
+      {{- if not (empty .Values.restore.nodeAffinity) }}
+      affinity:
+        nodeAffinity:
+          {{- toYaml .Values.restore.nodeAffinity | nindent 10 }}
+      {{- end }}
       volumes:
         - name: shared-volume
           emptyDir:

helm-chart/sefaria/values.yaml

@@ -45,6 +45,13 @@ restore:
   bucket: sefaria-mongo-backup
   # tarball:
   serviceAccount: database-backup-read
+  # Avoid GKE preemptible nodes during restore. Set nodeAffinity: {} to allow any pool.
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: cloud.google.com/gke-preemptible
+              operator: DoesNotExist
 # config to backup environment DB
 backup:
   mongo:
@@ -56,6 +63,13 @@ backup:
     archiveBucket: sefaria-mongo-archive
     serviceAccount: database-backup-write
     version: 4.4
+    # Avoid GKE preemptible nodes mid-dump. Set nodeAffinity: {} to allow any pool.
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: cloud.google.com/gke-preemptible
+                operator: DoesNotExist
   postgres:
     enabled: false
     version: 10.3