diff --git a/.gitignore b/.gitignore index a46e872c2..4356b4a8d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ target/* # Jetbrains .idea/ +dind.override.yaml diff --git a/charts/sourcegraph-executor/dind/README.md b/charts/sourcegraph-executor/dind/README.md index b6f5f8a87..2ff5b7eef 100644 --- a/charts/sourcegraph-executor/dind/README.md +++ b/charts/sourcegraph-executor/dind/README.md @@ -52,22 +52,38 @@ In addition to the documented values, the `executor` and `private-docker-registr | Key | Type | Default | Description | |-----|------|---------|-------------| +| dind.command | list | `["dockerd"]` | Command for the dind container. | +| dind.daemonConfig | object | `{"hosts":["tcp://127.0.0.1:2375"],"insecure-registries":["private-docker-registry:5000"],"mtu":1200,"registry-mirrors":["http://private-docker-registry:5000"],"tls":false}` | Docker daemon configuration passed as daemon.json to the dind sidecar. Learn more from: https://docs.docker.com/reference/cli/dockerd/#on-linux | +| dind.gVisor.command | list | `["/bin/sh","-c","ip link del docker0 2>/dev/null || true\necho 1 > /proc/sys/net/ipv4/ip_forward\ndev=$(ip route show default | awk '{for(i=1;i<=NF;i++) if($i==\"dev\"){print $(i+1); exit}}')\naddr=$(ip addr show dev \"$dev\" | awk '/inet /{gsub(/\\/.*/, \"\", $2); print $2; exit}')\niptables-legacy -t nat -A POSTROUTING -o \"$dev\" -j SNAT --to-source \"$addr\" -p tcp || true\niptables-legacy -t nat -A POSTROUTING -o \"$dev\" -j SNAT --to-source \"$addr\" -p udp || true\nexec dockerd\n"]` | Command for the dind container when gVisor is enabled. Overrides dind.command. Prepares the network environment that gVisor does not initialise automatically before handing off to dockerd. | +| dind.gVisor.daemonConfig | object | `{"features":{"containerd-snapshotter":false},"ip6tables":false,"iptables":false,"storage-driver":"vfs"}` | Extra daemon.json settings merged into dind.daemonConfig when gVisor is enabled. These defaults configure Docker to work within gVisor's kernel constraints. | +| dind.gVisor.enabled | bool | `false` | Enable gVisor sandbox (GKE only). Requires the GKE node pool to have sandbox type set to gvisor. When enabled, sets runtimeClassName: gvisor on executor pods and replaces privileged: true with explicit capabilities — these are intercepted in-sandbox and never granted to the host kernel. See: https://gvisor.dev/docs/tutorials/docker-in-gvisor/ | +| dind.gVisor.securityContext | object | `{"capabilities":{"add":["NET_ADMIN","SYS_ADMIN","AUDIT_WRITE","CHOWN","DAC_OVERRIDE","FOWNER","FSETID","KILL","MKNOD","NET_BIND_SERVICE","NET_RAW","SETFCAP","SETGID","SETPCAP","SETUID","SYS_CHROOT","SYS_PTRACE"]}}` | securityContext for the dind container when gVisor is enabled. Replaces privileged: true — gVisor intercepts these capabilities in-sandbox and never grants them to the host kernel. | | dind.image.registry | string | `"index.docker.io"` | | | dind.image.repository | string | `"docker"` | | -| dind.image.tag | string | `"20.10.22-dind"` | | -| executor.enabled | bool | `true` | | -| executor.env.EXECUTOR_FRONTEND_PASSWORD | object | `{"value":""}` | The shared secret configured in the Sourcegraph instance site config under executors.accessToken. Required. | -| executor.env.EXECUTOR_FRONTEND_URL | object | `{"value":""}` | The external URL of the Sourcegraph instance. Required. | -| executor.env.EXECUTOR_QUEUE_NAME | object | `{"value":""}` | The name of the queue to pull jobs from to. Possible values: batches and codeintel. **Either this or EXECUTOR_QUEUE_NAMES is required.** | -| executor.env.EXECUTOR_QUEUE_NAMES | object | `{"value":""}` | The comma-separated list of names of multiple queues to pull jobs from to. Possible values: batches and codeintel. **Either this or EXECUTOR_QUEUE_NAME is required.** | +| dind.image.tag | string | `"29.5.3-dind"` | | +| executor.env | object | `{}` | Extra environment variables to set on the executor container. | +| executor.frontendExistingSecret | string | `""` | Name of existing k8s Secret to use for frontend password. The k8s Secret must contain the key EXECUTOR_FRONTEND_PASSWORD matching the site config executors.accessToken value. frontendPassword is ignored if this is set. | +| executor.frontendPassword | string | `""` | The shared secret configured in the Sourcegraph instance site config under executors.accessToken. Required if frontendExistingSecret is not configured. | +| executor.frontendUrl | string | `""` | The external URL of the Sourcegraph instance. Required. | | executor.image.defaultTag | string | `"6.0.0@sha256:0be94a7c91f8273db10fdf46718c6596340ab2acc570e7b85353806e67a27508"` | | | executor.image.name | string | `"executor"` | | +| executor.log.format | string | `"json"` | | +| executor.log.level | string | `"warn"` | Possible values are dbug, info, warn, eror, crit. | +| executor.maximumNumJobs | int | `10` | The maximum amount of jobs that can be executed concurrently. | +| executor.maximumRuntimePerJob | string | `"30m"` | The maximum wall time that can be spent on a single job. | +| executor.queueName | string | `""` | The name of the queue to pull jobs from. Possible values: batches and codeintel. Either this or queueNames is required (when not using queues). | +| executor.queueNames | list | `[]` | The names of multiple queues to pull jobs from. Possible values: batches and codeintel. Either this or queueName is required (when not using queues). | | executor.replicaCount | int | `1` | | +| executor.resources | object | `{}` | Resource requests and limits for the executor container. Each queue can override this with its own resources field. | +| executor.storage.class | string | `""` | StorageClass for the ephemeral volume. Only used when type is ephemeral. Defaults to the cluster default StorageClass when empty. | +| executor.storage.size | string | `""` | Size of the scratch volume. emptyDir: sets sizeLimit (optional, leave empty for unlimited). ephemeral: sets the PVC storage request (required). | +| executor.storage.type | string | `"emptyDir"` | Type of scratch volume for job workspaces. One of: emptyDir, ephemeral. emptyDir: plain emptyDir, no storage class required. ephemeral: per-pod PVC via the cluster default storage class; size is required. | | privateDockerRegistry.enabled | bool | `true` | Whether to deploy the private registry. Only one registry is needed when deploying multiple executors. More information: https://docs.sourcegraph.com/admin/executors/deploy_executors#using-private-registries | | privateDockerRegistry.image.registry | string | `"index.docker.io"` | | -| privateDockerRegistry.image.repository | string | `"docker/regisry"` | | -| privateDockerRegistry.image.tag | int | `2` | | +| privateDockerRegistry.image.repository | string | `"registry"` | | +| privateDockerRegistry.image.tag | int | `3` | | | privateDockerRegistry.storageSize | string | `"10Gi"` | | +| queues | list | `[]` | Optional list of queues to deploy as standalone Deployments. When set, the single executor Deployment is not rendered. Each entry supports: name (required) — used as the deployment name suffix (executor-) queueName — sets EXECUTOR_QUEUE_NAME; defaults to name if omitted queueNames — sets EXECUTOR_QUEUE_NAMES (comma-joined); takes precedence over queueName when set replicaCount, resources, env (merged with executor.env, queue overrides) | | sourcegraph.affinity | object | `{}` | Affinity, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) | | sourcegraph.image.defaultTag | string | `"{{ .Chart.AppVersion }}"` | Global docker image tag | | sourcegraph.image.pullPolicy | string | `"IfNotPresent"` | Global docker image pull policy | diff --git a/charts/sourcegraph-executor/dind/examples/gcp/values.yaml b/charts/sourcegraph-executor/dind/examples/gcp/values.yaml new file mode 100644 index 000000000..feee1a29c --- /dev/null +++ b/charts/sourcegraph-executor/dind/examples/gcp/values.yaml @@ -0,0 +1,21 @@ +storageClass: + create: false + name: standard-rwo + +executor: + replicaCount: 4 + frontendUrl: "http://" + frontendPassword: "" + queueNames: ["batches", "codeintel"] + log: + format: "json_gcp" + storage: + type: ephemeral + size: 50Gi + class: premium-rwo + +dind: + # enable gVisor to run executor instances on GKE Sandbox + # for security hardening. + gVisor: + enabled: true diff --git a/charts/sourcegraph-executor/dind/templates/_helpers.tpl b/charts/sourcegraph-executor/dind/templates/_helpers.tpl index d2797759d..886851396 100644 --- a/charts/sourcegraph-executor/dind/templates/_helpers.tpl +++ b/charts/sourcegraph-executor/dind/templates/_helpers.tpl @@ -100,10 +100,10 @@ tolerations: {{- define "executor.name" -}} -{{- if .Values.executor.env.EXECUTOR_QUEUE_NAME.value -}} -executor-{{.Values.executor.env.EXECUTOR_QUEUE_NAME.value}} -{{- else if .Values.executor.env.EXECUTOR_QUEUE_NAMES.value -}} -executor-{{replace "," "-" .Values.executor.env.EXECUTOR_QUEUE_NAMES.value }} +{{- if .Values.executor.queueName -}} +executor-{{.Values.executor.queueName}} +{{- else if .Values.executor.queueNames -}} +executor-{{join "-" .Values.executor.queueNames }} {{- end }} {{- end }} @@ -113,3 +113,269 @@ deploy: sourcegraph sourcegraph-resource-requires: no-cluster-admin app.kubernetes.io/component: executor {{- end}} + +{{- define "dind.daemonConfig" -}} +{{- $config := .Values.dind.daemonConfig | deepCopy }} +{{- if .Values.dind.gVisor.enabled }} +{{- $config = mergeOverwrite $config .Values.dind.gVisor.daemonConfig }} +{{- end }} +{{- $config | toPrettyJson }} +{{- end }} + +{{/* +Render a single executor Deployment. +Usage: include "executor.deployment" (dict "root" $ "name" "executor-foo" "queueName" "foo" "queueNames" (list) "replicaCount" 1 "resources" $res "env" $env) +*/}} +{{- define "executor.deployment" -}} +{{- $r := .root -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .name }} + annotations: + description: Runs sourcegraph executors + kubectl.kubernetes.io/default-container: executor + labels: + {{- include "sourcegraph.labels" $r | nindent 4 }} + {{- if $r.Values.executor.labels }} + {{- toYaml $r.Values.executor.labels | nindent 4 }} + {{- end }} + app: {{ .name }} + deploy: sourcegraph + sourcegraph-resource-requires: no-cluster-admin + app.kubernetes.io/component: executor +spec: + selector: + matchLabels: + {{- include "sourcegraph.selectorLabels" $r | nindent 6 }} + app: {{ .name }} + minReadySeconds: 10 + replicas: {{ .replicaCount }} + revisionHistoryLimit: 10 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: executor + checksum/docker-config: {{ include "dind.daemonConfig" $r | sha256sum }} + {{- if $r.Values.sourcegraph.podAnnotations }} + {{- toYaml $r.Values.sourcegraph.podAnnotations | nindent 8 }} + {{- end }} + {{- if $r.Values.executor.podAnnotations }} + {{- toYaml $r.Values.executor.podAnnotations | nindent 8 }} + {{- end }} + labels: + {{- include "sourcegraph.selectorLabels" $r | nindent 8 }} + {{- if $r.Values.sourcegraph.podLabels }} + {{- toYaml $r.Values.sourcegraph.podLabels | nindent 8 }} + {{- end }} + {{- if $r.Values.executor.podLabels }} + {{- toYaml $r.Values.executor.podLabels | nindent 8 }} + {{- end }} + app: {{ .name }} + deploy: sourcegraph + sourcegraph-resource-requires: no-cluster-admin + app.kubernetes.io/component: executor + spec: + containers: + - name: executor + image: {{ include "sourcegraph.image" (list $r "executor") }} + imagePullPolicy: {{ $r.Values.sourcegraph.image.pullPolicy }} + livenessProbe: + httpGet: + path: /healthz + port: http-debug + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: http-debug + scheme: HTTP + periodSeconds: 5 + timeoutSeconds: 5 + ports: + - name: http-debug + containerPort: 8080 + terminationMessagePolicy: FallbackToLogsOnError + env: + - name: EXECUTOR_FRONTEND_URL + value: {{ $r.Values.executor.frontendUrl | quote }} + - name: EXECUTOR_FRONTEND_PASSWORD + {{- if $r.Values.executor.frontendExistingSecret }} + valueFrom: + secretKeyRef: + name: {{ $r.Values.executor.frontendExistingSecret }} + key: EXECUTOR_FRONTEND_PASSWORD + {{- else }} + value: {{ $r.Values.executor.frontendPassword | quote }} + {{- end }} + {{- if .queueNames }} + - name: EXECUTOR_QUEUE_NAMES + value: {{ join "," .queueNames | quote }} + {{- else }} + - name: EXECUTOR_QUEUE_NAME + value: {{ .queueName | quote }} + {{- end }} + - name: SRC_LOG_LEVEL + value: {{ $r.Values.executor.log.level | quote }} + - name: SRC_LOG_FORMAT + value: {{ $r.Values.executor.log.format | quote }} + - name: EXECUTOR_MAXIMUM_RUNTIME_PER_JOB + value: {{ $r.Values.executor.maximumRuntimePerJob | quote }} + - name: EXECUTOR_USE_FIRECRACKER + value: "false" + - name: EXECUTOR_USE_KUBERNETES + value: "false" + - name: EXECUTOR_HEALTH_SERVER_ADDR + value: ":8080" + - name: EXECUTOR_JOB_NUM_CPUS + value: "0" + - name: EXECUTOR_JOB_MEMORY + value: "0" + - name: DOCKER_HOST + value: tcp://localhost:2375 + - name: TMPDIR + value: /scratch + {{- range $name, $item := .env }} + - name: {{ $name }} + {{- $item | toYaml | nindent 14 }} + {{- end }} + volumeMounts: + - mountPath: /scratch + name: executor-scratch + {{- with .resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: dind + image: "{{ $r.Values.dind.image.registry }}/{{ $r.Values.dind.image.repository }}:{{ $r.Values.dind.image.tag }}" + imagePullPolicy: {{ $r.Values.sourcegraph.image.pullPolicy }} + securityContext: + {{- if $r.Values.dind.gVisor.enabled }} + {{- toYaml $r.Values.dind.gVisor.securityContext | nindent 12 }} + {{- else }} + privileged: true + {{- end }} + command: + {{- if $r.Values.dind.gVisor.enabled }} + {{- toYaml $r.Values.dind.gVisor.command | nindent 12 }} + {{- else }} + {{- toYaml $r.Values.dind.command | nindent 12 }} + {{- end }} + livenessProbe: + exec: + command: + - wget + - -qO- + - http://127.0.0.1:2375/_ping + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 5 + readinessProbe: + exec: + command: + - wget + - -qO- + - http://127.0.0.1:2375/_ping + initialDelaySeconds: 20 + periodSeconds: 5 + failureThreshold: 5 + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + ports: + - containerPort: 2375 + protocol: TCP + volumeMounts: + - mountPath: /scratch + name: executor-scratch + - mountPath: /etc/docker/daemon.json + subPath: daemon.json + name: docker-config + {{- if $r.Values.dind.gVisor.enabled }} + - mountPath: /var/lib/docker + name: docker + {{- end }} + enableServiceLinks: false + {{- if $r.Values.dind.gVisor.enabled }} + runtimeClassName: gvisor + {{- end }} + {{- with $r.Values.sourcegraph.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $r.Values.sourcegraph.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with include "sourcegraph.priorityClassName" (list $r "executor") | trim }}{{ . | nindent 6 }}{{- end }} + {{- with $r.Values.sourcegraph.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $r.Values.sourcegraph.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: executor-scratch + {{- if eq $r.Values.executor.storage.type "ephemeral" }} + ephemeral: + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + {{- if $r.Values.executor.storage.class }} + storageClassName: {{ $r.Values.executor.storage.class }} + {{- end }} + resources: + requests: + storage: {{ required "executor.storage.size is required when storage.type is ephemeral" $r.Values.executor.storage.size }} + {{- else }} + emptyDir: + {{- if $r.Values.executor.storage.size }} + sizeLimit: {{ $r.Values.executor.storage.size }} + {{- end }} + {{- end }} + - name: docker-config + configMap: + defaultMode: 420 + name: docker-config + {{- if $r.Values.dind.gVisor.enabled }} + - name: docker + emptyDir: {} + {{- end }} +{{- end }} + +{{/* +Validate that an env dict does not contain managed environment variable names. +Usage: include "executor.validateEnv" (list $envDict "label") +*/}} +{{- define "executor.validateEnv" -}} +{{- $envDict := index . 0 }} +{{- $label := index . 1 }} +{{- $managed := list + "EXECUTOR_FRONTEND_URL" + "EXECUTOR_FRONTEND_PASSWORD" + "EXECUTOR_QUEUE_NAME" + "EXECUTOR_QUEUE_NAMES" + "SRC_LOG_LEVEL" + "SRC_LOG_FORMAT" + "EXECUTOR_MAXIMUM_NUM_JOBS" + "EXECUTOR_MAXIMUM_RUNTIME_PER_JOB" + "EXECUTOR_DOCKER_ADD_HOST_GATEWAY" + "EXECUTOR_KEEP_WORKSPACES" -}} +{{- range $managed -}} +{{- if hasKey $envDict . -}} +{{- fail (printf "%s: env must not contain managed variable %s; use the structured executor fields instead" $label .) -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/sourcegraph-executor/dind/templates/executor/docker-daemon.ConfigMap.yaml b/charts/sourcegraph-executor/dind/templates/executor/docker-daemon.ConfigMap.yaml index f927fcf8f..ecd96f99b 100644 --- a/charts/sourcegraph-executor/dind/templates/executor/docker-daemon.ConfigMap.yaml +++ b/charts/sourcegraph-executor/dind/templates/executor/docker-daemon.ConfigMap.yaml @@ -1,9 +1,4 @@ -{{- if .Values.executor.enabled -}} apiVersion: v1 -data: - daemon.json: | - { "insecure-registries":["private-docker-registry:5000"] } - kind: ConfigMap metadata: labels: @@ -11,4 +6,6 @@ metadata: deploy: sourcegraph app.kubernetes.io/component: executor name: docker-config -{{- end }} +data: + daemon.json: | + {{- include "dind.daemonConfig" . | nindent 4 }} diff --git a/charts/sourcegraph-executor/dind/templates/executor/executor.Deployment.yaml b/charts/sourcegraph-executor/dind/templates/executor/executor.Deployment.yaml index 069481490..e251e60b2 100644 --- a/charts/sourcegraph-executor/dind/templates/executor/executor.Deployment.yaml +++ b/charts/sourcegraph-executor/dind/templates/executor/executor.Deployment.yaml @@ -1,149 +1,28 @@ -{{- if .Values.executor.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - description: Runs sourcegraph executors - kubectl.kubernetes.io/default-container: executor - labels: - {{- include "sourcegraph.labels" . | nindent 4 }} - {{- if .Values.executor.labels }} - {{- toYaml .Values.executor.labels | nindent 4 }} - {{- end }} - {{- include "executor.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - {{- include "sourcegraph.selectorLabels" . | nindent 6 }} - app: {{ include "executor.name" . }} - minReadySeconds: 10 - replicas: {{ .Values.executor.replicaCount }} - revisionHistoryLimit: 10 - strategy: - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 - type: RollingUpdate - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: executor - {{- if .Values.sourcegraph.podAnnotations }} - {{- toYaml .Values.sourcegraph.podAnnotations | nindent 8 }} - {{- end }} - {{- if .Values.executor.podAnnotations }} - {{- toYaml .Values.executor.podAnnotations | nindent 8 }} - {{- end }} - labels: - {{- include "sourcegraph.selectorLabels" . | nindent 8 }} - {{- if .Values.sourcegraph.podLabels }} - {{- toYaml .Values.sourcegraph.podLabels | nindent 8 }} - {{- end }} - {{- if .Values.executor.podLabels }} - {{- toYaml .Values.executor.podLabels | nindent 8 }} - {{- end }} - {{- include "executor.labels" . | nindent 8 }} - spec: - containers: - - name: executor - image: {{ include "sourcegraph.image" (list . "executor") }} - imagePullPolicy: {{ .Values.sourcegraph.image.pullPolicy }} - livenessProbe: - httpGet: - path: /healthz - port: http-debug - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /ready - port: http-debug - scheme: HTTP - periodSeconds: 5 - timeoutSeconds: 5 - ports: - - name: http-debug - containerPort: 6060 - terminationMessagePolicy: FallbackToLogsOnError - env: - {{- range $name, $item := .Values.executor.env }} - - name: {{ $name }} - {{- $item | toYaml | nindent 14 }} - {{- end }} - - name: EXECUTOR_USE_FIRECRACKER - value: "false" - - name: EXECUTOR_JOB_NUM_CPUS - value: "0" - - name: EXECUTOR_JOB_MEMORY - value: "0" - - name: DOCKER_HOST - value: tcp://localhost:2375 - - name: TMPDIR - value: /scratch - volumeMounts: - - mountPath: /scratch - name: executor-scratch - - name: dind - image: "{{ .Values.dind.image.registry}}/{{ .Values.dind.image.repository}}:{{ .Values.dind.image.tag}}" - imagePullPolicy: {{ .Values.sourcegraph.image.pullPolicy }} - securityContext: - privileged: true - command: - - 'dockerd' - - '--tls=false' - - '--mtu=1200' - - '--registry-mirror=http://executor:5000' - - '--host=tcp://0.0.0.0:2375' - livenessProbe: - tcpSocket: - port: 2375 - initialDelaySeconds: 5 - periodSeconds: 5 - failureThreshold: 5 - readinessProbe: - tcpSocket: - port: 2375 - initialDelaySeconds: 10 - periodSeconds: 5 - failureThreshold: 5 - env: - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - ports: - - containerPort: 2375 - protocol: TCP - volumeMounts: - - mountPath: /scratch - name: executor-scratch - - mountPath: /etc/docker/daemon.json - subPath: daemon.json - name: docker-config - {{- with .Values.sourcegraph.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.sourcegraph.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with include "sourcegraph.priorityClassName" (list . "executor") | trim }}{{ . | nindent 6 }}{{- end }} - {{- with .Values.sourcegraph.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.sourcegraph.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: executor-scratch - emptyDir: {} - - name: docker-config - configMap: - defaultMode: 420 - name: docker-config +{{- if .Values.queues }} +{{- range .Values.queues }} +{{- include "executor.validateEnv" (list $.Values.executor.env "executor.env") }} +{{- include "executor.validateEnv" (list (.env | default dict) (printf "queues[%s].env" .name)) }} +--- +{{ include "executor.deployment" (dict + "root" $ + "name" (printf "executor-%s" .name) + "queueName" (.queueName | default .name) + "queueNames" (.queueNames | default (list)) + "replicaCount" (.replicaCount | default $.Values.executor.replicaCount) + "resources" (.resources | default $.Values.executor.resources) + "env" (mergeOverwrite (deepCopy $.Values.executor.env) (.env | default dict)) +) }} +{{- end }} +{{- else }} +{{- include "executor.validateEnv" (list .Values.executor.env "executor.env") }} +--- +{{ include "executor.deployment" (dict + "root" . + "name" (include "executor.name" .) + "queueName" .Values.executor.queueName + "queueNames" .Values.executor.queueNames + "replicaCount" .Values.executor.replicaCount + "resources" .Values.executor.resources + "env" .Values.executor.env +) }} {{- end }} diff --git a/charts/sourcegraph-executor/dind/templates/executor/executor.Service.yaml b/charts/sourcegraph-executor/dind/templates/executor/executor.Service.yaml deleted file mode 100644 index 970de1a9c..000000000 --- a/charts/sourcegraph-executor/dind/templates/executor/executor.Service.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if .Values.executor.enabled -}} -apiVersion: v1 -kind: Service -metadata: - annotations: - prometheus.io/port: "6060" - sourcegraph.prometheus/scrape: "true" - {{- if .Values.executor.serviceAnnotations }} - {{- toYaml .Values.executor.serviceAnnotations | nindent 4 }} - {{- end }} - labels: - {{- include "executor.labels" . | nindent 4 }} - {{- if .Values.executor.serviceLabels }} - {{- toYaml .Values.executor.serviceLabels | nindent 4 }} - {{- end }} - name: executor -spec: - ports: - - name: http-debug - port: 6060 - targetPort: http-debug - selector: - {{- include "sourcegraph.selectorLabels" . | nindent 4 }} - app: {{include "executor.name" . }} - type: {{ .Values.executor.serviceType | default "ClusterIP" }} -{{- end }} diff --git a/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.PersistentVolumeClaim.yaml b/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.PersistentVolumeClaim.yaml deleted file mode 100644 index 619d5af9b..000000000 --- a/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.PersistentVolumeClaim.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if .Values.privateDockerRegistry.enabled -}} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - deploy: sourcegraph - app.kubernetes.io/component: private-docker-registry - name: private-docker-registry -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.privateDockerRegistry.storageSize }} - storageClassName: {{ .Values.storageClass.name }} - {{- if .Values.privateDockerRegistry.volumeName }} - volumeName: {{ .Values.privateDockerRegistry.volumeName }} - {{- end }} -{{- end }} diff --git a/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Deployment.yaml b/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Statefulset.yaml similarity index 85% rename from charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Deployment.yaml rename to charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Statefulset.yaml index 32554be69..fff6d3379 100644 --- a/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Deployment.yaml +++ b/charts/sourcegraph-executor/dind/templates/private-docker-registry/private-docker-registry.Statefulset.yaml @@ -1,6 +1,6 @@ {{- if .Values.privateDockerRegistry.enabled -}} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: private-docker-registry labels: @@ -11,13 +11,14 @@ metadata: deploy: sourcegraph app.kubernetes.io/component: private-docker-registry spec: - replicas: {{ .Values.privateDockerRegistry.replicaCount }} + replicas: 1 + serviceName: private-docker-registry selector: matchLabels: {{- include "sourcegraph.selectorLabels" . | nindent 6 }} app: private-docker-registry - strategy: - type: Recreate + updateStrategy: + type: RollingUpdate template: metadata: annotations: @@ -83,8 +84,17 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - volumes: - - name: cache - persistentVolumeClaim: - claimName: private-docker-registry + volumeClaimTemplates: + - metadata: + name: cache + labels: + deploy: sourcegraph + app.kubernetes.io/component: private-docker-registry + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.privateDockerRegistry.storageSize }} + storageClassName: {{ .Values.storageClass.name }} {{- end }} diff --git a/charts/sourcegraph-executor/dind/values.yaml b/charts/sourcegraph-executor/dind/values.yaml index eec0a03c1..e27885cf6 100644 --- a/charts/sourcegraph-executor/dind/values.yaml +++ b/charts/sourcegraph-executor/dind/values.yaml @@ -54,31 +54,150 @@ storageClass: # learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/storage/storage-classes/#allowed-topologies) allowedTopologies: {} +# -- Optional list of queues to deploy as standalone Deployments. +# When set, the single executor Deployment is not rendered. +# Each entry supports: +# name (required) — used as the deployment name suffix (executor-) +# queueName — sets EXECUTOR_QUEUE_NAME; defaults to name if omitted +# queueNames — sets EXECUTOR_QUEUE_NAMES (comma-joined); takes precedence over queueName when set +# replicaCount, resources, env (merged with executor.env, queue overrides) +queues: [] +# - name: codeintel +# queueName: codeintel +# replicaCount: 2 +# resources: +# requests: +# cpu: "2" +# memory: 4Gi +# limits: +# cpu: "4" +# memory: 8Gi +# env: {} +# - name: batches +# queueName: batches +# replicaCount: 1 +# resources: +# requests: +# cpu: "1" +# memory: 2Gi +# limits: +# cpu: "2" +# memory: 4Gi +# env: {} + executor: - enabled: true + # -- The external URL of the Sourcegraph instance. Required. + frontendUrl: "" + # -- The shared secret configured in the Sourcegraph instance site config under executors.accessToken. Required if frontendExistingSecret is not configured. + frontendPassword: "" + # -- Name of existing k8s Secret to use for frontend password. + # The k8s Secret must contain the key EXECUTOR_FRONTEND_PASSWORD matching the site config executors.accessToken value. + # frontendPassword is ignored if this is set. + frontendExistingSecret: "" + # -- The name of the queue to pull jobs from. Possible values: batches and codeintel. Either this or queueNames is required (when not using queues). + queueName: "" + # -- The names of multiple queues to pull jobs from. Possible values: batches and codeintel. Either this or queueName is required (when not using queues). + queueNames: [] + # -- The maximum amount of jobs that can be executed concurrently. + maximumNumJobs: 10 + # -- The maximum wall time that can be spent on a single job. + maximumRuntimePerJob: "30m" + log: + # -- Possible values are dbug, info, warn, eror, crit. + level: "warn" + format: "json" + # -- Resource requests and limits for the executor container. + # Each queue can override this with its own resources field. + resources: {} image: defaultTag: 6.0.0@sha256:0be94a7c91f8273db10fdf46718c6596340ab2acc570e7b85353806e67a27508 name: "executor" replicaCount: 1 - env: - # -- The external URL of the Sourcegraph instance. Required. - EXECUTOR_FRONTEND_URL: - value: "" - # -- The shared secret configured in the Sourcegraph instance site config under executors.accessToken. Required. - EXECUTOR_FRONTEND_PASSWORD: - value: "" - # -- The name of the queue to pull jobs from to. Possible values: batches and codeintel. **Either this or EXECUTOR_QUEUE_NAMES is required.** - EXECUTOR_QUEUE_NAME: - value: "" - # -- The comma-separated list of names of multiple queues to pull jobs from to. Possible values: batches and codeintel. **Either this or EXECUTOR_QUEUE_NAME is required.** - EXECUTOR_QUEUE_NAMES: - value: "" + # -- Extra environment variables to set on the executor container. + env: {} + storage: + # -- Type of scratch volume for job workspaces. One of: emptyDir, ephemeral. + # emptyDir: plain emptyDir, no storage class required. + # ephemeral: per-pod PVC via the cluster default storage class; size is required. + type: emptyDir + # -- Size of the scratch volume. + # emptyDir: sets sizeLimit (optional, leave empty for unlimited). + # ephemeral: sets the PVC storage request (required). + size: "" + # -- StorageClass for the ephemeral volume. Only used when type is ephemeral. + # Defaults to the cluster default StorageClass when empty. + class: "" dind: image: registry: index.docker.io repository: docker - tag: 20.10.22-dind + tag: 29.5.3-dind + # -- Command for the dind container. + command: + - dockerd + # -- Docker daemon configuration passed as daemon.json to the dind sidecar. + # Learn more from: https://docs.docker.com/reference/cli/dockerd/#on-linux + daemonConfig: + tls: false + mtu: 1200 + hosts: + - tcp://127.0.0.1:2375 + registry-mirrors: + - http://private-docker-registry:5000 + insecure-registries: + - private-docker-registry:5000 + gVisor: + # -- Enable gVisor sandbox (GKE only). Requires the GKE node pool to have sandbox type set to gvisor. + # When enabled, sets runtimeClassName: gvisor on executor pods and replaces privileged: true with + # explicit capabilities — these are intercepted in-sandbox and never granted to the host kernel. + # See: https://gvisor.dev/docs/tutorials/docker-in-gvisor/ + enabled: false + # -- Extra daemon.json settings merged into dind.daemonConfig when gVisor is enabled. + # These defaults configure Docker to work within gVisor's kernel constraints. + daemonConfig: + storage-driver: vfs + iptables: false + ip6tables: false + features: + containerd-snapshotter: false + # -- securityContext for the dind container when gVisor is enabled. + # Replaces privileged: true — gVisor intercepts these capabilities in-sandbox and never + # grants them to the host kernel. + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + - AUDIT_WRITE + - CHOWN + - DAC_OVERRIDE + - FOWNER + - FSETID + - KILL + - MKNOD + - NET_BIND_SERVICE + - NET_RAW + - SETFCAP + - SETGID + - SETPCAP + - SETUID + - SYS_CHROOT + - SYS_PTRACE + # -- Command for the dind container when gVisor is enabled. + # Overrides dind.command. Prepares the network environment that gVisor does not + # initialise automatically before handing off to dockerd. + command: + - /bin/sh + - -c + - | + ip link del docker0 2>/dev/null || true + echo 1 > /proc/sys/net/ipv4/ip_forward + dev=$(ip route show default | awk '{for(i=1;i<=NF;i++) if($i=="dev"){print $(i+1); exit}}') + addr=$(ip addr show dev "$dev" | awk '/inet /{gsub(/\/.*/, "", $2); print $2; exit}') + iptables-legacy -t nat -A POSTROUTING -o "$dev" -j SNAT --to-source "$addr" -p tcp || true + iptables-legacy -t nat -A POSTROUTING -o "$dev" -j SNAT --to-source "$addr" -p udp || true + exec dockerd privateDockerRegistry: # -- Whether to deploy the private registry. Only one registry is needed when deploying multiple executors. @@ -86,6 +205,6 @@ privateDockerRegistry: enabled: true image: registry: index.docker.io - repository: docker/regisry - tag: 2 + repository: registry + tag: 3 storageSize: 10Gi