docker update

Author: max-ostapenko

Makefile

@@ -14,3 +14,6 @@ tf_apply:
 
 bigquery_export_deploy:
 	cd infra/bigquery-export && npm run buildpack
+
+bigquery_export_spark_deploy:
+	cd infra/bigquery_export_spark && gcloud builds submit --region=global --tag us-docker.pkg.dev/httparchive/bigquery-spark-procedures/firestore_export:latest

infra/bigquery_export_spark/Dockerfile

@@ -1,5 +1,7 @@
+# Dataproc image example: https://cloud.google.com/dataproc-serverless/docs/guides/custom-containers
 # Recommendation: Use Debian 12.
-FROM python:3.12-slim
+FROM debian:12-slim
+# python:3.12-slim
 
 # Suppress interactive prompts
 ENV DEBIAN_FRONTEND=noninteractive
@@ -10,13 +12,20 @@ RUN apt update && apt install -y procps tini libjemalloc2
 # Enable jemalloc2 as default memory allocator
 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
 
-# Install packages.
-RUN pip install --no-cache-dir \
-    google-cloud-bigquery \
-    google-cloud-storage \
-    google-cloud-firestore \
-    grpcio \
-    google-cloud-secret-manager
+# Install and configure Miniconda3.
+ENV CONDA_HOME=/opt/miniforge3
+ENV PYSPARK_PYTHON=${CONDA_HOME}/bin/python
+ENV PATH=${CONDA_HOME}/bin:${PATH}
+ADD https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh .
+RUN bash Miniforge3-Linux-x86_64.sh -b -p /opt/miniforge3 \
+  && ${CONDA_HOME}/bin/conda config --system --set always_yes True \
+  && ${CONDA_HOME}/bin/conda config --system --set auto_update_conda False \
+  && ${CONDA_HOME}/bin/conda config --system --set channel_priority strict
+
+COPY . .
+
+# Install pip packages.
+RUN ${PYSPARK_PYTHON} -m pip install -r requirements.txt
 
 # Create the 'spark' group/user.
 # The GID and UID must be 1099. Home directory is required.

infra/bigquery_export_spark/requirements.txt

@@ -0,0 +1,3 @@
+google-cloud-bigquery==3.23
+google-cloud-storage==2.16
+google-cloud-firestore==2.20.1