Try GCSFuse mounting as a side car

Author: tarekmamdouh

http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml

@@ -19,12 +19,7 @@ services:
     volumes:
       # shared with welder
       - "/mnt/disks/work:${NOTEBOOKS_DIR}"
-    devices:
-      - /dev/fuse:/dev/fuse
-    cap_add:
-      - SYS_ADMIN
-    security_opt:
-      - apparmor:unconfined
+      - /mnt/disks/bucket:/usr/local/genomics
     restart: always
     environment:
       GOOGLE_PROJECT: "${GOOGLE_PROJECT}"

http/src/main/resources/init-resources/jupyter-docker-compose.yaml

@@ -36,12 +36,7 @@ services:
       - /hadoop:/hadoop
       - /hadoop_gcs_connector_metadata_cache:/hadoop_gcs_connector_metadata_cache
       - /usr/local/share/google/dataproc:/usr/local/share/google/dataproc
-    devices:
-      - /dev/fuse:/dev/fuse
-    cap_add:
-      - SYS_ADMIN
-    security_opt:
-      - apparmor:unconfined
+      - /mnt/disks/bucket:/usr/local/genomics
     restart: always
     environment:
       GOOGLE_PROJECT: "${GOOGLE_PROJECT}"

http/src/main/resources/init-resources/startup.sh

@@ -115,6 +115,12 @@ then
     mount -t ext4 -O discard,defaults /dev/${DISK_DEVICE_ID} ${WORK_DIRECTORY}
     chmod a+rwx /mnt/disks/work
 
+    # Start gcsfuse as a sidecar
+    mkdir -p /mnt/disks/bucket
+    docker exec $JUPYTER_SERVER_NAME /bin/bash -c "mkdir /usr/local/genomics"
+    docker run -d --name gcsfuse-container   --platform linux/amd64   --privileged   --env BUCKET_NAME=genomics-public-data   --device /dev/fuse   --security-opt apparmor=unconfined   -v /mnt/disks/bucket:/mnt/gcs-bucket:shared   -v /mnt/disks/bucket:/mnt/gcs-data   tarekmahmed/gcsfuse-container:latest
+
+
     # (1/6/22) Restart Jupyter Container to reset `NOTEBOOKS_DIR` for existing runtimes. This code can probably be removed after a year
     if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then
         echo "Restarting Jupyter Container $GOOGLE_PROJECT / $CLUSTER_NAME..."