From da4f6169905d2c33544c1d08e5cbef02c4d11648 Mon Sep 17 00:00:00 2001 From: Tim Fennell Date: Mon, 22 Jun 2026 16:13:43 -0600 Subject: [PATCH 1/2] perf: use jemalloc for make_examples in the runtime image make_examples spends a large share of time in allocation-heavy pileup and local-realignment work. Preloading jemalloc (vs glibc malloc) measurably reduces its wall-clock; it has no measurable effect on call_variants (TF inference), so the LD_PRELOAD is scoped to the make_examples-family wrappers only. Installed via the distro libjemalloc2 package; the bare soname (libjemalloc.so.2) keeps it architecture-portable. --- Dockerfile | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0d3c449d..92901e90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -102,7 +102,7 @@ ENV DV_BIN_PATH=/opt/deepvariant/bin # Install libraries RUN apt-get -y update && \ - apt-get install -y parallel python3-pip unzip && \ + apt-get install -y parallel python3-pip unzip libjemalloc2 && \ PATH="${HOME}/.local/bin:$PATH" python3 -m pip install absl-py==0.13.0 && \ apt-get clean autoclean && \ apt-get autoremove -y --purge && \ @@ -140,11 +140,17 @@ COPY --from=builder \ /opt/deepvariant/bin/ # Create shell wrappers for python zip files for easier use. +# +# The make_examples family is wrapped with LD_PRELOAD=libjemalloc.so.2: jemalloc +# meaningfully reduces make_examples wall-clock (its pileup/realignment work is +# allocation-heavy) while having no measurable effect on call_variants (TF +# inference), so the preload is scoped to just those wrappers. The bare soname +# keeps it architecture-portable (resolved from the default linker search path). RUN \ BASH_HEADER='#!/bin/bash' && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ - '/usr/bin/python3 /opt/deepvariant/bin/make_examples.zip "$@"' > \ + 'LD_PRELOAD=libjemalloc.so.2 /usr/bin/python3 /opt/deepvariant/bin/make_examples.zip "$@"' > \ /opt/deepvariant/bin/make_examples && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ @@ -168,7 +174,7 @@ RUN \ /opt/deepvariant/bin/runtime_by_region_vis && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ - '/usr/bin/python3 /opt/deepvariant/bin/multisample_make_examples.zip "$@"' > \ + 'LD_PRELOAD=libjemalloc.so.2 /usr/bin/python3 /opt/deepvariant/bin/multisample_make_examples.zip "$@"' > \ /opt/deepvariant/bin/multisample_make_examples && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ @@ -180,7 +186,7 @@ RUN \ /opt/deepvariant/bin/convert_to_saved_model && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ - '/usr/bin/python3 -u /opt/deepvariant/bin/make_examples_somatic.zip "$@"' > \ + 'LD_PRELOAD=libjemalloc.so.2 /usr/bin/python3 -u /opt/deepvariant/bin/make_examples_somatic.zip "$@"' > \ /opt/deepvariant/bin/make_examples_somatic && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ From a632502229e71ee68003ca881a7155d653f8c97a Mon Sep 17 00:00:00 2001 From: Tim Fennell Date: Tue, 23 Jun 2026 05:15:14 -0600 Subject: [PATCH 2/2] perf: use jemalloc for make_examples in the DeepTrio, DeepSomatic and pangenome images The runtime image already preloads jemalloc for its make_examples wrappers, but the DeepTrio, DeepSomatic, and pangenome-aware images build FROM ubuntu:22.04 rather than from that image, so they inherited neither the libjemalloc2 install nor the preload and ran make_examples on glibc malloc. Their make_examples does the same allocation-heavy pileup/realignment work, so they benefit from the same preload. This applies the identical pattern to each of the three Dockerfiles: install libjemalloc2 and prepend LD_PRELOAD=libjemalloc.so.2 to that image's make_examples wrapper only (deeptrio/make_examples, make_examples_somatic, make_examples_pangenome_aware_dv). call_variants and the other wrappers are left untouched, matching the runtime image. --- Dockerfile.deepsomatic | 10 ++++++++-- Dockerfile.deeptrio | 10 ++++++++-- Dockerfile.pangenome_aware_deepvariant | 10 ++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Dockerfile.deepsomatic b/Dockerfile.deepsomatic index 4d2f38f1..6ae7b5af 100644 --- a/Dockerfile.deepsomatic +++ b/Dockerfile.deepsomatic @@ -130,7 +130,7 @@ ENV DV_BIN_PATH=/opt/deepvariant/bin # Install libraries RUN apt-get -y update && \ - apt-get install -y parallel python3-pip unzip && \ + apt-get install -y parallel python3-pip unzip libjemalloc2 && \ PATH="${HOME}/.local/bin:$PATH" python3 -m pip install absl-py==0.13.0 && \ apt-get clean autoclean && \ apt-get autoremove -y --purge && \ @@ -164,11 +164,17 @@ COPY --from=builder \ /opt/deepvariant/bin/ # Create shell wrappers for python zip files for easier use. +# +# The make_examples family is wrapped with LD_PRELOAD=libjemalloc.so.2: jemalloc +# meaningfully reduces make_examples wall-clock (its pileup/realignment work is +# allocation-heavy) while having no measurable effect on call_variants (TF +# inference), so the preload is scoped to just those wrappers. The bare soname +# keeps it architecture-portable (resolved from the default linker search path). RUN \ BASH_HEADER='#!/bin/bash' && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ - 'python3 -u /opt/deepvariant/bin/make_examples_somatic.zip "$@"' > \ + 'LD_PRELOAD=libjemalloc.so.2 python3 -u /opt/deepvariant/bin/make_examples_somatic.zip "$@"' > \ /opt/deepvariant/bin/make_examples_somatic && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ diff --git a/Dockerfile.deeptrio b/Dockerfile.deeptrio index 051efa1e..cef15014 100644 --- a/Dockerfile.deeptrio +++ b/Dockerfile.deeptrio @@ -72,11 +72,17 @@ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTH update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 0 # Create shell wrappers for python zip files for easier use. +# +# The make_examples family is wrapped with LD_PRELOAD=libjemalloc.so.2: jemalloc +# meaningfully reduces make_examples wall-clock (its pileup/realignment work is +# allocation-heavy) while having no measurable effect on call_variants (TF +# inference), so the preload is scoped to just those wrappers. The bare soname +# keeps it architecture-portable (resolved from the default linker search path). RUN \ BASH_HEADER='#!/bin/bash' && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ - 'python3 /opt/deepvariant/bin/deeptrio/make_examples.zip "$@"' > \ + 'LD_PRELOAD=libjemalloc.so.2 python3 /opt/deepvariant/bin/deeptrio/make_examples.zip "$@"' > \ /opt/deepvariant/bin/deeptrio/make_examples && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ @@ -215,7 +221,7 @@ ENV PATH="${PATH}":/opt/conda/bin:/opt/conda/envs/bio/bin:/opt/deepvariant/bin/d ENV TF_USE_LEGACY_KERAS=1 RUN apt-get -y update && \ - apt-get install -y parallel python3-pip && \ + apt-get install -y parallel python3-pip libjemalloc2 && \ PATH="${HOME}/.local/bin:$PATH" python3 -m pip install absl-py==0.13.0 && \ apt-get clean autoclean && \ apt-get autoremove -y --purge && \ diff --git a/Dockerfile.pangenome_aware_deepvariant b/Dockerfile.pangenome_aware_deepvariant index 2026b069..034bab39 100644 --- a/Dockerfile.pangenome_aware_deepvariant +++ b/Dockerfile.pangenome_aware_deepvariant @@ -73,11 +73,17 @@ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTH update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 0 # Create shell wrappers for python zip files for easier use. +# +# The make_examples family is wrapped with LD_PRELOAD=libjemalloc.so.2: jemalloc +# meaningfully reduces make_examples wall-clock (its pileup/realignment work is +# allocation-heavy) while having no measurable effect on call_variants (TF +# inference), so the preload is scoped to just those wrappers. The bare soname +# keeps it architecture-portable (resolved from the default linker search path). RUN \ BASH_HEADER='#!/bin/bash' && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ - 'python3 -u /opt/deepvariant/bin/make_examples_pangenome_aware_dv.zip "$@"' > \ + 'LD_PRELOAD=libjemalloc.so.2 python3 -u /opt/deepvariant/bin/make_examples_pangenome_aware_dv.zip "$@"' > \ /opt/deepvariant/bin/make_examples_pangenome_aware_dv && \ printf "%s\n%s\n" \ "${BASH_HEADER}" \ @@ -140,7 +146,7 @@ ENV PATH="${PATH}":/opt/conda/bin:/opt/conda/envs/bio/bin:/opt/deepvariant/bin/p ENV TF_USE_LEGACY_KERAS=1 RUN apt-get -y update && \ - apt-get install -y parallel python3-pip && \ + apt-get install -y parallel python3-pip libjemalloc2 && \ PATH="${HOME}/.local/bin:$PATH" python3 -m pip install absl-py==0.13.0 && \ apt-get clean autoclean && \ apt-get autoremove -y --purge && \