From c7af759ab1db428ab85b08188b57fadb898408bc Mon Sep 17 00:00:00 2001
From: Siyao Meng <50227127+smengcl@users.noreply.github.com>
Date: Sun, 24 May 2026 00:14:26 -0700
Subject: [PATCH] HDDS-11234. Manage Netty native memory consumption

---
 hadoop-hdds/common/src/main/conf/ozone-env.sh     | 15 +++++++++++++++
 .../dist/src/shell/ozone/ozone-functions.sh       | 11 +++++++++++
 2 files changed, 26 insertions(+)

diff --git a/hadoop-hdds/common/src/main/conf/ozone-env.sh b/hadoop-hdds/common/src/main/conf/ozone-env.sh
index dd98331d78db..76b991702c82 100644
--- a/hadoop-hdds/common/src/main/conf/ozone-env.sh
+++ b/hadoop-hdds/common/src/main/conf/ozone-env.sh
@@ -249,6 +249,21 @@ export OZONE_OS_TYPE=${OZONE_OS_TYPE:-$(uname -s)}
 # to keep OZONE_IDENT_STRING untouched, then uncomment this line.
 # export OZONE_SECURE_IDENT_PRESERVE="true"
 
+###
+# Netty native (direct) memory caps (HDDS-11234)
+###
+# Both unshaded io.netty and the Ratis-shaded copy default their pooled
+# direct-memory ceiling to MaxDirectMemorySize (≈ -Xmx) per JVM, which
+# can let the resident size of a busy DataNode or S3 Gateway grow well
+# beyond the heap. To put a hard cap on each pool, export one or both
+# of the following before starting Ozone daemons. Values are raw byte
+# counts (suffixes like "m" or "g" are NOT supported by Netty's
+# property parser); for example, 536870912 = 512 MiB.
+#
+# For example, to cap each pool at 4 GiB on a DataNode with -Xmx16g:
+# export OZONE_NETTY_MAX_DIRECT_MEMORY=4294967296
+# export OZONE_RATIS_NETTY_MAX_DIRECT_MEMORY=4294967296
+
 ###
 # Ozone Manager specific parameters
 ###
diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
index 325d6daa50b2..4a4173f6d67f 100755
--- a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
+++ b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
@@ -1420,6 +1420,17 @@ function ozone_java_setup
     RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true ${RATIS_OPTS}"
   fi
 
+  # Opt-in caps on Netty's pooled direct-memory arena (HDDS-11234). Two
+  # properties are needed because Ozone runs both the unshaded io.netty
+  # *and* the Ratis-shaded copy in the same JVM, each with its own
+  # independent ceiling.
+  if [[ -n "${OZONE_NETTY_MAX_DIRECT_MEMORY:-}" ]]; then
+    OZONE_OPTS="-Dio.netty.maxDirectMemory=${OZONE_NETTY_MAX_DIRECT_MEMORY} ${OZONE_OPTS}"
+  fi
+  if [[ -n "${OZONE_RATIS_NETTY_MAX_DIRECT_MEMORY:-}" ]]; then
+    RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.maxDirectMemory=${OZONE_RATIS_NETTY_MAX_DIRECT_MEMORY} ${RATIS_OPTS}"
+  fi
+
   ozone_set_module_access_args
 }