HDDS-15306. Expose Disk Balancer CLI in top-level datanode help and improve usability

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -410,7 +410,7 @@ public final class HddsConfigKeys {
 
   public static final String HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY =
       "hdds.datanode.disk.balancer.enabled";
-  public static final boolean HDDS_DATANODE_DISK_BALANCER_ENABLED_DEFAULT = false;
+  public static final boolean HDDS_DATANODE_DISK_BALANCER_ENABLED_DEFAULT = true;
 
   public static final String HDDS_DATANODE_DNS_INTERFACE_KEY =
       "hdds.datanode.dns.interface";

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -218,11 +218,9 @@
   </property>
   <property>
     <name>hdds.datanode.disk.balancer.enabled</name>
-    <value>false</value>
+    <value>true</value>
     <tag>OZONE, DATANODE, DISKBALANCER</tag>
-    <description>If this property is set to true, then the Disk Balancer
-      feature is enabled on Datanodes, and users can use
-      this service. By default, this is disabled.
+    <description>By default Disk Balancer feature is enabled on Datanodes.
     </description>
   </property>
   <property>

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java

@@ -747,6 +747,7 @@ public static List<VolumeReportProto> buildVolumeReportProto(List<VolumeFixedUsa
       VolumeReportProto.Builder builder = VolumeReportProto.newBuilder()
           .setStorageId(volume.getStorageID())
           .setTotalCapacity(v.getUsage().getCapacity())
+          .setOzoneAvailable(v.getUsage().getAvailable())
           .setUsedSpace(v.getUsage().getUsedSpace())
           .setCommittedBytes(volume.getCommittedBytes())
           .setEffectiveUsedSpace(v.getEffectiveUsed())

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java

@@ -50,6 +50,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
@@ -279,7 +280,8 @@ public OzoneContainer(HddsDatanodeService hddsDatanodeService,
               config);
     } else {
       diskBalancerService = null;
-      LOG.info("Disk Balancer is disabled.");
+      LOG.info("Disk Balancer is not enabled. Please enable the " +
+          HddsConfigKeys.HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY + " configuration key.");
     }
 
     Duration recoveringContainerScrubbingSvcInterval =

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java

@@ -108,6 +108,7 @@ void setup() throws IOException {
             .setUtilization(TEST_UTILIZATION_1)
             .setCommittedBytes(TEST_COMMITTED_BYTES_1)
             .setTotalCapacity(TEST_TOTAL_CAPACITY)
+            .setOzoneAvailable(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_1)
             .setUsedSpace(TEST_USED_SPACE_1)
             .setEffectiveUsedSpace(TEST_EFFECTIVE_USED_SPACE_1)
             .build(),
@@ -117,6 +118,7 @@ void setup() throws IOException {
             .setUtilization(TEST_UTILIZATION_2)
             .setCommittedBytes(TEST_COMMITTED_BYTES_2)
             .setTotalCapacity(TEST_TOTAL_CAPACITY)
+            .setOzoneAvailable(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_2)
             .setUsedSpace(TEST_USED_SPACE_2)
             .setEffectiveUsedSpace(TEST_EFFECTIVE_USED_SPACE_2)
             .build()));
@@ -157,13 +159,15 @@ void testGetDiskBalancerInfoReport() throws IOException {
     assertEquals(TEST_UTILIZATION_1, volReport0.getUtilization());
     assertEquals(TEST_COMMITTED_BYTES_1, volReport0.getCommittedBytes());
     assertEquals(TEST_TOTAL_CAPACITY, volReport0.getTotalCapacity());
+    assertEquals(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_1, volReport0.getOzoneAvailable());
     assertEquals(TEST_USED_SPACE_1, volReport0.getUsedSpace());
     assertEquals(TEST_EFFECTIVE_USED_SPACE_1, volReport0.getEffectiveUsedSpace());
     assertEquals(TEST_STORAGE_ID_2, volReport1.getStorageId());
     assertEquals(TEST_STORAGE_PATH_2, volReport1.getStoragePath());
     assertEquals(TEST_UTILIZATION_2, volReport1.getUtilization());
     assertEquals(TEST_COMMITTED_BYTES_2, volReport1.getCommittedBytes());
     assertEquals(TEST_TOTAL_CAPACITY, volReport1.getTotalCapacity());
+    assertEquals(TEST_TOTAL_CAPACITY - TEST_USED_SPACE_2, volReport1.getOzoneAvailable());
     assertEquals(TEST_USED_SPACE_2, volReport1.getUsedSpace());
     assertEquals(TEST_EFFECTIVE_USED_SPACE_2, volReport1.getEffectiveUsedSpace());
   }

hadoop-hdds/docs/content/design/diskbalancer.md

@@ -165,7 +165,7 @@ This ensures DiskBalancer respects datanode lifecycle management and does not in
 
 ## Feature Flag
 
-The DiskBalancer feature is gated behind a feature flag (`hdds.datanode.disk.balancer.enabled`) to allow controlled rollout. By default, the feature is disabled. When disabled, the DiskBalancer service is not initialized on datanodes, and the CLI commands are hidden from the main help output to prevent accidental usage.
+The DiskBalancer feature is gated behind a feature flag (`hdds.datanode.disk.balancer.enabled`) to allow controlled rollout. By default, the feature is enabled. When disabled, the DiskBalancer service is not initialized on datanodes, and the CLI commands refuse to run until the flag is set back to true.
 
 ## DiskBalancer Metrics
 

hadoop-hdds/docs/content/feature/DiskBalancer.md

@@ -43,9 +43,9 @@ A disk is considered a candidate for balancing if its
 
 ## Feature Flag
 
-The Disk Balancer feature is introduced with a feature flag. By default, this feature is disabled.
+The Disk Balancer feature is introduced with a feature flag. By default, this feature is enabled.
 
-The feature can be **enabled** by setting the following property to `true` in the `ozone-site.xml` configuration file:
+The feature can be **disabled** by setting the following property to `false` in the `ozone-site.xml` configuration file:
 `hdds.datanode.disk.balancer.enabled = false`
 
 ### Authentication and Authorization
@@ -119,8 +119,8 @@ restart the datanode service for the changes to take effect.
 ## Command Line Usage
 The DiskBalancer is managed through the `ozone admin datanode diskbalancer` command.
 
-**Note:** This command is hidden from the main help message (`ozone admin datanode --help`). This is because the feature
-is currently considered experimental and is disabled by default. The command is, however, fully functional for those who wish to enable and use the feature.
+**Note:** DiskBalancer is enabled by default on datanodes. Use `hdds.datanode.disk.balancer.enabled=false` in
+`ozone-site.xml` to disable the service on datanodes and prevent CLI commands from running.
 
 ### Command Syntax
 
@@ -241,7 +241,7 @@ The DiskBalancer's behavior can be controlled using the following configuration
 
 | Property                                                       | Default Value                                                                          | Description                                                                                                                                                                                                                                                                                                                            |
 |----------------------------------------------------------------|----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `hdds.datanode.disk.balancer.enabled`                          | `false`                                                                                | If false, the DiskBalancer service on the Datanode is disabled. Configure it to true for diskBalancer to be enabled.                                                                                                                                                                                                                   |
+| `hdds.datanode.disk.balancer.enabled`                          | `true`                                                                                 | If false, the DiskBalancer service on the Datanode is disabled. By default, DiskBalancer is enabled on datanodes.                                                                                                                                                                                                                    |
 | `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0`                                                                                 | A percentage (0-100). A datanode is considered balanced if for each volume, its utilization differs from the average datanode utilization by no more than this threshold.                                                                                                                                                              |
 | `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec`    | `10`                                                                                   | The maximum bandwidth (in MB/s) that the balancer can use for moving data, to avoid impacting client I/O.                                                                                                                                                                                                                              |
 | `hdds.datanode.disk.balancer.parallel.thread`                  | `5`                                                                                    | The number of worker threads to use for moving containers in parallel.                                                                                                                                                                                                                                                                 |

hadoop-hdds/docs/content/feature/DiskBalancer.zh.md

@@ -39,9 +39,9 @@ summary: 数据节点的磁盘平衡器.
 
 ## 功能标志
 
-磁盘平衡器功能已通过功能标志引入。默认情况下，此功能处于禁用状态。
+磁盘平衡器功能已通过功能标志引入。默认情况下，此功能处于启用状态。
 
-可以通过在“ozone-site.xml”配置文件中将以下属性设置为“true”来**启用**该功能：
+可以通过在“ozone-site.xml”配置文件中将以下属性设置为“false”来**禁用**该功能：
 `hdds.datanode.disk.balancer.enabled = false`
 
 ### 身份验证和授权
@@ -115,8 +115,8 @@ DiskBalancer 命令通过 RPC 直接与数据节点通信，因此需要进行
 ## 命令行用法
 DiskBalancer 通过 `ozone admin datanode diskbalancer` 命令进行管理。
 
-**注意：**此命令在主帮助信息（`ozone admin datanode --help`）中隐藏。这是因为该功能目前处于实验阶段，默认禁用。隐藏该命令可防止意外使用，
-并为普通用户提供清晰的帮助输出。但是，对于希望启用和使用该功能的用户，该命令仍然完全可用。
+**注意：**DiskBalancer 在数据节点上默认启用。在 `ozone-site.xml` 中使用 `hdds.datanode.disk.balancer.enabled=false`
+可禁用数据节点上的服务并阻止 CLI 命令运行。
 
 ### 命令语法
 **启动 DiskBalancer：**
@@ -233,7 +233,7 @@ The DiskBalancer's behavior can be controlled using the following configuration
 
 | Property                                                    | Default Value                          | Description                                                                                                                                                                 |
 |-------------------------------------------------------------|----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `hdds.datanode.disk.balancer.enabled`                       | `false`                                | 如果为 false，则 Datanode 上的 DiskBalancer 服务将被禁用。将其配置为 true 可启用 DiskBalancer。                                                      |                                                            |                                                                                        |                                                                                                                                                                              |
+| `hdds.datanode.disk.balancer.enabled`                       | `true`                                 | 如果为 false，则 Datanode 上的 DiskBalancer 服务将被禁用。默认情况下，DiskBalancer 在 Datanode 上启用。                                                      |                                                            |                                                                                        |                                                                                                                                                                              |
 | `hdds.datanode.disk.balancer.volume.density.threshold.percent` | `10.0`                                 | 百分比（0-100）。如果对于每个卷，其利用率与平均数据节点利用率之差不超过此阈值，则认为数据节点处于平衡状态。    |
 | `hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec` | `10`                                   | 平衡器可用于移动数据的最大带宽（以 MB/s 为单位），以避免影响客户端 I/O。                                                                    |
 | `hdds.datanode.disk.balancer.parallel.thread`               | `5`                                    | 用于并行移动容器的工作线程数。                                                                                                       |

hadoop-hdds/interface-client/src/main/proto/hdds.proto

@@ -591,6 +591,7 @@ message VolumeReportProto {
     optional uint64 committedBytes = 5;
     optional uint64 effectiveUsedSpace = 6;
     optional double utilization = 7;
+    optional uint64 ozoneAvailable = 8;
 }
 
 message DatanodeDiskBalancerInfoProto {

hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DatanodeParameters.java

@@ -29,10 +29,22 @@ public class DatanodeParameters extends ItemsFromStdin {
   @CommandLine.Spec
   private CommandLine.Model.CommandSpec spec;
 
-  @CommandLine.Parameters(description = "Datanode addresses: one or more, separated by spaces." +
-      " To read from stdin, specify '-' and supply one item per line." +
-      "Port is optional and defaults to 19864 (CLIENT_RPC port). " +
-      "Examples: 'DN-1', 'DN-1:19864', '192.168.1.10'. ",
+  @CommandLine.Parameters(
+      description = {
+          "Datanode addresses: one or more on the command line, OR read from stdin with '-'.",
+          "Stdin usage:",
+          "  ozone admin datanode diskbalancer <subcommand> -",
+          "  Then type one datanode per line and end input:",
+          "    - Linux/macOS: Ctrl-D",
+          "    - Windows:     Ctrl-Z, then Enter",
+          "Examples:",
+          "  # Piped (recommended for scripts)",
+          "  echo -e \"DN-1\\nDN-2\" | ozone admin datanode diskbalancer status -",
+          "  # From file having list of dns to balance",
+          "  ozone admin datanode diskbalancer report - < datanode-lists.txt",
+          "Port is optional and defaults to 19864 (CLIENT_RPC port).",
+          "Address examples: 'DN-1', 'DN-1:19864', '192.168.1.10'."
+      },
       arity = "0..*",
       paramLabel = "<datanode address>")
 

hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java

@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.hdds.scm.cli.datanode;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY;
+
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import picocli.CommandLine.Command;
 
@@ -155,11 +157,10 @@
 
 @Command(
     name = "diskbalancer",
-    description = "DiskBalancer specific operations. It is disabled by default." +
-        " To enable it, set 'hdds.datanode.disk.balancer.enabled' as true",
+    description = "DiskBalancer specific operations to ensure even disk utilization." +
+        " It is enabled by default. Set " + HDDS_DATANODE_DISK_BALANCER_ENABLED_KEY + " to false to disable.",
     mixinStandardHelpOptions = true,
     versionProvider = HddsVersionProvider.class,
-    hidden = true,
     subcommands = {
         DiskBalancerStartSubcommand.class,
         DiskBalancerStopSubcommand.class,