From e10af3759f85fcf07cea67e0fc98bba7e1094d22 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 29 Jul 2026 20:08:13 -0400 Subject: [PATCH 1/2] Impl and tests for finalization status enum --- .../src/main/proto/hdds.proto | 12 +++- .../scm/server/SCMClientProtocolServer.java | 16 ++++- .../server/TestSCMClientProtocolServer.java | 44 +++++++++++++- .../admin/upgrade/FinalizeSubCommand.java | 3 +- .../ozone/admin/upgrade/StatusSubCommand.java | 36 ++++++----- .../admin/upgrade/TestFinalizeSubCommand.java | 16 ++--- .../admin/upgrade/TestStatusSubCommand.java | 59 +++++++++++++++---- .../hdds/upgrade/HddsUpgradeTestUtils.java | 2 +- .../hadoop/ozone/om/OMUpgradeTestUtils.java | 8 +-- .../ozone/om/TestOMUpgradeFinalization.java | 41 +++++++++++++ .../src/main/proto/OmClientProtocol.proto | 6 +- .../apache/hadoop/ozone/om/OzoneManager.java | 27 +++++++-- .../om/upgrade/OMUpgradeFinalizeService.java | 2 +- .../upgrade/TestOMUpgradeFinalizeService.java | 16 ++--- .../TestOzoneManagerRequestHandler.java | 6 +- 15 files changed, 225 insertions(+), 69 deletions(-) diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index fe19f300089d..7f19264cb0b5 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -430,9 +430,17 @@ message UpgradeFinalizationStatus { repeated string messages = 2; } +// Finalization status of a component or group of components. +enum FinalizationStatus { + UNFINALIZED = 1; + FINALIZED = 2; + PENDING = 3; + IN_PROGRESS = 4; +} + message UpgradeStatus { - optional bool hddsFinalized = 1; - optional bool scmFinalized = 2; + optional FinalizationStatus hddsFinalizationStatus = 1; + optional FinalizationStatus scmFinalizationStatus = 2; optional int32 numDatanodesFinalized = 3; optional int32 numDatanodesTotal = 4; optional uint32 scmApparentVersion = 5; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 623016457006..d4bc7804af93 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -1224,13 +1224,23 @@ public HddsProtos.UpgradeStatus queryUpgradeStatus() throws IOException { scm.getScmNodeManager().getDatanodeFinalizationCounts(); int finalizedDatanodes = datanodeFinalizationCounts.getNumFinalizedDatanodes(); int healthyDatanodes = datanodeFinalizationCounts.getTotalHealthyDatanodes(); - boolean hddsFinalized = scmFinalized && datanodeFinalizationCounts.allNodesFinalized(); + + HddsProtos.FinalizationStatus scmFinalizationStatus = + scmFinalized ? HddsProtos.FinalizationStatus.FINALIZED : HddsProtos.FinalizationStatus.UNFINALIZED; + HddsProtos.FinalizationStatus hddsFinalizationStatus; + if (!scmFinalized) { + hddsFinalizationStatus = HddsProtos.FinalizationStatus.UNFINALIZED; + } else if (datanodeFinalizationCounts.allNodesFinalized()) { + hddsFinalizationStatus = HddsProtos.FinalizationStatus.FINALIZED; + } else { + hddsFinalizationStatus = HddsProtos.FinalizationStatus.IN_PROGRESS; + } HddsProtos.UpgradeStatus result = HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(scmFinalized) + .setScmFinalizationStatus(scmFinalizationStatus) .setNumDatanodesFinalized(finalizedDatanodes) .setNumDatanodesTotal(healthyDatanodes) - .setHddsFinalized(hddsFinalized) + .setHddsFinalizationStatus(hddsFinalizationStatus) .setScmApparentVersion(scm.getVersionManager().getApparentVersion().serialize()) .setMinDatanodeApparentVersion(datanodeFinalizationCounts.getMinApparentVersion()) .setMaxDatanodeApparentVersion(datanodeFinalizationCounts.getMaxApparentVersion()) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java index 8ac74e4b86bd..ae0c226f91ec 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java @@ -39,6 +39,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import org.apache.hadoop.hdds.ComponentVersion; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.ReconfigurationHandler; @@ -49,10 +50,12 @@ import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl; +import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.ha.SCMHAManagerStub; import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails; +import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocolServerSideTranslatorPB; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; @@ -281,11 +284,48 @@ public void testQueryUpgradeStatus() throws Exception { HddsProtos.UpgradeStatus status = server.queryUpgradeStatus(); // SCM starts already finalized in tests - assertTrue(status.getScmFinalized()); + assertEquals(HddsProtos.FinalizationStatus.FINALIZED, status.getScmFinalizationStatus()); // No datanodes registered assertEquals(0, status.getNumDatanodesFinalized()); assertEquals(0, status.getNumDatanodesTotal()); - assertTrue(status.getHddsFinalized()); + assertEquals(HddsProtos.FinalizationStatus.FINALIZED, status.getHddsFinalizationStatus()); + } + + @Test + public void testQueryUpgradeStatusHddsInProgress() throws Exception { + // SCM is finalized but not all datanodes are, so HDDS is only partially finalized. + ScmVersionManager mockVersionManager = mock(ScmVersionManager.class); + when(mockVersionManager.needsFinalization()).thenReturn(false); + ComponentVersion apparentVersion = mock(ComponentVersion.class); + when(apparentVersion.serialize()).thenReturn(0); + when(mockVersionManager.getApparentVersion()).thenReturn(apparentVersion); + + NodeManager mockNodeManager = new MockNodeManager(false, 0) { + @Override + public DatanodeFinalizationCounts getDatanodeFinalizationCounts() { + return DatanodeFinalizationCounts.newBuilder() + .setNumFinalizedDatanodes(1) + .setTotalHealthyDatanodes(3) + .build(); + } + }; + + StorageContainerManager mockScm = mockStorageContainerManager(); + when(mockScm.getVersionManager()).thenReturn(mockVersionManager); + when(mockScm.getScmNodeManager()).thenReturn(mockNodeManager); + when(mockScm.getScmContext()).thenReturn(SCMContext.emptyContext()); + + SCMClientProtocolServer testServer = new SCMClientProtocolServer( + new OzoneConfiguration(), mockScm, mock(ReconfigurationHandler.class)); + try { + HddsProtos.UpgradeStatus status = testServer.queryUpgradeStatus(); + assertEquals(HddsProtos.FinalizationStatus.FINALIZED, status.getScmFinalizationStatus()); + assertEquals(HddsProtos.FinalizationStatus.IN_PROGRESS, status.getHddsFinalizationStatus()); + assertEquals(1, status.getNumDatanodesFinalized()); + assertEquals(3, status.getNumDatanodesTotal()); + } finally { + testServer.stop(); + } } @Test diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java index 232f3347e9e3..08c4a6d7aa03 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/FinalizeSubCommand.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.cli.AbstractSubcommand; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.FinalizationStatus; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.admin.om.OmAddressOptions; import org.apache.hadoop.ozone.client.rpc.RpcClient; @@ -91,7 +92,7 @@ private int waitForFinalization(OzoneManagerProtocol client) { if (status != null) { // Finalization checks before sleeping, so an already-finalized cluster returns without waiting. - if (status.getClusterFinalized()) { + if (status.getClusterFinalizationStatus() == FinalizationStatus.FINALIZED) { out().println("Finalization complete."); return 0; } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java index fa1ef06bb586..d86273207495 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/upgrade/StatusSubCommand.java @@ -75,9 +75,10 @@ public Integer call() throws Exception { /** Basic, non-verbose human-readable status. */ static void printBasic(QueryUpgradeStatusResponse status, PrintWriter out) { - out.println("Upgrade status:"); - out.println(" OM Finalized? " + status.getOmFinalized()); - out.println(" SCM Finalized? " + status.getHddsStatus().getScmFinalized()); + out.println("Upgrade finalization status:"); + out.println(" Cluster: " + status.getClusterFinalizationStatus().name()); + out.println(" OM: " + status.getOmFinalizationStatus().name()); + out.println(" SCM: " + status.getHddsStatus().getScmFinalizationStatus().name()); out.println(" Datanodes finalized: " + status.getHddsStatus().getNumDatanodesFinalized() + "/" + status.getHddsStatus().getNumDatanodesTotal()); } @@ -88,11 +89,12 @@ static void printBasic(QueryUpgradeStatusResponse status, PrintWriter out) { */ static void printVerbose(QueryUpgradeStatusResponse status, PrintWriter out) { HddsProtos.UpgradeStatus hdds = status.getHddsStatus(); - out.println("Upgrade status:"); - out.println(" OM Finalized? " + status.getOmFinalized()); + out.println("Upgrade finalization status:"); + out.println(" Cluster: " + status.getClusterFinalizationStatus().name()); + out.println(" OM: " + status.getOmFinalizationStatus().name()); out.println(" OM Apparent Version: " + OzoneManagerVersion.deserialize(status.getOmApparentVersion()).toString()); - out.println(" SCM Finalized? " + hdds.getScmFinalized()); + out.println(" SCM: " + hdds.getScmFinalizationStatus().name()); out.println(" SCM Apparent Version: " + HDDSVersion.deserialize(hdds.getScmApparentVersion()).toString()); out.println(" Datanodes finalized: " + hdds.getNumDatanodesFinalized() + "/" + hdds.getNumDatanodesTotal()); out.println(" Min Datanode Apparent Version: " @@ -109,9 +111,10 @@ protected OzoneManagerProtocol getClient() throws Exception { * JSON-friendly DTO mirroring {@link QueryUpgradeStatusResponse}. */ public static final class UpgradeStatusDto { - private boolean omFinalized; + private String clusterFinalizationStatus; + private String omFinalizationStatus; private String omApparentVersion; - private boolean scmFinalized; + private String scmFinalizationStatus; private String scmApparentVersion; private int datanodesFinalized; private int datanodesTotal; @@ -121,8 +124,9 @@ public static final class UpgradeStatusDto { public static UpgradeStatusDto from(QueryUpgradeStatusResponse status) { HddsProtos.UpgradeStatus hdds = status.getHddsStatus(); UpgradeStatusDto dto = new UpgradeStatusDto(); - dto.omFinalized = status.getOmFinalized(); - dto.scmFinalized = hdds.getScmFinalized(); + dto.clusterFinalizationStatus = status.getClusterFinalizationStatus().name(); + dto.omFinalizationStatus = status.getOmFinalizationStatus().name(); + dto.scmFinalizationStatus = hdds.getScmFinalizationStatus().name(); dto.datanodesFinalized = hdds.getNumDatanodesFinalized(); dto.datanodesTotal = hdds.getNumDatanodesTotal(); dto.omApparentVersion = OzoneManagerVersion.deserialize(status.getOmApparentVersion()).toString(); @@ -132,16 +136,20 @@ public static UpgradeStatusDto from(QueryUpgradeStatusResponse status) { return dto; } - public boolean isOmFinalized() { - return omFinalized; + public String getClusterFinalizationStatus() { + return clusterFinalizationStatus; + } + + public String getOmFinalizationStatus() { + return omFinalizationStatus; } public String getOmApparentVersion() { return omApparentVersion; } - public boolean isScmFinalized() { - return scmFinalized; + public String getScmFinalizationStatus() { + return scmFinalizationStatus; } public String getScmApparentVersion() { diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java index 056ce3b3b0c8..c9319078ef9a 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java @@ -286,8 +286,8 @@ public void testWaitFlagWithVerbosePrintsFullStatus() throws Exception { assertEquals(0, cmd.call()); String output = outContent.toString(DEFAULT_ENCODING); - assertTrue(output.contains("OM Finalized?")); - assertTrue(output.contains("SCM Finalized?")); + assertTrue(output.contains("OM:")); + assertTrue(output.contains("SCM:")); assertTrue(output.contains("OM Apparent Version:")); assertTrue(output.contains("SCM Apparent Version:")); assertTrue(output.contains("Min Datanode Apparent Version:")); @@ -297,10 +297,10 @@ public void testWaitFlagWithVerbosePrintsFullStatus() throws Exception { private static QueryUpgradeStatusResponse inProgressStatus(int dnFinalized, int dnTotal) { return QueryUpgradeStatusResponse.newBuilder() - .setOmFinalized(false) - .setClusterFinalized(false) + .setOmFinalizationStatus(HddsProtos.FinalizationStatus.UNFINALIZED) + .setClusterFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS) .setHddsStatus(HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(false) + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.UNFINALIZED) .setNumDatanodesFinalized(dnFinalized) .setNumDatanodesTotal(dnTotal) .build()) @@ -309,10 +309,10 @@ private static QueryUpgradeStatusResponse inProgressStatus(int dnFinalized, int private static QueryUpgradeStatusResponse finalizedStatus(int dnFinalized, int dnTotal) { return QueryUpgradeStatusResponse.newBuilder() - .setOmFinalized(true) - .setClusterFinalized(true) + .setOmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) + .setClusterFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setHddsStatus(HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(true) + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setNumDatanodesFinalized(dnFinalized) .setNumDatanodesTotal(dnTotal) .build()) diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java index 76ff43998e18..a70eecc67204 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java @@ -91,15 +91,16 @@ public void tearDown() { @Test public void testStatusCommandPrintsUpgradeStatus() throws Exception { HddsProtos.UpgradeStatus hddsStatus = HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(false) + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setNumDatanodesFinalized(1) .setNumDatanodesTotal(3) - .setHddsFinalized(true) + .setHddsFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS) .build(); OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response = OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder() - .setOmFinalized(false) + .setOmFinalizationStatus(HddsProtos.FinalizationStatus.UNFINALIZED) + .setClusterFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS) .setHddsStatus(hddsStatus) .build(); @@ -109,14 +110,43 @@ public void testStatusCommandPrintsUpgradeStatus() throws Exception { String output = outContent.toString(DEFAULT_ENCODING); assertTrue(output.contains("Upgrade status")); - assertTrue(output.contains("OM Finalized? false")); - assertTrue(output.contains("SCM Finalized? false")); + assertTrue(output.contains("Cluster: IN_PROGRESS")); + assertTrue(output.contains("OM: UNFINALIZED")); + assertTrue(output.contains("SCM: FINALIZED")); assertTrue(output.contains("Datanodes finalized: 1/3")); // Without --verbose the apparent versions are not shown. assertFalse(output.contains("Apparent Version")); verify(omClient).queryUpgradeStatus(); } + @Test + public void testStatusCommandPrintsPendingOm() throws Exception { + // OM has begun polling SCM (marker present) but is not finalized yet. + HddsProtos.UpgradeStatus hddsStatus = HddsProtos.UpgradeStatus.newBuilder() + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) + .setNumDatanodesFinalized(3) + .setNumDatanodesTotal(3) + .setHddsFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) + .build(); + + OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response = + OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder() + .setOmFinalizationStatus(HddsProtos.FinalizationStatus.PENDING) + .setClusterFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS) + .setHddsStatus(hddsStatus) + .build(); + + when(omClient.queryUpgradeStatus()).thenReturn(response); + new CommandLine(cmd).parseArgs(); + cmd.call(); + + String output = outContent.toString(DEFAULT_ENCODING); + assertTrue(output.contains("OM: PENDING")); + assertTrue(output.contains("Cluster: IN_PROGRESS")); + assertTrue(output.contains("SCM: FINALIZED")); + verify(omClient).queryUpgradeStatus(); + } + @Test public void testStatusCommandPropagatesException() throws Exception { when(omClient.queryUpgradeStatus()).thenThrow(new IOException("OM unavailable")); @@ -142,10 +172,11 @@ public void testJsonOutput() throws Exception { int hddsVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response = OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder() - .setOmFinalized(true) + .setOmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) + .setClusterFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setOmApparentVersion(omVersion) .setHddsStatus(HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(true) + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setNumDatanodesFinalized(2) .setNumDatanodesTotal(3) .setScmApparentVersion(hddsVersion) @@ -161,8 +192,9 @@ public void testJsonOutput() throws Exception { String jsonOutput = outContent.toString(DEFAULT_ENCODING); JsonNode root = JSON.readTree(jsonOutput); - assertTrue(root.path("omFinalized").asBoolean()); - assertTrue(root.path("scmFinalized").asBoolean()); + assertEquals("FINALIZED", root.path("clusterFinalizationStatus").asText()); + assertEquals("FINALIZED", root.path("omFinalizationStatus").asText()); + assertEquals("FINALIZED", root.path("scmFinalizationStatus").asText()); assertEquals(2, root.path("datanodesFinalized").asInt()); assertEquals(3, root.path("datanodesTotal").asInt()); assertEquals(OzoneManagerVersion.ZDU.toString(), root.path("omApparentVersion").asText()); @@ -185,10 +217,11 @@ public void testVerboseTextOutputIncludesVersions() throws Exception { int hddsVersion = HDDSVersion.SOFTWARE_VERSION.serialize(); OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response = OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder() - .setOmFinalized(true) + .setOmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) + .setClusterFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setOmApparentVersion(omVersion) .setHddsStatus(HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(true) + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setNumDatanodesFinalized(3) .setNumDatanodesTotal(3) .setScmApparentVersion(hddsVersion) @@ -202,9 +235,9 @@ public void testVerboseTextOutputIncludesVersions() throws Exception { assertEquals(0, cmd.call()); String output = outContent.toString(DEFAULT_ENCODING); - assertTrue(output.contains("OM Finalized?")); + assertTrue(output.contains("OM:")); assertTrue(output.contains("OM Apparent Version:")); - assertTrue(output.contains("SCM Finalized?")); + assertTrue(output.contains("SCM:")); assertTrue(output.contains("SCM Apparent Version:")); assertTrue(output.contains("Min Datanode Apparent Version:")); assertTrue(output.contains("Max Datanode Apparent Version:")); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java index 04defd997745..7fcc4cd53a58 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java @@ -51,7 +51,7 @@ public static void waitForFinalizationFromClient(StorageContainerLocationProtoco LambdaTestUtils.await(60_000, 1_000, () -> { HddsProtos.UpgradeStatus status = scmClient.queryUpgradeStatus(); LOG.info("Waiting for upgrade finalization to complete from client. Current status is:\n{}", status); - return status.getHddsFinalized(); + return status.getHddsFinalizationStatus() == HddsProtos.FinalizationStatus.FINALIZED; }); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java index 675bfdcac67a..e32392830647 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java @@ -45,12 +45,10 @@ public static void waitForFinalization(OzoneManagerProtocol omClient) try { QueryUpgradeStatusResponse status = omClient.queryUpgradeStatus(); HddsProtos.UpgradeStatus hdds = status.getHddsStatus(); - LOG.info("Finalization status: omFinalized={}, scmFinalized={}, datanodes={}/{}", - status.getOmFinalized(), hdds.getScmFinalized(), + LOG.info("Finalization status: om={}, scm={}, datanodes={}/{}", + status.getOmFinalizationStatus(), hdds.getScmFinalizationStatus(), hdds.getNumDatanodesFinalized(), hdds.getNumDatanodesTotal()); - return status.getOmFinalized() - && hdds.getScmFinalized() - && hdds.getNumDatanodesFinalized() == hdds.getNumDatanodesTotal(); + return status.getClusterFinalizationStatus() == HddsProtos.FinalizationStatus.FINALIZED; } catch (IOException e) { fail(e.getMessage()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index c2b7d0d3b736..08aab50f4fd6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om; import static org.apache.hadoop.ozone.OzoneConsts.APPARENT_VERSION_KEY; +import static org.apache.hadoop.ozone.OzoneConsts.FINALIZATION_IN_PROGRESS_KEY; import static org.apache.hadoop.ozone.om.TestOzoneManagerHAWithStoppedNodes.createKey; import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION; import static org.apache.ozone.test.GenericTestUtils.waitFor; @@ -31,6 +32,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.OzoneManagerVersion; @@ -163,6 +165,45 @@ void testFinalizationFromSnapshot() throws Exception { } } + /** + * OM's reported finalization status should move UNFINALIZED -> PENDING (once the in-progress + * marker is present) -> FINALIZED. + */ + @Test + void testOmFinalizationStatusTransitions() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OMConfigKeys.OZONE_OM_UPGRADE_FINALIZATION_CHECK_INTERVAL, "10ms"); + conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, INITIAL_VERSION.serialize()); + + try (MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(1) + .build()) { + cluster.waitForClusterToBeReady(); + OzoneManager om = cluster.getOzoneManager(); + + try (OzoneClient client = cluster.newClient()) { + OzoneManagerProtocol omClient = client.getObjectStore().getClientProxy().getOzoneManagerClient(); + + // Before finalization: no in-progress marker, OM reports UNFINALIZED. + assertNull(om.getMetadataManager().getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY)); + assertEquals(HddsProtos.FinalizationStatus.UNFINALIZED, + omClient.queryUpgradeStatus().getOmFinalizationStatus()); + + // With the in-progress marker present but the OM not yet finalized, OM reports PENDING. + om.getMetadataManager().getMetaTable().put(FINALIZATION_IN_PROGRESS_KEY, "ignored"); + assertEquals(HddsProtos.FinalizationStatus.PENDING, + omClient.queryUpgradeStatus().getOmFinalizationStatus()); + om.getMetadataManager().getMetaTable().delete(FINALIZATION_IN_PROGRESS_KEY); + + // After finalization completes, OM reports FINALIZED. + omClient.finalizeUpgrade(); + OMUpgradeTestUtils.waitForFinalization(omClient); + assertEquals(HddsProtos.FinalizationStatus.FINALIZED, + omClient.queryUpgradeStatus().getOmFinalizationStatus()); + } + } + } + private static MiniOzoneHAClusterImpl newCluster(OzoneConfiguration conf) throws IOException { conf.setInt(OMStorage.TESTING_INIT_APPARENT_VERSION_KEY, INITIAL_VERSION.serialize()); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index fec82cdda363..1349b646f956 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1666,10 +1666,10 @@ message QueryUpgradeStatusRequest { } message QueryUpgradeStatusResponse { - // True when OM, SCM and all healthy datanodes are finalized - optional bool clusterFinalized = 1; + // Aggregate status of OM, SCM and all healthy datanodes + optional hadoop.hdds.FinalizationStatus clusterFinalizationStatus = 1; optional hadoop.hdds.UpgradeStatus hddsStatus = 2; - optional bool omFinalized = 3; + optional hadoop.hdds.FinalizationStatus omFinalizationStatus = 3; optional uint32 omApparentVersion = 4; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 012e4e857a90..16543745c414 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3676,15 +3676,32 @@ public QueryUpgradeStatusResponse queryUpgradeStatus() throws IOException { throw e; } - boolean omFinalized = !versionManager.needsFinalization(); - boolean hddsFinalized = scmStatus.getHddsFinalized(); - boolean clusterFinalized = omFinalized && hddsFinalized; + HddsProtos.FinalizationStatus omFinalizationStatus; + if (!versionManager.needsFinalization()) { + omFinalizationStatus = HddsProtos.FinalizationStatus.FINALIZED; + } else if (metadataManager.getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY) != null) { + omFinalizationStatus = HddsProtos.FinalizationStatus.PENDING; + } else { + omFinalizationStatus = HddsProtos.FinalizationStatus.UNFINALIZED; + } + HddsProtos.FinalizationStatus hddsFinalizationStatus = scmStatus.getHddsFinalizationStatus(); + + HddsProtos.FinalizationStatus clusterFinalizationStatus; + if (omFinalizationStatus == HddsProtos.FinalizationStatus.FINALIZED + && hddsFinalizationStatus == HddsProtos.FinalizationStatus.FINALIZED) { + clusterFinalizationStatus = HddsProtos.FinalizationStatus.FINALIZED; + } else if (omFinalizationStatus == HddsProtos.FinalizationStatus.UNFINALIZED + && hddsFinalizationStatus == HddsProtos.FinalizationStatus.UNFINALIZED) { + clusterFinalizationStatus = HddsProtos.FinalizationStatus.UNFINALIZED; + } else { + clusterFinalizationStatus = HddsProtos.FinalizationStatus.IN_PROGRESS; + } return QueryUpgradeStatusResponse.newBuilder() - .setOmFinalized(omFinalized) + .setOmFinalizationStatus(omFinalizationStatus) .setHddsStatus(scmStatus) .setOmApparentVersion(versionManager.getApparentVersion().serialize()) - .setClusterFinalized(clusterFinalized) + .setClusterFinalizationStatus(clusterFinalizationStatus) .build(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java index 00cc567bf3d1..f45f963a45e1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMUpgradeFinalizeService.java @@ -117,7 +117,7 @@ public BackgroundTaskResult call() { } HddsProtos.UpgradeStatus upgradeStatus = scmClient.getContainerClient().queryUpgradeStatus(); - if (upgradeStatus.getHddsFinalized()) { + if (upgradeStatus.getHddsFinalizationStatus() == HddsProtos.FinalizationStatus.FINALIZED) { LOG.info("The SCM Upgrade has been finalized. OM will now finalize. Run count {}", run); OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java index ef0f6ef6eb85..b6ae9557459d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMUpgradeFinalizeService.java @@ -125,7 +125,7 @@ void testNoTasksSubmittedWhenFinalizationNotNeeded() throws Exception { /** * When the OM is the leader, finalization is needed, the finalization command is given and SCM reports - * hddsFinalized=true, a FinalizeUpgrade request should be submitted via Ratis. + * hddsFinalizationStatus=FINALIZED, a FinalizeUpgrade request should be submitted via Ratis. */ @Test void testFinalizationTriggeredWhenScmIsFinalizedAndFinalizationInProgress() throws Exception { @@ -133,8 +133,8 @@ void testFinalizationTriggeredWhenScmIsFinalizedAndFinalizationInProgress() thro when(versionManager.needsFinalization()).thenReturn(true); HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(true) - .setHddsFinalized(true) + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) + .setHddsFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setNumDatanodesFinalized(3) .setNumDatanodesTotal(3) .build(); @@ -156,7 +156,7 @@ void testFinalizationTriggeredWhenScmIsFinalizedAndFinalizationInProgress() thro } /** - * When SCM reports hddsFinalized=false (SCM is not yet finalized), + * When SCM reports hddsFinalizationStatus=UNFINALIZED (SCM is not yet finalized), * no Ratis request should be submitted. */ @Test @@ -165,8 +165,8 @@ void testFinalizationSkippedWhenScmNotYetFinalized() throws Exception { when(versionManager.needsFinalization()).thenReturn(true); HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(false) - .setHddsFinalized(false) + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.UNFINALIZED) + .setHddsFinalizationStatus(HddsProtos.FinalizationStatus.UNFINALIZED) .setNumDatanodesFinalized(0) .setNumDatanodesTotal(3) .build(); @@ -247,8 +247,8 @@ void testExceptionFromRatisSubmitIsHandledGracefully() throws Exception { when(versionManager.needsFinalization()).thenReturn(true); HddsProtos.UpgradeStatus scmStatus = HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(true) - .setHddsFinalized(true) + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) + .setHddsFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setNumDatanodesFinalized(3) .setNumDatanodesTotal(3) .build(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java index 764a82866484..622134889c6b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/protocolPB/TestOzoneManagerRequestHandler.java @@ -475,14 +475,14 @@ public void testQueryUpgradeStatusDispatch() throws IOException { OzoneManager ozoneManager = handler.getOzoneManager(); HddsProtos.UpgradeStatus hddsStatus = HddsProtos.UpgradeStatus.newBuilder() - .setScmFinalized(true) - .setHddsFinalized(false) + .setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) + .setHddsFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS) .setNumDatanodesFinalized(3) .setNumDatanodesTotal(3) .build(); OzoneManagerProtocolProtos.QueryUpgradeStatusResponse expected = OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder() - .setOmFinalized(true) + .setOmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) .setHddsStatus(hddsStatus) .build(); Mockito.when(ozoneManager.queryUpgradeStatus()).thenReturn(expected); From a70a45990baf92e80489c307d31738f2cec5f8fc Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 30 Jul 2026 11:44:18 -0400 Subject: [PATCH 2/2] Remove pending status, fix test assertion messages --- hadoop-hdds/interface-client/src/main/proto/hdds.proto | 3 +-- .../ozone/admin/upgrade/TestFinalizeSubCommand.java | 2 +- .../hadoop/ozone/admin/upgrade/TestStatusSubCommand.java | 8 ++++---- .../apache/hadoop/ozone/om/TestOMUpgradeFinalization.java | 6 +++--- .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 2 +- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto index 7f19264cb0b5..d3074d371b27 100644 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@ -434,8 +434,7 @@ message UpgradeFinalizationStatus { enum FinalizationStatus { UNFINALIZED = 1; FINALIZED = 2; - PENDING = 3; - IN_PROGRESS = 4; + IN_PROGRESS = 3; } message UpgradeStatus { diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java index c9319078ef9a..cb5010629e0b 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestFinalizeSubCommand.java @@ -167,7 +167,7 @@ public void testWaitFlagPollsUntilFinalized() throws Exception { String output = outContent.toString(DEFAULT_ENCODING); // While polling, the shared basic status output is printed on each in-progress poll. - assertTrue(output.contains("Upgrade status:")); + assertTrue(output.contains("Upgrade finalization status:")); assertTrue(output.contains("Finalization complete.")); verify(omClient, times(3)).queryUpgradeStatus(); } diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java index a70eecc67204..9f28b602bf18 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/ozone/admin/upgrade/TestStatusSubCommand.java @@ -109,7 +109,7 @@ public void testStatusCommandPrintsUpgradeStatus() throws Exception { cmd.call(); String output = outContent.toString(DEFAULT_ENCODING); - assertTrue(output.contains("Upgrade status")); + assertTrue(output.contains("Upgrade finalization status")); assertTrue(output.contains("Cluster: IN_PROGRESS")); assertTrue(output.contains("OM: UNFINALIZED")); assertTrue(output.contains("SCM: FINALIZED")); @@ -120,7 +120,7 @@ public void testStatusCommandPrintsUpgradeStatus() throws Exception { } @Test - public void testStatusCommandPrintsPendingOm() throws Exception { + public void testStatusCommandPrintsInProgressOm() throws Exception { // OM has begun polling SCM (marker present) but is not finalized yet. HddsProtos.UpgradeStatus hddsStatus = HddsProtos.UpgradeStatus.newBuilder() .setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED) @@ -131,7 +131,7 @@ public void testStatusCommandPrintsPendingOm() throws Exception { OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response = OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder() - .setOmFinalizationStatus(HddsProtos.FinalizationStatus.PENDING) + .setOmFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS) .setClusterFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS) .setHddsStatus(hddsStatus) .build(); @@ -141,7 +141,7 @@ public void testStatusCommandPrintsPendingOm() throws Exception { cmd.call(); String output = outContent.toString(DEFAULT_ENCODING); - assertTrue(output.contains("OM: PENDING")); + assertTrue(output.contains("OM: IN_PROGRESS")); assertTrue(output.contains("Cluster: IN_PROGRESS")); assertTrue(output.contains("SCM: FINALIZED")); verify(omClient).queryUpgradeStatus(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index 08aab50f4fd6..12fd48c95615 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -166,7 +166,7 @@ void testFinalizationFromSnapshot() throws Exception { } /** - * OM's reported finalization status should move UNFINALIZED -> PENDING (once the in-progress + * OM's reported finalization status should move UNFINALIZED -> IN_PROGRESS (once the in-progress * marker is present) -> FINALIZED. */ @Test @@ -189,9 +189,9 @@ void testOmFinalizationStatusTransitions() throws Exception { assertEquals(HddsProtos.FinalizationStatus.UNFINALIZED, omClient.queryUpgradeStatus().getOmFinalizationStatus()); - // With the in-progress marker present but the OM not yet finalized, OM reports PENDING. + // With the in-progress marker present but the OM not yet finalized, OM reports IN_PROGRESS. om.getMetadataManager().getMetaTable().put(FINALIZATION_IN_PROGRESS_KEY, "ignored"); - assertEquals(HddsProtos.FinalizationStatus.PENDING, + assertEquals(HddsProtos.FinalizationStatus.IN_PROGRESS, omClient.queryUpgradeStatus().getOmFinalizationStatus()); om.getMetadataManager().getMetaTable().delete(FINALIZATION_IN_PROGRESS_KEY); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 16543745c414..4d4542c30cdd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3680,7 +3680,7 @@ public QueryUpgradeStatusResponse queryUpgradeStatus() throws IOException { if (!versionManager.needsFinalization()) { omFinalizationStatus = HddsProtos.FinalizationStatus.FINALIZED; } else if (metadataManager.getMetaTable().get(FINALIZATION_IN_PROGRESS_KEY) != null) { - omFinalizationStatus = HddsProtos.FinalizationStatus.PENDING; + omFinalizationStatus = HddsProtos.FinalizationStatus.IN_PROGRESS; } else { omFinalizationStatus = HddsProtos.FinalizationStatus.UNFINALIZED; }