Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions hadoop-hdds/interface-client/src/main/proto/hdds.proto
Original file line number Diff line number Diff line change
Expand Up @@ -430,9 +430,17 @@ message UpgradeFinalizationStatus {
repeated string messages = 2;
}

// Finalization status of a component or group of components.
enum FinalizationStatus {
UNFINALIZED = 1;
FINALIZED = 2;
PENDING = 3;
IN_PROGRESS = 4;
}

message UpgradeStatus {
optional bool hddsFinalized = 1;
optional bool scmFinalized = 2;
optional FinalizationStatus hddsFinalizationStatus = 1;
optional FinalizationStatus scmFinalizationStatus = 2;
optional int32 numDatanodesFinalized = 3;
optional int32 numDatanodesTotal = 4;
optional uint32 scmApparentVersion = 5;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1224,13 +1224,23 @@ public HddsProtos.UpgradeStatus queryUpgradeStatus() throws IOException {
scm.getScmNodeManager().getDatanodeFinalizationCounts();
int finalizedDatanodes = datanodeFinalizationCounts.getNumFinalizedDatanodes();
int healthyDatanodes = datanodeFinalizationCounts.getTotalHealthyDatanodes();
boolean hddsFinalized = scmFinalized && datanodeFinalizationCounts.allNodesFinalized();

HddsProtos.FinalizationStatus scmFinalizationStatus =
scmFinalized ? HddsProtos.FinalizationStatus.FINALIZED : HddsProtos.FinalizationStatus.UNFINALIZED;
HddsProtos.FinalizationStatus hddsFinalizationStatus;
if (!scmFinalized) {
hddsFinalizationStatus = HddsProtos.FinalizationStatus.UNFINALIZED;
} else if (datanodeFinalizationCounts.allNodesFinalized()) {
hddsFinalizationStatus = HddsProtos.FinalizationStatus.FINALIZED;
} else {
hddsFinalizationStatus = HddsProtos.FinalizationStatus.IN_PROGRESS;
}

HddsProtos.UpgradeStatus result = HddsProtos.UpgradeStatus.newBuilder()
.setScmFinalized(scmFinalized)
.setScmFinalizationStatus(scmFinalizationStatus)
.setNumDatanodesFinalized(finalizedDatanodes)
.setNumDatanodesTotal(healthyDatanodes)
.setHddsFinalized(hddsFinalized)
.setHddsFinalizationStatus(hddsFinalizationStatus)
.setScmApparentVersion(scm.getVersionManager().getApparentVersion().serialize())
.setMinDatanodeApparentVersion(datanodeFinalizationCounts.getMinApparentVersion())
.setMaxDatanodeApparentVersion(datanodeFinalizationCounts.getMaxApparentVersion())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import org.apache.hadoop.hdds.ComponentVersion;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
Expand All @@ -49,10 +50,12 @@
import org.apache.hadoop.hdds.scm.HddsTestUtils;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl;
import org.apache.hadoop.hdds.scm.container.MockNodeManager;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMHAManagerStub;
import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager;
Expand Down Expand Up @@ -281,11 +284,48 @@ public void testQueryUpgradeStatus() throws Exception {
HddsProtos.UpgradeStatus status = server.queryUpgradeStatus();

// SCM starts already finalized in tests
assertTrue(status.getScmFinalized());
assertEquals(HddsProtos.FinalizationStatus.FINALIZED, status.getScmFinalizationStatus());
// No datanodes registered
assertEquals(0, status.getNumDatanodesFinalized());
assertEquals(0, status.getNumDatanodesTotal());
assertTrue(status.getHddsFinalized());
assertEquals(HddsProtos.FinalizationStatus.FINALIZED, status.getHddsFinalizationStatus());
}

@Test
public void testQueryUpgradeStatusHddsInProgress() throws Exception {
// SCM is finalized but not all datanodes are, so HDDS is only partially finalized.
ScmVersionManager mockVersionManager = mock(ScmVersionManager.class);
when(mockVersionManager.needsFinalization()).thenReturn(false);
ComponentVersion apparentVersion = mock(ComponentVersion.class);
when(apparentVersion.serialize()).thenReturn(0);
when(mockVersionManager.getApparentVersion()).thenReturn(apparentVersion);

NodeManager mockNodeManager = new MockNodeManager(false, 0) {
@Override
public DatanodeFinalizationCounts getDatanodeFinalizationCounts() {
return DatanodeFinalizationCounts.newBuilder()
.setNumFinalizedDatanodes(1)
.setTotalHealthyDatanodes(3)
.build();
}
};

StorageContainerManager mockScm = mockStorageContainerManager();
when(mockScm.getVersionManager()).thenReturn(mockVersionManager);
when(mockScm.getScmNodeManager()).thenReturn(mockNodeManager);
when(mockScm.getScmContext()).thenReturn(SCMContext.emptyContext());

SCMClientProtocolServer testServer = new SCMClientProtocolServer(
new OzoneConfiguration(), mockScm, mock(ReconfigurationHandler.class));
try {
HddsProtos.UpgradeStatus status = testServer.queryUpgradeStatus();
assertEquals(HddsProtos.FinalizationStatus.FINALIZED, status.getScmFinalizationStatus());
assertEquals(HddsProtos.FinalizationStatus.IN_PROGRESS, status.getHddsFinalizationStatus());
assertEquals(1, status.getNumDatanodesFinalized());
assertEquals(3, status.getNumDatanodesTotal());
} finally {
testServer.stop();
}
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.cli.AbstractSubcommand;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.FinalizationStatus;
import org.apache.hadoop.ozone.OzoneManagerVersion;
import org.apache.hadoop.ozone.admin.om.OmAddressOptions;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
Expand Down Expand Up @@ -91,7 +92,7 @@ private int waitForFinalization(OzoneManagerProtocol client) {

if (status != null) {
// Finalization checks before sleeping, so an already-finalized cluster returns without waiting.
if (status.getClusterFinalized()) {
if (status.getClusterFinalizationStatus() == FinalizationStatus.FINALIZED) {
out().println("Finalization complete.");
return 0;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,10 @@ public Integer call() throws Exception {

/** Basic, non-verbose human-readable status. */
static void printBasic(QueryUpgradeStatusResponse status, PrintWriter out) {
out.println("Upgrade status:");
out.println(" OM Finalized? " + status.getOmFinalized());
out.println(" SCM Finalized? " + status.getHddsStatus().getScmFinalized());
out.println("Upgrade finalization status:");
out.println(" Cluster: " + status.getClusterFinalizationStatus().name());
out.println(" OM: " + status.getOmFinalizationStatus().name());
out.println(" SCM: " + status.getHddsStatus().getScmFinalizationStatus().name());
out.println(" Datanodes finalized: " + status.getHddsStatus().getNumDatanodesFinalized()
+ "/" + status.getHddsStatus().getNumDatanodesTotal());
}
Expand All @@ -88,11 +89,12 @@ static void printBasic(QueryUpgradeStatusResponse status, PrintWriter out) {
*/
static void printVerbose(QueryUpgradeStatusResponse status, PrintWriter out) {
HddsProtos.UpgradeStatus hdds = status.getHddsStatus();
out.println("Upgrade status:");
out.println(" OM Finalized? " + status.getOmFinalized());
out.println("Upgrade finalization status:");
out.println(" Cluster: " + status.getClusterFinalizationStatus().name());
out.println(" OM: " + status.getOmFinalizationStatus().name());
out.println(" OM Apparent Version: "
+ OzoneManagerVersion.deserialize(status.getOmApparentVersion()).toString());
out.println(" SCM Finalized? " + hdds.getScmFinalized());
out.println(" SCM: " + hdds.getScmFinalizationStatus().name());
out.println(" SCM Apparent Version: " + HDDSVersion.deserialize(hdds.getScmApparentVersion()).toString());
out.println(" Datanodes finalized: " + hdds.getNumDatanodesFinalized() + "/" + hdds.getNumDatanodesTotal());
out.println(" Min Datanode Apparent Version: "
Expand All @@ -109,9 +111,10 @@ protected OzoneManagerProtocol getClient() throws Exception {
* JSON-friendly DTO mirroring {@link QueryUpgradeStatusResponse}.
*/
public static final class UpgradeStatusDto {
private boolean omFinalized;
private String clusterFinalizationStatus;
private String omFinalizationStatus;
private String omApparentVersion;
private boolean scmFinalized;
private String scmFinalizationStatus;
private String scmApparentVersion;
private int datanodesFinalized;
private int datanodesTotal;
Expand All @@ -121,8 +124,9 @@ public static final class UpgradeStatusDto {
public static UpgradeStatusDto from(QueryUpgradeStatusResponse status) {
HddsProtos.UpgradeStatus hdds = status.getHddsStatus();
UpgradeStatusDto dto = new UpgradeStatusDto();
dto.omFinalized = status.getOmFinalized();
dto.scmFinalized = hdds.getScmFinalized();
dto.clusterFinalizationStatus = status.getClusterFinalizationStatus().name();
dto.omFinalizationStatus = status.getOmFinalizationStatus().name();
dto.scmFinalizationStatus = hdds.getScmFinalizationStatus().name();
dto.datanodesFinalized = hdds.getNumDatanodesFinalized();
dto.datanodesTotal = hdds.getNumDatanodesTotal();
dto.omApparentVersion = OzoneManagerVersion.deserialize(status.getOmApparentVersion()).toString();
Expand All @@ -132,16 +136,20 @@ public static UpgradeStatusDto from(QueryUpgradeStatusResponse status) {
return dto;
}

public boolean isOmFinalized() {
return omFinalized;
public String getClusterFinalizationStatus() {
return clusterFinalizationStatus;
}

public String getOmFinalizationStatus() {
return omFinalizationStatus;
}

public String getOmApparentVersion() {
return omApparentVersion;
}

public boolean isScmFinalized() {
return scmFinalized;
public String getScmFinalizationStatus() {
return scmFinalizationStatus;
}

public String getScmApparentVersion() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -286,8 +286,8 @@ public void testWaitFlagWithVerbosePrintsFullStatus() throws Exception {
assertEquals(0, cmd.call());

String output = outContent.toString(DEFAULT_ENCODING);
assertTrue(output.contains("OM Finalized?"));
assertTrue(output.contains("SCM Finalized?"));
assertTrue(output.contains("OM:"));
assertTrue(output.contains("SCM:"));
assertTrue(output.contains("OM Apparent Version:"));
assertTrue(output.contains("SCM Apparent Version:"));
assertTrue(output.contains("Min Datanode Apparent Version:"));
Expand All @@ -297,10 +297,10 @@ public void testWaitFlagWithVerbosePrintsFullStatus() throws Exception {

private static QueryUpgradeStatusResponse inProgressStatus(int dnFinalized, int dnTotal) {
return QueryUpgradeStatusResponse.newBuilder()
.setOmFinalized(false)
.setClusterFinalized(false)
.setOmFinalizationStatus(HddsProtos.FinalizationStatus.UNFINALIZED)
.setClusterFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS)
.setHddsStatus(HddsProtos.UpgradeStatus.newBuilder()
.setScmFinalized(false)
.setScmFinalizationStatus(HddsProtos.FinalizationStatus.UNFINALIZED)
.setNumDatanodesFinalized(dnFinalized)
.setNumDatanodesTotal(dnTotal)
.build())
Expand All @@ -309,10 +309,10 @@ private static QueryUpgradeStatusResponse inProgressStatus(int dnFinalized, int

private static QueryUpgradeStatusResponse finalizedStatus(int dnFinalized, int dnTotal) {
return QueryUpgradeStatusResponse.newBuilder()
.setOmFinalized(true)
.setClusterFinalized(true)
.setOmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setClusterFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setHddsStatus(HddsProtos.UpgradeStatus.newBuilder()
.setScmFinalized(true)
.setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setNumDatanodesFinalized(dnFinalized)
.setNumDatanodesTotal(dnTotal)
.build())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,15 +91,16 @@ public void tearDown() {
@Test
public void testStatusCommandPrintsUpgradeStatus() throws Exception {
HddsProtos.UpgradeStatus hddsStatus = HddsProtos.UpgradeStatus.newBuilder()
.setScmFinalized(false)
.setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setNumDatanodesFinalized(1)
.setNumDatanodesTotal(3)
.setHddsFinalized(true)
.setHddsFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS)
.build();

OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response =
OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder()
.setOmFinalized(false)
.setOmFinalizationStatus(HddsProtos.FinalizationStatus.UNFINALIZED)
.setClusterFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS)
.setHddsStatus(hddsStatus)
.build();

Expand All @@ -109,14 +110,43 @@ public void testStatusCommandPrintsUpgradeStatus() throws Exception {

String output = outContent.toString(DEFAULT_ENCODING);
assertTrue(output.contains("Upgrade status"));
assertTrue(output.contains("OM Finalized? false"));
assertTrue(output.contains("SCM Finalized? false"));
assertTrue(output.contains("Cluster: IN_PROGRESS"));
assertTrue(output.contains("OM: UNFINALIZED"));
assertTrue(output.contains("SCM: FINALIZED"));
assertTrue(output.contains("Datanodes finalized: 1/3"));
// Without --verbose the apparent versions are not shown.
assertFalse(output.contains("Apparent Version"));
verify(omClient).queryUpgradeStatus();
}

@Test
public void testStatusCommandPrintsPendingOm() throws Exception {
// OM has begun polling SCM (marker present) but is not finalized yet.
HddsProtos.UpgradeStatus hddsStatus = HddsProtos.UpgradeStatus.newBuilder()
.setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setNumDatanodesFinalized(3)
.setNumDatanodesTotal(3)
.setHddsFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.build();

OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response =
OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder()
.setOmFinalizationStatus(HddsProtos.FinalizationStatus.PENDING)
.setClusterFinalizationStatus(HddsProtos.FinalizationStatus.IN_PROGRESS)
.setHddsStatus(hddsStatus)
.build();

when(omClient.queryUpgradeStatus()).thenReturn(response);
new CommandLine(cmd).parseArgs();
cmd.call();

String output = outContent.toString(DEFAULT_ENCODING);
assertTrue(output.contains("OM: PENDING"));
assertTrue(output.contains("Cluster: IN_PROGRESS"));
assertTrue(output.contains("SCM: FINALIZED"));
verify(omClient).queryUpgradeStatus();
}

@Test
public void testStatusCommandPropagatesException() throws Exception {
when(omClient.queryUpgradeStatus()).thenThrow(new IOException("OM unavailable"));
Expand All @@ -142,10 +172,11 @@ public void testJsonOutput() throws Exception {
int hddsVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response =
OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder()
.setOmFinalized(true)
.setOmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setClusterFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setOmApparentVersion(omVersion)
.setHddsStatus(HddsProtos.UpgradeStatus.newBuilder()
.setScmFinalized(true)
.setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setNumDatanodesFinalized(2)
.setNumDatanodesTotal(3)
.setScmApparentVersion(hddsVersion)
Expand All @@ -161,8 +192,9 @@ public void testJsonOutput() throws Exception {
String jsonOutput = outContent.toString(DEFAULT_ENCODING);

JsonNode root = JSON.readTree(jsonOutput);
assertTrue(root.path("omFinalized").asBoolean());
assertTrue(root.path("scmFinalized").asBoolean());
assertEquals("FINALIZED", root.path("clusterFinalizationStatus").asText());
assertEquals("FINALIZED", root.path("omFinalizationStatus").asText());
assertEquals("FINALIZED", root.path("scmFinalizationStatus").asText());
assertEquals(2, root.path("datanodesFinalized").asInt());
assertEquals(3, root.path("datanodesTotal").asInt());
assertEquals(OzoneManagerVersion.ZDU.toString(), root.path("omApparentVersion").asText());
Expand All @@ -185,10 +217,11 @@ public void testVerboseTextOutputIncludesVersions() throws Exception {
int hddsVersion = HDDSVersion.SOFTWARE_VERSION.serialize();
OzoneManagerProtocolProtos.QueryUpgradeStatusResponse response =
OzoneManagerProtocolProtos.QueryUpgradeStatusResponse.newBuilder()
.setOmFinalized(true)
.setOmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setClusterFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setOmApparentVersion(omVersion)
.setHddsStatus(HddsProtos.UpgradeStatus.newBuilder()
.setScmFinalized(true)
.setScmFinalizationStatus(HddsProtos.FinalizationStatus.FINALIZED)
.setNumDatanodesFinalized(3)
.setNumDatanodesTotal(3)
.setScmApparentVersion(hddsVersion)
Expand All @@ -202,9 +235,9 @@ public void testVerboseTextOutputIncludesVersions() throws Exception {
assertEquals(0, cmd.call());

String output = outContent.toString(DEFAULT_ENCODING);
assertTrue(output.contains("OM Finalized?"));
assertTrue(output.contains("OM:"));
assertTrue(output.contains("OM Apparent Version:"));
assertTrue(output.contains("SCM Finalized?"));
assertTrue(output.contains("SCM:"));
assertTrue(output.contains("SCM Apparent Version:"));
assertTrue(output.contains("Min Datanode Apparent Version:"));
assertTrue(output.contains("Max Datanode Apparent Version:"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public static void waitForFinalizationFromClient(StorageContainerLocationProtoco
LambdaTestUtils.await(60_000, 1_000, () -> {
HddsProtos.UpgradeStatus status = scmClient.queryUpgradeStatus();
LOG.info("Waiting for upgrade finalization to complete from client. Current status is:\n{}", status);
return status.getHddsFinalized();
return status.getHddsFinalizationStatus() == HddsProtos.FinalizationStatus.FINALIZED;
});
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,10 @@ public static void waitForFinalization(OzoneManagerProtocol omClient)
try {
QueryUpgradeStatusResponse status = omClient.queryUpgradeStatus();
HddsProtos.UpgradeStatus hdds = status.getHddsStatus();
LOG.info("Finalization status: omFinalized={}, scmFinalized={}, datanodes={}/{}",
status.getOmFinalized(), hdds.getScmFinalized(),
LOG.info("Finalization status: om={}, scm={}, datanodes={}/{}",
status.getOmFinalizationStatus(), hdds.getScmFinalizationStatus(),
hdds.getNumDatanodesFinalized(), hdds.getNumDatanodesTotal());
return status.getOmFinalized()
&& hdds.getScmFinalized()
&& hdds.getNumDatanodesFinalized() == hdds.getNumDatanodesTotal();
return status.getClusterFinalizationStatus() == HddsProtos.FinalizationStatus.FINALIZED;
} catch (IOException e) {
fail(e.getMessage());
}
Expand Down
Loading