diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java index e75e1a5c5b32..abd00e706837 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java @@ -146,17 +146,6 @@ public DatanodeID getID() { return id; } - /** - * Returns the DataNode UUID. - * - * @return UUID of DataNode - */ - // TODO: Remove this in follow-up Jira (HDDS-12015) - @Deprecated - public UUID getUuid() { - return id.getUuid(); - } - /** * Returns the string representation of DataNode UUID. * diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeID.java index 6a0ee0b43c36..33f86ea71321 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeID.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeID.java @@ -45,7 +45,7 @@ private DatanodeID(final UUID uuid) { } // Mainly used for JSON conversion - public String getID() { + public String getUuid() { return toString(); } @@ -121,11 +121,4 @@ private static HddsProtos.UUID toProto(final UUID id) { .setLeastSigBits(id.getLeastSignificantBits()) .build(); } - - // TODO: Remove this in follow-up Jira. (HDDS-12015) - // Exposing this temporarily to help with refactoring. - @Deprecated - public UUID getUuid() { - return uuid; - } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java index b9b9d679d63b..2aaa45d695de 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hdds.scm.container; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import java.util.UUID; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.server.JsonUtils; @@ -31,7 +31,7 @@ public final class ContainerReplicaInfo { private long containerID; private String state; private DatanodeDetails datanodeDetails; - private UUID placeOfBirth; + private DatanodeID placeOfBirth; private long sequenceId; private long keyCount; private long bytesUsed; @@ -46,7 +46,7 @@ public static ContainerReplicaInfo fromProto( .setState(proto.getState()) .setDatanodeDetails(DatanodeDetails .getFromProtoBuf(proto.getDatanodeDetails())) - .setPlaceOfBirth(UUID.fromString(proto.getPlaceOfBirth())) + .setPlaceOfBirth(DatanodeID.fromUuidString(proto.getPlaceOfBirth())) .setSequenceId(proto.getSequenceID()) .setKeyCount(proto.getKeyCount()) .setBytesUsed(proto.getBytesUsed()) @@ -71,7 +71,7 @@ public DatanodeDetails getDatanodeDetails() { return datanodeDetails; } - public UUID getPlaceOfBirth() { + public DatanodeID getPlaceOfBirth() { return placeOfBirth; } @@ -117,7 +117,7 @@ public Builder setDatanodeDetails(DatanodeDetails datanodeDetails) { return this; } - public Builder setPlaceOfBirth(UUID placeOfBirth) { + public Builder setPlaceOfBirth(DatanodeID placeOfBirth) { subject.placeOfBirth = placeOfBirth; return this; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java index 8556ce5f6d22..d1b5d99a5ecf 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.hdds.security.symmetric.SecretKeySignerClient; @@ -119,7 +118,7 @@ public ContainerProtos.ContainerChecksumInfo getContainerChecksumInfo(long conta public static Pipeline createSingleNodePipeline(DatanodeDetails dn) { return Pipeline.newBuilder() .setNodes(ImmutableList.of(dn)) - .setId(PipelineID.valueOf(dn.getUuid())) + .setId(dn.getID().toPipelineID()) .setState(Pipeline.PipelineState.CLOSED) .setReplicationConfig(StandaloneReplicationConfig.getInstance( HddsProtos.ReplicationFactor.ONE)).build(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index ad392a247d53..a4d0750b7fe3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@ -1066,7 +1066,7 @@ public DatanodeInfo getDatanodeInfo(DatanodeDetails dn) { return nodeStateManager.getNode(dn); } catch (NodeNotFoundException e) { LOG.warn("Cannot retrieve DatanodeInfo, datanode {} not found.", - dn.getUuid()); + dn.getID()); return null; } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java index a0f5a14725fa..0195bf80d1f0 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java @@ -164,7 +164,7 @@ public static NodeReportProto getRandomNodeReport() { public static NodeReportProto getRandomNodeReport(int numberOfStorageReport, int numberOfMetadataStorageReport) { DatanodeID nodeId = DatanodeID.randomID(); - return getRandomNodeReport(nodeId, File.separator + nodeId.getID(), + return getRandomNodeReport(nodeId, File.separator + nodeId.getUuid(), numberOfStorageReport, numberOfMetadataStorageReport); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java index 0f536b4b01cc..d10d951da90c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java @@ -109,7 +109,7 @@ public void testNodeCanBeAddedAndRetrieved() // Create a datanode, then add and retrieve it DatanodeDetails dn = generateDatanode(); nsm.addNode(dn, UpgradeUtils.defaultLayoutVersionProto()); - assertEquals(dn.getUuid(), nsm.getNode(dn).getUuid()); + assertEquals(dn.getID(), nsm.getNode(dn).getID()); // Now get the status of the newly added node and it should be // IN_SERVICE and HEALTHY NodeStatus expectedState = NodeStatus.inServiceHealthy(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java index 586718766b28..a947d65e0fe0 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineDatanodesIntersection.java @@ -122,12 +122,12 @@ public void testPipelineDatanodesIntersection(int nodeCount, LOG.info("This pipeline: " + pipeline.getId().toString() + " overlaps with previous pipeline: " + overlapPipeline.getId() + ". They share same set of datanodes as: " + - pipeline.getNodesInOrder().get(0).getUuid() + "/" + - pipeline.getNodesInOrder().get(1).getUuid() + "/" + - pipeline.getNodesInOrder().get(2).getUuid() + " and " + - overlapPipeline.getNodesInOrder().get(0).getUuid() + "/" + - overlapPipeline.getNodesInOrder().get(1).getUuid() + "/" + - overlapPipeline.getNodesInOrder().get(2).getUuid() + + pipeline.getNodesInOrder().get(0).getID() + "/" + + pipeline.getNodesInOrder().get(1).getID() + "/" + + pipeline.getNodesInOrder().get(2).getID() + " and " + + overlapPipeline.getNodesInOrder().get(0).getID() + "/" + + overlapPipeline.getNodesInOrder().get(1).getID() + "/" + + overlapPipeline.getNodesInOrder().get(2).getID() + " is the same."); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java index 7094a39c79ed..90b7747350bf 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementPolicy.java @@ -41,7 +41,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.UUID; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import org.apache.hadoop.hdds.client.RatisReplicationConfig; @@ -49,6 +48,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; @@ -189,7 +189,7 @@ public void testChooseNodeBasedOnNetworkTopology() { // nodeManager.getClusterNetworkTopologyMap(), anchor, excludedNodes); assertThat(excludedNodes).doesNotContain(nextNode); // next node should not be the same as anchor. - assertNotSame(anchor.getUuid(), nextNode.getUuid()); + assertNotSame(anchor.getID(), nextNode.getID()); // next node should be on the same rack based on topology. assertEquals(anchor.getNetworkLocation(), nextNode.getNetworkLocation()); } @@ -388,15 +388,14 @@ private NetworkTopology createNetworkTopologyOnDifRacks() { private DatanodeDetails overwriteLocationInNode( DatanodeDetails datanode, Node node) { - DatanodeDetails result = DatanodeDetails.newBuilder() - .setUuid(datanode.getUuid()) + return DatanodeDetails.newBuilder() + .setID(datanode.getID()) .setHostName(datanode.getHostName()) .setIpAddress(datanode.getIpAddress()) .addPort(datanode.getStandalonePort()) .addPort(datanode.getRatisPort()) .addPort(datanode.getRestPort()) .setNetworkLocation(node.getNetworkLocation()).build(); - return result; } private List overWriteLocationInNodes( @@ -427,7 +426,7 @@ public void testHeavyNodeShouldBeExcludedWithMinorityHeavy() // NODES should be sufficient. assertEquals(nodesRequired, pickedNodes1.size()); // make sure pipeline placement policy won't select duplicated NODES. - assertTrue(checkDuplicateNodesUUID(pickedNodes1)); + assertTrue(checkDuplicateNodesID(pickedNodes1)); // majority of healthy NODES are heavily engaged in pipelines. int majorityHeavy = healthyNodes.size() / 2 + 2; @@ -608,11 +607,11 @@ private List setupSkewedRacks() { return dns; } - private boolean checkDuplicateNodesUUID(List nodes) { - HashSet uuids = nodes.stream(). - map(DatanodeDetails::getUuid). + private boolean checkDuplicateNodesID(List nodes) { + HashSet ids = nodes.stream(). + map(DatanodeDetails::getID). collect(Collectors.toCollection(HashSet::new)); - return uuids.size() == nodes.size(); + return ids.size() == nodes.size(); } private void insertHeavyNodesIntoNodeManager( diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java index 5a850551c2b5..230c45f5d400 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/TopologySubcommand.java @@ -300,15 +300,15 @@ public List getPorts() { } private static class NodeTopologyFull extends NodeTopologyDefault { - private String uuid; + private final String id; NodeTopologyFull(DatanodeDetails node, String state) { super(node, state); - uuid = node.getUuid().toString(); + id = node.getID().toString(); } - public String getUuid() { - return uuid; + public String getId() { + return id; } } } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DecommissionStatusSubCommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DecommissionStatusSubCommand.java index 1c166a21316e..a435b5d01169 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DecommissionStatusSubCommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DecommissionStatusSubCommand.java @@ -59,9 +59,9 @@ public class DecommissionStatusSubCommand extends ScmSubcommand { @CommandLine.Mixin private NodeSelectionMixin nodeSelectionMixin; - @CommandLine.Spec + @CommandLine.Spec private CommandLine.Model.CommandSpec spec; - + @Override public void execute(ScmClient scmClient) throws IOException { if (!nodeSelectionMixin.getHostname().isEmpty()) { @@ -135,8 +135,7 @@ public void setErrorMessage(String errorMessage) { } private void printDetails(DatanodeDetails datanode) { - System.out.println(); - System.out.println("Datanode: " + datanode.getUuid().toString() + + System.out.println("\nDatanode: " + datanode.getUuidString() + " (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress() + "/" + datanode.getHostName() + ")"); } diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java index 650027afbee3..8be3ecd0168e 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java @@ -218,7 +218,7 @@ private void printDatanodeInfo(BasicDatanodeInfo dn) { .append("No pipelines in cluster.") .append(System.lineSeparator()); } - System.out.println("Datanode: " + datanode.getUuid().toString() + + System.out.println("Datanode: " + datanode.getUuidString() + " (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress() + "/" + datanode.getHostName() + "/" + relatedPipelineNum + " pipelines)"); diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java index f6cefaa9ad3c..2987b4059b21 100644 --- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/UsageInfoSubcommand.java @@ -81,7 +81,7 @@ public void execute(ScmClient scmClient) throws IOException { !Strings.isNullOrEmpty(exclusiveArguments.getIp()) ? exclusiveArguments.getIp() : !Strings.isNullOrEmpty(exclusiveArguments.getHostname()) ? exclusiveArguments.getHostname() : exclusiveArguments.address; //Fallback to deprecated --address for backward compatibility with older CLI. - + List infoList; if (count < 1) { throw new IOException("Count must be an integer greater than 0."); @@ -116,8 +116,8 @@ public void execute(ScmClient scmClient) throws IOException { * @param info Information such as Capacity, SCMUsed etc. */ private void printInfo(DatanodeUsage info) { - System.out.printf("%-24s: %s %n", "UUID", - info.getDatanodeDetails().getUuid()); + System.out.printf("%-24s: %s %n", "ID", + info.getDatanodeDetails().getUuidString()); System.out.printf("%-24s: %s %n", "IP Address", info.getDatanodeDetails().getIpAddress()); System.out.printf("%-24s: %s %n", "Hostname", @@ -161,7 +161,7 @@ private void printInfo(DatanodeUsage info) { info.getFreeSpaceToSpare() + " B", StringUtils.byteDesc(info.getFreeSpaceToSpare())); System.out.printf("%-24s: %s (%s) %n", "Reserved", - info.getReserved() + " B", + info.getReserved() + " B", StringUtils.byteDesc(info.getReserved())); System.out.println(); } @@ -230,7 +230,7 @@ private static class DatanodeUsage { if (proto.hasFreeSpaceToSpare()) { freeSpaceToSpare = proto.getFreeSpaceToSpare(); } - if (proto.hasReserved()) { + if (proto.hasReserved()) { reserved = proto.getReserved(); } } @@ -329,7 +329,7 @@ public long getPipelineCount() { return pipelineCount; } - public long getReserved() { + public long getReserved() { return reserved; } } diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java index 9433ce46c927..86e3bfdf2731 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestInfoSubCommand.java @@ -234,7 +234,7 @@ private void testReplicaIncludedInOutput(boolean includeIndex) // Ensure each DN UUID is mentioned in the message: for (DatanodeDetails dn : datanodes) { - Pattern uuidPattern = Pattern.compile(".*" + dn.getUuid().toString() + ".*", + Pattern uuidPattern = Pattern.compile(".*" + dn.getID().toString() + ".*", Pattern.DOTALL); assertThat(replica).matches(uuidPattern); } @@ -321,7 +321,7 @@ private void testJsonOutput() throws IOException { assertTrue(json.matches("(?s).*replicas.*")); for (DatanodeDetails dn : datanodes) { Pattern pattern = Pattern.compile( - ".*replicas.*" + dn.getUuid().toString() + ".*", Pattern.DOTALL); + ".*replicas.*" + dn.getID().toString() + ".*", Pattern.DOTALL); Matcher matcher = pattern.matcher(json); assertTrue(matcher.matches()); } @@ -340,7 +340,7 @@ private List getReplicas(boolean includeIndex) { .setContainerID(1) .setBytesUsed(1234) .setState("CLOSED") - .setPlaceOfBirth(dn.getUuid()) + .setPlaceOfBirth(dn.getID()) .setDatanodeDetails(dn) .setKeyCount(1) .setSequenceId(1); diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java index b104db6ef986..ceb2b6ac7471 100644 --- a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java +++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java @@ -115,7 +115,7 @@ public void testOutputDataFieldsAligning() throws IOException { // then String output = outContent.toString(CharEncoding.UTF_8); - assertThat(output).contains("UUID :"); + assertThat(output).contains("ID :"); assertThat(output).contains("IP Address :"); assertThat(output).contains("Hostname :"); assertThat(output).contains("Ozone Capacity :"); diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java index af4521387615..eb7ca54438a9 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.OzoneClientConfig; @@ -337,7 +338,7 @@ public void testSmallerThanChunkSize() throws IOException { HddsProtos.DatanodeDetailsProto member = blockList.getKeyLocations(0).getPipeline().getMembers(i); MockDatanodeStorage mockDatanodeStorage = - storages.get(getMatchingStorage(storages, member.getUuid())); + storages.get(getMatchingStorage(storages, DatanodeID.fromProto(member.getId()))); dns.add(mockDatanodeStorage); } String firstBlockData = dns.get(0).getFullBlockData(new BlockID( @@ -396,8 +397,8 @@ public void testPutBlockHasBlockGroupLen() throws IOException { for (int i = 0; i < dataBlocks + parityBlocks; i++) { MockDatanodeStorage mockDatanodeStorage = storages.get( getMatchingStorage(storages, - blockList.getKeyLocations(0).getPipeline().getMembers(i) - .getUuid())); + DatanodeID.fromProto(blockList.getKeyLocations(0).getPipeline().getMembers(i) + .getId()))); final OzoneKeyDetails keyDetails = bucket.getKey(keyName); ContainerProtos.BlockData block = mockDatanodeStorage.getBlock( @@ -421,11 +422,11 @@ public void testPutBlockHasBlockGroupLen() throws IOException { } private static DatanodeDetails getMatchingStorage( - Map storages, String uuid) { + Map storages, DatanodeID id) { Iterator iterator = storages.keySet().iterator(); while (iterator.hasNext()) { DatanodeDetails dn = iterator.next(); - if (dn.getUuid().toString().equals(uuid)) { + if (dn.getID().equals(id)) { return dn; } } diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot index 7642719d9d37..4dc9ce74106e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot +++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot @@ -160,7 +160,7 @@ Get All Container IDs Get Datanode Ozone Used Bytes Info [arguments] ${uuid} - ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].id'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add + ${output} = Execute export DATANODES=$(ozone admin datanode list --json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].id'); do ozone admin datanode usageinfo --uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.id.uuid) : .[0].ozoneUsed}'; done | jq -s add ${result} = Execute echo '${output}' | jq '. | to_entries | .[] | select(.key == "${uuid}") | .value' [return] ${result} diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java index 9a5a19318c7c..21a0303f32b1 100644 --- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java +++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java @@ -196,7 +196,7 @@ public void fail(MiniOzoneChaosCluster cluster) { try { cluster.restartHddsDatanode(dn, failureMode); } catch (Throwable t) { - LOG.error("Failed to restartNodes Datanode {}", dn.getUuid(), t); + LOG.error("Failed to restartNodes Datanode {}", dn.getID(), t); } }); } @@ -218,7 +218,7 @@ public void fail(MiniOzoneChaosCluster cluster) { cluster.restartHddsDatanode(dn, true); } } catch (Throwable t) { - LOG.error("Failed to shutdown Datanode {}", dn.getUuid(), t); + LOG.error("Failed to shutdown Datanode {}", dn.getID(), t); } }); } diff --git a/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/datanode/PipelineComponentUtil.java b/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/datanode/PipelineComponentUtil.java index 02d426180958..667057af467b 100644 --- a/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/datanode/PipelineComponentUtil.java +++ b/hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/datanode/PipelineComponentUtil.java @@ -68,7 +68,7 @@ public static void withDatanodesFromPipeline( Pipeline pipeline = pipelineSelection.get(); for (DatanodeDetails datanode : pipeline.getNodes()) { Component dn = - new Component(Type.DATANODE, datanode.getUuid().toString(), + new Component(Type.DATANODE, datanode.getID().toString(), datanode.getHostName(), 9882); func.apply(dn); } diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerHealthSummaryEndToEnd.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerHealthSummaryEndToEnd.java index fbfa650bac53..f1976f5a878a 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerHealthSummaryEndToEnd.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerHealthSummaryEndToEnd.java @@ -32,11 +32,11 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.stream.Collectors; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.XceiverClientManager; @@ -611,12 +611,12 @@ private List setupOverReplicatedContainers( // so the phantom persists for the duration of the test. // With 10m FCR, the real DN won't send a full report that changes replica counts. // Result: 2 replicas for RF1 → OVER_REPLICATED. - Set existingUuids = scmCm.getContainerReplicas(containerID) + Set existingIds = scmCm.getContainerReplicas(containerID) .stream() - .map(r -> r.getDatanodeDetails().getUuid()) + .map(r -> r.getDatanodeDetails().getID()) .collect(Collectors.toSet()); DatanodeDetails phantomDN = allDatanodes.stream() - .filter(d -> !existingUuids.contains(d.getUuid())) + .filter(d -> !existingIds.contains(d.getID())) .findFirst() .orElseThrow(() -> new AssertionError( "No spare DN available to inject phantom replica for " + containerID)); diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java index 249e81470d7b..c3e213c07eaf 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java @@ -604,7 +604,7 @@ public void testContainerHealthTaskDetectsOverReplicatedAndNegativeSize() DatanodeDetails primaryDn = pipeline.getFirstNode(); DatanodeDetails secondDn = cluster.getHddsDatanodes().stream() .map(HddsDatanodeService::getDatanodeDetails) - .filter(dd -> !dd.getUuid().equals(primaryDn.getUuid())) + .filter(dd -> !dd.getID().equals(primaryDn.getID())) .findFirst() .orElseThrow(() -> new AssertionError("No second datanode available")); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java index 5877542b92b2..804d27ade4bd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java @@ -65,7 +65,7 @@ static RaftServer.Division getRaftServerDivision( HddsDatanodeService dn, Pipeline pipeline) throws Exception { if (!pipeline.getNodes().contains(dn.getDatanodeDetails())) { throw new IllegalArgumentException("Pipeline:" + pipeline.getId() + - " not exist in datanode:" + dn.getDatanodeDetails().getUuid()); + " not exist in datanode:" + dn.getDatanodeDetails().getID()); } XceiverServerRatis server = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java index 5b89503788df..701a833f2e7c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerReplicationEndToEnd.java @@ -203,7 +203,7 @@ public void testContainerReplication() throws Exception { for (HddsDatanodeService dn : cluster.getHddsDatanodes()) { Predicate p = - i -> i.getUuid().equals(dn.getDatanodeDetails().getUuid()); + i -> i.getID().equals(dn.getDatanodeDetails().getID()); if (!pipeline.getNodes().stream().anyMatch(p)) { dnService = dn; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java index 54644189eced..e3bf613ed9e3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java @@ -130,7 +130,7 @@ static XceiverServerRatis newXceiverServerRatis( DatanodeDetails dn, OzoneConfiguration conf) throws IOException { conf.setInt(OzoneConfigKeys.HDDS_CONTAINER_RATIS_IPC_PORT, dn.getRatisPort().getValue()); - final String dir = testDir.resolve(dn.getUuid().toString()).toString(); + final String dir = testDir.resolve(dn.getID().toString()).toString(); conf.set(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATANODE_STORAGE_DIR, dir); final ContainerDispatcher dispatcher = new TestContainerDispatcher(); @@ -208,7 +208,7 @@ private HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId, ContainerProtos.ContainerType.values()) { handlers.put(containerType, Handler.getHandlerForContainerType(containerType, conf, - dd.getUuid().toString(), + dd.getID().toString(), containerSet, volumeSet, volumeChoosingPolicy, metrics, c -> { }, new ContainerChecksumTreeManager(conf))); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java index f02d6326022e..235cc553fb09 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java @@ -183,7 +183,7 @@ private HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId, ContainerProtos.ContainerType.values()) { handlers.put(containerType, Handler.getHandlerForContainerType(containerType, conf, - dd.getUuid().toString(), + dd.getID().toString(), containerSet, volumeSet, volumeChoosingPolicy, metrics, c -> { }, new ContainerChecksumTreeManager(conf))); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java index 4e69848b307d..de30c36a50e1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java @@ -241,7 +241,7 @@ private static ArgumentMatcher matchEmptyPipeline() { private static ArgumentMatcher matchPipeline(DatanodeDetails dn) { return argument -> argument != null && !argument.getNodes().isEmpty() - && argument.getNodes().get(0).getUuid().equals(dn.getUuid()); + && argument.getNodes().get(0).getID().equals(dn.getID()); } private static ArgumentMatcher matchEcPipeline() { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerDuringDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerDuringDecommissionAndMaintenance.java index 04c6273b8ae1..7709cf546145 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerDuringDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerDuringDecommissionAndMaintenance.java @@ -222,16 +222,16 @@ public void testDiskBalancerWithDecommissionAndMaintenanceNodes() // in DiskBalancer report and status (since we only queried IN_SERVICE nodes) boolean isDecommissionedDnInReport = reportProtoList.stream() .anyMatch(proto -> proto.getNode().getUuid(). - equals(dnToDecommission.getUuid().toString())); + equals(dnToDecommission.getID().toString())); boolean isMaintenanceDnInReport = reportProtoList.stream() .anyMatch(proto -> proto.getNode().getUuid(). - equals(dnToMaintenance.getUuid().toString())); + equals(dnToMaintenance.getID().toString())); boolean isDecommissionedDnInStatus = statusProtoList.stream() .anyMatch(proto -> proto.getNode().getUuid(). - equals(dnToDecommission.getUuid().toString())); + equals(dnToDecommission.getID().toString())); boolean isMaintenanceDnInStatus = statusProtoList.stream() .anyMatch(proto -> proto.getNode().getUuid(). - equals(dnToMaintenance.getUuid().toString())); + equals(dnToMaintenance.getID().toString())); // Assert that the decommissioned DN is not present in both report and status assertFalse(isDecommissionedDnInReport); @@ -262,10 +262,10 @@ public void testDiskBalancerWithDecommissionAndMaintenanceNodes() boolean isRecommissionedDnInReport = reportProtoList.stream() .anyMatch(proto -> proto.getNode().getUuid(). - equals(recommissionedDn.getUuid().toString())); + equals(recommissionedDn.getID().toString())); boolean isRecommissionedDnInStatus = statusProtoList.stream() .anyMatch(proto -> proto.getNode().getUuid(). - equals(recommissionedDn.getUuid().toString())); + equals(recommissionedDn.getID().toString())); // Verify that the recommissioned DN is included in both report and status assertTrue(isRecommissionedDnInReport); diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index dae0d4a48225..1200b1e4d561 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -245,7 +245,7 @@ public HddsDatanodeService getHddsDatanode(DatanodeDetails dn) } } throw new IOException( - "Not able to find datanode with datanode Id " + dn.getUuid()); + "Not able to find datanode with datanode Id " + dn.getID()); } @Override @@ -256,7 +256,7 @@ public int getHddsDatanodeIndex(DatanodeDetails dn) throws IOException { } } throw new IOException( - "Not able to find datanode with datanode Id " + dn.getUuid()); + "Not able to find datanode with datanode Id " + dn.getID()); } @Override diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java index 971bc2d27258..f5fe5aa14c0d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java @@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.recon.scm; -import java.util.UUID; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerReplicaHistoryProto; import org.apache.hadoop.hdds.scm.container.ContainerChecksums; @@ -31,8 +31,8 @@ * of one DN but later moved back to the same DN. */ public class ContainerReplicaHistory { - // Datanode UUID - private final UUID uuid; + // Datanode ID + private final DatanodeID id; // First reported time of the replica on this datanode private final Long firstSeenTime; // Last reported time of the replica @@ -42,9 +42,9 @@ public class ContainerReplicaHistory { private String state; private ContainerChecksums checksums; - public ContainerReplicaHistory(UUID id, Long firstSeenTime, + public ContainerReplicaHistory(DatanodeID id, Long firstSeenTime, Long lastSeenTime, long bcsId, String state, ContainerChecksums checksums) { - this.uuid = id; + this.id = id; this.firstSeenTime = firstSeenTime; this.lastSeenTime = lastSeenTime; this.bcsId = bcsId; @@ -60,8 +60,8 @@ public void setBcsId(long bcsId) { this.bcsId = bcsId; } - public UUID getUuid() { - return uuid; + public DatanodeID getId() { + return id; } public Long getFirstSeenTime() { @@ -98,13 +98,13 @@ public void setChecksums(ContainerChecksums checksums) { public static ContainerReplicaHistory fromProto( ContainerReplicaHistoryProto proto) { - return new ContainerReplicaHistory(UUID.fromString(proto.getUuid()), + return new ContainerReplicaHistory(DatanodeID.fromUuidString(proto.getUuid()), proto.getFirstSeenTime(), proto.getLastSeenTime(), proto.getBcsId(), proto.getState(), ContainerChecksums.of(proto.getDataChecksum())); } public ContainerReplicaHistoryProto toProto() { - return ContainerReplicaHistoryProto.newBuilder().setUuid(uuid.toString()) + return ContainerReplicaHistoryProto.newBuilder().setUuid(id.toString()) .setFirstSeenTime(firstSeenTime).setLastSeenTime(lastSeenTime) .setBcsId(bcsId).setState(state) .setDataChecksum(checksums.getDataChecksum()) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java index e42e0ccac903..4057093c3aa8 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; @@ -70,8 +69,8 @@ public class ReconContainerManager extends ContainerManagerImpl { private final ContainerHealthSchemaManager containerHealthSchemaManager; private final ReconContainerMetadataManager cdbServiceProvider; private final Table nodeDB; - // Container ID -> Datanode UUID -> Timestamp - private final Map> replicaHistoryMap; + // Container ID -> DatanodeID -> Timestamp + private final Map> replicaHistoryMap; // Pipeline -> # of open containers private final Map pipelineToOpenContainer; @@ -307,13 +306,13 @@ public void updateContainerReplica(ContainerID containerID, super.updateContainerReplica(containerID, replica); final long currTime = System.currentTimeMillis(); - final long id = containerID.getId(); + final long cid = containerID.getId(); final DatanodeDetails dnInfo = replica.getDatanodeDetails(); - final UUID uuid = dnInfo.getUuid(); + final DatanodeID id = dnInfo.getID(); - // Map from DataNode UUID to replica last seen time - final Map replicaLastSeenMap = - replicaHistoryMap.get(id); + // Map from DataNode ID to replica last seen time + final Map replicaLastSeenMap = + replicaHistoryMap.get(cid); boolean flushToDB = false; long bcsId = replica.getSequenceId() != null ? replica.getSequenceId() : -1; @@ -323,19 +322,19 @@ public void updateContainerReplica(ContainerID containerID, // If replica doesn't exist in in-memory map, add to DB and add to map if (replicaLastSeenMap == null) { // putIfAbsent to avoid TOCTOU - replicaHistoryMap.putIfAbsent(id, - new ConcurrentHashMap() {{ - put(uuid, new ContainerReplicaHistory(uuid, currTime, currTime, + replicaHistoryMap.putIfAbsent(cid, + new ConcurrentHashMap() {{ + put(id, new ContainerReplicaHistory(id, currTime, currTime, bcsId, state, checksums)); }}); flushToDB = true; } else { // ContainerID exists, update timestamp in memory - final ContainerReplicaHistory ts = replicaLastSeenMap.get(uuid); + final ContainerReplicaHistory ts = replicaLastSeenMap.get(id); if (ts == null) { // New Datanode - replicaLastSeenMap.put(uuid, - new ContainerReplicaHistory(uuid, currTime, currTime, bcsId, + replicaLastSeenMap.put(id, + new ContainerReplicaHistory(id, currTime, currTime, bcsId, state, checksums)); flushToDB = true; } else { @@ -348,7 +347,7 @@ public void updateContainerReplica(ContainerID containerID, } if (flushToDB) { - upsertContainerHistory(id, uuid, currTime, bcsId, state, checksums); + upsertContainerHistory(cid, id, currTime, bcsId, state, checksums); } } @@ -361,20 +360,20 @@ public void removeContainerReplica(ContainerID containerID, ContainerReplicaNotFoundException { super.removeContainerReplica(containerID, replica); - final long id = containerID.getId(); + final long cid = containerID.getId(); final DatanodeDetails dnInfo = replica.getDatanodeDetails(); - final UUID uuid = dnInfo.getUuid(); + final DatanodeID id = dnInfo.getID(); String state = replica.getState().toString(); - final Map replicaLastSeenMap = - replicaHistoryMap.get(id); + final Map replicaLastSeenMap = + replicaHistoryMap.get(cid); if (replicaLastSeenMap != null) { - final ContainerReplicaHistory ts = replicaLastSeenMap.get(uuid); + final ContainerReplicaHistory ts = replicaLastSeenMap.get(id); if (ts != null) { // Flush to DB, then remove from in-memory map - upsertContainerHistory(id, uuid, ts.getLastSeenTime(), ts.getBcsId(), + upsertContainerHistory(cid, id, ts.getLastSeenTime(), ts.getBcsId(), state, ts.getChecksums()); - replicaLastSeenMap.remove(uuid); + replicaLastSeenMap.remove(id); } } } @@ -385,13 +384,13 @@ public ContainerHealthSchemaManager getContainerSchemaManager() { } @VisibleForTesting - public Map> getReplicaHistoryMap() { + public Map> getReplicaHistoryMap() { return replicaHistoryMap; } public List getAllContainerHistory(long containerID) { // First, get the existing entries from DB - Map resMap; + Map resMap; try { resMap = cdbServiceProvider.getContainerReplicaHistory(containerID); } catch (IOException ex) { @@ -401,10 +400,10 @@ public List getAllContainerHistory(long containerID) { // Then, update the entries with the latest in-memory info, if available if (replicaHistoryMap != null) { - Map replicaLastSeenMap = + Map replicaLastSeenMap = replicaHistoryMap.get(containerID); if (replicaLastSeenMap != null) { - Map finalResMap = resMap; + Map finalResMap = resMap; replicaLastSeenMap.forEach((k, v) -> finalResMap.merge(k, v, (old, latest) -> latest)); resMap = finalResMap; @@ -413,19 +412,19 @@ public List getAllContainerHistory(long containerID) { // Finally, convert map to list for output List resList = new ArrayList<>(); - for (Map.Entry entry : resMap.entrySet()) { - final UUID uuid = entry.getKey(); + for (Map.Entry entry : resMap.entrySet()) { + final DatanodeID id = entry.getKey(); String hostname = "N/A"; // Attempt to retrieve hostname from NODES table if (nodeDB != null) { try { - final DatanodeDetails dnDetails = nodeDB.get(DatanodeID.of(uuid)); + final DatanodeDetails dnDetails = nodeDB.get(id); if (dnDetails != null) { hostname = dnDetails.getHostName(); } } catch (IOException ex) { LOG.debug("Unable to retrieve from NODES table of node {}. {}", - uuid, ex.getMessage()); + id, ex.getMessage()); } } final long firstSeenTime = entry.getValue().getFirstSeenTime(); @@ -434,7 +433,7 @@ public List getAllContainerHistory(long containerID) { String state = entry.getValue().getState(); long dataChecksum = entry.getValue().getDataChecksum(); - resList.add(new ContainerHistory(containerID, uuid.toString(), hostname, + resList.add(new ContainerHistory(containerID, id.toString(), hostname, firstSeenTime, lastSeenTime, bcsId, state, dataChecksum)); } return resList; @@ -469,15 +468,15 @@ public void flushReplicaHistoryMapToDB(boolean clearMap) { } } - public void upsertContainerHistory(long containerID, UUID uuid, long time, + public void upsertContainerHistory(long containerID, DatanodeID id, long time, long bcsId, String state, ContainerChecksums checksums) { - Map tsMap; + Map tsMap; try { tsMap = cdbServiceProvider.getContainerReplicaHistory(containerID); - ContainerReplicaHistory ts = tsMap.get(uuid); + ContainerReplicaHistory ts = tsMap.get(id); if (ts == null) { // New entry - tsMap.put(uuid, new ContainerReplicaHistory(uuid, time, time, bcsId, + tsMap.put(id, new ContainerReplicaHistory(id, time, time, bcsId, state, checksums)); } else { // Entry exists, update last seen time and put it back to DB. diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java index acdeaf430528..4ca02476654a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java @@ -19,8 +19,8 @@ import java.io.IOException; import java.util.Map; -import java.util.UUID; import org.apache.hadoop.hdds.annotation.InterfaceStability; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; @@ -95,7 +95,7 @@ void batchStoreContainerKeyCounts(BatchOperation batch, Long containerID, * @throws IOException */ void storeContainerReplicaHistory(Long containerID, - Map tsMap) throws IOException; + Map tsMap) throws IOException; /** * Batch version of storeContainerReplicaHistory. @@ -104,7 +104,7 @@ void storeContainerReplicaHistory(Long containerID, * @throws IOException */ void batchStoreContainerReplicaHistory( - Map> replicaHistoryMap) + Map> replicaHistoryMap) throws IOException; /** @@ -139,7 +139,7 @@ Integer getCountForContainerKeyPrefix( * @return A map of ContainerReplicaWithTimestamp of the given containerID. * @throws IOException */ - Map getContainerReplicaHistory( + Map getContainerReplicaHistory( Long containerID) throws IOException; /** diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java index 3d8b97d3676e..470b2d2a8127 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java @@ -34,10 +34,10 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import javax.inject.Inject; import javax.inject.Singleton; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.utils.db.BatchOperation; @@ -215,9 +215,9 @@ public void batchStoreContainerKeyCounts(BatchOperation batch, */ @Override public void storeContainerReplicaHistory(Long containerID, - Map tsMap) throws IOException { + Map tsMap) throws IOException { List tsList = new ArrayList<>(); - for (Map.Entry e : tsMap.entrySet()) { + for (Map.Entry e : tsMap.entrySet()) { tsList.add(e.getValue()); } @@ -233,17 +233,17 @@ public void storeContainerReplicaHistory(Long containerID, */ @Override public void batchStoreContainerReplicaHistory( - Map> replicaHistoryMap) + Map> replicaHistoryMap) throws IOException { try (BatchOperation batchOperation = containerDbStore.initBatchOperation()) { - for (Map.Entry> entry : + for (Map.Entry> entry : replicaHistoryMap.entrySet()) { final long containerId = entry.getKey(); - final Map tsMap = entry.getValue(); + final Map tsMap = entry.getValue(); List tsList = new ArrayList<>(); - for (Map.Entry e : tsMap.entrySet()) { + for (Map.Entry e : tsMap.entrySet()) { tsList.add(e.getValue()); } @@ -276,7 +276,7 @@ public long getKeyCountForContainer(Long containerID) throws IOException { * @throws IOException */ @Override - public Map getContainerReplicaHistory( + public Map getContainerReplicaHistory( Long containerID) throws IOException { final ContainerReplicaHistoryList tsList = @@ -286,12 +286,12 @@ public Map getContainerReplicaHistory( return new HashMap<>(); } - Map res = new HashMap<>(); + Map res = new HashMap<>(); // Populate result map with entries from the DB. // The list should be fairly short (< 10 entries). for (ContainerReplicaHistory ts : tsList.getList()) { - final UUID uuid = ts.getUuid(); - res.put(uuid, ts); + final DatanodeID id = ts.getId(); + res.put(id, ts); } return res; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java index aabf2a45a634..bae9ecbeba32 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java @@ -48,7 +48,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -175,10 +174,10 @@ public class TestContainerEndpoint { private static final long LOCAL_ID = 0L; private static final long KEY_ONE_SIZE = 500L; // 500 bytes - private UUID uuid1; - private UUID uuid2; - private UUID uuid3; - private UUID uuid4; + private DatanodeID id1; + private DatanodeID id2; + private DatanodeID id3; + private DatanodeID id4; private void initializeInjector() throws Exception { reconOMMetadataManager = getTestReconOmMetadataManager( @@ -763,10 +762,10 @@ public void testGetMissingContainers() throws IOException, TimeoutException { assertEquals(Collections.EMPTY_LIST, responseObject.getContainers()); putContainerInfos(5); - uuid1 = newDatanode("host1", "127.0.0.1"); - uuid2 = newDatanode("host2", "127.0.0.2"); - uuid3 = newDatanode("host3", "127.0.0.3"); - uuid4 = newDatanode("host4", "127.0.0.4"); + id1 = newDatanode("host1", "127.0.0.1"); + id2 = newDatanode("host2", "127.0.0.2"); + id3 = newDatanode("host3", "127.0.0.3"); + id4 = newDatanode("host4", "127.0.0.4"); createUnhealthyRecords(5, 0, 0, 0, 0); Response responseWithLimit = containerEndpoint.getMissingContainers(3); @@ -848,10 +847,10 @@ public void testUnhealthyContainers() throws IOException, TimeoutException { assertEquals(Collections.EMPTY_LIST, responseObject.getContainers()); putContainerInfos(15); - uuid1 = newDatanode("host1", "127.0.0.1"); - uuid2 = newDatanode("host2", "127.0.0.2"); - uuid3 = newDatanode("host3", "127.0.0.3"); - uuid4 = newDatanode("host4", "127.0.0.4"); + id1 = newDatanode("host1", "127.0.0.1"); + id2 = newDatanode("host2", "127.0.0.2"); + id3 = newDatanode("host3", "127.0.0.3"); + id4 = newDatanode("host4", "127.0.0.4"); createUnhealthyRecords(5, 4, 3, 2, 1); response = containerEndpoint.getUnhealthyContainers(1000, 0, 0); @@ -969,11 +968,13 @@ public void testUnhealthyContainersFilteredResponse() // Add unhealthy records putContainerInfos(5); - uuid1 = newDatanode("host1", "127.0.0.1"); - uuid2 = newDatanode("host2", "127.0.0.2"); - uuid3 = newDatanode("host3", "127.0.0.3"); - uuid4 = newDatanode("host4", "127.0.0.4"); + id1 = newDatanode("host1", "127.0.0.1"); + id2 = newDatanode("host2", "127.0.0.2"); + id3 = newDatanode("host3", "127.0.0.3"); + id4 = newDatanode("host4", "127.0.0.4"); createUnhealthyRecords(5, 4, 3, 2, 1); + // createEmptyMissingUnhealthyRecords(2); // For EMPTY_MISSING state + // createNegativeSizeUnhealthyRecords(2); // For NEGATIVE_SIZE state // Check for unhealthy containers response = containerEndpoint.getUnhealthyContainers(missing, 1000, 0, 0); @@ -1031,10 +1032,10 @@ public void testUnhealthyContainersPaging() // createUnhealthyRecords(5, 4, 3, 2) will create records for containers 1-14 // So we need to create 14 containers instead of just 6 putContainerInfos(14); // Changed from 6 to 14 - uuid1 = newDatanode("host1", "127.0.0.1"); - uuid2 = newDatanode("host2", "127.0.0.2"); - uuid3 = newDatanode("host3", "127.0.0.3"); - uuid4 = newDatanode("host4", "127.0.0.4"); + id1 = newDatanode("host1", "127.0.0.1"); + id2 = newDatanode("host2", "127.0.0.2"); + id3 = newDatanode("host3", "127.0.0.3"); + id4 = newDatanode("host4", "127.0.0.4"); createUnhealthyRecords(5, 4, 3, 2, 0); // Get first batch with no pagination (prevStartKey=0, prevLastKey=0) @@ -1072,10 +1073,10 @@ public void testUnhealthyContainersPaging() @Test public void testGetReplicaHistoryForContainer() throws IOException { // Add container history for container id 1 - final UUID u1 = newDatanode("host1", "127.0.0.1"); - final UUID u2 = newDatanode("host2", "127.0.0.2"); - final UUID u3 = newDatanode("host3", "127.0.0.3"); - final UUID u4 = newDatanode("host4", "127.0.0.4"); + final DatanodeID u1 = newDatanode("host1", "127.0.0.1"); + final DatanodeID u2 = newDatanode("host2", "127.0.0.2"); + final DatanodeID u3 = newDatanode("host3", "127.0.0.3"); + final DatanodeID u4 = newDatanode("host4", "127.0.0.4"); reconContainerManager.upsertContainerHistory(1L, u1, 1L, 1L, "OPEN", ContainerChecksums.of(1234L, 0L)); reconContainerManager.upsertContainerHistory(1L, u2, 2L, 1L, "OPEN", ContainerChecksums.of(1234L, 0L)); reconContainerManager.upsertContainerHistory(1L, u3, 3L, 1L, "OPEN", ContainerChecksums.of(1234L, 0L)); @@ -1116,15 +1117,15 @@ public void testGetReplicaHistoryForContainer() throws IOException { } } - UUID newDatanode(String hostName, String ipAddress) throws IOException { - final UUID uuid = UUID.randomUUID(); - reconContainerManager.getNodeDB().put(DatanodeID.of(uuid), + DatanodeID newDatanode(String hostName, String ipAddress) throws IOException { + final DatanodeID id = DatanodeID.randomID(); + reconContainerManager.getNodeDB().put(id, DatanodeDetails.newBuilder() - .setUuid(uuid) + .setID(id) .setHostName(hostName) .setIpAddress(ipAddress) .build()); - return uuid; + return id; } private void createUnhealthyRecords(int missing, int overRep, int underRep, @@ -1167,13 +1168,13 @@ private void createUnhealthyRecord(int id, String state, int expected, long differentChecksum = dataChecksumMismatch ? 2345L : 1234L; - reconContainerManager.upsertContainerHistory(cID, uuid1, 1L, 1L, + reconContainerManager.upsertContainerHistory(cID, id1, 1L, 1L, "UNHEALTHY", ContainerChecksums.of(differentChecksum, 0L)); - reconContainerManager.upsertContainerHistory(cID, uuid2, 2L, 1L, + reconContainerManager.upsertContainerHistory(cID, id2, 2L, 1L, "UNHEALTHY", ContainerChecksums.of(differentChecksum, 0L)); - reconContainerManager.upsertContainerHistory(cID, uuid3, 3L, 1L, + reconContainerManager.upsertContainerHistory(cID, id3, 3L, 1L, "UNHEALTHY", ContainerChecksums.of(1234L, 0L)); - reconContainerManager.upsertContainerHistory(cID, uuid4, 4L, 1L, + reconContainerManager.upsertContainerHistory(cID, id4, 4L, 1L, "UNHEALTHY", ContainerChecksums.of(1234L, 0L)); } @@ -1622,7 +1623,7 @@ public void testGetContainerInsightsNonOMContainersPipelineIsolation() throws IOException, TimeoutException { // Verifies fix for pipeline accumulation bug: containers missing in OM // should each have their own isolated pipeline list, not shared. - + // Create 3 different pipelines Pipeline[] pipelines = { getRandomPipeline(), getRandomPipeline(), getRandomPipeline() @@ -1630,7 +1631,7 @@ public void testGetContainerInsightsNonOMContainersPipelineIsolation() for (Pipeline p : pipelines) { reconPipelineManager.addPipeline(p); } - + // Create 3 containers in SCM with different pipelines (not in OM) long[] containerIds = {501L, 502L, 503L}; for (int i = 0; i < containerIds.length; i++) { @@ -1638,16 +1639,16 @@ public void testGetContainerInsightsNonOMContainersPipelineIsolation() assertFalse(reconContainerMetadataManager.doesContainerExists(containerIds[i]), "Container " + containerIds[i] + " should NOT exist in OM"); } - + // Call API Response response = containerEndpoint.getContainerMisMatchInsights(10, 500, "OM"); assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); - + Map responseMap = (Map) response.getEntity(); List discrepancies = (List) responseMap.get("containerDiscrepancyInfo"); assertNotNull(discrepancies); - + // Find containers in response ContainerDiscrepancyInfo[] infos = new ContainerDiscrepancyInfo[3]; for (int i = 0; i < containerIds.length; i++) { @@ -1658,12 +1659,12 @@ public void testGetContainerInsightsNonOMContainersPipelineIsolation() .orElseThrow(() -> new AssertionError( "Container " + id + " not found in mismatch list")); } - + // Verify pipeline isolation for each container List pipelineList1 = infos[0].getPipelines(); List pipelineList2 = infos[1].getPipelines(); List pipelineList3 = infos[2].getPipelines(); - + verifyPipelineIsolation(infos[0], containerIds[0], pipelines[0].getId(), Arrays.asList(pipelineList2, pipelineList3)); verifyPipelineIsolation(infos[1], containerIds[1], pipelines[1].getId(), @@ -2030,8 +2031,8 @@ public void testGetQuasiClosedContainersWithReplicas() throws Exception { ContainerID.valueOf(210L), HddsProtos.LifeCycleEvent.QUASI_CLOSE); // Register 2 datanodes and upsert replica history for container 210. - UUID dn1 = newDatanode("qc-host1", "10.0.0.1"); - UUID dn2 = newDatanode("qc-host2", "10.0.0.2"); + DatanodeID dn1 = newDatanode("qc-host1", "10.0.0.1"); + DatanodeID dn2 = newDatanode("qc-host2", "10.0.0.2"); reconContainerManager.upsertContainerHistory( 210L, dn1, 1L, 2L, "QUASI_CLOSED", ContainerChecksums.of(1111L, 0L)); reconContainerManager.upsertContainerHistory( diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 07dd11d8021e..db9301e4795e 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -333,8 +334,8 @@ public void setUp() throws Exception { initializeInjector(); isSetupDone = true; } - String datanodeId = datanodeDetails.getUuid().toString(); - String datanodeId2 = datanodeDetails2.getUuid().toString(); + String datanodeId = datanodeDetails.getID().toString(); + String datanodeId2 = datanodeDetails2.getID().toString(); containerReportsProto = ContainerReportsProto.newBuilder() .addReports( @@ -432,7 +433,7 @@ public void setUp() throws Exception { DatanodeDetailsProto datanodeDetailsProto3 = DatanodeDetailsProto.newBuilder() .setHostName(HOST3) - .setUuid(datanodeDetails3.getUuid().toString()) + .setId(datanodeDetails3.getID().toProto()) .setIpAddress(IP3) .build(); extendedDatanodeDetailsProto3 = @@ -1310,7 +1311,8 @@ public void testExplicitRemovalOfDecommissionedNode() throws Exception { nodeManager.setNodeOperationalState(dnDetailsInternal, NodeOperationalState.DECOMMISSIONED, 666L); - Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(datanodeDetails3.getUuid().toString())); + Response removedDNResponse = nodeEndpoint.removeDatanodes( + Collections.singletonList(datanodeDetails3.getID().toString())); RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); @@ -1326,7 +1328,7 @@ public void testExplicitRemovalOfDecommissionedNode() throws Exception { @Test public void testExplicitRemovalOfInvalidStateNode() { - String dnUUID = datanodeDetails2.getUuid().toString(); + String dnUUID = datanodeDetails2.getID().toString(); Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); @@ -1343,7 +1345,7 @@ public void testExplicitRemovalOfInvalidStateNode() { @Test public void testExplicitRemovalOfNonExistingNode() { - String dnUUID = datanodeDetails4.getUuid().toString(); + String dnUUID = datanodeDetails4.getID().toString(); Response removedDNResponse = nodeEndpoint.removeDatanodes(Arrays.asList(dnUUID)); RemoveDataNodesResponseWrapper removeDataNodesResponseWrapper = (RemoveDataNodesResponseWrapper) removedDNResponse.getEntity(); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java index be2fd9bd8247..e4325f9604fb 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenContainerCount.java @@ -225,7 +225,7 @@ public void setUp() throws Exception { initializeInjector(); isSetupDone = true; } - datanodeId = datanodeDetails.getUuid().toString(); + datanodeId = datanodeDetails.getID().toString(); // initialize container report builder = ContainerReportsProto.newBuilder(); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java index 1d871b9974b9..e1e89f180be6 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java @@ -32,10 +32,10 @@ import java.util.List; import java.util.Map; import java.util.NavigableSet; -import java.util.UUID; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; @@ -205,16 +205,16 @@ public void testUpdateAndRemoveContainerReplica() final ContainerID containerID1 = ContainerID.valueOf(cIDlong1); // Init DN01 - final UUID uuid1 = UUID.randomUUID(); + final DatanodeID id1 = DatanodeID.randomID(); final DatanodeDetails datanodeDetails1 = DatanodeDetails.newBuilder() - .setUuid(uuid1).setHostName("host1").setIpAddress("127.0.0.1").build(); + .setID(id1).setHostName("host1").setIpAddress("127.0.0.1").build(); ContainerReplica containerReplica1 = ContainerReplica.newBuilder() .setContainerID(containerID1).setContainerState(State.OPEN) .setDatanodeDetails(datanodeDetails1).setSequenceId(1001L) .setChecksums(ContainerChecksums.of(1234L, 0L)).build(); final ReconContainerManager containerManager = getContainerManager(); - final Map> repHistMap = + final Map> repHistMap = containerManager.getReplicaHistoryMap(); // Should be empty at the beginning assertEquals(0, repHistMap.size()); @@ -233,8 +233,8 @@ public void testUpdateAndRemoveContainerReplica() assertEquals(1, repHistMap.size()); // Should only have 1 entry for this replica (on DN01) assertEquals(1, repHistMap.get(cIDlong1).size()); - ContainerReplicaHistory repHist1 = repHistMap.get(cIDlong1).get(uuid1); - assertEquals(uuid1, repHist1.getUuid()); + ContainerReplicaHistory repHist1 = repHistMap.get(cIDlong1).get(id1); + assertEquals(id1, repHist1.getId()); // Because this is a new entry, first seen time equals last seen time assertEquals(repHist1.getLastSeenTime(), repHist1.getFirstSeenTime()); assertEquals(containerReplica1.getSequenceId().longValue(), @@ -253,9 +253,9 @@ public void testUpdateAndRemoveContainerReplica() assertEquals(1051L, repHist1.getBcsId()); // Init DN02 - final UUID uuid2 = UUID.randomUUID(); + final DatanodeID id2 = DatanodeID.randomID(); final DatanodeDetails datanodeDetails2 = DatanodeDetails.newBuilder() - .setUuid(uuid2).setHostName("host2").setIpAddress("127.0.0.2").build(); + .setID(id2).setHostName("host2").setIpAddress("127.0.0.2").build(); final ContainerReplica containerReplica2 = ContainerReplica.newBuilder() .setContainerID(containerID1).setContainerState(State.OPEN) .setDatanodeDetails(datanodeDetails2).setSequenceId(1051L) @@ -268,8 +268,8 @@ public void testUpdateAndRemoveContainerReplica() assertEquals(1, repHistMap.size()); // Should have 2 entries for this replica (on DN01 and DN02) assertEquals(2, repHistMap.get(cIDlong1).size()); - ContainerReplicaHistory repHist2 = repHistMap.get(cIDlong1).get(uuid2); - assertEquals(uuid2, repHist2.getUuid()); + ContainerReplicaHistory repHist2 = repHistMap.get(cIDlong1).get(id2); + assertEquals(id2, repHist2.getId()); // Because this is a new entry, first seen time equals last seen time assertEquals(repHist2.getLastSeenTime(), repHist2.getFirstSeenTime()); assertEquals(1051L, repHist2.getBcsId()); @@ -282,7 +282,7 @@ public void testUpdateAndRemoveContainerReplica() // Should have 1 entry for this replica assertEquals(1, repHistMap.get(cIDlong1).size()); // And the only entry should match DN02 - assertEquals(uuid2, + assertEquals(id2, repHistMap.get(cIDlong1).keySet().iterator().next()); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java index e0eb2f020ccb..9c93441457dd 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java @@ -232,7 +232,7 @@ public void testUpdateNodeOperationalStateFromScm() throws Exception { List nodes = reconNodeManager.getNodes(DECOMMISSIONING, null); assertEquals(1, nodes.size()); - assertEquals(datanodeDetails.getUuid(), nodes.get(0).getUuid()); + assertEquals(datanodeDetails.getID(), nodes.get(0).getID()); } @Test diff --git a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java index 0268281f83e0..324df519e593 100644 --- a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java +++ b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; @@ -131,7 +132,7 @@ public class DatanodeSimulator implements Callable, VaporSubcommand { private ConfigurationSource conf; private List datanodes; - private Map datanodesMap; + private Map datanodesMap; private ScheduledExecutorService heartbeatScheduler; private LayoutVersionProto layoutInfo; @@ -295,7 +296,7 @@ private void loadOrCreateDatanodes() throws UnknownHostException { datanodesMap = new HashMap<>(); for (DatanodeSimulationState datanode : datanodes) { - datanodesMap.put(datanode.getDatanodeDetails().getUuid(), datanode); + datanodesMap.put(datanode.getDatanodeDetails().getID(), datanode); } } @@ -317,8 +318,8 @@ private void growContainers() throws IOException { ReplicationFactor.THREE, "test"); for (DatanodeDetails datanode : cp.getPipeline().getNodeSet()) { - if (datanodesMap.containsKey(datanode.getUuid())) { - datanodesMap.get(datanode.getUuid()) + if (datanodesMap.containsKey(datanode.getID())) { + datanodesMap.get(datanode.getID()) .newContainer(cp.getContainerInfo().getContainerID()); totalAssignedContainers++; } @@ -458,7 +459,7 @@ private LayoutVersionProto createLayoutInfo() throws IOException { private DatanodeDetails randomDatanodeDetails(ConfigurationSource config) throws UnknownHostException { DatanodeDetails details = DatanodeDetails.newBuilder() - .setUuid(UUID.randomUUID()) + .setID(DatanodeID.randomID()) .build(); details.setInitialVersion(DatanodeVersion.CURRENT_VERSION); details.setCurrentVersion(DatanodeVersion.CURRENT_VERSION); diff --git a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java index 031c69a10e22..3652a8e58dd6 100644 --- a/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java +++ b/hadoop-ozone/vapor/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java @@ -33,7 +33,6 @@ import java.util.List; import java.util.Queue; import java.util.Random; -import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -49,6 +48,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.StorageTypeProto; @@ -796,7 +796,7 @@ private class FakeDatanode { public void register() throws IOException { SCMRegisteredResponseProto response = datanodeScmClient.register( datanodeDetails.getExtendedProtoBufMessage(), - createNodeReport(datanodeDetails.getUuid()), + createNodeReport(datanodeDetails.getID()), createContainerReport(), createPipelineReport(), UpgradeUtils.defaultLayoutVersionProto()); @@ -852,7 +852,6 @@ public DatanodeDetails getDatanodeDetails() { } private static DatanodeDetails createRandomDatanodeDetails() { - UUID uuid = UUID.randomUUID(); String ipAddress = RANDOM.nextInt(256) + "." + RANDOM.nextInt(256) + "." + RANDOM .nextInt(256) + "." + RANDOM.nextInt(256); @@ -861,7 +860,8 @@ private static DatanodeDetails createRandomDatanodeDetails() { DatanodeDetails.Port ratisPort = DatanodeDetails.newRatisPort(0); DatanodeDetails.Port restPort = DatanodeDetails.newRestPort(0); DatanodeDetails.Builder builder = DatanodeDetails.newBuilder(); - builder.setUuid(uuid).setHostName("localhost") + builder.setID(DatanodeID.randomID()) + .setHostName("localhost") .setIpAddress(ipAddress) .addPort(containerPort) .addPort(ratisPort) @@ -869,7 +869,7 @@ private static DatanodeDetails createRandomDatanodeDetails() { return builder.build(); } - private static NodeReportProto createNodeReport(UUID nodeId) { + private static NodeReportProto createNodeReport(DatanodeID nodeId) { List storageReports = new ArrayList<>(); List metadataStorageReports = new ArrayList<>(); @@ -881,7 +881,7 @@ private static NodeReportProto createNodeReport(UUID nodeId) { return nb.build(); } - private static StorageReportProto createStorageReport(UUID nodeId) { + private static StorageReportProto createStorageReport(DatanodeID nodeId) { StorageReportProto.Builder srb = StorageReportProto.newBuilder(); srb.setStorageUuid(nodeId.toString()) .setStorageLocation("/data")