Skip to content

HDDS-13482. Intermittent failure in TestContainerStateMachineFailures #10397

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
chungen0126 wants to merge 6 commits into
base: master
Choose a base branch
from
+28 −19
Draft

HDDS-13482. Intermittent failure in TestContainerStateMachineFailures #10397

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
63b20e0
HDDS-13482. Intermittent failure in TestContainerStateMachineFailures
chungen0126 May 31, 2026
6cdd22b
fix testApplyTransactionIdempotencyWithClosedContainer
chungen0126 May 31, 2026
9550d08
fix checkstyle
chungen0126 May 31, 2026
dc41163
rewrite testApplyTransactionFailure
chungen0126 Jun 1, 2026
19765cb
fix testApplyTransactionFailure
chungen0126 Jun 1, 2026
a1e6bd9
fix checkstyle
chungen0126 Jun 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 28 additions & 19 deletions ...t/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,14 @@
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NODE_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.QUASI_CLOSED;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
Expand All @@ -46,6 +47,7 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
Expand Down Expand Up @@ -100,7 +102,6 @@
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.LambdaTestUtils;
import org.apache.ozone.test.tag.Flaky;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.exceptions.StateMachineException;
import org.apache.ratis.server.storage.FileInfo;
Expand All @@ -109,11 +110,15 @@
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;

/**
* Tests the containerStateMachine failure handling.
*/
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class TestContainerStateMachineFailures {

private static MiniOzoneCluster cluster;
Expand All @@ -138,7 +143,9 @@ public static void init() throws Exception {
conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 200,
TimeUnit.MILLISECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 200, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS);
conf.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 1, TimeUnit.SECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, TimeUnit.SECONDS);
conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s");
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s");
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s");
Expand Down Expand Up @@ -254,7 +261,6 @@ public void testContainerStateMachineCloseOnMissingPipeline()
}

@Test
@Flaky("HDDS-12215")
public void testContainerStateMachineRestartWithDNChangePipeline()
throws Exception {
try (OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName)
Expand Down Expand Up @@ -304,7 +310,12 @@ public void testContainerStateMachineRestartWithDNChangePipeline()
}
}

// This test case is placed at the end because it resets the Ratis storage location.
// This causes pipelines to break. Those pipelines are closed passively
// via client-side retries rather than by the ScrubbingService.
// Running this test earlier would leave a dirty pipeline pool for subsequent tests.
@Test
@Order(Integer.MAX_VALUE)
public void testContainerStateMachineFailures() throws Exception {
OzoneOutputStream key =
objectStore.getVolume(volumeName).getBucket(bucketName)
Expand Down Expand Up @@ -476,16 +487,16 @@ public void testApplyTransactionFailure() throws Exception {
getHddsDatanodes().get(index), omKeyLocationInfo.getPipeline());
SimpleStateMachineStorage storage =
(SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
stateMachine.takeSnapshot();
long containerID = omKeyLocationInfo.getContainerID();
// delete the container db file
FileUtil.fullyDelete(new File(keyValueContainerData.getContainerPath()));
final FileInfo snapshot = getSnapshotFileInfo(storage);
final Path parentPath = snapshot.getPath();
// Since the snapshot threshold is set to 1, since there are
// applyTransactions, we should see snapshots
assertThat(parentPath.getParent().toFile().listFiles().length).isGreaterThan(0);
assertThat(Objects.requireNonNull(parentPath.getParent().toFile().listFiles()).length)
.isGreaterThan(0);
assertNotNull(snapshot);
long containerID = omKeyLocationInfo.getContainerID();
// delete the container db file
FileUtil.fullyDelete(new File(keyValueContainerData.getContainerPath()));
Pipeline pipeline = cluster.getStorageContainerLocationClient()
.getContainerWithPipeline(containerID).getPipeline();
XceiverClientSpi xceiverClient =
Expand All @@ -506,12 +517,13 @@ public void testApplyTransactionFailure() throws Exception {
xceiverClientManager.releaseClient(xceiverClient, false);
}
// Make sure the container is marked unhealthy
assertSame(dn.getDatanodeStateMachine()
GenericTestUtils.waitFor(() -> dn.getDatanodeStateMachine()
.getContainer().getContainerSet().getContainer(containerID)
.getContainerState(), UNHEALTHY);
.getContainerState() == UNHEALTHY, 100, 5000);
try {
// try to take a new snapshot, ideally it should just fail
stateMachine.takeSnapshot();
fail("Should have thrown StateMachineException because it is UNHEALTHY");
} catch (IOException ioe) {
assertInstanceOf(StateMachineException.class, ioe);
}
Expand All @@ -521,8 +533,7 @@ public void testApplyTransactionFailure() throws Exception {
try {
final FileInfo latestSnapshot = getSnapshotFileInfo(storage);
assertEquals(snapshot.getPath(), latestSnapshot.getPath());
} catch (Throwable e) {
assertFalse(snapshot.getPath().toFile().exists());
} catch (IOException ioe) {
}
}

Expand All @@ -532,7 +543,6 @@ public void testApplyTransactionFailure() throws Exception {
}

@Test
@Flaky("HDDS-6115")
void testApplyTransactionIdempotencyWithClosedContainer()
throws Exception {
OzoneOutputStream key =
Expand Down Expand Up @@ -590,6 +600,8 @@ void testApplyTransactionIdempotencyWithClosedContainer()
.getContainerState(),
ContainerProtos.ContainerDataProto.State.CLOSED);
assertTrue(stateMachine.isStateMachineHealthy());
GenericTestUtils.waitFor(() -> stateMachine.getLastAppliedTermIndex().getIndex() != markIndex1,
1000, 30000);
try {
stateMachine.takeSnapshot();
} finally {
Expand Down Expand Up @@ -618,7 +630,6 @@ void testApplyTransactionIdempotencyWithClosedContainer()
// not be marked unhealthy and pipeline should not fail if container gets
// closed here.
@Test
@Flaky("HDDS-13482")
void testWriteStateMachineDataIdempotencyWithClosedContainer()
throws Exception {
OzoneOutputStream key =
Expand Down Expand Up @@ -683,7 +694,7 @@ void testWriteStateMachineDataIdempotencyWithClosedContainer()
};
Runnable r2 = () -> {
try {
ByteString data = ByteString.copyFromUtf8("hello");
ByteString data = ByteString.copyFromUtf8("ratis");
ContainerProtos.ContainerCommandRequestProto.Builder writeChunkRequest =
ContainerTestHelper.newWriteChunkRequestBuilder(pipeline,
omKeyLocationInfo.getBlockID(), data.size());
Expand All @@ -698,7 +709,7 @@ void testWriteStateMachineDataIdempotencyWithClosedContainer()
failCount.incrementAndGet();
}
String message = e.getMessage();
assertThat(message).doesNotContain("hello");
assertThat(message).doesNotContain("ratis");
assertThat(message).contains(HddsUtils.REDACTED.toStringUtf8());
}
};
Expand Down Expand Up @@ -745,7 +756,6 @@ void testWriteStateMachineDataIdempotencyWithClosedContainer()
}

@Test
@Flaky("HDDS-14101")
void testContainerStateMachineSingleFailureRetry()
throws Exception {
try (OzoneOutputStream key = objectStore.getVolume(volumeName).getBucket(bucketName)
Expand Down Expand Up @@ -776,7 +786,6 @@ void testContainerStateMachineSingleFailureRetry()
}

@Test
@Flaky("HDDS-14101")
void testContainerStateMachineDualFailureRetry()
throws Exception {
OzoneOutputStream key =
Expand Down