From 8667d8c483776084ed8bc96e6a380fe518d2e593 Mon Sep 17 00:00:00 2001 From: Ren Koike Date: Thu, 28 May 2026 15:41:27 +0200 Subject: [PATCH] HDDS-15344. Extract test cases to OzoneFileSystemTests part 6. --- .../fs/ozone/AbstractOzoneFileSystemTest.java | 83 +------------------ .../AbstractRootedOzoneFileSystemTest.java | 80 +----------------- .../fs/ozone/OzoneFileSystemTestBase.java | 79 ++++++++++++++++++ 3 files changed, 87 insertions(+), 155 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index d4c098b7a623..efc79b7e08c9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -21,14 +21,11 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; -import static org.apache.hadoop.fs.CommonPathCapabilities.FS_ACLS; -import static org.apache.hadoop.fs.CommonPathCapabilities.FS_CHECKSUMS; import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import static org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames.OP_CREATE; import static org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames.OP_GET_FILE_STATUS; import static org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames.OP_MKDIRS; import static org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames.OP_OPEN; -import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE; import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; @@ -76,7 +73,6 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; @@ -387,26 +383,7 @@ public void testMakeDirsWithAnFakeDirectory() throws Exception { @Test public void testCreateWithInvalidPaths() throws Exception { assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout())); - - // Test for path with .. - Path parent = new Path("../../../../../d1/d2/"); - Path file1 = new Path(parent, "key1"); - checkInvalidPath(file1); - - // Test for path with : - file1 = new Path("/:/:"); - checkInvalidPath(file1); - - // Test for path with scheme and authority. - file1 = new Path(fs.getUri() + "/:/:"); - checkInvalidPath(file1); - } - - private void checkInvalidPath(Path path) { - InvalidPathException pathException = GenericTestUtils.assertThrows( - InvalidPathException.class, () -> fs.create(path, false) - ); - assertThat(pathException.getMessage()).contains("Invalid path Name"); + createWithInvalidPaths(); } @Test @@ -529,38 +506,7 @@ private void checkPath(Path path) { @Test public void testFileDelete() throws Exception { - Path grandparent = new Path("/testBatchDelete"); - Path parent = new Path(grandparent, "parent"); - Path childFolder = new Path(parent, "childFolder"); - // BatchSize is 5, so we're going to set a number that's not a - // multiple of 5. In order to test the final number of keys less than - // batchSize can also be deleted. - for (int i = 0; i < 8; i++) { - Path childFile = new Path(parent, "child" + i); - Path childFolderFile = new Path(childFolder, "child" + i); - ContractTestUtils.touch(fs, childFile); - ContractTestUtils.touch(fs, childFolderFile); - } - - assertEquals(1, fs.listStatus(grandparent).length); - assertEquals(9, fs.listStatus(parent).length); - assertEquals(8, fs.listStatus(childFolder).length); - - assertTrue(fs.delete(grandparent, true)); - assertFalse(fs.exists(grandparent)); - for (int i = 0; i < 8; i++) { - Path childFile = new Path(parent, "child" + i); - // Make sure all keys under testBatchDelete/parent should be deleted - assertFalse(fs.exists(childFile)); - - // Test to recursively delete child folder, make sure all keys under - // testBatchDelete/parent/childFolder should be deleted. - Path childFolderFile = new Path(childFolder, "child" + i); - assertFalse(fs.exists(childFolderFile)); - } - // Will get: WARN ozone.BasicOzoneFileSystem delete: Path does not exist. - // This will return false. - assertFalse(fs.delete(parent, true)); + fileDelete(ROOT); } @Test @@ -1468,33 +1414,12 @@ public void testListStatusOnLargeDirectoryForACLCheck() throws Exception { @Test public void testFileSystemDeclaresCapability() throws Throwable { - Path root = new Path(OZONE_URI_DELIMITER); - assertHasPathCapabilities(fs, root, FS_ACLS); - assertHasPathCapabilities(fs, root, FS_CHECKSUMS); + fileSystemDeclaresCapability(new Path(OZONE_URI_DELIMITER)); } @Test public void testSetTimes() throws Exception { - // Create a file - String testKeyName = "testKey1"; - Path path = new Path(OZONE_URI_DELIMITER, testKeyName); - try (FSDataOutputStream stream = fs.create(path)) { - stream.write(1); - } - - long mtime = 1000; - fs.setTimes(path, mtime, 2000); - - FileStatus fileStatus = fs.getFileStatus(path); - // verify that mtime is updated as expected. - assertEquals(mtime, fileStatus.getModificationTime()); - - long mtimeDontUpdate = -1; - fs.setTimes(path, mtimeDontUpdate, 2000); - - fileStatus = fs.getFileStatus(path); - // verify that mtime is NOT updated as expected. - assertEquals(mtime, fileStatus.getModificationTime()); + setTimes(ROOT); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index 92f3ffd918e0..8c3e7c94c2de 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -19,10 +19,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; -import static org.apache.hadoop.fs.CommonPathCapabilities.FS_ACLS; -import static org.apache.hadoop.fs.CommonPathCapabilities.FS_CHECKSUMS; import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; -import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE; import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS; import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS; @@ -79,7 +76,6 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.StreamCapabilities; @@ -1558,38 +1554,7 @@ void testGetTrashRoots() throws IOException { @Test void testFileDelete() throws Exception { - Path grandparent = new Path(bucketPath, "testBatchDelete"); - Path parent = new Path(grandparent, "parent"); - Path childFolder = new Path(parent, "childFolder"); - // BatchSize is 5, so we're going to set a number that's not a - // multiple of 5. In order to test the final number of keys less than - // batchSize can also be deleted. - for (int i = 0; i < 8; i++) { - Path childFile = new Path(parent, "child" + i); - Path childFolderFile = new Path(childFolder, "child" + i); - ContractTestUtils.touch(fs, childFile); - ContractTestUtils.touch(fs, childFolderFile); - } - - assertEquals(1, fs.listStatus(grandparent).length); - assertEquals(9, fs.listStatus(parent).length); - assertEquals(8, fs.listStatus(childFolder).length); - - assertTrue(fs.delete(grandparent, true)); - assertFalse(fs.exists(grandparent)); - for (int i = 0; i < 8; i++) { - Path childFile = new Path(parent, "child" + i); - // Make sure all keys under testBatchDelete/parent should be deleted - assertFalse(fs.exists(childFile)); - - // Test to recursively delete child folder, make sure all keys under - // testBatchDelete/parent/childFolder should be deleted. - Path childFolderFile = new Path(childFolder, "child" + i); - assertFalse(fs.exists(childFolderFile)); - } - // Will get: WARN ozone.BasicOzoneFileSystem delete: Path does not exist. - // This will return false. - assertFalse(fs.delete(parent, true)); + fileDelete(bucketPath); } /** @@ -1708,25 +1673,7 @@ && getOMMetrics().getNumTrashFilesDeletes() @Test void testCreateWithInvalidPaths() { assumeFalse(isBucketFSOptimized); - - // Test for path with .. - Path parent = new Path("../../../../../d1/d2/"); - Path file1 = new Path(parent, "key1"); - checkInvalidPath(file1); - - // Test for path with : - file1 = new Path("/:/:"); - checkInvalidPath(file1); - - // Test for path with scheme and authority. - file1 = new Path(fs.getUri() + "/:/:"); - checkInvalidPath(file1); - } - - private void checkInvalidPath(Path path) { - InvalidPathException exception = assertThrows(InvalidPathException.class, - () -> fs.create(path, false)); - assertThat(exception.getMessage()).contains("Invalid path Name"); + createWithInvalidPaths(); } @Test @@ -2043,8 +1990,7 @@ void testSnapshotRead() throws Exception { @Test void testFileSystemDeclaresCapability() throws Throwable { - assertHasPathCapabilities(fs, getBucketPath(), FS_ACLS); - assertHasPathCapabilities(fs, getBucketPath(), FS_CHECKSUMS); + fileSystemDeclaresCapability(getBucketPath()); } @Test @@ -2111,29 +2057,11 @@ void testSnapshotDiff() throws Exception { @Test void testSetTimes() throws Exception { - // Create a file OzoneBucket bucket1 = TestDataUtil.createVolumeAndBucket(client, bucketLayout); Path volumePath1 = new Path(OZONE_URI_DELIMITER, bucket1.getVolumeName()); Path bucketPath1 = new Path(volumePath1, bucket1.getName()); - Path path = new Path(bucketPath1, "key1"); - try (FSDataOutputStream stream = fs.create(path)) { - stream.write(1); - } - - long mtime = 1000; - fs.setTimes(path, mtime, 2000); - - FileStatus fileStatus = fs.getFileStatus(path); - // verify that mtime is updated as expected. - assertEquals(mtime, fileStatus.getModificationTime()); - - long mtimeDontUpdate = -1; - fs.setTimes(path, mtimeDontUpdate, 2000); - - fileStatus = fs.getFileStatus(path); - // verify that mtime is NOT updated as expected. - assertEquals(mtime, fileStatus.getModificationTime()); + setTimes(bucketPath1); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTestBase.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTestBase.java index a291e3c09352..dee6911eccdb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTestBase.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTestBase.java @@ -17,6 +17,9 @@ package org.apache.hadoop.fs.ozone; +import static org.apache.hadoop.fs.CommonPathCapabilities.FS_ACLS; +import static org.apache.hadoop.fs.CommonPathCapabilities.FS_CHECKSUMS; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; @@ -25,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -35,8 +39,10 @@ import java.util.Set; import java.util.TreeSet; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; @@ -302,6 +308,79 @@ void deleteCreatesFakeParentDir(Path root) throws IOException { assertTrue(fs.delete(grandparent, true)); } + void fileDelete(Path fsRootPath) throws Exception { + Path grandparent = new Path(fsRootPath, "testBatchDelete"); + Path parent = new Path(grandparent, "parent"); + Path childFolder = new Path(parent, "childFolder"); + FileSystem fs = getFs(); + // BatchSize is 5, so use a count that is not a multiple of 5. + for (int i = 0; i < 8; i++) { + Path childFile = new Path(parent, "child" + i); + Path childFolderFile = new Path(childFolder, "child" + i); + ContractTestUtils.touch(fs, childFile); + ContractTestUtils.touch(fs, childFolderFile); + } + + assertEquals(1, fs.listStatus(grandparent).length); + assertEquals(9, fs.listStatus(parent).length); + assertEquals(8, fs.listStatus(childFolder).length); + + assertTrue(fs.delete(grandparent, true)); + assertFalse(fs.exists(grandparent)); + for (int i = 0; i < 8; i++) { + // Make sure all keys under testBatchDelete/parent should be deleted. + assertFalse(fs.exists(new Path(parent, "child" + i))); + + // Test recursive delete of child folder. + assertFalse(fs.exists(new Path(childFolder, "child" + i))); + } + // Will get: WARN ozone.BasicOzoneFileSystem delete: Path does not exist. + assertFalse(fs.delete(parent, true)); + } + + void createWithInvalidPaths() { + // Test for path with .. + checkInvalidPath(new Path("../../../../../d1/d2/", "key1")); + + // Test for path with : + checkInvalidPath(new Path("/:/:")); + + // Test for path with scheme and authority. + checkInvalidPath(new Path(getFs().getUri() + "/:/:")); + } + + private void checkInvalidPath(Path testPath) { + InvalidPathException exception = assertThrows(InvalidPathException.class, + () -> getFs().create(testPath, false)); + assertThat(exception.getMessage()).contains("Invalid path Name"); + } + + void fileSystemDeclaresCapability(Path testPath) throws Throwable { + assertHasPathCapabilities(getFs(), testPath, FS_ACLS); + assertHasPathCapabilities(getFs(), testPath, FS_CHECKSUMS); + } + + void setTimes(Path testPathRoot) throws Exception { + Path path = new Path(testPathRoot, "testKey1"); + FileSystem fs = getFs(); + try (FSDataOutputStream stream = fs.create(path)) { + stream.write(1); + } + + long mtime = 1000; + fs.setTimes(path, mtime, 2000); + + FileStatus fileStatus = fs.getFileStatus(path); + // Verify that mtime is updated as expected. + assertEquals(mtime, fileStatus.getModificationTime()); + + fs.setTimes(path, -1, 2000); + + fileStatus = fs.getFileStatus(path); + // Verify that mtime is NOT updated as expected. + assertEquals(mtime, fileStatus.getModificationTime()); + } + void verifyListStatus(Path root, PathFilter filter) throws Exception { Path parent = new Path(root, "testListStatus"); Path file1 = new Path(parent, "key1");