Skip to content

HDDS-15344. Extract test cases to OzoneFileSystemTestBase part 6. #10401

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
len548 wants to merge 1 commit into
base: master
Choose a base branch
from
+87 −155
Open

HDDS-15344. Extract test cases to OzoneFileSystemTestBase part 6. #10401

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 4 additions & 79 deletions ...ntegration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,11 @@
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.fs.CommonPathCapabilities.FS_ACLS;
import static org.apache.hadoop.fs.CommonPathCapabilities.FS_CHECKSUMS;
import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
import static org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames.OP_CREATE;
import static org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames.OP_GET_FILE_STATUS;
import static org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames.OP_MKDIRS;
import static org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames.OP_OPEN;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
Expand Down Expand Up @@ -76,7 +73,6 @@
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
Expand Down Expand Up @@ -387,26 +383,7 @@ public void testMakeDirsWithAnFakeDirectory() throws Exception {
@Test
public void testCreateWithInvalidPaths() throws Exception {
assumeFalse(FILE_SYSTEM_OPTIMIZED.equals(getBucketLayout()));

// Test for path with ..
Path parent = new Path("../../../../../d1/d2/");
Path file1 = new Path(parent, "key1");
checkInvalidPath(file1);

// Test for path with :
file1 = new Path("/:/:");
checkInvalidPath(file1);

// Test for path with scheme and authority.
file1 = new Path(fs.getUri() + "/:/:");
checkInvalidPath(file1);
}

private void checkInvalidPath(Path path) {
InvalidPathException pathException = GenericTestUtils.assertThrows(
InvalidPathException.class, () -> fs.create(path, false)
);
assertThat(pathException.getMessage()).contains("Invalid path Name");
createWithInvalidPaths();
}

@Test
Expand Down Expand Up @@ -529,38 +506,7 @@ private void checkPath(Path path) {

@Test
public void testFileDelete() throws Exception {
Path grandparent = new Path("/testBatchDelete");
Path parent = new Path(grandparent, "parent");
Path childFolder = new Path(parent, "childFolder");
// BatchSize is 5, so we're going to set a number that's not a
// multiple of 5. In order to test the final number of keys less than
// batchSize can also be deleted.
for (int i = 0; i < 8; i++) {
Path childFile = new Path(parent, "child" + i);
Path childFolderFile = new Path(childFolder, "child" + i);
ContractTestUtils.touch(fs, childFile);
ContractTestUtils.touch(fs, childFolderFile);
}

assertEquals(1, fs.listStatus(grandparent).length);
assertEquals(9, fs.listStatus(parent).length);
assertEquals(8, fs.listStatus(childFolder).length);

assertTrue(fs.delete(grandparent, true));
assertFalse(fs.exists(grandparent));
for (int i = 0; i < 8; i++) {
Path childFile = new Path(parent, "child" + i);
// Make sure all keys under testBatchDelete/parent should be deleted
assertFalse(fs.exists(childFile));

// Test to recursively delete child folder, make sure all keys under
// testBatchDelete/parent/childFolder should be deleted.
Path childFolderFile = new Path(childFolder, "child" + i);
assertFalse(fs.exists(childFolderFile));
}
// Will get: WARN ozone.BasicOzoneFileSystem delete: Path does not exist.
// This will return false.
assertFalse(fs.delete(parent, true));
fileDelete(ROOT);
}

@Test
Expand Down Expand Up @@ -1468,33 +1414,12 @@ public void testListStatusOnLargeDirectoryForACLCheck() throws Exception {

@Test
public void testFileSystemDeclaresCapability() throws Throwable {
Path root = new Path(OZONE_URI_DELIMITER);
assertHasPathCapabilities(fs, root, FS_ACLS);
assertHasPathCapabilities(fs, root, FS_CHECKSUMS);
fileSystemDeclaresCapability(new Path(OZONE_URI_DELIMITER));
}

@Test
public void testSetTimes() throws Exception {
// Create a file
String testKeyName = "testKey1";
Path path = new Path(OZONE_URI_DELIMITER, testKeyName);
try (FSDataOutputStream stream = fs.create(path)) {
stream.write(1);
}

long mtime = 1000;
fs.setTimes(path, mtime, 2000);

FileStatus fileStatus = fs.getFileStatus(path);
// verify that mtime is updated as expected.
assertEquals(mtime, fileStatus.getModificationTime());

long mtimeDontUpdate = -1;
fs.setTimes(path, mtimeDontUpdate, 2000);

fileStatus = fs.getFileStatus(path);
// verify that mtime is NOT updated as expected.
assertEquals(mtime, fileStatus.getModificationTime());
setTimes(ROOT);
}

@Test
Expand Down
80 changes: 4 additions & 76 deletions ...tion-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,7 @@

import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.fs.CommonPathCapabilities.FS_ACLS;
import static org.apache.hadoop.fs.CommonPathCapabilities.FS_CHECKSUMS;
import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
Expand Down Expand Up @@ -79,7 +76,6 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.StreamCapabilities;
Expand Down Expand Up @@ -1558,38 +1554,7 @@ void testGetTrashRoots() throws IOException {

@Test
void testFileDelete() throws Exception {
Path grandparent = new Path(bucketPath, "testBatchDelete");
Path parent = new Path(grandparent, "parent");
Path childFolder = new Path(parent, "childFolder");
// BatchSize is 5, so we're going to set a number that's not a
// multiple of 5. In order to test the final number of keys less than
// batchSize can also be deleted.
for (int i = 0; i < 8; i++) {
Path childFile = new Path(parent, "child" + i);
Path childFolderFile = new Path(childFolder, "child" + i);
ContractTestUtils.touch(fs, childFile);
ContractTestUtils.touch(fs, childFolderFile);
}

assertEquals(1, fs.listStatus(grandparent).length);
assertEquals(9, fs.listStatus(parent).length);
assertEquals(8, fs.listStatus(childFolder).length);

assertTrue(fs.delete(grandparent, true));
assertFalse(fs.exists(grandparent));
for (int i = 0; i < 8; i++) {
Path childFile = new Path(parent, "child" + i);
// Make sure all keys under testBatchDelete/parent should be deleted
assertFalse(fs.exists(childFile));

// Test to recursively delete child folder, make sure all keys under
// testBatchDelete/parent/childFolder should be deleted.
Path childFolderFile = new Path(childFolder, "child" + i);
assertFalse(fs.exists(childFolderFile));
}
// Will get: WARN ozone.BasicOzoneFileSystem delete: Path does not exist.
// This will return false.
assertFalse(fs.delete(parent, true));
fileDelete(bucketPath);
}

/**
Expand Down Expand Up @@ -1708,25 +1673,7 @@ && getOMMetrics().getNumTrashFilesDeletes()
@Test
void testCreateWithInvalidPaths() {
assumeFalse(isBucketFSOptimized);

// Test for path with ..
Path parent = new Path("../../../../../d1/d2/");
Path file1 = new Path(parent, "key1");
checkInvalidPath(file1);

// Test for path with :
file1 = new Path("/:/:");
checkInvalidPath(file1);

// Test for path with scheme and authority.
file1 = new Path(fs.getUri() + "/:/:");
checkInvalidPath(file1);
}

private void checkInvalidPath(Path path) {
InvalidPathException exception = assertThrows(InvalidPathException.class,
() -> fs.create(path, false));
assertThat(exception.getMessage()).contains("Invalid path Name");
createWithInvalidPaths();
}

@Test
Expand Down Expand Up @@ -2043,8 +1990,7 @@ void testSnapshotRead() throws Exception {

@Test
void testFileSystemDeclaresCapability() throws Throwable {
assertHasPathCapabilities(fs, getBucketPath(), FS_ACLS);
assertHasPathCapabilities(fs, getBucketPath(), FS_CHECKSUMS);
fileSystemDeclaresCapability(getBucketPath());
}

@Test
Expand Down Expand Up @@ -2111,29 +2057,11 @@ void testSnapshotDiff() throws Exception {

@Test
void testSetTimes() throws Exception {
// Create a file
OzoneBucket bucket1 =
TestDataUtil.createVolumeAndBucket(client, bucketLayout);
Path volumePath1 = new Path(OZONE_URI_DELIMITER, bucket1.getVolumeName());
Path bucketPath1 = new Path(volumePath1, bucket1.getName());
Path path = new Path(bucketPath1, "key1");
try (FSDataOutputStream stream = fs.create(path)) {
stream.write(1);
}

long mtime = 1000;
fs.setTimes(path, mtime, 2000);

FileStatus fileStatus = fs.getFileStatus(path);
// verify that mtime is updated as expected.
assertEquals(mtime, fileStatus.getModificationTime());

long mtimeDontUpdate = -1;
fs.setTimes(path, mtimeDontUpdate, 2000);

fileStatus = fs.getFileStatus(path);
// verify that mtime is NOT updated as expected.
assertEquals(mtime, fileStatus.getModificationTime());
setTimes(bucketPath1);
}

@Test
Expand Down
79 changes: 79 additions & 0 deletions ...ne/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

package org.apache.hadoop.fs.ozone;

import static org.apache.hadoop.fs.CommonPathCapabilities.FS_ACLS;
import static org.apache.hadoop.fs.CommonPathCapabilities.FS_CHECKSUMS;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
Expand All @@ -25,6 +28,7 @@
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

Expand All @@ -35,8 +39,10 @@
import java.util.Set;
import java.util.TreeSet;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
Expand Down Expand Up @@ -302,6 +308,79 @@ void deleteCreatesFakeParentDir(Path root) throws IOException {
assertTrue(fs.delete(grandparent, true));
}

void fileDelete(Path fsRootPath) throws Exception {
Path grandparent = new Path(fsRootPath, "testBatchDelete");
Path parent = new Path(grandparent, "parent");
Path childFolder = new Path(parent, "childFolder");
FileSystem fs = getFs();
// BatchSize is 5, so use a count that is not a multiple of 5.
for (int i = 0; i < 8; i++) {
Path childFile = new Path(parent, "child" + i);
Path childFolderFile = new Path(childFolder, "child" + i);
ContractTestUtils.touch(fs, childFile);
ContractTestUtils.touch(fs, childFolderFile);
}

assertEquals(1, fs.listStatus(grandparent).length);
assertEquals(9, fs.listStatus(parent).length);
assertEquals(8, fs.listStatus(childFolder).length);

assertTrue(fs.delete(grandparent, true));
assertFalse(fs.exists(grandparent));
for (int i = 0; i < 8; i++) {
// Make sure all keys under testBatchDelete/parent should be deleted.
assertFalse(fs.exists(new Path(parent, "child" + i)));

// Test recursive delete of child folder.
assertFalse(fs.exists(new Path(childFolder, "child" + i)));
}
// Will get: WARN ozone.BasicOzoneFileSystem delete: Path does not exist.
assertFalse(fs.delete(parent, true));
}

void createWithInvalidPaths() {
// Test for path with ..
checkInvalidPath(new Path("../../../../../d1/d2/", "key1"));

// Test for path with :
checkInvalidPath(new Path("/:/:"));

// Test for path with scheme and authority.
checkInvalidPath(new Path(getFs().getUri() + "/:/:"));
}

private void checkInvalidPath(Path testPath) {
InvalidPathException exception = assertThrows(InvalidPathException.class,
() -> getFs().create(testPath, false));
assertThat(exception.getMessage()).contains("Invalid path Name");
}

void fileSystemDeclaresCapability(Path testPath) throws Throwable {
assertHasPathCapabilities(getFs(), testPath, FS_ACLS);
assertHasPathCapabilities(getFs(), testPath, FS_CHECKSUMS);
}

void setTimes(Path testPathRoot) throws Exception {
Path path = new Path(testPathRoot, "testKey1");
FileSystem fs = getFs();
try (FSDataOutputStream stream = fs.create(path)) {
stream.write(1);
}

long mtime = 1000;
fs.setTimes(path, mtime, 2000);

FileStatus fileStatus = fs.getFileStatus(path);
// Verify that mtime is updated as expected.
assertEquals(mtime, fileStatus.getModificationTime());

fs.setTimes(path, -1, 2000);

fileStatus = fs.getFileStatus(path);
// Verify that mtime is NOT updated as expected.
assertEquals(mtime, fileStatus.getModificationTime());
}

void verifyListStatus(Path root, PathFilter filter) throws Exception {
Path parent = new Path(root, "testListStatus");
Path file1 = new Path(parent, "key1");
Expand Down