From 73bb03141dc78e86637666f7a03f594b836cf637 Mon Sep 17 00:00:00 2001 From: Aryan Gupta Date: Wed, 29 Jul 2026 13:30:58 +0530 Subject: [PATCH] HDDS-16015. Make listStatusLight skip SCM pipeline refresh. --- .../apache/hadoop/ozone/om/KeyManager.java | 28 ++++++++++ .../hadoop/ozone/om/KeyManagerImpl.java | 34 ++++++++---- .../hadoop/ozone/om/OmMetadataReader.java | 36 ++++++++++++- .../apache/hadoop/ozone/om/OmSnapshot.java | 32 +++++++++-- .../apache/hadoop/ozone/om/OzoneManager.java | 11 ++-- .../hadoop/ozone/om/TestKeyManagerUnit.java | 53 +++++++++++++++++++ 6 files changed, 173 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index fdf4172c71b2..5e5c70d65eee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.service.CompactionService; import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; @@ -87,6 +88,33 @@ OmKeyInfo lookupKey(OmKeyArgs args, ResolvedBucket bucketLayout, OmKeyInfo getKeyInfo(OmKeyArgs args, ResolvedBucket buctket, String clientAddress) throws IOException; + /** + * List the status for a file or a directory and its contents. + * Optionally skips pipeline refresh for callers that only need metadata. + * + * @param args Key args + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @param clientAddress a hint to key manager, order the datanode in returned + * pipeline by distance between client and datanode. + * @param allowPartialPrefixes if partial prefixes should be allowed, + * this is needed in context of ListKeys + * @param refreshPipelineInfo if true, update pipeline location information + * using SCM-backed cache; if false, skip it + * @return list of file status + * @throws IOException if file or bucket or volume does not exist + */ + default List listStatus(OmKeyArgs args, boolean recursive, + String startKey, long numEntries, String clientAddress, + boolean allowPartialPrefixes, boolean refreshPipelineInfo) + throws IOException { + return listStatus(args, recursive, startKey, numEntries, clientAddress, + allowPartialPrefixes); + } + /** * Returns a list of keys represented by {@link OmKeyInfo} * in the given bucket. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 08b6d6abbf18..ca51b348c203 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1902,6 +1902,16 @@ public List listStatus(OmKeyArgs args, boolean recursive, public List listStatus(OmKeyArgs args, boolean recursive, String startKey, long numEntries, String clientAddress, boolean allowPartialPrefixes) throws IOException { + return listStatus(args, recursive, startKey, numEntries, clientAddress, + allowPartialPrefixes, true); + } + + @Override + @SuppressWarnings("methodlength") + public List listStatus(OmKeyArgs args, boolean recursive, + String startKey, long numEntries, String clientAddress, + boolean allowPartialPrefixes, boolean refreshPipelineInfo) + throws IOException { Objects.requireNonNull(args, "Key args can not be null"); String volumeName = args.getVolumeName(); String bucketName = args.getBucketName(); @@ -1920,7 +1930,8 @@ public List listStatus(OmKeyArgs args, boolean recursive, Collection statuses = statusHelper.listStatusFSO(args, startKey, numEntries, clientAddress, allowPartialPrefixes); - return buildFinalStatusList(statuses, args, clientAddress); + return buildFinalStatusList(statuses, args, clientAddress, + refreshPipelineInfo); } // A map sorted by OmKey to combine results from TableCache and DB. @@ -1988,7 +1999,9 @@ public List listStatus(OmKeyArgs args, boolean recursive, slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0])); } - refreshPipelineFromCache(keyInfoList); + if (refreshPipelineInfo) { + refreshPipelineFromCache(keyInfoList); + } if (args.getSortDatanodes()) { sortDatanodes(clientAddress, keyInfoList); @@ -2093,7 +2106,7 @@ private void findKeyInDbWithIterator(boolean recursive, String startKey, private List buildFinalStatusList( Collection statusesCollection, OmKeyArgs omKeyArgs, - String clientAddress) + String clientAddress, boolean refreshPipelineInfo) throws IOException { List fileStatusFinalList = new ArrayList<>(); List keyInfoList = new ArrayList<>(); @@ -2105,19 +2118,22 @@ private List buildFinalStatusList( fileStatusFinalList.add(fileStatus); } return sortPipelineInfo(fileStatusFinalList, keyInfoList, - omKeyArgs, clientAddress); + omKeyArgs, clientAddress, refreshPipelineInfo); } private List sortPipelineInfo( List fileStatusFinalList, List keyInfoList, - OmKeyArgs omKeyArgs, String clientAddress) throws IOException { + OmKeyArgs omKeyArgs, String clientAddress, boolean refreshPipelineInfo) + throws IOException { if (omKeyArgs.getLatestVersionLocation()) { slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0])); } - // refreshPipeline flag check has been removed as part of - // https://issues.apache.org/jira/browse/HDDS-3658. - // Please refer this jira for more details. - refreshPipelineFromCache(keyInfoList); + if (refreshPipelineInfo) { + // refreshPipeline flag check has been removed as part of + // https://issues.apache.org/jira/browse/HDDS-3658. + // Please refer this jira for more details. + refreshPipelineFromCache(keyInfoList); + } if (omKeyArgs.getSortDatanodes()) { sortDatanodes(clientAddress, keyInfoList); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java index 64f46089c066..2dfd1ef4e82a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java @@ -259,8 +259,40 @@ public List listStatus(OmKeyArgs args, boolean recursive, public List listStatusLight(OmKeyArgs args, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException { - List ozoneFileStatuses = - listStatus(args, recursive, startKey, numEntries, allowPartialPrefixes); + long maxListingPageSize = ozoneManager.getConfiguration().getInt( + OZONE_FS_LISTING_PAGE_SIZE_MAX, + OZONE_FS_LISTING_PAGE_SIZE_DEFAULT); + maxListingPageSize = OzoneConfigUtil.limitValue(numEntries, + OZONE_FS_LISTING_PAGE_SIZE, OZONE_FS_LISTING_PAGE_SIZE_MAX, + maxListingPageSize); + + ResolvedBucket bucket = ozoneManager.resolveBucketLink(args); + boolean auditSuccess = true; + Map auditMap = bucket.audit(args.toAuditMap()); + OmKeyArgs resolvedArgs = bucket.update(args); + + List ozoneFileStatuses; + try { + if (isAclEnabled) { + checkAcls(getResourceType(resolvedArgs), StoreType.OZONE, ACLType.READ, + bucket, resolvedArgs.getKeyName()); + } + metrics.incNumListStatus(); + ozoneFileStatuses = keyManager.listStatus(resolvedArgs, recursive, + startKey, maxListingPageSize, getClientAddress(), + allowPartialPrefixes, false); + } catch (Exception ex) { + metrics.incNumListStatusFails(); + auditSuccess = false; + audit.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_STATUS, + auditMap, ex)); + throw ex; + } finally { + if (auditSuccess) { + audit.logReadSuccess(buildAuditMessageForSuccess( + OMAction.LIST_STATUS, auditMap)); + } + } return ozoneFileStatuses.stream() .map(OzoneFileStatusLight::fromOzoneFileStatus) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java index 5147eafe628f..6d3a56f40ed0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java @@ -133,12 +133,13 @@ recursive, normalizeKeyName(startKey), numEntries, public List listStatusLight(OmKeyArgs args, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException { - - List ozoneFileStatuses = - listStatus(args, recursive, startKey, numEntries, allowPartialPrefixes); + List ozoneFileStatuses = omMetadataReader + .listStatusLight(normalizeOmKeyArgs(args), + recursive, normalizeKeyName(startKey), numEntries, + allowPartialPrefixes); return ozoneFileStatuses.stream() - .map(OzoneFileStatusLight::fromOzoneFileStatus) + .map(this::denormalizeOzoneFileStatusLight) .collect(Collectors.toList()); } @@ -285,6 +286,29 @@ private OzoneFileStatus denormalizeOzoneFileStatus( omKeyInfo, fileStatus.getBlockSize(), fileStatus.isDirectory()); } + private OzoneFileStatusLight denormalizeOzoneFileStatusLight( + OzoneFileStatusLight fileStatus) { + if (fileStatus == null || fileStatus.getKeyInfo() == null) { + return fileStatus; + } + BasicOmKeyInfo keyInfo = fileStatus.getKeyInfo(); + BasicOmKeyInfo denormalized = new BasicOmKeyInfo.Builder() + .setVolumeName(keyInfo.getVolumeName()) + .setBucketName(keyInfo.getBucketName()) + .setKeyName(denormalizeKeyName(keyInfo.getKeyName())) + .setDataSize(keyInfo.getDataSize()) + .setCreationTime(keyInfo.getCreationTime()) + .setModificationTime(keyInfo.getModificationTime()) + .setReplicationConfig(keyInfo.getReplicationConfig()) + .setIsFile(keyInfo.isFile()) + .setETag(keyInfo.getETag()) + .setOwnerName(keyInfo.getOwnerName()) + .setIsEncrypted(keyInfo.isEncrypted()) + .build(); + return new OzoneFileStatusLight( + denormalized, fileStatus.getBlockSize(), fileStatus.isDirectory()); + } + private KeyInfoWithVolumeContext denormalizeKeyInfoWithVolumeContext( KeyInfoWithVolumeContext k) { return new KeyInfoWithVolumeContext.Builder() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 1910d92e9690..ee6a29d2eee4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -4070,12 +4070,11 @@ public List listStatus(OmKeyArgs args, boolean recursive, public List listStatusLight(OmKeyArgs args, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException { - List ozoneFileStatuses = - listStatus(args, recursive, startKey, numEntries, allowPartialPrefixes); - - return ozoneFileStatuses.stream() - .map(OzoneFileStatusLight::fromOzoneFileStatus) - .collect(Collectors.toList()); + try (UncheckedAutoCloseableSupplier rcReader = + getReader(args)) { + return rcReader.get().listStatusLight( + args, recursive, startKey, numEntries, allowPartialPrefixes); + } } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 9b1844212073..07fef4b7674e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -79,12 +79,14 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.ozone.test.OzoneTestBase; import org.apache.ratis.util.ExitUtils; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -766,4 +768,55 @@ public void listStatus() throws Exception { null, Long.MAX_VALUE, client); verify(containerClient, times(1)).getContainerWithPipelineBatch(anySet()); } + + @Test + public void listStatusLightDoesNotRefreshPipeline() throws Exception { + String volume = volumeName(); + String bucket = "bucket-light"; + String keyPrefix = "key-light-"; + + OMRequestTestUtils.addVolumeToDB(volume, OzoneConsts.OZONE, metadataManager); + OMRequestTestUtils.addBucketToDB(volume, bucket, metadataManager); + + final Pipeline pipeline = MockPipeline.createPipeline(3); + for (long i = 1; i <= 5; i++) { + final long containerID = CONTAINER_ID.incrementAndGet(); + final OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(containerID, 1L)) + .setPipeline(pipeline) + .setOffset(0) + .setLength(1024) + .build(); + + OmKeyInfo keyInfo = new OmKeyInfo.Builder() + .setVolumeName(volume) + .setBucketName(bucket) + .setCreationTime(Time.now()) + .setOmKeyLocationInfos(singletonList( + new OmKeyLocationInfoGroup(0, new ArrayList<>()))) + .setReplicationConfig(RatisReplicationConfig + .getInstance(ReplicationFactor.THREE)) + .setKeyName(keyPrefix + i) + .setObjectID(i) + .setUpdateID(i) + .build(); + keyInfo.appendNewBlocks(singletonList(keyLocationInfo), false); + OMRequestTestUtils.addKeyToOM(metadataManager, keyInfo); + } + + OmKeyArgs.Builder builder = new OmKeyArgs.Builder() + .setVolumeName(volume) + .setBucketName(bucket) + .setKeyName(""); + + List fileStatusList; + try (UncheckedAutoCloseableSupplier rcReader = + om.getOmMetadataReader()) { + fileStatusList = rcReader.get().listStatusLight(builder.build(), false, + null, Long.MAX_VALUE, false); + } + + assertEquals(5, fileStatusList.size()); + verify(containerClient, times(0)).getContainerWithPipelineBatch(anySet()); + } }