diff --git a/tests/functional/countItems.js b/tests/functional/countItems.js index a72079c5..a52fc760 100644 --- a/tests/functional/countItems.js +++ b/tests/functional/countItems.js @@ -20,7 +20,7 @@ const USERSBUCKET = '__usersbucket'; const expectedCountItems = { objects: 90, versions: 60, - buckets: 9, + buckets: 10, dataManaged: { total: { curr: 15000, prev: 12000 }, byLocation: { @@ -100,6 +100,10 @@ const expectedDataMetrics = { objectCount: { current: 10n, deleteMarker: 0n, nonCurrent: 10n }, usedCapacity: { current: 1000n, nonCurrent: 1000n }, }, + [`bucket_test-bucket-empty_${testBucketCreationDate}`]: { + objectCount: { current: 0n, deleteMarker: 0n, nonCurrent: 0n }, + usedCapacity: { current: 0n, nonCurrent: 0n }, + }, 'location_secondary-location-1': { objectCount: { current: 30n, deleteMarker: 0n, nonCurrent: 30n }, usedCapacity: { current: 3000n, nonCurrent: 3000n }, }, @@ -199,6 +203,25 @@ function populateMongo(client, callback) { ], cb), callback); } +const emptyBucket = BucketInfo.fromObj({ ...testBucketMD, _name: 'test-bucket-empty' }); + +function populateEmptyBucket(client, callback) { + return async.series([ + next => client.createBucket(emptyBucket.getName(), emptyBucket, logger, next), + next => client.putObject( + USERSBUCKET, + `${emptyBucket.getOwner()}${constants.splitter}${emptyBucket.getName()}`, + testUserBucketInfo.value, + { + versioning: false, + versionId: null, + }, + logger, + next, + ), + ], callback); +} + jest.setTimeout(120000); describe('CountItems', () => { const oldEnv = process.env; @@ -222,6 +245,7 @@ describe('CountItems', () => { async.series([ next => client.setup(next), next => populateMongo(client, next), + next => populateEmptyBucket(client, next), ], done); setTimeoutSpy = jest.spyOn(global, 'setTimeout'); setTimeoutSpy.mockImplementation((callback, delay) => callback()); diff --git a/tests/unit/utils/S3UtilsMongoClient.js b/tests/unit/utils/S3UtilsMongoClient.js index 1f1929a4..01b5fa77 100644 --- a/tests/unit/utils/S3UtilsMongoClient.js +++ b/tests/unit/utils/S3UtilsMongoClient.js @@ -1059,6 +1059,50 @@ function uploadObjects(client, bucketName, objectList, callback) { }, callback); } +describe('S3UtilsMongoClient::_seedEmptyBucketMetrics', () => { + const bucketInfo = BucketInfo.fromObj({ ...testBucketMD }); + const locationConfig = { 'us-east-1': { objectId: 'us-east-1' } }; + const bucketResource = `test-bucket_${testBucketCreationDate}`; + + it('should seed zero-value bucket, account and location entries and return true', () => { + const collRes = { bucket: {}, location: {}, account: {} }; + const log = { warn: sinon.spy() }; + const seeded = mongoTestClient._seedEmptyBucketMetrics(collRes, bucketResource, bucketInfo, locationConfig, false, log); + assert.strictEqual(seeded, true); + assert.deepStrictEqual(Object.keys(collRes.bucket), [bucketResource]); + assert.deepStrictEqual(Object.keys(collRes.account), [testAccountCanonicalId]); + assert.deepStrictEqual(Object.keys(collRes.location), ['us-east-1']); + assert.strictEqual(log.warn.called, false); + }); + + it('should warn and seed nothing when no __usersbucket creation date is available', () => { + const collRes = { bucket: {}, location: {}, account: {} }; + const log = { warn: sinon.spy() }; + const seeded = mongoTestClient._seedEmptyBucketMetrics(collRes, undefined, bucketInfo, locationConfig, false, log); + assert.strictEqual(seeded, false); + assert.deepStrictEqual(collRes, { bucket: {}, location: {}, account: {} }); + assert.strictEqual(log.warn.calledOnce, true); + }); + + it('should warn and seed nothing when the scan had processing errors', () => { + const collRes = { bucket: {}, location: {}, account: {} }; + const log = { warn: sinon.spy() }; + const seeded = mongoTestClient._seedEmptyBucketMetrics(collRes, bucketResource, bucketInfo, locationConfig, true, log); + assert.strictEqual(seeded, false); + assert.deepStrictEqual(collRes, { bucket: {}, location: {}, account: {} }); + assert.strictEqual(log.warn.calledOnce, true); + }); + + it('should seed nothing when the bucket already produced metrics', () => { + const collRes = { bucket: { existing: {} }, location: {}, account: {} }; + const log = { warn: sinon.spy() }; + const seeded = mongoTestClient._seedEmptyBucketMetrics(collRes, bucketResource, bucketInfo, locationConfig, false, log); + assert.strictEqual(seeded, false); + assert.deepStrictEqual(collRes, { bucket: { existing: {} }, location: {}, account: {} }); + assert.strictEqual(log.warn.called, false); + }); +}); + describe('S3UtilsMongoClient, tests', () => { const hr = 1000 * 60 * 60; let client; @@ -1150,14 +1194,76 @@ describe('S3UtilsMongoClient, tests', () => { dataStore: 'us-east-1', }; + const zeroUsedCapacity = { + current: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUParts: 0n, + }; + const zeroObjectCount = { + current: 0n, + nonCurrent: 0n, + _currentCold: 0n, + _nonCurrentCold: 0n, + _currentRestored: 0n, + _currentRestoring: 0n, + _nonCurrentRestored: 0n, + _nonCurrentRestoring: 0n, + _incompleteMPUUploads: 0n, + deleteMarker: 0n, + }; + const zeroMetricsEntry = { + usedCapacity: zeroUsedCapacity, + objectCount: zeroObjectCount, + }; const tests = [ [ - 'getObjectMDStats() should return zero-result when no objects in the bucket', + 'getObjectMDStats() should seed zero-value metrics for an empty bucket, account and location', { bucketName: 'test-bucket', isVersioned: false, objectList: [], }, + { + dataManaged: { + locations: { 'us-east-1': { curr: 0, prev: 0 } }, + total: { curr: 0, prev: 0 }, + }, + objects: 0, + stalled: 0, + versions: 0, + dataMetrics: { + bucket: { + [`test-bucket_${testBucketCreationDate}`]: { + ...zeroMetricsEntry, + locations: { 'us-east-1': { ...zeroMetricsEntry } }, + }, + }, + location: { + 'us-east-1': { ...zeroMetricsEntry }, + }, + account: { + [testAccountCanonicalId]: { + ...zeroMetricsEntry, + locations: { 'us-east-1': { ...zeroMetricsEntry } }, + }, + }, + }, + }, + ], + [ + 'getObjectMDStats() should not seed metrics when the bucket has no __usersbucket creation date', + { + bucketName: 'test-bucket-no-usersbucket', + isVersioned: false, + objectList: [], + skipUsersBucket: true, + }, { dataManaged: { locations: {}, @@ -2806,20 +2912,23 @@ describe('S3UtilsMongoClient, tests', () => { isVersioned, objectList, inflights, + skipUsersBucket, } = testCase; return async.waterfall([ next => createBucket(client, bucketName, isVersioned, err => next(err)), - next => client.putObject( - USERSBUCKET, - `${testBucketMD._owner}${constants.splitter}${bucketName}`, - testUserBucketInfo.value, - { - versioning: false, - versionId: null, - }, - logger, - next, - ), + next => (skipUsersBucket + ? next() + : client.putObject( + USERSBUCKET, + `${testBucketMD._owner}${constants.splitter}${bucketName}`, + testUserBucketInfo.value, + { + versioning: false, + versionId: null, + }, + logger, + next, + )), next => uploadObjects(client, bucketName, objectList, err => next(err)), next => client.getBucketAttributes(bucketName, logger, next), (bucketInfo, next) => { @@ -2845,6 +2954,42 @@ describe('S3UtilsMongoClient, tests', () => { next => client.deleteBucket(bucketName, logger, next), ], done); })); + + it('getObjectMDStats() should not seed metrics when entries fail processing', done => { + const name = 'test-bucket-scan-errors'; + return async.waterfall([ + next => createBucket(client, name, false, err => next(err)), + next => client.putObject( + USERSBUCKET, + `${testBucketMD._owner}${constants.splitter}${name}`, + testUserBucketInfo.value, + { versioning: false, versionId: null }, + logger, + next, + ), + next => uploadObjects( + client, + name, + [{ name: 'obj1', ownerId: testAccountCanonicalId, lastModified: new Date(Date.now()) }], + err => next(err), + ), + next => client.getBucketAttributes(name, logger, next), + (bucketInfo, next) => { + // Every entry fails _processEntryData, so collRes stays empty even + // though the bucket has an object: seeding zero here would under-report. + const stub = sinon.stub(client, '_processEntryData').returns({ error: new Error('boom') }); + return client.getObjectMDStats(name, BucketInfo.fromObj(bucketInfo), false, logger, (err, res) => { + stub.restore(); + if (err) { + return next(err); + } + assert.deepStrictEqual(res.dataMetrics.bucket, {}, 'must not seed when entries failed processing'); + return next(); + }); + }, + next => client.deleteBucket(name, logger, next), + ], done); + }); }); describe('S3UtilsMongoClient, update inflight deltas', () => { diff --git a/utils/S3UtilsMongoClient.js b/utils/S3UtilsMongoClient.js index 2c9d0df8..3934a0b9 100644 --- a/utils/S3UtilsMongoClient.js +++ b/utils/S3UtilsMongoClient.js @@ -164,8 +164,10 @@ class S3UtilsMongoClient extends MongoClientInterface { }; let stalledCount = 0; let bucketKey; + let bucketResource; let inflightsPreScan = 0n; let accountBucket; + let scanHadErrors = false; const cmpDate = new Date(); cmpDate.setHours(cmpDate.getHours() - 1); @@ -184,7 +186,8 @@ class S3UtilsMongoClient extends MongoClientInterface { || bucketStatus.Status === 'Suspended')); if (bucketCreationDate) { - bucketKey = `bucket_${bucketName}_${new Date(bucketCreationDate).getTime()}`; + bucketResource = `${bucketName}_${new Date(bucketCreationDate).getTime()}`; + bucketKey = `bucket_${bucketResource}`; inflightsPreScan = await this.readStorageConsumptionInflights(bucketKey, log); } @@ -229,6 +232,7 @@ class S3UtilsMongoClient extends MongoClientInterface { error, }); monitoring.objectsCount.inc({ status: 'error' }); + scanHadErrors = true; return; } @@ -400,6 +404,10 @@ class S3UtilsMongoClient extends MongoClientInterface { }, ); + if (this._seedEmptyBucketMetrics(collRes, bucketResource, bucketInfo, locationConfig, scanHadErrors, log)) { + accountBucket = bucketInfo.getOwner(); + } + const retResult = this._handleResults(collRes, isVer); retResult.stalled = stalledCount; @@ -432,6 +440,50 @@ class S3UtilsMongoClient extends MongoClientInterface { } } + /** + * Seed zero-value metrics for empty buckets. + * @param{object} collRes - per-resource metrics accumulator, mutated in place + * @param{string} [bucketResource] - `${bucketName}_${creationDate}` key, absent when no __usersbucket date + * @param{object} bucketInfo - bucket attributes + * @param{object} locationConfig - locationConfig.json + * @param{boolean} scanHadErrors - whether any entry failed processing during the scan + * @param{object} log - werelogs logger + * @returns{boolean} true when zero metrics were seeded, false otherwise + */ + _seedEmptyBucketMetrics(collRes, bucketResource, bucketInfo, locationConfig, scanHadErrors, log) { + if (Object.keys(collRes.bucket).length > 0) { + return false; + } + if (scanHadErrors) { + // An empty collRes with processing errors means we could not account + // for the bucket's objects, not that it is empty: seeding zero would + // under-report a non-empty bucket, so leave it to the next scan. + log.warn('cannot seed empty-bucket metrics: the scan had processing errors, emptiness is unverified', { + method: 'getObjectMDStats', + bucketName: bucketInfo.getName(), + }); + return false; + } + if (!bucketResource) { + log.warn('cannot seed empty-bucket metrics: bucket has no __usersbucket creation date entry', { + method: 'getObjectMDStats', + bucketName: bucketInfo.getName(), + owner: bucketInfo.getOwner(), + }); + return false; + } + // eslint-disable-next-line no-param-reassign + collRes.bucket[bucketResource] = { ...baseMetricsObject, locations: {} }; + // eslint-disable-next-line no-param-reassign + collRes.account[bucketInfo.getOwner()] = { ...baseMetricsObject, locations: {} }; + const bucketLocation = bucketInfo.getLocationConstraint(); + if (bucketLocation && locationConfig && locationConfig[bucketLocation]) { + // eslint-disable-next-line no-param-reassign + collRes.location[locationConfig[bucketLocation].objectId] = { ...baseMetricsObject }; + } + return true; + } + /** * @param{string} bucketName - * @param{object} bucketInfo - bucket attributes