diff --git a/src/control-plane-services/api-keys/README.md b/src/control-plane-services/api-keys/README.md index ebc7084520..3a1f8fac3a 100644 --- a/src/control-plane-services/api-keys/README.md +++ b/src/control-plane-services/api-keys/README.md @@ -3,6 +3,39 @@ NVIDIA API Keys service manages the lifecycle of API keys (issue, authenticate, introspect, rotate, revoke). +## Synchronous bulk operations + +`SyncKeysV2OperationsService` provides key-owner, account-admin and platform-admin +entry points. HTTP integration with the actor controllers is pending. The key-owner +entry point fixes the user, and the account-admin entry point fixes the account. + +Set `operation` to `REVOKED` or `DELETE`. Each dimension not fixed by the actor +requires an `accounts`, `users`, or `issuerServices` array of distinct IDs or +the singleton `["*"]`. Platform requests selecting all three wildcards also +require `confirm_platform_wide: true`. Configure the array limit with +`apikeys.bulk-max-selector-values` (default 10). + +The service returns `KeyOperationResponse` after processing finishes. Selection +uses the submission time as its creation cutoff. Processing stops on failure, +and the exception's problem details include `bulkOperation` with `FAILED` status and counts +for eligible keys processed so far. `partialApplication` indicates that a mutation was +attempted. Earlier writes remain applied. Retries start a new selection with +a new cutoff. + +A failure between v2 and legacy writes can partially update one migrated key. +The operation reports failure until both representations are updated. Retrying +repeats the per-key mutation to repair partial writes. + +Selection fails when a legacy key lacks the creation time or issuer mapping +needed to process it. Correct the record or deployment mapping before retrying. + +Bulk mutations clear cached key rows and introspection responses on this instance +before the response, including after partial failure. Other replicas expire cached key +rows after 60 seconds and cached introspection responses after 60 seconds. +These layers can retain a successful decision for up to 120 seconds after a +mutation, plus in-flight request time. Existing caller-side HTTP caching can +add another 360 seconds for introspection or 60 seconds for evaluation. + ## Minimum Requirements - [Eclipse Temurin OpenJDK 25](https://adoptium.net/temurin/releases/) diff --git a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/caching/PrefetchingCache.java b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/caching/PrefetchingCache.java index 63b0b4484d..6009b8b2bb 100644 --- a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/caching/PrefetchingCache.java +++ b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/caching/PrefetchingCache.java @@ -52,8 +52,7 @@ public class PrefetchingCache { private final LoadingCache keyLastFetchTime; /** - * Methods invalidate all cache records. Not intended to be used in normal flows, only during - * testing. + * Clears cached values and prefetch tracking after management mutations. */ public void invalidate() { records.clear(); diff --git a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/config/NakProperties.java b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/config/NakProperties.java index 64890d198b..643f59b54a 100644 --- a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/config/NakProperties.java +++ b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/config/NakProperties.java @@ -59,6 +59,7 @@ public class NakProperties { private String dataDomainKey; @NotNull private Duration keepAfterExpiredDuration; + private int bulkMaxSelectorValues; @NotNull @Valid diff --git a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/persistance/dao/KeysDao.java b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/persistance/dao/KeysDao.java index 59b051fdf6..f885b81832 100644 --- a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/persistance/dao/KeysDao.java +++ b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/persistance/dao/KeysDao.java @@ -34,10 +34,14 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import org.springframework.data.cassandra.core.CassandraBatchOperations; import org.springframework.data.cassandra.core.CassandraTemplate; import org.springframework.data.cassandra.core.WriteResult; +import org.springframework.data.cassandra.core.cql.QueryOptions; +import org.springframework.data.cassandra.core.query.Criteria; +import org.springframework.data.cassandra.core.query.Query; import org.springframework.stereotype.Service; @Service @@ -160,6 +164,24 @@ public List list(KeyOwnerType ownerType, String ownerId) .toList(); } + /** Streams legacy rows in bounded driver pages, scanning when no owner is specified. */ + public Stream streamForBulk( + KeyOwnerType ownerType, String ownerId, String serviceId, int pageSize) { + var query = Query.empty(); + if (ownerId != null) { + query = query.and(Criteria.where("owner_type").is(ownerType)) + .and(Criteria.where("owner_id").is(ownerId)); + if (serviceId != null) { + query = query.and(Criteria.where("issuer_service_id").is(serviceId)); + } + } + return cassandraTemplate.stream(query.queryOptions(QueryOptions.builder().pageSize(pageSize).build()), + KeyByOwnerAndServiceModel.class) + .filter(row -> row.getKeyId() != null && row.getOwnerType() == ownerType) + .filter(row -> serviceId == null || serviceId.equals(row.getIssuerServiceId())) + .map(this::modelToValidatedVo); + } + private KeyVo modelToValidatedVo(KeyModel keyModel) { return keyConverter.modelToVo(keyModel); diff --git a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/persistance/dao/KeysV2Dao.java b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/persistance/dao/KeysV2Dao.java index ca49931b57..d9b2d4c656 100644 --- a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/persistance/dao/KeysV2Dao.java +++ b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/persistance/dao/KeysV2Dao.java @@ -30,6 +30,7 @@ import com.nvidia.apikeys.vo.KeysByAccountSliceVo; import com.nvidia.apikeys.vo.KeysByOwnerAndAccountAndServiceVo; import java.nio.ByteBuffer; +import java.time.Instant; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -38,7 +39,10 @@ import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.springframework.data.cassandra.core.CassandraTemplate; +import org.springframework.data.cassandra.core.cql.QueryOptions; import org.springframework.data.cassandra.core.query.CassandraPageRequest; +import org.springframework.data.cassandra.core.query.Criteria; +import org.springframework.data.cassandra.core.query.Query; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Slice; @@ -120,6 +124,26 @@ public KeysByAccountSliceVo list(String ncaId, int limit, String cursor) { limit, cursor); } + /** Streams indexed bulk matches in driver-sized pages; null dimensions are unrestricted. */ + public Stream streamForBulk( + String ncaId, KeyOwnerType ownerType, String ownerId, String issuerServiceId, + Instant cutoff, int pageSize) { + var query = Query.query(Criteria.where("owner_type").is(ownerType)) + .and(Criteria.where("created_at").lte(cutoff)); + if (ncaId != null) { + query = query.and(Criteria.where("nca_id").is(ncaId)); + } + if (ownerId != null) { + query = query.and(Criteria.where("owner_id").is(ownerId)); + } + if (issuerServiceId != null) { + query = query.and(Criteria.where("issuer_service_id").is(issuerServiceId)); + } + return cassandraTemplate.stream(query.queryOptions(QueryOptions.builder().pageSize(pageSize).build()), + KeysByOwnerAndAccountAndServiceModel.class) + .map(keyConverter::modelToVo); + } + public KeysByAccountSliceVo list( String ncaId, String issuerServiceId, int limit, String cursor) { return slice( diff --git a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/KeysV2BulkSelectionService.java b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/KeysV2BulkSelectionService.java new file mode 100644 index 0000000000..31c3e24b4b --- /dev/null +++ b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/KeysV2BulkSelectionService.java @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.apikeys.services; + +import static com.nvidia.apikeys.vo.KeyOwnerType.USER; + +import com.nvidia.apikeys.persistance.dao.KeysDao; +import com.nvidia.apikeys.persistance.dao.KeysV2Dao; +import com.nvidia.apikeys.vo.KeyByOwnerAndServiceVo; +import com.nvidia.apikeys.vo.KeysByOwnerAndAccountAndServiceVo; +import java.time.Instant; +import java.util.stream.Stream; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +/** Paged discovery for bulk selectors, including the configured legacy account. */ +@Service +public class KeysV2BulkSelectionService { + + private final KeysV2Dao keysV2Dao; + private final KeysDao keysDao; + private final LegacyAccountTranslator legacyAccount; + private final int pageSize; + + public KeysV2BulkSelectionService( + KeysV2Dao keysV2Dao, KeysDao keysDao, LegacyAccountTranslator legacyAccount, + @Value("${spring.cassandra.request.page-size:5000}") int pageSize) { + this.keysV2Dao = keysV2Dao; + this.keysDao = keysDao; + this.legacyAccount = legacyAccount; + this.pageSize = pageSize; + } + + /** Null dimensions select all values. The caller closes the returned stream. */ + public Stream select( + String account, String user, String issuer, Instant cutoff) { + var current = keysV2Dao.streamForBulk(account, USER, user, issuer, cutoff, pageSize); + try { + // Exclude migrated legacy copies before mutations can delete their v2 rows. + return Stream.concat(selectLegacy(account, user, issuer, cutoff), current); + } catch (RuntimeException | Error failure) { + current.close(); + throw failure; + } + } + + private Stream selectLegacy( + String account, String user, String issuer, Instant cutoff) { + if (account != null && !legacyAccount.isLegacy(account)) { + return Stream.empty(); + } + var legacyIssuer = issuer == null ? null + : legacyAccount.findLegacyIssuerServiceId(issuer).orElse(null); + if (issuer != null && legacyIssuer == null) { + return Stream.empty(); + } + return keysDao.streamForBulk(USER, user, legacyIssuer, pageSize) + .filter(key -> key.getCreatedAt() == null || !key.getCreatedAt().isAfter(cutoff)) + .map(this::validateAndTranslateLegacyKey) + // A migrated v2 row wins even when its creation time is after the cutoff. + .filter(key -> keysV2Dao.get(key.getNcaId(), USER, key.getOwnerId(), + key.getIssuerServiceId(), key.getKeyId()).isEmpty()); + } + + private KeysByOwnerAndAccountAndServiceVo validateAndTranslateLegacyKey(KeyByOwnerAndServiceVo key) { + if (legacyAccount.findLegacyIssuerServiceId(legacyAccount.toServiceId(key.getIssuerServiceId())) + .filter(key.getIssuerServiceId()::equals).isEmpty()) { + throw new IllegalStateException("Legacy key issuer has no configured mapping"); + } + return legacyAccount.fromLegacy(key); + } +} diff --git a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/KeysV2Service.java b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/KeysV2Service.java index d5a38cebda..021a46ed62 100644 --- a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/KeysV2Service.java +++ b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/KeysV2Service.java @@ -49,6 +49,7 @@ import com.nvidia.boot.exceptions.NotFoundException; import com.nvidia.boot.exceptions.TooManyRequestsException; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -153,22 +154,33 @@ public List listKeys( } /** - * Sets ACTIVE, SUSPENDED, or REVOKED. EXPIRED and REVOKED keys keep their status. + * Sets ACTIVE, SUSPENDED, or REVOKED. Revocation also denies retained expired keys and + * existing legacy copies, and rewrites both stored rows on retry. */ public KeyV2Vo updateKeyStatus( String ncaId, KeyOwnerType ownerType, String ownerId, String issuerServiceId, String keyId, KeyStatus status) { var stored = load(ncaId, ownerType, ownerId, issuerServiceId, keyId); var key = stored.key(); - if (key.getKeyStatus() == status) { + assertKeyScope(key, ncaId, ownerType, ownerId, issuerServiceId, keyId); + if (key.getKeyStatus() == status && status != KeyStatus.REVOKED) { return key; } - if (FINAL_KEY_STATUSES.contains(key.getKeyStatus())) { + if (FINAL_KEY_STATUSES.contains(key.getKeyStatus()) && status != KeyStatus.REVOKED) { throw new ConflictException(MESG_KEY_STATUS_FINAL.formatted(key.getKeyStatus())); } + var legacyCopy = status == KeyStatus.REVOKED && !stored.legacy() + ? findLegacyKeyRow(ncaId, ownerType, ownerId, issuerServiceId, keyId) + .map(row -> keysDao.getKeyByHash(row.getKeyHash()) + .orElseThrow(() -> new NotFoundException(MESG_KEY_NOT_FOUND))) + .map(legacyAccount::fromLegacy) + : Optional.empty(); + legacyCopy.ifPresent(copy -> assertKeyScope(copy, ncaId, ownerType, ownerId, issuerServiceId, keyId)); + var updated = key.toBuilder().keyStatus(status).build(); save(stored, updated); + legacyCopy.ifPresent(copy -> save(new StoredKey(copy, true), copy.toBuilder().keyStatus(status).build())); recordKeyAction(updated, switch (status) { case SUSPENDED -> SUSPEND; case REVOKED -> REVOKE; @@ -208,6 +220,12 @@ public void deleteKeyById( throw new NotFoundException(MESG_KEY_NOT_FOUND); } + v2Key.flatMap(row -> keysV2Dao.getKeyByHash(row.getKeyHash())) + .ifPresent(key -> assertKeyScope(key, ncaId, ownerType, ownerId, issuerServiceId, keyId)); + legacyKey.flatMap(row -> keysDao.getKeyByHash(row.getKeyHash())) + .map(legacyAccount::fromLegacy) + .ifPresent(key -> assertKeyScope(key, ncaId, ownerType, ownerId, issuerServiceId, keyId)); + v2Key.ifPresent(keysV2Dao::delete); legacyKey.ifPresent(keysDao::deleteKey); v2Key.ifPresentOrElse( @@ -308,6 +326,17 @@ private void save(StoredKey stored, KeyV2Vo updated) { } } + private static void assertKeyScope( + KeyV2Vo key, String ncaId, KeyOwnerType ownerType, String ownerId, + String issuerServiceId, String keyId) { + if (!Objects.equals(key.getNcaId(), ncaId) || key.getOwnerType() != ownerType + || !Objects.equals(key.getOwnerId(), ownerId) + || !Objects.equals(key.getIssuerServiceId(), issuerServiceId) + || !Objects.equals(key.getKeyId(), keyId)) { + throw new BadRequestException("Key does not match the requested scope"); + } + } + /** * The legacy account keeps owner status with the owner's legacy keys. */ diff --git a/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/SyncKeysV2OperationsService.java b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/SyncKeysV2OperationsService.java new file mode 100644 index 0000000000..982310a72a --- /dev/null +++ b/src/control-plane-services/api-keys/src/main/java/com/nvidia/apikeys/services/SyncKeysV2OperationsService.java @@ -0,0 +1,340 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.apikeys.services; + +import static com.nvidia.apikeys.vo.KeyOwnerType.USER; + +import com.nvidia.apikeys.caching.CachingKeysLoader; +import com.nvidia.apikeys.caching.IntrospectionLocalCache; +import com.nvidia.apikeys.dto.keysV2.AccountAdminKeysOperationRequest; +import com.nvidia.apikeys.dto.keysV2.KeyOperationResponse; +import com.nvidia.apikeys.dto.keysV2.KeyOwnerKeysOperationRequest; +import com.nvidia.apikeys.dto.keysV2.KeysOperationType; +import com.nvidia.apikeys.dto.keysV2.PlatformAdminKeysOperationRequest; +import com.nvidia.apikeys.utils.JsonUtils; +import com.nvidia.apikeys.vo.KeyOperationStatus; +import com.nvidia.apikeys.vo.KeyOwnerStatus; +import com.nvidia.apikeys.vo.KeyStatus; +import com.nvidia.apikeys.vo.KeysByOwnerAndAccountAndServiceVo; +import com.nvidia.boot.exceptions.BadRequestException; +import com.nvidia.boot.exceptions.BootResponseException; +import com.nvidia.boot.exceptions.ForbiddenException; +import com.nvidia.boot.exceptions.NotFoundException; +import java.time.Instant; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +/** + * Processes explicit or wildcard selections synchronously using paged, legacy-aware reads + * and the existing per-key operations. + */ +@Slf4j +@Service +public class SyncKeysV2OperationsService { + + private final KeysV2Service keys; + private final ServicesService services; + private final LegacyAccountTranslator legacyAccount; + private final MillisecondPrecisionClock clock; + private final int maxSelectorValues; + private final KeysV2BulkSelectionService selection; + private final CachingKeysLoader keyCache; + private final IntrospectionLocalCache introspectionCache; + + public SyncKeysV2OperationsService( + KeysV2Service keys, ServicesService services, LegacyAccountTranslator legacyAccount, + MillisecondPrecisionClock clock, + @Value("${apikeys.bulk-max-selector-values}") int maxSelectorValues, + KeysV2BulkSelectionService selection, CachingKeysLoader keyCache, + IntrospectionLocalCache introspectionCache) { + if (maxSelectorValues < 1) { + throw new IllegalArgumentException("max-selector-values must be positive"); + } + this.keys = keys; + this.services = services; + this.legacyAccount = legacyAccount; + this.clock = clock; + this.maxSelectorValues = maxSelectorValues; + this.selection = selection; + this.keyCache = keyCache; + this.introspectionCache = introspectionCache; + } + + public KeyOperationResponse operateAsKeyOwner( + String userId, KeyOwnerKeysOperationRequest request) { + return execute(Actor.KEY_OWNER, request.getOperation(), request.getAccounts(), + List.of(validateId(userId, "userId")), request.getIssuerServices(), null); + } + + public KeyOperationResponse operateAsAccountAdmin( + String ncaId, AccountAdminKeysOperationRequest request) { + return execute(Actor.ACCOUNT_ADMIN, request.getOperation(), + List.of(validateId(ncaId, "ncaId")), request.getUsers(), request.getIssuerServices(), null); + } + + public KeyOperationResponse operateAsPlatformAdmin(PlatformAdminKeysOperationRequest request) { + return execute(Actor.PLATFORM_ADMIN, request.getOperation(), request.getAccounts(), + request.getUsers(), request.getIssuerServices(), + request.getConfirmPlatformWide()); + } + + private KeyOperationResponse execute( + Actor actor, KeysOperationType operation, List accountIds, + List userIds, List issuerIds, Boolean confirmPlatformWide) { + var cutoff = clock.instant(); + var actorId = auditActorId(); + if (operation == null) { + throw new BadRequestException("operation is required"); + } + var accounts = validateSelection(accountIds, "accounts"); + var users = validateSelection(userIds, "users"); + var issuers = validateSelection(issuerIds, "issuerServices"); + if (actor == Actor.PLATFORM_ADMIN && isWildcard(accounts) && isWildcard(users) + && isWildcard(issuers) && !Boolean.TRUE.equals(confirmPlatformWide)) { + throw new BadRequestException("confirm_platform_wide must be true for a platform-wide operation"); + } + if (!isWildcard(issuers)) { + for (var issuer : issuers) { + assertRegisteredIssuer(issuer); + } + } + // Check every selected owner boundary before allowing any changes. + if (actor == Actor.KEY_OWNER) { + assertSelectedOwnerScopesActive(accounts, users.getFirst(), issuers, cutoff); + } + var operationId = UUID.randomUUID(); + return process(actor, actorId, operation, accounts, users, issuers, cutoff, operationId); + } + + private KeyOperationResponse process( + Actor actor, String actorId, KeysOperationType operation, List accounts, + List users, List issuers, Instant cutoff, UUID operationId) { + var result = KeyOperationResponse.builder() + .operationId(operationId).operation(operation).status(KeyOperationStatus.COMPLETED) + .matchedCount(0L).completedCount(0L).failedCount(0L).submittedAt(cutoff).build(); + boolean mutationAttempted = false; + try { + for (var account : accounts) { + for (var user : users) { + for (var issuer : issuers) { + try (var selected = select(account, user, issuer, cutoff)) { + var iterator = selected.iterator(); + while (iterator.hasNext()) { + var key = iterator.next(); + assertMatchesScope(key, account, user, issuer); + if (key.getCreatedAt() == null) { + throw new IllegalStateException("Selected key has no creation time"); + } + result.setMatchedCount(result.getMatchedCount() + 1); + try { + if (actor == Actor.KEY_OWNER) { + assertOwnerActive(key.getNcaId(), key.getOwnerId(), key.getIssuerServiceId()); + } + mutationAttempted = true; + switch (operation) { + case REVOKED -> keys.updateKeyStatus( + key.getNcaId(), USER, key.getOwnerId(), key.getIssuerServiceId(), + key.getKeyId(), KeyStatus.REVOKED); + case DELETE -> deleteSelectedKey(key); + } + result.setCompletedCount(result.getCompletedCount() + 1); + } catch (RuntimeException failure) { + result.setFailedCount(result.getFailedCount() + 1); + throw failure; + } + } + } + } + } + } + } catch (RuntimeException failure) { + result.setStatus(KeyOperationStatus.FAILED); + throw new BulkOperationException(result, mutationAttempted, failure); + } finally { + if (mutationAttempted) { + keyCache.invalidate(); + introspectionCache.invalidate(); + } + result.setUpdatedAt(clock.instant()); + log.info("Synchronous bulk operation finished: {}", JsonUtils.getRequestResponseJsonMapper() + .writeValueAsString(Map.of("actor", actor.name(), "actorId", actorId, + "accounts", accounts, "users", users, "issuerServices", issuers, + "bulkOperation", result))); + } + return result; + } + + private void deleteSelectedKey(KeysByOwnerAndAccountAndServiceVo key) { + try { + keys.deleteKeyById(key.getNcaId(), USER, key.getOwnerId(), key.getIssuerServiceId(), key.getKeyId()); + } catch (NotFoundException ignored) { + // A key deleted after selection already satisfies this operation. + } + } + + private Stream select( + String account, String user, String issuer, Instant cutoff) { + return selection.select("*".equals(account) ? null : account, + "*".equals(user) ? null : user, "*".equals(issuer) ? null : issuer, cutoff); + } + + private void assertSelectedOwnerScopesActive( + List accounts, String user, List issuers, Instant cutoff) { + for (var account : accounts) { + if (!"*".equals(account) && isWildcard(issuers)) { + assertOwnerAccountActive(account, user); + } + for (var issuer : issuers) { + if (!"*".equals(account) && !"*".equals(issuer)) { + assertOwnerActive(account, user, issuer); + } else { + try (var selected = select(account, user, issuer, cutoff)) { + selected.forEach(key -> { + assertMatchesScope(key, account, user, issuer); + if (key.getCreatedAt() == null) { + throw new IllegalStateException("Selected key has no creation time"); + } + assertOwnerActive(key.getNcaId(), user, key.getIssuerServiceId()); + }); + } + } + } + } + } + + private static String auditActorId() { + if (RequestContextHolder.getRequestAttributes() instanceof ServletRequestAttributes attributes) { + var request = attributes.getRequest(); + var actorId = request.getHeader("NVCF-ACTOR-ID"); + if (actorId != null) { + return actorId; + } + } + return ActionRecorder.VALUE_NOT_APPLICABLE; + } + + private List validateSelection(List selection, String dimension) { + if (selection == null || selection.isEmpty()) { + throw new BadRequestException(dimension + " must be nonempty"); + } + if (selection.size() > maxSelectorValues) { + throw new BadRequestException(dimension + " exceeds the configured selector limit"); + } + if (isWildcard(selection)) { + return List.of("*"); + } + if (selection.contains("*")) { + throw new BadRequestException(dimension + " wildcard must be the only value"); + } + var distinct = new HashSet(); + for (var value : selection) { + validateId(value, dimension); + if (!distinct.add(value)) { + throw new BadRequestException(dimension + " must not contain duplicate IDs"); + } + } + return List.copyOf(selection); + } + + private static boolean isWildcard(List selection) { + return selection.size() == 1 && "*".equals(selection.getFirst()); + } + + private static String validateId(String value, String dimension) { + if (value == null || value.isBlank() || "*".equals(value)) { + throw new BadRequestException(dimension + " contains an invalid ID"); + } + return value; + } + + private void assertRegisteredIssuer(String issuer) { + if (!issuer.equals(legacyAccount.toServiceId(issuer))) { + throw new BadRequestException("Use canonical issuer service IDs"); + } + var registered = services.getServiceById(issuer).isPresent() + || legacyAccount.findLegacyIssuerServiceId(issuer) + .flatMap(services::getServiceById).isPresent(); + if (!registered) { + throw new BadRequestException("Unknown issuer service"); + } + } + + private void assertOwnerActive(String account, String user, String issuer) { + assertOwnerAccountActive(account, user); + var issuerStatus = keys.getOwnerStatusByAccountAndService(account, USER, user, issuer); + if (issuerStatus.map(status -> status.getOwnerStatus() != KeyOwnerStatus.ACTIVE) + .orElse(false)) { + throw new ForbiddenException("Key owner suspended."); + } + } + + private void assertOwnerAccountActive(String account, String user) { + if (keys.getOwnerStatusByAccount(account, USER, user) + .map(status -> status.getOwnerStatus() != KeyOwnerStatus.ACTIVE).orElse(false)) { + throw new ForbiddenException("Key owner suspended."); + } + } + + private static void assertMatchesScope( + KeysByOwnerAndAccountAndServiceVo key, String account, String user, String issuer) { + if (!matches(account, key.getNcaId()) || key.getOwnerType() != USER + || !matches(user, key.getOwnerId()) + || !matches(issuer, key.getIssuerServiceId()) + || key.getKeyId() == null || key.getKeyId().isBlank()) { + throw new IllegalStateException("Selected key does not match the requested scope"); + } + } + + private static boolean matches(String selector, String value) { + return value != null && !value.isBlank() && !"*".equals(value) + && ("*".equals(selector) || Objects.equals(selector, value)); + } + + private enum Actor { + KEY_OWNER, ACCOUNT_ADMIN, PLATFORM_ADMIN + } + + public static final class BulkOperationException extends BootResponseException { + + public BulkOperationException(KeyOperationResponse result, boolean mutationAttempted, RuntimeException cause) { + super(statusFor(cause), mutationAttempted + ? "Bulk operation stopped. Some keys may have changed; reconcile state before retrying." + : "Bulk operation stopped before any key mutation was attempted.", + cause, BulkOperationException.class); + getBody().setProperty("partialApplication", mutationAttempted); + getBody().setProperty("bulkOperation", result); + } + + private static HttpStatus statusFor(RuntimeException cause) { + if (cause instanceof BootResponseException response) { + return HttpStatus.valueOf(response.getStatusCode().value()); + } + return HttpStatus.INTERNAL_SERVER_ERROR; + } + } +} diff --git a/src/control-plane-services/api-keys/src/main/resources/application.yaml b/src/control-plane-services/api-keys/src/main/resources/application.yaml index 8063d2f8ca..d8c185853e 100644 --- a/src/control-plane-services/api-keys/src/main/resources/application.yaml +++ b/src/control-plane-services/api-keys/src/main/resources/application.yaml @@ -164,6 +164,7 @@ springdoc: packages-to-scan: "com.nvidia.apikeys" apikeys: + bulk-max-selector-values: 10 jwks: private-jwks: ${kv.private-key-jwks.keys} jwe-key-mapping: diff --git a/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2BulkSelectionIntegrationTest.java b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2BulkSelectionIntegrationTest.java new file mode 100644 index 0000000000..aa947df00c --- /dev/null +++ b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2BulkSelectionIntegrationTest.java @@ -0,0 +1,357 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.apikeys.services; + +import static com.nvidia.apikeys.TestData.KEY_OWNER_VO_1; +import static com.nvidia.apikeys.TestData.KEY_V2_VO_1; +import static com.nvidia.apikeys.TestData.KEY_VO_1; +import static com.nvidia.apikeys.TestData.SERVICE_ID_1; +import static com.nvidia.apikeys.TestData.TEST_TIME; +import static com.nvidia.apikeys.vo.KeyOwnerType.USER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; + +import com.nvidia.apikeys.App; +import com.nvidia.apikeys.caching.CachingKeysLoader; +import com.nvidia.apikeys.caching.IntrospectionLocalCache; +import com.nvidia.apikeys.config.IntegrationTestConfiguration; +import com.nvidia.apikeys.config.IntegrationTestConfiguration.TestCleanerExtension; +import com.nvidia.apikeys.dto.keysV2.KeyOperationResponse; +import com.nvidia.apikeys.dto.keysV2.KeysOperationType; +import com.nvidia.apikeys.dto.keysV2.PlatformAdminKeysOperationRequest; +import com.nvidia.apikeys.persistance.dao.KeysDao; +import com.nvidia.apikeys.persistance.dao.KeysV2Dao; +import com.nvidia.apikeys.services.SyncKeysV2OperationsService.BulkOperationException; +import com.nvidia.apikeys.utils.TestClock; +import com.nvidia.apikeys.vo.KeyOperationStatus; +import com.nvidia.apikeys.vo.KeyStatus; +import com.nvidia.apikeys.vo.KeyV2Vo; +import com.nvidia.apikeys.vo.KeyVo; +import com.nvidia.apikeys.vo.KeysByOwnerAndAccountAndServiceVo; +import java.time.Clock; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; + +@ExtendWith(TestCleanerExtension.class) +@SpringBootTest(classes = App.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = "spring.profiles.active=integrationtest") +@ContextConfiguration(initializers = IntegrationTestConfiguration.Initializer.class) +class KeysV2BulkSelectionIntegrationTest { + + @Autowired + private CachingKeysLoader keyCache; + @Autowired + private IntrospectionLocalCache introspectionCache; + + private static final String LEGACY_ACCOUNT = "test-nca-id"; + private static final int PAGE_SIZE = 2; + + @Autowired + private KeysV2Dao keysV2Dao; + @MockitoSpyBean + private KeysDao keysDao; + @Autowired + private KeysV2Service keys; + @Autowired + private LegacyAccountTranslator legacyAccount; + @Autowired + private ServicesService services; + + @BeforeEach + void setUpClock() { + TestClock.setBaseClock(Clock.fixed(TEST_TIME, ZoneOffset.UTC)); + } + + @AfterEach + void resetClock() { + TestClock.resetToDefaults(); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, 2, 3, 4, 5, 6, 7}) + void selectsEveryWildcardCombinationAcrossPages(int wildcardMask) { + var seeded = seedV2(); + var account = (wildcardMask & 1) == 0 ? LEGACY_ACCOUNT : null; + var user = (wildcardMask & 2) == 0 ? "u1" : null; + var issuer = (wildcardMask & 4) == 0 ? "nvcf" : null; + var expected = seeded.stream() + .filter(key -> account == null || account.equals(key.getNcaId())) + .filter(key -> user == null || user.equals(key.getOwnerId())) + .filter(key -> issuer == null || issuer.equals(key.getIssuerServiceId())) + .map(KeyV2Vo::getKeyId).toList(); + + try (var selected = selector().select(account, user, issuer, TEST_TIME)) { + assertThat(selected).extracting(KeysByOwnerAndAccountAndServiceVo::getKeyId) + .containsExactlyInAnyOrderElementsOf(expected); + } + } + + @Test + void legacyScanMapsIssuersSkipsStatusOnlyRowsAndPrefersV2Copy() { + var owner = KEY_OWNER_VO_1.toBuilder().ownerId("u1").build(); + keysDao.save(KEY_VO_1.toBuilder().ownerId("u1").keyId("legacy-only") + .keyHash("legacy-only-hash").build(), owner); + keysDao.save(KEY_VO_1.toBuilder().ownerId("u1").keyId("migrated") + .keyHash("migrated-hash").build(), owner); + keysDao.save(KEY_VO_1.toBuilder().ownerId("u1").keyId("newer") + .keyHash("newer-hash").build(), owner); + keysDao.save(KEY_OWNER_VO_1.toBuilder().ownerId("status-only").build()); + keysV2Dao.save(key(LEGACY_ACCOUNT, "u1", "nvcf", "migrated")); + keysV2Dao.save(key(LEGACY_ACCOUNT, "u1", "nvcf", "newer").toBuilder() + .createdAt(TEST_TIME.plusMillis(1)).build()); + + try (var selected = selector().select(null, null, null, TEST_TIME)) { + assertThat(selected).extracting(KeysByOwnerAndAccountAndServiceVo::getKeyId) + .containsExactlyInAnyOrder("migrated", "legacy-only"); + } + try (var selected = selector().select(LEGACY_ACCOUNT, "u1", "nvcf", TEST_TIME)) { + assertThat(selected).allSatisfy(key -> { + assertThat(key.getNcaId()).isEqualTo(LEGACY_ACCOUNT); + assertThat(key.getIssuerServiceId()).isEqualTo("nvcf"); + }); + } + } + + @ParameterizedTest + @EnumSource(KeysOperationType.class) + void mutatesEverySelectedKeyWhileConsumingMultiplePages(KeysOperationType operation) { + var seeded = seedV2(); + var owner = KEY_OWNER_VO_1.toBuilder().ownerId("u1").build(); + var migrated = seeded.getFirst(); + keysDao.save(KEY_VO_1.toBuilder().ownerId("u1").keyId(migrated.getKeyId()) + .keyHash(migrated.getKeyHash()).build(), owner); + keysDao.save(KEY_VO_1.toBuilder().ownerId("u1").keyId("legacy-only") + .keyHash("legacy-only-hash").build(), owner); + keysDao.save(KEY_VO_1.toBuilder().ownerId("u1").keyId("legacy-later") + .keyHash("legacy-later-hash").createdAt(TEST_TIME.plusMillis(1)).build(), owner); + var bulk = new SyncKeysV2OperationsService(keys, services, legacyAccount, + new MillisecondPrecisionClock(Clock.fixed(TEST_TIME, ZoneOffset.UTC)), 10, selector(), + keyCache, introspectionCache); + + var response = bulk.operateAsPlatformAdmin(PlatformAdminKeysOperationRequest.builder() + .operation(operation).accounts(List.of("*")).users(List.of("*")) + .issuerServices(List.of("*")).confirmPlatformWide(true).build()); + + assertThat(response.getCompletedCount()).isEqualTo(seeded.size() + 1); + for (var key : seeded) { + var stored = keysV2Dao.getKeyByHash(key.getKeyHash()); + var management = keysV2Dao.get(key.getNcaId(), USER, key.getOwnerId(), + key.getIssuerServiceId(), key.getKeyId()); + if (operation == KeysOperationType.DELETE) { + assertThat(stored).isEmpty(); + assertThat(management).isEmpty(); + } else { + assertThat(stored).get().extracting(KeyV2Vo::getKeyStatus).isEqualTo(KeyStatus.REVOKED); + assertThat(management).get().extracting(KeysByOwnerAndAccountAndServiceVo::getKeyStatus) + .isEqualTo(KeyStatus.REVOKED); + } + } + var legacy = keysDao.getKeyByHash("legacy-only-hash"); + if (operation == KeysOperationType.DELETE) { + assertThat(legacy).isEmpty(); + assertThat(keysDao.getKeyByOwnerAndServiceAndId(USER, "u1", SERVICE_ID_1, "legacy-only")) + .isEmpty(); + assertThat(keysDao.getKeyByHash(migrated.getKeyHash())).isEmpty(); + assertThat(keysDao.getKeyByOwnerAndServiceAndId(USER, "u1", SERVICE_ID_1, migrated.getKeyId())) + .isEmpty(); + } else { + assertThat(legacy).get().extracting(key -> key.getKeyStatus()).isEqualTo(KeyStatus.REVOKED); + assertThat(keysDao.getKeyByOwnerAndServiceAndId(USER, "u1", SERVICE_ID_1, "legacy-only")) + .get().extracting(key -> key.getKeyStatus()).isEqualTo(KeyStatus.REVOKED); + } + assertThat(keysV2Dao.getKeyByHash("future-hash")).get() + .extracting(KeyV2Vo::getKeyStatus).isEqualTo(KeyStatus.ACTIVE); + assertThat(keysDao.getKeyByHash("legacy-later-hash")).get() + .extracting(key -> key.getKeyStatus()).isEqualTo(KeyStatus.ACTIVE); + } + + @ParameterizedTest + @EnumSource(KeysOperationType.class) + void unmappedLegacyIssuerFailsWithinCutoffInsteadOfBeingSkipped(KeysOperationType operation) { + var legacy = KEY_VO_1.toBuilder().issuerServiceId("unmapped") + .createdAt(TEST_TIME.plusMillis(1)).build(); + keysDao.save(legacy, KEY_OWNER_VO_1); + var bulk = new SyncKeysV2OperationsService(keys, services, legacyAccount, + new MillisecondPrecisionClock(Clock.fixed(TEST_TIME, ZoneOffset.UTC)), 10, selector(), + keyCache, introspectionCache); + var request = PlatformAdminKeysOperationRequest.builder().operation(operation) + .accounts(List.of("*")).users(List.of("*")).issuerServices(List.of("*")) + .confirmPlatformWide(true).build(); + + assertThat(bulk.operateAsPlatformAdmin(request).getMatchedCount()).isZero(); + assertThat(keysDao.getKeyByHash(legacy.getKeyHash())).contains(legacy); + + var eligible = legacy.toBuilder().createdAt(TEST_TIME).build(); + keysDao.save(eligible, KEY_OWNER_VO_1); + + var failure = assertThrows(BulkOperationException.class, () -> bulk.operateAsPlatformAdmin(request)); + + assertThat(failure.getCause()).isInstanceOf(IllegalStateException.class) + .hasMessage("Legacy key issuer has no configured mapping"); + assertThat(failure.getBody().getProperties()).containsEntry("partialApplication", false); + var result = (KeyOperationResponse) failure.getBody().getProperties().get("bulkOperation"); + assertThat(result.getStatus()).isEqualTo(KeyOperationStatus.FAILED); + assertThat(result.getCompletedCount()).isZero(); + assertThat(keysDao.getKeyByHash(eligible.getKeyHash())).contains(eligible); + assertThat(keysDao.getKeyByOwnerAndServiceAndId(USER, eligible.getOwnerId(), + "unmapped", eligible.getKeyId())).get() + .extracting(row -> row.getKeyStatus()).isEqualTo(KeyStatus.ACTIVE); + } + + @Test + void missingLegacyCreationTimeReportsPartialFailureWithoutSkippingKey() { + var valid = key(LEGACY_ACCOUNT, "valid", "nvcf", "valid-key"); + keysV2Dao.save(valid); + var invalid = KEY_VO_1.toBuilder().ownerId("invalid").keyId("invalid-key") + .keyHash("invalid-hash").createdAt(null).build(); + keysDao.save(invalid, KEY_OWNER_VO_1.toBuilder().ownerId("invalid").build()); + var bulk = new SyncKeysV2OperationsService(keys, services, legacyAccount, + new MillisecondPrecisionClock(Clock.fixed(TEST_TIME, ZoneOffset.UTC)), 10, selector(), + keyCache, introspectionCache); + + var failure = assertThrows(BulkOperationException.class, + () -> bulk.operateAsPlatformAdmin(PlatformAdminKeysOperationRequest.builder() + .operation(KeysOperationType.REVOKED).accounts(List.of(LEGACY_ACCOUNT)) + .users(List.of("valid", "invalid")).issuerServices(List.of("nvcf")).build())); + + assertThat(failure.getCause()).isInstanceOf(IllegalStateException.class) + .hasMessage("Selected key has no creation time"); + assertThat(failure.getBody().getProperties()).containsEntry("partialApplication", true); + var result = (KeyOperationResponse) failure.getBody().getProperties().get("bulkOperation"); + assertThat(result.getStatus()).isEqualTo(KeyOperationStatus.FAILED); + assertThat(result.getMatchedCount()).isOne(); + assertThat(result.getCompletedCount()).isOne(); + assertThat(result.getFailedCount()).isZero(); + assertThat(keysV2Dao.getKeyByHash(valid.getKeyHash())).get() + .extracting(KeyV2Vo::getKeyStatus).isEqualTo(KeyStatus.REVOKED); + assertThat(keysV2Dao.get(LEGACY_ACCOUNT, USER, "valid", "nvcf", valid.getKeyId())).get() + .extracting(KeysByOwnerAndAccountAndServiceVo::getKeyStatus).isEqualTo(KeyStatus.REVOKED); + assertThat(keysDao.getKeyByHash(invalid.getKeyHash())).contains(invalid); + assertThat(keysDao.getKeyByOwnerAndServiceAndId(USER, "invalid", SERVICE_ID_1, invalid.getKeyId())) + .hasValueSatisfying(row -> { + assertThat(row.getCreatedAt()).isNull(); + assertThat(row.getKeyStatus()).isEqualTo(KeyStatus.ACTIVE); + }); + } + + @ParameterizedTest + @EnumSource(KeysOperationType.class) + void retriesMigratedKeyAfterLegacyMutationFails(KeysOperationType operation) { + var migrated = key(LEGACY_ACCOUNT, "u1", "nvcf", "migrated"); + keysV2Dao.save(migrated); + var legacy = KEY_VO_1.toBuilder().ownerId("u1").keyId(migrated.getKeyId()) + .keyHash(migrated.getKeyHash()).build(); + keysDao.save(legacy, KEY_OWNER_VO_1.toBuilder().ownerId("u1").build()); + var writeFailure = new IllegalStateException("legacy write failed"); + if (operation == KeysOperationType.REVOKED) { + doThrow(writeFailure).doCallRealMethod().when(keysDao).save(any(KeyVo.class), any()); + } else { + doThrow(writeFailure).doCallRealMethod().when(keysDao).deleteKey(any()); + } + var bulk = new SyncKeysV2OperationsService(keys, services, legacyAccount, + new MillisecondPrecisionClock(Clock.fixed(TEST_TIME, ZoneOffset.UTC)), 10, selector(), + keyCache, introspectionCache); + var request = PlatformAdminKeysOperationRequest.builder().operation(operation) + .accounts(List.of(LEGACY_ACCOUNT)).users(List.of("u1")) + .issuerServices(List.of("nvcf")).build(); + + var failure = assertThrows(BulkOperationException.class, () -> bulk.operateAsPlatformAdmin(request)); + + assertThat(failure.getCause()).isSameAs(writeFailure); + assertThat(failure.getBody().getProperties()).containsEntry("partialApplication", true); + var failed = (KeyOperationResponse) failure.getBody().getProperties().get("bulkOperation"); + assertThat(failed.getStatus()).isEqualTo(KeyOperationStatus.FAILED); + assertThat(failed.getMatchedCount()).isOne(); + assertThat(failed.getCompletedCount()).isZero(); + assertThat(failed.getFailedCount()).isOne(); + assertV2MutationApplied(operation, migrated); + assertThat(keysDao.getKeyByHash(legacy.getKeyHash())).contains(legacy); + assertThat(keysDao.getKeyByOwnerAndServiceAndId(USER, "u1", SERVICE_ID_1, legacy.getKeyId())) + .get().extracting(row -> row.getKeyStatus()).isEqualTo(KeyStatus.ACTIVE); + + var retried = bulk.operateAsPlatformAdmin(request); + + assertThat(retried.getStatus()).isEqualTo(KeyOperationStatus.COMPLETED); + assertThat(retried.getMatchedCount()).isOne(); + assertThat(retried.getCompletedCount()).isOne(); + assertThat(retried.getFailedCount()).isZero(); + assertV2MutationApplied(operation, migrated); + var legacyLookup = keysDao.getKeyByHash(legacy.getKeyHash()); + var legacyManagement = keysDao.getKeyByOwnerAndServiceAndId(USER, "u1", SERVICE_ID_1, legacy.getKeyId()); + if (operation == KeysOperationType.DELETE) { + assertThat(legacyLookup).isEmpty(); + assertThat(legacyManagement).isEmpty(); + } else { + assertThat(legacyLookup).get().extracting(KeyVo::getKeyStatus).isEqualTo(KeyStatus.REVOKED); + assertThat(legacyManagement).get().extracting(row -> row.getKeyStatus()).isEqualTo(KeyStatus.REVOKED); + } + } + + private void assertV2MutationApplied(KeysOperationType operation, KeyV2Vo key) { + var lookup = keysV2Dao.getKeyByHash(key.getKeyHash()); + var management = keysV2Dao.get(key.getNcaId(), USER, key.getOwnerId(), key.getIssuerServiceId(), key.getKeyId()); + if (operation == KeysOperationType.DELETE) { + assertThat(lookup).isEmpty(); + assertThat(management).isEmpty(); + } else { + assertThat(lookup).get().extracting(KeyV2Vo::getKeyStatus).isEqualTo(KeyStatus.REVOKED); + assertThat(management).get().extracting(KeysByOwnerAndAccountAndServiceVo::getKeyStatus) + .isEqualTo(KeyStatus.REVOKED); + } + } + + private List seedV2() { + var seeded = new ArrayList(); + for (var account : List.of(LEGACY_ACCOUNT, "other-account")) { + for (var user : List.of("u1", "u2")) { + for (var issuer : List.of("nvcf", "nvct")) { + var key = key(account, user, issuer, account + "-" + user + "-" + issuer); + keysV2Dao.save(key); + seeded.add(key); + } + } + } + keysV2Dao.save(key(LEGACY_ACCOUNT, "u1", "nvcf", "future").toBuilder() + .createdAt(TEST_TIME.plusMillis(1)).build()); + return seeded; + } + + private KeysV2BulkSelectionService selector() { + return new KeysV2BulkSelectionService(keysV2Dao, keysDao, legacyAccount, PAGE_SIZE); + } + + private static KeyV2Vo key(String account, String user, String issuer, String id) { + return KEY_V2_VO_1.toBuilder().ncaId(account).ownerId(user).issuerServiceId(issuer) + .audienceServiceIds(Set.of(issuer)).keyId(id).keyHash(id + "-hash").build(); + } +} diff --git a/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2BulkSelectionServiceTest.java b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2BulkSelectionServiceTest.java new file mode 100644 index 0000000000..50ce4d7f5c --- /dev/null +++ b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2BulkSelectionServiceTest.java @@ -0,0 +1,138 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.apikeys.services; + +import static com.nvidia.apikeys.vo.KeyOwnerType.USER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import com.nvidia.apikeys.config.NakProperties; +import com.nvidia.apikeys.persistance.dao.KeysDao; +import com.nvidia.apikeys.persistance.dao.KeysV2Dao; +import com.nvidia.apikeys.vo.KeyByOwnerAndServiceVo; +import com.nvidia.apikeys.vo.KeysByOwnerAndAccountAndServiceVo; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class KeysV2BulkSelectionServiceTest { + + private static final Instant CUTOFF = Instant.parse("2026-10-21T12:00:00Z"); + private static final int PAGE_SIZE = 2; + + @Mock + private KeysV2Dao keysV2Dao; + @Mock + private KeysDao keysDao; + private KeysV2BulkSelectionService service; + + @BeforeEach + void setUp() { + var translator = new LegacyAccountTranslator(NakProperties.builder() + .ncaId("legacy").serviceIdMap(Map.of("issuer", "legacy-issuer")).build()); + service = new KeysV2BulkSelectionService(keysV2Dao, keysDao, translator, PAGE_SIZE); + } + + @Test + void explicitOtherAccountNeverReadsLegacy() { + var row = row("account", "key"); + when(keysV2Dao.streamForBulk("account", USER, "user", "issuer", CUTOFF, PAGE_SIZE)) + .thenReturn(Stream.of(row)); + try (var selected = service.select("account", "user", "issuer", CUTOFF)) { + assertThat(selected).containsExactly(row); + } + verifyNoInteractions(keysDao); + } + + @Test + void wildcardSelectionIncludesCanonicalLegacyAndExcludesLaterOrMigratedRows() { + var current = row("new-account", "current"); + var migrated = row("legacy", "migrated"); + when(keysV2Dao.streamForBulk(null, USER, null, null, CUTOFF, PAGE_SIZE)) + .thenReturn(Stream.of(current, migrated)); + when(keysDao.streamForBulk(USER, null, null, PAGE_SIZE)).thenReturn(Stream.of( + legacy("legacy-only"), legacy("migrated"), + legacy("newer").toBuilder().createdAt(CUTOFF.plusMillis(1)).build())); + when(keysV2Dao.get("legacy", USER, "user", "issuer", "migrated")) + .thenReturn(Optional.of(migrated)); + when(keysV2Dao.get("legacy", USER, "user", "issuer", "legacy-only")) + .thenReturn(Optional.empty()); + try (var selected = service.select(null, null, null, CUTOFF)) { + assertThat(selected).extracting(KeysByOwnerAndAccountAndServiceVo::getKeyId) + .containsExactly("legacy-only", "current", "migrated"); + } + } + + @Test + void unmappedCanonicalIssuerCannotSelectLegacy() { + when(keysV2Dao.streamForBulk(null, USER, null, "other", CUTOFF, PAGE_SIZE)) + .thenReturn(Stream.empty()); + try (var selected = service.select(null, null, "other", CUTOFF)) { + assertThat(selected).isEmpty(); + } + verifyNoInteractions(keysDao); + } + + @Test + void closesBothSourceStreamsWhenConsumptionStopsEarly() { + var currentClosed = new AtomicBoolean(); + var legacyClosed = new AtomicBoolean(); + when(keysV2Dao.streamForBulk(null, USER, null, null, CUTOFF, PAGE_SIZE)) + .thenReturn(Stream.of(row("account", "first"), row("account", "second")) + .onClose(() -> currentClosed.set(true))); + when(keysDao.streamForBulk(USER, null, null, PAGE_SIZE)) + .thenReturn(Stream.of(legacy("legacy")).onClose(() -> legacyClosed.set(true))); + try (var selected = service.select(null, null, null, CUTOFF)) { + assertThat(selected.findFirst()).isPresent(); + } + assertThat(currentClosed).isTrue(); + assertThat(legacyClosed).isTrue(); + } + + @Test + void closesV2StreamIfLegacyQueryFails() { + var closed = new AtomicBoolean(); + when(keysV2Dao.streamForBulk(null, USER, null, null, CUTOFF, PAGE_SIZE)) + .thenReturn(Stream.empty().onClose(() -> closed.set(true))); + when(keysDao.streamForBulk(USER, null, null, PAGE_SIZE)) + .thenThrow(new IllegalStateException("query failed")); + assertThatThrownBy(() -> service.select(null, null, null, CUTOFF)) + .isInstanceOf(IllegalStateException.class).hasMessage("query failed"); + assertThat(closed).isTrue(); + } + + private static KeysByOwnerAndAccountAndServiceVo row(String account, String keyId) { + return KeysByOwnerAndAccountAndServiceVo.builder().ncaId(account).ownerType(USER) + .ownerId("user").issuerServiceId("issuer").keyId(keyId).createdAt(CUTOFF).build(); + } + + private static KeyByOwnerAndServiceVo legacy(String keyId) { + return KeyByOwnerAndServiceVo.builder().ownerType(USER).ownerId("user") + .issuerServiceId("legacy-issuer").keyId(keyId).createdAt(CUTOFF).build(); + } +} diff --git a/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2ServiceIntegrationTest.java b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2ServiceIntegrationTest.java index 51245bc9be..0bc70bd124 100644 --- a/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2ServiceIntegrationTest.java +++ b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2ServiceIntegrationTest.java @@ -18,6 +18,7 @@ package com.nvidia.apikeys.services; import static com.nvidia.apikeys.TestData.API_KEY_HASH_1; +import static com.nvidia.apikeys.TestData.INTROSPECTION_REQUEST_1; import static com.nvidia.apikeys.TestData.KEY_AUTHZ_2; import static com.nvidia.apikeys.TestData.KEY_ID_1; import static com.nvidia.apikeys.TestData.KEY_ID_2; @@ -34,6 +35,7 @@ import static org.assertj.core.api.Assertions.assertThat; import com.nvidia.apikeys.App; +import com.nvidia.apikeys.caching.CachingKeysLoader; import com.nvidia.apikeys.config.IntegrationTestConfiguration; import com.nvidia.apikeys.config.IntegrationTestConfiguration.TestCleanerExtension; import com.nvidia.apikeys.persistance.dao.KeysDao; @@ -44,6 +46,7 @@ import com.nvidia.apikeys.persistance.models.OwnerStatusByAccountAndServiceModel; import com.nvidia.apikeys.persistance.models.OwnerStatusByAccountModel; import com.nvidia.apikeys.utils.TestClock; +import com.nvidia.apikeys.validators.IntrospectionValidator; import com.nvidia.apikeys.vo.KeyOwnerStatus; import com.nvidia.apikeys.vo.KeyStatus; import com.nvidia.apikeys.vo.KeyV2Vo; @@ -60,6 +63,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.resttestclient.autoconfigure.AutoConfigureTestRestTemplate; import org.springframework.boot.test.context.SpringBootTest; @@ -106,6 +111,12 @@ class KeysV2ServiceIntegrationTest { @Autowired private RowUpdateLockDao rowUpdateLockDao; + @Autowired + private CachingKeysLoader cachingKeysLoader; + + @Autowired + private IntrospectionValidator introspectionValidator; + @BeforeEach void setUp() { TestClock.setBaseClock(TestClock.fixed(TEST_TIME, ZoneId.systemDefault())); @@ -285,21 +296,19 @@ void updateKeyStatusOfLegacyKeyWritesLegacyTables() { } @Test - void updateKeyStatusOfMigratedKeyWritesKeysV2Only() { + void revokingMigratedKeyDeniesLegacyIntrospection() { keysDao.save(KEY_VO_1, KEY_OWNER_VO_1); keysV2Dao.save(LEGACY_KEY_V2); service.updateKeyStatus(LEGACY_NCA_ID, USER, USER_KEY_OWNER_ID_1, SERVICE_NAME, KEY_ID_1, KeyStatus.REVOKED); - assertThat(keysV2Dao.getKeyByHash(API_KEY_HASH_1)) - .get() - .extracting(KeyV2Vo::getKeyStatus) - .isEqualTo(KeyStatus.REVOKED); - assertThat(keysDao.getKeyByHash(API_KEY_HASH_1)) - .get() - .extracting(KeyVo::getKeyStatus) - .isEqualTo(KeyStatus.ACTIVE); + assertRevokedCopies(false, LEGACY_KEY_V2); + assertRevokedCopies(true, LEGACY_KEY_V2); + cachingKeysLoader.invalidate(); + assertThrowsExceptionWithDetails(NotFoundException.class, + () -> introspectionValidator.validate(INTROSPECTION_REQUEST_1, API_KEY_HASH_1), + "Key not found"); } @Test @@ -319,6 +328,68 @@ void updateKeyStatusRejectsChangeFromRevoked() { .isEqualTo(KeyStatus.REVOKED); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void revokeExpiredKeyUpdatesBothStoredCopies(boolean legacy) { + var expiredAt = TEST_TIME.minusSeconds(1); + var key = (legacy ? LEGACY_KEY_V2 : KEY_V2_VO_1).toBuilder().expiresAt(expiredAt).build(); + if (legacy) { + keysDao.save(KEY_VO_1.toBuilder().expiresAt(expiredAt).build(), KEY_OWNER_VO_1); + } else { + keysV2Dao.save(key); + } + assertThat(service.getKeyById(key.getNcaId(), USER, key.getOwnerId(), + key.getIssuerServiceId(), key.getKeyId())).get() + .extracting(KeyV2Vo::getKeyStatus).isEqualTo(KeyStatus.EXPIRED); + + service.updateKeyStatus(key.getNcaId(), USER, key.getOwnerId(), key.getIssuerServiceId(), + key.getKeyId(), KeyStatus.REVOKED); + + assertRevokedCopies(legacy, key); + assertThat(service.getKeyByHash(key.getKeyHash())).get() + .extracting(KeyV2Vo::getExpiresAt).isEqualTo(expiredAt); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void repeatedRevocationRepairsStaleManagementStatus(boolean legacy) { + var key = (legacy ? LEGACY_KEY_V2 : KEY_V2_VO_1).toBuilder() + .keyStatus(KeyStatus.REVOKED).build(); + if (legacy) { + keysDao.save(KEY_VO_1.toBuilder().keyStatus(KeyStatus.REVOKED).build(), KEY_OWNER_VO_1); + IntegrationTestConfiguration.CQL_SESSION.execute( + "UPDATE " + KEY_SPACE + ".keys_by_owner_and_service SET key_status = 'ACTIVE'" + + " WHERE owner_type = 'USER' AND owner_id = ? AND issuer_service_id = ? AND key_id = ?", + key.getOwnerId(), SERVICE_ID_1, key.getKeyId()); + } else { + keysV2Dao.save(key); + IntegrationTestConfiguration.CQL_SESSION.execute( + "UPDATE " + KEY_SPACE + ".keys_by_owner_and_account_and_service SET key_status = 'ACTIVE'" + + " WHERE nca_id = ? AND owner_type = 'USER' AND owner_id = ?" + + " AND issuer_service_id = ? AND key_id = ?", + key.getNcaId(), key.getOwnerId(), key.getIssuerServiceId(), key.getKeyId()); + } + + service.updateKeyStatus(key.getNcaId(), USER, key.getOwnerId(), key.getIssuerServiceId(), + key.getKeyId(), KeyStatus.REVOKED); + + assertRevokedCopies(legacy, key); + } + + private void assertRevokedCopies(boolean legacy, KeyV2Vo key) { + if (legacy) { + assertThat(keysDao.getKeyByHash(key.getKeyHash())).get() + .extracting(KeyVo::getKeyStatus).isEqualTo(KeyStatus.REVOKED); + assertThat(keysDao.getKeyByOwnerAndServiceAndId(USER, key.getOwnerId(), SERVICE_ID_1, key.getKeyId())) + .get().extracting(row -> row.getKeyStatus()).isEqualTo(KeyStatus.REVOKED); + } else { + assertThat(keysV2Dao.getKeyByHash(key.getKeyHash())).get() + .extracting(KeyV2Vo::getKeyStatus).isEqualTo(KeyStatus.REVOKED); + assertThat(keysV2Dao.get(key.getNcaId(), USER, key.getOwnerId(), key.getIssuerServiceId(), key.getKeyId())) + .get().extracting(KeysByOwnerAndAccountAndServiceVo::getKeyStatus).isEqualTo(KeyStatus.REVOKED); + } + } + @Test void updateKeyAuthorizationsOfV2KeyLocksKeysV2Row() { createKey(KEY_V2_VO_1); diff --git a/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2ServiceTest.java b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2ServiceTest.java index 305d52ad67..3b7ee709dc 100644 --- a/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2ServiceTest.java +++ b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/KeysV2ServiceTest.java @@ -75,6 +75,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -424,6 +425,76 @@ void updateKeyStatusToCurrentStatusWritesNothing() { verifyNoInteractions(actionRecorder); } + @ParameterizedTest + @MethodSource("keysWithMismatchedScope") + void mutationsRejectLookupKeyOutsideRequestedScope(KeyV2Vo key) { + when(keysV2Dao.get(NCA_ID_1, USER, USER_KEY_OWNER_ID_1, SERVICE_ID_1, KEY_ID_1)) + .thenReturn(Optional.of(KEY_V2_ROW_1)); + when(keysV2Dao.getKeyByHash(API_KEY_HASH_1)).thenReturn(Optional.of(key)); + + assertThrowsExceptionWithDetails(BadRequestException.class, + () -> service.updateKeyStatus(NCA_ID_1, USER, USER_KEY_OWNER_ID_1, SERVICE_ID_1, + KEY_ID_1, KeyStatus.REVOKED), "Key does not match the requested scope"); + assertThrowsExceptionWithDetails(BadRequestException.class, + () -> service.deleteKeyById(NCA_ID_1, USER, USER_KEY_OWNER_ID_1, SERVICE_ID_1, KEY_ID_1), + "Key does not match the requested scope"); + + verify(keysV2Dao, never()).save(any()); + verify(keysV2Dao, never()).delete(any()); + verifyNoInteractions(actionRecorder); + } + + private static Stream keysWithMismatchedScope() { + return Stream.of( + KEY_V2_VO_1.toBuilder().ncaId("other-account").build(), + KEY_V2_VO_1.toBuilder().ownerType(null).build(), + KEY_V2_VO_1.toBuilder().ownerId("other-user").build(), + KEY_V2_VO_1.toBuilder().issuerServiceId("other-service").build(), + KEY_V2_VO_1.toBuilder().keyId("other-key").build()); + } + + @Test + void mutationsValidateLegacyLookupBeforeChangingEitherMigratedCopy() { + var migrated = KeysByOwnerAndAccountAndServiceVo.from(LEGACY_KEY_V2); + when(keysV2Dao.get(LEGACY_NCA_ID, USER, USER_KEY_OWNER_ID_1, SERVICE_NAME, KEY_ID_1)) + .thenReturn(Optional.of(migrated)); + when(keysV2Dao.getKeyByHash(API_KEY_HASH_1)).thenReturn(Optional.of(LEGACY_KEY_V2)); + when(keysDao.getKeyByOwnerAndServiceAndId(USER, USER_KEY_OWNER_ID_1, SERVICE_ID_1, KEY_ID_1)) + .thenReturn(Optional.of(KEY_BY_OWNER_AND_SERVICE_VO_1)); + when(keysDao.getKeyByHash(API_KEY_HASH_1)) + .thenReturn(Optional.of(KEY_VO_1.toBuilder().ownerId("other-user").build())); + + assertThrowsExceptionWithDetails(BadRequestException.class, + () -> service.updateKeyStatus(LEGACY_NCA_ID, USER, USER_KEY_OWNER_ID_1, SERVICE_NAME, + KEY_ID_1, KeyStatus.REVOKED), "Key does not match the requested scope"); + assertThrowsExceptionWithDetails(BadRequestException.class, + () -> service.deleteKeyById(LEGACY_NCA_ID, USER, USER_KEY_OWNER_ID_1, SERVICE_NAME, KEY_ID_1), + "Key does not match the requested scope"); + + verify(keysV2Dao, never()).save(any()); + verify(keysDao, never()).save(any(KeyVo.class), any()); + verify(keysV2Dao, never()).delete(any()); + verify(keysDao, never()).deleteKey(any()); + verifyNoInteractions(actionRecorder); + } + + @Test + void revokeRejectsMissingLegacyLookupBeforeChangingMigratedCopy() { + when(keysV2Dao.get(LEGACY_NCA_ID, USER, USER_KEY_OWNER_ID_1, SERVICE_NAME, KEY_ID_1)) + .thenReturn(Optional.of(KeysByOwnerAndAccountAndServiceVo.from(LEGACY_KEY_V2))); + when(keysV2Dao.getKeyByHash(API_KEY_HASH_1)).thenReturn(Optional.of(LEGACY_KEY_V2)); + when(keysDao.getKeyByOwnerAndServiceAndId(USER, USER_KEY_OWNER_ID_1, SERVICE_ID_1, KEY_ID_1)) + .thenReturn(Optional.of(KEY_BY_OWNER_AND_SERVICE_VO_1)); + + assertThrowsExceptionWithDetails(NotFoundException.class, + () -> service.updateKeyStatus(LEGACY_NCA_ID, USER, USER_KEY_OWNER_ID_1, SERVICE_NAME, + KEY_ID_1, KeyStatus.REVOKED), "Key not found"); + + verify(keysV2Dao, never()).save(any()); + verify(keysDao, never()).save(any(KeyVo.class), any()); + verifyNoInteractions(actionRecorder); + } + @ParameterizedTest @EnumSource(value = KeyStatus.class, names = {"EXPIRED", "REVOKED"}) void updateKeyStatusRejectsFinalStatus(KeyStatus finalStatus) { diff --git a/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/SyncKeysV2OperationsServiceTest.java b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/SyncKeysV2OperationsServiceTest.java new file mode 100644 index 0000000000..5c6b531076 --- /dev/null +++ b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/SyncKeysV2OperationsServiceTest.java @@ -0,0 +1,410 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.apikeys.services; + +import static com.nvidia.apikeys.dto.keysV2.KeysOperationType.DELETE; +import static com.nvidia.apikeys.dto.keysV2.KeysOperationType.REVOKED; +import static com.nvidia.apikeys.vo.KeyOwnerType.USER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import com.nvidia.apikeys.caching.CachingKeysLoader; +import com.nvidia.apikeys.caching.IntrospectionLocalCache; +import com.nvidia.apikeys.config.NakProperties; +import com.nvidia.apikeys.dto.keysV2.AccountAdminKeysOperationRequest; +import com.nvidia.apikeys.dto.keysV2.KeyOperationResponse; +import com.nvidia.apikeys.dto.keysV2.KeyOwnerKeysOperationRequest; +import com.nvidia.apikeys.dto.keysV2.PlatformAdminKeysOperationRequest; +import com.nvidia.apikeys.persistance.models.OwnerStatusByAccountAndServiceModel; +import com.nvidia.apikeys.persistance.models.OwnerStatusByAccountModel; +import com.nvidia.apikeys.services.SyncKeysV2OperationsService.BulkOperationException; +import com.nvidia.apikeys.utils.JsonUtils; +import com.nvidia.apikeys.vo.KeyOperationStatus; +import com.nvidia.apikeys.vo.KeyOwnerStatus; +import com.nvidia.apikeys.vo.KeyStatus; +import com.nvidia.apikeys.vo.KeyV2Vo; +import com.nvidia.apikeys.vo.KeysByOwnerAndAccountAndServiceVo; +import com.nvidia.apikeys.vo.ServiceVo; +import com.nvidia.boot.exceptions.BadRequestException; +import com.nvidia.boot.exceptions.ConflictException; +import com.nvidia.boot.exceptions.ForbiddenException; +import com.nvidia.boot.exceptions.NotFoundException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.test.system.CapturedOutput; +import org.springframework.boot.test.system.OutputCaptureExtension; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +@ExtendWith({MockitoExtension.class, OutputCaptureExtension.class}) +class SyncKeysV2OperationsServiceTest { + + @Mock + private CachingKeysLoader keyCache; + @Mock + private IntrospectionLocalCache introspectionCache; + + private static final Instant CUTOFF = Instant.parse("2026-10-21T12:00:00Z"); + private static final String ISSUER = "cloud-functions"; + + @Mock + private KeysV2Service keys; + @Mock + private ServicesService services; + @Mock + private KeysV2BulkSelectionService selection; + @Mock + private MillisecondPrecisionClock clock; + private SyncKeysV2OperationsService service; + + @BeforeEach + void setUp() { + var legacy = new LegacyAccountTranslator(NakProperties.builder() + .ncaId("legacy").serviceIdMap(Map.of(ISSUER, "legacy-issuer")).build()); + lenient().when(clock.instant()).thenReturn(CUTOFF); + service = new SyncKeysV2OperationsService(keys, services, legacy, clock, 10, selection, keyCache, introspectionCache); + lenient().when(services.getServiceById(ISSUER)) + .thenReturn(Optional.of(ServiceVo.builder().serviceId(ISSUER).build())); + } + + @ParameterizedTest + @CsvSource({ + "KEY_OWNER, a*, u, cloud-functions", + "KEY_OWNER, a, user*, cloud-functions", + "KEY_OWNER, a, u, issuer*", + "ACCOUNT_ADMIN, a*, u, cloud-functions", + "ACCOUNT_ADMIN, a, user*, cloud-functions", + "ACCOUNT_ADMIN, a, u, issuer*", + "PLATFORM_ADMIN, a*, u, cloud-functions", + "PLATFORM_ADMIN, a, user*, cloud-functions", + "PLATFORM_ADMIN, a, u, issuer*" + }) + void treatsIdsContainingAsterisksAsLiteralValues( + String actor, String account, String user, String issuer) { + when(services.getServiceById(issuer)) + .thenReturn(Optional.of(ServiceVo.builder().serviceId(issuer).build())); + when(selection.select(account, user, issuer, CUTOFF)) + .thenReturn(Stream.of(row(account, user, "key", CUTOFF).toBuilder() + .issuerServiceId(issuer).build())); + + var response = switch (actor) { + case "KEY_OWNER" -> service.operateAsKeyOwner(user, + KeyOwnerKeysOperationRequest.builder().operation(DELETE) + .accounts(List.of(account)).issuerServices(List.of(issuer)).build()); + case "ACCOUNT_ADMIN" -> service.operateAsAccountAdmin(account, + AccountAdminKeysOperationRequest.builder().operation(DELETE) + .users(List.of(user)).issuerServices(List.of(issuer)).build()); + case "PLATFORM_ADMIN" -> service.operateAsPlatformAdmin( + PlatformAdminKeysOperationRequest.builder().operation(DELETE) + .accounts(List.of(account)).users(List.of(user)) + .issuerServices(List.of(issuer)).build()); + default -> throw new IllegalArgumentException(actor); + }; + + assertThat(response.getCompletedCount()).isOne(); + verify(keys).deleteKeyById(account, USER, user, issuer, "key"); + } + + @Test + void invalidLaterDimensionDoesNotPartiallyApplyEarlierValues() { + var request = accountRequest().toBuilder().issuerServices(List.of(ISSUER, "*")).build(); + assertThatThrownBy(() -> service.operateAsAccountAdmin("a", request)) + .isInstanceOf(BadRequestException.class); + verifyNoInteractions(keys); + } + + @Test + void rejectsUnknownAndLegacyIssuerIdsBeforeDiscovery() { + for (var issuer : List.of("unknown", "legacy-issuer")) { + assertThatThrownBy(() -> service.operateAsAccountAdmin("a", + accountRequest().toBuilder().issuerServices(List.of(issuer)).build())) + .isInstanceOf(BadRequestException.class); + } + verifyNoInteractions(keys); + } + + @Test + void validatesOperationAndFixedRouteDimension() { + assertThatThrownBy(() -> service.operateAsAccountAdmin("a", + accountRequest().toBuilder().operation(null).build())) + .isInstanceOf(BadRequestException.class); + var invalidAccount = assertThrows(BadRequestException.class, + () -> service.operateAsAccountAdmin("*", accountRequest())); + assertThat(invalidAccount.getBody().getDetail()).isEqualTo("ncaId contains an invalid ID"); + var invalidUser = assertThrows(BadRequestException.class, + () -> service.operateAsKeyOwner("*", KeyOwnerKeysOperationRequest.builder() + .operation(DELETE).accounts(List.of("a")).issuerServices(List.of(ISSUER)).build())); + assertThat(invalidUser.getBody().getDetail()).isEqualTo("userId contains an invalid ID"); + verifyNoInteractions(keys); + } + + @ParameterizedTest + @MethodSource("optionalReasons") + void acceptsOptionalReasonWithoutAdditionalRestrictions(String reason) { + assertThat(service.operateAsAccountAdmin("a", + accountRequest().toBuilder().reason(reason).build()) + .getCompletedCount()).isZero(); + } + + static Stream optionalReasons() { + return Stream.of(null, " ", "bad\nreason", "x".repeat(1025)); + } + + @ParameterizedTest + @ValueSource(strings = {" user ", "user\nid"}) + void preservesNonblankLiteralIds(String user) { + when(selection.select("a", user, ISSUER, CUTOFF)) + .thenReturn(Stream.of(row("a", user, "key", CUTOFF))); + + assertThat(service.operateAsAccountAdmin("a", accountRequest().toBuilder() + .operation(DELETE).users(List.of(user)).build()).getCompletedCount()).isOne(); + + verify(keys).deleteKeyById("a", USER, user, ISSUER, "key"); + } + + @Test + void auditIdentityDoesNotAddAHeaderLengthLimit() { + var request = new MockHttpServletRequest(); + request.addHeader("NVCF-ACTOR-ID", "a".repeat(257)); + RequestContextHolder.setRequestAttributes(new ServletRequestAttributes(request)); + try { + assertThat(service.operateAsAccountAdmin("a", accountRequest()).getCompletedCount()).isZero(); + } finally { + RequestContextHolder.resetRequestAttributes(); + } + } + + @ParameterizedTest + @ValueSource(ints = {0, 2}) + void returnsCompletedSharedResponse(int keyCount) { + var finishedAt = CUTOFF.plusSeconds(2); + when(clock.instant()).thenReturn(CUTOFF, finishedAt); + when(selection.select("a", "u", ISSUER, CUTOFF)).thenReturn( + java.util.stream.IntStream.range(0, keyCount) + .mapToObj(i -> row("a", "u", "key-" + i, CUTOFF))); + + var response = service.operateAsAccountAdmin("a", accountRequest()); + + assertThat(response.getOperationId()).isNotNull(); + assertThat(response.getOperation()).isEqualTo(REVOKED); + assertThat(response.getStatus()).isEqualTo(KeyOperationStatus.COMPLETED); + assertThat(response.getMatchedCount()).isEqualTo((long) keyCount); + assertThat(response.getCompletedCount()).isEqualTo((long) keyCount); + assertThat(response.getFailedCount()).isZero(); + assertThat(response.getSubmittedAt()).isEqualTo(CUTOFF); + assertThat(response.getUpdatedAt()).isEqualTo(finishedAt); + if (keyCount == 0) { + verifyNoInteractions(keyCache, introspectionCache); + } else { + verify(keyCache).invalidate(); + verify(introspectionCache).invalidate(); + } + } + + @Test + void checksEveryOwnerScopeBeforeAnyChanges() { + when(keys.getOwnerStatusByAccount("a", USER, "u")).thenReturn(Optional.empty()); + when(keys.getOwnerStatusByAccount("blocked", USER, "u")) + .thenReturn(Optional.of(OwnerStatusByAccountModel.builder() + .ncaId("blocked").ownerType(USER).ownerId("u") + .ownerStatus(KeyOwnerStatus.SUSPENDED).build())); + assertThatThrownBy(() -> service.operateAsKeyOwner("u", + KeyOwnerKeysOperationRequest.builder().operation(DELETE) + .accounts(List.of("a", "blocked")).issuerServices(List.of(ISSUER)).build())) + .isInstanceOf(ForbiddenException.class); + verifyNoInteractions(selection); + } + + @Test + void ownerCannotOperateWhenIssuerStatusIsSuspended() { + when(keys.getOwnerStatusByAccountAndService("a", USER, "u", ISSUER)) + .thenReturn(Optional.of(OwnerStatusByAccountAndServiceModel.builder() + .ncaId("a").ownerType(USER).ownerId("u").issuerServiceId(ISSUER) + .ownerStatus(KeyOwnerStatus.SUSPENDED).build())); + assertThatThrownBy(() -> service.operateAsKeyOwner("u", ownerRequest())) + .isInstanceOf(ForbiddenException.class); + verifyNoInteractions(selection); + } + + @Test + void rejectsMismatchedStoredScopeWithoutMutation() { + when(selection.select("a", "u", ISSUER, CUTOFF)) + .thenReturn(Stream.of(row("different-account", "u", "key", CUTOFF))); + var failure = assertThrows(BulkOperationException.class, + () -> service.operateAsAccountAdmin("a", accountRequest())); + assertThat(failure.getBody().getProperties()).containsEntry("partialApplication", false); + verify(keys, never()).updateKeyStatus(anyString(), any(), anyString(), anyString(), + anyString(), any()); + } + + @Test + void laterMutationFailureRetainsCauseAndStopsFurtherKeys() { + when(selection.select("a", "u", ISSUER, CUTOFF)).thenReturn(Stream.of( + row("a", "u", "first", CUTOFF), row("a", "u", "failure", CUTOFF), + row("a", "u", "last", CUTOFF))); + when(keys.updateKeyStatus("a", USER, "u", ISSUER, "first", KeyStatus.REVOKED)) + .thenReturn(KeyV2Vo.builder().ncaId("a").ownerType(USER).ownerId("u") + .issuerServiceId(ISSUER).keyId("first").keyStatus(KeyStatus.REVOKED).build()); + var cause = new ConflictException("terminal key status"); + when(keys.updateKeyStatus("a", USER, "u", ISSUER, "failure", KeyStatus.REVOKED)) + .thenThrow(cause); + var failure = assertThrows(BulkOperationException.class, + () -> service.operateAsAccountAdmin("a", accountRequest())); + assertThat(failure.getStatusCode().value()).isEqualTo(409); + assertThat(failure.getCause()).isSameAs(cause); + assertThat(failure.getBody().getProperties()).containsEntry("partialApplication", true); + var result = failedResult(failure); + assertThat(result.getMatchedCount()).isEqualTo(2); + assertThat(result.getCompletedCount()).isOne(); + assertThat(result.getFailedCount()).isOne(); + verify(keyCache).invalidate(); + verify(introspectionCache).invalidate(); + verify(keys).updateKeyStatus("a", USER, "u", ISSUER, "first", KeyStatus.REVOKED); + verify(keys, never()).updateKeyStatus("a", USER, "u", ISSUER, "last", KeyStatus.REVOKED); + } + + @Test + void firstMutationWithLostAcknowledgementIsAlsoPartial() { + when(selection.select("a", "u", ISSUER, CUTOFF)) + .thenReturn(Stream.of(row("a", "u", "key", CUTOFF))); + doThrow(new IllegalStateException("acknowledgement lost")).when(keys) + .deleteKeyById("a", USER, "u", ISSUER, "key"); + var failure = assertThrows(BulkOperationException.class, + () -> service.operateAsAccountAdmin("a", + accountRequest().toBuilder().operation(DELETE).build())); + assertThat(failure.getBody().getProperties()).containsEntry("partialApplication", true); + var result = failedResult(failure); + assertThat(result.getMatchedCount()).isOne(); + assertThat(result.getCompletedCount()).isZero(); + assertThat(result.getFailedCount()).isOne(); + } + + @Test + void laterDiscoveryFailureCanFollowSuccessfulMutation() { + when(selection.select("a", "u", ISSUER, CUTOFF)) + .thenReturn(Stream.of(row("a", "u", "key", CUTOFF))); + when(selection.select("a", "v", ISSUER, CUTOFF)) + .thenThrow(new IllegalStateException("selection unavailable")); + var failure = assertThrows(BulkOperationException.class, + () -> service.operateAsAccountAdmin("a", + accountRequest().toBuilder().users(List.of("u", "v")).build())); + assertThat(failure.getBody().getProperties()).containsEntry("partialApplication", true); + var result = failedResult(failure); + assertThat(result.getMatchedCount()).isOne(); + assertThat(result.getCompletedCount()).isOne(); + assertThat(result.getFailedCount()).isZero(); + verify(keys).updateKeyStatus("a", USER, "u", ISSUER, "key", KeyStatus.REVOKED); + } + + @Test + void selectedKeyDeletedConcurrentlyIsAlreadyComplete() { + when(selection.select("a", "u", ISSUER, CUTOFF)) + .thenReturn(Stream.of(row("a", "u", "deleted", CUTOFF))); + doThrow(new NotFoundException("Key not found")).when(keys) + .deleteKeyById("a", USER, "u", ISSUER, "deleted"); + + var result = service.operateAsAccountAdmin("a", accountRequest().toBuilder().operation(DELETE).build()); + + assertThat(result.getStatus()).isEqualTo(KeyOperationStatus.COMPLETED); + assertThat(result.getMatchedCount()).isOne(); + assertThat(result.getCompletedCount()).isOne(); + assertThat(result.getFailedCount()).isZero(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void emitsCompleteAuditOutcomeWithEscapedIdentity(boolean fail, CapturedOutput output) { + var request = new MockHttpServletRequest(); + request.addHeader("NVCF-ACTOR-ID", "actor\nid"); + RequestContextHolder.setRequestAttributes(new ServletRequestAttributes(request)); + when(selection.select("a", "u", ISSUER, CUTOFF)) + .thenReturn(Stream.of(row("a", "u", "key", CUTOFF))); + if (fail) { + when(keys.updateKeyStatus("a", USER, "u", ISSUER, "key", KeyStatus.REVOKED)) + .thenThrow(new IllegalStateException("write failed")); + } + try { + var result = fail + ? failedResult(assertThrows(BulkOperationException.class, + () -> service.operateAsAccountAdmin("a", accountRequest()))) + : service.operateAsAccountAdmin("a", accountRequest()); + var prefix = "Synchronous bulk operation finished: "; + var events = output.getOut().lines().filter(line -> line.contains(prefix)).toList(); + assertThat(events).hasSize(1); + var event = events.getFirst(); + var mapper = JsonUtils.getRequestResponseJsonMapper(); + var json = mapper.readTree(event.substring(event.indexOf(prefix) + prefix.length())); + assertThat(json.get("actor").asString()).isEqualTo("ACCOUNT_ADMIN"); + assertThat(json.get("actorId").asString()).isEqualTo("actor\nid"); + assertThat(json.get("accounts")).isEqualTo(mapper.valueToTree(List.of("a"))); + assertThat(json.get("users")).isEqualTo(mapper.valueToTree(List.of("u"))); + assertThat(json.get("issuerServices")).isEqualTo(mapper.valueToTree(List.of(ISSUER))); + assertThat(mapper.treeToValue(json.get("bulkOperation"), KeyOperationResponse.class)).isEqualTo(result); + } finally { + RequestContextHolder.resetRequestAttributes(); + } + } + + private static KeyOperationResponse failedResult(BulkOperationException failure) { + var result = (KeyOperationResponse) failure.getBody().getProperties().get("bulkOperation"); + assertThat(result.getOperationId()).isNotNull(); + assertThat(result.getStatus()).isEqualTo(KeyOperationStatus.FAILED); + assertThat(result.getSubmittedAt()).isEqualTo(CUTOFF); + assertThat(result.getUpdatedAt()).isEqualTo(CUTOFF); + return result; + } + + private static AccountAdminKeysOperationRequest accountRequest() { + return AccountAdminKeysOperationRequest.builder().operation(REVOKED) + .users(List.of("u")).issuerServices(List.of(ISSUER)).build(); + } + + private static KeyOwnerKeysOperationRequest ownerRequest() { + return KeyOwnerKeysOperationRequest.builder().operation(DELETE) + .accounts(List.of("a")).issuerServices(List.of(ISSUER)).build(); + } + + private static KeysByOwnerAndAccountAndServiceVo row( + String account, String user, String id, Instant created) { + return KeysByOwnerAndAccountAndServiceVo.builder().ncaId(account).ownerType(USER) + .ownerId(user).issuerServiceId(ISSUER).keyId(id).createdAt(created) + .keyStatus(KeyStatus.ACTIVE).build(); + } +} diff --git a/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/SyncKeysV2WildcardOperationsServiceTest.java b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/SyncKeysV2WildcardOperationsServiceTest.java new file mode 100644 index 0000000000..b1b552c2cb --- /dev/null +++ b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/services/SyncKeysV2WildcardOperationsServiceTest.java @@ -0,0 +1,366 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.apikeys.services; + +import static com.nvidia.apikeys.dto.keysV2.KeysOperationType.DELETE; +import static com.nvidia.apikeys.dto.keysV2.KeysOperationType.REVOKED; +import static com.nvidia.apikeys.vo.KeyOwnerType.USER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import com.nvidia.apikeys.caching.CachingKeysLoader; +import com.nvidia.apikeys.caching.IntrospectionLocalCache; +import com.nvidia.apikeys.config.NakProperties; +import com.nvidia.apikeys.dto.keysV2.AccountAdminKeysOperationRequest; +import com.nvidia.apikeys.dto.keysV2.KeyOwnerKeysOperationRequest; +import com.nvidia.apikeys.dto.keysV2.KeysOperationType; +import com.nvidia.apikeys.dto.keysV2.PlatformAdminKeysOperationRequest; +import com.nvidia.apikeys.dto.keysV2.KeyOperationResponse; +import com.nvidia.apikeys.persistance.models.OwnerStatusByAccountAndServiceModel; +import com.nvidia.apikeys.persistance.models.OwnerStatusByAccountModel; +import com.nvidia.apikeys.services.SyncKeysV2OperationsService.BulkOperationException; +import com.nvidia.apikeys.vo.KeyOwnerStatus; +import com.nvidia.apikeys.vo.KeyStatus; +import com.nvidia.apikeys.vo.KeysByOwnerAndAccountAndServiceVo; +import com.nvidia.apikeys.vo.ServiceVo; +import com.nvidia.boot.exceptions.BadRequestException; +import com.nvidia.boot.exceptions.ForbiddenException; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class SyncKeysV2WildcardOperationsServiceTest { + + @Mock + private CachingKeysLoader keyCache; + @Mock + private IntrospectionLocalCache introspectionCache; + + private static final Instant CUTOFF = Instant.parse("2026-10-26T12:00:00Z"); + private static final List ALL = List.of("*"); + private static final List ACCOUNTS = List.of("a1", "a2"); + private static final List USERS = List.of("u1", "u2"); + private static final List ISSUERS = List.of("s1", "s2"); + + @Mock private KeysV2Service keys; + @Mock private ServicesService services; + @Mock private KeysV2BulkSelectionService selection; + private SyncKeysV2OperationsService service; + private final List mutations = new ArrayList<>(); + + @BeforeEach + void setUp() { + var legacy = new LegacyAccountTranslator(NakProperties.builder() + .ncaId("legacy").serviceIdMap(Map.of("s1", "legacy-s1")).build()); + var clock = new MillisecondPrecisionClock(Clock.fixed(CUTOFF, ZoneOffset.UTC)); + service = new SyncKeysV2OperationsService(keys, services, legacy, clock, 10, selection, keyCache, introspectionCache); + lenient().when(services.getServiceById(anyString())).thenAnswer(call -> + Optional.of(ServiceVo.builder().serviceId(call.getArgument(0)).build())); + lenient().doAnswer(call -> { + mutations.add(new Mutation(DELETE, call.getArgument(0), call.getArgument(2), + call.getArgument(3), call.getArgument(4))); + return null; + }).when(keys).deleteKeyById(anyString(), eq(USER), anyString(), anyString(), anyString()); + lenient().doAnswer(call -> { + assertThat(call.getArgument(5, KeyStatus.class)).isEqualTo(KeyStatus.REVOKED); + mutations.add(new Mutation(REVOKED, call.getArgument(0), call.getArgument(2), + call.getArgument(3), call.getArgument(4))); + return null; + }).when(keys).updateKeyStatus(anyString(), eq(USER), anyString(), anyString(), anyString(), any()); + } + + @ParameterizedTest + @MethodSource("selectorCombinations") + void selectsEveryMatchingKeyWithinTheActorBoundary(String actor, KeysOperationType operation, + boolean everyAccount, boolean everyUser, boolean everyIssuer) { + var fixtures = new ArrayList(); + for (var account : List.of("a1", "a2", "a3")) { + for (var user : List.of("u1", "u2", "u3")) { + for (var issuer : List.of("s1", "s2", "s3")) { + fixtures.add(row(account, user, issuer, "first")); + fixtures.add(row(account, user, issuer, "second")); + } + } + } + stubSelection(fixtures); + var accounts = actor.equals("ACCOUNT_ADMIN") ? List.of("a1") : everyAccount ? ALL : ACCOUNTS; + var users = actor.equals("KEY_OWNER") ? List.of("u1") : everyUser ? ALL : USERS; + var issuers = everyIssuer ? ALL : ISSUERS; + var expected = fixtures.stream() + .filter(key -> includes(accounts, key.getNcaId()) && includes(users, key.getOwnerId()) + && includes(issuers, key.getIssuerServiceId())) + .map(key -> mutation(operation, key)).toList(); + + var response = operate(actor, operation, accounts, users, issuers, + everyAccount && everyUser && everyIssuer ? true : null); + + assertThat(response.getCompletedCount()).isEqualTo(expected.size()); + assertThat(mutations).containsExactlyInAnyOrderElementsOf(expected); + assertThat(response.getSubmittedAt()).isEqualTo(CUTOFF); + verify(selection, atLeastOnce()).select(any(), any(), any(), eq(CUTOFF)); + } + + static Stream selectorCombinations() { + var cases = new ArrayList(); + for (var operation : KeysOperationType.values()) { + for (int mask = 0; mask < 8; mask++) { + boolean accounts = (mask & 1) != 0; + boolean users = (mask & 2) != 0; + boolean issuers = (mask & 4) != 0; + cases.add(Arguments.of("PLATFORM_ADMIN", operation, accounts, users, issuers)); + if (!users) { + cases.add(Arguments.of("KEY_OWNER", operation, accounts, false, issuers)); + } + if (!accounts) { + cases.add(Arguments.of("ACCOUNT_ADMIN", operation, false, users, issuers)); + } + } + } + return cases.stream(); + } + + @ParameterizedTest + @MethodSource("invalidSelectors") + void rejectsInvalidSelectorsBeforeDiscoveryOrMutation(String dimension, List values) { + var request = PlatformAdminKeysOperationRequest.builder().operation(DELETE) + .accounts(ACCOUNTS).users(USERS).issuerServices(ISSUERS).build(); + switch (dimension) { + case "accounts" -> request.setAccounts(values); + case "users" -> request.setUsers(values); + case "issuerServices" -> request.setIssuerServices(values); + default -> throw new IllegalArgumentException(dimension); + } + + assertThrows(BadRequestException.class, () -> service.operateAsPlatformAdmin(request)); + + verifyNoInteractions(selection, keys); + } + + static Stream invalidSelectors() { + List> invalid = Arrays.asList(null, List.of(), List.of("*", "id"), + List.of("id", "*"), List.of("id", "id"), List.of("*", "*"), + Arrays.asList("id", null), List.of(" "), + java.util.stream.IntStream.range(0, 11).mapToObj(i -> "id" + i).toList()); + return Stream.of("accounts", "users", "issuerServices") + .flatMap(dimension -> invalid.stream().map(values -> Arguments.of(dimension, values))); + } + + @ParameterizedTest + @MethodSource("missingPlatformConfirmations") + void deploymentWideSelectionRequiresExplicitTrue(KeysOperationType operation, Boolean confirmation) { + assertThrows(BadRequestException.class, + () -> operate("PLATFORM_ADMIN", operation, ALL, ALL, ALL, confirmation)); + verifyNoInteractions(selection, keys); + } + + static Stream missingPlatformConfirmations() { + return Stream.of(DELETE, REVOKED).flatMap(operation -> + Arrays.asList((Boolean) null, false).stream() + .map(confirmation -> Arguments.of(operation, confirmation))); + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3, 4, 5, 6}) + void partialWildcardsDoNotRequirePlatformWideConfirmation(int mask) { + stubSelection(List.of(row("a1", "u1", "s1", "key"))); + var response = operate("PLATFORM_ADMIN", DELETE, + (mask & 1) != 0 ? ALL : ACCOUNTS, + (mask & 2) != 0 ? ALL : USERS, + (mask & 4) != 0 ? ALL : ISSUERS, false); + assertThat(response.getCompletedCount()).isOne(); + assertThat(mutations).containsExactly(new Mutation(DELETE, "a1", "u1", "s1", "key")); + } + + @ParameterizedTest + @CsvSource({"accounts,a*,u1,s1", "users,a1,user*,s1", "issuerServices,a1,u1,issuer*"}) + void embeddedAsterisksStayLiteralWhenOtherDimensionsAreWildcards( + String dimension, String account, String user, String issuer) { + var exact = row(account, user, issuer, "exact"); + stubSelection(List.of(exact, row(account.replace("*", "suffix"), user.replace("*", "suffix"), + issuer.replace("*", "suffix"), "different"))); + var response = operate("PLATFORM_ADMIN", DELETE, + dimension.equals("accounts") ? List.of(account) : ALL, + dimension.equals("users") ? List.of(user) : ALL, + dimension.equals("issuerServices") ? List.of(issuer) : ALL, null); + assertThat(response.getCompletedCount()).isOne(); + assertThat(mutations).containsExactly(mutation(DELETE, exact)); + } + + @ParameterizedTest + @EnumSource(KeysOperationType.class) + void processesSelectionAndClosesItsStream(KeysOperationType operation) { + var first = row("a1", "u1", "s1", "first"); + var second = row("a2", "u2", "s2", "second"); + var closed = new AtomicBoolean(); + when(selection.select(null, null, null, CUTOFF)).thenAnswer(call -> + Stream.of(first, second).onClose(() -> closed.set(true))); + + var response = operate("PLATFORM_ADMIN", operation, ALL, ALL, ALL, true); + + assertThat(response.getCompletedCount()).isEqualTo(2); + assertThat(mutations).containsExactly(mutation(operation, first), mutation(operation, second)); + assertThat(closed).isTrue(); + } + + @Test + void emptyWildcardSelectionCompletesWithoutMutations() { + when(selection.select(null, null, null, CUTOFF)).thenReturn(Stream.empty()); + var response = operate("PLATFORM_ADMIN", DELETE, ALL, ALL, ALL, true); + assertThat(response.getCompletedCount()).isZero(); + assertThat(response.getOperationId()).isNotNull(); + verifyNoInteractions(keys); + } + + @ParameterizedTest + @MethodSource("outOfScopeKeys") + void rejectsDiscoveredKeysOutsideTheRequestedScope(String actor, List accounts, + List users, List issuers, KeysByOwnerAndAccountAndServiceVo wrongKey) { + when(selection.select(any(), any(), any(), any())).thenAnswer(call -> Stream.of(wrongKey)); + var failure = assertThrows(RuntimeException.class, + () -> operate(actor, DELETE, accounts, users, issuers, null)); + assertThat(failure instanceof BulkOperationException ? failure.getCause() : failure) + .isInstanceOf(IllegalStateException.class); + assertThat(mutations).isEmpty(); + } + + static Stream outOfScopeKeys() { + return Stream.of( + Arguments.of("KEY_OWNER", ALL, List.of("u1"), ALL, row("a1", "other", "s1", "key")), + Arguments.of("ACCOUNT_ADMIN", List.of("a1"), ALL, ALL, row("other", "u1", "s1", "key")), + Arguments.of("PLATFORM_ADMIN", ACCOUNTS, ALL, ALL, row("other", "u1", "s1", "key")), + Arguments.of("PLATFORM_ADMIN", ALL, USERS, ALL, row("a1", "other", "s1", "key")), + Arguments.of("PLATFORM_ADMIN", ALL, ALL, ISSUERS, row("a1", "u1", "other", "key")), + Arguments.of("ACCOUNT_ADMIN", List.of("a1"), ALL, ALL, + row("a1", "u1", "s1", "key").toBuilder().ownerType(null).build())); + } + + @ParameterizedTest + @ValueSource(strings = {"account", "issuer"}) + void ownerWildcardPreflightRejectsSuspensionBeforeAnyMutation(String suspendedScope) { + stubSelection(List.of(row("a1", "u1", "s1", "allowed"), row("a2", "u1", "s2", "blocked"))); + when(keys.getOwnerStatusByAccount("a1", USER, "u1")).thenReturn(Optional.empty()); + when(keys.getOwnerStatusByAccountAndService("a1", USER, "u1", "s1")) + .thenReturn(Optional.empty()); + if (suspendedScope.equals("account")) { + when(keys.getOwnerStatusByAccount("a2", USER, "u1")).thenReturn(Optional.of( + OwnerStatusByAccountModel.builder().ncaId("a2").ownerType(USER).ownerId("u1") + .ownerStatus(KeyOwnerStatus.SUSPENDED).build())); + lenient().when(keys.getOwnerStatusByAccountAndService("a2", USER, "u1", "s2")) + .thenReturn(Optional.of(OwnerStatusByAccountAndServiceModel.builder() + .ncaId("a2").ownerType(USER).ownerId("u1").issuerServiceId("s2") + .ownerStatus(KeyOwnerStatus.ACTIVE).build())); + } else { + when(keys.getOwnerStatusByAccount("a2", USER, "u1")).thenReturn(Optional.empty()); + when(keys.getOwnerStatusByAccountAndService("a2", USER, "u1", "s2")) + .thenReturn(Optional.of(OwnerStatusByAccountAndServiceModel.builder() + .ncaId("a2").ownerType(USER).ownerId("u1").issuerServiceId("s2") + .ownerStatus(KeyOwnerStatus.SUSPENDED).build())); + } + assertThrows(ForbiddenException.class, + () -> operate("KEY_OWNER", DELETE, ALL, List.of("u1"), ALL, null)); + assertThat(mutations).isEmpty(); + } + + @ParameterizedTest + @CsvSource({"ACCOUNT_ADMIN,DELETE", "ACCOUNT_ADMIN,REVOKED", "PLATFORM_ADMIN,DELETE", "PLATFORM_ADMIN,REVOKED"}) + void administratorsCanManageSuspendedUsersWithWildcards(String actor, KeysOperationType operation) { + var key = row("a1", "u1", "s1", "key"); + stubSelection(List.of(key)); + lenient().when(keys.getOwnerStatusByAccount(anyString(), eq(USER), anyString())) + .thenReturn(Optional.of(OwnerStatusByAccountModel.builder() + .ncaId("a1").ownerType(USER).ownerId("u1") + .ownerStatus(KeyOwnerStatus.SUSPENDED).build())); + lenient().when(keys.getOwnerStatusByAccountAndService(anyString(), eq(USER), anyString(), anyString())) + .thenReturn(Optional.of(OwnerStatusByAccountAndServiceModel.builder() + .ncaId("a1").ownerType(USER).ownerId("u1").issuerServiceId("s1") + .ownerStatus(KeyOwnerStatus.SUSPENDED).build())); + var response = operate(actor, operation, actor.equals("ACCOUNT_ADMIN") ? List.of("a1") : ALL, + ALL, ALL, actor.equals("PLATFORM_ADMIN") ? true : null); + assertThat(response.getCompletedCount()).isOne(); + assertThat(mutations).containsExactly(mutation(operation, key)); + } + + private void stubSelection(List fixtures) { + when(selection.select(any(), any(), any(), eq(CUTOFF))).thenAnswer(call -> { + String account = call.getArgument(0); + String user = call.getArgument(1); + String issuer = call.getArgument(2); + return fixtures.stream().filter(key -> (account == null || account.equals(key.getNcaId())) + && (user == null || user.equals(key.getOwnerId())) + && (issuer == null || issuer.equals(key.getIssuerServiceId()))); + }); + } + + private KeyOperationResponse operate(String actor, KeysOperationType operation, + List accounts, List users, List issuers, Boolean confirmation) { + return switch (actor) { + case "KEY_OWNER" -> service.operateAsKeyOwner(users.getFirst(), KeyOwnerKeysOperationRequest.builder() + .operation(operation).accounts(accounts).issuerServices(issuers).build()); + case "ACCOUNT_ADMIN" -> service.operateAsAccountAdmin(accounts.getFirst(), + AccountAdminKeysOperationRequest.builder().operation(operation) + .users(users).issuerServices(issuers).build()); + case "PLATFORM_ADMIN" -> service.operateAsPlatformAdmin(PlatformAdminKeysOperationRequest.builder() + .operation(operation).accounts(accounts).users(users).issuerServices(issuers) + .confirmPlatformWide(confirmation).build()); + default -> throw new IllegalArgumentException(actor); + }; + } + + private static boolean includes(List selection, String value) { + return selection.equals(ALL) || selection.contains(value); + } + + private static KeysByOwnerAndAccountAndServiceVo row(String account, String user, String issuer, String keyId) { + return KeysByOwnerAndAccountAndServiceVo.builder().ncaId(account).ownerType(USER).ownerId(user) + .issuerServiceId(issuer).keyId(keyId).createdAt(CUTOFF).keyStatus(KeyStatus.ACTIVE).build(); + } + + private static Mutation mutation(KeysOperationType operation, KeysByOwnerAndAccountAndServiceVo key) { + return new Mutation(operation, key.getNcaId(), key.getOwnerId(), key.getIssuerServiceId(), key.getKeyId()); + } + + private record Mutation(KeysOperationType operation, String account, String user, String issuer, String keyId) { + } +}