Skip to content

Commit 0437da1

Browse files
theghost5800theghost5800
committed
Add pagination for long queries in CustomControllerClients
JIRA:LMCROSSITXSADEPLOY-3432
1 parent 636ca41 commit 0437da1

8 files changed

Lines changed: 1974 additions & 60 deletions

File tree

‎multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/cf/clients/CustomControllerClient.java‎

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
public abstract class CustomControllerClient {
1919

20+
protected static final int MAX_URI_QUERY_LENGTH = 4000;
21+
2022
private final WebClient webClient;
2123
private String correlationId = StringUtils.EMPTY;
2224
private final CloudControllerHeaderConfiguration headerConfiguration;
@@ -34,17 +36,55 @@ protected CustomControllerClient(ApplicationConfiguration configuration, WebClie
3436
this.headerConfiguration = new CloudControllerHeaderConfiguration(configuration.getVersion());
3537
}
3638

37-
protected <T> List<T> getListOfResources(ResourcesResponseMapper<T> responseMapper, String uri, Object... urlVariables) {
38-
PaginationV3 pagination = addPageOfResources(uri, responseMapper, urlVariables);
39+
protected <T> List<T> getListOfResources(ResourcesResponseMapper<T> responseMapper, String uri) {
40+
PaginationV3 pagination = addPageOfResources(uri, responseMapper);
3941
while (!StringUtils.isEmpty(pagination.getNextUri())) {
4042
pagination = addPageOfResources(pagination.getNextUri(), responseMapper);
4143
}
4244
return responseMapper.getMappedResources();
4345
}
4446

45-
private PaginationV3 addPageOfResources(String uri, ResourcesResponseMapper<?> responseMapper, Object... urlVariables) {
47+
protected <T> List<T> getListOfResourcesInBatches(ResourcesResponseMapper<T> responseMapper, String uriPrefix, String batchParamPrefix,
48+
List<String> batchValues) {
49+
int fixedUriLength = uriPrefix.length() + batchParamPrefix.length();
50+
List<List<String>> batches = splitIntoBatches(batchValues, fixedUriLength);
51+
return batches.stream()
52+
.map(batch -> {
53+
String uri = uriPrefix + batchParamPrefix + String.join(",", batch);
54+
return getListOfResources(responseMapper, uri);
55+
})
56+
.flatMap(List::stream)
57+
.toList();
58+
}
59+
60+
List<List<String>> splitIntoBatches(List<String> values, int fixedUriLength) {
61+
int maxBatchLength = Math.max(1, MAX_URI_QUERY_LENGTH - fixedUriLength);
62+
List<List<String>> batches = new ArrayList<>();
63+
List<String> currentBatch = new ArrayList<>();
64+
int currentLength = 0;
65+
66+
for (String value : values) {
67+
// Account for the comma separator between values
68+
int addedLength = currentBatch.isEmpty() ? value.length() : value.length() + 1;
69+
if (!currentBatch.isEmpty() && currentLength + addedLength > maxBatchLength) {
70+
batches.add(currentBatch);
71+
currentBatch = new ArrayList<>();
72+
currentLength = 0;
73+
addedLength = value.length();
74+
}
75+
currentBatch.add(value);
76+
currentLength += addedLength;
77+
}
78+
79+
if (!currentBatch.isEmpty()) {
80+
batches.add(currentBatch);
81+
}
82+
return batches;
83+
}
84+
85+
private PaginationV3 addPageOfResources(String uri, ResourcesResponseMapper<?> responseMapper) {
4686
String responseString = webClient.get()
47-
.uri(uri, urlVariables)
87+
.uri(uri)
4888
.headers(httpHeaders -> httpHeaders.addAll(generateRequestHeaders()))
4989
.retrieve()
5090
.bodyToMono(String.class)

‎multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/cf/clients/CustomServiceKeysClient.java‎

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,10 @@
1818
import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration;
1919

2020
public class CustomServiceKeysClient extends CustomControllerClient {
21-
2221
private static final String SERVICE_KEYS_RESOURCE_BASE_URI = "/v3/service_credential_bindings";
2322
private static final String SERVICE_KEYS_BY_METADATA_SELECTOR_URI = SERVICE_KEYS_RESOURCE_BASE_URI + "?type=key&label_selector={value}";
2423
private static final String INCLUDE_SERVICE_INSTANCE_RESOURCES_PARAM = "&include=service_instance";
25-
24+
private static final String SERVICE_INSTANCE_GUIDS_PARAM_PREFIX = "&service_instance_guids=";
2625
private final CloudEntityResourceMapper resourceMapper = new CloudEntityResourceMapper();
2726

2827
public CustomServiceKeysClient(ApplicationConfiguration configuration, WebClientFactory webClientFactory, CloudCredentials credentials,
@@ -35,17 +34,13 @@ public List<DeployedMtaServiceKey> getServiceKeysByMetadataAndExistingGuids(
3534
String mtaId,
3635
String mtaNamespace,
3736
List<String> existingServiceGuids) {
38-
3937
String labelSelector = buildMtaMetadataLabelSelector(spaceGuid, mtaId, mtaNamespace);
40-
4138
List<String> allServiceGuids = existingServiceGuids.stream()
4239
.filter(Objects::nonNull)
4340
.toList();
44-
4541
if (allServiceGuids.isEmpty()) {
4642
return List.of();
4743
}
48-
4944
return new CustomControllerClientErrorHandler()
5045
.handleErrorsOrReturnResult(
5146
() -> getServiceKeysByMetadataInternal(labelSelector, allServiceGuids)
@@ -57,15 +52,11 @@ public List<DeployedMtaServiceKey> getServiceKeysByMetadataAndManagedServices(
5752
String mtaId,
5853
String mtaNamespace,
5954
List<DeployedMtaService> services) {
60-
6155
String labelSelector = buildMtaMetadataLabelSelector(spaceGuid, mtaId, mtaNamespace);
62-
6356
List<String> managedGuids = extractManagedServiceGuids(services);
64-
6557
if (managedGuids.isEmpty()) {
6658
return List.of();
6759
}
68-
6960
return new CustomControllerClientErrorHandler()
7061
.handleErrorsOrReturnResult(
7162
() -> getServiceKeysByMetadataInternal(labelSelector, managedGuids)
@@ -75,7 +66,6 @@ public List<DeployedMtaServiceKey> getServiceKeysByMetadataAndManagedServices(
7566
private String buildMtaMetadataLabelSelector(String spaceGuid,
7667
String mtaId,
7768
String mtaNamespace) {
78-
7969
return MtaMetadataCriteriaBuilder.builder()
8070
.label(MtaMetadataLabels.SPACE_GUID)
8171
.hasValue(spaceGuid)
@@ -98,13 +88,12 @@ private List<String> extractManagedServiceGuids(List<DeployedMtaService> service
9888
}
9989

10090
private List<DeployedMtaServiceKey> getServiceKeysByMetadataInternal(String labelSelector, List<String> guids) {
101-
102-
String uriSuffix = INCLUDE_SERVICE_INSTANCE_RESOURCES_PARAM
103-
+ "&service_instance_guids=" + String.join(",", guids);
104-
105-
return getListOfResources(new ServiceKeysResponseMapper(),
106-
SERVICE_KEYS_BY_METADATA_SELECTOR_URI + uriSuffix,
107-
labelSelector);
91+
String expandedUriPrefix = SERVICE_KEYS_BY_METADATA_SELECTOR_URI.replace("{value}", labelSelector)
92+
+ INCLUDE_SERVICE_INSTANCE_RESOURCES_PARAM;
93+
return getListOfResourcesInBatches(new ServiceKeysResponseMapper(),
94+
expandedUriPrefix,
95+
SERVICE_INSTANCE_GUIDS_PARAM_PREFIX,
96+
guids);
10897
}
10998

11099
private List<DeployedMtaService> getManagedServices(List<DeployedMtaService> services) {
@@ -132,12 +121,10 @@ public List<DeployedMtaServiceKey> getMappedResources() {
132121

133122
public Map<String, CloudServiceInstance> getIncludedServiceInstancesMapping() {
134123
List<Object> serviceInstances = getIncludedResources().getOrDefault("service_instances", Collections.emptyList());
135-
136124
return serviceInstances.stream()
137125
.distinct()
138126
.map(service -> (Map<String, Object>) service)
139127
.collect(Collectors.toMap(service -> (String) service.get("guid"), resourceMapper::mapService));
140-
141128
}
142129
}
143-
}
130+
}

‎multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/cf/clients/ServiceInstanceRoutesGetter.java‎

Lines changed: 8 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
package org.cloudfoundry.multiapps.controller.core.cf.clients;
22

3-
import java.util.ArrayList;
4-
import java.util.Collection;
53
import java.util.List;
64
import java.util.Map;
75
import java.util.stream.Collectors;
@@ -13,48 +11,23 @@
1311

1412
public class ServiceInstanceRoutesGetter extends CustomControllerClient {
1513

16-
private static final int MAX_CHAR_LENGTH_FOR_PARAMS_IN_REQUEST = 4000;
14+
private static final String SERVICE_ROUTE_BINDINGS_URI_PREFIX = "/v3/service_route_bindings?";
15+
private static final String ROUTE_GUIDS_PARAM_PREFIX = "route_guids=";
1716

1817
public ServiceInstanceRoutesGetter(ApplicationConfiguration configuration, WebClientFactory webClientFactory,
1918
CloudCredentials credentials, String correlationId) {
2019
super(configuration, webClientFactory, credentials, correlationId);
2120
}
2221

23-
public List<ServiceRouteBinding> getServiceRouteBindings(Collection<String> routeGuids) {
22+
public List<ServiceRouteBinding> getServiceRouteBindings(List<String> routeGuids) {
2423
return new CustomControllerClientErrorHandler().handleErrorsOrReturnResult(() -> doGetServiceRouteBindings(routeGuids));
2524
}
2625

27-
private List<ServiceRouteBinding> doGetServiceRouteBindings(Collection<String> routeGuids) {
28-
var batchedRouteGuids = getBatchedRouteGuids(routeGuids);
29-
var responseMapper = new ServiceRouteBindingsResponseMapper();
30-
return batchedRouteGuids.stream()
31-
.map(this::getServiceRouteBindingsUrl)
32-
.map(url -> getListOfResources(responseMapper, url))
33-
.flatMap(List::stream)
34-
.collect(Collectors.toList());
35-
}
36-
37-
private List<List<String>> getBatchedRouteGuids(Collection<String> routeGuids) {
38-
List<List<String>> batches = new ArrayList<>();
39-
int currentBatchLength = 0, currentBatchIndex = 0;
40-
batches.add(new ArrayList<>());
41-
42-
for (String routeGuid : routeGuids) {
43-
int elementLength = routeGuid.length();
44-
if (elementLength + currentBatchLength >= MAX_CHAR_LENGTH_FOR_PARAMS_IN_REQUEST) {
45-
batches.add(new ArrayList<>());
46-
currentBatchIndex++;
47-
currentBatchLength = 0;
48-
}
49-
batches.get(currentBatchIndex)
50-
.add(routeGuid);
51-
currentBatchLength += elementLength;
52-
}
53-
return batches;
54-
}
55-
56-
private String getServiceRouteBindingsUrl(Collection<String> routeGuids) {
57-
return "/v3/service_route_bindings?route_guids=" + String.join(",", routeGuids);
26+
private List<ServiceRouteBinding> doGetServiceRouteBindings(List<String> routeGuids) {
27+
return getListOfResourcesInBatches(new ServiceRouteBindingsResponseMapper(),
28+
SERVICE_ROUTE_BINDINGS_URI_PREFIX,
29+
ROUTE_GUIDS_PARAM_PREFIX,
30+
routeGuids);
5831
}
5932

6033
protected static class ServiceRouteBindingsResponseMapper extends ResourcesResponseMapper<ServiceRouteBinding> {

0 commit comments

Comments
 (0)