Skip to content

Commit c209dfd

Browse files
csviricsviri
authored
improve: add sharding sample as integration test (#3460)
Adds sample integration test for sharding. Signed-off-by: Attila Mészáros <a_meszaros@apple.com>
1 parent a1919fe commit c209dfd

3 files changed

Lines changed: 197 additions & 0 deletions

File tree

‎operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/shardselector/ShardSelectorIT.java‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
* Copyright Java Operator SDK Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.javaoperatorsdk.operator.baseapi.shardselector;
17+
18+
import java.time.Duration;
19+
20+
import org.junit.jupiter.api.AfterEach;
21+
import org.junit.jupiter.api.BeforeEach;
22+
import org.junit.jupiter.api.Test;
23+
24+
import io.fabric8.kubeapitest.junit.EnableKubeAPIServer;
25+
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
26+
import io.fabric8.kubernetes.client.KubernetesClient;
27+
import io.fabric8.kubernetes.client.KubernetesClientBuilder;
28+
import io.javaoperatorsdk.annotation.Sample;
29+
import io.javaoperatorsdk.operator.Operator;
30+
import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension;
31+
32+
import static org.assertj.core.api.Assertions.assertThat;
33+
import static org.awaitility.Awaitility.await;
34+
35+
@Sample(
36+
tldr = "Shard Selector for Splitting Resources Across Operator Instances",
37+
description =
38+
"""
39+
Demonstrates how to shard custom resources across multiple operator instances using shard \
40+
selectors. Two operators watch the same resource type, each configured with a shard \
41+
selector covering one evenly split half of the UID hash space. The test verifies that a \
42+
given custom resource is reconciled by exactly one instance, never by both and never by \
43+
neither. Sharding relies on the Kubernetes API server 'ShardedListAndWatch' alpha feature.
44+
""")
45+
@EnableKubeAPIServer(
46+
kubeAPIVersion = "1.36.*",
47+
apiServerFlags = {"--feature-gates=ShardedListAndWatch=true"},
48+
updateKubeConfigFile = true)
49+
class ShardSelectorIT {
50+
51+
// The two selectors split the 64-bit UID hash space in half: [0x0, 0x8000000000000000) and
52+
// [0x8000000000000000, 0x10000000000000000). Together they cover the whole space with no overlap,
53+
// so every resource is owned by exactly one shard.
54+
private static final String SHARD1 =
55+
"shardRange(object.metadata.uid, '0x0000000000000000', '0x8000000000000000')";
56+
private static final String SHARD2 =
57+
"shardRange(object.metadata.uid, '0x8000000000000000', '0x10000000000000000')";
58+
59+
private static final String TEST_RESOURCE_NAME = "shard-test1";
60+
private static final Duration EVENT_SETTLE_WINDOW = Duration.ofMillis(500);
61+
62+
private final KubernetesClient adminClient = new KubernetesClientBuilder().build();
63+
64+
private Operator operator1;
65+
private Operator operator2;
66+
private ShardSelectorTestReconciler reconciler1;
67+
private ShardSelectorTestReconciler reconciler2;
68+
69+
@BeforeEach
70+
void beforeEach() {
71+
LocallyRunOperatorExtension.applyCrd(ShardSelectorTestCustomResource.class, adminClient);
72+
reconciler1 = startOperatorForShard(SHARD1);
73+
reconciler2 = startOperatorForShard(SHARD2);
74+
}
75+
76+
@AfterEach
77+
void cleanup() {
78+
if (operator1 != null) {
79+
operator1.stop();
80+
}
81+
if (operator2 != null) {
82+
operator2.stop();
83+
}
84+
adminClient.resource(testCustomResource()).delete();
85+
}
86+
87+
@Test
88+
void onlyOneShardReconcilesTheResource() {
89+
adminClient.resource(testCustomResource()).create();
90+
91+
// The condition must hold for the whole settle window: exactly one shard ever reconciles the
92+
// resource, so the other shard has no chance to (incorrectly) pick it up later.
93+
await()
94+
.atMost(Duration.ofSeconds(30))
95+
.during(EVENT_SETTLE_WINDOW)
96+
.untilAsserted(
97+
() -> {
98+
int executions1 = reconciler1.getNumberOfExecutions();
99+
int executions2 = reconciler2.getNumberOfExecutions();
100+
// exactly one shard owns the resource
101+
assertThat((executions1 == 0) ^ (executions2 == 0)).isTrue();
102+
});
103+
}
104+
105+
private ShardSelectorTestReconciler startOperatorForShard(String shardSelector) {
106+
var reconciler = new ShardSelectorTestReconciler();
107+
var operator = new Operator(o -> o.withKubernetesClient(new KubernetesClientBuilder().build()));
108+
operator.register(reconciler, o -> o.withShardSelector(shardSelector));
109+
operator.start();
110+
if (operator1 == null) {
111+
operator1 = operator;
112+
} else {
113+
operator2 = operator;
114+
}
115+
return reconciler;
116+
}
117+
118+
private ShardSelectorTestCustomResource testCustomResource() {
119+
var resource = new ShardSelectorTestCustomResource();
120+
resource.setMetadata(new ObjectMetaBuilder().withName(TEST_RESOURCE_NAME).build());
121+
return resource;
122+
}
123+
}

‎operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/shardselector/ShardSelectorTestCustomResource.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright Java Operator SDK Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.javaoperatorsdk.operator.baseapi.shardselector;
17+
18+
import io.fabric8.kubernetes.api.model.Namespaced;
19+
import io.fabric8.kubernetes.client.CustomResource;
20+
import io.fabric8.kubernetes.model.annotation.Group;
21+
import io.fabric8.kubernetes.model.annotation.ShortNames;
22+
import io.fabric8.kubernetes.model.annotation.Version;
23+
24+
@Group("sample.javaoperatorsdk")
25+
@Version("v1")
26+
@ShortNames("sst")
27+
public class ShardSelectorTestCustomResource extends CustomResource<Void, Void>
28+
implements Namespaced {}

‎operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/shardselector/ShardSelectorTestReconciler.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright Java Operator SDK Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.javaoperatorsdk.operator.baseapi.shardselector;
17+
18+
import java.util.concurrent.atomic.AtomicInteger;
19+
20+
import io.javaoperatorsdk.operator.api.reconciler.*;
21+
import io.javaoperatorsdk.operator.support.TestExecutionInfoProvider;
22+
23+
/**
24+
* The shard selector is intentionally not set through the {@link
25+
* io.javaoperatorsdk.operator.api.config.informer.Informer} annotation: the test registers the same
26+
* reconciler on two operator instances and overrides the shard selector per instance so that the
27+
* two shards split the resources evenly.
28+
*/
29+
@ControllerConfiguration
30+
public class ShardSelectorTestReconciler
31+
implements Reconciler<ShardSelectorTestCustomResource>, TestExecutionInfoProvider {
32+
33+
private final AtomicInteger numberOfExecutions = new AtomicInteger(0);
34+
35+
@Override
36+
public UpdateControl<ShardSelectorTestCustomResource> reconcile(
37+
ShardSelectorTestCustomResource resource, Context<ShardSelectorTestCustomResource> context) {
38+
numberOfExecutions.addAndGet(1);
39+
return UpdateControl.noUpdate();
40+
}
41+
42+
@Override
43+
public int getNumberOfExecutions() {
44+
return numberOfExecutions.get();
45+
}
46+
}

0 commit comments

Comments
 (0)