Skip to content

Commit bc78b31

Browse files
ServerSideHannesServerSideHannes
committed
refactor: separate CI cluster-test from local docker-compose
CI (cluster-test.yml): - Self-contained like helm-install-test.yml - Uses helm/kind-action directly - Simple Redis, 3 replicas, low CPU limits - Full load test with encryption verification Local (docker-compose.cluster.yml): - Kept for local development with full redis-ha - Removed CI-specific conditional code - Uses --wait for helm install
1 parent 99689ae commit bc78b31

2 files changed

Lines changed: 263 additions & 136 deletions

File tree

‎.github/workflows/cluster-test.yml‎

Lines changed: 245 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,47 +12,265 @@ on:
1212
jobs:
1313
cluster-test:
1414
runs-on: ubuntu-latest
15-
timeout-minutes: 40
15+
timeout-minutes: 20
1616
steps:
1717
- name: Checkout
1818
uses: actions/checkout@v6
1919

2020
- name: Set up Docker Buildx
2121
uses: docker/setup-buildx-action@v3
2222

23-
- name: Cache Docker layers
24-
uses: actions/cache@v4
25-
with:
26-
path: /tmp/.buildx-cache
27-
key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile', 'pyproject.toml') }}
28-
restore-keys: |
29-
${{ runner.os }}-buildx-
30-
31-
- name: Pre-build s3proxy image (cached)
23+
- name: Build s3proxy image
3224
uses: docker/build-push-action@v6
3325
with:
3426
context: .
3527
load: true
3628
tags: s3proxy:latest
37-
cache-from: type=local,src=/tmp/.buildx-cache
38-
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
39-
40-
- name: Pre-pull Kind node image
41-
run: docker pull kindest/node:v1.29.2
4229

43-
- name: Cache Helm dependencies
44-
uses: actions/cache@v4
30+
- name: Create Kind cluster
31+
uses: helm/kind-action@v1
4532
with:
46-
path: manifests/charts
47-
key: ${{ runner.os }}-helm-deps-${{ hashFiles('manifests/Chart.lock') }}
48-
restore-keys: |
49-
${{ runner.os }}-helm-deps-
33+
node_image: kindest/node:v1.29.2
34+
cluster_name: cluster-test
35+
36+
- name: Load image into Kind
37+
run: kind load docker-image s3proxy:latest --name cluster-test
38+
39+
- name: Create namespace
40+
run: kubectl create namespace s3proxy
41+
42+
- name: Deploy MinIO
43+
run: |
44+
cat <<EOF | kubectl apply -n s3proxy -f -
45+
apiVersion: apps/v1
46+
kind: Deployment
47+
metadata:
48+
name: minio
49+
spec:
50+
replicas: 1
51+
selector:
52+
matchLabels:
53+
app: minio
54+
template:
55+
metadata:
56+
labels:
57+
app: minio
58+
spec:
59+
containers:
60+
- name: minio
61+
image: minio/minio:latest
62+
args: ["server", "/data"]
63+
env:
64+
- name: MINIO_ROOT_USER
65+
value: minioadmin
66+
- name: MINIO_ROOT_PASSWORD
67+
value: minioadmin
68+
ports:
69+
- containerPort: 9000
70+
---
71+
apiVersion: v1
72+
kind: Service
73+
metadata:
74+
name: minio
75+
spec:
76+
selector:
77+
app: minio
78+
ports:
79+
- port: 9000
80+
EOF
81+
kubectl wait --for=condition=ready pod -l app=minio -n s3proxy --timeout=120s
82+
83+
- name: Deploy Redis
84+
run: |
85+
cat <<EOF | kubectl apply -n s3proxy -f -
86+
apiVersion: apps/v1
87+
kind: Deployment
88+
metadata:
89+
name: redis
90+
spec:
91+
replicas: 1
92+
selector:
93+
matchLabels:
94+
app: redis
95+
template:
96+
metadata:
97+
labels:
98+
app: redis
99+
spec:
100+
containers:
101+
- name: redis
102+
image: redis:7-alpine
103+
ports:
104+
- containerPort: 6379
105+
resources:
106+
limits:
107+
memory: 128Mi
108+
cpu: 100m
109+
---
110+
apiVersion: v1
111+
kind: Service
112+
metadata:
113+
name: redis
114+
spec:
115+
selector:
116+
app: redis
117+
ports:
118+
- port: 6379
119+
EOF
120+
kubectl wait --for=condition=ready pod -l app=redis -n s3proxy --timeout=120s
121+
122+
- name: Build Helm dependencies
123+
run: helm dependency build manifests/
124+
125+
- name: Install s3proxy chart
126+
run: |
127+
helm install s3proxy manifests/ \
128+
--namespace s3proxy \
129+
--set image.repository=s3proxy \
130+
--set image.tag=latest \
131+
--set image.pullPolicy=IfNotPresent \
132+
--set s3.host="http://minio:9000" \
133+
--set secrets.encryptKey="test-encryption-key-32chars!!" \
134+
--set secrets.awsAccessKeyId=minioadmin \
135+
--set secrets.awsSecretAccessKey=minioadmin \
136+
--set redis-ha.enabled=false \
137+
--set externalRedis.url="redis://redis:6379/0" \
138+
--set replicaCount=3 \
139+
--set resources.limits.cpu=100m \
140+
--set resources.requests.cpu=50m \
141+
--wait \
142+
--timeout 5m
143+
144+
- name: Verify pods are running
145+
run: |
146+
kubectl get pods -n s3proxy
147+
POD_COUNT=$(kubectl get pods -n s3proxy -l app.kubernetes.io/name=s3proxy-python --no-headers | grep Running | wc -l)
148+
if [ "$POD_COUNT" -lt 3 ]; then
149+
echo "Expected 3 s3proxy pods, got $POD_COUNT"
150+
exit 1
151+
fi
152+
echo "✓ All 3 s3proxy pods running"
153+
154+
- name: Run load test
155+
run: |
156+
# Save pod names for load balancing check
157+
PODS=$(kubectl get pods -n s3proxy -l app.kubernetes.io/name=s3proxy-python -o jsonpath='{.items[*].metadata.name}')
158+
echo "Found pods: $PODS"
159+
160+
# Save starting log line counts
161+
for pod in $PODS; do
162+
kubectl logs $pod -n s3proxy 2>/dev/null | wc -l > /tmp/$pod.start
163+
done
164+
165+
# Run the load test
166+
kubectl run s3-load-test -n s3proxy --rm -i --restart=Never \
167+
--image=amazon/aws-cli:latest \
168+
--env="AWS_ACCESS_KEY_ID=minioadmin" \
169+
--env="AWS_SECRET_ACCESS_KEY=minioadmin" \
170+
--env="AWS_DEFAULT_REGION=us-east-1" \
171+
--command -- /bin/sh -c '
172+
set -e
173+
ENDPOINT="http://s3proxy-python:4433"
174+
175+
echo "=== Creating test bucket ==="
176+
aws --endpoint-url $ENDPOINT s3 mb s3://load-test-bucket || true
177+
178+
echo "=== Generating 10MB test files ==="
179+
mkdir -p /tmp/testfiles
180+
for i in 1 2 3; do
181+
dd if=/dev/urandom of=/tmp/testfiles/file-$i.bin bs=1M count=10 2>/dev/null &
182+
done
183+
wait
184+
ls -lh /tmp/testfiles/
185+
186+
echo "=== Starting concurrent uploads ==="
187+
START=$(date +%s)
188+
for i in 1 2 3; do
189+
aws --endpoint-url $ENDPOINT s3 cp /tmp/testfiles/file-$i.bin s3://load-test-bucket/file-$i.bin &
190+
done
191+
wait
192+
END=$(date +%s)
193+
echo "=== Uploads complete in $((END - START))s ==="
194+
195+
echo "=== Listing bucket ==="
196+
aws --endpoint-url $ENDPOINT s3 ls s3://load-test-bucket/
197+
198+
echo "=== Downloading and verifying ==="
199+
mkdir -p /tmp/downloads
200+
for i in 1 2 3; do
201+
aws --endpoint-url $ENDPOINT s3 cp s3://load-test-bucket/file-$i.bin /tmp/downloads/file-$i.bin &
202+
done
203+
wait
204+
205+
echo "=== Comparing checksums ==="
206+
ORIG_SUMS=$(md5sum /tmp/testfiles/*.bin | cut -d" " -f1 | sort)
207+
DOWN_SUMS=$(md5sum /tmp/downloads/*.bin | cut -d" " -f1 | sort)
208+
209+
if [ "$ORIG_SUMS" = "$DOWN_SUMS" ]; then
210+
echo "✓ Checksums match - round-trip successful"
211+
else
212+
echo "✗ Checksum mismatch!"
213+
exit 1
214+
fi
215+
216+
echo "=== Verifying encryption ==="
217+
dd if=/dev/urandom of=/tmp/encrypt-test.bin bs=1K count=100 2>/dev/null
218+
ORIG_SIZE=$(stat -c%s /tmp/encrypt-test.bin)
219+
ORIG_MD5=$(md5sum /tmp/encrypt-test.bin | cut -c1-32)
220+
221+
aws --endpoint-url $ENDPOINT s3 cp /tmp/encrypt-test.bin s3://load-test-bucket/encrypt-test.bin
222+
aws --endpoint-url http://minio:9000 s3 cp s3://load-test-bucket/encrypt-test.bin /tmp/raw.bin 2>/dev/null || true
223+
224+
if [ -f /tmp/raw.bin ]; then
225+
RAW_SIZE=$(stat -c%s /tmp/raw.bin)
226+
RAW_MD5=$(md5sum /tmp/raw.bin | cut -c1-32)
227+
EXPECTED_SIZE=$((ORIG_SIZE + 28))
228+
229+
if [ "$RAW_SIZE" = "$EXPECTED_SIZE" ] && [ "$ORIG_MD5" != "$RAW_MD5" ]; then
230+
echo "✓ Encryption verified - size +28 bytes (GCM overhead), content differs"
231+
else
232+
echo "✗ Encryption check failed"
233+
exit 1
234+
fi
235+
fi
236+
237+
echo ""
238+
echo "✓ All tests passed!"
239+
'
240+
241+
- name: Check load balancing
242+
run: |
243+
PODS=$(kubectl get pods -n s3proxy -l app.kubernetes.io/name=s3proxy-python -o jsonpath='{.items[*].metadata.name}')
244+
PODS_HIT=0
245+
246+
for pod in $PODS; do
247+
START_LINE=$(cat /tmp/$pod.start 2>/dev/null || echo "0")
248+
REQUEST_COUNT=$(kubectl logs $pod -n s3proxy 2>/dev/null | tail -n +$((START_LINE + 1)) | grep -cE "GET|POST|PUT|HEAD" || echo "0")
249+
if [ "$REQUEST_COUNT" -gt 0 ]; then
250+
PODS_HIT=$((PODS_HIT + 1))
251+
echo "✓ Pod $pod: received $REQUEST_COUNT requests"
252+
else
253+
echo " Pod $pod: received 0 requests"
254+
fi
255+
done
50256
51-
- name: Run cluster test
52-
run: make cluster-test
257+
if [ "$PODS_HIT" -ge 2 ]; then
258+
echo "✓ Load balancing verified - traffic distributed across $PODS_HIT pods"
259+
else
260+
echo "⚠ Traffic went to only $PODS_HIT pod(s)"
261+
fi
53262
54-
# Avoid cache growing unbounded
55-
- name: Move cache
263+
- name: Show logs on failure
264+
if: failure()
56265
run: |
57-
rm -rf /tmp/.buildx-cache
58-
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
266+
echo "=== Pod Status ==="
267+
kubectl get pods -n s3proxy -o wide
268+
echo ""
269+
echo "=== S3Proxy Logs ==="
270+
kubectl logs -l app.kubernetes.io/name=s3proxy-python -n s3proxy --tail=100
271+
echo ""
272+
echo "=== MinIO Logs ==="
273+
kubectl logs -l app=minio -n s3proxy --tail=50
274+
echo ""
275+
echo "=== Events ==="
276+
kubectl get events -n s3proxy --sort-by=.lastTimestamp

0 commit comments

Comments
 (0)