Skip to content

Commit c0e86c5

Browse files
vid277cdxker
vid277
authored and
cdxker
committed
feature: add chaos testing
1 parent 43af1d7 commit c0e86c5

3 files changed

Lines changed: 292 additions & 6 deletions

File tree

‎.github/workflows/integ-tests.yml‎

Lines changed: 214 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,6 @@ jobs:
1616
- name: Checkout repository
1717
uses: actions/checkout@v4
1818

19-
- name: Cache Rust dependencies
20-
uses: useblacksmith/cache@v5
21-
2219
- name: Cache Rust dependencies
2320
uses: useblacksmith/cache@v5
2421
with:
@@ -76,9 +73,6 @@ jobs:
7673
- name: Checkout repository
7774
uses: actions/checkout@v4
7875

79-
- name: Cache Rust dependencies
80-
uses: useblacksmith/cache@v5
81-
8276
- name: Cache Rust dependencies
8377
uses: useblacksmith/cache@v5
8478
with:
@@ -170,3 +164,217 @@ jobs:
170164
run: |
171165
docker compose -f docker-compose.yaml logs --no-color || true
172166
docker compose -f docker-compose.yaml down --volumes --remove-orphans || true
167+
168+
dkg_test_chaos_network:
169+
runs-on: blacksmith-16vcpu-ubuntu-2204
170+
env:
171+
MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
172+
MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
173+
IS_TESTNET: "true"
174+
steps:
175+
- name: Checkout repository
176+
uses: actions/checkout@v4
177+
178+
- name: Cache Rust dependencies
179+
uses: useblacksmith/cache@v5
180+
with:
181+
path: |
182+
~/.cargo/registry
183+
~/.cargo/git
184+
target
185+
key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}
186+
187+
- name: Set up Rust toolchain
188+
run: |
189+
rustup update 1.87
190+
rustup default 1.87
191+
192+
- name: Install dependencies
193+
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang
194+
195+
- name: Create NodeCluster (3 nodes)
196+
run: ./scripts/clear_dkg_keys.sh
197+
198+
- name: Start 5-node stack with network chaos
199+
run: |
200+
docker compose -f docker-compose.yaml -f docker-compose.chaos-network.yaml up -d --build
201+
202+
- name: Wait for gRPC endpoint
203+
run: |
204+
echo "Waiting for node to expose port 50051..."
205+
for i in {1..30}; do
206+
if nc -z localhost 50051; then
207+
echo "Port 50051 is open."
208+
break
209+
fi
210+
echo "Port not ready yet – retry #$i"
211+
sleep 5
212+
done
213+
214+
- name: Run end-to-end integration tests with network chaos
215+
run: cargo run --bin integration-tests check-dkg --port-range 50051-50055
216+
217+
- name: Shutdown stack & print logs
218+
run: |
219+
docker compose -f docker-compose.yaml -f docker-compose.chaos-network.yaml logs --no-color || true
220+
docker compose -f docker-compose.yaml -f docker-compose.chaos-network.yaml down --volumes --remove-orphans || true
221+
222+
dkg_test_chaos_nodes:
223+
runs-on: blacksmith-16vcpu-ubuntu-2204
224+
env:
225+
MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
226+
MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
227+
IS_TESTNET: "true"
228+
steps:
229+
- name: Checkout repository
230+
uses: actions/checkout@v4
231+
232+
- name: Cache Rust dependencies
233+
uses: useblacksmith/cache@v5
234+
with:
235+
path: |
236+
~/.cargo/registry
237+
~/.cargo/git
238+
target
239+
key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}
240+
241+
- name: Set up Rust toolchain
242+
run: |
243+
rustup update 1.87
244+
rustup default 1.87
245+
246+
- name: Install dependencies
247+
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang
248+
249+
- name: Create NodeCluster (3 nodes)
250+
run: ./scripts/clear_dkg_keys.sh
251+
252+
- name: Start 5-node stack with node chaos
253+
run: |
254+
docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml up -d --build
255+
256+
- name: Wait for gRPC endpoint
257+
run: |
258+
echo "Waiting for node to expose port 50051..."
259+
for i in {1..30}; do
260+
if nc -z localhost 50051; then
261+
echo "Port 50051 is open."
262+
break
263+
fi
264+
echo "Port not ready yet – retry #$i"
265+
sleep 5
266+
done
267+
268+
- name: Run end-to-end integration tests with node chaos
269+
run: cargo run --bin integration-tests check-dkg --port-range 50051-50055
270+
271+
- name: Shutdown stack & print logs
272+
run: |
273+
docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml logs --no-color || true
274+
docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml down --volumes --remove-orphans || true
275+
276+
end-to-end-deposit-withdrawl-chaos-network:
277+
runs-on: blacksmith-16vcpu-ubuntu-2204
278+
env:
279+
MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
280+
MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
281+
IS_TESTNET: "true"
282+
steps:
283+
- name: Checkout repository
284+
uses: actions/checkout@v4
285+
286+
- name: Cache Rust dependencies
287+
uses: useblacksmith/cache@v5
288+
with:
289+
path: |
290+
~/.cargo/registry
291+
~/.cargo/git
292+
target
293+
key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}
294+
295+
- name: Set up Rust toolchain
296+
run: |
297+
rustup update 1.87
298+
rustup default 1.87
299+
300+
- name: Install dependencies
301+
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang
302+
303+
- run: cargo clean
304+
305+
- name: Start 5-node stack with network chaos
306+
run: |
307+
docker compose -f docker-compose.yaml -f docker-compose.chaos-network.yaml up -d --build
308+
309+
- name: Wait for gRPC endpoint
310+
run: |
311+
echo "Waiting for node to expose port 50051..."
312+
for i in {1..30}; do
313+
if nc -z localhost 50051; then
314+
echo "Port 50051 is open."
315+
break
316+
fi
317+
echo "Port not ready yet – retry #$i"
318+
sleep 5
319+
done
320+
321+
- name: Run end-to-end integration tests with network chaos
322+
run: cargo run --bin integration-tests test
323+
324+
- name: Shutdown stack & print logs
325+
run: |
326+
docker compose -f docker-compose.yaml -f docker-compose.chaos-network.yaml logs --no-color || true
327+
docker compose -f docker-compose.yaml -f docker-compose.chaos-network.yaml down --volumes --remove-orphans || true
328+
329+
end-to-end-deposit-withdrawl-chaos-nodes:
330+
runs-on: blacksmith-16vcpu-ubuntu-2204
331+
env:
332+
MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
333+
MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
334+
IS_TESTNET: "true"
335+
steps:
336+
- name: Checkout repository
337+
uses: actions/checkout@v4
338+
339+
- name: Cache Rust dependencies
340+
uses: useblacksmith/cache@v5
341+
with:
342+
path: |
343+
~/.cargo/registry
344+
~/.cargo/git
345+
target
346+
key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}
347+
348+
- name: Set up Rust toolchain
349+
run: |
350+
rustup update 1.87
351+
rustup default 1.87
352+
353+
- name: Install dependencies
354+
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang
355+
356+
- run: cargo clean
357+
358+
- name: Start 5-node stack with node chaos
359+
run: |
360+
docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml up -d --build
361+
362+
- name: Wait for gRPC endpoint
363+
run: |
364+
echo "Waiting for node to expose port 50051..."
365+
for i in {1..30}; do
366+
if nc -z localhost 50051; then
367+
echo "Port 50051 is open."
368+
break
369+
fi
370+
echo "Port not ready yet – retry #$i"
371+
sleep 5
372+
done
373+
374+
- name: Run end-to-end integration tests with node chaos
375+
run: cargo run --bin integration-tests test
376+
377+
- name: Shutdown stack & print logs
378+
run: |
379+
docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml logs --no-color || true
380+
docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml down --volumes --remove-orphans || true

‎docker-compose.chaos-network.yaml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
services:
2+
node1:
3+
cap_add:
4+
- NET_ADMIN
5+
entrypoint: >
6+
/bin/bash -c "apt-get update && apt-get install -y --no-install-recommends iproute2 iptables && \
7+
modprobe sch_netem 2>/dev/null || true && \
8+
tc qdisc add dev eth0 root netem delay 200ms loss 10% 2>/dev/null || echo 'tc netem not available, continuing...' && \
9+
exec /app/cli run --key-file-path /app/configs/node_1.json --config-file-path /app/configs/node_1.yaml --use-mock-oracle"
10+
11+
node2:
12+
cap_add:
13+
- NET_ADMIN
14+
entrypoint: >
15+
/bin/bash -c "apt-get update && apt-get install -y --no-install-recommends iproute2 iptables && \
16+
modprobe sch_netem 2>/dev/null || true && \
17+
tc qdisc add dev eth0 root netem delay 200ms loss 10% 2>/dev/null || echo 'tc netem not available, continuing...' && \
18+
exec /app/cli run --key-file-path /app/configs/node_2.json --config-file-path /app/configs/node_2.yaml --use-mock-oracle"
19+
20+
node3:
21+
cap_add:
22+
- NET_ADMIN
23+
entrypoint: >
24+
/bin/bash -c "apt-get update && apt-get install -y --no-install-recommends iproute2 iptables && \
25+
modprobe sch_netem 2>/dev/null || true && \
26+
tc qdisc add dev eth0 root netem delay 200ms loss 10% 2>/dev/null || echo 'tc netem not available, continuing...' && \
27+
iptables -A OUTPUT -d node4 -j DROP && \
28+
exec /app/cli run --key-file-path /app/configs/node_3.json --config-file-path /app/configs/node_3.yaml --use-mock-oracle"
29+
30+
node4:
31+
cap_add:
32+
- NET_ADMIN
33+
entrypoint: >
34+
/bin/bash -c "apt-get update && apt-get install -y --no-install-recommends iproute2 iptables && \
35+
modprobe sch_netem 2>/dev/null || true && \
36+
tc qdisc add dev eth0 root netem delay 200ms loss 10% 2>/dev/null || echo 'tc netem not available, continuing...' && \
37+
iptables -A OUTPUT -d node3 -j DROP && \
38+
exec /app/cli run --key-file-path /app/configs/node_4.json --config-file-path /app/configs/node_4.yaml --use-mock-oracle"
39+
40+
node5:
41+
cap_add:
42+
- NET_ADMIN
43+
entrypoint: >
44+
/bin/bash -c "apt-get update && apt-get install -y --no-install-recommends iproute2 iptables && \
45+
modprobe sch_netem 2>/dev/null || true && \
46+
tc qdisc add dev eth0 root netem delay 200ms loss 10% 2>/dev/null || echo 'tc netem not available, continuing...' && \
47+
exec /app/cli run --key-file-path /app/configs/node_5.json --config-file-path /app/configs/node_5.yaml --use-mock-oracle"

‎docker-compose.chaos-nodes.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
services:
2+
# Apply memory/CPU pressure limits to each node container
3+
node1:
4+
mem_limit: 128m
5+
cpus: "0.5"
6+
7+
node2:
8+
mem_limit: 128m
9+
cpus: "0.5"
10+
11+
node3:
12+
mem_limit: 128m
13+
cpus: "0.5"
14+
15+
node4:
16+
mem_limit: 128m
17+
cpus: "0.5"
18+
19+
node5:
20+
mem_limit: 128m
21+
cpus: "0.5"
22+
23+
chaos-killer:
24+
image: docker:24.0-cli
25+
volumes:
26+
- /var/run/docker.sock:/var/run/docker.sock
27+
entrypoint: >
28+
/bin/sh -c 'set -e; echo "Chaos-killer started"; while true; do TARGET=$(docker ps --format "{{.Names}}" | grep -E "node[0-9]+$" | shuf -n 1); echo "[KILLER] Restarting $TARGET"; docker restart $TARGET; INTERVAL=$(shuf -i 10-40 -n 1); echo "[KILLER] sleeping $INTERVAL s"; sleep $INTERVAL; done'
29+
networks:
30+
- vaultnet
31+
restart: unless-stopped

0 commit comments

Comments
 (0)