Skip to content

Commit 6e763e3

Browse files
MartinRixhamMartinRixham
committed
Etcd cluster.
1 parent 2100228 commit 6e763e3

4 files changed

Lines changed: 283 additions & 19 deletions

File tree

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
create-stack:
22
aws cloudformation create-stack --stack-name asyncdb --template-body file://cloudformation.json --capabilities CAPABILITY_NAMED_IAM
33

4+
update-stack:
5+
aws cloudformation update-stack --stack-name asyncdb --template-body file://cloudformation.json --capabilities CAPABILITY_NAMED_IAM
6+
47
delete-stack:
58
aws cloudformation delete-stack --stack-name asyncdb
69

‎cloudformation.json‎

Lines changed: 225 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@
99
"ECSAMI": {
1010
"Type": "AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>",
1111
"Default": "/aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id"
12+
},
13+
"EtcdAmiId": {
14+
"Type": "AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>",
15+
"Default": "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2"
1216
}
1317
},
1418
"Resources": {
@@ -34,7 +38,7 @@
3438
"Type": "AWS::EC2::Subnet",
3539
"Properties": {
3640
"VpcId": { "Ref": "VPC" },
37-
"CidrBlock": "10.0.1.0/24",
41+
"CidrBlock": "10.0.0.0/24",
3842
"AvailabilityZone": { "Fn::Select": [ 0, { "Fn::GetAZs": "" } ] },
3943
"MapPublicIpOnLaunch": true
4044
}
@@ -43,7 +47,7 @@
4347
"Type": "AWS::EC2::Subnet",
4448
"Properties": {
4549
"VpcId": { "Ref": "VPC" },
46-
"CidrBlock": "10.0.2.0/24",
50+
"CidrBlock": "10.0.1.0/24",
4751
"AvailabilityZone": { "Fn::Select": [ 1, { "Fn::GetAZs": "" } ] },
4852
"MapPublicIpOnLaunch": true
4953
}
@@ -52,7 +56,7 @@
5256
"Type": "AWS::EC2::Subnet",
5357
"Properties": {
5458
"VpcId": { "Ref": "VPC" },
55-
"CidrBlock": "10.0.3.0/24",
59+
"CidrBlock": "10.0.2.0/24",
5660
"AvailabilityZone": { "Fn::Select": [ 2, { "Fn::GetAZs": "" } ] },
5761
"MapPublicIpOnLaunch": true
5862
}
@@ -118,7 +122,7 @@
118122
"Type": "AWS::EC2::SecurityGroup",
119123
"Properties": {
120124
"VpcId": { "Ref": "VPC" },
121-
"GroupDescription": "Allow inbound traffic from application load balancer",
125+
"GroupDescription": "Allow inbound traffic from application load balancer",
122126
"SecurityGroupIngress": [
123127
{
124128
"IpProtocol": "tcp",
@@ -135,6 +139,36 @@
135139
]
136140
}
137141
},
142+
"EtcdLoadBalancerSecurityGroup": {
143+
"Type": "AWS::EC2::SecurityGroup",
144+
"Properties": {
145+
"VpcId": { "Ref": "VPC" },
146+
"GroupDescription": "Allow etcd traffic",
147+
"SecurityGroupIngress": [
148+
{
149+
"IpProtocol": "tcp",
150+
"FromPort": 2379,
151+
"ToPort": 2380,
152+
"CidrIp": "0.0.0.0/0"
153+
}
154+
]
155+
}
156+
},
157+
"EtcdSecurityGroup": {
158+
"Type": "AWS::EC2::SecurityGroup",
159+
"Properties": {
160+
"VpcId": { "Ref": "VPC" },
161+
"GroupDescription": "Allow etcd traffic",
162+
"SecurityGroupIngress": [
163+
{
164+
"IpProtocol": "tcp",
165+
"FromPort": 2379,
166+
"ToPort": 2380,
167+
"CidrIp": "0.0.0.0/0"
168+
}
169+
]
170+
}
171+
},
138172
"InstanceRole": {
139173
"Type": "AWS::IAM::Role",
140174
"Properties": {
@@ -173,20 +207,20 @@
173207
"UserData": {
174208
"Fn::Base64": {
175209
"Fn::Join": [
176-
"",
210+
"\n",
177211
[
178-
"#! /bin/bash\n",
179-
"sudo yum update\n",
180-
"sudo yum -y install unzip\n",
181-
"curl \"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip\" -o \"awscliv2.zip\"\n",
182-
"unzip awscliv2.zip\n",
183-
"./aws/install\n",
184-
"REGISTRY_URL=332187735950.dkr.ecr.eu-west-2.amazonaws.com\n",
185-
"VERSION=0.0.2\n",
186-
"IMAGE=$REGISTRY_URL/asyncdb:$VERSION\n",
187-
"aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin $REGISTRY_URL\n",
188-
"docker pull $IMAGE\n",
189-
"docker run -d -p 80:80 $IMAGE\n"
212+
"#! /bin/bash",
213+
"sudo yum update",
214+
"sudo yum -y install unzip",
215+
"curl \"https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip\" -o \"awscliv2.zip\"",
216+
"unzip awscliv2.zip",
217+
"./aws/install",
218+
"REGISTRY_URL=332187735950.dkr.ecr.eu-west-2.amazonaws.com",
219+
"VERSION=0.0.2",
220+
"IMAGE=$REGISTRY_URL/asyncdb:$VERSION",
221+
"aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin $REGISTRY_URL",
222+
"docker pull $IMAGE",
223+
"docker run -d -p 80:80 $IMAGE"
190224
]
191225
]
192226
}
@@ -199,7 +233,11 @@
199233
"Properties": {
200234
"Name": "ClusterALB",
201235
"Scheme": "internet-facing",
202-
"Subnets": [{ "Ref": "PublicSubnet1" }, { "Ref": "PublicSubnet2" }, { "Ref": "PublicSubnet3" }],
236+
"Subnets": [
237+
{ "Ref": "PublicSubnet1" },
238+
{ "Ref": "PublicSubnet2" },
239+
{ "Ref": "PublicSubnet3" }
240+
],
203241
"SecurityGroups": [{ "Ref": "ALBSecurityGroup" }]
204242
}
205243
},
@@ -230,7 +268,11 @@
230268
"AutoScalingGroup": {
231269
"Type": "AWS::AutoScaling::AutoScalingGroup",
232270
"Properties": {
233-
"VPCZoneIdentifier": [ { "Ref": "PublicSubnet1" }, { "Ref": "PublicSubnet2" }, { "Ref": "PublicSubnet3" } ],
271+
"VPCZoneIdentifier": [
272+
{ "Ref": "PublicSubnet1" },
273+
{ "Ref": "PublicSubnet2" },
274+
{ "Ref": "PublicSubnet3" }
275+
],
234276
"LaunchTemplate": {
235277
"LaunchTemplateId": { "Ref": "LaunchTemplate" },
236278
"Version": { "Fn::GetAtt": [ "LaunchTemplate", "LatestVersionNumber" ] }
@@ -240,6 +282,170 @@
240282
"MaxSize": "4",
241283
"TargetGroupARNs":[{ "Ref":"ALBTargetGroup" } ]
242284
}
285+
},
286+
"DiscoveryTokenLambdaRole": {
287+
"Type": "AWS::IAM::Role",
288+
"Properties": {
289+
"AssumeRolePolicyDocument": {
290+
"Version": "2012-10-17",
291+
"Statement": [
292+
{
293+
"Effect": "Allow",
294+
"Principal": {
295+
"Service": "lambda.amazonaws.com"
296+
},
297+
"Action": "sts:AssumeRole"
298+
}
299+
]
300+
},
301+
"ManagedPolicyArns": [
302+
"arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
303+
]
304+
}
305+
},
306+
"DiscoveryTokenLambda": {
307+
"Type": "AWS::Lambda::Function",
308+
"Properties": {
309+
"Handler": "index.handler",
310+
"Role": { "Fn::GetAtt": ["DiscoveryTokenLambdaRole", "Arn"] },
311+
"Runtime": "python3.9",
312+
"Timeout": 30,
313+
"Code": {
314+
"ZipFile": {
315+
"Fn::Join": [
316+
"\n",
317+
[
318+
"import urllib.request",
319+
"import cfnresponse",
320+
"def handler(event, context):",
321+
" try:",
322+
" if event['RequestType'] in ('Create','Update'):",
323+
" url = \"https://discovery.etcd.io/new?size=3\"",
324+
" token = urllib.request.urlopen(url).read().decode().strip()",
325+
" cfnresponse.send(event, context, cfnresponse.SUCCESS,",
326+
" { 'DiscoveryURL': token })",
327+
" else:",
328+
" cfnresponse.send(event, context, cfnresponse.SUCCESS, {})",
329+
" except Exception as e:",
330+
" print(\"Error:\", e)",
331+
" cfnresponse.send(event, context, cfnresponse.FAILED, {})"
332+
]
333+
]
334+
}
335+
}
336+
}
337+
},
338+
"DiscoveryTokenCustomResource": {
339+
"Type": "Custom::EtcdDiscovery",
340+
"Properties": {
341+
"ServiceToken": { "Fn::GetAtt": ["DiscoveryTokenLambda", "Arn"] }
342+
}
343+
},
344+
"EtcdLaunchTemplate": {
345+
"Type": "AWS::EC2::LaunchTemplate",
346+
"Properties": {
347+
"LaunchTemplateData": {
348+
"ImageId": { "Ref": "EtcdAmiId" },
349+
"InstanceType": { "Ref": "InstanceType" },
350+
"SecurityGroupIds": [{ "Ref": "EtcdSecurityGroup" }],
351+
"UserData": {
352+
"Fn::Base64": {
353+
"Fn::Join": [
354+
"\n",
355+
[
356+
"#!/bin/bash",
357+
"yum install -y wget",
358+
"ETCD_VER=v3.5.10",
359+
"DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download",
360+
"wget ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz",
361+
"tar xvf etcd-${ETCD_VER}-linux-amd64.tar.gz",
362+
"mv etcd-${ETCD_VER}-linux-amd64/etcd* /usr/local/bin/",
363+
"PRIVATE_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)",
364+
"DISCOVERY_URL='${DiscoveryTokenCustomResource.DiscoveryURL}'",
365+
"nohup etcd \\",
366+
" --name ${PRIVATE_IP} \\",
367+
" --initial-advertise-peer-urls http://${PRIVATE_IP}:2380 \\",
368+
" --listen-peer-urls http://${PRIVATE_IP}:2380 \\",
369+
" --listen-client-urls http://${PRIVATE_IP}:2379,http://127.0.0.1:2379 \\",
370+
" --advertise-client-urls http://${PRIVATE_IP}:2379 \\",
371+
" --discovery ${DISCOVERY_URL} &"
372+
]
373+
]
374+
}
375+
}
376+
}
377+
}
378+
},
379+
"EtcdAutoScalingGroup": {
380+
"Type": "AWS::AutoScaling::AutoScalingGroup",
381+
"Properties": {
382+
"VPCZoneIdentifier": [
383+
{ "Ref": "PublicSubnet1" },
384+
{ "Ref": "PublicSubnet2" },
385+
{ "Ref": "PublicSubnet3" }
386+
],
387+
"LaunchTemplate": {
388+
"LaunchTemplateId": { "Ref": "EtcdLaunchTemplate" },
389+
"Version": { "Fn::GetAtt": ["EtcdLaunchTemplate", "LatestVersionNumber"] }
390+
},
391+
"MinSize": "3",
392+
"MaxSize": "3",
393+
"DesiredCapacity": "3",
394+
"HealthCheckType": "EC2",
395+
"Tags": [
396+
{
397+
"Key": "Name",
398+
"Value": "etcd-node",
399+
"PropagateAtLaunch": true
400+
}
401+
]
402+
}
403+
},
404+
"EtcdTargetGroup": {
405+
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
406+
"Properties": {
407+
"VpcId": { "Ref": "VPC" },
408+
"Port": 2379,
409+
"Protocol": "TCP",
410+
"TargetType": "instance",
411+
"HealthCheckProtocol": "TCP"
412+
}
413+
},
414+
"EtcdNLB": {
415+
"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
416+
"Properties": {
417+
"Subnets": [
418+
{ "Ref": "PublicSubnet1" },
419+
{ "Ref": "PublicSubnet2" },
420+
{ "Ref": "PublicSubnet3" }
421+
],
422+
"SecurityGroups": [{ "Ref": "EtcdLoadBalancerSecurityGroup" }],
423+
"Scheme": "internet-facing",
424+
"Type": "network"
425+
}
426+
},
427+
"EtcdListener": {
428+
"Type": "AWS::ElasticLoadBalancingV2::Listener",
429+
"Properties": {
430+
"LoadBalancerArn": { "Ref": "EtcdNLB" },
431+
"Port": 2379,
432+
"Protocol": "TCP",
433+
"DefaultActions": [
434+
{
435+
"Type": "forward",
436+
"TargetGroupArn": { "Ref": "EtcdTargetGroup" }
437+
}
438+
]
439+
}
440+
},
441+
"EtcdASGAttachment": {
442+
"Type": "AWS::AutoScaling::LifecycleHook",
443+
"Properties": {
444+
"AutoScalingGroupName": { "Ref": "EtcdAutoScalingGroup" },
445+
"LifecycleTransition": "autoscaling:EC2_INSTANCE_LAUNCHING",
446+
"DefaultResult": "CONTINUE",
447+
"HeartbeatTimeout": 300
448+
}
243449
}
244450
}
245451
}

‎test-etcd.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import grpc
2+
import socket
3+
from etcd3.etcdrpc import KVStub, RangeRequest
4+
5+
def test_etcd(elb_hostname: str):
6+
# Resolve ELB hostname to IPv4 manually
7+
ip = socket.gethostbyname(elb_hostname)
8+
target = f"{ip}:2379"
9+
print(f"Connecting to {target}")
10+
11+
# Connect directly to the IP
12+
channel = grpc.insecure_channel(target)
13+
14+
kv = KVStub(channel)
15+
request = RangeRequest(key=b"test")
16+
response = kv.Range(request, timeout=5)
17+
18+
print("Cluster responded OK")
19+
print("KVs returned:", len(response.kvs))
20+
21+
if __name__ == "__main__":
22+
elb = "asyncd-EtcdN-On7N9ya3YCJL-caba21a201f8acff.elb.eu-west-2.amazonaws.com"
23+
test_etcd(elb)
24+

‎test-etcd.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/usr/bin/env bash
2+
# test-etcd-cluster.sh
3+
# Usage: ./test-etcd-cluster.sh <LOAD_BALANCER_DNS_NAME>
4+
5+
LB_DNS=$1
6+
7+
if [ -z "$LB_DNS" ]; then
8+
echo "Usage: $0 <LOAD_BALANCER_DNS_NAME>"
9+
exit 1
10+
fi
11+
12+
echo "Checking etcd cluster via Load Balancer: $LB_DNS"
13+
14+
# 1. Check health
15+
echo "==> Checking cluster health..."
16+
curl -s http://$LB_DNS:2379/health | jq .
17+
18+
# 2. Write a test key
19+
echo "==> Writing test key..."
20+
curl -s http://$LB_DNS:2379/v2/keys/testkey -XPUT -d value="hello-etcd" | jq .
21+
22+
# 3. Read the test key
23+
echo "==> Reading test key..."
24+
curl -s http://$LB_DNS:2379/v2/keys/testkey | jq .
25+
26+
# 4. Delete the test key
27+
echo "==> Deleting test key..."
28+
curl -s http://$LB_DNS:2379/v2/keys/testkey -XDELETE | jq .
29+
30+
echo "Done."
31+

0 commit comments

Comments
 (0)