Skip to content

Commit b697b36

Browse files
committed
Merge branch 'main' of https://github.com/aws-samples/aws-do-eks into main
2 parents 8671faa + b9797e1 commit b697b36

77 files changed

Lines changed: 4964 additions & 18 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.env

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,11 @@ export https_proxy=
1414
export no_proxy=localhost
1515

1616
# AWS settings
17-
## AWS_PROFILE - name of AWS settings profile AWS_PROFILE=default(default)|aws-do-eks|...
18-
export AWS_PROFILE=default
17+
if [ -f ${HOME}/.aws/credentials ]; then
18+
## AWS_PROFILE - name of AWS CLI settings profile to use AWS_PROFILE=default(default)|aws-do-eks|...
19+
export AWS_PROFILE=default
20+
fi
21+
## If no AWS CLI credentials are configured, then the instance profile is in effect
1922
## AWS_REGION - will be set to AWS_DEFAULT_REGION if not set externally.
2023
export AWS_DEFAULT_REGION=us-east-1
2124
if [ "${AWS_REGION}" == "" ]; then
@@ -62,5 +65,7 @@ export VOL_MAP="-v ${HOME}/.aws:/root/.aws -v ${HOME}/.kube:/root/.kube -v $(pwd
6265
## Network [optional] - Network name including the --net switch. Example --net mynet
6366
#export NETWORK=
6467
## RUN_OPTS [optional] - additional options to specify with the run comman. Example -e POSTGRES_DB=dbname
65-
export RUN_OPTS="-e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e AWS_PROFILE=$AWS_PROFILE -e REGION=$REGION"
66-
68+
export RUN_OPTS="-e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e REGION=$REGION -e AWS_DEFAULT_REGION=$REGION"
69+
if [ -f ${HOME}/.aws/credentials ]; then
70+
export RUN_OPTS="${RUN_OPTS} -e AWS_PROFILE=$AWS_PROFILE"
71+
fi

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,5 @@ Container-Root/eks/deployment/kube-ops-view/kube-ops-view
55
Container-Root/eks/deployment/distributed-training/pytorch/habana/deepspeed-bert/efs-get-data.yaml
66
Container-Root/eks/deployment/distributed-training/pytorch/habana/deepspeed-bert/deepspeed-bert.yaml
77
Container-Root/eks/deployment/distributed-training/tensorflow/habana/mpijob-mnist.yaml
8+
Container-Root/eks/deployment/csi/fsx/fsx-storage-class.yaml
9+
Container-Root/eks/deployment/hpc/do-gromacs/to/kubernetes/app

Container-Root/eks/deployment/cluster-autoscaler/cluster-autoscaler-template.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,10 +144,10 @@ spec:
144144
resources:
145145
limits:
146146
cpu: 100m
147-
memory: 500Mi
147+
memory: 800Mi
148148
requests:
149149
cpu: 100m
150-
memory: 500Mi
150+
memory: 800Mi
151151
command:
152152
- ./cluster-autoscaler
153153
- --v=4

Container-Root/eks/deployment/csi/fsx/deploy.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
# you can also use static provisioning using the manifests in this folder or following instructions from here:
1111
# https://github.com/kubernetes-sigs/aws-fsx-csi-driver/blob/master/examples/kubernetes/static_provisioning/README.md
1212

13-
. fsx.conf
13+
source fsx.conf
1414

1515
# FSx Policy
1616
echo ""
Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,31 @@
11
#!/bin/bash
22

3+
source ../../../eks.conf
4+
35
# FSX Configuration
46
# All settings are required
57

68
## IAM Policy to provide access to FSx, if configured policy name does not exist, it will be created.
79
export FSX_POLICY_NAME=fsx-csi
810
export FSX_POLICY_DOC=file://fsx-policy.json
911

12+
## Derive
13+
asg1_name=$(eksctl get nodegroups --cluster $CLUSTER_NAME | grep -v NAME | head -n 1 | awk '{print $10}')
14+
launch_template_name=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name=$asg1_name | jq -r .AutoScalingGroups[].MixedInstancesPolicy.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateName)
15+
instance_profile_name=$(aws ec2 describe-launch-template-versions --versions '$Default' --launch-template-name=$launch_template_name | jq -r .LaunchTemplateVersions[].LaunchTemplateData.IamInstanceProfile.Name)
16+
instance1_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name=$asg1_name | jq -r .AutoScalingGroups[].Instances[0].InstanceId)
17+
subnet_id=$(aws ec2 describe-instances --instance-id=$instance1_id | jq -r .Reservations[0].Instances[0].SubnetId)
18+
1019
## Instance profiles of EKS node groups that will have access to FSx. Space separated string enclosed in ().
11-
export EKS_INSTANCE_PROFILE_NAMES=()
20+
export EKS_INSTANCE_PROFILE_NAMES=($instance_profile_name)
1221

1322
## Subnet to use for all EKS nodes and FSx volumes
14-
export FSX_SUBNET_ID=
23+
24+
export FSX_SUBNET_ID=$subnet_id
1525

1626
## Security group name for access to FSx volumes. Will be created if it does not exist.
1727
export FSX_SECURITY_GROUP_NAME=eks-fsx-sg
1828

1929
## Name of FSX storage class to create or update
2030
export FSX_STORAGE_CLASS_NAME=fsx-sc
31+
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
*.sh
2+
.env
3+
.fun
4+
build.sh
5+
exec.sh
6+
logs.sh
7+
pull.sh
8+
push.sh
9+
run.sh
10+
status.sh
11+
stop.sh
12+
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/bin/bash
2+
3+
# Source helper functions
4+
source .fun
5+
6+
# Proxy settings [optional] - set if your network requires a proxy to connect to the Internet
7+
export http_proxy=
8+
export https_proxy=
9+
export no_proxy=localhost
10+
11+
# AWS settings
12+
## AWS_PROFILE - name of AWS settings profile AWS_PROFILE=default(default)|aws-do-eks|...
13+
if [ -f ${HOME}/.aws/credentials ]; then
14+
export AWS_PROFILE=${AWS_PROFILE:-default}
15+
fi
16+
## AWS_REGION - will be set to AWS_DEFAULT_REGION if not set externally.
17+
export AWS_DEFAULT_REGION=us-east-1
18+
if [ "${AWS_REGION}" == "" ]; then
19+
export AWS_REGION=$AWS_DEFAULT_REGION
20+
fi
21+
22+
# TO - Target Orchestrator, TO=docker|kubernetes(default)
23+
## docker - choose when running locally or on ec2
24+
## kubernetes - choose when running on a local, remote Kubernetes cluster or EKS
25+
export TO=kubernetes
26+
27+
# Docker image settings
28+
## REGISTRY: [optional] - Docker registry path including trailing "/". Example: registry.company.com/demo/
29+
## If REGISTRY==default, then the default elastic container registry in the account for the current region will be used
30+
export REGISTRY=default
31+
## Set default registry if needed
32+
if [ "$REGISTRY" == "default" ]; then
33+
export REGION=${AWS_REGION}
34+
export ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
35+
if [ "$ACCOUNT" == "" ]; then
36+
export REGISTRY=""
37+
else
38+
export REGISTRY=${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com/
39+
fi
40+
fi
41+
## Add trailing forward slash if needed
42+
if [ -n "${REGISTRY}" ]; then
43+
if [ "${REGISTRY: -1}" != "/" ]; then
44+
export REGISTRY="${REGISTRY}/"
45+
fi
46+
fi
47+
48+
## IMAGE: <required> - Docker image name for this project. Example: myapp
49+
export IMAGE=do-gromacs
50+
## VERSION: [optional] - Version tag for this Docker image. Example: v20180302
51+
#export VERSION=v$(date +%Y%m%d)
52+
export VERSION=v20220910
53+
export TAG=$(if [ -z "${VERSION}" ]; then echo ""; else echo ":${VERSION}"; fi)
54+
55+
## IMAGE Build Options
56+
export SPACK_VERSION=v0.18.0
57+
export BUILD_IMAGE=spack/amazon-linux:${SPACK_VERSION}
58+
export SPACK_CACHE_VERSION=releases/v0.18
59+
export SPACK_TARGET=x86_64_v3
60+
export RUN_IMAGE="public.ecr.aws/amazonlinux/amazonlinux:latest"
61+
62+
## BUILD_OPTS: [optional] - arguments for the docker image build command
63+
export BUILD_OPTS="--build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} --build-arg no_proxy=${no_proxy} --build-arg SPACK_VERSION=${SPACK_VERSION} --build-arg BUILD_IMAGE=${BUILD_IMAGE} --build-arg SPACK_CACHE_VERSION=${SPACK_CACHE_VERSION} --build-arg SPACK_TARGET=${SPACK_TARGET} --build-arg RUN_IMAGE=${RUN_IMAGE}"
64+
65+
# Docker TO settings
66+
## CONTAINER_NAME: [optional] - Name of the Docker container including the --name switch. Example --name myapp
67+
export CONTAINER=${IMAGE}
68+
export CONTAINER_NAME="--name ${CONTAINER}"
69+
## Port map [optional] - Mapping of external to internal ports including the -p switch. Example -p 80:8080
70+
export PORT_MAP="-p 80:8080"
71+
## Volume map [optional] - Mapping of external to internal paths including the -v switch. Example $(pwd):/wd
72+
export VOL_MAP="-v $(pwd):/wd"
73+
## Network [optional] - Network name including the --net switch. Example --net mynet
74+
export NETWORK=
75+
## RUN_OPTS [optional] - additional options to specify with the run comman. Example -e POSTGRES_DB=dbname
76+
export RUN_OPTS="-e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION -e AWS_REGION=$AWS_REGION"
77+
if [ -f ${HOME}/.aws/credentials ]; then
78+
export RUN_OPTS="$RUN_OPTS -e AWS_PROFILE=$AWS_PROFILE"
79+
fi
80+
81+
# Kubernetes TO settings
82+
export K8S_NAMESPACE=gromacs
83+
export K8S_PVC_NAME=fsx-pvc
84+
export K8S_PVC_SIZE=100Gi
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
3+
# Helper functions
4+
## Detect current operating system
5+
function os
6+
{
7+
UNAME=$(uname -a)
8+
if [ $(echo $UNAME | awk '{print $1}') == "Darwin" ]; then
9+
export OPERATING_SYSTEM="MacOS"
10+
elif [ $(echo $UNAME | awk '{print $1}') == "Linux" ]; then
11+
export OPERATING_SYSTEM="Linux"
12+
elif [ ${UNAME:0:5} == "MINGW" ]; then
13+
export OPERATING_SYSTEM="Windows"
14+
export MSYS_NO_PATHCONV=1 # turn off path conversion
15+
else
16+
export OPERATING_SYSTEM="Other"
17+
fi
18+
}
19+
## End os function
20+
os
21+
22+
## Determine current host IP address
23+
function hostip
24+
{
25+
case "${OPERATING_SYSTEM}" in
26+
"Linux")
27+
export HOST_IP=$(hostname -I | tr " " "\n" | head -1) # Linux
28+
;;
29+
"MacOS")
30+
export HOST_IP=$(ifconfig | grep -v 127.0.0.1 | grep -v inet6 | grep inet | head -n 1 | awk '{print $2}') # Mac OS
31+
;;
32+
"Windows")
33+
export HOST_IP=$( ((ipconfig | grep IPv4 | grep 10.187 | tail -1) && (ipconfig | grep IPv4 | grep 3. | head -1)) | tail -1 | awk '{print $14}' ) # Git bash
34+
;;
35+
*)
36+
export HOST_IP=$(hostname)
37+
;;
38+
esac
39+
}
40+
## End hostip function
41+
hostip
42+
43+
## generate_kubernetes_manifests function
44+
function generate_kubernetes_manifests
45+
{
46+
echo "Generating Kubernetes manifests ..."
47+
if [ -d "to/kubernetes/app" ]; then
48+
rm -f to/kubernetes/app/*.yaml
49+
else
50+
mkdir -p "to/kubernetes/app"
51+
fi
52+
CMD="BASE_PATH=$(pwd); cd to/kubernetes/template; for f in *.yaml; do cat \$f | envsubst > \${BASE_PATH}/to/kubernetes/app/\$f; done; cd \${BASE_PATH}"
53+
if [ "${VERBOSE}" == "true" ]; then
54+
echo "${CMD}"
55+
fi
56+
eval "${CMD}"
57+
}
58+
## End generate_kubernetes_manifests function
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Container-Root/app/manifests/*.yaml
2+
wd/*

0 commit comments

Comments
 (0)