Merge pull request #11 from aws-samples/deepspeed

Deepspeed and distributed training examples

Author: iankouls-aws

.env

@@ -13,19 +13,40 @@ export http_proxy=
 export https_proxy=
 export no_proxy=localhost
 
+# AWS settings
+## AWS_PROFILE - name of AWS settings profile AWS_PROFILE=default(default)|aws-do-eks|...
+export AWS_PROFILE=default
+## AWS_REGION - will be set to AWS_DEFAULT_REGION if not set externally.
+export AWS_DEFAULT_REGION=us-east-1
+if [ "${AWS_REGION}" == "" ]; then
+	export AWS_REGION=$AWS_DEFAULT_REGION
+fi
 # Docker image settings
 ## REGISTRY: [optional] - Docker registry path including trailing "/". Example: registry.company.com/demo/
-export REGISTRY=
+## If REGISTRY==default, then the default elastic container registry in the account for the current region will be used 
+export REGISTRY=default
+## Set default registry if needed
+if [ "$REGISTRY" == "default" ]; then
+    export REGION=${AWS_REGION}
+    export ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
+    if [ "$ACCOUNT" == "" ]; then
+	export REGISTRY=""
+    else
+    	export REGISTRY=${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com/
+    fi
+fi
+## Add trailing forward slash if needed
 if [ -n "${REGISTRY}" ]; then
   if [ "${REGISTRY: -1}" != "/" ]; then
     export REGISTRY="${REGISTRY}/"
   fi
 fi
+
 ## IMAGE: <required> - Docker image name for this project. Example: myapp
 export IMAGE=aws-do-eks
 ## VERSION: [optional] - Version tag for this Docker image. Example: v20180302
 #export VERSION=v$(date +%Y%m%d)
-export VERSION=v3-20220707
+export VERSION=v4-20220801
 export TAG=$(if [ -z "${VERSION}" ]; then echo ""; else echo ":${VERSION}"; fi) 
 ## BUILD_OPTS: [optional] - arguments for the docker image build command
 export BUILD_OPTS="--progress plain --build-arg http_proxy=${http_proxy} --build-arg https_proxy=${https_proxy} --build-arg no_proxy=${no_proxy}"
@@ -37,9 +58,9 @@ export CONTAINER_NAME="--name ${CONTAINER}"
 ## Port map [optional] - Mapping of external to internal ports including the -p switch. Example -p 80:8080 
 #export PORT_MAP="-p 80:8080"
 ## Volume map [optional] - Mapping of external to internal paths including the -v switch. Example $(pwd):/wd
-export VOL_MAP="-v ${HOME}/.aws:/root/.aws -v ${HOME}/.kube:/root/.kube -v $(pwd)/wd/conf/eks.conf:/eks/eks.conf -v $(pwd)/wd/conf/eks.yaml:/eks/eks.yaml -v $(pwd):/aws-do-eks"
+export VOL_MAP="-v ${HOME}/.aws:/root/.aws -v ${HOME}/.kube:/root/.kube -v $(pwd)/wd/conf/eks.conf:/eks/eks.conf -v $(pwd)/wd/conf/eks.yaml:/eks/eks.yaml -v $(pwd):/aws-do-eks -v /var/run/docker.sock:/var/run/docker.sock"
 ## Network [optional] - Network name including the --net switch. Example --net mynet
 #export NETWORK=
 ## RUN_OPTS [optional] - additional options to specify with the run comman. Example -e POSTGRES_DB=dbname
-export RUN_OPTS="-e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy"
+export RUN_OPTS="-e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e AWS_PROFILE=$AWS_PROFILE -e REGION=$REGION"
 

.gitignore

@@ -2,3 +2,6 @@ Container-Root/eks/deployment/cluster-autoscaler/cluster-autoscaler.yaml
 Container-Root/eks/deployment/aws-load-balancer-controller/aws-load-balancer-controller.yaml
 Container-Root/eks/deployment/kube-ops-view/kube-ops-view
 .DS_Store
+Container-Root/eks/deployment/distributed-training/pytorch/habana/deepspeed-bert/efs-get-data.yaml
+Container-Root/eks/deployment/distributed-training/pytorch/habana/deepspeed-bert/deepspeed-bert.yaml
+Container-Root/eks/deployment/distributed-training/tensorflow/habana/mpijob-mnist.yaml

Container-Root/eks/deployment/aws-load-balancer-controller/ingress.yaml.template

@@ -0,0 +1,19 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  namespace: ${NAMESPACE}
+  name: ${INGRESS_NAME}
+  annotations:
+    alb.ingress.kubernetes.io/scheme: internet-facing
+    alb.ingress.kubernetes.io/target-type: ip
+spec:
+  ingressClassName: alb
+  rules:
+        - path: /
+          pathType: Prefix
+          backend:
+            service:
+              name: ${SERVICE_NAME}
+              port:
+                number: ${SERVICE_PORT}

Container-Root/eks/deployment/cluster-autoscaler/set-iam-role.sh

@@ -6,28 +6,27 @@ source ../../eks.conf
 IAM_POLICY=$(aws iam list-policies --no-paginate | grep PolicyName | grep AmazonEKSClusterAutoscalerPolicy)
 
 if [ "$IAM_POLICY" == "" ]; then
-	echo "Creating Cluster Autoscaler IAM Policy ..."
-	json_out=$(aws iam create-policy --policy-name AmazonEKSClusterAutoscalerPolicy --policy-document file://cluster-autoscaler-policy.json)
-	arn=$(echo $json_out | jq '.Policy.Arn')
-	POLICY_ARN=$(echo $arn | sed -e 's/\"//g')
-	echo "POLICY_ARN=$POLICY_ARN"
+        echo "Creating Cluster Autoscaler IAM Policy ..."
+        json_out=$(aws iam create-policy --policy-name AmazonEKSClusterAutoscalerPolicy --policy-document file://cluster-autoscaler-policy.json)
+        arn=$(echo $json_out | jq '.Policy.Arn')
+        POLICY_ARN=$(echo $arn | sed -e 's/\"//g')
+        echo "POLICY_ARN=$POLICY_ARN"
 else
-	echo "IAM Policy $IAM_POLICY alreqady exists"
+        echo "IAM Policy $IAM_POLICY already exists"
+        POLICY_ARN=$(aws iam list-policies --no-paginate | grep AmazonEKSClusterAutoscalerPolicy | grep Arn | cut -d "\"" -f 4)
 fi
 
 # Create service account cluster-autoscaler with attached policy if it does not exist
 output=$(eksctl get iamserviceaccount --cluster $CLUSTER_NAME --output json)
 clean_out=$(echo ${output##*[} )
 json_out="[ $clean_out"
 IAM_SA_NAMES=$(echo $json_out | jq -r '.[].metadata.name')
-IAM_SA_NAME=$(echo $IAM_SA_NAMES | grep cluster-autoscaler) 
+IAM_SA_NAME=$(echo $IAM_SA_NAMES | grep cluster-autoscaler)
 
 if [ "$IAM_SA_NAME" == "" ]; then
-	echo "Creating IAM Service Account cluster-autoscaler ..."
-	eksctl create iamserviceaccount --cluster=$CLUSTER_NAME --namespace=kube-system --name=cluster-autoscaler \
-					--attach-policy-arn=$POLICY_ARN --override-existing-serviceaccounts --approve 
+        echo "Creating IAM Service Account cluster-autoscaler ..."
+        eksctl create iamserviceaccount --cluster=$CLUSTER_NAME --namespace=kube-system --name=cluster-autoscaler \
+                                        --attach-policy-arn=$POLICY_ARN --override-existing-serviceaccounts --approve
 else
-	echo "IAM Service Account $IAM_SA_NAME already exists"
+        echo "IAM Service Account $IAM_SA_NAME already exists"
 fi
-
-