Add leases permission to mpi-operator role, update EFA tests

Author: iankouls-aws

Container-Root/eks/deployment/efa-device-plugin/test-efa.yaml

@@ -13,13 +13,14 @@ spec:
           imagePullPolicy: IfNotPresent
           restartPolicy: OnFailure
           containers:
+          #- image: <account>.dkr.ecr.<region>.amazonaws.com/cuda-efa-nccl-tests:ubuntu18.04
           - image: public.ecr.aws/w6p6i9i7/aws-efa-nccl-rdma:base-cudnn8-cuda11-ubuntu18.04
             name: efa-info-launcher
             env:
              - name: LD_LIBRARY_PATH
                value: /opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/nvidia/lib:$LD_LIBRARY_PATH
              - name: PATH
-               value: $PATH:/opt/amazon/efa/bin
+               value: $PATH:/opt/amazon/efa/bin:/usr/bin
              - name: XLA_FLAGS
                value: "--xla_gpu_cuda_data_dir=/usr/local/cuda"
              - name: TF_XLA_FLAGS
@@ -31,7 +32,7 @@ spec:
             - --allow-run-as-root
             - --tag-output
             - -np
-            - "16"
+            - "2"
             - -bind-to
             - none
             - -map-by
@@ -49,14 +50,16 @@ spec:
             - -x
             - NCCL_ALGO=RING
             - -x
-            - FI_EFA_USE_DEVICE_RDMA=1
+            - FI_EFA_USE_DEVICE_RDMA=0
             - -x
-            - RDMAV_FORK_SAFE=1
+            - RDMAV_FORK_SAFE=0
             - -x
             - NCCL_DEBUG
             - --mca
             - pml
             - ^cm
+            - --mca
+            - pml_rsh_agent=ssh
             - --oversubscribe
             - /opt/amazon/efa/bin/fi_info
             - -p
@@ -69,16 +72,17 @@ spec:
         spec:
           imagePullPolicy: IfNotPresent
           containers:
+          #- image: <account>.dkr.ecr.<region>.amazonaws.com/cuda-efa-nccl-tests:ubuntu18.04
           - image: public.ecr.aws/w6p6i9i7/aws-efa-nccl-rdma:base-cudnn8-cuda11-ubuntu18.04
             name: efa-info-worker
             resources:
               limits:
-                nvidia.com/gpu: 8
-                hugepages-2Mi: 5120Mi
-                vpc.amazonaws.com/efa: 4
-                memory: 8000Mi
+                nvidia.com/gpu: 1
+                #hugepages-2Mi: 5120Mi
+                vpc.amazonaws.com/efa: 1
+                #memory: 8000Mi
               requests:
-                nvidia.com/gpu: 8
-                hugepages-2Mi: 5120Mi
-                vpc.amazonaws.com/efa: 4
-                memory: 8000Mi
+                nvidia.com/gpu: 1
+                #hugepages-2Mi: 5120Mi
+                vpc.amazonaws.com/efa: 1
+                #memory: 8000Mi

Container-Root/eks/deployment/efa-device-plugin/test-nccl-efa-rdma.yaml

@@ -0,0 +1,92 @@
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+  name: test-nccl-efa
+spec:
+  runPolicy:
+    cleanPodPolicy: Running
+    backoffLimit: 20
+  slotsPerWorker: 1
+  mpiReplicaSpecs:
+    Launcher:
+      replicas: 1
+      template:
+         spec:
+          imagePullPolicy: IfNotPresent
+          restartPolicy: OnFailure
+          containers:
+          #- image: <account>.dkr.ecr.us-west-2.amazonaws.com/cuda-efa-nccl-tests:ubuntu18.04
+          - image: public.ecr.aws/w6p6i9i7/aws-efa-nccl-rdma:base-cudnn8-cuda11-ubuntu18.04
+            name: test-nccl-efa-launcher
+            command:
+            - /opt/amazon/openmpi/bin/mpirun
+            - --allow-run-as-root
+            - --tag-output
+            - -np
+            - "2"
+            - -bind-to
+            - none
+            - -map-by
+            - slot
+            - -x
+            - LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
+            - -x
+            - NCCL_DEBUG=INFO
+            - -x
+            - NCCL_ALGO=Ring
+            - -x
+            - FI_PROVIDER=efa
+            - -x
+            - FI_EFA_USE_DEVICE_RDMA=1
+            - -x
+            - RDMAV_FORK_SAFE=1
+            - -x
+            - NCCL_SHM_DISABLE=0
+            - --mca
+            - pml
+            - ^cm
+            - --oversubscribe
+            - /opt/nccl-tests/build/all_reduce_perf
+            - -b
+            - "1"
+            - -e
+            - 1G
+            - -f
+            - "2"
+            - -t
+            - "1"
+            - -g
+            - "1"
+            - -c
+            - "1"
+            - -n
+            - "100"
+    Worker:
+      replicas: 2
+      template:
+        spec:
+          #nodeSelector:
+            #node.kubernetes.io/instance-type: "g4dn.metal"
+          imagePullPolicy: IfNotPresent
+          containers:
+          #- image: <account>.dkr.ecr.us-west-2.amazonaws.com/cuda-efa-nccl-tests:ubuntu18.04
+          - image: public.ecr.aws/w6p6i9i7/aws-efa-nccl-rdma:base-cudnn8-cuda11-ubuntu18.04
+            name: test-nccl-efa-worker
+            volumeMounts:
+            - name: shmem
+              mountPath: /dev/shm
+            resources:
+              limits:
+                nvidia.com/gpu: 1
+                #hugepages-2Mi: 5120Mi
+                vpc.amazonaws.com/efa: 1
+                #memory: 8000Mi
+              requests:
+                nvidia.com/gpu: 1
+                #hugepages-2Mi: 5120Mi
+                vpc.amazonaws.com/efa: 1
+                #memory: 8000Mi
+          volumes:
+          - name: shmem
+            hostPath:
+              path: /dev/shm

Container-Root/eks/deployment/efa-device-plugin/test-nccl-efa.yaml

@@ -18,13 +18,6 @@ spec:
           #- image: <account>.dkr.ecr.us-west-2.amazonaws.com/cuda-efa-nccl-tests:ubuntu18.04
           - image: public.ecr.aws/w6p6i9i7/aws-efa-nccl-rdma:base-cudnn8-cuda11-ubuntu18.04
             name: test-nccl-efa-launcher
-            #env:
-            #- name: LD_LIBRARY_PATH
-            #  value: /usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
-              #value: /opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/nvidia/lib:$LD_LIBRARY_PATH
-            #- name: PATH
-            #  value: /opt/amazon/openmpi/bin/:/opt/amazon/efa/bin:/usr/bin:/usr/local/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-            #  value: $PATH:/opt/amazon/efa/bin
             command:
             - /opt/amazon/openmpi/bin/mpirun
             - --allow-run-as-root
@@ -36,19 +29,17 @@ spec:
             - -map-by
             - slot
             - -x
-            - PATH
-            - -x
-            - LD_LIBRARY_PATH
+            - LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
             - -x
             - NCCL_DEBUG=INFO
             - -x
             - NCCL_ALGO=Ring
             - -x
             - FI_PROVIDER=efa
             - -x
-            - FI_EFA_USE_DEVICE_RDMA=1
+            - FI_EFA_USE_DEVICE_RDMA=0
             - -x
-            - RDMAV_FORK_SAFE=1
+            - RDMAV_FORK_SAFE=0
             - -x
             - NCCL_SHM_DISABLE=0
             - --mca
@@ -75,9 +66,7 @@ spec:
       template:
         spec:
           #nodeSelector:
-            #beta.kubernetes.io/instance-type: "p4d.24xlarge"
-            #beta.kubernetes.io/instance-type: "p3dn.24xlarge"
-            #beta.kubernetes.io/instance-type: "g4dn.metal"
+            #node.kubernetes.io/instance-type: "g4dn.metal"
           imagePullPolicy: IfNotPresent
           containers:
           #- image: <account>.dkr.ecr.us-west-2.amazonaws.com/cuda-efa-nccl-tests:ubuntu18.04
@@ -91,12 +80,12 @@ spec:
                 nvidia.com/gpu: 1
                 #hugepages-2Mi: 5120Mi
                 vpc.amazonaws.com/efa: 1
-                memory: 8000Mi
+                #memory: 8000Mi
               requests:
                 nvidia.com/gpu: 1
                 #hugepages-2Mi: 5120Mi
                 vpc.amazonaws.com/efa: 1
-                memory: 8000Mi
+                #memory: 8000Mi
           volumes:
           - name: shmem
             hostPath:

Container-Root/eks/deployment/kubeflow/mpi-operator/clusterrole-mpi-operator.yaml

@@ -0,0 +1,105 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+  labels:
+    app: mpi-operator
+    app.kubernetes.io/component: mpijob
+    app.kubernetes.io/name: mpi-operator
+    kustomize.component: mpi-operator
+  name: mpi-operator
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  - services
+  verbs:
+  - create
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - delete
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods/exec
+  verbs:
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  verbs:
+  - create
+  - list
+  - update
+  - watch
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+  - list
+  - update
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - kubeflow.org
+  resources:
+  - mpijobs
+  - mpijobs/finalizers
+  - mpijobs/status
+  verbs:
+  - '*'
+- apiGroups:
+  - scheduling.incubator.k8s.io
+  - scheduling.sigs.dev
+  resources:
+  - queues
+  - podgroups
+  verbs:
+  - '*'
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - delete
+  - update