cloudtools/hail-ci-build.sh at master · Nealelab/cloudtools · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
#!/bin/bash

set -ex

gcloud auth activate-service-account \
     hail-ci-0-1@broad-ctsa.iam.gserviceaccount.com \
     --key-file=/secrets/hail-ci-0-1.key

gcloud config set project broad-ctsa

shutdown_cluster() {
    trap "" INT TERM
    set +e
    for CLUSTER_NAME in $CLUSTERS
    do
        time gcloud dataproc clusters delete ${CLUSTER_NAME} --async
    done
}
trap shutdown_cluster EXIT

trap "exit 42" INT TERM

for PY_VERSION in 2 3
do
    PIP=pip${PY_VERSION}

    $PIP install ./

    # remove any symlinks to a default python
    ls -al $(which python)
    rm -rf $(which python)

    CLUSTER_NAME_0_2=cloudtools-ci-$(LC_CTYPE=C LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 8)
    CLUSTER_NAME_0_1=cloudtools-ci-$(LC_CTYPE=C LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 8)

    CLUSTERS="${CLUSTERS} ${CLUSTER_NAME_0_2} ${CLUSTER_NAME_0_1}"

    # check binary exists
    time cluster start --help

    SHA=$(cluster latest --sha 0.1 2.0.2)
    JAR=$(cluster latest --jar 0.1 2.0.2)
    ZIP=$(cluster latest --zip 0.1 2.0.2)
    gsutil ls $JAR
    gsutil ls $ZIP
    SHA=$(cluster latest --sha 0.2 2.2.0)
    JAR=$(cluster latest --jar 0.2 2.2.0)
    ZIP=$(cluster latest --zip 0.2 2.2.0)
    gsutil ls $JAR
    gsutil ls $ZIP

    # check 0.2 cluster starts and runs hail
    time cluster start ${CLUSTER_NAME_0_2} \
         --version 0.2 \
         --spark 2.2.0 \
         --max-idle 40m \
         --bucket=hail-ci-0-1-dataproc-staging-bucket
    time cluster submit ${CLUSTER_NAME_0_2} \
         cluster-sanity-check-0.2.py
    time cluster stop --async ${CLUSTER_NAME_0_2}

    # check 0.1 cluster starts and runs hail
    time cluster start ${CLUSTER_NAME_0_1} \
         --version 0.1 \
         --spark 2.0.2 \
         --max-idle 40m \
         --bucket=hail-ci-0-1-dataproc-staging-bucket
    time cluster submit ${CLUSTER_NAME_0_1} \
         cluster-sanity-check-0.1.py
    time cluster stop --async ${CLUSTER_NAME_0_1}

    yes | $PIP uninstall cloudtools
done