Merge pull request #83 from plessbd/gw2020-additions

Add historical data for gateways2020

Author: aebruno

database/create-xdmod-db.sh

@@ -11,7 +11,7 @@ EOF
 
 echo "Creating xdmod databases.."
 
-for db in mod_hpcdb mod_logger mod_shredder moddb modw modw_aggregates modw_filters modw_supremm modw_etl modw_jobefficiency
+for db in mod_hpcdb mod_logger mod_shredder moddb modw modw_aggregates modw_filters modw_supremm modw_etl modw_jobefficiency modw_cloud
 do
     echo "Creating $db database.."
     echo "create database if not exists $db" | mysql -uroot

xdmod/Dockerfile

@@ -6,6 +6,7 @@ RUN /build/install.sh && rm -rf /build
 COPY conf/httpd.conf /etc/httpd/conf.d/xdmod.conf
 COPY conf/simplesamlphp /etc/xdmod/simplesamlphp
 COPY hierarchy.csv /srv/xdmod/hierarchy.csv
+COPY historical/ /srv/xdmod/historical
 COPY scripts/ /srv/xdmod/scripts
 COPY bin/sendmail /usr/sbin/sendmail
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh

xdmod/historical/add-historical.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+if [ `whoami` != 'root' ]; then
+    echo "ERROR: This script should only be run as root" >&2
+    exit 1
+fi
+historicaldir='/srv/xdmod/historical'
+if [ -f $historicaldir/.hasrun ]; then
+    echo "ERROR: This has already been run once. Trying to do it again will cause errors" >&2
+    exit 2
+fi
+touch $historicaldir/.hasrun
+tar -zxf $historicaldir/historical-data.tgz -C $historicaldir/
+jq --argfile f1 /etc/xdmod/resources.json --argfile f2 $historicaldir/resources.json -n '$f1 + $f2' | tee /etc/xdmod/resources.json 
+jq --argfile f1 /etc/xdmod/resource_specs.json --argfile f2 $historicaldir/resource_specs.json -n '$f1 + $f2' | tee /etc/xdmod/resource_specs.json 
+types='Cloud Jobs Storage'
+for resource in $historicaldir/Jobs/*.log; do
+    sudo -u xdmod xdmod-shredder -r `basename $resource .log` -f slurm -i $resource;
+done
+last_modified_start_date=$(date +'%F %T')
+sudo -u xdmod xdmod-shredder -r cumulonimbus -d $historicaldir/Cloud/ -f openstack
+sudo -u xdmod xdmod-ingestor --last-modified-start-date "$last_modified_start_date"
+for storage_dir in $historicaldir/Storage/*; do
+   sudo -u xdmod xdmod-shredder -f storage -r $(basename $storage_dir) -d $storage_dir
+done
+last_modified_start_date=$(date +'%F %T')
+sudo -u xdmod xdmod-ingestor --datatype storage
+cat $historicaldir/names.slurm.csv $historicaldir/names.csv > /tmp/historical.names.csv
+sudo -u xdmod xdmod-import-csv -t names -i /tmp/historical.names.csv
+sudo -u xdmod xdmod-ingestor

xdmod/historical/historical-data.tgz

@@ -0,0 +1,5 @@
+cgray,Carl,Gray
+sfoster,Stephanie,Foster
+csimmons,Charles,Simmons
+astewart,Andrea,Stewart
+hpcadmin,,HPC Administrators

xdmod/historical/names.slurm.csv

@@ -0,0 +1,32 @@
+[
+    {
+        "resource": "compsci",
+        "processors": 9000,
+        "nodes": 900,
+        "ppn": 10
+    },
+    {
+        "resource": "maths",
+        "processors": 5500,
+        "nodes": 550,
+        "ppn": 10
+    },
+    {
+        "resource": "humanities",
+        "processors": 2500,
+        "nodes": 250,
+        "ppn": 10
+    },
+    {
+        "resource": "parallelfs",
+        "processors": 1,
+        "nodes": 1,
+        "ppn": 1
+    },
+    {
+        "resource": "cumulonimbus",
+        "processors": 10000,
+        "nodes": 1000,
+        "ppn": 10
+    }
+]

xdmod/historical/resource_specs.json

@@ -0,0 +1,27 @@
+[
+    {
+        "resource": "compsci",
+        "resource_type": "HPC",
+        "name": "Computer Science"
+    },
+    {
+        "resource": "maths",
+        "resource_type": "HPC",
+        "name": "Mathematics"
+    },
+    {
+        "resource": "humanities",
+        "resource_type": "HPC",
+        "name": "Humanities"
+    },
+    {
+        "resource": "parallelfs",
+        "resource_type": "Disk",
+        "name": "GPFS Parallel FS"
+    },
+    {
+        "resource": "cumulonimbus",
+        "resource_type": "Cloud",
+        "name": "Cumulonimbus Cloud"
+    }
+]