Merge pull request #68 from dabercro/v011

V011

Author: Daniel Abercrombie

.gitignore

@@ -2,6 +2,7 @@
 *~
 build/*
 test/*/*
+!test/sl?/Dockerfile
 test/*.cache.json
 dist/*
 cmstoolbox.egg-info/*

.travis.yml

@@ -3,8 +3,6 @@ cache: pip
 matrix:
   include:
     - python: 2.6
-      env:
-        - nodoc=true
     - python: 2.7
       env:
         - nodoc=true

Jenkinsfile

@@ -0,0 +1,32 @@
+def run(os) {
+  return {
+
+    docker.build("opsspace-${os}:${env.BUILD_ID}", "test/${os}").inside('-u root:root') {
+
+      stage("${os}: Copy Source") {
+        sh """
+           test ! -d ${os} || rm -rf ${os}
+           mkdir ${os}
+           cp --parents `git ls-files` ${os}
+           """
+      }
+
+      stage("${os}: Installation") {
+        sh "cd ${os}; python setup.py install"
+      }
+
+      stage("${os}: Unit Tests") {
+        sh "cd ${os}; opsspace-test"
+      }
+    }
+  }
+}
+
+def osList = ['sl6', 'sl7']
+
+node {
+  checkout scm
+  parallel osList.collectEntries{
+    ["${it}": run(it)]
+  }
+}

bin/copy-coverage-html

@@ -0,0 +1,53 @@
+#! /usr/bin/env perl
+
+use strict;
+use warnings;
+
+use v5.10;
+
+use Cwd;
+use File::Path qw(make_path);
+use File::Basename;
+use File::Spec::Functions;
+use File::Find;
+
+# If not in a location with a test directory, exit
+if (! -f 'test/.coverage') {
+    exit 1;
+}
+
+my $outdir = pop @ARGV or die 'Need a destination';
+make_path($outdir);
+
+my @inits = sort { length $a <=> length $b } `find . -type f -name '__init__.py'`;
+
+# Get the name of the package
+my ($pack) = shift @ARGV || ($inits[0] =~ m|\./(.*)/__init__\.py|);
+
+# Create the htmlcov directory
+`cd test && coverage html`;
+
+# And get all of the files that we'll want
+my @files = glob 'test/htmlcov/*';
+
+for (@files) {
+
+    open(my $in_fh, '<', $_);
+
+    s/.*(${pack}.*)/$1/;
+
+    my $base = fileparse $_;
+
+    open(my $out_fh, '>', catfile($outdir, $base));
+
+    for (<$in_fh>) {
+        if (/$pack/) {
+            # Fix href
+            s/(href=").*(${pack}.*?\.html)/$1$2/;
+            # Fix text
+            s|/.*/(${pack}.*\.py)|$1|;
+        }
+        print $out_fh $_;
+    }
+
+}

bin/opsspace-test

@@ -52,7 +52,7 @@ _do_test () {
 _doc_test () {
 
     # Check if we should test build Sphinx documentation
-    if [ -f docs/conf.py ] && [ "$nodoc" != "true" ]
+    if [ -f docs/conf.py ] && [ "$nodoc" != "true" ] && [ "$pyvers" != "26" ]
     then
 
         test -f docs/requirements.txt && pip install -r docs/requirements.txt
@@ -82,14 +82,14 @@ _unit_test () {
 
 
     # Run the test for all files that start with "test_". They should be executable.
-    for packagetest in test_*
+    for packagetest in test_*[^~]
     do
 
         # Run the coverage on top of the unit tests
         if [ ! -z $docoverage ] && [ "${packagetest##*.}" == "py" ]
         then
 
-            _do_test "coverage run --source $package -a $packagetest"
+            _do_test "coverage run --source ${cover:-$package} -a $packagetest"
 
         else 
 

cmstoolbox/__init__.py

@@ -2,4 +2,4 @@
 
 __all__ = []
 
-__version__ = '0.9.8'
+__version__ = '0.11.2'

cmstoolbox/samstatus.py

@@ -10,6 +10,29 @@
 
 from .webtools import get_json
 
+
+def _is_problem(success, data):
+    """
+    :param float success: the rate of success desired
+    :param list data: Data from the WLCG SAM server
+    :returns: True if the success rate of SAM test is below success
+    :rtype: bool
+    """
+    # if success is less then 85% call it a problem
+    items = 0
+    bad = 0
+    for item in data:
+        stat = item[1]
+        if stat == 'WHITE':
+            continue
+
+        items += 1
+        if stat in ['CRITICAL', 'WARNING']:
+            bad += 1
+
+    return float(bad)/float(items) > (1.0 - success)
+
+
 def is_sam_good(site, time_span=24, success=0.85):
     """
     Checks the SAM tests for success rate, and returns True if SAM tests were passing
@@ -33,12 +56,18 @@ def is_sam_good(site, time_span=24, success=0.85):
     srm_host_name = ''
     ce_host_name = ''
     ce_flavour = ''
-    for item in gen_json['data']['results'][0]['flavours']:
-        if item['servicename'] == 'SRM':
-            srm_host_name = item['hosts'][0]['hostname']
-        else:
-            ce_host_name = item['hosts'][0]['hostname']
-            ce_flavour = item['servicename']
+
+    try:
+        for item in gen_json['data']['results'][0]['flavours']:
+            if item['servicename'] == 'SRM':
+                srm_host_name = item['hosts'][0]['hostname']
+            else:
+                ce_host_name = item['hosts'][0]['hostname']
+                ce_flavour = item['servicename']
+
+    except (KeyError, IndexError):
+        # If there's something wrong with the JSON file, go through
+        return True
 
     get_data = lambda flav, host: get_json('wlcg-sam-cms.cern.ch',
                                            '/dashboard/request.py/getTestResults',
@@ -47,33 +76,16 @@ def is_sam_good(site, time_span=24, success=0.85):
                                             'hostname': host,
                                             'time_range': time_span_str})
 
-    def is_problem(success, data):
-        """
-        :param float success: the rate of success desired
-        :param list data: Data from the WLCG SAM server
-        :returns: True if the success rate of SAM test is below success
-        :rtype: bool
-        """
-		# if success is less then 85% call it a problem
-        items = 0
-        bad = 0
-        for item in data:
-            stat = item[1]
-            if stat == 'WHITE':
-                continue
-
-            items += 1
-            if stat in ['CRITICAL', 'WARNING']:
-                bad += 1
-
-        return float(bad)/float(items) > (1.0 - success)
-
-    for probe in get_data('SRM', srm_host_name)['data']:
-        if is_problem(success, probe[1]):
-            return False
-
-    for probe in get_data(ce_flavour, ce_host_name)['data']:
-        if 'WN-xrootd-access' in probe[0] and is_problem(success, probe[1]):
-            return False
+    try:
+        for probe in get_data('SRM', srm_host_name)['data']:
+            if _is_problem(success, probe[1]):
+                return False
+
+        for probe in get_data(ce_flavour, ce_host_name)['data']:
+            if 'WN-xrootd-access' in probe[0] and _is_problem(success, probe[1]):
+                return False
+
+    except KeyError:
+        pass
 
     return True

cmstoolbox/webtools.py

@@ -60,8 +60,11 @@ def get_json(host, request, params='', body='', headers=None,
 
     if use_https:
         use_port = port or 443
-        cert_file = kwargs.get('cert_file', os.getenv('X509_USER_PROXY')) \
-            if use_cert else None
+        cert_file = kwargs.get(
+            'cert_file',
+            os.environ.get('X509_USER_PROXY',
+                           '/tmp/x509up_u%i' % os.geteuid())
+            ) if use_cert else None
 
         # Python 2.7.something verifies HTTPS connections,
         # but earlier version of Python do not

test/requirements26.txt

@@ -1,3 +1,4 @@
 astroid<1.3
 pylint<1.4
 testfixtures<6
+setuptools==36.8.0

test/sl6/Dockerfile

@@ -0,0 +1,10 @@
+FROM sl:6
+
+RUN yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm
+
+RUN yum -y install \
+    python-pip \
+    python-devel \
+    git perl
+
+RUN pip install -U 'pip==9.0.1'