Merge branch 'master' into master

Author: Daniel Abercrombie

setup.py

@@ -32,6 +32,9 @@
         'validators',
         'tabulate',
         'pymongo<3.5.0',
-        'cx_Oracle'
+        'cx_Oracle',
+        'pandas',
+        'keras',
+        'tensorflow'
         ]
     )

workflowwebtools/__init__.py

@@ -4,6 +4,6 @@
 :author: Daniel Abercrombie <dabercro@mit.edu>
 """
 
-__version__ = '0.8.2'
+__version__ = '0.9.4'
 
 __all__ = []

workflowwebtools/actionshistorylink.py

@@ -34,7 +34,8 @@ def dump_json(file_name=None):
 
     for workflow in actions:
         for subtask in history.get_step_list(workflow):
-            if actions[workflow]['Action'] in ['acdc', 'recovery']:
+            action = actions[workflow]['Action']
+            if action in ['acdc', 'recovery']:
                 parameters = actions[workflow]['Parameters'].get(
                     '/'.join(subtask.split('/')[2:]), {})
             else:
@@ -52,7 +53,10 @@ def dump_json(file_name=None):
                 'parameters':
                     parameters
                 }
-            output[subtask]['parameters']['action'] = actions[workflow]['Action']
+
+            output[subtask]['parameters']['action'] = \
+                action if action != 'special' else \
+                actions[workflow]['Parameters']['action']
 
     if file_name:
         with open(file_name, 'w') as output_file:

workflowwebtools/classifyerrors.py

@@ -8,10 +8,11 @@
 
 import re
 
+from collections import defaultdict
+
 from .procedures import PROCEDURES
-from .globalerrors import check_session
 
-def classifyerror(errorcode, workflow, session=None):
+def classifyerror(errorcode, workflow):
     """
     Return the most relevant characteristics of an error code for this session.
     This will include things like:
@@ -24,8 +25,7 @@ def classifyerror(errorcode, workflow, session=None):
        More error types should be added to this function as needed
 
     :param int errorcode: The error code that we want to classify
-    :param str workflow: the workflow that we want to get the errors from
-    :param cherrypy.Session session: Is the user's cherrypy session
+    :param workflowinfo.WorkflowInfo workflow: the workflow that we want to get the errors from
     :returns: A tuple of strings describing the key characteristics of the errorcode.
               These strings are good for printing directly in web browsers.
               The first string is the types of errors reported with this error code.
@@ -36,7 +36,7 @@ def classifyerror(errorcode, workflow, session=None):
 
     procedure = PROCEDURES.get(errorcode, {})
 
-    logs = check_session(session).get_workflow(workflow).get_explanation(str(errorcode))
+    logs = workflow.get_explanation(str(errorcode))
 
     error_re = re.compile(r'[\w\s]+ \(Exit code: (\d+)\)')
     error_types = {}
@@ -79,26 +79,29 @@ def classifyerror(errorcode, workflow, session=None):
             additional_actions_string.replace(' |br| |br| ', '<br>'))
 
 
-def get_max_errorcode(workflow, session=None):
+def get_max_errorcode(workflow):
     """
     Get the errorcode with the most errors for a session
 
-    :param str workflow: Is the primary name of the workflow
-    :param cherrypy.Session session: Is the user's cherrypy session
+    :param workflowinfo.WorkflowInfo workflow: the workflow that we want to get the errors from
     :returns: The error code that appears most often for this workflow
     :rtype: int
     """
 
+    errors = workflow.get_errors(True)
+    errors_summed = defaultdict(int)
 
-    curs, _, allerrors, _ = check_session(session).info
-
-    num_errors = []
-
-    for errorcode in allerrors:
-        output = curs.execute("SELECT SUM(numbererrors) FROM workflows WHERE "
-                              "stepname LIKE '/{0}/%' AND errorcode={1}".\
-                                  format(workflow, errorcode))
+    for codes in errors.values():
+        for errorcode, sites in codes.items():
+            numcode = -1 if errorcode == 'NotReported' else int(errorcode)
+            for num in sites.values():
+                errors_summed[numcode] += num
 
-        num_errors.append(output[0])
+    output = 0
+    max_num = 0
+    for code, num in errors_summed.items():
+        if num > max_num:
+            max_num = num
+            output = code
 
-    return allerrors[num_errors.index(max(num_errors))]
+    return output

workflowwebtools/errorutils.py

@@ -59,7 +59,6 @@ def open_location(data_location):
             "SELECT NAME FROM CMS_UNIFIED_ADMIN.workflow WHERE lower(STATUS) LIKE '%manual%'")
         wkfs = [row for row, in oracle_cursor]
         oracle_db_conn.close()
-        cherrypy.log('Number of workflows from database: %i' % len(wkfs))
         return errors_from_list(wkfs)
 
     raw = None

workflowwebtools/predict/evaluate.py

@@ -1,18 +1,189 @@
+# pylint: disable=missing-docstring, too-complex, invalid-name, too-many-branches, too-many-locals
+
 """
 A module that evaluates a model and returns the prediction
 """
 
-#from cmstoolbox import sitereadiness
+import os
+import random
+import itertools
+
+import numpy as np
+import pandas as pd
+import keras as K
+
+
+def modified_site_name(site):
+    site_name = site.split('_')[:-1]
+    s = ''
+    for i in site_name:
+        s = s+i+'_'
+    s = s.rstrip('_')
+    return s
+
+def build_table(df, template_table):
+    sparse_df = template_table.copy()
+
+    tier0_sites, tier1_sites, tier2_sites, tier3_sites = [], [], [], []
+    for i in sparse_df.keys():
+        if i != 'NA':
+            if i[1] == '0':
+                tier0_sites.append(i)
+            elif i[1] == '1':
+                tier1_sites.append(i)
+            elif i[1] == '2':
+                tier2_sites.append(i)
+            elif i[1] == '3':
+                tier3_sites.append(i)
+
+    n0, n1, n2, n3 = len(tier0_sites), len(tier1_sites), len(tier2_sites), len(tier3_sites)
+    for exit_code, site_dict in zip(df.keys(), df.values()):
+        exit_code = int(exit_code)
+        for site, count in site_dict.items():
+
+            chosen_site = None
+            site_present_in_training_data = site in sparse_df.keys()
+            if not site_present_in_training_data:
+
+                site = modified_site_name(site)
+                cond = site in sparse_df.keys()
+                if cond:
+                    chosen_site = site
+
+                print "Detected a site %s which was not present in the training dataset" % site
+                print "We would use a proxy site for this based on whether it is T1, T2 or T3"
+                tier = site.split("_")[0][1]
+                if chosen_site is None:
+                    if tier == '1':
+                        chosen_num = random.randint(0, n1-1)
+                        chosen_site = tier1_sites[chosen_num]
+                    elif tier == '2':
+                        chosen_num = random.randint(0, n2-1)
+                        chosen_site = tier2_sites[chosen_num]
+                        print 'The chosen site is ', chosen_site
+                    elif tier == '3':
+                        chosen_num = random.randint(0, n3-1)
+                        chosen_site = tier3_sites[chosen_num]
+                        print 'The chosen site is ', chosen_site
+                    elif tier == '0':
+                        chosen_num = random.randint(0, n0-1)
+                        chosen_site = tier0_sites[chosen_num]
+                        print 'The chosen site is ', chosen_site
+                    else:
+                        continue
+                if chosen_site is None:
+                    chosen_site = site
+
+                if np.isnan(count) or np.isnan(sparse_df.loc[exit_code, chosen_site]):
+                    sparse_df.loc[exit_code, chosen_site] = 0
+                else:
+                    sparse_df.loc[exit_code, chosen_site] = count
+
+    return sparse_df
+
+
+def list_of_sites(x):
+    return [item.keys() for item in x] or ['NA']
+
+def build_table_flatten(x):
+    d_outer = []
+
+    for column in x:
+
+        for item in x[column]:
+            d_outer.append(item)
+
+    return d_outer
+
+
+def pred(errors):
+    # Needs all of these files to be local
+    for filename in ['sparse_table.csv', 'actionfile.txt', 'my_model.h5']:
+        if not os.path.exists(filename):
+            return ['TBD']
+
+    df = pd.DataFrame(columns=('workflow', 'errors'))
+    base_data = []
+    for i in errors:
+        it = i.items()
+        base_data.extend(it)
+
+    for i, dat in enumerate(base_data):
+        workflow, error_dict = dat[0], dat[1]
+
+        if 'NotReported' in error_dict:
+            error_dict[-1] = error_dict.pop('NotReported')
+
+        df.loc[i] = [workflow, error_dict]
+
+    template_table = pd.read_csv("sparse_table.csv").set_index("Unnamed: 0")
+    template_table[:] = 0
+    df['errors_sites_exit_codes'] = df['errors'].apply(lambda x: x.keys() if x else ['0'])
+
+    df['errors_sites_dict'] = df['errors'].apply(lambda x: x.values() if x else [{'NA': 0}])
+
+    df['errors_sites_list'] = df['errors_sites_dict'].apply(list_of_sites)
+
+    list2d = df['errors_sites_exit_codes'].tolist()
+
+    sites_exit_codes = sorted(set(list(itertools.chain.from_iterable(list2d))), key=int)
+    sites_exit_codes = [str(x) for x in sites_exit_codes]
+
+    list2d_step1 = df['errors_sites_list'].tolist()
+    list2d_step2 = list(itertools.chain.from_iterable(list2d_step1))
+    site_names = sorted(set(list(itertools.chain.from_iterable(list2d_step2))))
+    site_names = [str(x) for x in site_names]
+
+    df['table_sites'] = df['errors'].apply(
+        lambda x: build_table(x, template_table))
+    df['table_sites_flatten'] = df['table_sites'].apply(build_table_flatten)
+    x_dataframe = df.loc[:, "table_sites_flatten"]
+    x_matrix = x_dataframe.values
+    feature_size = len(x_matrix[0])
+    res = []
+    clip_length = len(x_matrix[0])
+    for i in x_matrix:
+        i_clipped = i[:clip_length]
+        res.extend(i_clipped)
+
+    res = np.asarray(res).reshape(-1, feature_size)
+    mask = ~np.any(pd.isnull(res), axis=1)
+    res = res[mask]
+
+    model = K.models.load_model('my_model.h5')
+    predicted_actions_encoded = model.predict(np.array(np.asfarray(res)))
+    predicted_actions_encoded = np.round(predicted_actions_encoded)
+
+    action_code_dictionary = {}
+    a = np.genfromtxt("actionfile.txt", delimiter='\t', dtype=str)
+    b = list(i.split('   ') for i in a)
+    for i in b:
+
+        action_code_dictionary[int(i[1])] = i[0]
+
+    predicted_actions = []
+    for i in predicted_actions_encoded:
+        pos = np.argmax(i)
+
+        if pos in action_code_dictionary:
+            predicted_actions.append(action_code_dictionary[pos])
+        else:
+            predicted_actions.append(-1)
+
+    K.backend.clear_session()
 
+    return predicted_actions
 
 
-def predict(errors): # pylint: disable=unused-argument
+def predict(wf_obj):
     """
     Takes the errors for a workflow and makes an action prediction
-    :param workflowwebtool.workflowinfo.WorkflowInfo errors:
+    :param workflowwebtool.workflowinfo.WorkflowInfo wf_obj:
         The WorkflowInfo object that we want to perform a prediction on
     :returns: Prediction results to be passed back to a browser
     :rtype: dict
     """
 
-    return {'Action': 'TBD'}
+    return {
+        'Action': pred([wf_obj.get_errors(True)])[0]
+    }