Remove iteritems and use old scikit-learn

Author: dabercro

setup.py

@@ -24,7 +24,7 @@
         'mako',
         'numpy>=1.6.1',
         'scipy==1.1.0',
-        'sklearn',
+        'scikit-learn==0.20.3',
         'passlib>=1.6',
         'bcrypt',
         'pyOpenSSL',

workflowwebtools/clusterworkflows.py

@@ -208,7 +208,7 @@ def get_clustered_group(workflow, clusterer, session=None):
     group = predictions.get(workflow)
 
     if group is not None:
-        for wkf, cluster in predictions.iteritems():
+        for wkf, cluster in predictions.items():
             if cluster == group and wkf != workflow:
                 output.append(wkf)
 

workflowwebtools/errorutils.py

@@ -90,7 +90,7 @@ def open_location(data_location):
         return raw
 
     return errors_from_list([
-        workflow for workflow, statuses in raw.iteritems()
+        workflow for workflow, statuses in raw.items()
         if True in ['manual' in status for status in statuses]
     ])
 

workflowwebtools/globalerrors.py

@@ -405,20 +405,20 @@ def group_errors(input_errors, grouping_function, **kwargs):
 
     output = default_errors_format()
 
-    for subgroup, values in input_errors.iteritems():
+    for subgroup, values in input_errors.items():
 
         group = grouping_function(subgroup)
 
         # We have three variables for everything, so we can write this by hand
         # Not ideal
-        for row, row_val in values['errors'].iteritems():
-            for col, numerrors in row_val.iteritems():
+        for row, row_val in values['errors'].items():
+            for col, numerrors in row_val.items():
                 output[group]['errors'][row][col] += numerrors
 
         output[group]['sub'][subgroup] = values
         output[group]['total'] += values['total']
 
-        for key, func in kwargs.iteritems():
+        for key, func in kwargs.items():
             output[group][key] = func(group)
 
     return output

workflowwebtools/manageactions.py

@@ -36,7 +36,7 @@ def extract_reasons_params(action, **kwargs):
     if not isinstance(tasks_to_do, list):
         tasks_to_do = [tasks_to_do]
 
-    for key, item in kwargs.iteritems():
+    for key, item in kwargs.items():
 
         if 'shortreason' in key:
             short_re = item or reasonsmanip.DEFAULT_SHORT
@@ -140,7 +140,7 @@ def submitaction(user, workflows, action, session=None, **kwargs):
                 # Get any existing thing (most likely not there)
                 step_params = wf_params.get(short_step_name, {})
 
-                for key, val in all_steps.iteritems():
+                for key, val in all_steps.items():
                     # This also includes if the key value is set but blank
                     if not step_params.get(key):
                         step_params[key] = val
@@ -315,7 +315,7 @@ def fix_sites(**kwargs):
 
     coll = get_actions_collection()
 
-    for task, value in params.iteritems():
+    for task, value in params.items():
         split_task = task.split('/')
         workflow = split_task[1]
         subtask = '/'.join(split_task[2:])

workflowwebtools/paramsregression.py

@@ -42,7 +42,7 @@ def convert_to_dense(errors, keys=None, allerrors=None, allsites=None):
         allsites = set()
 
         for status in keys:
-            for error, sites in errors[status].iteritems():
+            for error, sites in errors[status].items():
                 allerrors.add(int(error))
                 for site in sites:
                     allsites.add(site)
@@ -99,7 +99,7 @@ def get_classifier(raw_data, parameter, **kwargs):
             matrix = raw_data[key]['errors'][status]
             # Only do this for sparse matrices
             if not isinstance(matrix, list):
-                for error, sites in matrix.iteritems():
+                for error, sites in matrix.items():
                     allerrors.add(int(error))
                     for site in sites:
                         allsites.add(site)

workflowwebtools/statuses.py

@@ -34,5 +34,5 @@ def get_manual_workflows(location):
     """
 
     return [workflow for workflow, statuses
-            in open_statuses(location).iteritems()
+            in open_statuses(location).items()
             if True in ['manual' in status for status in statuses]]

workflowwebtools/workflowinfo.py

@@ -133,11 +133,11 @@ def errors_for_workflow(workflow, url='cmsweb.cern.ch'):
     if not result['result']:
         return output
 
-    for step, stepdata in result['result'][0].get(workflow, {}).iteritems():
+    for step, stepdata in result['result'][0].get(workflow, {}).items():
         errors = {}
-        for code, codedata in stepdata.get('jobfailed', {}).iteritems():
+        for code, codedata in stepdata.get('jobfailed', {}).items():
             sites = {}
-            for site, sitedata in codedata.iteritems():
+            for site, sitedata in codedata.items():
                 if sitedata['errorCount']:
                     sites[site] = sitedata['errorCount']
 
@@ -261,7 +261,7 @@ def get_workflow_parameters(self):
                               use_https=True, use_cert=True)
 
             for params in result['result']:
-                for key, item in params.iteritems():
+                for key, item in params.items():
                     if key == self.workflow:
                         return item
 
@@ -351,13 +351,13 @@ def get_failure_rate(self):
 
         nsuccess = 0
         nfailure = 0
-        for agent, agentdata in wf_agents.iteritems():
+        for agent, agentdata in wf_agents.items():
             status = agentdata.get('status', {})
             if not status: continue
 
             nsuccess += status.get('success', 0)
 
-            for ftype, num in status.get('failure', {}).iteritems():
+            for ftype, num in status.get('failure', {}).items():
                 nfailure += num
 
         try:
@@ -407,7 +407,7 @@ def get_recovery_info(self):
             task = doc['fileset_name']
             # For each task, we have the following keys:
             # sites - a set of sites that the recovery docs say to run on.
-            for replica, info in doc['files'].iteritems():
+            for replica, info in doc['files'].items():
                 # For fake files, just return the site whitelist
                 if replica.startswith('MCFakeFile'):
                     locations = site_white_list
@@ -478,14 +478,14 @@ def get_explanation(self, errorcode, step=''):
         if self.explanations is None:
             self.explanations = defaultdict(lambda: defaultdict(lambda: []))
             result = self._get_jobdetail()
-            for stepname, stepdata in result['result'][0].get(self.workflow, {}).iteritems():
+            for stepname, stepdata in result['result'][0].get(self.workflow, {}).items():
                 # Get the errors from both 'jobfailed' and 'submitfailed' details
                 for error, site in [(error, site) for status in ['jobfailed', 'submitfailed'] \
                                         for error, site in stepdata.get(status, {}).items()]:
                     if error == '0':
                         continue
 
-                    for sitename, samples in site.iteritems():
+                    for sitename, samples in site.items():
                         for detail in [values for sample in samples['samples']
                                        for errs in sample['errors'].values()
                                        for values in errs]:
@@ -566,7 +566,7 @@ def get_workflows_requesttime(self):
         request = self.get_requests()
 
         return [(workflow, time.mktime(datetime.datetime(*value['RequestDate']).timetuple())) \
-                    for workflow, value in request.iteritems()]
+                    for workflow, value in request.items()]
 
     def get_workflows(self):
         """

workflowwebtools/workflowtools.py

@@ -195,11 +195,11 @@ def getworkflows(self, prepid):
              "status": self.get_status(workflow),
              "errors": obj['obj'].sum_errors()
             }
-            for workflow, obj in workflow_objs.iteritems()
+            for workflow, obj in workflow_objs.items()
             ]
 
         self.lock.acquire()
-        for workflow, obj in workflow_objs.iteritems():
+        for workflow, obj in workflow_objs.items():
             if workflow not in self.workflows:
                 self.workflows[workflow] = obj['obj']
         self.lock.release()
@@ -316,7 +316,7 @@ def getreasons(self):
                 'long': longreason
             }
             for shortreason, longreason in
-            sorted(reasonsmanip.reasons_list().iteritems())
+            sorted(reasonsmanip.reasons_list().items())
         ]
 
 
@@ -329,17 +329,17 @@ def workflowerrors(self, workflow):
         output = []
 
         # Need to track total sites, so we need to do nested loops here
-        for step, ecs in sorted(errors.iteritems()):
+        for step, ecs in sorted(errors.items()):
             allsites = set()
 
             codes = []
             for code, sites in sorted([
                     (-1 if code == 'NotReported' else int(code), sites)
-                    for code, sites in ecs.iteritems()]):
+                    for code, sites in ecs.items()]):
 
                 sites = {
                     site: (num or int(code < 0))
-                    for site, num in sorted(sites.iteritems())
+                    for site, num in sorted(sites.items())
                 }
                 allsites.update(sites.keys())
 
@@ -667,7 +667,7 @@ def submitaction(self, workflows='', action='', **kwargs):
             for workflow in workflows:
                 # Check sites of recovered workflows
                 if check_actions[workflow]['Action'] in ['acdc', 'recovery']:
-                    for subtask, params in check_actions[workflow]['Parameters'].iteritems():
+                    for subtask, params in check_actions[workflow]['Parameters'].items():
                         # Empty sites are noted
                         if not params.get('sites'):
                             blank_sites_subtask.append('/%s/%s' % (workflow, subtask))