unified overflow mapping in cache info

Author: vlimant

Unified/equalizor.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 from assignSession import *
-from utils import workflowInfo, getWorkflows, global_SI, sendEmail, componentInfo, getDatasetPresence, monitor_dir, monitor_pub_dir, reqmgr_url, campaignInfo, unifiedConfiguration, sendLog, do_html_in_each_module, base_eos_dir, eosRead, eosFile, agent_speed_draining
+from utils import workflowInfo, getWorkflows, global_SI, sendEmail, componentInfo, getDatasetPresence, monitor_dir, monitor_pub_dir, reqmgr_url, campaignInfo, unifiedConfiguration, sendLog, do_html_in_each_module, base_eos_dir, eosRead, eosFile, agent_speed_draining, cacheInfo
 import reqMgrClient
 import json
 import os, sys
@@ -25,10 +25,10 @@ def equalizor(url , specific = None, options=None):
 
     ## start from scratch
     modifications = defaultdict(dict)
-    ## define regionality site => fallback allowed. feed on an ssb metric ??
-    mapping = defaultdict(list)
-    reversed_mapping = defaultdict(list)
-    regions = defaultdict(list)
+
+    cache = cacheInfo()
+    mapping = cache.get('overflow_mapping')
+    reversed_mapping = cache.get('overflow_reverse_mapping')
 
     UC = unifiedConfiguration()
     over_rides = []
@@ -40,160 +40,12 @@ def equalizor(url , specific = None, options=None):
     if options.hlt: use_HLT = True
     if use_HLT: over_rides.append('T2_CH_CERN_HLT')
 
-    use_CSCS = ('T0_CH_CSCS_HPC' in UC.get("site_for_overflow"))
-    if options.cscs: use_CSCS = True
-    if use_CSCS: over_rides.append('T0_CH_CSCS_HPC')
-    
-
     SI = global_SI( over_rides )
-    print sorted(SI.all_sites)
-    print sorted(SI.sites_T0s)
-
     CI = campaignInfo()
 
-    #sites_to_consider = SI.all_sites
-    sites_to_consider = SI.sites_ready
-    for site in sites_to_consider:
-        region = site.split('_')[1]
-        if not region in ['US'
-                          ,'DE','IT','FR',
-                          'ES',
-                          'UK',
-                          'RU'### latest addition
-                          ]: continue
-        regions[region] = [region] 
-
-    def site_in_depletion(s):
-        return True
-        if s in SI.sites_pressure:
-            (m, r, pressure) = SI.sites_pressure[s]
-            if float(m) < float(r):
-                print s,m,r,"lacking pressure"
-                return True
-            else:
-                print s,m,r,"pressure"
-                pass
-                
-        return False
-
-    for site in sites_to_consider:
-        region = site.split('_')[1]
-        ## fallback to the region, to site with on-going low pressure
-        within_region = [fb for fb in sites_to_consider if any([('_%s_'%(reg) in fb and fb!=site and site_in_depletion(fb))for reg in regions[region]]) ]
-        #print site,region, within_region
-        mapping[site] = within_region
-    
-
-    for site in sites_to_consider:
-        if site.split('_')[1] == 'US': ## to all site in the US
-            ## add NERSC 
-            mapping[site].append('T3_US_NERSC')
-            mapping[site].append('T3_US_SDSC')
-            mapping[site].append('T3_US_TACC')
-            mapping[site].append('T3_US_PSC')
-            ## add OSG            
-            mapping[site].append('T3_US_OSG')
-            #mapping[site].append('T3_US_Colorado')
-            pass
-
-    if use_HLT:
-        mapping['T2_CH_CERN'].append('T2_CH_CERN_HLT')
-
-    if use_T0:
-        ## who can read from T0
-        mapping['T2_CH_CERN'].append('T0_CH_CERN')
-        mapping['T1_IT_CNAF'].append('T0_CH_CERN')
-        mapping['T1_FR_CCIN2P3'].append('T0_CH_CERN')
-        mapping['T1_DE_KIT'].append('T0_CH_CERN')
-
-    if use_CSCS: 
-	## analog config to T0: 
-	mapping['T2_CH_CERN'].append('T0_CH_CSCS_HPC')
-        mapping['T1_IT_CNAF'].append('T0_CH_CSCS_HPC')
-        mapping['T1_FR_CCIN2P3'].append('T0_CH_CSCS_HPC')
-        mapping['T1_DE_KIT'].append('T0_CH_CSCS_HPC')
-
-    ## temptatively
-    mapping['T0_CH_CERN'].append( 'T2_CH_CERN' )
-
-    ## all europ can read from CERN
-    for reg in ['IT','DE','UK','FR','BE','ES']:
-        mapping['T2_CH_CERN'].extend([fb for fb in sites_to_consider if '_%s_'%reg in fb])
-        pass
-
-    ## all europ T1 among each others
-    europ_t1 = [site for site in sites_to_consider if site.startswith('T1') and any([reg in site for reg in ['IT','DE','UK','FR','ES','RU']])]
-    #print europ_t1
-    for one in europ_t1:
-        for two in europ_t1:
-            if one==two: continue
-            mapping[one].append(two)
-            pass
-        ## all EU T1 can read from T0
-        mapping['T0_CH_CERN'].append( one )
-        
-    mapping['T0_CH_CERN'].append( 'T1_US_FNAL' )
-    #mapping['T1_IT_CNAF'].append( 'T1_US_FNAL' )
-    #mapping['T1_IT_CNAF'].extend( [site for site in SI.sites_ready if '_US_' in site] ) ## all US can read from CNAF
-    mapping['T1_IT_CNAF'].append( 'T2_CH_CERN' )
-    mapping['T1_DE_KIT'].append( 'T2_CH_CERN' )
-    mapping['T2_CH_CERN'].append( 'T1_IT_CNAF' )
-    mapping['T2_CH_CERN'].append( 'T1_US_FNAL' )
-    mapping['T2_CH_CERN'].append('T3_CH_CERN_HelixNebula')
-    mapping['T2_CH_CERN'].append('T3_CH_CERN_HelixNebula_REHA')
-    
-
-    for site in sites_to_consider:
-        if '_US_' in site:
-            mapping[site].append('T2_CH_CERN')
     ## make them appear as OK to use
     force_sites = []
 
-    ## overflow CERN to underutilized T1s
-    upcoming = json.loads( eosRead('%s/GQ.json'%monitor_dir) )
-    for possible in SI.sites_T1s:
-        if not possible in upcoming:
-            mapping['T2_CH_CERN'].append(possible)
-            pass
-
-    take_site_out = UC.get('site_out_of_overflow')
-
-    for site,fallbacks in mapping.items():
-        mapping[site] = list(set(fallbacks))
-    
-
-    ### mapping is a dictionnary where
-    # key can read from site in values. 
-    ### reverserd mapping is a dictionnary where
-    # key can be read by site in values.
-    ## create the reverse mapping for the condor module
-    for site,fallbacks in mapping.items():
-        if site in take_site_out:
-            print "taking",site,"out of overflow source by unified configuration"
-            mapping.pop(site)
-            continue
-        for fb in fallbacks:
-            if fb == site: 
-                ## remove self
-                mapping[site].remove(fb)
-                continue
-            if fb in take_site_out:
-                ## remove those to be removed
-                print "taking",fb,"out of overflow destination by unified configuration"
-                mapping[site].remove(fb)
-                continue
-            if not site in reversed_mapping[fb]:
-                reversed_mapping[fb].append(site)
-
-    #for site in mapping.keys():
-    #    mapping[site] = list(set(mapping[site]))
-
-    ## this is the fallback mapping
-    #print "Direct mapping : site => overflow"
-    #print json.dumps( mapping, indent=2)
-    #print "Reverse mapping : dest <= from origin"
-    #print json.dumps( reversed_mapping, indent=2)
-
     altered_tasks = set()
 
     def running_idle( wfi , task_name):