infrastructure to exclude CNAF-archived datasets

Author: Yutaro Iiyama

lib/common/interface/mysqlstore.py

@@ -524,12 +524,12 @@ def _do_load_files(self, dataset): #override
             dataset.files.add(lfile)
 
     def _do_check_if_on(self, datasetname, sitename): #override
-        query = 'SELECT 1 FROM `datasets` AS d'
+        query = 'SELECT COUNT(*) FROM `datasets` AS d'
         query += ' INNER JOIN `dataset_replicas` AS dr ON dr.`dataset_id` = d.`id`'
         query += ' INNER JOIN `sites` AS s ON s.`id` = dr.`site_id`'
-        query += ' WHERE  d.`name` = %s and s.`name` = %s' % (datasetname, sitename)
+        query += ' WHERE  d.`name` = %s and s.`name` = %s'
 
-        return self._mysql.query(query)[0]
+        return (self._mysql.query(query, datasetname, sitename)[0] != 0)
 
     def _do_find_block_of(self, fullpath, datasets): #override
         query = 'SELECT d.`name`, b.`name` FROM `files` AS f'

lib/detox/configuration.py

@@ -4,6 +4,6 @@
     activity_indicator = '/home/cmsprod/public_html/IntelROCCS/Detox/inActionLock.txt',
     deletion_per_iteration = 0.01, # fraction of quota to delete per iteration
     deletion_volume_per_request = 50, # size to delete per deletion request in TB
-    exclude_if_on = [], # if a dataset has a replica on these [sites], don't consider it for deletions. Introduced because of CNAF indicent.
+    exclude_if_on = ['T1_IT_CNAF_MSS'], # if a dataset has a replica on these [sites], don't consider it for deletions. Introduced because of CNAF indicent.
     time_shift = 0. # number of days in the future from which to evaluate the policies
 )

lib/detox/main.py

@@ -103,14 +103,11 @@ def _execute_policy(self, policy, is_test, comment):
         # possible use case: a tape site has had a water indicident, for example
         # Communication with database is needed because we do not have all (tape) replicas in memory 
         if len(detox_config.main.exclude_if_on) > 0:
-            excluded_replicas = []
             for replica in all_replicas:
                 ds_name = replica.dataset.name
                 for sitename in detox_config.main.exclude_if_on:
                     if self.inventory_manager.store.check_if_on(ds_name,sitename):
-                        excluded_replicas.append(replica)
-            for excluded_replica in excluded_replicas:
-                all_replicas.remove(excluded_replica)
+                        replica.dataset.demand['on_protected_site'] = True
 
         logger.info('Saving site and dataset states.')
 

lib/detox/variables.py

@@ -104,6 +104,16 @@ def _get(self, dataset):
         except KeyError:
             return 0.
 
+class DatasetOnProtectedSite(DatasetAttr):
+    def __init__(self):
+        DatasetAttr.__init__(self, Attr.BOOL_TYPE)
+
+    def _get(self, dataset):
+        try:
+            return dataset.demand['on_protected_site']
+        except KeyError:
+            return False
+
 class ReplicaSize(DatasetReplicaAttr):
     def __init__(self):
         DatasetReplicaAttr.__init__(self, Attr.NUMERIC_TYPE)
@@ -272,6 +282,7 @@ def _get(self, site):
     'dataset.usage_rank': DatasetUsageRank(),
     'dataset.demand_rank': DatasetDemandRank(),
     'dataset.release': DatasetRelease(),
+    'dataset.on_protected_site': DatasetOnProtectedSite(),
     'replica.is_last_transfer_source': ReplicaIsLastSource(),
     'replica.size': ReplicaSize(),
     'replica.incomplete': ReplicaIncomplete(),