Merge pull request #11 from maierbenedikt/production

Delta update preliminary version

Author: yiiyama

bin/dynamod

@@ -6,7 +6,7 @@
 HOURS_PER_LOGFILE=24
 DETOX_INTERVAL=6
 DEALER_INTERVAL=1
-INVENTORY_FULL_UPDATE_INTERVAL=6
+INVENTORY_FULL_UPDATE_INTERVAL=12
 NUM_CRASHES=0
 
 while [ $# -gt 0 ]
@@ -90,13 +90,13 @@ do
     then
       echo "Performing the full update of inventory"
       $DYNAMO_BASE/bin/execlib common/inventory.py updatefull --site @disk
-#      $DYNAMO_BASE/bin/execlib common/inventory.py updatefull --site T1_DE_KIT_MSS T1_ES_PIC_MSS T1_FR_CCIN2P3_MSS
-#      $DYNAMO_BASE/bin/execlib common/inventory.py updatefull --site T1_IT_CNAF_MSS T1_RU_JINR_MSS T1_UK_RAL_MSS
-#      $DYNAMO_BASE/bin/execlib common/inventory.py updatefull --site T1_US_FNAL_MSS
-#      $DYNAMO_BASE/bin/execlib common/inventory.py updatefull --site T0_CH_CERN_MSS
-#    else
-#      echo "Updating the inventory"
-#      $DYNAMO_BASE/bin/execlib common/inventory.py update
+      $DYNAMO_BASE/bin/execlib common/inventory.py updatefull --site T1_DE_KIT_MSS T1_ES_PIC_MSS T1_FR_CCIN2P3_MSS
+      $DYNAMO_BASE/bin/execlib common/inventory.py updatefull --site T1_IT_CNAF_MSS T1_RU_JINR_MSS T1_UK_RAL_MSS
+      $DYNAMO_BASE/bin/execlib common/inventory.py updatefull --site T1_US_FNAL_MSS
+      $DYNAMO_BASE/bin/execlib common/inventory.py updatefull --site T0_CH_CERN_MSS
+    else
+      echo "Updating the inventory"
+      $DYNAMO_BASE/bin/execlib common/inventory.py update
     fi
 
     IHOUR=$(($IHOUR+1))

lib/common/interface/mysqlstore.py

@@ -1237,7 +1237,17 @@ def _do_update_blockreplicas(self, replica_list): #override
 
         sites = list(set([r.site for r in replica_list])) # list of sites
         datasets = list(set([r.block.dataset for r in replica_list])) # list of datasets
-        groups = list(set([r.block.group for r in replica_list])) # list of groups
+        # Load all groups - otherwise it won't pass obj.name in the make_map function if group == None. 
+        # This must be appended separately in make_group_map
+        groups = []
+        for name, olname in self._mysql.xquery('SELECT `name`, `olevel` FROM `groups`'):
+            if olname == 'Dataset':
+                olevel = Dataset
+            else:
+                olevel = Block
+
+            group = Group(name, olevel)
+            groups.append(group)
 
         site_id_map = {}
         self._make_site_map(sites, site_id_map = site_id_map)
@@ -1246,6 +1256,7 @@ def _do_update_blockreplicas(self, replica_list): #override
         dataset_id_map = {}
         self._make_dataset_map(datasets, dataset_id_map = dataset_id_map)
 
+        block_name_to_id = {}
         for dataset in datasets:
             for block_id, block_name in self._mysql.xquery('SELECT `id`, `name` FROM `blocks` WHERE `dataset_id` = %s', dataset_id_map[dataset]):
                 block_name_to_id[Block.translate_name(block_name)] = block_id