purging delete_not_in

Author: Yutaro Iiyama

bin/registry_users.py

@@ -36,23 +36,29 @@
 
     domain_id = store.query('SELECT `id` FROM `domains` WHERE `name` = \'cern.ch\'')[0]
 
-    query = 'INSERT INTO `users` (`name`, `domain_id`, `email`, `dn`) VALUES (%s, ' + str(domain_id) +', %s, %s) ON DUPLICATE KEY UPDATE `email` = `email`, `dn` = `dn`'
+    known_users = {}
+    for name, email, dn in store.xquery('SELECT `name`, `email`, `dn` FROM `users` WHERE `domain_id` = %s', domain_id):
+        known_users[name] = (email, dn)
 
-    names = []
+    updated_users = []
 
     for user_info in sitedb._make_request('people'):
         name = user_info[0]
         email = user_info[1]
         dn = user_info[4]
 
-        if dn is None:
-            continue
+        try:
+            known_user = known_users.pop(name)
+        except KeyError:
+            updated_users.append((name, domain_id, email, dn))
+        else:
+            if known_user != (email, dn):
+                updated_users.append((name, domain_id, email, dn))
 
-        names.append(name)
- 
-        store.query(query, name, email, dn)
+    store.insert_many('users', ('name', 'domain_id', 'email', 'dn'), None, updated_users, do_update = True)
 
-    store.delete_not_in('users', 'name', names)
+    if len(known_users) != 0:
+        store.delete_in('users', 'name', known_users.iterkeys())
 
 elif args.command[0] == 'service':
     if args.command[1] == 'add':

lib/common/interface/mysql.py

@@ -164,7 +164,7 @@ def xquery(self, sql, *args):
         cursor.close()
         return
 
-    def execute_many(self, sqlbase, key, pool, additional_conditions = [], exec_on_match = True, order_by = ''):
+    def execute_many(self, sqlbase, key, pool, additional_conditions = [], order_by = ''):
         result = []
 
         if type(key) is tuple:
@@ -178,10 +178,6 @@ def execute_many(self, sqlbase, key, pool, additional_conditions = [], exec_on_m
         for add in additional_conditions:
             sqlbase += add + ' AND '
         sqlbase += key_str
-        if exec_on_match:
-            sqlbase += ' IN '
-        else:
-            sqlbase += ' NOT IN '
 
         def execute(pool_expr):
             sql = sqlbase + pool_expr
@@ -196,26 +192,20 @@ def execute(pool_expr):
 
         return result
 
-    def select_many(self, table, fields, key, pool, additional_conditions = [], select_match = True, order_by = ''):
+    def select_many(self, table, fields, key, pool, additional_conditions = [], order_by = ''):
         if type(fields) is str:
             fields_str = '`%s`' % fields
         else:
             fields_str = ','.join('`%s`' % f for f in fields)
 
         sqlbase = 'SELECT {fields} FROM `{table}`'.format(fields = fields_str, table = table)
 
-        return self.execute_many(sqlbase, key, pool, additional_conditions, select_match, order_by = order_by)
+        return self.execute_many(sqlbase, key, pool, additional_conditions, order_by = order_by)
 
-    def delete_many(self, table, key, pool, additional_conditions = [], delete_match = True):
+    def delete_many(self, table, key, pool, additional_conditions = []):
         sqlbase = 'DELETE FROM `{table}`'.format(table = table)
 
-        self.execute_many(sqlbase, key, pool, additional_conditions, delete_match)
-
-    def delete_in(self, table, key, pool, additional_conditions = []):
-        self.delete_many(table, key, pool, additional_conditions = additional_conditions, delete_match = True)
-
-    def delete_not_in(self, table, key, pool, additional_conditions = []):
-        self.delete_many(table, key, pool, additional_conditions = additional_conditions, delete_match = False)
+        self.execute_many(sqlbase, key, pool, additional_conditions)
 
     def insert_many(self, table, fields, mapping, objects, do_update = True):
         """

lib/common/interface/mysqlhistory.py

@@ -153,7 +153,7 @@ def _do_make_deletion_entry(self, run_number, site, operation_id, approved, data
 
         site_id = self._mysql.query('SELECT `id` FROM `sites` WHERE `name` LIKE %s', site.name)[0]
 
-        dataset_ids = self._mysql.select_many('datasets', ('id',), 'name', [d.name for d in datasets])
+        dataset_ids = self._mysql.select_many('datasets', ('id',), 'name', (d.name for d in datasets))
 
         self._mysql.query('INSERT INTO `deletion_requests` (`id`, `run_id`, `timestamp`, `approved`, `site_id`, `size`) VALUES (%s, %s, NOW(), %s, %s, %s)', operation_id, run_number, approved, site_id, size)
 

lib/common/interface/mysqlstore.py

@@ -749,22 +749,20 @@ def _do_save_datasets(self, datasets): #override
         # speedup - fetch all blocks and files of updated datasets
         # if size, num_file, or is_open of a block or a file is updated, its dataset also is
 
-        pool = [did for did, dataset in datasets_to_update if dataset.blocks is not None]
-        block_entries = dict((i, []) for i in pool)
+        block_entries = dict((i, []) for i in did for did, dataset in datasets_to_update if dataset.blocks is not None)
 
         _dataset_id = 0
-        for entry in self._mysql.select_many('blocks', ('dataset_id', 'id', 'name', 'size', 'num_files', 'is_open'), 'dataset_id', pool, order_by = '`dataset_id`'):
+        for entry in self._mysql.select_many('blocks', ('dataset_id', 'id', 'name', 'size', 'num_files', 'is_open'), 'dataset_id', block_entries.iterkeys(), order_by = '`dataset_id`'):
             if entry[0] != _dataset_id:
                 _dataset_id = entry[0]
                 entry_list = block_entries[entry[0]] = []
 
             entry_list.append(entry[1:])
 
-        pool = [did for did, dataset in datasets_to_update if dataset.files is not None]
-        file_entries = dict((i, []) for i in pool)
+        file_entries = dict((i, []) for i in did for did, dataset in datasets_to_update if dataset.files is not None)
 
         _dataset_id = 0
-        for entry in self._mysql.select_many('files', ('dataset_id', 'id', 'size', 'name'), 'dataset_id', pool, order_by = '`dataset_id`'):
+        for entry in self._mysql.select_many('files', ('dataset_id', 'id', 'size', 'name'), 'dataset_id', file_entries.iterkeys(), order_by = '`dataset_id`'):
             if entry[0] != _dataset_id:
                 _dataset_id = entry[0]
                 entry_list = file_entries[entry[0]] = []
@@ -1153,7 +1151,7 @@ def _do_delete_datasets(self, datasets): #override
         """
         Delete everything related to the datasets
         """
-        dataset_ids = self._mysql.select_many('datasets', 'id', 'name', [d.name for d in datasets])
+        dataset_ids = self._mysql.select_many('datasets', 'id', 'name', (d.name for d in datasets))
 
         ids_str = ','.join(['%d' % i for i in dataset_ids])
 
@@ -1179,9 +1177,9 @@ def _do_delete_block(self, block): #override
     def _do_delete_datasetreplicas(self, site, datasets, delete_blockreplicas): #override
         site_id = self._mysql.query('SELECT `id` FROM `sites` WHERE `name` LIKE %s', site.name)[0]
 
-        dataset_ids = self._mysql.select_many('datasets', 'id', 'name', [d.name for d in datasets])
+        dataset_ids = self._mysql.select_many('datasets', 'id', 'name', (d.name for d in datasets))
 
-        self._mysql.delete_in('dataset_replicas', 'dataset_id', dataset_ids, additional_conditions = ['site_id = %d' % site_id])
+        self._mysql.delete_many('dataset_replicas', 'dataset_id', dataset_ids, additional_conditions = ['site_id = %d' % site_id])
 
         if delete_blockreplicas:
             ids_str = ','.join(['%d' % i for i in dataset_ids])

lib/common/interface/sitedb.py

@@ -81,12 +81,12 @@ def get_site_list(self, sites, filt = '*'): #override
 
                 sites[name] = site
 
-    def _make_request(self, resource):
+    def _make_request(self, resource, options = []):
         """
         Make a single API call to SiteDB, strip the "header" and return the body JSON.
         """
 
-        resp = self._interface.make_request(resource)
+        resp = self._interface.make_request(resource, options)
 
         return resp['result']