refactored group and user data sources

Author: Yutaro Iiyama

bin/registry_users.py

@@ -42,11 +42,10 @@
 
     updated_users = []
 
-    for user_info in sitedb._make_request('people'):
-        name = user_info[0]
-        email = user_info[1]
-        dn = user_info[4]
+    sitedb_users = {}
+    sitedb.get_user_list(sitedb_users)
 
+    for name, email, dn in sitedb_users.itervalues():
         try:
             known_user = known_users.pop(name)
         except KeyError:

lib/common/configuration.py

@@ -121,7 +121,7 @@
 )
 
 activitylock = Configuration(
-    default_user = 'paus'
+    default_user = 'dynamo@dynamo.mit.edu'
 )
 
 tape_sites = ['T1_*_MSS', 'T0_CH_CERN_MSS']

lib/common/interface/activitylock.py

@@ -2,6 +2,7 @@
 import logging
 
 from common.interface.mysql import MySQL
+from common.interface.classes import default_interface
 import common.configuration as config
 
 logger = logging.getLogger(__name__)
@@ -21,6 +22,8 @@ def __init__(self, application, service = 'dynamo', asuser = '', db_params = con
         else:
             self.user = config.activitylock.default_user
 
+        self._userinfo = default_interface['user_source']()
+
     def __enter__(self):
         self.lock()
 
@@ -51,6 +54,13 @@ def lock(self):
             self._mysql.query('UNLOCK TABLES')
             time.sleep(60)
 
+        if self._mysql.query('SELECT COUNT(*) FROM `users` WHERE `name` = %s', self.user)[0] == 0:
+            user_data = self._userinfo.get_user(self.user)
+            if user_data is None:
+                raise RuntimeError('Invalid user %s used for activity lock' % self.user)
+
+            self._mysql.query('INSERT INTO `users` (`name`, `email`, `dn`) VALUES (%s, %s, %s)', *user_data)
+
         query = 'INSERT INTO `activity_lock` (`user_id`, `service_id`, `application`, `timestamp`, `note`)'
         query += ' SELECT `users`.`id`, `services`.`id`, %s, NOW(), \'Dynamo running\' FROM `users`, `services`'
         query += ' WHERE `users`.`name` = %s AND `services`.`name` = %s'

lib/common/interface/classes.py

@@ -20,13 +20,11 @@ def __call__(self):
         return obj
 
 
-class DummyInterface(object):
-    def __init__(self):
-        pass
-
 default_interface = {
     'dataset_source': Generator('phedexdbsssb', 'PhEDExDBSSSB'),
     'site_source': Generator('phedexdbsssb', 'PhEDExDBSSSB'),
+    'user_source': Generator('sitedb', 'SiteDB'),
+    'group_source': Generator('phedexdbsssb', 'PhEDExDBSSSB'),
     'replica_source': Generator('phedexdbsssb', 'PhEDExDBSSSB'),
     'copy': Generator('phedexdbsssb', 'PhEDExDBSSSB'),
     'deletion': Generator('phedexdbsssb', 'PhEDExDBSSSB'),

lib/common/interface/groupinfo.py

@@ -0,0 +1,16 @@
+class GroupInfoSourceInterface(object):
+    """
+    Interface specs for group information authority.
+    """
+
+    def __init__(self):
+        pass
+
+    def get_group_list(self, groups, filt = '*'):
+        """
+        Fill the list of groups with groups that match the wildcard name.
+
+        @param groups  {name: Group} to be filled. Information of the groups already in the list will be updated.
+        @param filt    A wildcard string or a list of wildcard strings.
+        """
+        pass

lib/common/interface/phedexdbsssb.py

@@ -11,6 +11,7 @@
 from common.interface.copy import CopyInterface
 from common.interface.deletion import DeletionInterface
 from common.interface.siteinfo import SiteInfoSourceInterface
+from common.interface.groupinfo import GroupInfoSourceInterface
 from common.interface.replicainfo import ReplicaInfoSourceInterface
 from common.interface.datasetinfo import DatasetInfoSourceInterface
 from common.interface.webservice import RESTService, GET, POST
@@ -27,7 +28,7 @@
 # PhEDEx does not document a hard limit on the length of POST request list.
 # 10000 was experimentally verified to be OK.
 
-class PhEDExDBSSSB(CopyInterface, DeletionInterface, SiteInfoSourceInterface, ReplicaInfoSourceInterface, DatasetInfoSourceInterface):
+class PhEDExDBSSSB(CopyInterface, DeletionInterface, SiteInfoSourceInterface, GroupInfoSourceInterface, ReplicaInfoSourceInterface, DatasetInfoSourceInterface):
     """
     Interface to PhEDEx/DBS/SSB using datasvc REST API.
     """
@@ -36,6 +37,7 @@ def __init__(self, phedex_url = config.phedex.url_base, dbs_url = config.dbs.url
         CopyInterface.__init__(self)
         DeletionInterface.__init__(self)
         SiteInfoSourceInterface.__init__(self)
+        GroupInfoSourceInterface.__init__(self)
         ReplicaInfoSourceInterface.__init__(self)
         DatasetInfoSourceInterface.__init__(self)
 
@@ -465,7 +467,7 @@ def set_site_status(self, sites): #override (SiteInfoSourceInterface)
                 else:
                     site.status = Site.STAT_READY
 
-    def get_group_list(self, groups, filt = '*'): #override (SiteInfoSourceInterface)
+    def get_group_list(self, groups, filt = '*'): #override (GroupInfoSourceInterface)
         logger.info('get_group_list  Fetching the list of groups from PhEDEx')
         source = self._make_phedex_request('groups')
 

lib/common/interface/sitedb.py

@@ -4,17 +4,18 @@
 import datetime
 
 from common.interface.siteinfo import SiteInfoSourceInterface
+from common.interface.userinfo import UserInfoSourceInterface
 from common.interface.webservice import RESTService, GET, POST
 from common.dataformat import Site
 import common.configuration as config
 
 logger = logging.getLogger(__name__)
 
-class SiteDB(SiteInfoSourceInterface):
+class SiteDB(SiteInfoSourceInterface, UserInfoSourceInterface):
     def __init__(self):
         self._interface = RESTService(config.sitedb.url_base)
 
-    def get_site_list(self, sites, filt = '*'): #override
+    def get_site_list(self, sites, filt = '*'): #override (SiteInfoSourceInterface)
         """
         Fill the list of sites with sites that match the wildcard name.
         Arguments:
@@ -81,6 +82,30 @@ def get_site_list(self, sites, filt = '*'): #override
 
                 sites[name] = site
 
+    def get_user(self, name): #override (UserInfoSourceInterface)
+        result = self._interface.make_request('people', ['match=%s' % name])
+
+        if len(result) == 0:
+            return None
+        else:
+            user_info = result[0]
+            name = user_info[0]
+            email = user_info[1]
+            dn = user_info[4]
+
+            return (name, email, dn)
+
+    def get_user_list(self, users, filt = '*'): #override (UserInfoSourceInterface)
+        result = self._interface.make_request('people')
+
+        for user in users:
+            user_info = result[0]
+            name = user_info[0]
+            email = user_info[1]
+            dn = user_info[4]
+            
+            users[name] = (name, email, dn)
+
     def _make_request(self, resource, options = []):
         """
         Make a single API call to SiteDB, strip the "header" and return the body JSON.
@@ -118,3 +143,8 @@ def _make_request(self, resource, options = []):
                     continue
 
                 out.write('UPDATE `sites` SET storage = %f, cpu = %f WHERE `name` LIKE \'%s\';\n' % (site.storage, site.cpu, site.name))
+
+    elif args.command == 'api':
+        result = interface._make_request(args.options[0], args.options[1:])
+
+        pprint.pprint(result)

lib/common/interface/siteinfo.py

@@ -23,15 +23,6 @@ def set_site_status(self, sites):
         """
         pass
 
-    def get_group_list(self, groups, filt = '*'):
-        """
-        Fill the list of groups with groups that match the wildcard name.
-        Arguments:
-          groups: the name->group dict to be filled. Information of the groups already in the list will be updated.
-          filt: a wildcard string or a list of wildcard strings.
-        """
-        pass
-
 
 if __name__ == '__main__':
 

lib/common/interface/userinfo.py

@@ -0,0 +1,21 @@
+class UserInfoSourceInterface(object):
+    """
+    Interface specs for user data authority.
+    """
+        
+    def __init__(self):
+        pass
+
+    def get_user(self, name):
+        """
+        Return info of a single user in the expected format. Return None for invalid user name.
+        """
+        pass
+
+    def get_user_list(self, users, filt = '*'):
+        """
+        Fill the given list with users with names matching the filter.
+        @param users   {name: userdata} to be filled with user information in the expected format.
+        @param filt    Wildcard pattern.
+        """
+        pass

lib/common/inventory.py

@@ -22,7 +22,7 @@ class ConsistencyError(Exception):
 class InventoryManager(object):
     """Bookkeeping class to bridge the communication between remote and local data sources."""
 
-    def __init__(self, load_data = True, store_cls = None, site_source_cls = None, dataset_source_cls = None, replica_source_cls = None):
+    def __init__(self, load_data = True, store_cls = None, site_source_cls = None, group_source_cls = None, dataset_source_cls = None, replica_source_cls = None):
         if store_cls:
             self.store = store_cls()
         else:
@@ -33,6 +33,11 @@ def __init__(self, load_data = True, store_cls = None, site_source_cls = None, d
         else:
             self.site_source = default_interface['site_source']()
 
+        if group_source_cls:
+            self.group_source = group_source_cls()
+        else:
+            self.group_source = default_interface['group_source']()
+
         if dataset_source_cls:
             self.dataset_source = dataset_source_cls()
         else:
@@ -148,7 +153,7 @@ def update(self, dataset_filter = '*', load_first = True, make_snapshot = True,
 
             self.site_source.set_site_status(self.sites)
 
-            self.site_source.get_group_list(self.groups, filt = config.inventory.included_groups)
+            self.group_source.get_group_list(self.groups, filt = config.inventory.included_groups)
 
             if from_delta:
                 if last_update == 0: