[#471][#472] allow save, load, and autoload of configuration

Author: d-w-moore

README.rst

@@ -314,6 +314,37 @@ This may be useful for Python programs in which frequent flushing of write updat
 with descriptors on such objects possibly being held open for indeterminately long lifetimes -- yet the eventual
 application of those updates prior to the teardown of the Python interpreter is required.
 
+The current value of the setting is global in scope (ie applies to all sessions, whenever created) and is always
+consulted for the creation of any data object handle to govern that handle's cleanup behavior.
+
+Python iRODS Client Settings File
+---------------------------------
+
+As of v1.1.9, Python iRODS client configuration can be saved in, and loaded from, a settings file.
+
+If the settings file exists, each of its lines contains (a) a dotted name identifying a particular configuration setting
+to be assigned within the PRC, potentially changing its runtime behavior; and (b) the specific value, in Python "repr"-style
+format, that should be assigned into it.
+
+An example follows:
+
+  data_objects.auto_close   True
+
+New dotted names may be created following the example of the one valid example created thus far,
+code:`data_objects.auto_close`, initialized in :code:`irods/client_configuration/__init__.py`. Each such name should correspond
+to a globally set value which the PRC routinely checks when performing the affected library function.
+
+The use of a settings file can be indicated, and the path to that file determined, by setting the environment variable:
+:code:`PYTHON_IRODSCLIENT_CONFIGURATION_PATH`.  If this variable is present but empty, this denotes use of a default settings
+file path of :code:~/.python-irodsclient`;  if the variable's value is of nonzero length, the value should be an absolute path
+to the settings file whose use is desired.  Also, if the variable is set, auto-load of settings will be performed, meaning
+that the act of importing :code:`irods` or any of its submodules will cause the automatic loading the settings from the
+settings file, assuming it exists.  (Failure to find the file at the indicated path will be logged as a warning.)
+
+Settings can also be saved and loaded manually using the save() and load() functions in the :code:`irods.client_configuration`
+module.  Each of these functions accepts an optional :code:`file` parameter which, if set to a non-empty string, will override
+the settings file path currently "in force" (i.e., the CONFIG_DEFAULT_PATH, as optionally overridden by the environment variable
+PYTHON_IRODSCLIENT_CONFIGURATION_PATH).
 
 Computing and Retrieving Checksums
 ----------------------------------

irods/__init__.py

@@ -1,6 +1,11 @@
 from .version import __version__
 
 import logging
+import os
+
+# This has no effect if basicConfig() was previously called.
+logging.basicConfig()
+
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.NullHandler())
 gHandler = None
@@ -50,3 +55,18 @@ def client_logging(flag=True,handler=None):
 
 PAM_AUTH_PLUGIN = 'PAM'
 PAM_AUTH_SCHEME = PAM_AUTH_PLUGIN.lower()
+
+DEFAULT_CONFIG_PATH = os.path.expanduser('~/.python_irodsclient')
+settings_path_environment_variable = 'PYTHON_IRODSCLIENT_CONFIGURATION_PATH'
+
+def get_settings_path():
+    env_var = os.environ.get(settings_path_environment_variable)
+    return DEFAULT_CONFIG_PATH if not env_var else env_var
+
+from . import client_configuration
+
+client_configuration.preserve_defaults()
+
+# If the settings path variable is not set in the environment, a value of None is passed,
+# and thus no settings file is auto-loaded.
+client_configuration.autoload(_file_to_load = os.environ.get(settings_path_environment_variable))

irods/client_configuration/__init__.py

@@ -3,16 +3,30 @@
 import copy
 import io
 import logging
+import os
 import re
 import sys
 import types
 
+# Duplicate here for convenience
+from .. import DEFAULT_CONFIG_PATH
+
 logger = logging.Logger(__name__)
 
 class iRODSConfiguration(object):
     __slots__ = ()
 
 def getter(category, setting):
+    """A programmatic way of allowing the current value of the specified setting to be
+    given indirectly (through an extra call indirection) as the default value of a parameter.
+
+    Returns a lambda that, when called, will yield the setting's value. In the closure of
+    that lambda, the Python builtin function globals() is used to access (in a read-only
+    capacity) the namespace dict of the irods.client_configuration module.
+
+    See the irods.manager.data_object_manager.DataObjectManager.open(...) method signature
+    for a usage example.
+    """
     return lambda:getattr(globals()[category], setting)
 
 # #############################################################################
@@ -48,3 +62,162 @@ def __init__(self):
 # listed in the __slots__ member of the category class.
 
 data_objects = DataObjects()
+
+def _var_items(root):
+    if isinstance(root,types.ModuleType):
+        return [(i,v) for i,v in vars(root).items()
+                if isinstance(v,iRODSConfiguration)]
+    if isinstance(root,iRODSConfiguration):
+        return [(i, getattr(root,i)) for i in root.__slots__]
+    return []
+
+def save(root = None, string='', file = ''):
+    """Save the current configuration.
+
+    When called simply as save(), this function simply writes all client settings into
+    a configuration file.
+
+    The 'root' and 'string' parameters are not likely to be overridden when called from an
+    application. They should usually only vary from the defaults when save() recurses into itself.
+    However, for due explanation's sake: 'root' specifies at which subtree node to start writing,
+    None denoting the top level; and 'string' specifies a prefix for the dotted prefix name,
+    which should be empty for an invocation that references the settings' top level namespace.
+    Both of these defaults are in effect when calling save() without explicit parameters.
+
+    The configuration file path will normally be the value of DEFAULT_CONFIG_PATH,
+    but this can be overridden by supplying a non-empty string in the 'file' parameter.
+    """
+    _file = None
+    auto_close_settings = False
+    try:
+        if not file:
+            from .. import get_settings_path
+            file = get_settings_path()
+        if isinstance(file,str):
+            _file = open(file,'w')
+            auto_close_settings = True
+        else:
+            _file = file # assume file-like object if not a string
+        if root is None:
+            root = sys.modules[__name__]
+        for k,v in _var_items(root):
+            dotted_string = string + ("." if string else "") + k
+            if isinstance(v,iRODSConfiguration):
+                save(root = v, string = dotted_string, file = _file)
+            else:
+                print(dotted_string, repr(v), sep='\t\t', file = _file)
+        return file
+    finally:
+        if _file and auto_close_settings:
+            _file.close()
+
+def _load_config_line(root, setting, value):
+    arr = [_.strip() for _ in setting.split('.')]
+    # Compute the object referred to by the dotted name.
+    attr = ''
+    for i in filter(None,arr):
+        if attr:
+            root = getattr(root,attr)
+        attr = i
+    # Assign into the current setting of the dotted name (effectively <root>.<attr>)
+    # using the loaded value.
+    if attr:
+        return setattr(root, attr, ast.literal_eval(value))
+    error_message = 'Bad setting: root = {root!r}, setting = {setting!r}, value = {value!r}'.format(**locals())
+    raise RuntimeError (error_message)
+
+# The following regular expression is used to match a configuration file line of the form:
+# ---------------------------------------------------------------
+#         <optional whitespace>
+#  key:   <dotted-name specification>
+#         <whitespace of length 1 or more>
+#  value: <A Python value which can be given to ast.literal_eval(); e.g. 5, True, or 'some_string'>
+#         <optional whitespace>
+
+_key_value_pattern = re.compile(r'\s*(?P<key>\w+(\.\w+)+)\s+(?P<value>\S.*?)\s*$')
+
+class _ConfigLoadError:
+    """
+    Exceptions that subclass this type can be thrown by the load() function if
+    their classes are listed in the failure_modes parameter of that function.
+    """
+
+class NoConfigError(Exception, _ConfigLoadError): pass
+class BadConfigError(Exception, _ConfigLoadError): pass
+
+def load(root = None, file = '', failure_modes = (), logging_level = logging.WARNING):
+    """Load the current configuration.
+
+    An example of a valid line in a configuration file is this:
+
+        data_objects.auto_close  True
+
+    When this function is called without parameters, it reads all client settings from
+    a configuration file (the path given by DEFAULT_CONFIG_PATH, since file = '' in such
+    an invocation) and assigns the repr()-style Python value given into the dotted-string
+    configuration entry given.
+
+    The 'file' parameter, when set to a non-empty string, provides an override for
+    the config-file path default.
+
+    As with save(), 'root' refers to the starting location in the settings tree, with
+    a value of None denoting the top tree node (ie the namespace containing *all* settings).
+    There are as yet no imagined use-cases for an application developer to pass in an
+    explicit 'root' override.
+
+    'failure_modes' is an iterable containing desired exception types to be thrown if,
+    for example, the configuration file is missing (NoConfigError) or contains an improperly
+    formatted line (BadConfigError).
+
+    'logging_level' governs the internally logged messages and can be used to e.g. quiet the
+    call's logging output.
+    """
+    def _existing_config(path):
+        if os.path.isfile(path):
+            return open(path,'r')
+        message = 'Config file not available at %r' % (path,)
+        logging.getLogger(__name__).log(logging_level, message)
+        if NoConfigError in failure_modes:
+            raise NoConfigError(message)
+        return io.StringIO()
+
+    _file = None
+    try:
+        if not file:
+            from .. import get_settings_path
+            file = get_settings_path()
+
+        _file = _existing_config(file)
+
+        if root is None:
+            root = sys.modules[__name__]
+
+        for line_number, line in enumerate(_file.readlines()):
+            line = line.strip()
+            match = _key_value_pattern.match(line)
+            if not match:
+                if line != '':
+                    # Log only the invalid lines that contain non-whitespace characters.
+                    message = 'Invalid configuration format at line %d: %r' % (line_number+1, line)
+                    logging.getLogger(__name__).log(logging_level, message)
+                    if BadConfigError in failure_modes:
+                        raise BadConfigError(message)
+                continue
+            _load_config_line(root, match.group('key'), match.group('value'))
+    finally:
+        if _file:
+            _file.close()
+
+default_config_dict = {}
+
+def preserve_defaults():
+    default_config_dict.update((k,copy.deepcopy(v)) for k,v in globals().items() if isinstance(v,iRODSConfiguration))
+
+def autoload(_file_to_load):
+    if _file_to_load is not None:
+        load(file = _file_to_load)
+
+def new_default_config():
+    module = types.ModuleType('_')
+    module.__dict__.update(default_config_dict)
+    return module

irods/connection.py

@@ -12,7 +12,6 @@
 from ast import literal_eval as safe_eval
 import re
 
-
 PAM_PW_ESC_PATTERN = re.compile(r'([@=&;])')
 
 

irods/test/data_obj_test.py

@@ -41,11 +41,12 @@ def is_localhost_synonym(name):
 import irods.test.helpers as helpers
 import irods.test.modules as test_modules
 import irods.keywords as kw
+import irods.client_configuration as config
 from irods.manager import data_object_manager
 from irods.message import RErrorStack
 from irods.message import ( ET, XML_Parser_Type, default_XML_parser, current_XML_parser )
 from datetime import datetime
-from tempfile import NamedTemporaryFile, mktemp
+from tempfile import NamedTemporaryFile, gettempdir
 from irods.test.helpers import (unique_name, my_function_name)
 from irods.ticket import Ticket
 import irods.parallel
@@ -1911,6 +1912,52 @@ def test_data_objects_auto_close_on_function_exit__issue_456(self):
         data_object_path, expected_content = test_module.test(return_locals = ('name','expected_content'))
         self._auto_close_test(data_object_path, expected_content)
 
+    @unittest.skipIf(helpers.configuration_file_exists(),"test would overwrite pre-existing configuration.")
+    def test_settings_save_and_autoload__issue_471(self):
+        import irods.test.modules.test_saving_and_loading_of_settings__issue_471 as test_module
+        truth = int(time.time())
+        test_output = test_module.test(truth)
+        self.assertEqual(test_output, str(truth))
+
+    def test_settings_load_and_save_471(self):
+        from irods import settings_path_environment_variable, get_settings_path, DEFAULT_CONFIG_PATH
+        settings_path = get_settings_path()
+        with helpers.file_backed_up(settings_path, require_that_file_exists = False):
+
+            RANDOM_VALUE=int(time.time())
+            config.data_objects.auto_close = RANDOM_VALUE
+
+            # Create empty settings file.
+            with open(settings_path,'w'):
+                pass
+
+            # For "silent" loading.
+            load_logging_options =  {'logging_level':logging.DEBUG}
+
+            config.load(**load_logging_options)
+
+            # Load from empty settings should change nothing.
+            self.assertTrue(config.data_objects.auto_close, RANDOM_VALUE)
+
+            os.unlink(settings_path)
+            config.load(**load_logging_options)
+            # Load from nonexistent settings file should change nothing.
+            self.assertTrue(config.data_objects.auto_close, RANDOM_VALUE)
+
+            with helpers.environment_variable_backed_up(settings_path_environment_variable):
+                os.environ.pop(settings_path_environment_variable,None)
+                tmp_path = os.path.join(gettempdir(),'.prc')
+                for i, test_path in enumerate([None, '', tmp_path]):
+                    if test_path is not None:
+                        os.environ[settings_path_environment_variable] = test_path
+                    # Check that load and save work as expected.
+                    config.data_objects.auto_close = RANDOM_VALUE - i - 1
+                    saved_path = config.save()
+                    # File path should be as expected.
+                    self.assertEqual(saved_path, (DEFAULT_CONFIG_PATH if not test_path else test_path))
+                    config.data_objects.auto_close = RANDOM_VALUE
+                    config.load(**load_logging_options)
+                    self.assertTrue(config.data_objects.auto_close, RANDOM_VALUE - i - 1)
 
 if __name__ == '__main__':
     # let the tests find the parent irods lib