Merge branch 'develop'

Author: RKrahl

CHANGES.rst

@@ -2,6 +2,20 @@ Changelog
 =========
 
 
+.. _changes-1_6_0:
+
+1.6.0 (2024-12-06)
+~~~~~~~~~~~~~~~~~~
+
+New features
+------------
+
++ `#166`_: Add new custom IDS method
+  :meth:`icat.client.Client.restoreData`.
+
+.. _#166: https://github.com/icatproject/python-icat/pull/166
+
+
 .. _changes-1_5_1:
 
 1.5.1 (2024-10-25)

doc/src/client.rst

@@ -154,4 +154,6 @@ manages the interaction with an ICAT service as a client.
 
     .. automethod:: deleteData
 
+    .. automethod:: restoreData
+
 .. _ICAT SOAP Manual: https://repo.icatproject.org/site/icat/server/4.10.0/soap.html

src/icat/client.py

@@ -1000,5 +1000,40 @@ def deleteData(self, objs):
             objs = DataSelection(objs)
         self.ids.delete(objs)
 
+    def restoreData(self, objs):
+        """Request IDS to restore data.
+
+        Check the status of the data, request a restore if needed and
+        wait for the restore to complete.
+
+        :param objs: either a dict having some of the keys
+            `investigationIds`, `datasetIds`, and `datafileIds`
+            with a list of object ids as value respectively, or a list
+            of entity objects, or a data selection.
+        :type objs: :class:`dict`, :class:`list` of
+            :class:`icat.entity.Entity`, or
+            :class:`icat.ids.DataSelection`
+
+        .. versionadded:: 1.6.0
+        """
+        if not self.ids:
+            raise RuntimeError("no IDS.")
+        if not isinstance(objs, DataSelection):
+            objs = DataSelection(objs)
+        while True:
+            self.autoRefresh()
+            status = self.ids.getStatus(objs)
+            if status == "ONLINE":
+                break
+            elif status == "RESTORING":
+                pass
+            elif status == "ARCHIVED":
+                self.ids.restore(objs)
+            else:
+                # Should never happen
+                raise IDSResponseError("unexpected response from "
+                                       "IDS getStatus() call: %s" % status)
+            time.sleep(30)
+
 
 atexit.register(Client.cleanupall)

tests/conftest.py

@@ -48,9 +48,28 @@
 logging.getLogger('suds.client').setLevel(logging.CRITICAL)
 logging.getLogger('suds').setLevel(logging.ERROR)
 
+_skip_slow = True
 testdir = Path(__file__).resolve().parent
 testdatadir = testdir / "data"
 
+def pytest_addoption(parser):
+    parser.addoption("--no-skip-slow", action="store_true", default=False,
+                     help="do not skip slow tests.")
+
+def pytest_configure(config):
+    global _skip_slow
+    _skip_slow = not config.getoption("--no-skip-slow")
+    config.addinivalue_line("markers", "slow: mark a test as slow, "
+                            "the test will be skipped unless --no-skip-slow "
+                            "is set on the command line")
+
+def pytest_runtest_setup(item):
+    """Skip slow tests by default.
+    """
+    marker = item.get_closest_marker("slow")
+    if marker is not None and _skip_slow:
+        pytest.skip("skip slow test")
+
 def _skip(reason):
     if Version(pytest.__version__) >= '3.3.0':
         pytest.skip(reason, allow_module_level=True)

tests/test_06_ids.py

@@ -3,23 +3,31 @@
 
 import datetime
 import filecmp
+import logging
 import time
 import zipfile
 import pytest
 import icat
+import icat.client
 import icat.config
-from icat.ids import DataSelection
+from icat.ids import IDSClient, DataSelection
 from icat.query import Query
 from conftest import DummyDatafile, UtcTimezone
 from conftest import getConfig, tmpSessionId, tmpClient
 
+logger = logging.getLogger(__name__)
 
-@pytest.fixture(scope="module")
-def client(setupicat):
-    client, conf = getConfig(ids="mandatory")
-    client.login(conf.auth, conf.credentials)
-    yield client
-    query = "SELECT df FROM Datafile df WHERE df.location IS NOT NULL"
+GiB = 1073741824
+
+class LoggingIDSClient(IDSClient):
+    """Modified version of IDSClient that logs some calls.
+    """
+    def getStatus(self, selection):
+        status = super().getStatus(selection)
+        logger.debug("getStatus(%s): %s", selection, status, stacklevel=2)
+        return status
+
+def _delete_datafiles(client, query):
     while True:
         try:
             client.deleteData(client.search(query))
@@ -28,6 +36,45 @@ def client(setupicat):
         else:
             break
 
+@pytest.fixture(scope="module")
+def cleanup(setupicat):
+    client, conf = getConfig(confSection="root", ids="mandatory")
+    client.login(conf.auth, conf.credentials)
+    yield
+    query = "SELECT df FROM Datafile df WHERE df.location IS NOT NULL"
+    _delete_datafiles(client, query)
+
+@pytest.fixture(scope="function")
+def client(monkeypatch, cleanup):
+    monkeypatch.setattr(icat.client, "IDSClient", LoggingIDSClient)
+    client, conf = getConfig(ids="mandatory")
+    client.login(conf.auth, conf.credentials)
+    yield client
+
+@pytest.fixture(scope="function")
+def dataset(client, cleanup_objs):
+    """A dataset to be used in the test.
+
+    The dataset will be eventually be deleted after the test.
+    """
+    inv = client.assertedSearch(Query(client, "Investigation", conditions={
+        "name": "= '10100601-ST'",
+    }))[0]
+    dstype = client.assertedSearch(Query(client, "DatasetType", conditions={
+        "name": "= 'raw'",
+    }))[0]
+    dataset = client.new("Dataset",
+                         name="e208343", complete=False,
+                         investigation=inv, type=dstype)
+    dataset.create()
+    cleanup_objs.append(dataset)
+    yield dataset
+    query = Query(client, "Datafile", conditions={
+        "dataset.id": "= %d" % dataset.id,
+        "location": "IS NOT NULL",
+    })
+    _delete_datafiles(client, query)
+
 
 # ============================ testdata ============================
 
@@ -392,6 +439,55 @@ def test_restore(client, case):
     # outcome of the restore() call.
     print("Status of dataset %s is now %s" % (case['dsname'], status))
 
+@pytest.mark.parametrize(("case"), markeddatasets)
+def test_restoreDataCall(client, case):
+    """Test the high level call restoreData().
+
+    This is essentially a no-op as the dataset in question will
+    already be ONLINE.  It only tests that the call does not throw an
+    error.
+    """
+    dataset = getDataset(client, case)
+    client.restoreData([dataset])
+    status = client.ids.getStatus(DataSelection([dataset]))
+    assert status == "ONLINE"
+
+@pytest.mark.parametrize(("case"), markeddatasets)
+def test_restoreDataCallSelection(client, case):
+    """Test the high level call restoreData().
+
+    Same as last test, but now pass a DataSelection as argument.
+    """
+    selection = DataSelection([getDataset(client, case)])
+    client.restoreData(selection)
+    status = client.ids.getStatus(selection)
+    assert status == "ONLINE"
+
+@pytest.mark.slow
+def test_restoreData(tmpdirsec, client, dataset):
+    """Test restoring data with the high level call restoreData().
+
+    This test archives a dataset and calls restoreData() to restore it
+    again.  The size of the dataset is large enough so that restoring
+    takes some time, so that we actually can observe the call to wait
+    until the restoring is finished.  As a result, the test is rather
+    slow.  It is marked as such and thus disabled by default.
+    """
+    if not client.ids.isTwoLevel():
+        pytest.skip("This IDS does not use two levels of data storage")
+    f = DummyDatafile(tmpdirsec, "e208343.nxs", GiB)
+    query = Query(client, "DatafileFormat", conditions={
+        "name": "= 'NeXus'",
+    })
+    datafileformat = client.assertedSearch(query)[0]
+    datafile = client.new("Datafile", name=f.fname.name,
+                          dataset=dataset, datafileFormat=datafileformat)
+    client.putData(f.fname, datafile)
+    client.ids.archive(DataSelection([dataset]))
+    client.restoreData([dataset])
+    status = client.ids.getStatus(DataSelection([dataset]))
+    assert status == "ONLINE"
+
 @pytest.mark.parametrize(("case"), markeddatasets)
 def test_reset(client, case):
     """Call reset() on a dataset.