Merge pull request #938 from davidhassell/kerchunk-read

Read Kerchunk and file-like object datasets

Author: davidhassell

Changelog.rst

@@ -3,6 +3,10 @@ Version NEXTVERSION
 
 **2026-??-??**
 
+* Read Kerchunk datasets with `cf.read`
+  (https://github.com/NCAS-CMS/cf-python/issues/936)
+* Read open file handle datasets with `cf.read`
+  (https://github.com/NCAS-CMS/cf-python/issues/937)
 * Write UGRID datasets with `cf.write`
   (https://github.com/NCAS-CMS/cf-python/issues/697)
 * Support for HEALPix grids

cf/read_write/read.py

@@ -20,8 +20,8 @@
 class read(cfdm.read):
     """Read field or domain constructs from files.
 
-    The following file formats are supported: netCDF, CDL, Zarr, PP,
-    and UM fields file.
+    The following file formats are supported: netCDF, CDL, Zarr,
+    Kerchunk, PP, and UM fields file.
 
     NetCDF and Zarr datasets may be on local disk, on an OPeNDAP
     server, or in an S3 object store.
@@ -144,7 +144,7 @@ class read(cfdm.read):
 
     :Parameters:
 
-        {{read datasets: (arbitrarily nested sequence of) `str`}}
+        {{read datasets:}}
 
         {{read recursive: `bool`, optional}}
 
@@ -162,6 +162,7 @@ class read(cfdm.read):
             ``'netCDF'``    A netCDF-3 or netCDF-4 dataset
             ``'CDL'``       A text CDL file of a netCDF dataset
             ``'Zarr'``      A Zarr v2 (xarray) or Zarr v3 dataset
+            ``'Kerchunk'``  A Kerchunked dataset
             ``'UM'``        A UM fields file or PP dataset
             ==============  ==========================================
 

cf/read_write/um/umread.py

@@ -3549,6 +3549,13 @@ def read(
                 "(only Field constructs)"
             )
 
+        representation = self.dataset_representation(dataset)
+        if representation != "path":
+            raise NotImplementedError(
+                "Can't yet read Field constructs from a UM or PP "
+                f"{representation!r} dataset: {dataset!r}"
+            )
+
         if not _stash2standard_name:
             # --------------------------------------------------------
             # Create the STASH code to standard_name conversion
@@ -3605,75 +3612,17 @@ def read(
                 # Return now if there are valid file types
                 return []
 
-        # Parse the 'storage_options' keyword parameter
-        if storage_options is None:
-            storage_options = {}
-        elif filesystem is not None:
-            raise ValueError(
-                "Can't set both storage_options and filesystem keywords"
+        if storage_options is not None:
+            raise NotImplementedError(
+                "Can't yet open PP/UM files with file system storage options"
             )
 
-        storage_protocol = None
-
         if filesystem is not None:
-            # --------------------------------------------------------
-            # A pre-authenticated filesystem was provided: open the
-            # dataset as a file-like object and pass it to the backend.
-            # --------------------------------------------------------
             raise NotImplementedError(
-                "Can't yet open PP/UM files from a remote file system"
+                "Can't yet open PP/UM files from a pre-defined file system"
             )
 
-            try:
-                dataset = filesystem.open(dataset, "rb")
-            except AttributeError:
-                raise AttributeError(
-                    f"The 'filesystem' object {filesystem!r} does not have "
-                    "an 'open' method. Please provide a valid filesystem "
-                    "object (e.g. an fsspec filesystem instance)."
-                )
-            except Exception as exc:
-                raise OSError(
-                    f"Failed to open {dataset!r} using the provided "
-                    f"'filesystem' object {filesystem!r}: {exc}"
-                ) from exc
-
-        else:
-            from uritools import urisplit
-
-            u = urisplit(dataset)
-            if u.scheme == "s3":
-                # ----------------------------------------------------
-                # Dataset is an s3://... string.
-                # ----------------------------------------------------
-                raise NotImplementedError(
-                    "Can't yet open PP/UM files from an s3 object store"
-                )
-
-                import fsspec
-
-                client_kwargs = storage_options.get("client_kwargs", {})
-                if (
-                    "endpoint_url" not in storage_options
-                    and "endpoint_url" not in client_kwargs
-                ):
-                    authority = u.authority
-                    if not authority:
-                        authority = ""
-
-                    storage_options["endpoint_url"] = f"https://{authority}"
-
-                filesystem = fsspec.filesystem(
-                    protocol=u.scheme, **storage_options
-                )
-                dataset = filesystem.open(u.path[1:], "rb")
-
-        if not storage_options:
-            storage_options = None
-
-        if filesystem is not None:
-            storage_protocol = filesystem.protocol
-            storage_options = filesystem.storage_options
+        storage_protocol = None
 
         f = self.dataset_open(dataset, parse=True)
 
@@ -3835,6 +3784,41 @@ def dataset_open(self, filename, parse=True):
             parse=parse,
         )
 
+    @classmethod
+    def dataset_representation(cls, dataset):
+        """Return the logical representation type of the input dataset.
+
+        .. versionadded:: NEXTVERSION
+
+        :Parameters:
+
+            dataset:
+                The dataset. May be a string-valued path or a
+                file-like object.
+
+        :Returns:
+
+            `str`
+                The dataset representation:
+
+                * ``'path'``: A string-valued path.
+
+                * ``'file_handle'``: An open file handle (such as
+                  returned by `fsspec.filesystem.open`)
+
+                * ``'unknown'``: Anything else.
+
+        """
+        # Strings (Paths)
+        if isinstance(dataset, str):
+            return "path"
+
+        # Check for a "binary stream" (file handle)
+        if hasattr(dataset, "read") and hasattr(dataset, "seek"):
+            return "file_handle"
+
+        return "unknown"
+
 
 """
 Problems:

cf/test/example_field_0.kerchunk

@@ -0,0 +1 @@
+{"version":1,"refs":{"lat\/0":"base64:eF5jYMABDjQwNBwIcmNwCzpwgMHBAQAxqAWx","lat_bnds\/0.0":"base64:eF5jYCAVNMAZYX5+dkDg5xd2AAQcQAAAZ+II3Q==","lon\/0":"base64:eF5jYCAONDxQ2BDwwUPCLCAmOTOvqNQBCgB9ngjU","lon_bnds\/0.0":"base64:eF5jYKAyaACCBw8aGhQUHjzYsKGBwc0tLCwhIS0tJ6egoLi4jMEBFQAA\/noSOQ==","q\/0.0":["example_field_0.nc",17755,100],"q\/0.1":"base64:eF6z7v4lGfX4l4\/QxJO77ufUsaQsrTNvflvm6f88K8x6dlbngZp9PL2\/9\/DdP7LH8ZT9fKbpK0t0\/RVKEhaaSmnnfZy8qXfD5KeucfrtkxbNmLt8U8WGbbvWttkjAQAjCS4Q","q\/1.0":"base64:eF6TzNB+88v69hsGIGCZ2B5Yd\/9nIIgd9lZsR5Z\/3Q4Qm6\/mptye3iw5EFvX\/lxryfQ9rSD2JtPvryfnlbwGseeuW7yxYtHkhfOAwB4K3IEAAKyvI6g=","q\/1.1":"base64:eF6TtLaeY909x5IBCFju3z9yP+fIFhA7zN9fwf+5wgkQm6+3d0Pv7w1lILbu9OlF01cW7QOxN+XlZeR9zJgPYq\/d1DRx0qKFXfOAwB4K3IEAAGAXIr4=","time\/0":"\u0000\u0000\u0000\u0000\u0000\u0000?@",".zgroup":"{\"zarr_format\":2}",".zattrs":"{\"Conventions\":\"CF-1.12\"}","lat\/.zarray":"{\"shape\":[5],\"chunks\":[5],\"dtype\":\"<f8\",\"fill_value\":null,\"order\":\"C\",\"filters\":[{\"id\":\"shuffle\",\"elementsize\":8},{\"id\":\"zlib\",\"level\":4}],\"dimension_separator\":\".\",\"compressor\":null,\"zarr_format\":2}","lat\/.zattrs":"{\"_ARRAY_DIMENSIONS\":[\"lat\"],\"units\":\"degrees_north\",\"standard_name\":\"latitude\",\"bounds\":\"lat_bnds\"}","lat_bnds\/.zarray":"{\"shape\":[5,2],\"chunks\":[5,2],\"dtype\":\"<f8\",\"fill_value\":null,\"order\":\"C\",\"filters\":[{\"id\":\"shuffle\",\"elementsize\":8},{\"id\":\"zlib\",\"level\":4}],\"dimension_separator\":\".\",\"compressor\":null,\"zarr_format\":2}","lat_bnds\/.zattrs":"{\"_ARRAY_DIMENSIONS\":[\"lat\",\"bounds2\"]}","lon\/.zarray":"{\"shape\":[8],\"chunks\":[8],\"dtype\":\"<f8\",\"fill_value\":null,\"order\":\"C\",\"filters\":[{\"id\":\"shuffle\",\"elementsize\":8},{\"id\":\"zlib\",\"level\":4}],\"dimension_separator\":\".\",\"compressor\":null,\"zarr_format\":2}","lon\/.zattrs":"{\"_ARRAY_DIMENSIONS\":[\"lon\"],\"units\":\"degrees_east\",\"standard_name\":\"longitude\",\"bounds\":\"lon_bnds\"}","lon_bnds\/.zarray":"{\"shape\":[8,2],\"chunks\":[8,2],\"dtype\":\"<f8\",\"fill_value\":null,\"order\":\"C\",\"filters\":[{\"id\":\"shuffle\",\"elementsize\":8},{\"id\":\"zlib\",\"level\":4}],\"dimension_separator\":\".\",\"compressor\":null,\"zarr_format\":2}","lon_bnds\/.zattrs":"{\"_ARRAY_DIMENSIONS\":[\"lon\",\"bounds2\"]}","q\/.zarray":"{\"shape\":[5,8],\"chunks\":[3,4],\"dtype\":\"<f8\",\"fill_value\":null,\"order\":\"C\",\"filters\":[{\"id\":\"shuffle\",\"elementsize\":8},{\"id\":\"zlib\",\"level\":4}],\"dimension_separator\":\".\",\"compressor\":null,\"zarr_format\":2}","q\/.zattrs":"{\"_ARRAY_DIMENSIONS\":[\"lat\",\"lon\"],\"project\":\"research\",\"standard_name\":\"specific_humidity\",\"units\":\"1\",\"coordinates\":\"time\",\"cell_methods\":\"area: mean\"}","time\/.zarray":"{\"shape\":[],\"chunks\":[],\"dtype\":\"<f8\",\"fill_value\":null,\"order\":\"C\",\"filters\":null,\"dimension_separator\":\".\",\"compressor\":null,\"zarr_format\":2}","time\/.zattrs":"{\"_ARRAY_DIMENSIONS\":[],\"units\":\"days since 2018-12-01\",\"standard_name\":\"time\"}"}}

cf/test/test_kerchunk.py

@@ -0,0 +1,99 @@
+import datetime
+import faulthandler
+import json
+import os
+import unittest
+
+import fsspec
+
+faulthandler.enable()  # to debug seg faults and timeouts
+
+
+import cf
+
+warnings = False
+
+
+kerchunk_file = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "example_field_0.kerchunk"
+)
+
+fs = fsspec.filesystem("reference", fo=kerchunk_file)
+kerchunk_mapper = fs.get_mapper()
+
+
+class read_writeTest(unittest.TestCase):
+    """Test the reading and writing of field constructs from/to disk."""
+
+    netcdf = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), "example_field_0.nc"
+    )
+    kerchunk = kerchunk_mapper
+
+    def setUp(self):
+        """Preparations called immediately before each test method."""
+        # Disable log messages to silence expected warnings
+        cf.LOG_LEVEL("DISABLE")
+        # Note: to enable all messages for given methods, lines or
+        # calls (those without a 'verbose' option to do the same)
+        # e.g. to debug them, wrap them (for methods, start-to-end
+        # internally) as follows: cf.LOG_LEVEL('DEBUG')
+        #
+        # < ... test code ... >
+        # cf.log_level('DISABLE')
+
+    def test_kerchunk_read(self):
+        """Test cf.read with Kerchunk."""
+        f = cf.read(self.netcdf)[0]
+
+        k = cf.read(self.kerchunk, dask_chunks=3)
+        self.assertEqual(len(k), 1)
+        self.assertTrue(k[0].equals(f))
+        self.assertGreater(k[0].data.npartitions, 1)
+
+        k = cf.read([self.kerchunk, self.kerchunk], dask_chunks=3)
+        self.assertEqual(len(k), 2)
+        self.assertTrue(k[0].equals(k[-1]))
+
+        k = cf.read([self.kerchunk, self.kerchunk, self.netcdf], dask_chunks=3)
+        self.assertEqual(len(k), 3)
+        self.assertTrue(k[0].equals(k[-1]))
+        self.assertTrue(k[1].equals(k[-1]))
+
+    def test_kerchunk_original_filenames(self):
+        """Test original_filenames with Kerchunk."""
+        k = cf.read(self.kerchunk)[0]
+        self.assertEqual(k.get_original_filenames(), set())
+
+    def test_read_dict(self):
+        """Test cf.read with an Kerchunk dictionary."""
+        with open(kerchunk_file, "r") as fh:
+            d = json.load(fh)
+
+        with self.assertRaises(ValueError):
+            cf.read(d)
+
+        fs = fsspec.filesystem("reference", fo=d)
+        kerchunk = fs.get_mapper()
+        self.assertEqual(len(cf.read(kerchunk)), 1)
+
+    def test_read_bytes(self):
+        """Test cf.read with a Kerchunk raw bytes representation."""
+        with open(kerchunk_file, "r") as fh:
+            d = json.load(fh)
+
+        b = json.dumps(d).encode("utf-8")
+        with self.assertRaises(ValueError):
+            cf.read(b)
+
+        d = json.loads(b)
+        fs = fsspec.filesystem("reference", fo=d)
+        kerchunk = fs.get_mapper()
+        self.assertEqual(len(cf.read(kerchunk)), 1)
+
+
+if __name__ == "__main__":
+    print("Run date:", datetime.datetime.now())
+    cf.environment()
+    print("")
+    unittest.main(verbosity=2)

cf/test/test_pp.py

@@ -145,6 +145,15 @@ def test_PP_um_version(self):
         f = cf.read(self.ppfile, um={"version": "6.6.3"})[0]
         self.assertEqual(f.get_property("um_version"), "6.6.3")
 
+    def test_PP_file_object(self):
+        # Can't yet read PP/UM from file-like objects
+        with open(self.ppfile, "rb") as fh:
+            with self.assertRaises(NotImplementedError):
+                cf.read(fh)
+
+            # Check that the file has been rewound
+            self.assertEqual(fh.tell(), 0)
+
 
 if __name__ == "__main__":
     print("Run date:", datetime.datetime.now())

docs/source/introduction.rst

@@ -67,8 +67,8 @@ may nonetheless be modified in memory.
 The `cf` package can:
 
 * read :term:`field constructs <field construct>` and :term:`domain
-  constructs <domain construct>` from netCDF, CDL, Zarr, PP and UM
-  datasets with a choice of netCDF backends,
+  constructs <domain construct>` from netCDF, CDL, Zarr, Kerchunk, PP
+  and UM datasets with a choice of netCDF backends,
 
 * read files from OPeNDAP servers and S3 object stores,
 

docs/source/tutorial.rst

@@ -139,15 +139,15 @@ The following file types can be read:
 
 ..
 
-* `CFA-netCDF
-  <https://github.com/NCAS-CMS/cfa-conventions/blob/master/source/cfa.md>`_
-  files at version 0.6 or later.
+* Datasets in `Kerchunk <https://fsspec.github.io/kerchunk>`_ format.
 
 ..
 
 * :ref:`PP and UM fields files <PP-and-UM-fields-files>`, whose
   contents are mapped into field constructs.
 
+..
+
 Note that when reading netCDF4 files that contain :ref:`hierachical
 groups <Hierarchical-groups>`, the group structure is saved via the
 :ref:`netCDF interface <NetCDF-interface>` so that it may be re-used,

requirements.txt

@@ -1,4 +1,4 @@
-netCDF4==1.7.2
+netCDF4>=1.7.2
 cftime>=1.6.4
 numpy>=2.0.0
 cfdm>=1.13.0.0, <1.13.1.0

setup.py

@@ -178,7 +178,7 @@ def compile():
 
 The ``cf`` package can:
 
-* read field and domain constructs from netCDF, CDL, Zarr, PP and UM datasets,
+* read field and domain constructs from netCDF, CDL, Zarr, Kerchunk, PP and UM datasets,
 
 * be fully flexible with respect to dataset storage chunking,