Break down datasets into 2 chunks

Author: treigerm

src/climatebenchpress/data_loader/datasets/ifs_humidity.py

@@ -13,8 +13,6 @@
 from .abc import Dataset
 from .ifs_uncompressed import load_hplp_data, regrid_to_regular
 
-BASE_URL = "https://object-store.os-api.cci1.ecmwf.int/esiwacebucket"
-
 
 class IFSHumidityDataset(Dataset):
     """Dataset for the humidity field of the uncompressed IFS data.
@@ -48,7 +46,15 @@ def download(download_path: Path, progress: bool = True):
     @staticmethod
     def open(download_path: Path) -> xr.Dataset:
         ds = xr.open_dataset(download_path / "ifs_humidity.zarr")
-        ds = ds.isel(time=slice(0, 1)).chunk(-1)
+        num_levels = ds["level"].size
+        ds = ds.isel(time=slice(0, 1)).chunk(
+            {
+                "latitude": -1,
+                "longitude": -1,
+                "time": -1,
+                "level": (num_levels // 2) + 1,
+            }
+        )
 
         # Needed to make the dataset CF-compliant.
         ds.longitude.attrs["axis"] = "X"