diff --git a/.conda/env_dev.yml b/.conda/env_dev.yml new file mode 100644 index 0000000..9580206 --- /dev/null +++ b/.conda/env_dev.yml @@ -0,0 +1,16 @@ +channels: + - accessnri + - conda-forge + - coecms + - nodefaults + +dependencies: + - mule + - numpy <= 1.23.4 # https://stackoverflow.com/a/75148219/21024780 + - scitools-iris + - xarray + - versioneer + - pytest + - pytest-cov + - pytest-xdist + - hypothesis \ No newline at end of file diff --git a/.gitignore b/.gitignore index 96ca83b..97b8d67 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ test_data/ .coverage -*.sh __pycache__/ *.py[cod] .ruff_cache/ diff --git a/README.md b/README.md index 18318dd..8777daf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,48 @@ -[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) - # replace_landsurface -This repository contains Python scripts for use by Regional Nesting Suites to replace specific land surface fields in static input data. +## About + +`replace_landsurface` is a `Python` utility to be used within ACCESS-NRI versions of the Regional Nesting Suites to replace specific land surface initial/boundary conditions. + + +## Development/Testing instructions +For development/testing, it is recommended to install `replace_landsurface` as a development package within a `micromamba`/`conda` testing environment. + +### Clone replace_landsurface GitHub repo +``` +git clone git@github.com:ACCESS-NRI/replace_landsurface.git +``` + +### Create a micromamba/conda testing environment +> [!TIP] +> In the following instructions `micromamba` can be replaced with `conda`. + +``` +cd replace_landsurface +micromamba env create -n replace_landsurface_dev --file .conda/env_dev.yml +micromamba activate replace_landsurface_dev +``` + +### Install replace_landsurface as a development package +``` +pip install --no-deps --no-build-isolation -e . +``` + +### Running the tests + +The test suite currently includes only integration tests. + +To manually run the tests, from the `replace_landsurface` directory, you can: + +1. Activate your [micromamba/conda testing environment](#create-a-micromamba-conda-testing-environment) +2. Run the following command: + ``` + pytest -n 4 + ``` + +> [!TIP] +> The `-n 4` option is a [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/) option to run the tests in parallel across 4 different workers. + +> [!IMPORTANT] +> Integration tests are designed to be run on `Gadi`. +> If you run tests on a local machine, the integration tests will be skipped. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4b36a01..0449a71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,7 @@ dependencies = [ Repository = "https://github.com/ACCESS-NRI/replace_landsurface" [project.scripts] -hres_eccb = "replace_landsurface.hres_eccb:main" -hres_ic = "replace_landsurface.hres_ic:main" +replace_landsurface = "replace_landsurface.replace_landsurface:main" [build-system] build-backend = "setuptools.build_meta" diff --git a/src/replace_landsurface/hres_eccb.py b/src/replace_landsurface/hres_eccb.py deleted file mode 100755 index b792da1..0000000 --- a/src/replace_landsurface/hres_eccb.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2024 ACCESS-NRI (https://www.access-nri.org.au/) -# See the top-level COPYRIGHT.txt file for details. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Created by: Chermelle Engel - -""" -Replace the land/surface fields in the ec_cb000 file with higher-resolution -era5-land or BARRA2-R data (if requested). -""" - -import argparse -import shutil -from pathlib import Path - -import pandas - -from replace_landsurface import replace_landsurface_with_BARRA2R_IC, replace_landsurface_with_ERA5land_IC - -def main(): - """ - The main function that creates a worker pool and generates single GRIB files - for requested date/times in parallel. - - Parameters - ---------- - None. The arguments are given via the command-line - - Returns - ------- - None. The ec_cb000 file is updated and overwritten - """ - - # Parse the command-line arguments - parser = argparse.ArgumentParser() - parser.add_argument('--mask', required=True, type=Path) - parser.add_argument('--file', required=True, type=Path) - parser.add_argument('--start', required=True, type=pandas.to_datetime) - parser.add_argument('--type', default="era5land") - parser.add_argument('--hres_ic', type=Path) - args = parser.parse_args() - print(args) - - # Convert the date/time to a formatted string - t = args.start.strftime("%Y%m%dT%H%MZ") - print(args.mask, args.file, t) - - # If necessary replace ERA5 land/surface fields with higher-resolution options - if "era5land" in args.type: - replace_landsurface_with_ERA5land_IC.swap_land_era5land(args.mask, args.file, t) - shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) - elif "barra" in args.type: - replace_landsurface_with_BARRA2R_IC.swap_land_barra(args.mask, args.file, t) - shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) - elif "astart" in args.type: - print("Fields not swapped out for ECCB files when using start dump as replacement option.") - else: - print("No need to swap out IC") - -if __name__ == '__main__': - main() - diff --git a/src/replace_landsurface/hres_ic.py b/src/replace_landsurface/replace_landsurface.py similarity index 84% rename from src/replace_landsurface/hres_ic.py rename to src/replace_landsurface/replace_landsurface.py index d29ce6e..5604d9b 100755 --- a/src/replace_landsurface/hres_ic.py +++ b/src/replace_landsurface/replace_landsurface.py @@ -6,8 +6,7 @@ # Created by: Chermelle Engel """ -Replace the land/surface fields in the astart file with higher-resolution -era5-land or BARRA2-R data (if requested). +Replace the land-surface fields in the astart file with higher-resolution data """ import argparse @@ -25,16 +24,17 @@ def main(): """ - The main function that creates a worker pool and generates single GRIB files - for requested date/times in parallel. + Calls the command line argument parser and process the arguments using the right function. Parameters ---------- - None. The arguments are given via the command-line + None + The arguments are given via the command-line Returns ------- - None. The astart file is updated and overwritten + None + An output file is written """ # Parse the command-line arguments diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py new file mode 100644 index 0000000..db238e5 --- /dev/null +++ b/tests/integration/test_integration.py @@ -0,0 +1,146 @@ +import contextlib +import filecmp +import os +import shutil +import socket +from unittest.mock import patch +import pytest + +# If not on Gadi, skip the tests because the test data is not available +GADI_HOSTNAME = "gadi.nci.org.au" +hostname = socket.gethostname() +# Marker to skip tests if not on Gadi +skip_marker = pytest.mark.skipif( + not hostname.endswith(GADI_HOSTNAME), + reason=f"Skipping integration tests because they cannot be executed on {hostname}.\n" + "Integration tests are specifically designed to run on Gadi (gadi.nci.org.au).", +) +# Marker to suppress warnings +warning_marker = pytest.mark.filterwarnings("ignore::Warning") +# Apply the markers to all tests in this file +pytestmark = [skip_marker, warning_marker] + +############################################ +## === Integration tests setup === ## +############################################ +TEST_DATA_DIR = "/g/data/vk83/testing/data/replace_landsurface/integration_tests" +INPUT_DIR = os.path.join(TEST_DATA_DIR, "input_data") +OUTPUT_DIR = os.path.join(TEST_DATA_DIR, "expected_outputs") +DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") +# Set the ROSE_DATA environment variable to the driving data directory +os.environ["ROSE_DATA"] = str(DRIVING_DATA_DIR) +from replace_landsurface import replace_landsurface # importing here because we need to set the ROSE_DATA env variable before importing # noqa + + +############################################ +## === Integration tests === ## +############################################ +def get_test_args(num, start, _type): + test_dir = f"test_{num}" + hres_ic = ( + os.path.join(INPUT_DIR, test_dir, "hres_ic") + if _type == "astart" + else "NOT_USED" + ) + return [ + "script_name", + "--file", + os.path.join(INPUT_DIR, test_dir, "file" + ".tmp"), + "--mask", + os.path.join(INPUT_DIR, test_dir, "mask"), + "--start", + start, + "--type", + _type, + "--hres_ic", + hres_ic, + ] + + +def get_error_msg(num, output, expected_output): + return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" + + +@pytest.fixture +def mock_sys_argv(): + @contextlib.contextmanager + def _mock_sys_argv(num, start, _type): + with patch("sys.argv", get_test_args(num, start, _type)): + yield mock_sys_argv + + return _mock_sys_argv + + +@pytest.fixture(scope="module") +def working_dir(tmp_path_factory): + return tmp_path_factory.mktemp("replace_landsurface_integration_tests") + + +@pytest.fixture() +def get_output_path(working_dir): + def _get_output_path(num): + return os.path.join(working_dir, f"output_{num}") + + return _get_output_path + + +@pytest.fixture() +def get_expected_output_path(): + def _get_expected_output_path(num): + return os.path.join(OUTPUT_DIR, f"output_{num}") + + return _get_expected_output_path + +@pytest.fixture(scope="module") +def original_shutil_move(): + return shutil.move + + +@pytest.fixture() +def new_shutil_move(original_shutil_move, get_output_path): + def _new_shutil_move(num): + def _wrapper(src, dst, **kwargs): + output_path = get_output_path(num) + return original_shutil_move(src=src, dst=output_path, **kwargs) + + return _wrapper + + return _new_shutil_move + + +@pytest.mark.parametrize( + "num, start, _type", + [ + (1, "202202260000", "era5land"), + (2, "202008090000", "barra"), + (3, "202112310000", "astart"), + (4, "202305040000", "era5land"), + ], + ids=[ + "replace_landsurface_era5land", + "replace_landsurface_barra", + "replace_landsurface_astart", + "replace_landsurface_era5land_2", + ], +) +def test_replace_landsurface( + new_shutil_move, + get_output_path, + get_expected_output_path, + mock_sys_argv, + num, + start, + _type, +): + """ + Test the replace_landsurface entry point + """ + with mock_sys_argv(num, start, _type): + with patch("shutil.move", side_effect=new_shutil_move(num)): + replace_landsurface.main() + output = get_output_path(num) + expected_output = get_expected_output_path(num) + # Compare the output file with the expected output + assert filecmp.cmp(output, expected_output), get_error_msg( + num, output, expected_output + ) \ No newline at end of file