From c9deb3b31312191b7844271616b216931d1e345b Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 14 Feb 2025 15:10:47 +1100 Subject: [PATCH 01/29] Added integration test script Added integration tests for hres_ic Added development environment --- .conda/env_dev.yml | 14 ++ .gitignore | 1 - integration/INTEGRATION_README.md | 80 ++++++++++ integration/integration_tests.sh | 142 ++++++++++++++++++ .../replace_landsurface_with_ERA5land_IC.py | 4 - 5 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 .conda/env_dev.yml create mode 100644 integration/INTEGRATION_README.md create mode 100755 integration/integration_tests.sh diff --git a/.conda/env_dev.yml b/.conda/env_dev.yml new file mode 100644 index 0000000..38fbb06 --- /dev/null +++ b/.conda/env_dev.yml @@ -0,0 +1,14 @@ +channels: + - accessnri + - conda-forge + - coecms + - nodefaults + +dependencies: + - mule + - numpy <= 1.23.4 # https://stackoverflow.com/a/75148219/21024780 + - scitools-iris + - xarray + - versioneer + - pytest + - hypothesis \ No newline at end of file diff --git a/.gitignore b/.gitignore index 96ca83b..97b8d67 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ test_data/ .coverage -*.sh __pycache__/ *.py[cod] .ruff_cache/ diff --git a/integration/INTEGRATION_README.md b/integration/INTEGRATION_README.md new file mode 100644 index 0000000..bdfb3f9 --- /dev/null +++ b/integration/INTEGRATION_README.md @@ -0,0 +1,80 @@ +# Integration tests for replace_landsurface + +`integration_tests.sh` is a basic binary compatibility test script for replace_landsurface, +that compares the outputs of a specific version of the `replace_landsurface` package with their expected outputs. +The script warns the user if the outputs do not match their respective expected versions. + +The tests are designed to be run on Gadi within a Python environment where `replace_landsurface` is installed as a development package (for instructions refer to the Installation paragraph in the README). + + + + +## Data choices +Three types of reference data are available for use in the tests, called "full", +"intermediate", and "light". Each group of data contains a fields file, and +netCDF files produced from converting the fields file using various `um2nc` options. + +### Full +The "full" fields file is an output file from an ESM1.5 simulation. +Running tests with the "full" data is slower and more resource-intensive. + +### Intermediate (default) +The "intermediate" data contains a fields file, generated as a subset of the +the variables from the "full" data. These variables were selected to ensure +different portions of code within `um2nc` are used in the integration tests. +The included variables are: + +* m01s00i024 Surface temperature (a simple 2D field) +* m01s00i407 Pressure on model rho levels +* m01s00i408 Pressure on model theta levels +* m01s02i288 Aerosol optical thickness from biomass burning (a variable on pseudo_levels) +* m01s03i209 Eastward wind (tests hardcoded variable name changes) +* m01s03i321 Canopy water on tiles (a tiled variable) +* m01s05i216 Precipitation (a simple 2D field) +* m01s08i208 Soil moisture (land only data) +* m01s08i223 Soil moisture on soil levels +* m01s30i204 Temperature on PLEV grid (requires masking) +* m01s30i301 Heaviside (used for masking) + +### Light +The "light" data contains a minimal subset of variables from the "full" data +fields file, and can be used for faster but less in-depth testing. It includes: + +* m01s30i204 Temperature on PLEV grid +* m01s05i216 Precipitation + +### Comparison data +For each of the above data choices, the following netCDF variants were produced: + +* `mask`: produced with the `--nohist` flag only. +* `nomask`: produced with the `--nomask` and `--nohist` flags. +* `hist`: produced with no flags. These files will have a conversion datestamp +in their history attribute. + +## Data versions +The `um2nc` version to compare against can be selected with the `-v` flag. +If omitted, the tests will be performed against the latest released version. + +Available versions for comparison are: +* 0 + +### Version `0` +Version `0` netCDF outputs were created using the `um2netcdf.py` script available +prior to the development of `um2nc`: https://github.com/ACCESS-NRI/um2nc-standalone/blob/f62105b45eb39d2beed5a7ac71f439ff90f0f00c/src/um2netcdf.py + +The conversion was performed within the following `payu1.1.5` environment, active on Gadi: +https://github.com/ACCESS-NRI/payu-condaenv/releases/tag/1.1.5 + +All test data is located in `/g/data/vk83/testing/data/um2nc/integration-tests`. diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh new file mode 100755 index 0000000..78e6252 --- /dev/null +++ b/integration/integration_tests.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +# Basic binary compatibility test script for replace_landsurface. +# See INTEGRATION_README.md for details on test usage, data and options. + +TEST_DATA_DIR=/g/data/tm70/dm5220/projects/replace_landsurface/test_data +INPUT_DIR=$TEST_DATA_DIR/input_data +OUTPUT_DIR=$TEST_DATA_DIR/expected_outputs +DRIVING_DATA_DIR=$TEST_DATA_DIR/driving_data +CLEAN_OUTPUT=true + +# Create a temporary work directory +WORK_DIR=$(mktemp -d) +echo -e "Work directory: $WORK_DIR\n" +# Trap signals to clean up WORK directory depending on exit status +functrap() { + code="$?" + if ([ "$code" -eq 0 ] && $CLEAN_OUTPUT) || [ "$code" -eq 2 ]; then + rm -rf "$WORK_DIR" + echo "Work directory cleaned up." + fi +} +# Separate the cases when the script is interrupted, from the cases when it's not +trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM +trap functrap EXIT + +#Set up the work directory as a copy of the test data directory +echo "Setting up work directory..." +cp -r $INPUT_DIR/* $WORK_DIR +cd $WORK_DIR + + +function usage { + cat << EOF +Basic binary compatibility test script for 'replace_landsurface'. +Checks that the outputs of the 'replace_landsurface' package entry points correspond to the expected outputs. + +Usage: regression_tests.sh [-k/--keep] + +Options: +-k, --keep Keep output data upon test completion. + If absent, output data will only be kept for failed tests. +EOF +} + +while getopts ":-:hk" opt; do + case ${opt} in + -) + case ${OPTARG} in + help) + usage + exit 0 + ;; + keep) + CLEAN_OUTPUT=false + ;; + *) + echo "Invalid option: \"--${OPTARG}\"." >&2 + usage + exit 1 + ;; + esac + ;; + h) + usage + exit 0 + ;; + k) + CLEAN_OUTPUT=true + ;; + \?) + echo "Invalid option: \"-${OPTARG}\"." >&2 + usage + exit 1 + ;; + esac +done + +# Check that no additional arguments were passed. +if [[ -n "${@:$OPTIND:1}" ]]; then + echo "Invalid positional argument: \"${@:$OPTIND:1}\"." >&2 + exit 1 +fi + +# # ----------------------------------------------------------------- +# # Run the tests +# # ----------------------------------------------------------------- +echo "Running tests..." + +export ROSE_DATA=$DRIVING_DATA_DIR +run_command() { + MASK=$WORK_DIR/$test_dir/mask + FILE=$WORK_DIR/$test_dir/file + HRES_IC=$WORK_DIR/$test_dir/hres_ic + eval "$entry_point --mask $MASK --file ${FILE}.tmp --start $START --hres_ic $HRES_IC --type $TYPE" +} +compare() { + cmp $WORK_DIR/$test_dir/file $OUTPUT_DIR/$test_dir + if [ $? -ne 0 ]; then + echo "Test ${test_dir#test} failed." + exit 1 + fi +} +# # ----------------------------------------------------------------- +# Test hres_ic +entry_point=hres_ic + +# Test 1: 'era5land' +# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d1000/qrparm.mask +# FILE=g/data/tm70/cbe563/test_replace_land_surface/ERA5LAND/GAL9_astart +# HRES_IC=NOT_SET +test_dir=test_1 +TYPE=era5land +START=202202260000 + +echo "### Test 1: hres_ic, type 'era5land' ###" +run_command +compare + +# # Test 2: 'barra' +# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask +# FILE=/g/data/tm70/cbe563/test_replace_land_surface/BARRAR2/GAL9_astart +# HRES_IC=NOT_SET +test_dir=test_2 +TYPE=barra +START=202008090000 + +echo "### Test 2: hres_ic, type 'barra' ###" +run_command +compare + +# # Test 2: 'barra' +# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask +# FILE=/g/data/tm70/cbe563/test_replace_land_surface/ASTART/RAL3P2_astart +# HRES_IC=/scratch/tm70/cbe563/cylc-run/u-dg768.worked/share/cycle/20220226T0000Z/Lismore/d0198/RAL3P2/ics/RAL3P2_astart +test_dir=test_3 +TYPE=astart +START=202112310000 + +echo "### Test 3: hres_ic, type 'astart' ###" +run_command +compare \ No newline at end of file diff --git a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py index 8d3bb92..b744279 100755 --- a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py @@ -215,10 +215,6 @@ def swap_land_era5land(mask_fullpath, ic_file_fullpath, ic_date): The file is replaced with a version of itself holding the higher-resolution data. """ - - # create name of file to be replaced - ic_file = ic_file_fullpath.parts[-1].replace('.tmp', '') - # create date/time useful information #print(ic_date) yyyy = ic_date[0:4] From 284cf26ebd585458c7fcf6a01203a8fc9f16a432 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 09:42:45 +1100 Subject: [PATCH 02/29] Added parallelization for tests with dynamic capture of PIDs and exit statuses --- integration/integration_tests.sh | 45 +++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 78e6252..2802668 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -101,7 +101,12 @@ compare() { exit 1 fi } +run_test() { + run_command + compare +} # # ----------------------------------------------------------------- +declare -A pids # Associative array (similar to a dictionary in Python) # Test hres_ic entry_point=hres_ic @@ -114,8 +119,7 @@ TYPE=era5land START=202202260000 echo "### Test 1: hres_ic, type 'era5land' ###" -run_command -compare +run_test & pids[1]+=$! # # Test 2: 'barra' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask @@ -126,8 +130,7 @@ TYPE=barra START=202008090000 echo "### Test 2: hres_ic, type 'barra' ###" -run_command -compare +run_test & pids[2]+=$! # # Test 2: 'barra' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask @@ -138,5 +141,35 @@ TYPE=astart START=202112310000 echo "### Test 3: hres_ic, type 'astart' ###" -run_command -compare \ No newline at end of file +run_test & pids[3]+=$! + + +# Capture the exit status of each background processes dynamically +declare -A exit_statuses # Associative array (similar to a dictionary in Python) +for pid in "${pids[@]}"; do + # Find which test the pid corresponds to + for key in "${!pids[@]}"; do + if [[ ${pids[$key]} == $pid ]]; then + test_num=$key + break + fi + done + wait $pid # Waits for the next job in the list to finish (not necessarily in order because it's used within a loop) + exit_status=$? + exit_statuses[$test_num]=$exit_status +done + +# Exit with a status 1 if any of the tests failed +exit_value=0 +for ind in ${!exit_statuses[@]}; do + if [ ${exit_statuses[$ind]} -ne 0 ]; then + echo "Test $ind failed." + exit_value=1 + fi +done + +if [ $exit_value -eq 0 ]; then + echo "All tests passed." +else + exit 1 +fi \ No newline at end of file From e7329856a3f7d7dad8ff958bd163eefe95b61fcc Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 09:48:17 +1100 Subject: [PATCH 03/29] Redirected tests STDOUT to /dev/null --- integration/integration_tests.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 2802668..93e8bdf 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -9,6 +9,8 @@ OUTPUT_DIR=$TEST_DATA_DIR/expected_outputs DRIVING_DATA_DIR=$TEST_DATA_DIR/driving_data CLEAN_OUTPUT=true +declare -A pids # Associative array (similar to a dictionary in Python) + # Create a temporary work directory WORK_DIR=$(mktemp -d) echo -e "Work directory: $WORK_DIR\n" @@ -106,7 +108,6 @@ run_test() { compare } # # ----------------------------------------------------------------- -declare -A pids # Associative array (similar to a dictionary in Python) # Test hres_ic entry_point=hres_ic @@ -119,7 +120,7 @@ TYPE=era5land START=202202260000 echo "### Test 1: hres_ic, type 'era5land' ###" -run_test & pids[1]+=$! +run_test > /dev/null & pids[1]+=$! # # Test 2: 'barra' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask @@ -130,7 +131,7 @@ TYPE=barra START=202008090000 echo "### Test 2: hres_ic, type 'barra' ###" -run_test & pids[2]+=$! +run_test > /dev/null & pids[2]+=$! # # Test 2: 'barra' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask @@ -141,7 +142,7 @@ TYPE=astart START=202112310000 echo "### Test 3: hres_ic, type 'astart' ###" -run_test & pids[3]+=$! +run_test > /dev/null & pids[3]+=$! # Capture the exit status of each background processes dynamically From 37c3ff0906db81c6a53048af7edcff316828cc5e Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 10:24:32 +1100 Subject: [PATCH 04/29] Updated trap function. Updated compare function. Updated paths --- integration/integration_tests.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 93e8bdf..6d0a45e 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -3,7 +3,7 @@ # Basic binary compatibility test script for replace_landsurface. # See INTEGRATION_README.md for details on test usage, data and options. -TEST_DATA_DIR=/g/data/tm70/dm5220/projects/replace_landsurface/test_data +TEST_DATA_DIR=/g/data/vk83/testing/data/replace_landsurface/integration_tests INPUT_DIR=$TEST_DATA_DIR/input_data OUTPUT_DIR=$TEST_DATA_DIR/expected_outputs DRIVING_DATA_DIR=$TEST_DATA_DIR/driving_data @@ -16,14 +16,19 @@ WORK_DIR=$(mktemp -d) echo -e "Work directory: $WORK_DIR\n" # Trap signals to clean up WORK directory depending on exit status functrap() { - code="$?" - if ([ "$code" -eq 0 ] && $CLEAN_OUTPUT) || [ "$code" -eq 2 ]; then + exit_code="$?" + if [ "$exit_code" -eq 0 ] && $CLEAN_OUTPUT; then # Job completed successfully and CLEAN_OUTPUT is true rm -rf "$WORK_DIR" echo "Work directory cleaned up." + elif [ "$exit_code" -eq 2 ]; then # Job was terminated preventively (did not fail) + kill -9 $(jobs -p) &> /dev/null # Kill all background jobs + rm -rf "$WORK_DIR" + echo "Script was terminated preventively." + echo "Work directory cleaned up. Background jobs killed." fi } # Separate the cases when the script is interrupted, from the cases when it's not -trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM +trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM SIGSTOP trap functrap EXIT #Set up the work directory as a copy of the test data directory @@ -97,9 +102,10 @@ run_command() { eval "$entry_point --mask $MASK --file ${FILE}.tmp --start $START --hres_ic $HRES_IC --type $TYPE" } compare() { - cmp $WORK_DIR/$test_dir/file $OUTPUT_DIR/$test_dir + test_num=${test_dir#test_} + cmp $WORK_DIR/$test_dir/file $OUTPUT_DIR/output_${test_num} if [ $? -ne 0 ]; then - echo "Test ${test_dir#test} failed." + echo "Test $test_num failed." exit 1 fi } @@ -144,7 +150,6 @@ START=202112310000 echo "### Test 3: hres_ic, type 'astart' ###" run_test > /dev/null & pids[3]+=$! - # Capture the exit status of each background processes dynamically declare -A exit_statuses # Associative array (similar to a dictionary in Python) for pid in "${pids[@]}"; do From a1e85dd07e16cc6d60a7d0de8348961e286d2d86 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 16:37:41 +1100 Subject: [PATCH 05/29] Minor fix to a comment --- integration/integration_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 6d0a45e..42851c2 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -119,7 +119,7 @@ entry_point=hres_ic # Test 1: 'era5land' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d1000/qrparm.mask -# FILE=g/data/tm70/cbe563/test_replace_land_surface/ERA5LAND/GAL9_astart +# FILE=/g/data/tm70/cbe563/test_replace_land_surface/ERA5LAND/GAL9_astart # HRES_IC=NOT_SET test_dir=test_1 TYPE=era5land @@ -139,7 +139,7 @@ START=202008090000 echo "### Test 2: hres_ic, type 'barra' ###" run_test > /dev/null & pids[2]+=$! -# # Test 2: 'barra' +# # Test 3: 'astart' # MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask # FILE=/g/data/tm70/cbe563/test_replace_land_surface/ASTART/RAL3P2_astart # HRES_IC=/scratch/tm70/cbe563/cylc-run/u-dg768.worked/share/cycle/20220226T0000Z/Lismore/d0198/RAL3P2/ics/RAL3P2_astart From 670197093cd1ba73cbfca04a489a797910229241 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Mon, 24 Feb 2025 09:08:28 +1100 Subject: [PATCH 06/29] Updated integration README --- integration/INTEGRATION_README.md | 88 +++++++------------------------ 1 file changed, 18 insertions(+), 70 deletions(-) diff --git a/integration/INTEGRATION_README.md b/integration/INTEGRATION_README.md index bdfb3f9..f568606 100644 --- a/integration/INTEGRATION_README.md +++ b/integration/INTEGRATION_README.md @@ -1,80 +1,28 @@ # Integration tests for replace_landsurface -`integration_tests.sh` is a basic binary compatibility test script for replace_landsurface, -that compares the outputs of a specific version of the `replace_landsurface` package with their expected outputs. -The script warns the user if the outputs do not match their respective expected versions. +`integration_tests.sh` is a basic binary compatibility test script for the replace_landsurface package. -The tests are designed to be run on Gadi within a Python environment where `replace_landsurface` is installed as a development package (for instructions refer to the Installation paragraph in the README). - - - - -## Data choices -Three types of reference data are available for use in the tests, called "full", -"intermediate", and "light". Each group of data contains a fields file, and -netCDF files produced from converting the fields file using various `um2nc` options. - -### Full -The "full" fields file is an output file from an ESM1.5 simulation. -Running tests with the "full" data is slower and more resource-intensive. +The script executes various tests in parallel, for all the entry points of the package, using different options to test multiple cases. -### Intermediate (default) -The "intermediate" data contains a fields file, generated as a subset of the -the variables from the "full" data. These variables were selected to ensure -different portions of code within `um2nc` are used in the integration tests. -The included variables are: +All multiple tests are executed to completion, even if any of them fails. +When all the tests are completed, the script fails if any of the tested returned a non-zero exit code, and a message is printed with information about the failed test. -* m01s00i024 Surface temperature (a simple 2D field) -* m01s00i407 Pressure on model rho levels -* m01s00i408 Pressure on model theta levels -* m01s02i288 Aerosol optical thickness from biomass burning (a variable on pseudo_levels) -* m01s03i209 Eastward wind (tests hardcoded variable name changes) -* m01s03i321 Canopy water on tiles (a tiled variable) -* m01s05i216 Precipitation (a simple 2D field) -* m01s08i208 Soil moisture (land only data) -* m01s08i223 Soil moisture on soil levels -* m01s30i204 Temperature on PLEV grid (requires masking) -* m01s30i301 Heaviside (used for masking) - -### Light -The "light" data contains a minimal subset of variables from the "full" data -fields file, and can be used for faster but less in-depth testing. It includes: - -* m01s30i204 Temperature on PLEV grid -* m01s05i216 Precipitation - -### Comparison data -For each of the above data choices, the following netCDF variants were produced: - -* `mask`: produced with the `--nohist` flag only. -* `nomask`: produced with the `--nomask` and `--nohist` flags. -* `hist`: produced with no flags. These files will have a conversion datestamp -in their history attribute. +The tests are designed to be run on Gadi within a Python environment where `replace_landsurface` is installed as a development package (for instructions refer to the Installation paragraph in the README). -## Data versions -The `um2nc` version to compare against can be selected with the `-v` flag. -If omitted, the tests will be performed against the latest released version. +Usage: + regression_tests.sh [-h] [--keep] -Available versions for comparison are: -* 0 +Options: + -h, --help Print this usage information and exit. + -k, --keep Force output data to be kept upon test completion. + The default behaviour is to keep the output data only for failed test sessions and delete it if all tests pass. -### Version `0` -Version `0` netCDF outputs were created using the `um2netcdf.py` script available -prior to the development of `um2nc`: https://github.com/ACCESS-NRI/um2nc-standalone/blob/f62105b45eb39d2beed5a7ac71f439ff90f0f00c/src/um2netcdf.py +All test data is located in `/g/data/vk83/testing/data/replace_landsurface/integration_tests`. -The conversion was performed within the following `payu1.1.5` environment, active on Gadi: -https://github.com/ACCESS-NRI/payu-condaenv/releases/tag/1.1.5 +### Tests performed -All test data is located in `/g/data/vk83/testing/data/um2nc/integration-tests`. +- Test 1: hres_ic with `--type era5land` +- Test 2: hres_ic with `--type barra` +- Test 3: hres_ic with `--type astart` +- Test 4: hres_eccb with `--type era5land` (CURRENTLY NOT AVAILABLE) +- Test 5: hres_eccb with `--type barra` (CURRENTLY NOT AVAILABLE) \ No newline at end of file From fb5ffa8c7c45bedc4a6359cb2edc22c557ba9f8f Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Mon, 24 Feb 2025 09:27:15 +1100 Subject: [PATCH 07/29] Updated README --- README.md | 40 +++++++++++++++++++++++++++++--- integration/integration_tests.sh | 3 ++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 18318dd..42548af 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,39 @@ -[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) - # replace_landsurface -This repository contains Python scripts for use by Regional Nesting Suites to replace specific land surface fields in static input data. +## About + +`replace_landsurface` is a `Python` utility to be used within ACCESS-NRI versions of the Regional Nesting Suites to replace specific land surface initial/boundary conditions. + + +## Development/Testing instructions +For development/testing, it is recommended to install `replace_landsurface` as a development package within a `micromamba`/`conda` testing environment. + +### Clone replace_landsurface GitHub repo +``` +git clone git@github.com:ACCESS-NRI/replace_landsurface.git +``` + +### Create a micromamba/conda testing environment +> [!TIP] +> In the following instructions `micromamba` can be replaced with `conda`. + +``` +cd replace_landsurface +micromamba env create -n replace_landsurface_dev --file .conda/env_dev.yml +micromamba activate replace_landsurface_dev +``` + +### Install replace_landsurface as a development package +``` +pip install --no-deps --no-build-isolation -e . +``` + +### Running the tests + +#### Integration tests +To manually run the integration tests, from the `replace_landsurface` directory, you can: +1. Activate your [micromamba/conda testing environment](#create-a-micromamba-conda-testing-environment) +2. Run the following command: + ``` + bash integration/integration_tests.sh + ``` \ No newline at end of file diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh index 42851c2..da731b4 100755 --- a/integration/integration_tests.sh +++ b/integration/integration_tests.sh @@ -31,11 +31,12 @@ functrap() { trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM SIGSTOP trap functrap EXIT +start_time=$(date +%s) #Set up the work directory as a copy of the test data directory echo "Setting up work directory..." cp -r $INPUT_DIR/* $WORK_DIR cd $WORK_DIR - +echo "Elapsed time for data copy: $(($(date +%s) - start_time)) seconds" function usage { cat << EOF From 731e0bcacf26e4feb5922108cdda3ad8782e91da Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Mon, 24 Feb 2025 18:12:02 +1100 Subject: [PATCH 08/29] Added dependencies to dev environment --- .conda/env_dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.conda/env_dev.yml b/.conda/env_dev.yml index 38fbb06..9580206 100644 --- a/.conda/env_dev.yml +++ b/.conda/env_dev.yml @@ -11,4 +11,6 @@ dependencies: - xarray - versioneer - pytest + - pytest-cov + - pytest-xdist - hypothesis \ No newline at end of file From 2f9785669f4c95502d78791124b6038ef52426bb Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Tue, 25 Feb 2025 18:01:01 +1100 Subject: [PATCH 09/29] Added tests/test_integration.py to be run with pytest. Before the actual test run, there is a setup step that sets up the work directory and data paths Currently the tests are not meant to be run in parallel yet, as the setup should be re-structured to be run only once (before the test run) and not for each parallel worker. --- tests/integration/test_integration.py | 122 ++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 tests/integration/test_integration.py diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py new file mode 100644 index 0000000..834cca5 --- /dev/null +++ b/tests/integration/test_integration.py @@ -0,0 +1,122 @@ +import filecmp +import os +import pytest +import shutil +import socket +import tempfile +from unittest.mock import patch +from datetime import datetime + +# If not on Gadi, fail the test because the test data is not available +GADI_HOSTNAME = "gadi.nci.org.au" +hostname = socket.gethostname() +if not hostname.endswith(GADI_HOSTNAME): + raise EnvironmentError( + f"Test not supported to be run on {hostname}. This test must be run on Gadi (gadi.nci.org.au)." + ) + +############################################ +## === Integration tests setup === ## +############################################ +print("\n## === Integration tests setup started === ##") +TEST_DATA_DIR = '/g/data/vk83/testing/data/replace_landsurface/integration_tests' +INPUT_DIR = os.path.join(TEST_DATA_DIR,'input_data') +OUTPUT_DIR = os.path.join(TEST_DATA_DIR,'expected_outputs') +DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR,'driving_data') +# Set the ROSE_DATA environment variable to the driving data directory +os.environ["ROSE_DATA"] = "/g/data/vk83/testing/data/replace_landsurface/integration_tests/driving_data" +from replace_landsurface import hres_ic # importing here because we need to set the ROSE_DATA env variable before importing # noqa + +# Set up working directory path' +working_dir_prefix='replace_landsurface_integration_tests_' +working_dir_date=datetime.now().strftime("%Y%m%d%H%M%S") +WORKING_DIR = os.path.join(tempfile.gettempdir(), working_dir_prefix + working_dir_date) +# Copy input data to the working directory +shutil.copytree(INPUT_DIR, WORKING_DIR) +# Change current working directory to the working directory +os.chdir(WORKING_DIR) +print("## === Integration tests setup complete! === ##") +############################################ +## === Integration tests === ## +############################################ +# Suppress warnings +pytestmark = pytest.mark.filterwarnings("ignore::Warning") + +def get_error_msg(num, output, expected_output): + return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" +# Define the command-line arguments for the tests +def get_test_args(num, start, _type): + test_dir = f'test_{num}' + _hres_ic = os.path.join(WORKING_DIR,test_dir,'hres_ic') if _type == 'astart' else 'NOT_USED' + return [ + "script_name", + '--mask', + os.path.join(WORKING_DIR,test_dir,'mask'), + '--file', + os.path.join(WORKING_DIR,test_dir,'file'+'.tmp'), + '--start', + start, + '--type', + _type, + '--hres_ic', + _hres_ic, + ] + +def get_output_path(num): + return os.path.join(WORKING_DIR,f'test_{num}','file') + +def get_expected_output_path(num): + return os.path.join(OUTPUT_DIR,f'output_{num}') + +# Set the ROSE_DATA environment variable to the driving data directory in all tests +@pytest.fixture(autouse=True) +def mock_rose_data(monkeypatch): + monkeypatch.setenv("ROSE_DATA", DRIVING_DATA_DIR) + +@patch("sys.argv", get_test_args(1,'202202260000','era5land')) +def test_hres_ic_era5land(): + """ + Test the hres_ic entry point with '--type era5land' + """ + num=1 + hres_ic.main() + output = get_output_path(num) + expected_output = get_expected_output_path(num) + # # Compare the output file with the expected output + assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) + +@patch("sys.argv", get_test_args(2,'202008090000','barra')) +def test_hres_ic_barra(): + """ + Test the hres_ic entry point with '--type barra' + """ + num=2 + hres_ic.main() + output = get_output_path(num) + expected_output = get_expected_output_path(num) + # # Compare the output file with the expected output + assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) + +@patch("sys.argv", get_test_args(3,'202112310000','astart')) +def test_hres_ic_astart(): + """ + Test the hres_ic entry point with '--type astart' + """ + num=3 + hres_ic.main() + output = get_output_path(num) + expected_output = get_expected_output_path(num) + # # Compare the output file with the expected output + assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) + +def test_hres_eccb_era5land(): + """ + Test the hres_eccb entry point with '--type era5land' + """ + pass + +def test_hres_eccb_barra(): + """ + Test the hres_eccb entry point with '--type barra' + """ + pass \ No newline at end of file From 8d225ced613fbb1bcf9308fcb98f79161908069c Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 00:00:02 +1100 Subject: [PATCH 10/29] Re-factored test to: - Mock the shutil.move function so the outputs can be sent to a custom path and there is no need for copying the input data - Use fixtures - Use mark.parameterize - Use built-in tmp_path_factory fixture - Enable parallelized tests Tests can now be run with 'pytest -n | auto' to run in parallel. --- tests/integration/test_integration.py | 192 ++++++++++++++++---------- 1 file changed, 119 insertions(+), 73 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 834cca5..a6bc479 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -1,113 +1,158 @@ +import contextlib import filecmp import os -import pytest import shutil import socket -import tempfile from unittest.mock import patch -from datetime import datetime +import pytest -# If not on Gadi, fail the test because the test data is not available +# If not on Gadi, skip the tests because the test data is not available GADI_HOSTNAME = "gadi.nci.org.au" hostname = socket.gethostname() -if not hostname.endswith(GADI_HOSTNAME): - raise EnvironmentError( - f"Test not supported to be run on {hostname}. This test must be run on Gadi (gadi.nci.org.au)." - ) +# Marker to skip tests if not on Gadi +skip_marker = pytest.mark.skipif( + not hostname.endswith(GADI_HOSTNAME), + reason=f"Skipping integration tests because they cannot be executed on {hostname}.\n" + "Integration tests are specifically designed to run on Gadi (gadi.nci.org.au).", +) +# Marker to suppress warnings +warning_marker = pytest.mark.filterwarnings("ignore::Warning") +# Apply the markers to all tests in this file +pytestmark = [skip_marker, warning_marker] ############################################ ## === Integration tests setup === ## ############################################ -print("\n## === Integration tests setup started === ##") -TEST_DATA_DIR = '/g/data/vk83/testing/data/replace_landsurface/integration_tests' -INPUT_DIR = os.path.join(TEST_DATA_DIR,'input_data') -OUTPUT_DIR = os.path.join(TEST_DATA_DIR,'expected_outputs') -DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR,'driving_data') +TEST_DATA_DIR = "/g/data/vk83/testing/data/replace_landsurface/integration_tests" +INPUT_DIR = os.path.join(TEST_DATA_DIR, "input_data") +OUTPUT_DIR = os.path.join(TEST_DATA_DIR, "expected_outputs") +DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") # Set the ROSE_DATA environment variable to the driving data directory -os.environ["ROSE_DATA"] = "/g/data/vk83/testing/data/replace_landsurface/integration_tests/driving_data" -from replace_landsurface import hres_ic # importing here because we need to set the ROSE_DATA env variable before importing # noqa - -# Set up working directory path' -working_dir_prefix='replace_landsurface_integration_tests_' -working_dir_date=datetime.now().strftime("%Y%m%d%H%M%S") -WORKING_DIR = os.path.join(tempfile.gettempdir(), working_dir_prefix + working_dir_date) -# Copy input data to the working directory -shutil.copytree(INPUT_DIR, WORKING_DIR) -# Change current working directory to the working directory -os.chdir(WORKING_DIR) -print("## === Integration tests setup complete! === ##") +os.environ["ROSE_DATA"] = DRIVING_DATA_DIR +from replace_landsurface import ( + hres_ic, +) # importing here because we need to set the ROSE_DATA env variable before importing # noqa + + ############################################ ## === Integration tests === ## ############################################ -# Suppress warnings -pytestmark = pytest.mark.filterwarnings("ignore::Warning") - -def get_error_msg(num, output, expected_output): - return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" -# Define the command-line arguments for the tests def get_test_args(num, start, _type): - test_dir = f'test_{num}' - _hres_ic = os.path.join(WORKING_DIR,test_dir,'hres_ic') if _type == 'astart' else 'NOT_USED' + test_dir = f"test_{num}" + _hres_ic = ( + os.path.join(INPUT_DIR, test_dir, "hres_ic") + if _type == "astart" + else "NOT_USED" + ) return [ "script_name", - '--mask', - os.path.join(WORKING_DIR,test_dir,'mask'), - '--file', - os.path.join(WORKING_DIR,test_dir,'file'+'.tmp'), - '--start', + "--mask", + os.path.join(INPUT_DIR, test_dir, "mask"), + "--file", + os.path.join(INPUT_DIR, test_dir, "file" + ".tmp"), + "--start", start, - '--type', + "--type", _type, - '--hres_ic', + "--hres_ic", _hres_ic, ] -def get_output_path(num): - return os.path.join(WORKING_DIR,f'test_{num}','file') -def get_expected_output_path(num): - return os.path.join(OUTPUT_DIR,f'output_{num}') +@pytest.fixture +def mock_sys_argv(): + @contextlib.contextmanager + def _mock_sys_argv(num, start, _type): + with patch("sys.argv", get_test_args(num, start, _type)): + yield mock_sys_argv + + return _mock_sys_argv + + +@pytest.fixture() +def get_error_msg(): + def _get_error_msg(num, output, expected_output): + return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" + + return _get_error_msg + + +@pytest.fixture(scope="module") +def working_dir(tmp_path_factory): + return tmp_path_factory.mktemp("replace_landsurface_integration_tests") + + +@pytest.fixture() +def get_output_path(working_dir): + def _get_output_path(num): + return os.path.join(working_dir, f"output_{num}") + + return _get_output_path + + +@pytest.fixture() +def get_expected_output_path(): + def _get_expected_output_path(num): + return os.path.join(OUTPUT_DIR, f"output_{num}") + + return _get_expected_output_path + # Set the ROSE_DATA environment variable to the driving data directory in all tests @pytest.fixture(autouse=True) def mock_rose_data(monkeypatch): monkeypatch.setenv("ROSE_DATA", DRIVING_DATA_DIR) -@patch("sys.argv", get_test_args(1,'202202260000','era5land')) -def test_hres_ic_era5land(): - """ - Test the hres_ic entry point with '--type era5land' - """ - num=1 - hres_ic.main() - output = get_output_path(num) - expected_output = get_expected_output_path(num) - # # Compare the output file with the expected output - assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) -@patch("sys.argv", get_test_args(2,'202008090000','barra')) -def test_hres_ic_barra(): - """ - Test the hres_ic entry point with '--type barra' - """ - num=2 - hres_ic.main() - output = get_output_path(num) - expected_output = get_expected_output_path(num) - # # Compare the output file with the expected output - assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) +@pytest.fixture(scope="module") +def original_shutil_move(): + return shutil.move -@patch("sys.argv", get_test_args(3,'202112310000','astart')) -def test_hres_ic_astart(): + +@pytest.fixture() +def new_shutil_move(original_shutil_move, get_output_path, get_expected_output_path): + def _new_shutil_move(num): + def _wrapper(src, dst, **kwargs): + output_path = get_output_path(num) + return original_shutil_move(src=src, dst=output_path, **kwargs) + + return _wrapper + + return _new_shutil_move + + +@pytest.mark.parametrize( + "num, start, _type", + [ + (1, "202202260000", "era5land"), + (2, "202008090000", "barra"), + (3, "202112310000", "astart"), + ], + ids=["hres_ic_era5land", "hres_ic_barra", "hres_ic_astart"], +) +def test_hres_ic_era5land( + new_shutil_move, + get_output_path, + get_expected_output_path, + get_error_msg, + mock_sys_argv, + num, + start, + _type, +): """ - Test the hres_ic entry point with '--type astart' + Test the hres_ic entry point with '--type era5land' """ - num=3 - hres_ic.main() + with mock_sys_argv(num, start, _type): + with patch("shutil.move", side_effect=new_shutil_move(num)): + hres_ic.main() output = get_output_path(num) expected_output = get_expected_output_path(num) # # Compare the output file with the expected output - assert filecmp.cmp(output, expected_output), get_error_msg(num, output, expected_output) + assert filecmp.cmp(output, expected_output), get_error_msg( + num, output, expected_output + ) + def test_hres_eccb_era5land(): """ @@ -115,8 +160,9 @@ def test_hres_eccb_era5land(): """ pass + def test_hres_eccb_barra(): """ Test the hres_eccb entry point with '--type barra' """ - pass \ No newline at end of file + pass From ad7ea52077d6a4ef7100fa4bfa4cb218fc0f5001 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 11:46:32 +1100 Subject: [PATCH 11/29] Added tests for hres_eccb entry point --- .../replace_landsurface_with_BARRA2R_IC.py | 2 - tests/integration/test_integration.py | 52 ++++++++++++++----- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py index a420754..4f64835 100755 --- a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py @@ -152,7 +152,6 @@ def get_BARRA_nc_data(ncfname, FIELDN, wanted_dt, NLAYERS, bounds): sys.exit(1) d.close() - return data.data @@ -202,7 +201,6 @@ def swap_land_barra(mask_fullpath, ec_cb_file_fullpath, ic_date): indir = os.path.join(BARRA_DIR, '1hr',BARRA_FIELDN, 'latest') barra_files = glob(os.path.join(indir, BARRA_FIELDN + '*' + yyyy + mm + '*nc')) barra_fname = indir + '/' + barra_files[0].split('/')[-1] - # Work out the grid bounds using the surface temperature file bounds = bounding_box(barra_fname, mask_fullpath.as_posix(), "land_binary_mask") diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index a6bc479..1cba67c 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -128,9 +128,13 @@ def _wrapper(src, dst, **kwargs): (2, "202008090000", "barra"), (3, "202112310000", "astart"), ], - ids=["hres_ic_era5land", "hres_ic_barra", "hres_ic_astart"], + ids=[ + "hres_ic_era5land", + "hres_ic_barra", + "hres_ic_astart", + ], ) -def test_hres_ic_era5land( +def test_hres_ic( new_shutil_move, get_output_path, get_expected_output_path, @@ -141,7 +145,7 @@ def test_hres_ic_era5land( _type, ): """ - Test the hres_ic entry point with '--type era5land' + Test the hres_ic entry point """ with mock_sys_argv(num, start, _type): with patch("shutil.move", side_effect=new_shutil_move(num)): @@ -153,16 +157,36 @@ def test_hres_ic_era5land( num, output, expected_output ) - -def test_hres_eccb_era5land(): - """ - Test the hres_eccb entry point with '--type era5land' - """ - pass - - -def test_hres_eccb_barra(): +@pytest.mark.parametrize( + "num, start, _type", + [ + (4, "202305040000", "era5land"), + # (5, "202403050000", "barra"), + ], + ids=[ + "hres_eccb_era5land", + # "hres_eccb_barra", + ], +) +def test_hres_eccb( + new_shutil_move, + get_output_path, + get_expected_output_path, + get_error_msg, + mock_sys_argv, + num, + start, + _type, +): """ - Test the hres_eccb entry point with '--type barra' + Test the hres_eccb entry point """ - pass + with mock_sys_argv(num, start, _type): + with patch("shutil.move", side_effect=new_shutil_move(num)): + hres_ic.main() + output = get_output_path(num) + expected_output = get_expected_output_path(num) + # # Compare the output file with the expected output + assert filecmp.cmp(output, expected_output), get_error_msg( + num, output, expected_output + ) \ No newline at end of file From 3b035ebbbb0a7b9c76165b2e3f703dfca4aae108 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 12:11:19 +1100 Subject: [PATCH 12/29] Deleted old ibash ntegration tests --- integration/INTEGRATION_README.md | 28 ----- integration/integration_tests.sh | 182 ------------------------------ 2 files changed, 210 deletions(-) delete mode 100644 integration/INTEGRATION_README.md delete mode 100755 integration/integration_tests.sh diff --git a/integration/INTEGRATION_README.md b/integration/INTEGRATION_README.md deleted file mode 100644 index f568606..0000000 --- a/integration/INTEGRATION_README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Integration tests for replace_landsurface - -`integration_tests.sh` is a basic binary compatibility test script for the replace_landsurface package. - -The script executes various tests in parallel, for all the entry points of the package, using different options to test multiple cases. - -All multiple tests are executed to completion, even if any of them fails. -When all the tests are completed, the script fails if any of the tested returned a non-zero exit code, and a message is printed with information about the failed test. - -The tests are designed to be run on Gadi within a Python environment where `replace_landsurface` is installed as a development package (for instructions refer to the Installation paragraph in the README). - -Usage: - regression_tests.sh [-h] [--keep] - -Options: - -h, --help Print this usage information and exit. - -k, --keep Force output data to be kept upon test completion. - The default behaviour is to keep the output data only for failed test sessions and delete it if all tests pass. - -All test data is located in `/g/data/vk83/testing/data/replace_landsurface/integration_tests`. - -### Tests performed - -- Test 1: hres_ic with `--type era5land` -- Test 2: hres_ic with `--type barra` -- Test 3: hres_ic with `--type astart` -- Test 4: hres_eccb with `--type era5land` (CURRENTLY NOT AVAILABLE) -- Test 5: hres_eccb with `--type barra` (CURRENTLY NOT AVAILABLE) \ No newline at end of file diff --git a/integration/integration_tests.sh b/integration/integration_tests.sh deleted file mode 100755 index da731b4..0000000 --- a/integration/integration_tests.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/bash - -# Basic binary compatibility test script for replace_landsurface. -# See INTEGRATION_README.md for details on test usage, data and options. - -TEST_DATA_DIR=/g/data/vk83/testing/data/replace_landsurface/integration_tests -INPUT_DIR=$TEST_DATA_DIR/input_data -OUTPUT_DIR=$TEST_DATA_DIR/expected_outputs -DRIVING_DATA_DIR=$TEST_DATA_DIR/driving_data -CLEAN_OUTPUT=true - -declare -A pids # Associative array (similar to a dictionary in Python) - -# Create a temporary work directory -WORK_DIR=$(mktemp -d) -echo -e "Work directory: $WORK_DIR\n" -# Trap signals to clean up WORK directory depending on exit status -functrap() { - exit_code="$?" - if [ "$exit_code" -eq 0 ] && $CLEAN_OUTPUT; then # Job completed successfully and CLEAN_OUTPUT is true - rm -rf "$WORK_DIR" - echo "Work directory cleaned up." - elif [ "$exit_code" -eq 2 ]; then # Job was terminated preventively (did not fail) - kill -9 $(jobs -p) &> /dev/null # Kill all background jobs - rm -rf "$WORK_DIR" - echo "Script was terminated preventively." - echo "Work directory cleaned up. Background jobs killed." - fi -} -# Separate the cases when the script is interrupted, from the cases when it's not -trap "exit 2" SIGHUP SIGINT SIGQUIT SIGILL SIGABRT SIGTERM SIGSTOP -trap functrap EXIT - -start_time=$(date +%s) -#Set up the work directory as a copy of the test data directory -echo "Setting up work directory..." -cp -r $INPUT_DIR/* $WORK_DIR -cd $WORK_DIR -echo "Elapsed time for data copy: $(($(date +%s) - start_time)) seconds" - -function usage { - cat << EOF -Basic binary compatibility test script for 'replace_landsurface'. -Checks that the outputs of the 'replace_landsurface' package entry points correspond to the expected outputs. - -Usage: regression_tests.sh [-k/--keep] - -Options: --k, --keep Keep output data upon test completion. - If absent, output data will only be kept for failed tests. -EOF -} - -while getopts ":-:hk" opt; do - case ${opt} in - -) - case ${OPTARG} in - help) - usage - exit 0 - ;; - keep) - CLEAN_OUTPUT=false - ;; - *) - echo "Invalid option: \"--${OPTARG}\"." >&2 - usage - exit 1 - ;; - esac - ;; - h) - usage - exit 0 - ;; - k) - CLEAN_OUTPUT=true - ;; - \?) - echo "Invalid option: \"-${OPTARG}\"." >&2 - usage - exit 1 - ;; - esac -done - -# Check that no additional arguments were passed. -if [[ -n "${@:$OPTIND:1}" ]]; then - echo "Invalid positional argument: \"${@:$OPTIND:1}\"." >&2 - exit 1 -fi - -# # ----------------------------------------------------------------- -# # Run the tests -# # ----------------------------------------------------------------- -echo "Running tests..." - -export ROSE_DATA=$DRIVING_DATA_DIR -run_command() { - MASK=$WORK_DIR/$test_dir/mask - FILE=$WORK_DIR/$test_dir/file - HRES_IC=$WORK_DIR/$test_dir/hres_ic - eval "$entry_point --mask $MASK --file ${FILE}.tmp --start $START --hres_ic $HRES_IC --type $TYPE" -} -compare() { - test_num=${test_dir#test_} - cmp $WORK_DIR/$test_dir/file $OUTPUT_DIR/output_${test_num} - if [ $? -ne 0 ]; then - echo "Test $test_num failed." - exit 1 - fi -} -run_test() { - run_command - compare -} -# # ----------------------------------------------------------------- -# Test hres_ic -entry_point=hres_ic - -# Test 1: 'era5land' -# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d1000/qrparm.mask -# FILE=/g/data/tm70/cbe563/test_replace_land_surface/ERA5LAND/GAL9_astart -# HRES_IC=NOT_SET -test_dir=test_1 -TYPE=era5land -START=202202260000 - -echo "### Test 1: hres_ic, type 'era5land' ###" -run_test > /dev/null & pids[1]+=$! - -# # Test 2: 'barra' -# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask -# FILE=/g/data/tm70/cbe563/test_replace_land_surface/BARRAR2/GAL9_astart -# HRES_IC=NOT_SET -test_dir=test_2 -TYPE=barra -START=202008090000 - -echo "### Test 2: hres_ic, type 'barra' ###" -run_test > /dev/null & pids[2]+=$! - -# # Test 3: 'astart' -# MASK=/scratch/tm70/cbe563/cylc-run/u-dg767/share/data/ancils/Lismore/d0198/qrparm.mask -# FILE=/g/data/tm70/cbe563/test_replace_land_surface/ASTART/RAL3P2_astart -# HRES_IC=/scratch/tm70/cbe563/cylc-run/u-dg768.worked/share/cycle/20220226T0000Z/Lismore/d0198/RAL3P2/ics/RAL3P2_astart -test_dir=test_3 -TYPE=astart -START=202112310000 - -echo "### Test 3: hres_ic, type 'astart' ###" -run_test > /dev/null & pids[3]+=$! - -# Capture the exit status of each background processes dynamically -declare -A exit_statuses # Associative array (similar to a dictionary in Python) -for pid in "${pids[@]}"; do - # Find which test the pid corresponds to - for key in "${!pids[@]}"; do - if [[ ${pids[$key]} == $pid ]]; then - test_num=$key - break - fi - done - wait $pid # Waits for the next job in the list to finish (not necessarily in order because it's used within a loop) - exit_status=$? - exit_statuses[$test_num]=$exit_status -done - -# Exit with a status 1 if any of the tests failed -exit_value=0 -for ind in ${!exit_statuses[@]}; do - if [ ${exit_statuses[$ind]} -ne 0 ]; then - echo "Test $ind failed." - exit_value=1 - fi -done - -if [ $exit_value -eq 0 ]; then - echo "All tests passed." -else - exit 1 -fi \ No newline at end of file From a75033630d4fc07ed5f76b909ca610d4d35a5f66 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 12:57:33 +1100 Subject: [PATCH 13/29] Revert changes on src/replace_landsurface files --- .../replace_landsurface_with_BARRA2R_IC.py | 2 ++ .../replace_landsurface_with_ERA5land_IC.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py index 4f64835..a420754 100755 --- a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py @@ -152,6 +152,7 @@ def get_BARRA_nc_data(ncfname, FIELDN, wanted_dt, NLAYERS, bounds): sys.exit(1) d.close() + return data.data @@ -201,6 +202,7 @@ def swap_land_barra(mask_fullpath, ec_cb_file_fullpath, ic_date): indir = os.path.join(BARRA_DIR, '1hr',BARRA_FIELDN, 'latest') barra_files = glob(os.path.join(indir, BARRA_FIELDN + '*' + yyyy + mm + '*nc')) barra_fname = indir + '/' + barra_files[0].split('/')[-1] + # Work out the grid bounds using the surface temperature file bounds = bounding_box(barra_fname, mask_fullpath.as_posix(), "land_binary_mask") diff --git a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py index b744279..8d3bb92 100755 --- a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py @@ -215,6 +215,10 @@ def swap_land_era5land(mask_fullpath, ic_file_fullpath, ic_date): The file is replaced with a version of itself holding the higher-resolution data. """ + + # create name of file to be replaced + ic_file = ic_file_fullpath.parts[-1].replace('.tmp', '') + # create date/time useful information #print(ic_date) yyyy = ic_date[0:4] From fdbebc537cf235424822c80f7346ef6a9c5eafed Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Thu, 27 Feb 2025 17:04:37 +1100 Subject: [PATCH 14/29] Incorporated Tom's suggestions --- tests/integration/test_integration.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 1cba67c..68f91b4 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -4,6 +4,7 @@ import shutil import socket from unittest.mock import patch +from pathlib import Path import pytest # If not on Gadi, skip the tests because the test data is not available @@ -23,12 +24,12 @@ ############################################ ## === Integration tests setup === ## ############################################ -TEST_DATA_DIR = "/g/data/vk83/testing/data/replace_landsurface/integration_tests" -INPUT_DIR = os.path.join(TEST_DATA_DIR, "input_data") -OUTPUT_DIR = os.path.join(TEST_DATA_DIR, "expected_outputs") -DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") +TEST_DATA_DIR = Path("/g/data/vk83/testing/data/replace_landsurface/integration_tests") +INPUT_DIR = TEST_DATA_DIR / "input_data" +OUTPUT_DIR = TEST_DATA_DIR / "expected_outputs" +DRIVING_DATA_DIR = TEST_DATA_DIR / "driving_data" # Set the ROSE_DATA environment variable to the driving data directory -os.environ["ROSE_DATA"] = DRIVING_DATA_DIR +os.environ["ROSE_DATA"] = str(DRIVING_DATA_DIR) from replace_landsurface import ( hres_ic, ) # importing here because we need to set the ROSE_DATA env variable before importing # noqa @@ -152,7 +153,7 @@ def test_hres_ic( hres_ic.main() output = get_output_path(num) expected_output = get_expected_output_path(num) - # # Compare the output file with the expected output + # Compare the output file with the expected output assert filecmp.cmp(output, expected_output), get_error_msg( num, output, expected_output ) @@ -186,7 +187,7 @@ def test_hres_eccb( hres_ic.main() output = get_output_path(num) expected_output = get_expected_output_path(num) - # # Compare the output file with the expected output + # Compare the output file with the expected output assert filecmp.cmp(output, expected_output), get_error_msg( num, output, expected_output ) \ No newline at end of file From 80f9957b1d680adae0541668050d0b2877de91ba Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 14:49:56 +1100 Subject: [PATCH 15/29] Updated README --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 42548af..8777daf 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,19 @@ pip install --no-deps --no-build-isolation -e . ### Running the tests -#### Integration tests -To manually run the integration tests, from the `replace_landsurface` directory, you can: +The test suite currently includes only integration tests. + +To manually run the tests, from the `replace_landsurface` directory, you can: + 1. Activate your [micromamba/conda testing environment](#create-a-micromamba-conda-testing-environment) 2. Run the following command: ``` - bash integration/integration_tests.sh - ``` \ No newline at end of file + pytest -n 4 + ``` + +> [!TIP] +> The `-n 4` option is a [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/) option to run the tests in parallel across 4 different workers. + +> [!IMPORTANT] +> Integration tests are designed to be run on `Gadi`. +> If you run tests on a local machine, the integration tests will be skipped. \ No newline at end of file From ce903760a9edeb0f97fed794f886f0dd9d6d569e Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 14:59:27 +1100 Subject: [PATCH 16/29] Removed fixture to set env variable --- tests/integration/test_integration.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 68f91b4..d7adff4 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -98,13 +98,6 @@ def _get_expected_output_path(num): return _get_expected_output_path - -# Set the ROSE_DATA environment variable to the driving data directory in all tests -@pytest.fixture(autouse=True) -def mock_rose_data(monkeypatch): - monkeypatch.setenv("ROSE_DATA", DRIVING_DATA_DIR) - - @pytest.fixture(scope="module") def original_shutil_move(): return shutil.move From 0806a01e21db39651f97885f9459116d1d3c8161 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 15:05:02 +1100 Subject: [PATCH 17/29] Turned get_error_message fixture to a normal function --- tests/integration/test_integration.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index d7adff4..0dbe736 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -60,6 +60,10 @@ def get_test_args(num, start, _type): ] +def get_error_msg(num, output, expected_output): + return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" + + @pytest.fixture def mock_sys_argv(): @contextlib.contextmanager @@ -70,14 +74,6 @@ def _mock_sys_argv(num, start, _type): return _mock_sys_argv -@pytest.fixture() -def get_error_msg(): - def _get_error_msg(num, output, expected_output): - return f"Test {num}: Test output '{output}' does not match the expected output '{expected_output}'!" - - return _get_error_msg - - @pytest.fixture(scope="module") def working_dir(tmp_path_factory): return tmp_path_factory.mktemp("replace_landsurface_integration_tests") @@ -132,7 +128,6 @@ def test_hres_ic( new_shutil_move, get_output_path, get_expected_output_path, - get_error_msg, mock_sys_argv, num, start, @@ -166,7 +161,6 @@ def test_hres_eccb( new_shutil_move, get_output_path, get_expected_output_path, - get_error_msg, mock_sys_argv, num, start, From abe59455e27fa70badb3a6da82ff58d7a3cc0bf5 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 15:08:06 +1100 Subject: [PATCH 18/29] Removed unused variable from fixture --- tests/integration/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 0dbe736..b8d786e 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -100,7 +100,7 @@ def original_shutil_move(): @pytest.fixture() -def new_shutil_move(original_shutil_move, get_output_path, get_expected_output_path): +def new_shutil_move(original_shutil_move, get_output_path): def _new_shutil_move(num): def _wrapper(src, dst, **kwargs): output_path = get_output_path(num) From 71800c3e37da437497c3a24b71152c1a5aa78e08 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 15:33:48 +1100 Subject: [PATCH 19/29] Turned all path concatenation to 'os.path.join' --- tests/integration/test_integration.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index b8d786e..14857ea 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -4,7 +4,6 @@ import shutil import socket from unittest.mock import patch -from pathlib import Path import pytest # If not on Gadi, skip the tests because the test data is not available @@ -24,15 +23,13 @@ ############################################ ## === Integration tests setup === ## ############################################ -TEST_DATA_DIR = Path("/g/data/vk83/testing/data/replace_landsurface/integration_tests") -INPUT_DIR = TEST_DATA_DIR / "input_data" -OUTPUT_DIR = TEST_DATA_DIR / "expected_outputs" -DRIVING_DATA_DIR = TEST_DATA_DIR / "driving_data" +TEST_DATA_DIR = "/g/data/vk83/testing/data/replace_landsurface/integration_tests" +INPUT_DIR = os.path.join(TEST_DATA_DIR, "input_data") +OUTPUT_DIR = os.path.join(TEST_DATA_DIR, "expected_outputs") +DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") # Set the ROSE_DATA environment variable to the driving data directory os.environ["ROSE_DATA"] = str(DRIVING_DATA_DIR) -from replace_landsurface import ( - hres_ic, -) # importing here because we need to set the ROSE_DATA env variable before importing # noqa +from replace_landsurface import hres_ic # importing here because we need to set the ROSE_DATA env variable before importing # noqa ############################################ From deb4432e23dd9675ea8bd32c87c716c591acda47 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Fri, 28 Feb 2025 19:53:39 +1100 Subject: [PATCH 20/29] Fixed hres_eccb test to call the right function --- tests/integration/test_integration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 14857ea..61050b3 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -29,7 +29,7 @@ DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") # Set the ROSE_DATA environment variable to the driving data directory os.environ["ROSE_DATA"] = str(DRIVING_DATA_DIR) -from replace_landsurface import hres_ic # importing here because we need to set the ROSE_DATA env variable before importing # noqa +from replace_landsurface import hres_ic, hres_eccb # importing here because we need to set the ROSE_DATA env variable before importing # noqa ############################################ @@ -168,7 +168,7 @@ def test_hres_eccb( """ with mock_sys_argv(num, start, _type): with patch("shutil.move", side_effect=new_shutil_move(num)): - hres_ic.main() + hres_eccb.main() output = get_output_path(num) expected_output = get_expected_output_path(num) # Compare the output file with the expected output From 24359ce86b4a5307418e47bde24e02c39447f20b Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 1 Mar 2025 09:43:07 +1100 Subject: [PATCH 21/29] Removed hres_eccb script and renamed 'hres_ic' to 'replace_landsurface' to make it general. Also changed the entry points to only include replace_landsurface (old 'hres_ic'). --- pyproject.toml | 3 +- src/replace_landsurface/hres_eccb.py | 63 ------------------- .../{hres_ic.py => replace_landsurface.py} | 0 tests/integration/test_integration.py | 32 +++++----- 4 files changed, 16 insertions(+), 82 deletions(-) delete mode 100755 src/replace_landsurface/hres_eccb.py rename src/replace_landsurface/{hres_ic.py => replace_landsurface.py} (100%) diff --git a/pyproject.toml b/pyproject.toml index 4b36a01..0449a71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,7 @@ dependencies = [ Repository = "https://github.com/ACCESS-NRI/replace_landsurface" [project.scripts] -hres_eccb = "replace_landsurface.hres_eccb:main" -hres_ic = "replace_landsurface.hres_ic:main" +replace_landsurface = "replace_landsurface.replace_landsurface:main" [build-system] build-backend = "setuptools.build_meta" diff --git a/src/replace_landsurface/hres_eccb.py b/src/replace_landsurface/hres_eccb.py deleted file mode 100755 index b792da1..0000000 --- a/src/replace_landsurface/hres_eccb.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2024 ACCESS-NRI (https://www.access-nri.org.au/) -# See the top-level COPYRIGHT.txt file for details. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Created by: Chermelle Engel - -""" -Replace the land/surface fields in the ec_cb000 file with higher-resolution -era5-land or BARRA2-R data (if requested). -""" - -import argparse -import shutil -from pathlib import Path - -import pandas - -from replace_landsurface import replace_landsurface_with_BARRA2R_IC, replace_landsurface_with_ERA5land_IC - -def main(): - """ - The main function that creates a worker pool and generates single GRIB files - for requested date/times in parallel. - - Parameters - ---------- - None. The arguments are given via the command-line - - Returns - ------- - None. The ec_cb000 file is updated and overwritten - """ - - # Parse the command-line arguments - parser = argparse.ArgumentParser() - parser.add_argument('--mask', required=True, type=Path) - parser.add_argument('--file', required=True, type=Path) - parser.add_argument('--start', required=True, type=pandas.to_datetime) - parser.add_argument('--type', default="era5land") - parser.add_argument('--hres_ic', type=Path) - args = parser.parse_args() - print(args) - - # Convert the date/time to a formatted string - t = args.start.strftime("%Y%m%dT%H%MZ") - print(args.mask, args.file, t) - - # If necessary replace ERA5 land/surface fields with higher-resolution options - if "era5land" in args.type: - replace_landsurface_with_ERA5land_IC.swap_land_era5land(args.mask, args.file, t) - shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) - elif "barra" in args.type: - replace_landsurface_with_BARRA2R_IC.swap_land_barra(args.mask, args.file, t) - shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) - elif "astart" in args.type: - print("Fields not swapped out for ECCB files when using start dump as replacement option.") - else: - print("No need to swap out IC") - -if __name__ == '__main__': - main() - diff --git a/src/replace_landsurface/hres_ic.py b/src/replace_landsurface/replace_landsurface.py similarity index 100% rename from src/replace_landsurface/hres_ic.py rename to src/replace_landsurface/replace_landsurface.py diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 61050b3..687987b 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -29,7 +29,7 @@ DRIVING_DATA_DIR = os.path.join(TEST_DATA_DIR, "driving_data") # Set the ROSE_DATA environment variable to the driving data directory os.environ["ROSE_DATA"] = str(DRIVING_DATA_DIR) -from replace_landsurface import hres_ic, hres_eccb # importing here because we need to set the ROSE_DATA env variable before importing # noqa +from replace_landsurface import replace_landsurface # importing here because we need to set the ROSE_DATA env variable before importing # noqa ############################################ @@ -37,23 +37,23 @@ ############################################ def get_test_args(num, start, _type): test_dir = f"test_{num}" - _hres_ic = ( + hres_ic = ( os.path.join(INPUT_DIR, test_dir, "hres_ic") if _type == "astart" else "NOT_USED" ) return [ "script_name", - "--mask", - os.path.join(INPUT_DIR, test_dir, "mask"), "--file", os.path.join(INPUT_DIR, test_dir, "file" + ".tmp"), + "--mask", + os.path.join(INPUT_DIR, test_dir, "mask"), "--start", start, "--type", _type, "--hres_ic", - _hres_ic, + hres_ic, ] @@ -116,12 +116,12 @@ def _wrapper(src, dst, **kwargs): (3, "202112310000", "astart"), ], ids=[ - "hres_ic_era5land", - "hres_ic_barra", - "hres_ic_astart", + "replace_landsurface_era5land", + "replace_landsurface_barra", + "replace_landsurface_astart", ], ) -def test_hres_ic( +def test_replace_landsurface( new_shutil_move, get_output_path, get_expected_output_path, @@ -131,11 +131,11 @@ def test_hres_ic( _type, ): """ - Test the hres_ic entry point + Test the replace_landsurface entry point """ with mock_sys_argv(num, start, _type): with patch("shutil.move", side_effect=new_shutil_move(num)): - hres_ic.main() + replace_landsurface.main() output = get_output_path(num) expected_output = get_expected_output_path(num) # Compare the output file with the expected output @@ -147,14 +147,12 @@ def test_hres_ic( "num, start, _type", [ (4, "202305040000", "era5land"), - # (5, "202403050000", "barra"), ], ids=[ - "hres_eccb_era5land", - # "hres_eccb_barra", + "replace_landsurface_era5land_2", ], ) -def test_hres_eccb( +def test_replace_landsurface_2( new_shutil_move, get_output_path, get_expected_output_path, @@ -164,11 +162,11 @@ def test_hres_eccb( _type, ): """ - Test the hres_eccb entry point + Test the replace_landsurface entry point """ with mock_sys_argv(num, start, _type): with patch("shutil.move", side_effect=new_shutil_move(num)): - hres_eccb.main() + replace_landsurface.main() output = get_output_path(num) expected_output = get_expected_output_path(num) # Compare the output file with the expected output From 8eec4fe384641d480d66ab3051d14b89f4c46a52 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 1 Mar 2025 10:12:28 +1100 Subject: [PATCH 22/29] Embedded old 'test_hres_eccb' test (test n. 4) within the 'test_replace_landsurface' function --- tests/integration/test_integration.py | 33 ++------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 687987b..db238e5 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -114,11 +114,13 @@ def _wrapper(src, dst, **kwargs): (1, "202202260000", "era5land"), (2, "202008090000", "barra"), (3, "202112310000", "astart"), + (4, "202305040000", "era5land"), ], ids=[ "replace_landsurface_era5land", "replace_landsurface_barra", "replace_landsurface_astart", + "replace_landsurface_era5land_2", ], ) def test_replace_landsurface( @@ -129,37 +131,6 @@ def test_replace_landsurface( num, start, _type, -): - """ - Test the replace_landsurface entry point - """ - with mock_sys_argv(num, start, _type): - with patch("shutil.move", side_effect=new_shutil_move(num)): - replace_landsurface.main() - output = get_output_path(num) - expected_output = get_expected_output_path(num) - # Compare the output file with the expected output - assert filecmp.cmp(output, expected_output), get_error_msg( - num, output, expected_output - ) - -@pytest.mark.parametrize( - "num, start, _type", - [ - (4, "202305040000", "era5land"), - ], - ids=[ - "replace_landsurface_era5land_2", - ], -) -def test_replace_landsurface_2( - new_shutil_move, - get_output_path, - get_expected_output_path, - mock_sys_argv, - num, - start, - _type, ): """ Test the replace_landsurface entry point From 3b0df76b3f4b5b2a1e3a3396deb2b8b93e38e0c8 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 1 Mar 2025 10:21:04 +1100 Subject: [PATCH 23/29] Updated docstrings --- src/replace_landsurface/replace_landsurface.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/replace_landsurface/replace_landsurface.py b/src/replace_landsurface/replace_landsurface.py index d29ce6e..5604d9b 100755 --- a/src/replace_landsurface/replace_landsurface.py +++ b/src/replace_landsurface/replace_landsurface.py @@ -6,8 +6,7 @@ # Created by: Chermelle Engel """ -Replace the land/surface fields in the astart file with higher-resolution -era5-land or BARRA2-R data (if requested). +Replace the land-surface fields in the astart file with higher-resolution data """ import argparse @@ -25,16 +24,17 @@ def main(): """ - The main function that creates a worker pool and generates single GRIB files - for requested date/times in parallel. + Calls the command line argument parser and process the arguments using the right function. Parameters ---------- - None. The arguments are given via the command-line + None + The arguments are given via the command-line Returns ------- - None. The astart file is updated and overwritten + None + An output file is written """ # Parse the command-line arguments From b0b6ee701e123b9d8d423fcfe3b70526932c54da Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 12:30:50 +1100 Subject: [PATCH 24/29] Restructured hres_ic.py and replace_landsurface_with_BARRA2R_IC.py: - Simplified code removing redundant portions - Deleted redundant MASK input - Turned the class bounding_box to the function get_bounding_box, that returns the BoundingBox namedtuple. Simplified the way to get boundary box extents. - Restructured the get_barra_data function to use xarray DataArray.sel method, together with slice and 'method' keyword for more efficient and simpler data retrieval. --- .conda/env_dev.yml | 1 + .../replace_landsurface.py | 23 +- .../replace_landsurface_with_BARRA2R_IC.py | 224 +++++++++--------- 3 files changed, 124 insertions(+), 124 deletions(-) diff --git a/.conda/env_dev.yml b/.conda/env_dev.yml index 9580206..8f3c4e1 100644 --- a/.conda/env_dev.yml +++ b/.conda/env_dev.yml @@ -10,6 +10,7 @@ dependencies: - scitools-iris - xarray - versioneer + - ipykernel - pytest - pytest-cov - pytest-xdist diff --git a/src/replace_landsurface/replace_landsurface.py b/src/replace_landsurface/replace_landsurface.py index 5604d9b..778bfae 100755 --- a/src/replace_landsurface/replace_landsurface.py +++ b/src/replace_landsurface/replace_landsurface.py @@ -8,13 +8,17 @@ """ Replace the land-surface fields in the astart file with higher-resolution data """ +#TODO: delete after testing +# barra_fname='/g/data/vk83/testing/data/replace_landsurface/integration_tests/driving_data/etc/barra_r2/1hr/ts/latest/ts_AUS-11_ERA5_historical_hres_BOM_BARRA-R2_v1_1hr_202008-202008.nc' +# fname='/g/data/tm70/cbe563/test_replace_land_surface/BARRAR2/GAL9_astart' +# mask='/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask' +# date='202008090320' +#TODO: delete after testing import argparse import shutil from pathlib import Path -import pandas - from replace_landsurface import ( replace_landsurface_with_BARRA2R_IC, replace_landsurface_with_ERA5land_IC, @@ -39,29 +43,22 @@ def main(): # Parse the command-line arguments parser = argparse.ArgumentParser() - parser.add_argument('--mask', required=True, type=Path) parser.add_argument('--file', required=True, type=Path) - parser.add_argument('--start', required=True, type=pandas.to_datetime) + parser.add_argument('--start', required=True, type=str, help="A date in the format 'YYYYmmddHHMM'.") parser.add_argument('--type', default="era5land") parser.add_argument('--hres_ic', type=Path) args = parser.parse_args() - print(args) - - # Convert the date/time to a formatted string - t = args.start.strftime("%Y%m%dT%H%MZ") - print(args.mask, args.file, t) # If necessary replace ERA5 land/surface fields with higher-resolution options if "era5land" in args.type: - replace_landsurface_with_ERA5land_IC.swap_land_era5land(args.mask, args.file, t) + replace_landsurface_with_ERA5land_IC.swap_land_era5land(args.file, args.start) shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) elif "barra" in args.type: - replace_landsurface_with_BARRA2R_IC.swap_land_barra(args.mask, args.file, t) + replace_landsurface_with_BARRA2R_IC.swap_land_barra(args.file, args.start) shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) elif "astart" in args.type: - replace_landsurface_with_FF_IC.swap_land_ff(args.mask, args.file, args.hres_ic,t) + replace_landsurface_with_FF_IC.swap_land_ff(args.file, args.hres_ic, args.start) shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) - else: print("No need to swap out IC") diff --git a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py index a420754..6cd4fcc 100755 --- a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py @@ -9,115 +9,119 @@ import sys from glob import glob from pathlib import Path +from collections import namedtuple import iris import mule import numpy as np import xarray as xr +import warnings + +STASH_LAND_MASK = 30 ROSE_DATA = os.environ.get('ROSE_DATA', "") # Base directory of the ERA5-land archive on NCI BARRA_DIR = os.path.join(ROSE_DATA, 'etc', 'barra_r2') - +BoundingBox = namedtuple('BoundingBox', ('latmin', 'latmax', 'lonmin', 'lonmax')) class ReplaceOperator(mule.DataOperator): """ Mule operator for replacing the data""" def __init__(self): pass def new_field(self, sources): - #print('new_field') return sources[0] def transform(self, sources, result): - #print('transform') return sources[1] -class bounding_box(): - """ Container class to hold spatial extent information.""" - def __init__(self, ncfname, maskfname, var): - """ - Initialization function for bounding_box class - - Parameters - ---------- - ncfname : POSIX string - POSIX path to input data (from the NetCDF archive) - maskfname : POSIX string - POSIX path to mask information to define the data to be cut out - var : string - The name of the mask variable that defines the spatial extent - - Returns - ------- - None. The variables describing the spatial extent are definied within bounding box object. - - """ - # Read in the mask and get the minimum/maximum latitude and longitude information - if Path(maskfname).exists(): - d = iris.load(maskfname, var) - d = d[0] - d = xr.DataArray.from_iris(d) - - lons = d['longitude'].data - lonmin = np.min(lons) - lonmax = np.max(lons) - lats = d['latitude'].data - latmin = np.min(lats) - latmax = np.max(lats) - d.close() - else: - print(f'ERROR: File {maskfname} not found', file=sys.stderr) - raise - - # Read in the file from the high-res netcdf archive - if Path(ncfname).exists(): - d = xr.open_dataset(ncfname) - else: - print(f'ERROR: File {ncfname} not found', file=sys.stderr) - sys.exit(1) - - - # Get the longitude information - lons = d['lon'].data - - # Coping with numerical inaccuracy - adj = (lons[1] - lons[0])/2. - - # Work out which longitudes define the minimum/maximum extents of the grid of interest - lonmin_index = np.argwhere( (lons > lonmin - adj) & (lons < lonmin + adj))[0][0] - lonmax_index = np.argwhere( (lons > lonmax - adj) & (lons < lonmax + adj))[0][0] +def get_bounding_box(mule_file): + """ Get spatial extent information for the replacement. + + Parameters + ---------- + mule_file : mule.UMFile object + Input UM fields file - # Get the latitude information - lats = d['lat'].data + Returns + ------- + namedtuple + The BoundingBox namedtuple + """ + # Get extent from the land mask (stash 30) if present, otherwise use the first field + # because the file extent sometimes is not correct + try: + field = [f for f in mule_file.fields if f.lbuser4 == STASH_LAND_MASK][0] + except IndexError: + field = mule_file.fields[0] + # Get the latitude extent + dlat = field.bdy + nlat = mule_file.integer_constants.num_rows + latmin = round(field.bzy + dlat, 4) # rounded to 4 decimal points to avoid numerical inaccuracy + latmax = round(latmin + (nlat-1)*dlat, 4) # rounded to 4 decimal points to avoid numerical inaccuracy + lat_extent=(latmin, latmax) + + # Get the latitude extent + dlon = field.bdx + nlon = mule_file.integer_constants.num_cols + lonmin = round(field.bzx + dlon, 4) # rounded to 4 decimal points to avoid numerical inaccuracy + lonmax = round(lonmin + (nlon-1)*dlon, 4) # rounded to 4 decimal points to avoid numerical inaccuracy + lon_extent=(lonmin, lonmax) + + # TODO: delete after testing + # ff=xr.DataArray.from_iris(iris.load(mask)[0]) + # latmin_orig=ff.latitude.min().data + # latmax_orig=ff.latitude.max().data + # lonmin_orig=ff.longitude.min().data + # lonmax_orig=ff.longitude.max().data + + # # Read in the file from the high-res netcdf archive + # if Path(barra_fname).exists(): + # d = xr.open_dataset(barra_fname) + # else: + # print(f'ERROR: File {barra_fname} not found', file=sys.stderr) + # sys.exit(1) + + # # Coping with numerical inaccuracy + # adj = (lons[1] - lons[0])/2. + + # # Get the longitude information + # lons = d['lon'].data + # # Work out which longitudes define the minimum/maximum extents of the grid of interest + # lonmin_index = np.argwhere( (lons > lonmin_orig - adj) & (lons < lonmin_orig + adj))[0][0] + # lonmax_index = np.argwhere( (lons > lonmax_orig - adj) & (lons < lonmax_orig + adj))[0][0] + # lon_extent_orig = (lons[lonmin_index],lons[lonmax_index]) + + # # Get the latitude information + # lats = d['lat'].data + # # Work out which latitudes define the minimum/maximum extents of the grid of interest + # latmin_index = np.argwhere( (lats > latmin_orig - adj) & (lats < latmin_orig + adj))[0][0] + # latmax_index = np.argwhere( (lats > latmax_orig - adj) & (lats < latmax_orig + adj))[0][0] + # lat_extent_orig = (lats[latmin_index],lats[latmax_index]) - # Work out which longitudes define the minimum/maximum extents of the grid of interest - # use the same adjustment as for longitude - latmin_index = np.argwhere( (lats > latmin - adj) & (lats < latmin + adj))[0][0] - latmax_index = np.argwhere( (lats > latmax - adj) & (lats < latmax + adj))[0][0] + # print(lon_extent) + # print(lon_extent_orig) + # print(lat_extent) + # print(lat_extent_orig) + # TODO: delete after testing - # Set the boundaries - self.lonmin = lonmin_index - self.lonmax = lonmax_index - self.latmin = latmin_index - self.latmax = latmax_index + # Return the boundaries + return BoundingBox(latmin, latmax, lonmin, lonmax) -def get_BARRA_nc_data(ncfname, FIELDN, wanted_dt, NLAYERS, bounds): +def get_barra_data(barra_fname, FIELDN, date, bounds): """ Function to get the BARA2-R data for a single land/surface variable. Parameters ---------- - ncfname : string + barra_fname : string The name of the file to read FIELDN : string The name of the variable in the file to read - wanted_dt : string - The date-time required in "%Y%m%d%H%M" format - NLAYERS : int - The number of layers in the multi-resolution grid (1 or more) - bounds : bounding_box object - A bounding box object defining the spatial extent to keep + date : string + The date in the format "YYYYmmddHHMM". + bounds : tuple + Tuple containing the spatial extent for the data in the form (latmin, latmax, lonmin, lonmax). Returns ------- @@ -126,48 +130,47 @@ def get_BARRA_nc_data(ncfname, FIELDN, wanted_dt, NLAYERS, bounds): """ # Open the file containing the data - if Path(ncfname).exists(): - d = xr.open_dataset(ncfname) + if Path(barra_fname).exists(): + d = xr.open_dataset(barra_fname) else: - print(f'ERROR: File {ncfname} not found', file=sys.stderr) - sys.exit(1) - - # Find the array index for the date/time of interest - times = d['time'].dt.strftime("%Y%m%d%H%M").data - TM = times.tolist().index(wanted_dt) - - # retrieve the spatial extends of interest - lonmin_index, lonmax_index = bounds.lonmin, bounds.lonmax - latmin_index, latmax_index = bounds.latmin, bounds.latmax + raise FileNotFoundError(f"File '{barra_fname}' not found.") - # Read the data + # Get the data + # Select the variable try: - if NLAYERS>1: - data = d[FIELDN][TM, :, latmin_index:latmax_index+1, lonmin_index:lonmax_index+1] - else: - data = d[FIELDN][TM, latmin_index:latmax_index+1, lonmin_index:lonmax_index+1] + data = d[FIELDN] except KeyError: - print(fname) - print(f'ERROR: Variable temp not found in file', file=sys.stderr) - sys.exit(1) - - d.close() - + raise ValueError(f"Variable '{FIELDN}' not found in high-resolution file '{barra_fname}'.") + # Select the date + try: + data = data.sel(time=date) + except KeyError: + # If the exact date is not found select the nearest date and send a warning message + data = data.sel(time=date, method='nearest') + actual_date = data.time.values.astype('datetime64[m]').tolist().strftime('%Y%m%d%H%M') + warnings.warn( + f"The date '{date}' was not found in the high-resolution file '{barra_fname}'.\n" + f"Using the nearest date '{actual_date}' instead." + ) + # Select the spatial extent + data = data.sel( + lat=slice(bounds.latmin, bounds.latmax), + lon=slice(bounds.lonmin, bounds.lonmax), + ) + return data.data -def swap_land_barra(mask_fullpath, ec_cb_file_fullpath, ic_date): +def swap_land_barra(ec_cb_file_fullpath, date): """ Function to get the BARRA2-R data for all land/surface variables. Parameters ---------- - mask_fullpath : Path - Path to the mask defining the spatial extent ic_file_fullpath : string Path to file with the coarser resolution data to be replaced with ".tmp" appended at end - ic_date : string - The date-time required in "%Y%m%d%H%M" format + date : string + The date in the format "YYYYmmddHHMM". Returns ------- @@ -180,9 +183,8 @@ def swap_land_barra(mask_fullpath, ec_cb_file_fullpath, ic_date): ec_cb_file = ec_cb_file_fullpath.parts[-1].replace('.tmp', '') # create date/time useful information - yyyy = ic_date[0:4] - mm = ic_date[4:6] - ic_z_date = ic_date.replace('T', '').replace('Z', '') + yyyy = date[0:4] + mm = date[4:6] # Path to input file ff_in = ec_cb_file_fullpath.as_posix().replace('.tmp', '') @@ -204,10 +206,10 @@ def swap_land_barra(mask_fullpath, ec_cb_file_fullpath, ic_date): barra_fname = indir + '/' + barra_files[0].split('/')[-1] # Work out the grid bounds using the surface temperature file - bounds = bounding_box(barra_fname, mask_fullpath.as_posix(), "land_binary_mask") + bounds = get_bounding_box(mf_in) # Read in the surface temperature data (and keep to use for replacement) - data = get_BARRA_nc_data(barra_fname, BARRA_FIELDN, ic_z_date ,-1 ,bounds) + data = get_barra_data(barra_fname, BARRA_FIELDN, date, bounds) surface_temp = data.copy() # Read in the soil moisture data (and keep to use for replacement) @@ -215,7 +217,7 @@ def swap_land_barra(mask_fullpath, ec_cb_file_fullpath, ic_date): indir = os.path.join(BARRA_DIR, '3hr',BARRA_FIELDN, 'latest') barra_files = glob(os.path.join(indir, BARRA_FIELDN + '*' + yyyy + mm + '*nc')) barra_fname = indir + '/' + barra_files[0].split('/')[-1] - data = get_BARRA_nc_data(barra_fname, BARRA_FIELDN, ic_date.replace('T', '').replace('Z', ''), 4, bounds) + data = get_barra_data(barra_fname, BARRA_FIELDN, date, bounds) mrsol = data.copy() # Read in the soil temperature data (and keep to use for replacement) @@ -223,7 +225,7 @@ def swap_land_barra(mask_fullpath, ec_cb_file_fullpath, ic_date): indir = os.path.join(BARRA_DIR, '3hr',BARRA_FIELDN, 'latest') barra_files = glob(os.path.join(indir, BARRA_FIELDN + '*' + yyyy + mm + '*nc')) barra_fname = indir + '/' + barra_files[0].split('/')[-1] - data = get_BARRA_nc_data(barra_fname, BARRA_FIELDN, ic_date.replace('T', '').replace('Z', ''), 4, bounds) + data = get_barra_data(barra_fname, BARRA_FIELDN, date, bounds) tsl = data.copy() # Set up the output file From ee36b8b3c9857be63b12804ee2a660397c3e486f Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 17:00:45 +1100 Subject: [PATCH 25/29] Removed unused imports --- src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py index 6cd4fcc..e617cd9 100755 --- a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py @@ -6,18 +6,15 @@ # Created by: Chermelle Engel import os -import sys from glob import glob from pathlib import Path from collections import namedtuple -import iris import mule import numpy as np import xarray as xr import warnings - STASH_LAND_MASK = 30 ROSE_DATA = os.environ.get('ROSE_DATA', "") # Base directory of the ERA5-land archive on NCI From bfab0b533785e8046619dccf98bba46d634123f5 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 22:26:44 +1100 Subject: [PATCH 26/29] - Removed --mask dependency from replace_landsurface_with_ERA5land_IC and make it consistent with the BARRA2R script Updated BARRA2R script --- .../replace_landsurface.py | 6 - .../replace_landsurface_with_BARRA2R_IC.py | 84 ++--- .../replace_landsurface_with_ERA5land_IC.py | 289 +++++++----------- 3 files changed, 135 insertions(+), 244 deletions(-) diff --git a/src/replace_landsurface/replace_landsurface.py b/src/replace_landsurface/replace_landsurface.py index 778bfae..1028f54 100755 --- a/src/replace_landsurface/replace_landsurface.py +++ b/src/replace_landsurface/replace_landsurface.py @@ -8,12 +8,6 @@ """ Replace the land-surface fields in the astart file with higher-resolution data """ -#TODO: delete after testing -# barra_fname='/g/data/vk83/testing/data/replace_landsurface/integration_tests/driving_data/etc/barra_r2/1hr/ts/latest/ts_AUS-11_ERA5_historical_hres_BOM_BARRA-R2_v1_1hr_202008-202008.nc' -# fname='/g/data/tm70/cbe563/test_replace_land_surface/BARRAR2/GAL9_astart' -# mask='/scratch/tm70/cbe563/cylc-run/u-dg767.b/share/data/ancils/Lismore/d1100/qrparm.mask' -# date='202008090320' -#TODO: delete after testing import argparse import shutil diff --git a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py index e617cd9..26d4911 100755 --- a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py @@ -6,6 +6,7 @@ # Created by: Chermelle Engel import os +import warnings from glob import glob from pathlib import Path from collections import namedtuple @@ -13,14 +14,15 @@ import mule import numpy as np import xarray as xr -import warnings +DECIMAL_PRECISION = 4 STASH_LAND_MASK = 30 ROSE_DATA = os.environ.get('ROSE_DATA', "") # Base directory of the ERA5-land archive on NCI BARRA_DIR = os.path.join(ROSE_DATA, 'etc', 'barra_r2') BoundingBox = namedtuple('BoundingBox', ('latmin', 'latmax', 'lonmin', 'lonmax')) + class ReplaceOperator(mule.DataOperator): """ Mule operator for replacing the data""" def __init__(self): @@ -53,66 +55,26 @@ def get_bounding_box(mule_file): # Get the latitude extent dlat = field.bdy nlat = mule_file.integer_constants.num_rows - latmin = round(field.bzy + dlat, 4) # rounded to 4 decimal points to avoid numerical inaccuracy - latmax = round(latmin + (nlat-1)*dlat, 4) # rounded to 4 decimal points to avoid numerical inaccuracy - lat_extent=(latmin, latmax) + latmin = round(field.bzy + dlat, DECIMAL_PRECISION) # rounded to avoid numerical inaccuracy + latmax = round(latmin + (nlat-1)*dlat, DECIMAL_PRECISION) # rounded to avoid numerical inaccuracy # Get the latitude extent dlon = field.bdx nlon = mule_file.integer_constants.num_cols - lonmin = round(field.bzx + dlon, 4) # rounded to 4 decimal points to avoid numerical inaccuracy - lonmax = round(lonmin + (nlon-1)*dlon, 4) # rounded to 4 decimal points to avoid numerical inaccuracy - lon_extent=(lonmin, lonmax) - - # TODO: delete after testing - # ff=xr.DataArray.from_iris(iris.load(mask)[0]) - # latmin_orig=ff.latitude.min().data - # latmax_orig=ff.latitude.max().data - # lonmin_orig=ff.longitude.min().data - # lonmax_orig=ff.longitude.max().data - - # # Read in the file from the high-res netcdf archive - # if Path(barra_fname).exists(): - # d = xr.open_dataset(barra_fname) - # else: - # print(f'ERROR: File {barra_fname} not found', file=sys.stderr) - # sys.exit(1) + lonmin = round(field.bzx + dlon, DECIMAL_PRECISION) # rounded to avoid numerical inaccuracy + lonmax = round(lonmin + (nlon-1)*dlon, DECIMAL_PRECISION) # rounded to avoid numerical inaccuracy - # # Coping with numerical inaccuracy - # adj = (lons[1] - lons[0])/2. - - # # Get the longitude information - # lons = d['lon'].data - # # Work out which longitudes define the minimum/maximum extents of the grid of interest - # lonmin_index = np.argwhere( (lons > lonmin_orig - adj) & (lons < lonmin_orig + adj))[0][0] - # lonmax_index = np.argwhere( (lons > lonmax_orig - adj) & (lons < lonmax_orig + adj))[0][0] - # lon_extent_orig = (lons[lonmin_index],lons[lonmax_index]) - - # # Get the latitude information - # lats = d['lat'].data - # # Work out which latitudes define the minimum/maximum extents of the grid of interest - # latmin_index = np.argwhere( (lats > latmin_orig - adj) & (lats < latmin_orig + adj))[0][0] - # latmax_index = np.argwhere( (lats > latmax_orig - adj) & (lats < latmax_orig + adj))[0][0] - # lat_extent_orig = (lats[latmin_index],lats[latmax_index]) - - # print(lon_extent) - # print(lon_extent_orig) - # print(lat_extent) - # print(lat_extent_orig) - # TODO: delete after testing - - # Return the boundaries return BoundingBox(latmin, latmax, lonmin, lonmax) -def get_barra_data(barra_fname, FIELDN, date, bounds): +def get_barra_data(fname, FIELDN, date, bounds): """ Function to get the BARA2-R data for a single land/surface variable. Parameters ---------- - barra_fname : string - The name of the file to read + fname : string + The name of the high-res file to read FIELDN : string The name of the variable in the file to read date : string @@ -127,17 +89,17 @@ def get_barra_data(barra_fname, FIELDN, date, bounds): """ # Open the file containing the data - if Path(barra_fname).exists(): - d = xr.open_dataset(barra_fname) + if Path(fname).exists(): + d = xr.open_dataset(fname) else: - raise FileNotFoundError(f"File '{barra_fname}' not found.") + raise FileNotFoundError(f"File '{fname}' not found.") # Get the data # Select the variable try: data = d[FIELDN] except KeyError: - raise ValueError(f"Variable '{FIELDN}' not found in high-resolution file '{barra_fname}'.") + raise ValueError(f"Variable '{FIELDN}' not found in high-resolution file '{fname}'.") # Select the date try: data = data.sel(time=date) @@ -146,7 +108,7 @@ def get_barra_data(barra_fname, FIELDN, date, bounds): data = data.sel(time=date, method='nearest') actual_date = data.time.values.astype('datetime64[m]').tolist().strftime('%Y%m%d%H%M') warnings.warn( - f"The date '{date}' was not found in the high-resolution file '{barra_fname}'.\n" + f"The date '{date}' was not found in the high-resolution file '{fname}'.\n" f"Using the nearest date '{actual_date}' instead." ) # Select the spatial extent @@ -174,10 +136,6 @@ def swap_land_barra(ec_cb_file_fullpath, date): None. The file is replaced with a version of itself holding the higher-resolution data. """ - - - # create name of file to be replaced - ec_cb_file = ec_cb_file_fullpath.parts[-1].replace('.tmp', '') # create date/time useful information yyyy = date[0:4] @@ -185,14 +143,18 @@ def swap_land_barra(ec_cb_file_fullpath, date): # Path to input file ff_in = ec_cb_file_fullpath.as_posix().replace('.tmp', '') - # Path to output file ff_out = ec_cb_file_fullpath.as_posix() - #print(ff_in, ff_out) # Read input file mf_in = mule.load_umfile(ff_in) + # Set up the output file + mf_out = mf_in.copy() + + # Get spatial extent for the replacement + bounds = get_bounding_box(mf_in) + # Create Mule Replacement Operator replace = ReplaceOperator() @@ -202,8 +164,6 @@ def swap_land_barra(ec_cb_file_fullpath, date): barra_files = glob(os.path.join(indir, BARRA_FIELDN + '*' + yyyy + mm + '*nc')) barra_fname = indir + '/' + barra_files[0].split('/')[-1] - # Work out the grid bounds using the surface temperature file - bounds = get_bounding_box(mf_in) # Read in the surface temperature data (and keep to use for replacement) data = get_barra_data(barra_fname, BARRA_FIELDN, date, bounds) @@ -225,8 +185,6 @@ def swap_land_barra(ec_cb_file_fullpath, date): data = get_barra_data(barra_fname, BARRA_FIELDN, date, bounds) tsl = data.copy() - # Set up the output file - mf_out = mf_in.copy() # For each field in the input write to the output file (but modify as required) for f in mf_in.fields: diff --git a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py index 8d3bb92..3169bfc 100755 --- a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py @@ -6,22 +6,25 @@ # Created by: Chermelle Engel import os -import sys +import warnings from glob import glob from pathlib import Path +from collections import namedtuple -import iris import mule import numpy as np import xarray as xr +DECIMAL_PRECISION = 4 +STASH_LAND_MASK = 30 ROSE_DATA = os.environ.get('ROSE_DATA', "") # Base directory of the ERA5-land archive on NCI ERA_DIR = os.path.join(ROSE_DATA, 'etc', 'era5_land') # The depths of soil for the conversion -##########multipliers=[7.*10., 21.*10., 72.*10., 189.*10.] -multipliers = [10.*10., 25.*10., 65.*10., 200.*10.] +SOIL_MULTIPLIERS = (100., 250., 650., 2000.) + +BoundingBox = namedtuple('BoundingBox', ('latmin', 'latmax', 'lonmin', 'lonmax')) class ReplaceOperator(mule.DataOperator): """ Mule operator for replacing the data""" @@ -34,100 +37,53 @@ def transform(self, sources, result): #print('transform') return sources[1] -class bounding_box(): - """ Container class to hold spatial extent information.""" - def __init__(self, ncfname, maskfname, var): - """ - Initialization function for bounding_box class - - Parameters - ---------- - ncfname : POSIX string - POSIX path to input data (from the NetCDF archive) - maskfname : POSIX string - POSIX path to mask information to define the data to be cut out - var : string - The name of the mask variable that defines the spatial extent - - Returns - ------- - None. The variables describing the spatial extent are definied within bounding box object. - - """ - - # Read in the mask and get the minimum/maximum latitude and longitude information - if Path(maskfname).exists(): - d = iris.load(maskfname, var) - d = d[0] - d = xr.DataArray.from_iris(d) - - lons = d['longitude'].data - lonmin = np.min(lons) - lonmax = np.max(lons) - - if lonmax > 180.: - lonmax = lonmax-360. - - lats = d['latitude'].data - latmin = np.min(lats) - latmax = np.max(lats) - d.close() - - else: - print(f'ERROR: File {maskfname} not found', file=sys.stderr) - raise - - # Read in the file from the high-res netcdf archive - if Path(ncfname).exists(): - d = xr.open_dataset(ncfname) - else: - print(f'ERROR: File {ncfname} not found', file=sys.stderr) - sys.exit(1) - - # Get the longitude information - lons = d['longitude'].data - - # Coping with numerical inaccuracy - adj = (lons[1] - lons[0])/2. - - # Work out which longitudes define the minimum/maximum extents of the grid of interest - lonmin_index = np.argwhere((lons > lonmin - adj) & (lons < lonmin + adj))[0][0] - lonmax_index = np.argwhere((lons > lonmax - adj) & (lons < lonmax + adj))[0][0] - - # Get the latitude information - lats = d['latitude'].data - - # Work out which longitudes define the minimum/maximum extents of the grid of interest - # use the same adjustment as for longitude - latmin_index = np.argwhere((lats > latmin - adj) & (lats < latmin + adj))[0][0] - latmax_index = np.argwhere((lats > latmax - adj) & (lats < latmax + adj))[0][0] +def get_bounding_box(mule_file): + """ Get spatial extent information for the replacement. + + Parameters + ---------- + mule_file : mule.UMFile object + Input UM fields file - # Swap the latitude min/max if upside down (is upside down for era5-land) - if latmax_index < latmin_index: - tmp_index=latmin_index - latmin_index=latmax_index - latmax_index=tmp_index + Returns + ------- + namedtuple + The BoundingBox namedtuple + """ + # Get extent from the land mask (stash 30) if present, otherwise use the first field + # because the file extent sometimes is not correct + try: + field = [f for f in mule_file.fields if f.lbuser4 == STASH_LAND_MASK][0] + except IndexError: + field = mule_file.fields[0] + # Get the latitude extent + dlat = field.bdy + nlat = mule_file.integer_constants.num_rows + latmin = round(field.bzy + dlat, DECIMAL_PRECISION) # rounded to avoid numerical inaccuracy + latmax = round(latmin + (nlat-1)*dlat, DECIMAL_PRECISION) # rounded to avoid numerical inaccuracy + + # Get the latitude extent + dlon = field.bdx + nlon = mule_file.integer_constants.num_cols + lonmin = round(field.bzx + dlon, DECIMAL_PRECISION) # rounded to avoid numerical inaccuracy + lonmax = round(lonmin + (nlon-1)*dlon, DECIMAL_PRECISION) # rounded to avoid numerical inaccuracy - # Set the boundaries - self.lonmin=lonmin_index - self.lonmax=lonmax_index - self.latmin=latmin_index - self.latmax=latmax_index + return BoundingBox(latmin, latmax, lonmin, lonmax) -def get_ERA_nc_data(ncfname, FIELDN, wanted_dt, bounds): +def get_era_data(fname, FIELDN, date, bounds): """ - Function to get the ERA5-land data for a single land/surface variable. + Function to get the BARA2-R data for a single land/surface variable. Parameters ---------- - ncfname : string - The name of the file to read + fname : string + The name of the high-res file to read FIELDN : string The name of the variable in the file to read - wanted_dt : string - The date-time required in "%Y%m%d%H%M" format - bounds : bounding_box object - A bounding box object defining the spatial extent to keep + date : string + The date in the format "YYYYmmddHHMM". + bounds : tuple + Tuple containing the spatial extent for the data in the form (latmin, latmax, lonmin, lonmax). Returns ------- @@ -135,60 +91,52 @@ def get_ERA_nc_data(ncfname, FIELDN, wanted_dt, bounds): A 2-D numpy array containg the field data for the date/time and spatial extent """ - # retrieve the spatial extends of interest - lonmin_index, lonmax_index = bounds.lonmin, bounds.lonmax - latmin_index, latmax_index = bounds.latmin, bounds.latmax - # Open the file containing the data - #print(ncfname, FIELDN) - if Path(ncfname).exists(): - d = xr.open_dataset(ncfname) - else: - print(f'ERROR: File {ncfname} not found', file=sys.stderr) - sys.exit(1) - - # Find the array index for the date/time of interest - times=d['time'].dt.strftime("%Y%m%d%H%M").data - TM=times.tolist().index(wanted_dt) - #print(TM) - - # Read the data - if lonmin_index < lonmax_index: - - # Grid does not wrap around. Simple read. - - try: - data=d[FIELDN][TM, latmin_index:latmax_index+1, lonmin_index:lonmax_index+1] - data=data.data - - except KeyError: - print(fname) - print(f'ERROR: Variable temp not found in file', file=sys.stderr) - sys.exit(1) + if Path(fname).exists(): + data = xr.open_dataset(fname) else: - - # Data required wraps around the input grid. Read in sections and patch together. - - try: - data_left=d[FIELDN][TM, latmin_index:latmax_index+1, lonmin_index:].data - data_right=d[FIELDN][TM, latmin_index:latmax_index+1, 0:lonmax_index+1].data - data=np.concatenate((data_left, data_right), axis=1) - - except KeyError: - print(fname) - print(f'ERROR: Variable temp not found in file', file=sys.stderr) - sys.exit(1) - - d.close() - - # Flip the data vertically because the era5-land latitudes are reversed in direction to the UM FF - return data[::-1, :] - -def replace_in_ff(f, generic_era5_fname, ERA_FIELDN, multiplier, ic_z_date, mf_out, replace, bounds): + raise FileNotFoundError(f"File '{fname}' not found.") + + # Get the data + + # Get data in the same structure as UM files: ascending latitude (-90 90) and positive longitude (0 360) + if data.latitude[0] > data.latitude[1]: # descending latitude + data = data.reindex(latitude=data.latitude[::-1]) # flip the latitude + if any(data.longitude < 0): # negative longitude (-180 180) + # longitude = np.round(data.longitude, DECIMAL_PRECISION) + longitude = data.longitude + new_index = [lon for lon in longitude if lon >= 0] + [lon for lon in longitude if lon < 0] # move negative longitudes to the end + data = data.reindex(longitude=new_index) + new_longitude = [lon if lon >= 0 else (360+lon) for lon in new_index] # convert negative longitudes to positives > 180 + data = data.assign_coords(longitude=new_longitude) + # Select the variable + try: + data = data[FIELDN] + except KeyError: + raise ValueError(f"Variable '{FIELDN}' not found in high-resolution file '{fname}'.") + # Select the date + try: + data = data.sel(time=date) + except KeyError: + # If the exact date is not found select the nearest date and send a warning message + data = data.sel(time=date, method='nearest') + actual_date = data.time.values.astype('datetime64[m]').tolist().strftime('%Y%m%d%H%M') + warnings.warn( + f"The date '{date}' was not found in the high-resolution file '{fname}'.\n" + f"Using the nearest date '{actual_date}' instead." + ) + # Select the spatial extent + data = data.sel( + latitude=slice(bounds.latmin, bounds.latmax), + longitude=slice(bounds.lonmin, bounds.lonmax), + ) + return data.data + +def replace_in_ff(f, generic_era5_fname, ERA_FIELDN, multiplier, date, mf_out, replace, bounds): current_data = f.get_data() era5_fname = generic_era5_fname.replace('FIELDN', ERA_FIELDN) - data = get_ERA_nc_data(era5_fname, ERA_FIELDN, ic_z_date, bounds) + data = get_era_data(era5_fname, ERA_FIELDN, date, bounds) if multiplier < 0: pass else: @@ -196,18 +144,16 @@ def replace_in_ff(f, generic_era5_fname, ERA_FIELDN, multiplier, ic_z_date, mf_o data = np.where(np.isnan(data), current_data, data) mf_out.fields.append(replace([f, data])) -def swap_land_era5land(mask_fullpath, ic_file_fullpath, ic_date): +def swap_land_era5land(ic_file_fullpath, date): """ Function to get the ERA5-land data for all land/surface variables. Parameters ---------- - mask_fullpath : Path - Path to the mask defining the spatial extent ic_file_fullpath : string Path to file with the coarser resolution data to be replaced with ".tmp" appended at end - ic_date : string - The date-time required in "%Y%m%d%H%M" format + date : string + The date in the format "YYYYmmddHHMM". Returns ------- @@ -220,67 +166,60 @@ def swap_land_era5land(mask_fullpath, ic_file_fullpath, ic_date): ic_file = ic_file_fullpath.parts[-1].replace('.tmp', '') # create date/time useful information - #print(ic_date) - yyyy = ic_date[0:4] - mm = ic_date[4:6] - ic_z_date = ic_date.replace('T', '').replace('Z', '') - - - # Find one "swvl1" file in the archive and create a generic filename - ERA_FIELDN = 'swvl1' - land_yes = os.path.join(ERA_DIR, ERA_FIELDN, yyyy) - era_files = glob(os.path.join(land_yes, ERA_FIELDN + '*' + yyyy + mm + '*nc')) - era5_fname = os.path.join(land_yes, os.path.basename(era_files[0])) - generic_era5_fname = era5_fname.replace('swvl1', 'FIELDN') + yyyy = date[0:4] + mm = date[4:6] # Path to input file ff_in = ic_file_fullpath.as_posix().replace('.tmp', '') - # Path to output file ff_out = ic_file_fullpath.as_posix() - #print(ff_in, ff_out) - + # Read input file mf_in = mule.load_umfile(ff_in) - - # Create Mule Replacement Operator - replace = ReplaceOperator() - - # Define spatial extent of grid required - bounds = bounding_box(era5_fname, mask_fullpath.as_posix(), "land_binary_mask") # Set up the output file mf_out = mf_in.copy() + + # Get spatial extent for the replacement + bounds = get_bounding_box(mf_in) + + # Create Mule Replacement Operator + replace = ReplaceOperator() + # Find one "swvl1" file in the archive and create a generic filename + ERA_FIELDN = 'swvl1' + indir = os.path.join(ERA_DIR, ERA_FIELDN, yyyy) + era_files = glob(os.path.join(indir, ERA_FIELDN + '*' + yyyy + mm + '*nc')) + era5_fname = os.path.join(indir, os.path.basename(era_files[0])) + generic_era5_fname = era5_fname.replace('swvl1', 'FIELDN') + # For each field in the input write to the output file (but modify as required) for f in mf_in.fields: - - #print(f.lbuser4, f.lblev, f.lblrec, f.lbhr, f.lbcode) if f.lbuser4 == 9: # replace coarse soil moisture with high-res information if f.lblev == 4: - replace_in_ff(f, generic_era5_fname, 'swvl4', multipliers[3], ic_z_date, mf_out, replace, bounds) + replace_in_ff(f, generic_era5_fname, 'swvl4', SOIL_MULTIPLIERS[3], date, mf_out, replace, bounds) elif f.lblev == 3: - replace_in_ff(f, generic_era5_fname, 'swvl3', multipliers[2], ic_z_date, mf_out, replace, bounds) + replace_in_ff(f, generic_era5_fname, 'swvl3', SOIL_MULTIPLIERS[2], date, mf_out, replace, bounds) elif f.lblev == 2: - replace_in_ff(f, generic_era5_fname, 'swvl2', multipliers[1], ic_z_date, mf_out, replace, bounds) + replace_in_ff(f, generic_era5_fname, 'swvl2', SOIL_MULTIPLIERS[1], date, mf_out, replace, bounds) elif f.lblev == 1: - replace_in_ff(f, generic_era5_fname, 'swvl1', multipliers[0], ic_z_date, mf_out, replace, bounds) + replace_in_ff(f, generic_era5_fname, 'swvl1', SOIL_MULTIPLIERS[0], date, mf_out, replace, bounds) elif f.lbuser4 == 20: # soil temperature if f.lblev == 4: - replace_in_ff(f, generic_era5_fname, 'stl4', -1, ic_z_date, mf_out, replace, bounds) + replace_in_ff(f, generic_era5_fname, 'stl4', -1, date, mf_out, replace, bounds) elif f.lblev == 3: - replace_in_ff(f, generic_era5_fname, 'stl3', -1, ic_z_date, mf_out, replace, bounds) + replace_in_ff(f, generic_era5_fname, 'stl3', -1, date, mf_out, replace, bounds) elif f.lblev == 2: - replace_in_ff(f, generic_era5_fname, 'stl2', -1, ic_z_date, mf_out, replace, bounds) + replace_in_ff(f, generic_era5_fname, 'stl2', -1, date, mf_out, replace, bounds) elif f.lblev == 1: - replace_in_ff(f, generic_era5_fname, 'stl1', -1, ic_z_date, mf_out, replace, bounds) + replace_in_ff(f, generic_era5_fname, 'stl1', -1, date, mf_out, replace, bounds) elif f.lbuser4 == 24: - replace_in_ff(f, generic_era5_fname, 'skt', -1, ic_z_date, mf_out, replace, bounds) + replace_in_ff(f, generic_era5_fname, 'skt', -1, date, mf_out, replace, bounds) else: mf_out.fields.append(f) From ac1f9c432a7da9c5a0cdc162f789bf75235a23e5 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sat, 22 Feb 2025 22:47:52 +1100 Subject: [PATCH 27/29] - Removed dependency of replace_landsurface_with_FF_IC from --mask and other redundant variables - General minor updates --- .../replace_landsurface.py | 2 +- .../replace_landsurface_with_BARRA2R_IC.py | 2 -- .../replace_landsurface_with_ERA5land_IC.py | 2 -- .../replace_landsurface_with_FF_IC.py | 28 ++++++------------- 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/replace_landsurface/replace_landsurface.py b/src/replace_landsurface/replace_landsurface.py index 1028f54..74a3a6d 100755 --- a/src/replace_landsurface/replace_landsurface.py +++ b/src/replace_landsurface/replace_landsurface.py @@ -51,7 +51,7 @@ def main(): replace_landsurface_with_BARRA2R_IC.swap_land_barra(args.file, args.start) shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) elif "astart" in args.type: - replace_landsurface_with_FF_IC.swap_land_ff(args.file, args.hres_ic, args.start) + replace_landsurface_with_FF_IC.swap_land_ff(args.file, args.hres_ic) shutil.move(args.file.as_posix(), args.file.as_posix().replace('.tmp', '')) else: print("No need to swap out IC") diff --git a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py index 26d4911..1c129c6 100755 --- a/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py @@ -188,8 +188,6 @@ def swap_land_barra(ec_cb_file_fullpath, date): # For each field in the input write to the output file (but modify as required) for f in mf_in.fields: - - #print(f.lbuser4, f.lblev, f.lblrec, f.lbhr, f.lbcode) if f.lbuser4 == 9: # replace coarse soil moisture with high-res information current_data = f.get_data() diff --git a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py index 3169bfc..89f3c4f 100755 --- a/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py @@ -31,10 +31,8 @@ class ReplaceOperator(mule.DataOperator): def __init__(self): pass def new_field(self, sources): - #print('new_field') return sources[0] def transform(self, sources, result): - #print('transform') return sources[1] def get_bounding_box(mule_file): diff --git a/src/replace_landsurface/replace_landsurface_with_FF_IC.py b/src/replace_landsurface/replace_landsurface_with_FF_IC.py index a5b3bc5..86da010 100755 --- a/src/replace_landsurface/replace_landsurface_with_FF_IC.py +++ b/src/replace_landsurface/replace_landsurface_with_FF_IC.py @@ -12,63 +12,53 @@ class ReplaceOperator(mule.DataOperator): def __init__(self): pass def new_field(self, sources): - print('new_field') return sources[0] def transform(self, sources, result): - print('transform') return sources[1] def replace_in_ff_from_ff(f, sf, mf_out, replace): - replacement_data = sf.get_data() mf_out.fields.append(replace([f, replacement_data])) -def swap_land_ff(mask_fullpath, ic_file_fullpath, source_fullpath, ic_date): +def swap_land_ff(ic_file_fullpath, source_fullpath): """ Function to get the land/surface data from another fields file into the start dump. Parameters ---------- - mask_fullpath : Path - Path to the mask defining the spatial extent ic_file_fullpath : Path Path to file with the coarser resolution data to be replaced with ".tmp" appended at end source_fullpath : Path Path to source fields file to take the land/surface data from - ic_date : string - The date-time required in "%Y%m%d%H%M" format Returns ------- None. The file is replaced with a version of itself holding the higher-resolution data. """ - - print(mask_fullpath, ic_file_fullpath, source_fullpath,ic_date) # Path to input file ff_in = ic_file_fullpath.as_posix().replace('.tmp', '') - # Path to input file - sf_in = source_fullpath.as_posix() - # Path to output file ff_out = ic_file_fullpath.as_posix() - print(ff_in, ff_out) - + + # Path to source input file + sf_in = source_fullpath.as_posix() + # Read input file mf_in = mule.load_umfile(ff_in) + # Read source file msf_in = mule.load_umfile(sf_in) - # Create Mule Replacement Operator - replace = ReplaceOperator() - # Set up the output file mf_out = mf_in.copy() + + # Create Mule Replacement Operator + replace = ReplaceOperator() # For each field in the input write to the output file (but modify as required) for f,sf in zip(mf_in.fields,msf_in.fields): - if f.lbuser4 in [9, 20, 24]: replace_in_ff_from_ff(f, sf, mf_out, replace) else: From 40eb164b1be0579fd4a977a195b3413b1d66c2a9 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sun, 2 Mar 2025 18:05:40 +1100 Subject: [PATCH 28/29] Removed mask input from regression tests --- src/replace_landsurface.egg-info/PKG-INFO | 267 ++++++++++++++++++ src/replace_landsurface.egg-info/SOURCES.txt | 16 ++ .../dependency_links.txt | 1 + .../entry_points.txt | 2 + src/replace_landsurface.egg-info/requires.txt | 6 + .../top_level.txt | 1 + tests/integration/test_integration.py | 2 - 7 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 src/replace_landsurface.egg-info/PKG-INFO create mode 100644 src/replace_landsurface.egg-info/SOURCES.txt create mode 100644 src/replace_landsurface.egg-info/dependency_links.txt create mode 100644 src/replace_landsurface.egg-info/entry_points.txt create mode 100644 src/replace_landsurface.egg-info/requires.txt create mode 100644 src/replace_landsurface.egg-info/top_level.txt diff --git a/src/replace_landsurface.egg-info/PKG-INFO b/src/replace_landsurface.egg-info/PKG-INFO new file mode 100644 index 0000000..83c8d38 --- /dev/null +++ b/src/replace_landsurface.egg-info/PKG-INFO @@ -0,0 +1,267 @@ +Metadata-Version: 2.2 +Name: replace-landsurface +Version: 1.0.1+23.g3b0df76.dirty +Summary: Package to replace specific land surface fields to be used within ACCESS-RAM suites. +Author-email: Chermelle Engel , Davide Marchegiani , Paul Leopardi +Maintainer-email: ACCESS-NRI +License: Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Project-URL: Repository, https://github.com/ACCESS-NRI/replace_landsurface +Keywords: ACCESS-RAM,Regional Nesting Suite,Replace Land Surface +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: python<=3.12,>=3.10 +Requires-Dist: mule +Requires-Dist: numpy==1.23.4 +Requires-Dist: scitools-iris +Requires-Dist: xarray +Requires-Dist: versioneer + +# replace_landsurface + +## About + +`replace_landsurface` is a `Python` utility to be used within ACCESS-NRI versions of the Regional Nesting Suites to replace specific land surface initial/boundary conditions. + + +## Development/Testing instructions +For development/testing, it is recommended to install `replace_landsurface` as a development package within a `micromamba`/`conda` testing environment. + +### Clone replace_landsurface GitHub repo +``` +git clone git@github.com:ACCESS-NRI/replace_landsurface.git +``` + +### Create a micromamba/conda testing environment +> [!TIP] +> In the following instructions `micromamba` can be replaced with `conda`. + +``` +cd replace_landsurface +micromamba env create -n replace_landsurface_dev --file .conda/env_dev.yml +micromamba activate replace_landsurface_dev +``` + +### Install replace_landsurface as a development package +``` +pip install --no-deps --no-build-isolation -e . +``` + +### Running the tests + +The test suite currently includes only integration tests. + +To manually run the tests, from the `replace_landsurface` directory, you can: + +1. Activate your [micromamba/conda testing environment](#create-a-micromamba-conda-testing-environment) +2. Run the following command: + ``` + pytest -n 4 + ``` + +> [!TIP] +> The `-n 4` option is a [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/) option to run the tests in parallel across 4 different workers. + +> [!IMPORTANT] +> Integration tests are designed to be run on `Gadi`. +> If you run tests on a local machine, the integration tests will be skipped. diff --git a/src/replace_landsurface.egg-info/SOURCES.txt b/src/replace_landsurface.egg-info/SOURCES.txt new file mode 100644 index 0000000..ce9bcef --- /dev/null +++ b/src/replace_landsurface.egg-info/SOURCES.txt @@ -0,0 +1,16 @@ +LICENSE +README.md +pyproject.toml +setup.py +src/replace_landsurface/__init__.py +src/replace_landsurface/_version.py +src/replace_landsurface/replace_landsurface.py +src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py +src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py +src/replace_landsurface/replace_landsurface_with_FF_IC.py +src/replace_landsurface.egg-info/PKG-INFO +src/replace_landsurface.egg-info/SOURCES.txt +src/replace_landsurface.egg-info/dependency_links.txt +src/replace_landsurface.egg-info/entry_points.txt +src/replace_landsurface.egg-info/requires.txt +src/replace_landsurface.egg-info/top_level.txt \ No newline at end of file diff --git a/src/replace_landsurface.egg-info/dependency_links.txt b/src/replace_landsurface.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/replace_landsurface.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/replace_landsurface.egg-info/entry_points.txt b/src/replace_landsurface.egg-info/entry_points.txt new file mode 100644 index 0000000..74b85e1 --- /dev/null +++ b/src/replace_landsurface.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +replace_landsurface = replace_landsurface.replace_landsurface:main diff --git a/src/replace_landsurface.egg-info/requires.txt b/src/replace_landsurface.egg-info/requires.txt new file mode 100644 index 0000000..dc222dc --- /dev/null +++ b/src/replace_landsurface.egg-info/requires.txt @@ -0,0 +1,6 @@ +python<=3.12,>=3.10 +mule +numpy==1.23.4 +scitools-iris +xarray +versioneer diff --git a/src/replace_landsurface.egg-info/top_level.txt b/src/replace_landsurface.egg-info/top_level.txt new file mode 100644 index 0000000..cc1fdb7 --- /dev/null +++ b/src/replace_landsurface.egg-info/top_level.txt @@ -0,0 +1 @@ +replace_landsurface diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index db238e5..3317d83 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -46,8 +46,6 @@ def get_test_args(num, start, _type): "script_name", "--file", os.path.join(INPUT_DIR, test_dir, "file" + ".tmp"), - "--mask", - os.path.join(INPUT_DIR, test_dir, "mask"), "--start", start, "--type", From 897295e2cde482f2f19ac8b53c92ee688967c050 Mon Sep 17 00:00:00 2001 From: Davide Marchegiani Date: Sun, 2 Mar 2025 19:05:25 +1100 Subject: [PATCH 29/29] Removed egg-info folder Updated gitignore --- .gitignore | 4 +- src/replace_landsurface.egg-info/PKG-INFO | 267 ------------------ src/replace_landsurface.egg-info/SOURCES.txt | 16 -- .../dependency_links.txt | 1 - .../entry_points.txt | 2 - src/replace_landsurface.egg-info/requires.txt | 6 - .../top_level.txt | 1 - 7 files changed, 3 insertions(+), 294 deletions(-) delete mode 100644 src/replace_landsurface.egg-info/PKG-INFO delete mode 100644 src/replace_landsurface.egg-info/SOURCES.txt delete mode 100644 src/replace_landsurface.egg-info/dependency_links.txt delete mode 100644 src/replace_landsurface.egg-info/entry_points.txt delete mode 100644 src/replace_landsurface.egg-info/requires.txt delete mode 100644 src/replace_landsurface.egg-info/top_level.txt diff --git a/.gitignore b/.gitignore index 97b8d67..072df77 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ __pycache__/ .ruff_cache/ .mypy_cache/ .pytest_cache/ -.vscode/ \ No newline at end of file +.vscode/ +build/ +*.egg-info/ \ No newline at end of file diff --git a/src/replace_landsurface.egg-info/PKG-INFO b/src/replace_landsurface.egg-info/PKG-INFO deleted file mode 100644 index 83c8d38..0000000 --- a/src/replace_landsurface.egg-info/PKG-INFO +++ /dev/null @@ -1,267 +0,0 @@ -Metadata-Version: 2.2 -Name: replace-landsurface -Version: 1.0.1+23.g3b0df76.dirty -Summary: Package to replace specific land surface fields to be used within ACCESS-RAM suites. -Author-email: Chermelle Engel , Davide Marchegiani , Paul Leopardi -Maintainer-email: ACCESS-NRI -License: Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -Project-URL: Repository, https://github.com/ACCESS-NRI/replace_landsurface -Keywords: ACCESS-RAM,Regional Nesting Suite,Replace Land Surface -Description-Content-Type: text/markdown -License-File: LICENSE -Requires-Dist: python<=3.12,>=3.10 -Requires-Dist: mule -Requires-Dist: numpy==1.23.4 -Requires-Dist: scitools-iris -Requires-Dist: xarray -Requires-Dist: versioneer - -# replace_landsurface - -## About - -`replace_landsurface` is a `Python` utility to be used within ACCESS-NRI versions of the Regional Nesting Suites to replace specific land surface initial/boundary conditions. - - -## Development/Testing instructions -For development/testing, it is recommended to install `replace_landsurface` as a development package within a `micromamba`/`conda` testing environment. - -### Clone replace_landsurface GitHub repo -``` -git clone git@github.com:ACCESS-NRI/replace_landsurface.git -``` - -### Create a micromamba/conda testing environment -> [!TIP] -> In the following instructions `micromamba` can be replaced with `conda`. - -``` -cd replace_landsurface -micromamba env create -n replace_landsurface_dev --file .conda/env_dev.yml -micromamba activate replace_landsurface_dev -``` - -### Install replace_landsurface as a development package -``` -pip install --no-deps --no-build-isolation -e . -``` - -### Running the tests - -The test suite currently includes only integration tests. - -To manually run the tests, from the `replace_landsurface` directory, you can: - -1. Activate your [micromamba/conda testing environment](#create-a-micromamba-conda-testing-environment) -2. Run the following command: - ``` - pytest -n 4 - ``` - -> [!TIP] -> The `-n 4` option is a [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/) option to run the tests in parallel across 4 different workers. - -> [!IMPORTANT] -> Integration tests are designed to be run on `Gadi`. -> If you run tests on a local machine, the integration tests will be skipped. diff --git a/src/replace_landsurface.egg-info/SOURCES.txt b/src/replace_landsurface.egg-info/SOURCES.txt deleted file mode 100644 index ce9bcef..0000000 --- a/src/replace_landsurface.egg-info/SOURCES.txt +++ /dev/null @@ -1,16 +0,0 @@ -LICENSE -README.md -pyproject.toml -setup.py -src/replace_landsurface/__init__.py -src/replace_landsurface/_version.py -src/replace_landsurface/replace_landsurface.py -src/replace_landsurface/replace_landsurface_with_BARRA2R_IC.py -src/replace_landsurface/replace_landsurface_with_ERA5land_IC.py -src/replace_landsurface/replace_landsurface_with_FF_IC.py -src/replace_landsurface.egg-info/PKG-INFO -src/replace_landsurface.egg-info/SOURCES.txt -src/replace_landsurface.egg-info/dependency_links.txt -src/replace_landsurface.egg-info/entry_points.txt -src/replace_landsurface.egg-info/requires.txt -src/replace_landsurface.egg-info/top_level.txt \ No newline at end of file diff --git a/src/replace_landsurface.egg-info/dependency_links.txt b/src/replace_landsurface.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/src/replace_landsurface.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/replace_landsurface.egg-info/entry_points.txt b/src/replace_landsurface.egg-info/entry_points.txt deleted file mode 100644 index 74b85e1..0000000 --- a/src/replace_landsurface.egg-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -replace_landsurface = replace_landsurface.replace_landsurface:main diff --git a/src/replace_landsurface.egg-info/requires.txt b/src/replace_landsurface.egg-info/requires.txt deleted file mode 100644 index dc222dc..0000000 --- a/src/replace_landsurface.egg-info/requires.txt +++ /dev/null @@ -1,6 +0,0 @@ -python<=3.12,>=3.10 -mule -numpy==1.23.4 -scitools-iris -xarray -versioneer diff --git a/src/replace_landsurface.egg-info/top_level.txt b/src/replace_landsurface.egg-info/top_level.txt deleted file mode 100644 index cc1fdb7..0000000 --- a/src/replace_landsurface.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -replace_landsurface