From 4729e2f3e2c2547a90c15ad73f48675e2e5fa010 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 20 May 2026 16:43:48 +0100 Subject: [PATCH 1/5] add nginx test script by varsiha --- varsiha.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 varsiha.py diff --git a/varsiha.py b/varsiha.py new file mode 100644 index 00000000..c0c8d9ac --- /dev/null +++ b/varsiha.py @@ -0,0 +1,72 @@ +import concurrent.futures +import numpy as np +import fsspec +import pyfive + +def load_from_https(uri): + """ + opening https file from uri using fsspec and pyfive + """ + + client_kwargs = {'auth': None} + fs = fsspec.filesystem('http', **client_kwargs) + http_file = fs.open(uri, 'rb') + + ds = pyfive.File(http_file) + print(f"Dataset loaded from https with Pyfive: {uri}") + return ds + +def _iterate_range(i): + """ + to iterate over various slices of the dataset + """ + return ds[0:i] # np.min(ds[0:i]) + + +# ------------------------------------------ +# --- Configuration which server testing --- +# ------------------------------------------ + +current_test = 'CEDA' + +# Define your environments +servers = { + 'CEDA': { + 'uri': "https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/AerChemMIP/MOHC/UKESM1-0-LL/ssp370SST-lowNTCF/r1i1p1f2/Amon/cl/gn/latest/cl_Amon_UKESM1-0-LL_ssp370SST-lowNTCF_r1i1p1f2_gn_205001-209912.nc", + 'var': "cl" + }, + 'DKRZ': { + 'uri': "http://esgf3.dkrz.de/thredds/fileServer/cmip6/RFMIP/MPI-M/MPI-ESM1-2-LR/piClim-spAer-anthro/r1i1p1f2/Amon/clw/gn/v20190710/clw_Amon_MPI-ESM1-2-LR_piClim-spAer-anthro_r1i1p1f2_gn_184901-187912.nc", + 'var': "clw" + }, + 'JASMIN': { + 'uri': "https://gws-access.jasmin.ac.uk/public/canari/varsiha/clw_Amon_MPI-ESM1-2-LR_piClim-spAer-anthro_r1i1p1f2_gn_184901-187912.nc", + 'var': "clw" + } +} + +if current_test not in servers: + raise ValueError(f"Unknown server: {current_test}. Choose from {list(servers.keys())}") + +# Extract settings +config = servers[current_test] +print(f"--- Running Test on {current_test} ---") + +file_obj = load_from_https(config['uri']) +ds = file_obj[config['var']] + +with concurrent.futures.ThreadPoolExecutor( + max_workers=10) as executor: + futures = [] + for i in range(10): + print("Thread ", i, " submitted") + future = executor.submit(_iterate_range, i) + futures.append(future) + print("-- Checking completed threads-- ") + for future in concurrent.futures.as_completed(futures): + try: + s = future.result() + print("Thread ", np.shape(s)[0]," completed with Success") + except Exception as exc: + print(exc) + #raise From 6a1759923b229a012fb9fa1bb8d30c48ee3bb13c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 20 May 2026 16:44:34 +0100 Subject: [PATCH 2/5] run nginx test --- .github/workflows/run-tests-nginx.yml | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/run-tests-nginx.yml diff --git a/.github/workflows/run-tests-nginx.yml b/.github/workflows/run-tests-nginx.yml new file mode 100644 index 00000000..82defe80 --- /dev/null +++ b/.github/workflows/run-tests-nginx.yml @@ -0,0 +1,38 @@ +name: Test + +on: + push: + branches: + - main + - varsiha + schedule: + - cron: '0 0 * * *' # nightly + +# Required shell entrypoint to have properly configured bash shell +defaults: + run: + shell: bash -l {0} + +jobs: + linux: + runs-on: "ubuntu-latest" + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + fail-fast: false + name: Linux Python ${{ matrix.python-version }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: conda-incubator/setup-miniconda@v4 + with: + activate-environment: activestorage + environment-file: environment.yml + python-version: ${{ matrix.python-version }} + miniforge-version: "latest" + use-mamba: true + - run: conda --version + - run: python -V + - run: conda list + - run: python varsiha.py From df6a9b60fb76b3ddad72cc334d2881bc8dce843b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 27 May 2026 16:23:04 +0100 Subject: [PATCH 3/5] up maworkers to 100 --- varsiha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varsiha.py b/varsiha.py index c0c8d9ac..086eee64 100644 --- a/varsiha.py +++ b/varsiha.py @@ -56,7 +56,7 @@ def _iterate_range(i): ds = file_obj[config['var']] with concurrent.futures.ThreadPoolExecutor( - max_workers=10) as executor: + max_workers=100) as executor: futures = [] for i in range(10): print("Thread ", i, " submitted") From aefc064d3b8fd5e072014fe992111a91297f7ba0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 27 May 2026 16:30:51 +0100 Subject: [PATCH 4/5] up range exec to 100 --- varsiha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varsiha.py b/varsiha.py index 086eee64..3c23538a 100644 --- a/varsiha.py +++ b/varsiha.py @@ -58,7 +58,7 @@ def _iterate_range(i): with concurrent.futures.ThreadPoolExecutor( max_workers=100) as executor: futures = [] - for i in range(10): + for i in range(100): print("Thread ", i, " submitted") future = executor.submit(_iterate_range, i) futures.append(future) From cb36b318628aa543b8cb1b24f3949160948c9f2f Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 27 May 2026 16:42:41 +0100 Subject: [PATCH 5/5] drop range to 50 --- varsiha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varsiha.py b/varsiha.py index 3c23538a..e00c4336 100644 --- a/varsiha.py +++ b/varsiha.py @@ -58,7 +58,7 @@ def _iterate_range(i): with concurrent.futures.ThreadPoolExecutor( max_workers=100) as executor: futures = [] - for i in range(100): + for i in range(50): print("Thread ", i, " submitted") future = executor.submit(_iterate_range, i) futures.append(future)