Merge branch 'main' into add_rpath

Author: jedwards4b

.github/workflows/autotools.yml

@@ -52,6 +52,6 @@ jobs:
         cd $GITHUB_WORKSPACE
         make check
 
-#    - name: Setup tmate session
-#      if: ${{ failure() }}
-#      uses: mxschmitt/action-tmate@v3
+    - name: Setup tmate session
+      if: ${{ failure() }}
+      uses: mxschmitt/action-tmate@v3

.github/workflows/intel.yml

@@ -16,8 +16,8 @@ jobs:
       FC: mpiifort
       # Versions should match the github tag names
       PNETCDF_VERSION: checkpoint.1.12.3
-      NETCDF_C_VERSION: v4.9.1
-      NETCDF_FORTRAN_VERSION: v4.6.0
+      NETCDF_C_VERSION: v4.9.2
+      NETCDF_FORTRAN_VERSION: v4.6.1
       HDF5_VERSION: hdf5_1_12_2
     steps:
     - uses: actions/checkout@v3
@@ -27,6 +27,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install libcurl4-gnutls-dev
         sudo apt-get install libz-dev
+        echo "/opt/intel/oneapi/compiler/2023.2.1/linux/bin/intel64/" >> $GITHUB_PATH
     - name: cache intel compiler
       id: cache-intel
       uses: actions/cache@v3
@@ -74,7 +75,7 @@ jobs:
         export PATH=$GITHUB_WORKSPACE/hdf5/bin:$GITHUB_WORKSPACE/netcdf/bin:$PATH
         export LDFLAGS="$LDFLAGS -L$GITHUB_WORKSPACE/hdf5/lib  -L/usr/lib/x86_64-linux-gnu/ -lcurl"
         export CPPFLAGS="$CPPFLAGS -I$GITHUB_WORKSPACE/hdf5/include -I$GITHUB_WORKSPACE/netcdf/include"
-        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/hdf5/lib:/usr/lib/x86_64-linux-gnu/"
+        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/hdf5/lib:/usr/lib/x86_64-linux-gnu/:/opt/intel/oneapi/compiler/2023.2.1/linux/compiler/lib/intel64_lin/"
         printenv >> $GITHUB_ENV
     - name: build-netcdf-c
       if: steps.cache-netcdf-c.outputs.cache-hit != 'true'
@@ -125,6 +126,6 @@ jobs:
         popd
       #     the following can be used by developers to login to the github server in case of errors
       #     see https://github.com/marketplace/actions/debugging-with-tmate for further details
-#    - name: Setup tmate session
-#      if: ${{ failure() }}
-#      uses: mxschmitt/action-tmate@v3
+    - name: Setup tmate session
+      if: ${{ failure() }}
+      uses: mxschmitt/action-tmate@v3

.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml

@@ -13,8 +13,8 @@ jobs:
       LDFLAGS: "-L${GITHUB_WORKSPACE}/mpich/lib "
       # Note issue https://github.com/NCAR/ParallelIO/issues/1889 netcdf integration currently only works with netcdf 4.7.4
       NETCDF_C_VERSION: v4.9.2
-      NETCDF_FORTRAN_VERSION: v4.6.0
-      MPICH_VERSION: v4.1
+      NETCDF_FORTRAN_VERSION: v4.6.1
+      MPICH_VERSION: v4.1.2
       HDF5_VERSION: hdf5_1_12_2
       FFLAGS: "-fallow-argument-mismatch"
       FCFLAGS: "-fallow-argument-mismatch"

.github/workflows/netcdf_pnetcdf_openmpi.yml

@@ -14,9 +14,9 @@ jobs:
     env:
 #      CPPFLAGS: "-I${GITHUB_WORKSPACE}/openmpi/include -I${GITHUB_WORKSPACE}/hdf5/include -I${GITHUB_WORKSPACE}/netcdf/include -I${GITHUB_WORKSPACE}/pnetcdf/include"
 #      LDFLAGS: "-L${GITHUB_WORKSPACE}/openmpi/lib -L${GITHUB_WORKSPACE}/hdf5/lib -L${GITHUB_WORKSPACE}/netcdf/lib -L${GITHUB_WORKSPACE}/pnetcdf/lib"
-      NETCDF_C_VERSION: v4.9.0
-      NETCDF_FORTRAN_VERSION: v4.6.0
-      OPENMPI_VERSION: v4.1.4
+      NETCDF_C_VERSION: v4.9.2
+      NETCDF_FORTRAN_VERSION: v4.6.1
+      OPENMPI_VERSION: v4.1.5
       PNETCDF_VERSION: checkpoint.1.12.3
       HDF5_VERSION: hdf5_1_12_2
       FCFLAGS: "-fallow-argument-mismatch"

.github/workflows/withspack.yml

@@ -59,8 +59,8 @@ jobs:
         # the || true prevents a fail if parallelio is not installed
         #        spack uninstall -y parallelio@=develop+pnetcdf+fortran ^mpich || true
         #        spack dev-build -d $GITHUB_WORKSPACE parallelio@=develop+pnetcdf+fortran ^mpich
-        spack uninstall -y parallelio@develop+pnetcdf+fortran ^mpich || true
-        spack dev-build -d $GITHUB_WORKSPACE parallelio@develop+pnetcdf+fortran ^mpich
+        spack uninstall -y parallelio@=develop+pnetcdf+fortran ^mpich || true
+        spack dev-build -d $GITHUB_WORKSPACE parallelio@=develop+pnetcdf+fortran ^mpich
 
     - name: Test parallelio
       run: |

CMakeLists.txt

@@ -357,8 +357,9 @@ int main() {return 0;}" HAVE_SZIP_WRITE)
 ###
 # Check to see if parallel filters are supported by HDF5/netcdf-c.
 ###
-check_function_exists(nc_inq_filter_avail HAVE_PAR_FILTERS)
-
+if (HAVE_NETCDF_PAR)
+  check_function_exists(nc_inq_filter_avail HAVE_PAR_FILTERS)
+endif()
 ###
 # Check to see if this is netcdf-c-4.7.2, which won't work.
 ###

examples/c/CMakeLists.txt

@@ -51,7 +51,7 @@ if (PIO_USE_MPISERIAL)
 else ()
   add_mpi_test(examplePio EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/examplePio NUMPROCS 4 TIMEOUT 60)
   add_mpi_test(example1 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/example1 NUMPROCS 4 TIMEOUT 60)
-  add_mpi_test(piorcw EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/piorcw  NUMPROCS 16 TIMEOUT 600)
+  #add_mpi_test(piorcw EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/piorcw  NUMPROCS 16 TIMEOUT 600)
   #add_mpi_test(darray_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_async NUMPROCS 5 TIMEOUT 60)
   add_mpi_test(darray_no_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_no_async NUMPROCS 4 TIMEOUT 60)
 endif ()

examples/c/piorcw.c

@@ -13,7 +13,7 @@ char *cvarw, *cvarr;
 typedef struct dimlist
 {
     char name[16];
-    int value;
+    PIO_Offset value;
 } dimlist;
 
 
@@ -78,7 +78,7 @@ int rcw_write_darray(int iosys, int rank)
     for(int i=0; i < maplen[rank]; i++)
         dsum += dvarw[i];
     if(dsum != rank)
-        printf("%d: dvarwsum = %d\n",rank, dsum);
+        printf("%d: dvarwsum = %g\n",rank, dsum);
 
     ierr = PIOc_write_darray(ncid, varid, ioid, maplen[rank], dvarw, NULL);
     free(maplen);
@@ -109,7 +109,7 @@ int rcw_read_darray(int iosys,int rank)
     char dimname[PIO_MAX_NAME];
     char varname[PIO_MAX_NAME];
 
-    int ndims, nvars, natts, unlimdim;
+    int i, nvars, natts, unlimdim;
     dimlist *dim;
 
     ierr = MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
@@ -123,18 +123,15 @@ int rcw_read_darray(int iosys,int rank)
 
     dim = (dimlist *) malloc(ndims*sizeof(dimlist));
     for(i=0; i<ndims; i++){
-        ierr = PIOc_inq_dim(ncid, i, dim[i].name, dim[i].value);
+        ierr = PIOc_inq_dim(ncid, i, dim[i].name, &(dim[i].value));
         if(ierr || debug) printf("%d %d i=%d\n",__LINE__,ierr, i);
     }
 
 
 
 
     /* TODO: support multiple variables and types*/
-    if(myvarname != NULL)
-        sprintf(varname,"%s",myvarname);
-    else
-        sprintf(varname,"var%4.4d",0);
+    sprintf(varname,"var%4.4d",0);
     ierr = PIOc_inq_varid(ncid, varname, &varid);
     if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
 
@@ -180,7 +177,7 @@ int rcw_read_darray(int iosys,int rank)
         for(int i=0; i < maplen[rank]; i++)
             dsum += dvarr[i];
         if(dsum != rank)
-            printf("%d: dsum = %d\n",rank, dsum);
+            printf("%d: dsum = %g\n",rank, dsum);
         break;
     case PIO_INT:
         ivarr = malloc(sizeof(int)*maplen[rank]);
@@ -221,7 +218,6 @@ int rcw_read_darray(int iosys,int rank)
 
 int main(int argc, char *argv[])
 {
-    struct arguments arguments;
     int ierr;
     int rank;
     int comm_size;

src/clib/pio.h

@@ -124,7 +124,7 @@ typedef struct var_desc_t
     int record;
 
     /** ID of each outstanding pnetcdf request for this variable. */
-    int *request;
+//    int *request;
 
     /** Number of requests pending with pnetcdf. */
     int nreqs;
@@ -784,7 +784,7 @@ enum PIO_ERROR_HANDLERS
 #define PIO_FILL_UINT64 NC_FILL_UINT64 /**< Default fill value for this type. */
 
 #define PIO_EINDEP  (-203)  /**< independent access error. */
-
+#define PIO_EINSUFFBUF (-219)         /**< Insufficient buffer size (pnetcdf only) */
 #define PIO_FIRST_ERROR_CODE (-500)  /**< The first error code for PIO. */
 #define PIO_EBADIOTYPE  (-500)       /**< Bad IOTYPE error. */
 #define PIO_EVARDIMMISMATCH (-501)   /**< Variable dimensions do not match in a multivar call. */

src/clib/pio_darray_int.c

@@ -427,6 +427,8 @@ write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *vari
         int rrcnt = 0; /* Number of subarray requests (pnetcdf only). */
         PIO_Offset *startlist[num_regions]; /* Array of start arrays for ncmpi_iput_varn(). */
         PIO_Offset *countlist[num_regions]; /* Array of count  arrays for ncmpi_iput_varn(). */
+
+        ierr = ncmpi_wait_all(file->fh, NC_REQ_ALL, NULL, NULL);
 #endif /* _PNETCDF */
 
         /* Process each region of data to be written. */
@@ -643,27 +645,10 @@ write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *vari
                         else /* don't flush yet, accumulate the request size */
                             vard_llen += llen;
 #else
-                        if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0)
-                        {
-                            if (!(vdesc->request = realloc(vdesc->request, sizeof(int) *
-                                                           (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK))))
-                                return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
 
-                            for (int i = vdesc->nreqs; i < vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK; i++)
-                                vdesc->request[i] = NC_REQ_NULL;
-                        }
-
-                        /* Write, in non-blocking fashion, a list of subarrays. */
-//                        PLOG((3, "about to call ncmpi_iput_varn() varids[%d] = %d rrcnt = %d, llen = %d",
-//                              nv, varids[nv], rrcnt, llen));
-//                        for(int i=0;i < llen; i++)
-//                            PLOG((3, "bufptr[%d] = %d",i,((int *)bufptr)[i]));
                         ierr = ncmpi_iput_varn(file->fh, varids[nv], rrcnt, startlist, countlist,
-                                               bufptr, llen, iodesc->mpitype, &vdesc->request[vdesc->nreqs]);
+                                               bufptr, llen, iodesc->mpitype, NULL);
 
-                        /* keeps wait calls in sync */
-                        if (vdesc->request[vdesc->nreqs] == NC_REQ_NULL)
-                            vdesc->request[vdesc->nreqs] = PIO_REQ_NULL;
 
                         vdesc->nreqs++;
 #endif
@@ -1750,58 +1735,34 @@ flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize)
      * limit, then flush to disk. */
     if (force || (usage >= pio_pnetcdf_buffer_size_limit))
     {
-        int rcnt;
         int  maxreq;
-        int reqcnt;
         maxreq = 0;
-        reqcnt = 0;
-        rcnt = 0;
 
         for (int i = 0; i < file->nvars; i++)
         {
             if ((ierr = get_var_desc(i, &file->varlist, &vdesc)))
                 return pio_err(NULL, file, ierr, __FILE__, __LINE__);
-            reqcnt += vdesc->nreqs;
             if (vdesc->nreqs > 0)
                 maxreq = i;
         }
-        int request[reqcnt];
-        int status[reqcnt];
-
         if (file->varlist)
         {
             for (int i = 0; i <= maxreq; i++)
             {
                 if ((ierr = get_var_desc(i, &file->varlist, &vdesc)))
                     return pio_err(NULL, file, ierr, __FILE__, __LINE__);
-#ifdef MPIO_ONESIDED
-                /*onesided optimization requires that all of the requests in a wait_all call represent
-                  a contiguous block of data in the file */
-                if (rcnt > 0 && (prev_record != vdesc->record || vdesc->nreqs == 0))
-                {
-                    ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
-                    rcnt = 0;
-                }
-                prev_record = vdesc->record;
-#endif
-                for (reqcnt = 0; reqcnt < vdesc->nreqs; reqcnt++)
-                    request[rcnt++] = max(vdesc->request[reqcnt], NC_REQ_NULL);
-                PLOG((3,"flush_output_buffer rcnt=%d",rcnt));
 
-                if (vdesc->request != NULL)
-                    free(vdesc->request);
-                vdesc->request = NULL;
                 vdesc->nreqs = 0;
 
 #ifdef FLUSH_EVERY_VAR
-                ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
+                ierr = ncmpi_wait_all(file->fh, NC_REQ_ALL, NULL, NULL);
+                //ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
                 rcnt = 0;
 #endif
             }
         }
-
-        if (rcnt > 0)
-            ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
+        /* make sure all buffers are now clean */
+        ierr = ncmpi_wait_all(file->fh, NC_REQ_ALL, NULL, NULL);
 
         /* Release resources. */
         if (file->iobuf)