Merge pull request #1997 from DLR-AMR/feature-shmem_init_error_checking

Feature: shmem init error checking

Author: Davknapp

src/t8_cmesh/t8_cmesh_internal/t8_cmesh_commit.cxx

@@ -241,7 +241,7 @@ t8_cmesh_commit_partitioned_new (t8_cmesh_t cmesh, sc_MPI_Comm comm)
     /* TODO: reset cmesh */
     return;
   }
-  t8_shmem_init (comm);
+  SC_CHECK_ABORT (t8_shmem_init (comm) > 0, "Error in shared memory setup. Could not build Cmesh.");
   t8_cmesh_set_shmem_type (comm); /* TODO: do we actually need the shared array? */
   t8_stash_attribute_sort (cmesh->stash);
 

src/t8_cmesh/t8_cmesh_internal/t8_cmesh_partition.cxx

@@ -195,7 +195,7 @@ t8_cmesh_gather_treecount_ext (const t8_cmesh_t cmesh, sc_MPI_Comm comm, const i
 
   tree_offset = cmesh->first_tree_shared ? -cmesh->first_tree - 1 : cmesh->first_tree;
   if (cmesh->tree_offsets == NULL) {
-    t8_shmem_init (comm);
+    SC_CHECK_ABORT (t8_shmem_init (comm) > 0, "Error in shared memory setup.");
     t8_shmem_set_type (comm, T8_SHMEM_BEST_TYPE);
     /* Only allocate the shmem array, if it is not already allocated */
     cmesh->tree_offsets = t8_cmesh_alloc_offsets (cmesh->mpisize, comm);

src/t8_cmesh/t8_cmesh_io/t8_cmesh_save.cxx

@@ -834,7 +834,7 @@ t8_cmesh_load_and_distribute (const char *fileprefix, const int num_files, sc_MP
   T8_ASSERT (mpisize >= num_files);
 
   /* Try to set the comm type */
-  t8_shmem_init (comm);
+  SC_CHECK_ABORT (t8_shmem_init (comm) > 0, "Error in shared memory setup. Could not load cmesh.");
   t8_shmem_set_type (comm, T8_SHMEM_BEST_TYPE);
 
   /* Use cmesh_bcast, if only one process loads the cmesh: */

src/t8_data/t8_shmem.c

@@ -60,9 +60,18 @@ t8_shmem_array_is_initialized (const t8_shmem_array_t array)
 }
 #endif
 
-void
+int
 t8_shmem_init (sc_MPI_Comm comm)
 {
+#ifndef T8_ENABLE_MPI
+  // If we do not use MPI, there is nothing to do.
+  // We only have a single process.
+  return 1;
+#endif
+#ifndef SC_ENABLE_MPICOMMSHARED
+  SC_ABORT ("Trying to use shared memory but SC_ENABLE_MPICOMMSHARED is not set. This should not happen if you use MPI "
+            "v.3.0 or higher. Maybe related to https://github.com/DLR-AMR/t8code/pull/1996.");
+#endif
   /* Check whether intranode and internode comms are set
    * for the current communicator. */
   sc_MPI_Comm intranode;
@@ -73,8 +82,12 @@ t8_shmem_init (sc_MPI_Comm comm)
   if (intranode == sc_MPI_COMM_NULL || internode == sc_MPI_COMM_NULL) {
     /* The inter/intra comms are not set. We need to set them to 
      * initialize shared memory usage. */
-    sc_mpi_comm_get_and_attach (comm);
+    return sc_mpi_comm_get_and_attach (comm);
   }
+  int intranode_size;
+  const int mpiret = sc_MPI_Comm_size (intranode, &intranode_size);
+  SC_CHECK_MPI (mpiret);
+  return intranode_size;
 }
 
 void

src/t8_data/t8_shmem.h

@@ -53,8 +53,11 @@ T8_EXTERN_C_BEGIN ();
  * \note This function needs to be called to enable shared memory usage for a communicator.
  * \note Calling this function multiple times with the same communicator is safe and does
  *  not change the behaviour.
+ * \return                    If the intranode communicator cannot be
+ *                            obtained, return 0.
+ *                            Otherwise return size of intranode communicator.
  */
-void
+int
 t8_shmem_init (sc_MPI_Comm comm);
 
 #if T8_ENABLE_DEBUG

src/t8_forest/t8_forest_partition.cxx

@@ -146,7 +146,7 @@ t8_forest_partition_create_offsets (t8_forest_t forest)
   t8_debugf ("Building offsets for forest %p\n", (void *) forest);
   comm = forest->mpicomm;
   /* Set the shmem array type of comm */
-  t8_shmem_init (comm);
+  SC_CHECK_ABORT (t8_shmem_init (comm) > 0, "Error in shared memory setup. Could not partition forest.");
   t8_shmem_set_type (comm, T8_SHMEM_BEST_TYPE);
   /* Initialize the offset array as a shmem array
    * holding mpisize+1 many t8_gloidx_t */
@@ -322,7 +322,7 @@ t8_forest_partition_create_first_desc (t8_forest_t forest)
 
   if (forest->global_first_desc == NULL) {
     /* Set the shmem array type of comm */
-    t8_shmem_init (comm);
+    SC_CHECK_ABORT (t8_shmem_init (comm) > 0, "Error in shared memory setup. Could not partition forest.");
     t8_shmem_set_type (comm, T8_SHMEM_BEST_TYPE);
     /* Initialize the offset array as a shmem array
      * holding mpisize+1 many t8_linearidx_t to store the elements linear ids */
@@ -425,7 +425,7 @@ t8_forest_partition_create_tree_offsets (t8_forest_t forest)
 
   if (forest->tree_offsets == NULL) {
     /* Set the shmem array type of comm */
-    t8_shmem_init (comm);
+    SC_CHECK_ABORT (t8_shmem_init (comm) > 0, "Error in shared memory setup. Could not partition forest.");
     t8_shmem_set_type (comm, T8_SHMEM_BEST_TYPE);
     /* Only allocate the shmem array, if it is not already allocated */
     t8_shmem_array_init (&forest->tree_offsets, sizeof (t8_gloidx_t), forest->mpisize + 1, comm);
@@ -496,8 +496,9 @@ t8_forest_partition_compute_new_offset (t8_forest_t forest)
   const t8_gloidx_t global_num_leaf_elements = forest_from->global_num_leaf_elements;
   const t8_weight_fcn_t *weight_fcn = forest->weight_function;
 
-  // Initialize the shmem array.
-  t8_shmem_init (comm);
+  T8_ASSERT (forest->element_offsets == NULL);
+  /* Set the shmem array type to comm */
+  SC_CHECK_ABORT (t8_shmem_init (comm) > 0, "Error in shared memory setup. Could not partition forest.");
   t8_shmem_set_type (comm, T8_SHMEM_BEST_TYPE);
   t8_shmem_array_init (&forest->element_offsets, sizeof (t8_gloidx_t), mpisize + 1, comm);
 

src/t8_vtk/t8_with_vtk/t8_vtk_reader.cxx

@@ -259,7 +259,7 @@ t8_vtk_partition (t8_cmesh_t cmesh, const int mpirank, const int mpisize, t8_glo
   t8_gloidx_t first_tree = 0;
   t8_gloidx_t last_tree = 1;
   /* Compute the global id of the first tree on each proc. */
-  t8_shmem_init (comm);
+  SC_CHECK_ABORT (t8_shmem_init (comm) > 0, "Error in shared memory setup in vtk output.");
   t8_shmem_set_type (comm, T8_SHMEM_BEST_TYPE);
   t8_shmem_array_t offsets = NULL;
   t8_shmem_array_init (&offsets, sizeof (t8_gloidx_t), mpisize + 1, comm);

test/t8_cmesh/t8_gtest_cmesh_set_partition_offsets.cxx

@@ -116,7 +116,8 @@ TEST_P (cmesh_set_partition_offsets_nocommit, test_set_offsets)
    * the array corresponds to any valid partition.
    * We use the offset_concentrate function to build an offset array for a partition
    * that concentrates all trees at one process. */
-  t8_shmem_init (sc_MPI_COMM_WORLD);
+  const int intranode_size = t8_shmem_init (sc_MPI_COMM_WORLD);
+  ASSERT_GT (intranode_size, 0) << "Could not initialize shared memory.";
   t8_shmem_array_t shmem_array = t8_cmesh_offset_concentrate (main_process, sc_MPI_COMM_WORLD, inum_trees);
 
   /* Set the partition offsets */
@@ -135,7 +136,9 @@ TEST_P (cmesh_set_partition_offsets_commit, test_set_offsets)
    * the array corresponds to any valid partition.
    * We use the offset_concentrate function to build an offset array for a partition
    * that concentrates all trees at one process. */
-  t8_shmem_init (comm);
+  const int intranode_size = t8_shmem_init (comm);
+  ASSERT_GT (intranode_size, 0) << "Could not initialize shared memory.";
+
   t8_shmem_array_t shmem_array = t8_cmesh_offset_concentrate (main_process, comm, inum_trees);
 
   /* Set the partition offsets */

test/t8_data/t8_gtest_shmem.cxx

@@ -58,7 +58,8 @@ TEST_P (shmem, test_shmem_init_finalize)
   int mpiret;
 
   /* setup shared memory usage */
-  t8_shmem_init (comm);
+  const int intrasize_from_init = t8_shmem_init (comm);
+  ASSERT_GT (intrasize_from_init, 0) << "Error in t8_shmem_init. No intranode communicator set.";
 
   /* Get intranode and internode comm */
   sc_mpi_comm_get_node_comms (comm, &intranode, &internode);
@@ -113,7 +114,9 @@ TEST_P (shmem, test_sc_shmem_alloc)
   t8_debugf ("Checking shared memory type %s.\n", sc_shmem_type_to_string[shmem_type]);
 
   /* setup shared memory usage */
-  t8_shmem_init (comm);
+  const int intranode_size = t8_shmem_init (comm);
+  ASSERT_GT (intranode_size, 0) << "Could not initialize shared memory.";
+
   t8_shmem_set_type (comm, shmem_type);
 
 #if T8_ENABLE_MPI
@@ -196,7 +199,8 @@ TEST_P (shmem, test_shmem_array_allgatherv)
   const sc_shmem_type_t shmem_type = (sc_shmem_type_t) shmem_type_int;
 
   /* setup shared memory usage */
-  t8_shmem_init (comm);
+  const int intranode_size = t8_shmem_init (comm);
+  ASSERT_GT (intranode_size, 0) << "Could not initialize shared memory.";
   t8_shmem_set_type (comm, shmem_type);
 
 #if T8_ENABLE_MPI
@@ -246,7 +250,8 @@ TEST_P (shmem, test_shmem_array_prefix)
   const sc_shmem_type_t shmem_type = (sc_shmem_type_t) shmem_type_int;
 
   /* setup shared memory usage */
-  t8_shmem_init (comm);
+  const int intranode_size = t8_shmem_init (comm);
+  ASSERT_GT (intranode_size, 0) << "Could not initialize shared memory.";
   t8_shmem_set_type (comm, shmem_type);
 
 #if T8_ENABLE_MPI
@@ -291,7 +296,8 @@ TEST_P (shmem, test_shmem_array)
   const sc_shmem_type_t shmem_type = (sc_shmem_type_t) shmem_type_int;
 
   /* setup shared memory usage */
-  t8_shmem_init (comm);
+  const int intranode_size = t8_shmem_init (comm);
+  ASSERT_GT (intranode_size, 0) << "Could not initialize shared memory.";
   t8_shmem_set_type (comm, shmem_type);
 
 #if T8_ENABLE_MPI