MPI: Init or Thread Multiple (#1298)

Simplify MPI init options to be independent of PSATD selection. Our
use of FFTD only uses the non-threaded FFTW routines.

Thread multiple is currently only needed for async I/O with AMReX
plotfiles.

Some MPI implementations allow to set the requested thread level with
`mpiexec` and/or pre-compiled variants of the lib and/or through an
environment variable, according to the standard. For example,
`MPICH_THREADLEVEL_DEFAULT` in MPICH (and derivatives have similar
names). This is done through an intricate upgrade/downgrade logic for
the provided argument (sec. 12.4.3). We print a warning if the
requested support level is not provided and a note if a stricter
support level is provided.

1 files changed, 58 insertions, 0 deletions

diff --git a/Source/Utils/MPIInitHelpers.cpp b/Source/Utils/MPIInitHelpers.cpp
new file mode 100644
index 000000000..ccf28134c
--- /dev/null
+++ b/Source/Utils/MPIInitHelpers.cpp
@@ -0,0 +1,58 @@
+/* Copyright 2020 Axel Huebl
+ *
+ * This file is part of WarpX.
+ *
+ * License: BSD-3-Clause-LBNL
+ */
+#include "MPIInitHelpers.H"
+
+#include <AMReX.H>
+#include <AMReX_ParallelDescriptor.H>
+#include <AMReX_Print.H>
+
+#include <string>
+#include <utility>
+
+
+namespace utils
+{
+    std::pair< int, int >
+    warpx_mpi_init (int argc, char* argv[])
+    {
+        int thread_required = -1;
+        int thread_provided = -1;
+#ifdef AMREX_USE_MPI
+        thread_required = MPI_THREAD_SINGLE;  // equiv. to MPI_Init
+#   ifdef AMREX_USE_OMP
+        thread_required = MPI_THREAD_FUNNELED;
+#   endif
+#   ifdef AMREX_MPI_THREAD_MULTIPLE  // i.e. for async_io
+        thread_required = MPI_THREAD_MULTIPLE;
+#   endif
+        MPI_Init_thread(&argc, &argv, thread_required, &thread_provided);
+#endif
+        return std::make_pair(thread_required, thread_provided);
+    }
+
+    void
+    warpx_check_mpi_thread_level (std::pair< int, int > const mpi_thread_levels)
+    {
+#ifdef AMREX_USE_MPI
+        auto const thread_required = mpi_thread_levels.first;
+        auto const thread_provided = mpi_thread_levels.second;
+        auto mtn = amrex::ParallelDescriptor::mpi_level_to_string;
+
+        if( thread_provided < thread_required )
+            amrex::Print() << "WARNING: Provided MPI thread safety level ("
+                           << mtn(thread_provided) << ") is LOWER than requested "
+                           << mtn(thread_required) << "). This might lead to undefined "
+                           << "results in asynchronous operations (e.g. async_io).";
+        if( thread_provided > thread_required )
+            amrex::Print() << "NOTE: Provided MPI thread safety level ("
+                           << mtn(thread_provided) << ") is stricter than requested "
+                           << mtn(thread_required) << "). This might reduce multi-node "
+                           << "communication performance.";
+#endif
+    }
+
+} // namespace utils