From f462adc324cf466b60d62bc8c06c035221b75861 Mon Sep 17 00:00:00 2001
From: KZhu-ME <86268612+KZhu-ME@users.noreply.github.com>
Date: Fri, 9 Jul 2021 20:13:56 -0700
Subject: Feature pass MPI comm from Python script to AMReX during
 initialization (#2034)

* Added functionality to pass mpi comm from python script to amrex during initialization

* Fixed missing _ in MPI._sizeof()

* Added functions to get the current processor's rank and total number of processors

* Renamed MPI_Comm to _MPI_Comm_type and defined _MPI_Comm_type in except statement

* Updated comment to explain why mpi4py needs to be imported before loading libwarpx

* Removed ifdef flags that prevent amrex_init_with_inited_mpi from being declared when MPI is off

* Changed amrex_init_with_inited_mpi to be declared even when not using mpi, but will be defined to be functionally the same as amrex_init

* Defined MPI = None to signify whether MPI is used, to add another check when initializing amrex

* Changed ifdef blocks in WarpXWrappers.cpp/h to fix compile errors.
Added ifdef block to conditionally declare amrex_init_with_inited_mpi in WarpXWrappers.h to prevent compile error when not using MPI. Removed ifdef block to declare/define same function in WarpXWrappers.cpp since function needs to be declared even when MPI is not used, but will never be called in that case.

* Changed BL_USE_MPI to AMREX_USE_MPI and removed incorrect MPI=None statement

* Changed BL_USE_MPI to AMREX_USE_MPI

* Added test to verify correct passing of MPI communicator to amrex

* Added ability to pass mpi_comm to sim.step

* Change test to check for differeing outputs when passed different inputs

* Removed obsolete comments refactored program to use more shared code

* Refactored comments

* Updated description to match test

* Removed unecessary imports and updated comments
---
 Source/Python/WarpXWrappers.cpp | 10 ++++++++--
 Source/Python/WarpXWrappers.h   |  8 ++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

(limited to 'Source/Python')

diff --git a/Source/Python/WarpXWrappers.cpp b/Source/Python/WarpXWrappers.cpp
index 60d675d3a..47942a55b 100644
--- a/Source/Python/WarpXWrappers.cpp
+++ b/Source/Python/WarpXWrappers.cpp
@@ -142,12 +142,10 @@ extern "C"
         warpx_amrex_init(argc, argv);
     }
 
-#ifdef BL_USE_MPI
     void amrex_init_with_inited_mpi (int argc, char* argv[], MPI_Comm mpicomm)
     {
         warpx_amrex_init(argc, argv, true, mpicomm);
     }
-#endif
 
     void amrex_finalize (int /*finalize_mpi*/)
     {
@@ -496,6 +494,14 @@ extern "C"
         return warpx.finestLevel ();
     }
 
+    int warpx_getMyProc () {
+        return amrex::ParallelDescriptor::MyProc();
+    }
+
+    int warpx_getNProcs () {
+        return amrex::ParallelDescriptor::NProcs();
+    }
+
     void mypc_Redistribute () {
         auto & mypc = WarpX::GetInstance().GetPartContainer();
         mypc.Redistribute();
diff --git a/Source/Python/WarpXWrappers.h b/Source/Python/WarpXWrappers.h
index a720f0f59..30fa81968 100644
--- a/Source/Python/WarpXWrappers.h
+++ b/Source/Python/WarpXWrappers.h
@@ -11,7 +11,7 @@
 #include <AMReX_Config.H>
 #include <AMReX_REAL.H>
 
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
 #   include <mpi.h>
 #endif
 
@@ -34,7 +34,7 @@ extern "C" {
 
     void amrex_init (int argc, char* argv[]);
 
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
     void amrex_init_with_inited_mpi (int argc, char* argv[], MPI_Comm mpicomm);
 #endif
 
@@ -113,6 +113,10 @@ extern "C" {
 
   int warpx_finestLevel ();
 
+  int warpx_getMyProc ();
+  int warpx_getNProcs ();
+
+
   void mypc_Redistribute ();
 
 #ifdef __cplusplus
-- 
cgit v1.2.3