Merge `FillBoundary` and `NodalSync` (`OneStep_nosub`, `OneStep_sub1`) (#3013)

* Add argument for synchronizing nodal points

* Add the option to overridesync in FillBoundaryE / FillBoundaryB

* Remove explicit synchronization

* Cleanup calls to nodal synchronization

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Improve const Correctness

* Cleaning

* Merge NodalSync and FillBoundary First

Restarting the implementation from scratch.
To-do: remove redundant FillBoundary calls.

* Merge Only for OneStep_nosub and OneStep_sub1

* Fix Checksums of Python_wrappers

* Define sync_nodal_points as WarpX Private Member Variable

* Keep Calls to NodalSync for B

* Keep Calls to NodalSync for F

* Reset Benchmark of reduced_diags_single_precision

* OneStep_sub1: Remove All Calls to NodalSync

* OneStep_nosub: Remove All Calls to NodalSync

* Reset Benchmark of divb_cleaning_3d

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Edoardo Zoni <ezoni@lbl.gov>

3 files changed, 45 insertions, 34 deletions

diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp
index 94583fbe3..539976a0a 100644
--- a/Source/Parallelization/WarpXComm.cpp
+++ b/Source/Parallelization/WarpXComm.cpp
@@ -456,38 +456,38 @@ void WarpX::UpdateCurrentNodalToStag (amrex::MultiFab& dst, amrex::MultiFab cons
 }
 
 void
-WarpX::FillBoundaryB (IntVect ng)
+WarpX::FillBoundaryB (IntVect ng, const bool nodal_sync)
 {
     for (int lev = 0; lev <= finest_level; ++lev)
     {
-        FillBoundaryB(lev, ng);
+        FillBoundaryB(lev, ng, nodal_sync);
     }
 }
 
 void
-WarpX::FillBoundaryE (IntVect ng)
+WarpX::FillBoundaryE (IntVect ng, const bool nodal_sync)
 {
     for (int lev = 0; lev <= finest_level; ++lev)
     {
-        FillBoundaryE(lev, ng);
+        FillBoundaryE(lev, ng, nodal_sync);
     }
 }
 
 void
-WarpX::FillBoundaryF (IntVect ng)
+WarpX::FillBoundaryF (IntVect ng, const bool nodal_sync)
 {
     for (int lev = 0; lev <= finest_level; ++lev)
     {
-        FillBoundaryF(lev, ng);
+        FillBoundaryF(lev, ng, nodal_sync);
     }
 }
 
 void
-WarpX::FillBoundaryG (IntVect ng)
+WarpX::FillBoundaryG (IntVect ng, const bool nodal_sync)
 {
     for (int lev = 0; lev <= finest_level; ++lev)
     {
-        FillBoundaryG(lev, ng);
+        FillBoundaryG(lev, ng, nodal_sync);
     }
 }
 
@@ -511,14 +511,14 @@ WarpX::FillBoundaryE_avg (IntVect ng)
 
 
 void
-WarpX::FillBoundaryE(int lev, IntVect ng)
+WarpX::FillBoundaryE (int lev, IntVect ng, const bool nodal_sync)
 {
-    FillBoundaryE(lev, PatchType::fine, ng);
-    if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng);
+    FillBoundaryE(lev, PatchType::fine, ng, nodal_sync);
+    if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync);
 }
 
 void
-WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::IntVect ng)
+WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::IntVect ng, const bool nodal_sync)
 {
     std::array<amrex::MultiFab*,3> mf;
     amrex::Periodicity period;
@@ -563,19 +563,19 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In
             "Error: in FillBoundaryE, requested more guard cells than allocated");
 
         const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng;
-        WarpXCommUtil::FillBoundary(*mf[i], nghost, period);
+        WarpXCommUtil::FillBoundary(*mf[i], nghost, period, nodal_sync);
     }
 }
 
 void
-WarpX::FillBoundaryB (int lev, IntVect ng)
+WarpX::FillBoundaryB (int lev, IntVect ng, const bool nodal_sync)
 {
-    FillBoundaryB(lev, PatchType::fine, ng);
-    if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng);
+    FillBoundaryB(lev, PatchType::fine, ng, nodal_sync);
+    if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync);
 }
 
 void
-WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::IntVect ng)
+WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::IntVect ng, const bool nodal_sync)
 {
     std::array<amrex::MultiFab*,3> mf;
     amrex::Periodicity period;
@@ -620,7 +620,7 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In
             "Error: in FillBoundaryB, requested more guard cells than allocated");
 
         const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng;
-        WarpXCommUtil::FillBoundary(*mf[i], nghost, period);
+        WarpXCommUtil::FillBoundary(*mf[i], nghost, period, nodal_sync);
     }
 }
 
@@ -730,14 +730,14 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng)
 }
 
 void
-WarpX::FillBoundaryF (int lev, IntVect ng)
+WarpX::FillBoundaryF (int lev, IntVect ng, const bool nodal_sync)
 {
-    FillBoundaryF(lev, PatchType::fine, ng);
-    if (lev > 0) FillBoundaryF(lev, PatchType::coarse, ng);
+    FillBoundaryF(lev, PatchType::fine, ng, nodal_sync);
+    if (lev > 0) FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync);
 }
 
 void
-WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng)
+WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, const bool nodal_sync)
 {
     if (patch_type == PatchType::fine)
     {
@@ -751,7 +751,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng)
         {
             const amrex::Periodicity& period = Geom(lev).periodicity();
             const amrex::IntVect& nghost = (safe_guard_cells) ? F_fp[lev]->nGrowVect() : ng;
-            WarpXCommUtil::FillBoundary(*F_fp[lev], nghost, period);
+            WarpXCommUtil::FillBoundary(*F_fp[lev], nghost, period, nodal_sync);
         }
     }
     else if (patch_type == PatchType::coarse)
@@ -766,22 +766,22 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng)
         {
             const amrex::Periodicity& period = Geom(lev-1).periodicity();
             const amrex::IntVect& nghost = (safe_guard_cells) ? F_cp[lev]->nGrowVect() : ng;
-            WarpXCommUtil::FillBoundary(*F_cp[lev], nghost, period);
+            WarpXCommUtil::FillBoundary(*F_cp[lev], nghost, period, nodal_sync);
         }
     }
 }
 
-void WarpX::FillBoundaryG (int lev, IntVect ng)
+void WarpX::FillBoundaryG (int lev, IntVect ng, const bool nodal_sync)
 {
-    FillBoundaryG(lev, PatchType::fine, ng);
+    FillBoundaryG(lev, PatchType::fine, ng, nodal_sync);
 
     if (lev > 0)
     {
-        FillBoundaryG(lev, PatchType::coarse, ng);
+        FillBoundaryG(lev, PatchType::coarse, ng, nodal_sync);
     }
 }
 
-void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng)
+void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, const bool nodal_sync)
 {
     if (patch_type == PatchType::fine)
     {
@@ -795,7 +795,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng)
         {
             const amrex::Periodicity& period = Geom(lev).periodicity();
             const amrex::IntVect& nghost = (safe_guard_cells) ? G_fp[lev]->nGrowVect() : ng;
-            WarpXCommUtil::FillBoundary(*G_fp[lev], nghost, period);
+            WarpXCommUtil::FillBoundary(*G_fp[lev], nghost, period, nodal_sync);
         }
     }
     else if (patch_type == PatchType::coarse)
@@ -810,7 +810,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng)
         {
             const amrex::Periodicity& period = Geom(lev-1).periodicity();
             const amrex::IntVect& nghost = (safe_guard_cells) ? G_cp[lev]->nGrowVect() : ng;
-            WarpXCommUtil::FillBoundary(*G_cp[lev], nghost, period);
+            WarpXCommUtil::FillBoundary(*G_cp[lev], nghost, period, nodal_sync);
         }
     }
 }
diff --git a/Source/Parallelization/WarpXCommUtil.H b/Source/Parallelization/WarpXCommUtil.H
index 2a5ec5390..09d102a56 100644
--- a/Source/Parallelization/WarpXCommUtil.H
+++ b/Source/Parallelization/WarpXCommUtil.H
@@ -59,7 +59,8 @@ void FillBoundary (amrex::MultiFab&          mf,
 
 void FillBoundary (amrex::MultiFab&          mf,
                    amrex::IntVect            ng,
-                   const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
+                   const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic(),
+                   const bool                nodal_sync = false);
 
 void FillBoundary (amrex::iMultiFab&         mf,
                    const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
diff --git a/Source/Parallelization/WarpXCommUtil.cpp b/Source/Parallelization/WarpXCommUtil.cpp
index d395e7628..104aa5aae 100644
--- a/Source/Parallelization/WarpXCommUtil.cpp
+++ b/Source/Parallelization/WarpXCommUtil.cpp
@@ -93,7 +93,8 @@ void FillBoundary (amrex::MultiFab& mf, const amrex::Periodicity& period)
 
 void FillBoundary (amrex::MultiFab&          mf,
                    amrex::IntVect            ng,
-                   const amrex::Periodicity& period)
+                   const amrex::Periodicity& period,
+                   const bool                nodal_sync)
 {
     BL_PROFILE("WarpXCommUtil::FillBoundary");
 
@@ -106,13 +107,22 @@ void FillBoundary (amrex::MultiFab&          mf,
 
         mixedCopy(mf_tmp, mf, 0, 0, mf.nComp(), mf.nGrowVect());
 
-        mf_tmp.FillBoundary(ng, period);
+        if (nodal_sync) {
+            mf_tmp.FillBoundaryAndSync(0, mf.nComp(), ng, period);
+        } else {
+            mf_tmp.FillBoundary(ng, period);
+        }
 
         mixedCopy(mf, mf_tmp, 0, 0, mf.nComp(), mf.nGrowVect());
     }
     else
     {
-        mf.FillBoundary(ng, period);
+
+        if (nodal_sync) {
+            mf.FillBoundaryAndSync(0, mf.nComp(), ng, period);
+        } else {
+            mf.FillBoundary(ng, period);
+        }
     }
 }