Option to do single precision mesh communication. (#2294)

* option to use single precision guard cell exhanges

* add missing files

* fix namespace issue

* change precision of comms to float

* ParmParse the single_precision_comms flag

* set back to real

* test

* make sure dst is filled

* nGrow -> nGrowVect

* restore float

* don't override valid cells

* single precision mesh

* whitespace

* wrap SumBoundary

* Wrap additional uses of FillBoundary.

* catch missing copies of ParallelCopy

* missing OverrideSyncs

* add wrapper for iMultifab

* fix typo

* moar typos

* typo

* strip single_precision_mesh option

* fix original copy

* update fusible syntax

Co-authored-by: Weiqun Zhang <WeiqunZhang@lbl.gov>

* Fix: Single Precision Builds

Should not copy around data for `do_single_precision_comms`

* Docs: warpx.do_single_precision_comms

* initialize this tmp multifab to 0.0

* fix tiny profile label

* remove orig copies, they are only correct for FillBoundary

* loosen tolerance for space charge tests for single precision

* missing some setVal

* another missing setVal

* missing setVal

* add wrapper for new version of sumboundary

* add explicit cast to silence compiler warning

* add a test for single precision comms

* revert change to test precision

* add benchmark for single precision comms test

* restore tolerance I removed by mistake

* tolerance

* copyright headers

* drop tolerance for single precision tests in default analysis script

* missing python module

* bump tol again

* fix bad merge

* Apply suggestions from code review

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

Co-authored-by: Remi Lehe <remi.lehe@normalesup.org>
Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>
Co-authored-by: Weiqun Zhang <WeiqunZhang@lbl.gov>

1 files changed, 15 insertions, 13 deletions

diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp
index 700dcdce4..1d06408a2 100644
--- a/Source/BoundaryConditions/PML.cpp
+++ b/Source/BoundaryConditions/PML.cpp
@@ -17,6 +17,7 @@
 #include "Utils/WarpXConst.H"
 #include "Utils/WarpXProfilerWrapper.H"
 #include "WarpX.H"
+#include "Parallelization/WarpXCommUtil.H"
 
 #include <AMReX.H>
 #include <AMReX_Algorithm.H>
@@ -1003,6 +1004,7 @@ PML::Exchange (MultiFab& pml, MultiFab& reg, const Geometry& geom,
 
     // Create temporary MultiFab to copy to and from the PML
     MultiFab tmpregmf(reg.boxArray(), reg.DistributionMap(), ncp, ngr);
+    tmpregmf.setVal(0.0);
 
     // Create the sum of the split fields, in the PML
     MultiFab totpmlmf(pml.boxArray(), pml.DistributionMap(), 1, 0); // Allocate
@@ -1015,7 +1017,7 @@ PML::Exchange (MultiFab& pml, MultiFab& reg, const Geometry& geom,
     if (do_pml_in_domain){
         // Valid cells of the PML and of the regular grid overlap
         // Copy from valid cells of the PML to valid cells of the regular grid
-        reg.ParallelCopy(totpmlmf, 0, 0, 1, IntVect(0), IntVect(0), period);
+        WarpXCommUtil::ParallelCopy(reg, totpmlmf, 0, 0, 1, IntVect(0), IntVect(0), period);
     } else {
         // Valid cells of the PML only overlap with guard cells of regular grid
         // (and outermost valid cell of the regular grid, for nodal direction)
@@ -1023,7 +1025,7 @@ PML::Exchange (MultiFab& pml, MultiFab& reg, const Geometry& geom,
         // but avoid updating the outermost valid cell
         if (ngr.max() > 0) {
             MultiFab::Copy(tmpregmf, reg, 0, 0, 1, ngr);
-            tmpregmf.ParallelCopy(totpmlmf, 0, 0, 1, IntVect(0), ngr, period);
+            WarpXCommUtil::ParallelCopy(tmpregmf, totpmlmf, 0, 0, 1, IntVect(0), ngr, period);
 #ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
@@ -1056,9 +1058,9 @@ PML::Exchange (MultiFab& pml, MultiFab& reg, const Geometry& geom,
         // Where valid cells of tmpregmf overlap with PML valid cells,
         // copy the PML (this is order to avoid overwriting PML valid cells,
         // in the next `ParallelCopy`)
-        tmpregmf.ParallelCopy(pml,0, 0, ncp, IntVect(0), IntVect(0), period);
+        WarpXCommUtil::ParallelCopy(tmpregmf, pml,0, 0, ncp, IntVect(0), IntVect(0), period);
     }
-    pml.ParallelCopy(tmpregmf, 0, 0, ncp, IntVect(0), ngp, period);
+    WarpXCommUtil::ParallelCopy(pml, tmpregmf, 0, 0, ncp, IntVect(0), ngp, period);
 }
 
 
@@ -1068,7 +1070,7 @@ PML::CopyToPML (MultiFab& pml, MultiFab& reg, const Geometry& geom)
   const IntVect& ngp = pml.nGrowVect();
   const auto& period = geom.periodicity();
 
-  pml.ParallelCopy(reg, 0, 0, 1, IntVect(0), ngp, period);
+  WarpXCommUtil::ParallelCopy(pml, reg, 0, 0, 1, IntVect(0), ngp, period);
 }
 
 void
@@ -1094,13 +1096,13 @@ PML::FillBoundaryE (PatchType patch_type)
     {
         const auto& period = m_geom->periodicity();
         Vector<MultiFab*> mf{pml_E_fp[0].get(),pml_E_fp[1].get(),pml_E_fp[2].get()};
-        amrex::FillBoundary(mf, period);
+        WarpXCommUtil::FillBoundary(mf, period);
     }
     else if (patch_type == PatchType::coarse && pml_E_cp[0] && pml_E_cp[0]->nGrowVect().max() > 0)
     {
         const auto& period = m_cgeom->periodicity();
         Vector<MultiFab*> mf{pml_E_cp[0].get(),pml_E_cp[1].get(),pml_E_cp[2].get()};
-        amrex::FillBoundary(mf, period);
+        WarpXCommUtil::FillBoundary(mf, period);
     }
 }
 
@@ -1118,13 +1120,13 @@ PML::FillBoundaryB (PatchType patch_type)
     {
         const auto& period = m_geom->periodicity();
         Vector<MultiFab*> mf{pml_B_fp[0].get(),pml_B_fp[1].get(),pml_B_fp[2].get()};
-        amrex::FillBoundary(mf, period);
+        WarpXCommUtil::FillBoundary(mf, period);
     }
     else if (patch_type == PatchType::coarse && pml_B_cp[0])
     {
         const auto& period = m_cgeom->periodicity();
         Vector<MultiFab*> mf{pml_B_cp[0].get(),pml_B_cp[1].get(),pml_B_cp[2].get()};
-        amrex::FillBoundary(mf, period);
+        WarpXCommUtil::FillBoundary(mf, period);
     }
 }
 
@@ -1141,12 +1143,12 @@ PML::FillBoundaryF (PatchType patch_type)
     if (patch_type == PatchType::fine && pml_F_fp && pml_F_fp->nGrowVect().max() > 0)
     {
         const auto& period = m_geom->periodicity();
-        pml_F_fp->FillBoundary(period);
+        WarpXCommUtil::FillBoundary(*pml_F_fp, period);
     }
     else if (patch_type == PatchType::coarse && pml_F_cp && pml_F_cp->nGrowVect().max() > 0)
     {
         const auto& period = m_cgeom->periodicity();
-        pml_F_cp->FillBoundary(period);
+        WarpXCommUtil::FillBoundary(*pml_F_cp, period);
     }
 }
 
@@ -1163,12 +1165,12 @@ PML::FillBoundaryG (PatchType patch_type)
     if (patch_type == PatchType::fine && pml_G_fp && pml_G_fp->nGrowVect().max() > 0)
     {
         const auto& period = m_geom->periodicity();
-        pml_G_fp->FillBoundary(period);
+        WarpXCommUtil::FillBoundary(*pml_G_fp, period);
     }
     else if (patch_type == PatchType::coarse && pml_G_cp && pml_G_cp->nGrowVect().max() > 0)
     {
         const auto& period = m_cgeom->periodicity();
-        pml_G_cp->FillBoundary(period);
+        WarpXCommUtil::FillBoundary(*pml_G_cp, period);
     }
 }