Option to do single precision mesh communication. (#2294)

* option to use single precision guard cell exhanges

* add missing files

* fix namespace issue

* change precision of comms to float

* ParmParse the single_precision_comms flag

* set back to real

* test

* make sure dst is filled

* nGrow -> nGrowVect

* restore float

* don't override valid cells

* single precision mesh

* whitespace

* wrap SumBoundary

* Wrap additional uses of FillBoundary.

* catch missing copies of ParallelCopy

* missing OverrideSyncs

* add wrapper for iMultifab

* fix typo

* moar typos

* typo

* strip single_precision_mesh option

* fix original copy

* update fusible syntax

Co-authored-by: Weiqun Zhang <WeiqunZhang@lbl.gov>

* Fix: Single Precision Builds

Should not copy around data for `do_single_precision_comms`

* Docs: warpx.do_single_precision_comms

* initialize this tmp multifab to 0.0

* fix tiny profile label

* remove orig copies, they are only correct for FillBoundary

* loosen tolerance for space charge tests for single precision

* missing some setVal

* another missing setVal

* missing setVal

* add wrapper for new version of sumboundary

* add explicit cast to silence compiler warning

* add a test for single precision comms

* revert change to test precision

* add benchmark for single precision comms test

* restore tolerance I removed by mistake

* tolerance

* copyright headers

* drop tolerance for single precision tests in default analysis script

* missing python module

* bump tol again

* fix bad merge

* Apply suggestions from code review

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

Co-authored-by: Remi Lehe <remi.lehe@normalesup.org>
Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>
Co-authored-by: Weiqun Zhang <WeiqunZhang@lbl.gov>

1 files changed, 16 insertions, 15 deletions

diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp
index 979079e71..d8b6cdd16 100644
--- a/Source/Initialization/WarpXInitData.cpp
+++ b/Source/Initialization/WarpXInitData.cpp
@@ -18,6 +18,7 @@
 #include "Filter/BilinearFilter.H"
 #include "Filter/NCIGodfreyFilter.H"
 #include "Particles/MultiParticleContainer.H"
+#include "Parallelization/WarpXCommUtil.H"
 #include "Utils/WarpXAlgorithmSelection.H"
 #include "Utils/WarpXConst.H"
 #include "Utils/WarpXProfilerWrapper.H"
@@ -493,24 +494,24 @@ WarpX::InitLevelData (int lev, Real /*time*/)
             ComputeDistanceToEB();
 
             const auto &period = Geom(lev).periodicity();
-            m_edge_lengths[lev][0]->FillBoundary(guard_cells.ng_alloc_EB, period);
-            m_edge_lengths[lev][1]->FillBoundary(guard_cells.ng_alloc_EB, period);
-            m_edge_lengths[lev][2]->FillBoundary(guard_cells.ng_alloc_EB, period);
-            m_face_areas[lev][0]->FillBoundary(guard_cells.ng_alloc_EB, period);
-            m_face_areas[lev][1]->FillBoundary(guard_cells.ng_alloc_EB, period);
-            m_face_areas[lev][2]->FillBoundary(guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][0], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][1], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][2], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_face_areas[lev][0], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_face_areas[lev][1], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_face_areas[lev][2], guard_cells.ng_alloc_EB, period);
 
             if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) {
-                m_area_mod[lev][0]->FillBoundary(guard_cells.ng_alloc_EB, period);
-                m_area_mod[lev][1]->FillBoundary(guard_cells.ng_alloc_EB, period);
-                m_area_mod[lev][2]->FillBoundary(guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_area_mod[lev][0], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_area_mod[lev][1], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_area_mod[lev][2], guard_cells.ng_alloc_EB, period);
                 MarkCells();
-                m_flag_info_face[lev][0]->FillBoundary(guard_cells.ng_alloc_EB, period);
-                m_flag_info_face[lev][1]->FillBoundary(guard_cells.ng_alloc_EB, period);
-                m_flag_info_face[lev][2]->FillBoundary(guard_cells.ng_alloc_EB, period);
-                m_flag_ext_face[lev][0]->FillBoundary(guard_cells.ng_alloc_EB, period);
-                m_flag_ext_face[lev][1]->FillBoundary(guard_cells.ng_alloc_EB, period);
-                m_flag_ext_face[lev][2]->FillBoundary(guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][0], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][1], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][2], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][0], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][1], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][2], guard_cells.ng_alloc_EB, period);
                 ComputeFaceExtensions();
             }
         }