Option to do single precision mesh communication. (#2294)

* option to use single precision guard cell exhanges

* add missing files

* fix namespace issue

* change precision of comms to float

* ParmParse the single_precision_comms flag

* set back to real

* test

* make sure dst is filled

* nGrow -> nGrowVect

* restore float

* don't override valid cells

* single precision mesh

* whitespace

* wrap SumBoundary

* Wrap additional uses of FillBoundary.

* catch missing copies of ParallelCopy

* missing OverrideSyncs

* add wrapper for iMultifab

* fix typo

* moar typos

* typo

* strip single_precision_mesh option

* fix original copy

* update fusible syntax

Co-authored-by: Weiqun Zhang <WeiqunZhang@lbl.gov>

* Fix: Single Precision Builds

Should not copy around data for `do_single_precision_comms`

* Docs: warpx.do_single_precision_comms

* initialize this tmp multifab to 0.0

* fix tiny profile label

* remove orig copies, they are only correct for FillBoundary

* loosen tolerance for space charge tests for single precision

* missing some setVal

* another missing setVal

* missing setVal

* add wrapper for new version of sumboundary

* add explicit cast to silence compiler warning

* add a test for single precision comms

* revert change to test precision

* add benchmark for single precision comms test

* restore tolerance I removed by mistake

* tolerance

* copyright headers

* drop tolerance for single precision tests in default analysis script

* missing python module

* bump tol again

* fix bad merge

* Apply suggestions from code review

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

Co-authored-by: Remi Lehe <remi.lehe@normalesup.org>
Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>
Co-authored-by: Weiqun Zhang <WeiqunZhang@lbl.gov>

1 files changed, 94 insertions, 0 deletions

diff --git a/Source/Parallelization/WarpXCommUtil.H b/Source/Parallelization/WarpXCommUtil.H
new file mode 100644
index 000000000..2a5ec5390
--- /dev/null
+++ b/Source/Parallelization/WarpXCommUtil.H
@@ -0,0 +1,94 @@
+/* Copyright 2019 Andrew Myers
+ *
+ * This file is part of WarpX.
+ *
+ * License: BSD-3-Clause-LBNL
+ */
+#ifndef WARPX_COMMUTIL_H_
+#define WARPX_COMMUTIL_H_
+
+#include <AMReX_FabArray.H>
+#include <AMReX_Gpu.H>
+#include <AMReX_iMultiFab.H>
+#include <AMReX_MultiFab.H>
+#include <AMReX_Periodicity.H>
+#include <AMReX_TypeTraits.H>
+
+#include "WarpX.H"
+
+namespace WarpXCommUtil
+{
+
+using comm_float_type = float;
+
+template <class FAB1, class FAB2>
+void
+mixedCopy (amrex::FabArray<FAB1>& dst, amrex::FabArray<FAB2> const& src, int srccomp, int dstcomp, int numcomp, const amrex::IntVect& nghost)
+{
+    auto const& srcma = src.const_arrays();
+    auto const& dstma = dst.arrays();
+    ParallelFor(dst, nghost, numcomp,
+    [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
+    {
+        dstma[box_no](i,j,k,dstcomp+n) = (typename FAB1::value_type) srcma[box_no](i,j,k,srccomp+n);
+    });
+    amrex::Gpu::synchronize();
+}
+
+void ParallelCopy (amrex::MultiFab&            dst,
+                   const amrex::MultiFab&      src,
+                   int                         src_comp,
+                   int                         dst_comp,
+                   int                         num_comp,
+                   const amrex::IntVect&       src_nghost,
+                   const amrex::IntVect&       dst_nghost,
+                   const amrex::Periodicity&   period = amrex::Periodicity::NonPeriodic(),
+                   amrex::FabArrayBase::CpOp   op = amrex::FabArrayBase::COPY);
+
+void ParallelAdd (amrex::MultiFab&            dst,
+                  const amrex::MultiFab&      src,
+                  int                         src_comp,
+                  int                         dst_comp,
+                  int                         num_comp,
+                  const amrex::IntVect&       src_nghost,
+                  const amrex::IntVect&       dst_nghost,
+                  const amrex::Periodicity&   period = amrex::Periodicity::NonPeriodic());
+
+void FillBoundary (amrex::MultiFab&          mf,
+                   const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
+
+void FillBoundary (amrex::MultiFab&          mf,
+                   amrex::IntVect            ng,
+                   const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
+
+void FillBoundary (amrex::iMultiFab&         mf,
+                   const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
+
+void FillBoundary (amrex::iMultiFab&         mf,
+                   amrex::IntVect            ng,
+                   const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
+
+void
+FillBoundary (amrex::Vector<amrex::MultiFab*> const& mf, const amrex::Periodicity& period);
+
+void SumBoundary (amrex::MultiFab&          mf,
+                  const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
+
+void SumBoundary (amrex::MultiFab&          mf,
+                  int                       start_comp,
+                  int                       num_comps,
+                  amrex::IntVect            ng,
+                  const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
+
+void SumBoundary (amrex::MultiFab&          mf,
+                  int                       start_comp,
+                  int                       num_comps,
+                  amrex::IntVect            src_ng,
+                  amrex::IntVect            dst_ng,
+                  const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
+
+void OverrideSync (amrex::MultiFab&          mf,
+                   const amrex::Periodicity& period = amrex::Periodicity::NonPeriodic());
+}
+
+#endif