Merge remote-tracking branch 'ECPwarpx/dev' into fix_picmi_diagnostics

20 files changed, 60 insertions, 42 deletions

diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp
index e7540c8a9..bd29d1b65 100644
--- a/Source/BoundaryConditions/WarpXEvolvePML.cpp
+++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp
@@ -233,7 +233,9 @@ WarpX::DampJPML (int lev, PatchType patch_type)
     }
 }
 
-/* \brief Copy the current J from the regular grid to the PML */
+/**
+ * \brief Copy the current J from the regular grid to the PML
+ */
 void
 WarpX::CopyJPML ()
 {
diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp
index bb42fc463..05c2066de 100644
--- a/Source/Diagnostics/WarpXOpenPMD.cpp
+++ b/Source/Diagnostics/WarpXOpenPMD.cpp
@@ -165,14 +165,14 @@ WarpXOpenPMDPlot::SavePlotFile (const std::unique_ptr<WarpXParticleContainer>& p
   SetupPos(currSpecies, counter.GetTotalNumParticles());
   SetupRealProperties(currSpecies, write_real_comp, real_comp_names, counter.GetTotalNumParticles());
 
+  // forces the files created by all processors! this is the key to resolve RZ storage issue!!
+  m_Series->flush();
   for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++)
     {
       //long numParticles = counter.m_ParticleSizeAtRank[currentLevel]
       unsigned long long const numParticles = counter.m_ParticleSizeAtRank[currentLevel];
       unsigned long long offset = counter.m_ParticleOffsetAtRank[currentLevel];
 
-      //if return after this, all is fine (although nothing useful is written)
-
       if (0 == numParticles)
           return;
 
@@ -355,11 +355,9 @@ WarpXOpenPMDPlot::WriteOpenPMDFields( //const std::string& filename,
     const std::string& varname = varnames[icomp];
     std::string field_name = varname;
     std::string comp_name = openPMD::MeshRecordComponent::SCALAR;
-    bool is_vector = false;
     for (const char* vector_field: {"E", "B", "j"}){
         for (const char* comp: {"x", "y", "z"}){
             if (varname[0] == *vector_field && varname[1] == *comp ){
-                is_vector = true;
                 field_name = varname[0] + varname.substr(2); // Strip component
                 comp_name = varname[1];
             }
diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H
index bc7f90fac..b7aed9e40 100644
--- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H
+++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H
@@ -3,7 +3,8 @@
 
 #include <SpectralBaseAlgorithm.H>
 
-/* \brief Class that updates the field in spectral space
+/**
+ * \brief Class that updates the field in spectral space
  * and stores the coefficients of the corresponding update equation.
  */
 class PsatdAlgorithm : public SpectralBaseAlgorithm
diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.H b/Source/FieldSolver/SpectralSolver/SpectralFieldData.H
index 01ca11083..dc83d279d 100644
--- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.H
+++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.H
@@ -8,20 +8,20 @@
 // Declare type for spectral fields
 using SpectralField = amrex::FabArray< amrex::BaseFab <Complex> >;
 
-/* Index for the regular fields, when stored in spectral space */
+/** Index for the regular fields, when stored in spectral space */
 struct SpectralFieldIndex {
   enum { Ex=0, Ey, Ez, Bx, By, Bz, Jx, Jy, Jz, rho_old, rho_new, n_fields };
   // n_fields is automatically the total number of fields
 };
 
-/* Index for the PML fields, when stored in spectral space */
+/** Index for the PML fields, when stored in spectral space */
 struct SpectralPMLIndex {
   enum { Exy=0, Exz, Eyx, Eyz, Ezx, Ezy,
            Bxy, Bxz, Byx, Byz, Bzx, Bzy, n_fields };
   // n_fields is automatically the total number of fields
 };
 
-/* \brief Class that stores the fields in spectral space, and performs the
+/** \brief Class that stores the fields in spectral space, and performs the
  *  Fourier transforms between real space and spectral space
  */
 class SpectralFieldData
diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H
index a73356dca..eb07e8fe6 100644
--- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H
+++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H
@@ -19,7 +19,8 @@ struct ShiftType {
     enum{ TransformFromCellCentered=0, TransformToCellCentered=1 };
 };
 
-/* \brief Class that represents the spectral space.
+/**
+ * \brief Class that represents the spectral space.
  *
  * (Contains info about the size of the spectral space corresponding
  * to each box in `realspace_ba`, as well as the value of the
diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.H b/Source/FieldSolver/SpectralSolver/SpectralSolver.H
index c570b017b..bd92d003f 100644
--- a/Source/FieldSolver/SpectralSolver/SpectralSolver.H
+++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.H
@@ -4,7 +4,8 @@
 #include <SpectralBaseAlgorithm.H>
 #include <SpectralFieldData.H>
 
-/* \brief Top-level class for the electromagnetic spectral solver
+/**
+ * \brief Top-level class for the electromagnetic spectral solver
  *
  * Stores the field in spectral space, and has member functions
  * to Fourier-transform the fields between real space and spectral space
@@ -26,7 +27,8 @@ class SpectralSolver
                         const amrex::RealVect dx, const amrex::Real dt,
                         const bool pml=false );
 
-        /* \brief Transform the component `i_comp` of MultiFab `mf`
+        /**
+         * \brief Transform the component `i_comp` of MultiFab `mf`
          *  to spectral space, and store the corresponding result internally
          *  (in the spectral field specified by `field_index`) */
         void ForwardTransform( const amrex::MultiFab& mf,
@@ -36,8 +38,10 @@ class SpectralSolver
             field_data.ForwardTransform( mf, field_index, i_comp );
         };
 
-        /* \brief Transform spectral field specified by `field_index` back to
-         * real space, and store it in the component `i_comp` of `mf` */
+        /**
+         * \brief Transform spectral field specified by `field_index` back to
+         * real space, and store it in the component `i_comp` of `mf`
+         */
         void BackwardTransform( amrex::MultiFab& mf,
                                 const int field_index,
                                 const int i_comp=0 ){
@@ -45,7 +49,9 @@ class SpectralSolver
             field_data.BackwardTransform( mf, field_index, i_comp );
         };
 
-        /* \brief Update the fields in spectral space, over one timestep */
+        /**
+         * \brief Update the fields in spectral space, over one timestep
+         */
         void pushSpectralFields(){
             BL_PROFILE("SpectralSolver::pushSpectralFields");
             // Virtual function: the actual function used here depends
diff --git a/Source/Make.WarpX b/Source/Make.WarpX
index fa708a62a..de6eed9da 100644
--- a/Source/Make.WarpX
+++ b/Source/Make.WarpX
@@ -124,7 +124,7 @@ endif
 
 ifeq ($(USE_OPENPMD), TRUE)
    # try pkg-config query
-   ifeq (0, $(shell pkg-config "openPMD >= 0.9.0"; echo $$?))
+   ifeq (0, $(shell pkg-config "openPMD >= 0.10.3"; echo $$?))
        CXXFLAGS += $(shell pkg-config --cflags openPMD)
        LIBRARY_LOCATIONS += $(shell pkg-config --variable=libdir openPMD)
        libraries += $(shell pkg-config --libs-only-l openPMD)
diff --git a/Source/Parallelization/WarpXSumGuardCells.H b/Source/Parallelization/WarpXSumGuardCells.H
index ce353c2b6..36eb4ed6c 100644
--- a/Source/Parallelization/WarpXSumGuardCells.H
+++ b/Source/Parallelization/WarpXSumGuardCells.H
@@ -3,7 +3,7 @@
 
 #include <AMReX_MultiFab.H>
 
-/* \brief Sum the values of `mf`, where the different boxes overlap
+/** \brief Sum the values of `mf`, where the different boxes overlap
  * (i.e. in the guard cells)
  *
  * This is typically called for the sources of the Maxwell equations (J/rho)
@@ -28,7 +28,7 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period,
     mf.SumBoundary(icomp, ncomp, n_updated_guards, period);
 }
 
-/* \brief Sum the values of `src` where the different boxes overlap
+/** \brief Sum the values of `src` where the different boxes overlap
  * (i.e. in the guard cells) and copy them into `dst`
  *
  * This is typically called for the sources of the Maxwell equations (J/rho)
diff --git a/Source/Particles/Collision/ComputeTemperature.H b/Source/Particles/Collision/ComputeTemperature.H
index 3cc96fb52..770510d74 100644
--- a/Source/Particles/Collision/ComputeTemperature.H
+++ b/Source/Particles/Collision/ComputeTemperature.H
@@ -4,6 +4,7 @@
 #include <WarpXConst.H>
 
 template <typename T_index, typename T_R>
+AMREX_GPU_HOST_DEVICE
 T_R ComputeTemperature (
     T_index const Is, T_index const Ie, T_index const *I,
     T_R const *ux, T_R const *uy, T_R const *uz, T_R const m )
diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H
index c1502e311..870dbcd33 100644
--- a/Source/Particles/Deposition/CurrentDeposition.H
+++ b/Source/Particles/Deposition/CurrentDeposition.H
@@ -7,7 +7,8 @@
 #include <AMReX_Array4.H>
 #include <AMReX_REAL.H>
 
-/* \brief Current Deposition for thread thread_num
+/**
+ * \brief Current Deposition for thread thread_num
  * /param xp, yp, zp   : Pointer to arrays of particle positions.
  * \param wp           : Pointer to array of particle weights.
  * \param uxp uyp uzp  : Pointer to arrays of particle momentum.
@@ -219,7 +220,9 @@ void doDepositionShapeN(const amrex::ParticleReal * const xp,
         );
 }
 
-/* \brief Esirkepov Current Deposition for thread thread_num
+/**
+ * \brief Esirkepov Current Deposition for thread thread_num
+ *
  * \param xp, yp, zp   : Pointer to arrays of particle positions.
  * \param wp           : Pointer to array of particle weights.
  * \param uxp uyp uzp  : Pointer to arrays of particle momentum.
diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H
index b4fc84908..57c5d1a4a 100644
--- a/Source/Particles/Gather/FieldGather.H
+++ b/Source/Particles/Gather/FieldGather.H
@@ -4,7 +4,8 @@
 #include "ShapeFactors.H"
 #include <WarpX_Complex.H>
 
-/* \brief Field gather for particles handled by thread thread_num
+/**
+ * \brief Field gather for particles handled by thread thread_num
  * \param xp, yp, zp   : Pointer to arrays of particle positions.
  * \param Exp, Eyp, Ezp: Pointer to array of electric field on particles.
  * \param Bxp, Byp, Bzp: Pointer to array of magnetic field on particles.
diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H
index f0dfa4c83..ae73a74e4 100644
--- a/Source/Particles/Pusher/GetAndSetPosition.H
+++ b/Source/Particles/Pusher/GetAndSetPosition.H
@@ -7,7 +7,7 @@
 
 #ifndef WARPX_DIM_RZ
 
-/* \brief Extract the particle's coordinates from the ParticleType struct `p`,
+/** \brief Extract the particle's coordinates from the ParticleType struct `p`,
  *        and stores them in the variables `x`, `y`, `z`. */
 AMREX_GPU_HOST_DEVICE AMREX_INLINE
 void GetPosition(
@@ -25,7 +25,7 @@ void GetPosition(
 #endif
 }
 
-/* \brief Set the particle's coordinates in the ParticleType struct `p`,
+/** \brief Set the particle's coordinates in the ParticleType struct `p`,
  *        from their values in the variables `x`, `y`, `z`. */
 AMREX_GPU_HOST_DEVICE AMREX_INLINE
 void SetPosition(
@@ -44,7 +44,7 @@ void SetPosition(
 
 # elif defined WARPX_DIM_RZ
 
-/* \brief Extract the particle's coordinates from `theta` and the attributes
+/** \brief Extract the particle's coordinates from `theta` and the attributes
  *         of the ParticleType struct `p` (which contains the radius),
  *         and store them in the variables `x`, `y`, `z` */
 AMREX_GPU_HOST_DEVICE AMREX_INLINE
@@ -58,7 +58,7 @@ void GetCartesianPositionFromCylindrical(
     z = p.pos(1);
 }
 
-/* \brief Set the particle's cylindrical coordinates by setting `theta`
+/** \brief Set the particle's cylindrical coordinates by setting `theta`
  *        and the attributes of the ParticleType struct `p` (which stores the radius),
  *        from the values of `x`, `y`, `z` */
 AMREX_GPU_HOST_DEVICE AMREX_INLINE
diff --git a/Source/Particles/Pusher/UpdateMomentumBoris.H b/Source/Particles/Pusher/UpdateMomentumBoris.H
index 205cc9a71..160f38ade 100644
--- a/Source/Particles/Pusher/UpdateMomentumBoris.H
+++ b/Source/Particles/Pusher/UpdateMomentumBoris.H
@@ -3,7 +3,7 @@
 
 #include <AMReX_REAL.H>
 
-/* \brief Push the particle's positions over one timestep,
+/** \brief Push the particle's positions over one timestep,
  *    given the value of its momenta `ux`, `uy`, `uz` */
 AMREX_GPU_HOST_DEVICE AMREX_INLINE
 void UpdateMomentumBoris(
diff --git a/Source/Particles/Pusher/UpdateMomentumVay.H b/Source/Particles/Pusher/UpdateMomentumVay.H
index 433a891c5..f7ec79d89 100644
--- a/Source/Particles/Pusher/UpdateMomentumVay.H
+++ b/Source/Particles/Pusher/UpdateMomentumVay.H
@@ -5,7 +5,7 @@
 #include <WarpXConst.H>
 #include <AMReX_REAL.H>
 
-/* \brief Push the particle's positions over one timestep,
+/** \brief Push the particle's positions over one timestep,
  *    given the value of its momenta `ux`, `uy`, `uz` */
 AMREX_GPU_HOST_DEVICE AMREX_INLINE
 void UpdateMomentumVay(
diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H
index da0e9cdf9..9943128f1 100644
--- a/Source/Particles/Pusher/UpdatePosition.H
+++ b/Source/Particles/Pusher/UpdatePosition.H
@@ -5,7 +5,7 @@
 #include <WarpXConst.H>
 #include <AMReX_REAL.H>
 
-/* \brief Push the particle's positions over one timestep,
+/** \brief Push the particle's positions over one timestep,
  *    given the value of its momenta `ux`, `uy`, `uz` */
 AMREX_GPU_HOST_DEVICE AMREX_INLINE
 void UpdatePosition(
diff --git a/Source/Particles/Pusher/UpdatePositionPhoton.H b/Source/Particles/Pusher/UpdatePositionPhoton.H
index f95c2b09d..1a0bd114f 100644
--- a/Source/Particles/Pusher/UpdatePositionPhoton.H
+++ b/Source/Particles/Pusher/UpdatePositionPhoton.H
@@ -6,8 +6,10 @@
 #include <AMReX_FArrayBox.H>
 #include <AMReX_REAL.H>
 
-/* \brief Push the position of a photon particle over one timestep,
- *    given the value of its momenta `ux`, `uy`, `uz` */
+/**
+ * \brief Push the position of a photon particle over one timestep,
+ *    given the value of its momenta `ux`, `uy`, `uz`
+ */
 AMREX_GPU_HOST_DEVICE AMREX_INLINE
 void UpdatePositionPhoton(
     amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z,
diff --git a/Source/Particles/Sorting/SortingUtils.H b/Source/Particles/Sorting/SortingUtils.H
index 80eaaf9cb..35bc059aa 100644
--- a/Source/Particles/Sorting/SortingUtils.H
+++ b/Source/Particles/Sorting/SortingUtils.H
@@ -8,7 +8,7 @@
     #include <thrust/distance.h>
 #endif
 
-/* \brief Fill the elements of the input vector with consecutive integer,
+/** \brief Fill the elements of the input vector with consecutive integer,
  *        starting from 0
  *
  * \param[inout] v Vector of integers, to be filled by this routine
@@ -24,7 +24,7 @@ void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector<long>& v )
 #endif
 }
 
-/* \brief Find the indices that would reorder the elements of `predicate`
+/** \brief Find the indices that would reorder the elements of `predicate`
  * so that the elements with non-zero value precede the other elements
  *
  * \param[in, out] index_begin Point to the beginning of the vector which is
@@ -56,7 +56,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin,
     return sep;
 }
 
-/* \brief Return the number of elements between `first` and `last`
+/** \brief Return the number of elements between `first` and `last`
  *
  * \param[in] fist Points to a position in a vector
  * \param[in] last Points to another position in a vector
@@ -74,7 +74,7 @@ int iteratorDistance(ForwardIterator const first,
 #endif
 }
 
-/* \brief Functor that fills the elements of the particle array `inexflag`
+/** \brief Functor that fills the elements of the particle array `inexflag`
  *  with the value of the spatial array `bmasks`, at the corresponding particle position.
  *
  * \param[in] pti Contains information on the particle positions
@@ -124,7 +124,7 @@ class fillBufferFlag
         amrex::Array4<int const> m_buffer_mask;
 };
 
-/* \brief Functor that fills the elements of the particle array `inexflag`
+/** \brief Functor that fills the elements of the particle array `inexflag`
  *  with the value of the spatial array `bmasks`, at the corresponding particle position.
  *
  * Contrary to `fillBufferFlag`, here this is done only for the particles that
@@ -186,7 +186,7 @@ class fillBufferFlagRemainingParticles
         long const* m_indices_ptr;
 };
 
-/* \brief Functor that copies the elements of `src` into `dst`,
+/** \brief Functor that copies the elements of `src` into `dst`,
  *       while reordering them according to `indices`
  *
  * \param[in] src Source vector
diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp
index e6a719ec5..15a6cff9b 100644
--- a/Source/Particles/WarpXParticleContainer.cpp
+++ b/Source/Particles/WarpXParticleContainer.cpp
@@ -173,7 +173,6 @@ WarpXParticleContainer::AddNParticles (int lev,
 
     //  Add to grid 0 and tile 0
     // Redistribute() will move them to proper places.
-    std::pair<int,int> key {0,0};
     auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
 
     std::size_t np = iend-ibegin;
diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp
index 4b66a0809..08272089e 100644
--- a/Source/Utils/WarpXAlgorithmSelection.cpp
+++ b/Source/Utils/WarpXAlgorithmSelection.cpp
@@ -18,14 +18,18 @@ const std::map<std::string, int> maxwell_solver_algo_to_int = {
 const std::map<std::string, int> particle_pusher_algo_to_int = {
     {"boris",   ParticlePusherAlgo::Boris },
     {"vay",     ParticlePusherAlgo::Vay },
-    {"higuera",     ParticlePusherAlgo::HigueraCary },
+    {"higuera", ParticlePusherAlgo::HigueraCary },
     {"default", ParticlePusherAlgo::Boris }
 };
 
 const std::map<std::string, int> current_deposition_algo_to_int = {
-    {"esirkepov",            CurrentDepositionAlgo::Esirkepov },
-    {"direct",               CurrentDepositionAlgo::Direct },
-    {"default",              CurrentDepositionAlgo::Esirkepov }
+    {"esirkepov", CurrentDepositionAlgo::Esirkepov },
+    {"direct",    CurrentDepositionAlgo::Direct },
+#ifdef WARPX_USE_PSATD
+    {"default",   CurrentDepositionAlgo::Direct }
+#else
+    {"default",   CurrentDepositionAlgo::Esirkepov }
+#endif
 };
 
 const std::map<std::string, int> charge_deposition_algo_to_int = {
diff --git a/Source/main.cpp b/Source/main.cpp
index cb183bc8d..19413da7a 100644
--- a/Source/main.cpp
+++ b/Source/main.cpp
@@ -16,7 +16,7 @@ int main(int argc, char* argv[])
 #if defined(_OPENMP) && defined(WARPX_USE_PSATD)
     int provided;
     MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &provided);
-    assert(provided >= MPI_THREAD_FUNNELED);
+    AMREX_ALWAYS_ASSERT(provided >= MPI_THREAD_FUNNELED);
 #else
     MPI_Init(&argc, &argv);
 #endif