Add timers in routines that depend on cell-related work (#1692)

* Add timers

* eol

* AtomicAdd

* lev argument for getCosts

* style

* style

* wip

* eol

* .ipynb

* passing down lev

* eol

* passing lev

* eol

* Update Source/Particles/Collision/PairWiseCoulombCollision.cpp

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

* Add for QED and ionization routines

* eol

* remove unneeded

* mfi-->pti

* move cost

* eol

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

1 files changed, 31 insertions, 30 deletions

diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp
index 25835be6a..630703b3b 100644
--- a/Source/BoundaryConditions/PML.cpp
+++ b/Source/BoundaryConditions/PML.cpp
@@ -425,7 +425,7 @@ MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt)
     }
 }
 
-PML::PML (const BoxArray& grid_ba, const DistributionMapping& /*grid_dm*/,
+PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& /*grid_dm*/,
           const Geometry* geom, const Geometry* cgeom,
           int ncell, int delta, amrex::IntVect ref_ratio,
           Real dt, int nox_fft, int noy_fft, int noz_fft, bool do_nodal,
@@ -568,7 +568,7 @@ PML::PML (const BoxArray& grid_ba, const DistributionMapping& /*grid_dm*/,
         Array<Real,3> const v_galilean_zero = {0., 0., 0.};
         Array<Real,3> const v_comoving_zero = {0., 0., 0.};
         realspace_ba.enclosedCells().grow(nge); // cell-centered + guard cells
-        spectral_solver_fp = std::make_unique<SpectralSolver>(realspace_ba, dm,
+        spectral_solver_fp = std::make_unique<SpectralSolver>(lev, realspace_ba, dm,
             nox_fft, noy_fft, noz_fft, do_nodal, v_galilean_zero, v_comoving_zero, dx, dt, in_pml );
 #endif
     }
@@ -667,7 +667,7 @@ PML::PML (const BoxArray& grid_ba, const DistributionMapping& /*grid_dm*/,
             const bool in_pml = true; // Tells spectral solver to use split-PML equations
 
             realspace_cba.enclosedCells().grow(nge); // cell-centered + guard cells
-            spectral_solver_cp = std::make_unique<SpectralSolver>(realspace_cba, cdm,
+            spectral_solver_cp = std::make_unique<SpectralSolver>(lev, realspace_cba, cdm,
                 nox_fft, noy_fft, noz_fft, do_nodal, v_galilean_zero, v_comoving_zero, cdx, dt, in_pml );
 #endif
         }
@@ -1124,17 +1124,18 @@ PML::Restart (const std::string& dir)
 
 #ifdef WARPX_USE_PSATD
 void
-PML::PushPSATD () {
+PML::PushPSATD (const int lev) {
 
     // Update the fields on the fine and coarse patch
-    PushPMLPSATDSinglePatch( *spectral_solver_fp, pml_E_fp, pml_B_fp );
+    PushPMLPSATDSinglePatch( lev, *spectral_solver_fp, pml_E_fp, pml_B_fp );
     if (spectral_solver_cp) {
-        PushPMLPSATDSinglePatch( *spectral_solver_cp, pml_E_cp, pml_B_cp );
+        PushPMLPSATDSinglePatch( lev, *spectral_solver_cp, pml_E_cp, pml_B_cp );
     }
 }
 
 void
 PushPMLPSATDSinglePatch (
+    const int lev,
     SpectralSolver& solver,
     std::array<std::unique_ptr<amrex::MultiFab>,3>& pml_E,
     std::array<std::unique_ptr<amrex::MultiFab>,3>& pml_B ) {
@@ -1146,32 +1147,32 @@ PushPMLPSATDSinglePatch (
     // (Exy, Ezx, etc.) and the component (PMLComp::xy, PMComp::zx, etc.)
     // of the MultiFabs (e.g. pml_E) is dictated by the
     // function that damps the PML
-    solver.ForwardTransform(*pml_E[0], SpIdx::Exy, PMLComp::xy);
-    solver.ForwardTransform(*pml_E[0], SpIdx::Exz, PMLComp::xz);
-    solver.ForwardTransform(*pml_E[1], SpIdx::Eyz, PMLComp::yz);
-    solver.ForwardTransform(*pml_E[1], SpIdx::Eyx, PMLComp::yx);
-    solver.ForwardTransform(*pml_E[2], SpIdx::Ezx, PMLComp::zx);
-    solver.ForwardTransform(*pml_E[2], SpIdx::Ezy, PMLComp::zy);
-    solver.ForwardTransform(*pml_B[0], SpIdx::Bxy, PMLComp::xy);
-    solver.ForwardTransform(*pml_B[0], SpIdx::Bxz, PMLComp::xz);
-    solver.ForwardTransform(*pml_B[1], SpIdx::Byz, PMLComp::yz);
-    solver.ForwardTransform(*pml_B[1], SpIdx::Byx, PMLComp::yx);
-    solver.ForwardTransform(*pml_B[2], SpIdx::Bzx, PMLComp::zx);
-    solver.ForwardTransform(*pml_B[2], SpIdx::Bzy, PMLComp::zy);
+    solver.ForwardTransform(lev, *pml_E[0], SpIdx::Exy, PMLComp::xy);
+    solver.ForwardTransform(lev, *pml_E[0], SpIdx::Exz, PMLComp::xz);
+    solver.ForwardTransform(lev, *pml_E[1], SpIdx::Eyz, PMLComp::yz);
+    solver.ForwardTransform(lev, *pml_E[1], SpIdx::Eyx, PMLComp::yx);
+    solver.ForwardTransform(lev, *pml_E[2], SpIdx::Ezx, PMLComp::zx);
+    solver.ForwardTransform(lev, *pml_E[2], SpIdx::Ezy, PMLComp::zy);
+    solver.ForwardTransform(lev, *pml_B[0], SpIdx::Bxy, PMLComp::xy);
+    solver.ForwardTransform(lev, *pml_B[0], SpIdx::Bxz, PMLComp::xz);
+    solver.ForwardTransform(lev, *pml_B[1], SpIdx::Byz, PMLComp::yz);
+    solver.ForwardTransform(lev, *pml_B[1], SpIdx::Byx, PMLComp::yx);
+    solver.ForwardTransform(lev, *pml_B[2], SpIdx::Bzx, PMLComp::zx);
+    solver.ForwardTransform(lev, *pml_B[2], SpIdx::Bzy, PMLComp::zy);
     // Advance fields in spectral space
     solver.pushSpectralFields();
     // Perform backward Fourier Transform
-    solver.BackwardTransform(*pml_E[0], SpIdx::Exy, PMLComp::xy);
-    solver.BackwardTransform(*pml_E[0], SpIdx::Exz, PMLComp::xz);
-    solver.BackwardTransform(*pml_E[1], SpIdx::Eyz, PMLComp::yz);
-    solver.BackwardTransform(*pml_E[1], SpIdx::Eyx, PMLComp::yx);
-    solver.BackwardTransform(*pml_E[2], SpIdx::Ezx, PMLComp::zx);
-    solver.BackwardTransform(*pml_E[2], SpIdx::Ezy, PMLComp::zy);
-    solver.BackwardTransform(*pml_B[0], SpIdx::Bxy, PMLComp::xy);
-    solver.BackwardTransform(*pml_B[0], SpIdx::Bxz, PMLComp::xz);
-    solver.BackwardTransform(*pml_B[1], SpIdx::Byz, PMLComp::yz);
-    solver.BackwardTransform(*pml_B[1], SpIdx::Byx, PMLComp::yx);
-    solver.BackwardTransform(*pml_B[2], SpIdx::Bzx, PMLComp::zx);
-    solver.BackwardTransform(*pml_B[2], SpIdx::Bzy, PMLComp::zy);
+    solver.BackwardTransform(lev, *pml_E[0], SpIdx::Exy, PMLComp::xy);
+    solver.BackwardTransform(lev, *pml_E[0], SpIdx::Exz, PMLComp::xz);
+    solver.BackwardTransform(lev, *pml_E[1], SpIdx::Eyz, PMLComp::yz);
+    solver.BackwardTransform(lev, *pml_E[1], SpIdx::Eyx, PMLComp::yx);
+    solver.BackwardTransform(lev, *pml_E[2], SpIdx::Ezx, PMLComp::zx);
+    solver.BackwardTransform(lev, *pml_E[2], SpIdx::Ezy, PMLComp::zy);
+    solver.BackwardTransform(lev, *pml_B[0], SpIdx::Bxy, PMLComp::xy);
+    solver.BackwardTransform(lev, *pml_B[0], SpIdx::Bxz, PMLComp::xz);
+    solver.BackwardTransform(lev, *pml_B[1], SpIdx::Byz, PMLComp::yz);
+    solver.BackwardTransform(lev, *pml_B[1], SpIdx::Byx, PMLComp::yx);
+    solver.BackwardTransform(lev, *pml_B[2], SpIdx::Bzx, PMLComp::zx);
+    solver.BackwardTransform(lev, *pml_B[2], SpIdx::Bzy, PMLComp::zy);
 }
 #endif