Add timers in routines that depend on cell-related work (#1692)

* Add timers

* eol

* AtomicAdd

* lev argument for getCosts

* style

* style

* wip

* eol

* .ipynb

* passing down lev

* eol

* passing lev

* eol

* Update Source/Particles/Collision/PairWiseCoulombCollision.cpp

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

* Add for QED and ionization routines

* eol

* remove unneeded

* mfi-->pti

* move cost

* eol

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

1 files changed, 16 insertions, 14 deletions

diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.cpp
index 0a278a54a..379bb6369 100644
--- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.cpp
+++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.cpp
@@ -881,6 +881,7 @@ void PsatdAlgorithm::InitializeSpectralCoefficients (
 
 void
 PsatdAlgorithm::CurrentCorrection (
+    const int lev,
     SpectralFieldData& field_data,
     std::array<std::unique_ptr<amrex::MultiFab>,3>& current,
     const std::unique_ptr<amrex::MultiFab>& rho)
@@ -891,11 +892,11 @@ PsatdAlgorithm::CurrentCorrection (
     using Idx = SpectralFieldIndex;
 
     // Forward Fourier transform of J and rho
-    field_data.ForwardTransform(*current[0], Idx::Jx, 0);
-    field_data.ForwardTransform(*current[1], Idx::Jy, 0);
-    field_data.ForwardTransform(*current[2], Idx::Jz, 0);
-    field_data.ForwardTransform(*rho, Idx::rho_old, 0);
-    field_data.ForwardTransform(*rho, Idx::rho_new, 1);
+    field_data.ForwardTransform(lev, *current[0], Idx::Jx, 0);
+    field_data.ForwardTransform(lev, *current[1], Idx::Jy, 0);
+    field_data.ForwardTransform(lev, *current[2], Idx::Jz, 0);
+    field_data.ForwardTransform(lev, *rho, Idx::rho_old, 0);
+    field_data.ForwardTransform(lev, *rho, Idx::rho_new, 1);
 
     // Loop over boxes
     for (amrex::MFIter mfi(field_data.fields); mfi.isValid(); ++mfi){
@@ -989,13 +990,14 @@ PsatdAlgorithm::CurrentCorrection (
     }
 
     // Backward Fourier transform of J
-    field_data.BackwardTransform(*current[0], Idx::Jx, 0);
-    field_data.BackwardTransform(*current[1], Idx::Jy, 0);
-    field_data.BackwardTransform(*current[2], Idx::Jz, 0);
+    field_data.BackwardTransform(lev, *current[0], Idx::Jx, 0);
+    field_data.BackwardTransform(lev, *current[1], Idx::Jy, 0);
+    field_data.BackwardTransform(lev, *current[2], Idx::Jz, 0);
 }
 
 void
 PsatdAlgorithm::VayDeposition (
+    const int lev,
     SpectralFieldData& field_data,
     std::array<std::unique_ptr<amrex::MultiFab>,3>& current)
 {
@@ -1007,9 +1009,9 @@ PsatdAlgorithm::VayDeposition (
     // Forward Fourier transform of D (temporarily stored in current):
     // D is nodal and does not match the staggering of J, therefore we pass the
     // actual staggering of D (IntVect(1)) to the ForwardTransform function
-    field_data.ForwardTransform(*current[0], Idx::Jx, 0, IntVect(1));
-    field_data.ForwardTransform(*current[1], Idx::Jy, 0, IntVect(1));
-    field_data.ForwardTransform(*current[2], Idx::Jz, 0, IntVect(1));
+    field_data.ForwardTransform(lev, *current[0], Idx::Jx, 0, IntVect(1));
+    field_data.ForwardTransform(lev, *current[1], Idx::Jy, 0, IntVect(1));
+    field_data.ForwardTransform(lev, *current[2], Idx::Jz, 0, IntVect(1));
 
     // Loop over boxes
     for (amrex::MFIter mfi(field_data.fields); mfi.isValid(); ++mfi)
@@ -1065,9 +1067,9 @@ PsatdAlgorithm::VayDeposition (
     }
 
     // Backward Fourier transform of J
-    field_data.BackwardTransform(*current[0], Idx::Jx, 0);
-    field_data.BackwardTransform(*current[1], Idx::Jy, 0);
-    field_data.BackwardTransform(*current[2], Idx::Jz, 0);
+    field_data.BackwardTransform(lev, *current[0], Idx::Jx, 0);
+    field_data.BackwardTransform(lev, *current[1], Idx::Jy, 0);
+    field_data.BackwardTransform(lev, *current[2], Idx::Jz, 0);
 }
 
 #endif // WARPX_USE_PSATD