Move KernelTimer.H to ablastr/parallelization (#3863)

* move KernelTimer to ablastr

* add missing folder

* fixed bug

* fixed bug

* really fixed the bug

* kernelTimer -> KernelTimer

8 files changed, 48 insertions, 39 deletions

diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H
index 7f6cccea5..88d952919 100644
--- a/Source/Particles/Deposition/ChargeDeposition.H
+++ b/Source/Particles/Deposition/ChargeDeposition.H
@@ -8,7 +8,7 @@
 #ifndef CHARGEDEPOSITION_H_
 #define CHARGEDEPOSITION_H_
 
-#include "Parallelization/KernelTimer.H"
+#include "ablastr/parallelization/KernelTimer.H"
 #include "Particles/Pusher/GetAndSetPosition.H"
 #include "Particles/ShapeFactors.H"
 #include "Utils/WarpXAlgorithmSelection.H"
@@ -98,8 +98,9 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition,
         np_to_depose,
         [=] AMREX_GPU_DEVICE (long ip) {
 #if defined(WARPX_USE_GPUCLOCK)
-            KernelTimer kernelTimer(cost && load_balance_costs_update_algo
-                                 == LoadBalanceCostsUpdateAlgo::GpuClock, cost_real);
+            const auto KernelTimer = ablastr::parallelization::KernelTimer(
+                cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock),
+                cost_real);
 #endif
             // --- Get particle quantities
             amrex::Real wq = q*wp[ip]*invvol;
diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H
index ccc27ca96..8a3aae004 100644
--- a/Source/Particles/Deposition/CurrentDeposition.H
+++ b/Source/Particles/Deposition/CurrentDeposition.H
@@ -8,7 +8,7 @@
 #ifndef CURRENTDEPOSITION_H_
 #define CURRENTDEPOSITION_H_
 
-#include "Parallelization/KernelTimer.H"
+#include "ablastr/parallelization/KernelTimer.H"
 #include "Particles/Pusher/GetAndSetPosition.H"
 #include "Particles/ShapeFactors.H"
 #include "Utils/WarpXAlgorithmSelection.H"
@@ -125,8 +125,9 @@ void doDepositionShapeN(const GetParticlePosition& GetPosition,
         np_to_depose,
         [=] AMREX_GPU_DEVICE (long ip) {
 #if defined(WARPX_USE_GPUCLOCK)
-            KernelTimer kernelTimer(cost && load_balance_costs_update_algo
-                                 == LoadBalanceCostsUpdateAlgo::GpuClock, cost_real);
+            const auto KernelTimer = ablastr::parallelization::KernelTimer(
+                cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock),
+                cost_real);
 #endif
 
             // --- Get particle quantities
@@ -441,8 +442,9 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition,
         np_to_depose,
         [=] AMREX_GPU_DEVICE (long const ip) {
 #if defined(WARPX_USE_GPUCLOCK)
-            KernelTimer kernelTimer(cost && load_balance_costs_update_algo
-                                 == LoadBalanceCostsUpdateAlgo::GpuClock, cost_real);
+            const auto KernelTimer = ablastr::parallelization::KernelTimer(
+                cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock),
+                cost_real);
 #endif
 
             // --- Get particle quantities
@@ -836,8 +838,9 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition,
     amrex::ParallelFor(np_to_depose, [=] AMREX_GPU_DEVICE (long ip)
     {
 #if defined(WARPX_USE_GPUCLOCK)
-        KernelTimer kernelTimer(cost && load_balance_costs_update_algo
-                             == LoadBalanceCostsUpdateAlgo::GpuClock, cost_real);
+        const auto KernelTimer = ablastr::parallelization::KernelTimer(
+            cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock),
+            cost_real);
 #endif
 
         // Inverse of Lorentz factor gamma
diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp
index 69efd62dd..518d86def 100644
--- a/Source/WarpX.cpp
+++ b/Source/WarpX.cpp
@@ -1106,6 +1106,10 @@ WarpX::ReadParameters ()
             utils::parser::queryWithParser(
                 pp_algo, "costs_heuristic_particles_wt", costs_heuristic_particles_wt);
         }
+#   ifndef WARPX_USE_GPUCLOCK
+        WARPX_ALWAYS_ASSERT_WITH_MESSAGE(WarpX::load_balance_costs_update_algo!=LoadBalanceCostsUpdateAlgo::GpuClock,
+            "`algo.load_balance_costs_update = gpuclock` requires to compile with `-DWarpX_GPUCLOCK=ON`.");
+#   endif // !WARPX_USE_GPUCLOCK
 
         // Parse algo.particle_shape and check that input is acceptable
         // (do this only if there is at least one particle or laser species)
diff --git a/Source/ablastr/CMakeLists.txt b/Source/ablastr/CMakeLists.txt
index 0224f1a8b..297267a06 100644
--- a/Source/ablastr/CMakeLists.txt
+++ b/Source/ablastr/CMakeLists.txt
@@ -1,5 +1,6 @@
 #add_subdirectory(fields)
 add_subdirectory(coarsen)
+#add_subdirectory(parallelization)
 #add_subdirectory(particles)
 #add_subdirectory(profiler)
 add_subdirectory(utils)
diff --git a/Source/ablastr/Make.package b/Source/ablastr/Make.package
index b10d9f629..2d848e63a 100644
--- a/Source/ablastr/Make.package
+++ b/Source/ablastr/Make.package
@@ -1,6 +1,7 @@
 #CEXE_sources += ParticleBoundaries.cpp
 
 include $(WARPX_HOME)/Source/ablastr/coarsen/Make.package
+include $(WARPX_HOME)/Source/ablastr/parallelization/Make.package
 include $(WARPX_HOME)/Source/ablastr/particles/Make.package
 include $(WARPX_HOME)/Source/ablastr/utils/Make.package
 include $(WARPX_HOME)/Source/ablastr/warn_manager/Make.package
diff --git a/Source/Parallelization/KernelTimer.H b/Source/ablastr/parallelization/KernelTimer.H
index 3a5c8bada..0d11220da 100644
--- a/Source/Parallelization/KernelTimer.H
+++ b/Source/ablastr/parallelization/KernelTimer.H
@@ -4,19 +4,22 @@
  *
  * License: BSD-3-Clause-LBNL
  */
-#ifndef KERNELTIMER_H_
-#define KERNELTIMER_H_
+#ifndef ABLASTR_KERNELTIMER_H_
+#define ABLASTR_KERNELTIMER_H_
 
-#include "Utils/TextMsg.H"
-#include "WarpX.H"
+#include "ablastr/utils/TextMsg.H"
 
 #include <AMReX.H>
+#include <AMReX_BLassert.H>
+#include <AMReX_GpuAtomic.H>
+#include <AMReX_GpuQualifiers.H>
 #include <AMReX_REAL.H>
 
-// #ifdef WARPX_USE_GPUCLOCK
-
 #include <climits>
 
+namespace ablastr::parallelization
+{
+
 /**
  * \brief Defines a timer object to be used on GPU; measures summed thread cycles.
  */
@@ -30,31 +33,24 @@ public:
      */
     AMREX_GPU_DEVICE
     KernelTimer
-#if (defined AMREX_USE_GPU)
     (const bool do_timing, amrex::Real* cost)
+#if (defined AMREX_USE_GPU)
         : m_do_timing(do_timing)
-#else
-    (const bool do_timing, amrex::Real* cost)
 #endif
     {
 #if (defined AMREX_USE_GPU)
-#   if (defined WARPX_USE_GPUCLOCK)
-        if (m_do_timing && cost) {
-            m_cost = cost;
-#       if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
-            // Start the timer
-            m_wt = clock64();
+    if (m_do_timing && cost) {
+        m_cost = cost;
+#   if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
+        // Start the timer
+        m_wt = clock64();
 
-#       elif defined(AMREX_USE_DPCPP)
-            // To be updated
-            AMREX_ALWAYS_ASSERT_WITH_MESSAGE( m_do_timing == false,
-                                              "KernelTimer not yet supported for this hardware." );
-#       endif
-        }
-#   else // WARPX_USE_GPUCLOCK
-        AMREX_ALWAYS_ASSERT_WITH_MESSAGE( m_do_timing == false,
-                                          "`algo.load_balance_costs_update = gpuclock` requires to compile with `-DWarpX_GPUCLOCK=ON`.");
-#   endif // WARPX_USE_GPUCLOCK
+#   elif defined(AMREX_USE_DPCPP)
+        // To be updated
+        ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(m_do_timing == false,
+            "KernelTimer not yet supported for this hardware.");
+#   endif
+    }
 #else  // AMREX_USE_GPU
         amrex::ignore_unused(do_timing, cost);
 #endif // AMREX_USE_GPU
@@ -72,7 +68,7 @@ public:
             // To be updated
 #   endif
         }
-#endif
+#endif //AMREX_USE_GPU
     }
 
 #if (defined AMREX_USE_GPU)
@@ -85,6 +81,9 @@ private:
 
     //! Store the time difference (cost) from a single thread.
     long long int m_wt;
-#endif
+#endif //AMREX_USE_GPU
 };
-#endif
+
+} // namespace ablastr::parallelization
+
+#endif // ABLASTR_KERNELTIMER_H_
diff --git a/Source/ablastr/parallelization/Make.package b/Source/ablastr/parallelization/Make.package
new file mode 100644
index 000000000..9b248b7fa
--- /dev/null
+++ b/Source/ablastr/parallelization/Make.package
@@ -0,0 +1 @@
+VPATH_LOCATIONS   += $(WARPX_HOME)/Source/ablastr/parallelization
diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H
index e7de36936..75e641cf3 100644
--- a/Source/ablastr/particles/DepositCharge.H
+++ b/Source/ablastr/particles/DepositCharge.H
@@ -8,7 +8,6 @@
 #define ABLASTR_DEPOSIT_CHARGE_H_
 
 #include "ablastr/profiler/ProfilerWrapper.H"
-#include "Parallelization/KernelTimer.H"
 #include "Particles/Pusher/GetAndSetPosition.H"
 #include "Particles/ShapeFactors.H"
 #include "Particles/Deposition/ChargeDeposition.H"