Merge branch 'merged_overlap_pml' of https://github.com/ablelly/WarpX into merged_overlap_pml

1 files changed, 25 insertions, 41 deletions

diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp
index dae18002d..609a50a17 100644
--- a/Source/FieldSolver/WarpXPushFieldsEM.cpp
+++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp
@@ -61,7 +61,7 @@ WarpX::PushPSATD (amrex::Real a_dt)
     for (int lev = 0; lev <= finest_level; ++lev) {
         AMREX_ALWAYS_ASSERT_WITH_MESSAGE(dt[lev] == a_dt, "dt must be consistent");
         if (fft_hybrid_mpi_decomposition){
-#ifndef AMREX_USE_CUDA // Only available on CPU
+#ifdef WARPX_USE_PSATD_HYBRID
             PushPSATD_hybridFFT(lev, a_dt);
 #endif
         } else {
@@ -161,33 +161,29 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt)
         auto const& Eyfab = Ey->array(mfi);
         auto const& Ezfab = Ez->array(mfi);
         if (do_nodal) {
-            amrex::ParallelFor(tbx,
+            amrex::ParallelFor(tbx, tby, tbz,
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_bx_nodal(j,k,l,Bxfab,Eyfab,Ezfab,dtsdy,dtsdz);
-            });
-            amrex::ParallelFor(tby,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_by_nodal(j,k,l,Byfab,Exfab,Ezfab,dtsdx,dtsdz);
-            });
-            amrex::ParallelFor(tbz,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_bz_nodal(j,k,l,Bzfab,Exfab,Eyfab,dtsdx,dtsdy);
             });
         } else if (WarpX::maxwell_fdtd_solver_id == 0) {
-            amrex::ParallelFor(tbx,
+            amrex::ParallelFor(tbx, tby, tbz,
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_bx_yee(j,k,l,Bxfab,Eyfab,Ezfab,dtsdy,dtsdz);
-            });
-            amrex::ParallelFor(tby,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_by_yee(j,k,l,Byfab,Exfab,Ezfab,dtsdx,dtsdz);
-            });
-            amrex::ParallelFor(tbz,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_bz_yee(j,k,l,Bzfab,Exfab,Eyfab,dtsdx,dtsdy,dxinv,xmin);
@@ -200,23 +196,21 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt)
                                              betaxy, betaxz, betayx, betayz, betazx, betazy,
                                              gammax, gammay, gammaz,
                                              alphax, alphay, alphaz);
-            amrex::ParallelFor(tbx,
+            amrex::ParallelFor(tbx, tby, tbz,
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_bx_ckc(j,k,l,Bxfab,Eyfab,Ezfab,
                                   betaxy, betaxz, betayx, betayz, betazx, betazy,
                                   gammax, gammay, gammaz,
                                   alphax, alphay, alphaz);
-            });
-            amrex::ParallelFor(tby,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_by_ckc(j,k,l,Byfab,Exfab,Ezfab,
                                   betaxy, betaxz, betayx, betayz, betazx, betazy,
                                   gammax, gammay, gammaz,
                                   alphax, alphay, alphaz);
-            });
-            amrex::ParallelFor(tbz,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_bz_ckc(j,k,l,Bzfab,Exfab,Eyfab,
@@ -358,33 +352,29 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt)
         auto const& jzfab = jz->array(mfi);
 
         if (do_nodal) {
-            amrex::ParallelFor(tex,
+            amrex::ParallelFor(tex, tey, tez,
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_ex_nodal(j,k,l,Exfab,Byfab,Bzfab,jxfab,mu_c2_dt,dtsdy_c2,dtsdz_c2);
-            });
-            amrex::ParallelFor(tey,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_ey_nodal(j,k,l,Eyfab,Bxfab,Bzfab,jyfab,mu_c2_dt,dtsdx_c2,dtsdz_c2);
-            });
-            amrex::ParallelFor(tez,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_ez_nodal(j,k,l,Ezfab,Bxfab,Byfab,jzfab,mu_c2_dt,dtsdx_c2,dtsdy_c2);
             });
         } else {
-            amrex::ParallelFor(tex,
+            amrex::ParallelFor(tex, tey, tez,
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_ex_yee(j,k,l,Exfab,Byfab,Bzfab,jxfab,mu_c2_dt,dtsdy_c2,dtsdz_c2);
-            });
-            amrex::ParallelFor(tey,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_ey_yee(j,k,l,Eyfab,Bxfab,Bzfab,jyfab,mu_c2_dt,dtsdx_c2,dtsdz_c2,xmin);
-            });
-            amrex::ParallelFor(tez,
+            },
             [=] AMREX_GPU_DEVICE (int j, int k, int l)
             {
                 warpx_push_ez_yee(j,k,l,Ezfab,Bxfab,Byfab,jzfab,mu_c2_dt,dtsdx_c2,dtsdy_c2,dxinv,xmin);
@@ -395,17 +385,15 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt)
         {
             auto const& Ffab = F->array(mfi);
             if (WarpX::maxwell_fdtd_solver_id == 0) {
-                amrex::ParallelFor(tex,
+                amrex::ParallelFor(tex, tey, tez,
                 [=] AMREX_GPU_DEVICE (int j, int k, int l)
                 {
                     warpx_push_ex_f_yee(j,k,l,Exfab,Ffab,dtsdx_c2);
-                });
-                amrex::ParallelFor(tey,
+                },
                 [=] AMREX_GPU_DEVICE (int j, int k, int l)
                 {
                     warpx_push_ey_f_yee(j,k,l,Eyfab,Ffab,dtsdy_c2);
-                });
-                amrex::ParallelFor(tez,
+                },
                 [=] AMREX_GPU_DEVICE (int j, int k, int l)
                 {
                     warpx_push_ez_f_yee(j,k,l,Ezfab,Ffab,dtsdz_c2);
@@ -419,23 +407,21 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt)
                                                  betaxy, betaxz, betayx, betayz, betazx, betazy,
                                                  gammax, gammay, gammaz,
                                                  alphax, alphay, alphaz);
-                amrex::ParallelFor(tex,
+                amrex::ParallelFor(tex, tey, tez,
                 [=] AMREX_GPU_DEVICE (int j, int k, int l)
                 {
                     warpx_push_ex_f_ckc(j,k,l,Exfab,Ffab,
                                         betaxy, betaxz, betayx, betayz, betazx, betazy,
                                         gammax, gammay, gammaz,
                                         alphax, alphay, alphaz);
-                });
-                amrex::ParallelFor(tey,
+                },
                 [=] AMREX_GPU_DEVICE (int j, int k, int l)
                 {
                     warpx_push_ey_f_ckc(j,k,l,Eyfab,Ffab,
                                         betaxy, betaxz, betayx, betayz, betazx, betazy,
                                         gammax, gammay, gammaz,
                                         alphax, alphay, alphaz);
-                });
-                amrex::ParallelFor(tez,
+                },
                 [=] AMREX_GPU_DEVICE (int j, int k, int l)
                 {
                     warpx_push_ez_f_ckc(j,k,l,Ezfab,Ffab,
@@ -663,7 +649,7 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu
 
         // Rescale current in r-z mode since the inverse volume factor was not
         // included in the current deposition.
-        amrex::ParallelFor(tbr,
+        amrex::ParallelFor(tbr, tbt, tbz,
         [=] AMREX_GPU_DEVICE (int i, int j, int k)
         {
             // Wrap the current density deposited in the guard cells around
@@ -677,8 +663,7 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu
             // between on axis and off-axis factors
             const amrex::Real r = std::abs(rmin + (i - irmin + 0.5)*dr);
             Jr_arr(i,j,0) /= (2.*MathConst::pi*r);
-        });
-        amrex::ParallelFor(tbt,
+        },
         [=] AMREX_GPU_DEVICE (int i, int j, int k)
         {
             // Wrap the current density deposited in the guard cells around
@@ -696,8 +681,7 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu
             } else {
                 Jt_arr(i,j,0) /= (2.*MathConst::pi*r);
             }
-        });
-        amrex::ParallelFor(tbz,
+        },
         [=] AMREX_GPU_DEVICE (int i, int j, int k)
         {
             // Wrap the current density deposited in the guard cells around