From 892958903f96367df47242b4f6c63b20fa11221e Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Tue, 21 Feb 2023 09:03:05 -0800 Subject: GetExternalEBField: Use AMReX's CompileTimeOption ParallelFor (#3696) This greatly improves the performance on AMD and Intel GPUs when GetExternalEBField is not used. --- Source/Particles/PhysicalParticleContainer.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 3559797da..1390fdda6 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2482,7 +2482,14 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const auto t_do_not_gather = do_not_gather; - amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + enum exteb_flags : int { no_exteb, has_exteb }; + + int exteb_runtime_flag = getExternalEB.isNoOp() ? no_exteb : has_exteb; + + amrex::ParallelFor(TypeList>{}, + {exteb_runtime_flag}, + np, [=,getExternalEB=getExternalEB] + AMREX_GPU_DEVICE (long ip, auto exteb_control) { amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); @@ -2498,8 +2505,11 @@ PhysicalParticleContainer::PushP (int lev, Real dt, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } + // Externally applied E and B-field in Cartesian co-ordinates - getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + if constexpr (exteb_control == has_exteb) { + getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + } if (do_crr) { amrex::Real qp = q; @@ -2703,8 +2713,9 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, amrex::ParallelFor(TypeList, CompileTimeOptions>{}, {exteb_runtime_flag, qed_runtime_flag}, - np_to_push, [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, - [[maybe_unused]] auto qed_control) + np_to_push, [=,getExternalEB=getExternalEB] + AMREX_GPU_DEVICE (long ip, auto exteb_control, + [[maybe_unused]] auto qed_control) { amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); @@ -2731,9 +2742,8 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, nox, galerkin_interpolation); } - auto const& externeb_fn = getExternalEB; // Have to do this for nvcc if constexpr (exteb_control == has_exteb) { - externeb_fn(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } scaleFields(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp); -- cgit v1.2.3