diff options
-rw-r--r-- | Source/LaserParticleContainer.cpp | 6 | ||||
-rw-r--r-- | Source/PhysicalParticleContainer.H | 8 | ||||
-rw-r--r-- | Source/PhysicalParticleContainer.cpp | 24 | ||||
-rw-r--r-- | Source/WarpX.H | 9 | ||||
-rw-r--r-- | Source/WarpXEvolve.cpp | 14 | ||||
-rw-r--r-- | Source/WarpXMove.cpp | 6 | ||||
-rw-r--r-- | Source/WarpXParticleContainer.cpp | 28 |
7 files changed, 60 insertions, 35 deletions
diff --git a/Source/LaserParticleContainer.cpp b/Source/LaserParticleContainer.cpp index 6ee79e9aa..cf669d32d 100644 --- a/Source/LaserParticleContainer.cpp +++ b/Source/LaserParticleContainer.cpp @@ -432,7 +432,11 @@ LaserParticleContainer::Evolve (int lev, if (cost) { const Box& tbx = pti.tilebox(); wt = (amrex::second() - wt) / tbx.d_numPts(); - (*cost)[pti].plus(wt, tbx); + FArrayBox* costfab = cost->fabPtr(pti); + AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tbx, work_box, + { + costfab->plus(wt, work_box); + }); } } } diff --git a/Source/PhysicalParticleContainer.H b/Source/PhysicalParticleContainer.H index d2648421d..bd6015437 100644 --- a/Source/PhysicalParticleContainer.H +++ b/Source/PhysicalParticleContainer.H @@ -80,6 +80,10 @@ public: // Inject particles in Box 'part_box' virtual void AddParticles (int lev); void AddPlasma(int lev, amrex::RealBox part_realbox = amrex::RealBox()); + void AddPlasmaCPU (int lev, amrex::RealBox part_realbox); +#ifdef AMREX_USE_GPU + void AddPlasmaGPU (int lev, amrex::RealBox part_realbox); +#endif void MapParticletoBoostedFrame(amrex::Real& x, amrex::Real& y, amrex::Real& z, std::array<amrex::Real, 3>& u); @@ -110,10 +114,6 @@ protected: const amrex::RealBox& tile_real_box, const amrex::RealBox& particle_real_box); - void AddPlasmaCPU (int lev, amrex::RealBox part_realbox); -#ifdef AMREX_USE_GPU - void AddPlasmaGPU (int lev, amrex::RealBox part_realbox); -#endif int GetRefineFac(const amrex::Real x, const amrex::Real y, const amrex::Real z); std::unique_ptr<amrex::IArrayBox> m_refined_injection_mask = nullptr; diff --git a/Source/PhysicalParticleContainer.cpp b/Source/PhysicalParticleContainer.cpp index f76c74579..c4fdd4464 100644 --- a/Source/PhysicalParticleContainer.cpp +++ b/Source/PhysicalParticleContainer.cpp @@ -437,7 +437,11 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) if (cost) { wt = (amrex::second() - wt) / tile_box.d_numPts(); - (*cost)[mfi].plus(wt, tile_box); + FArrayBox* costfab = cost->fabPtr(mfi); + AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tile_box, work_box, + { + costfab->plus(wt, work_box); + }); } } } @@ -674,7 +678,11 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) if (cost) { wt = (amrex::second() - wt) / tile_box.d_numPts(); - (*cost)[mfi].plus(wt, tile_box); + FArrayBox* costfab = cost->fabPtr(mfi); + AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tile_box, work_box, + { + costfab->plus(wt, work_box); + }); } } } @@ -981,7 +989,11 @@ PhysicalParticleContainer::FieldGather (int lev, if (cost) { const Box& tbx = pti.tilebox(); wt = (amrex::second() - wt) / tbx.d_numPts(); - (*cost)[pti].plus(wt, tbx); + FArrayBox* costfab = cost->fabPtr(pti); + AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tbx, work_box, + { + costfab->plus(wt, work_box); + }); } } } @@ -1406,7 +1418,11 @@ PhysicalParticleContainer::Evolve (int lev, if (cost) { const Box& tbx = pti.tilebox(); wt = (amrex::second() - wt) / tbx.d_numPts(); - (*cost)[pti].plus(wt, tbx); + FArrayBox* costfab = cost->fabPtr(pti); + AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tbx, work_box, + { + costfab->plus(wt, work_box); + }); } } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 4d74d165f..5678ac79f 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -144,9 +144,13 @@ public: void EvolveB (int lev, amrex::Real dt); void EvolveF ( amrex::Real dt, DtType dt_type); void EvolveF (int lev, amrex::Real dt, DtType dt_type); + void EvolveB (int lev, PatchType patch_type, amrex::Real dt); + void EvolveE (int lev, PatchType patch_type, amrex::Real dt); + void EvolveF (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type); void DampPML (); void DampPML (int lev); + void DampPML (int lev, PatchType patch_type); void PushParticlesandDepose (int lev, amrex::Real cur_time); void PushParticlesandDepose ( amrex::Real cur_time); @@ -253,11 +257,6 @@ private: void FillBoundaryE (int lev, PatchType patch_type); void FillBoundaryF (int lev, PatchType patch_type); - void EvolveB (int lev, PatchType patch_type, amrex::Real dt); - void EvolveE (int lev, PatchType patch_type, amrex::Real dt); - void EvolveF (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type); - void DampPML (int lev, PatchType patch_type); - void OneStep_nosub (amrex::Real t); void OneStep_sub1 (amrex::Real t); diff --git a/Source/WarpXEvolve.cpp b/Source/WarpXEvolve.cpp index 38a630880..6d917e36d 100644 --- a/Source/WarpXEvolve.cpp +++ b/Source/WarpXEvolve.cpp @@ -525,8 +525,12 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real dt) if (cost) { Box cbx = mfi.tilebox(IntVect{AMREX_D_DECL(0,0,0)}); if (patch_type == PatchType::coarse) cbx.refine(rr); - wt = (amrex::second() - wt) / cbx.d_numPts(); - (*cost)[mfi].plus(wt, cbx); + wt = (amrex::second() - wt) / cbx.d_numPts();\ + FArrayBox* costfab = cost->fabPtr(mfi); + AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( cbx, work_box, + { + costfab->plus(wt, work_box); + }); } } @@ -668,7 +672,11 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real dt) Box cbx = mfi.tilebox(IntVect{AMREX_D_DECL(0,0,0)}); if (patch_type == PatchType::coarse) cbx.refine(rr); wt = (amrex::second() - wt) / cbx.d_numPts(); - (*cost)[mfi].plus(wt, cbx); + FArrayBox* costfab = cost->fabPtr(mfi); + AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( cbx, work_box, + { + costfab->plus(wt, work_box); + }); } } diff --git a/Source/WarpXMove.cpp b/Source/WarpXMove.cpp index 899f021f8..908c70573 100644 --- a/Source/WarpXMove.cpp +++ b/Source/WarpXMove.cpp @@ -222,15 +222,15 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir) #endif for (MFIter mfi(tmpmf); mfi.isValid(); ++mfi ) { - FArrayBox* dstfab = &(mf[mfi]); - FArrayBox* srcfab = &(tmpmf[mfi]); + FArrayBox* dstfab = mf.fabPtr(mfi); + FArrayBox* srcfab = tmpmf.fabPtr(mfi); Box outbox = mfi.fabbox(); outbox &= adjBox; AMREX_LAUNCH_HOST_DEVICE_LAMBDA(outbox, toutbox, { srcfab->setVal(0.0, toutbox, 0, nc); }); - Box dstBox = dstfab->box(); + Box dstBox = mf[mfi].box(); if (num_shift > 0) { dstBox.growHi(dir, -num_shift); } else { diff --git a/Source/WarpXParticleContainer.cpp b/Source/WarpXParticleContainer.cpp index 7c15f9d7d..54199eeb7 100644 --- a/Source/WarpXParticleContainer.cpp +++ b/Source/WarpXParticleContainer.cpp @@ -277,21 +277,21 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, BL_PROFILE_VAR_START(blp_accumulate); FArrayBox const* local_jx_const_ptr = local_jx[thread_num].get(); - FArrayBox* global_jx_ptr = &jx[pti]; + FArrayBox* global_jx_ptr = jx.fabPtr(pti); AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tbx, thread_bx, { global_jx_ptr->atomicAdd(*local_jx_const_ptr, thread_bx, thread_bx, 0, 0, 1); }); FArrayBox const* local_jy_const_ptr = local_jy[thread_num].get(); - FArrayBox* global_jy_ptr = &jy[pti]; + FArrayBox* global_jy_ptr = jy.fabPtr(pti); AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tby, thread_bx, { global_jy_ptr->atomicAdd(*local_jy_const_ptr, thread_bx, thread_bx, 0, 0, 1); }); FArrayBox const* local_jz_const_ptr = local_jz[thread_num].get(); - FArrayBox* global_jz_ptr = &jz[pti]; + FArrayBox* global_jz_ptr = jz.fabPtr(pti); AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tbz, thread_bx, { global_jz_ptr->atomicAdd(*local_jz_const_ptr, thread_bx, thread_bx, 0, 0, 1); @@ -364,28 +364,24 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, &lvect,&WarpX::current_deposition_algo); BL_PROFILE_VAR_STOP(blp_pxr_cd); - FArrayBox& cjxfab = (*cjx)[pti]; - FArrayBox& cjyfab = (*cjy)[pti]; - FArrayBox& cjzfab = (*cjz)[pti]; - BL_PROFILE_VAR_START(blp_accumulate); FArrayBox const* local_jx_const_ptr = local_jx[thread_num].get(); - FArrayBox* global_jx_ptr = &cjxfab; + FArrayBox* global_jx_ptr = cjx->fabPtr(pti); AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tbx, thread_bx, { global_jx_ptr->atomicAdd(*local_jx_const_ptr, thread_bx, thread_bx, 0, 0, 1); }); FArrayBox const* local_jy_const_ptr = local_jy[thread_num].get(); - FArrayBox* global_jy_ptr = &cjyfab; + FArrayBox* global_jy_ptr = cjx->fabPtr(pti); AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tby, thread_bx, { global_jy_ptr->atomicAdd(*local_jy_const_ptr, thread_bx, thread_bx, 0, 0, 1); }); FArrayBox const* local_jz_const_ptr = local_jz[thread_num].get(); - FArrayBox* global_jz_ptr = &cjzfab; + FArrayBox* global_jz_ptr = cjx->fabPtr(pti); AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tbz, thread_bx, { global_jz_ptr->atomicAdd(*local_jz_const_ptr, thread_bx, thread_bx, 0, 0, 1); @@ -420,7 +416,6 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, // Deposit charge for particles that are not in the current buffers if (np_current > 0) { - FArrayBox& rhofab = (*rhomf)[pti]; const std::array<Real, 3>& xyzmin = xyzmin_tile; tile_box.grow(ngRho); local_rho[thread_num]->resize(tile_box); @@ -457,7 +452,7 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, const int ncomp = 1; FArrayBox const* local_fab = local_rho[thread_num].get(); - FArrayBox* global_fab = &rhofab; + FArrayBox* global_fab = rhomf->fabPtr(pti); BL_PROFILE_VAR_START(blp_accumulate); AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tile_box, tbx, { @@ -508,11 +503,10 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, &WarpX::nox,&WarpX::noy,&WarpX::noz, &lvect, &WarpX::charge_deposition_algo); BL_PROFILE_VAR_STOP(blp_pxr_chd); - FArrayBox& crhofab = (*crhomf)[pti]; const int ncomp = 1; FArrayBox const* local_fab = local_rho[thread_num].get(); - FArrayBox* global_fab = &crhofab; + FArrayBox* global_fab = crhomf->fabPtr(pti); BL_PROFILE_VAR_START(blp_accumulate); AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tile_box, tbx, { @@ -878,7 +872,11 @@ WarpXParticleContainer::PushX (int lev, Real dt) if (cost) { const Box& tbx = pti.tilebox(); wt = (amrex::second() - wt) / tbx.d_numPts(); - (*cost)[pti].plus(wt, tbx); + FArrayBox* costfab = cost->fabPtr(pti); + AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tbx, work_box, + { + costfab->plus(wt, work_box); + }); } } } |