diff options
Diffstat (limited to 'Source/Particles')
-rw-r--r-- | Source/Particles/MultiParticleContainer.H | 1 | ||||
-rw-r--r-- | Source/Particles/PhysicalParticleContainer.cpp | 51 | ||||
-rw-r--r-- | Source/Particles/WarpXParticleContainer.H | 15 | ||||
-rw-r--r-- | Source/Particles/WarpXParticleContainer.cpp | 130 |
4 files changed, 24 insertions, 173 deletions
diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index aaed96423..1aff7edfb 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -123,7 +123,6 @@ public: /// /// This deposits the particle charge onto a node-centered MultiFab and returns a unique ptr /// to it. The charge density is accumulated over all the particles in the MultiParticleContainer - /// This version uses PICSAR routines to perform the deposition. /// std::unique_ptr<amrex::MultiFab> GetChargeDensity(int lev, bool local = false); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 73acd60fa..015482e3f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -932,7 +932,7 @@ PhysicalParticleContainer::Evolve (int lev, { BL_PROFILE("PPC::Evolve()"); BL_PROFILE_VAR_NS("PPC::Evolve::Copy", blp_copy); - BL_PROFILE_VAR_NS("PICSAR::FieldGather", blp_pxr_fg); + BL_PROFILE_VAR_NS("PPC::FieldGather", blp_fg); BL_PROFILE_VAR_NS("PPC::ParticlePush", blp_ppc_pp); BL_PROFILE_VAR_NS("PPC::Evolve::partition", blp_partition); @@ -1198,7 +1198,7 @@ PhysicalParticleContainer::Evolve (int lev, // // Field Gather of Aux Data (i.e., the full solution) // - BL_PROFILE_VAR_START(blp_pxr_fg); + BL_PROFILE_VAR_START(blp_fg); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, exfab, eyfab, ezfab, bxfab, byfab, bzfab, Ex.nGrow(), e_is_nodal, 0, np_gather, thread_num, lev, lev); @@ -1278,7 +1278,7 @@ PhysicalParticleContainer::Evolve (int lev, thread_num, lev, lev-1); } - BL_PROFILE_VAR_STOP(blp_pxr_fg); + BL_PROFILE_VAR_STOP(blp_fg); // // Particle Push @@ -1291,36 +1291,25 @@ PhysicalParticleContainer::Evolve (int lev, // // Current Deposition // - if (WarpX::use_picsar_deposition) { - // Deposit inside domains - DepositCurrentFortran(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, - 0, np_current, thread_num, - lev, lev, dt); - if (has_buffer){ - // Deposit in buffers - DepositCurrentFortran(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt); - } + + int* AMREX_RESTRICT ion_lev; + if (do_field_ionization){ + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); } else { - int* AMREX_RESTRICT ion_lev; - if (do_field_ionization){ - ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); - } else { - ion_lev = nullptr; - } - - // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, - 0, np_current, thread_num, - lev, lev, dt); - if (has_buffer){ - // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt); - } + ion_lev = nullptr; } + + // Deposit inside domains + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt); + if (has_buffer){ + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt); + } + // // copy particle data back diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index c4bb5e410..7393f7301 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -205,21 +205,6 @@ public: int depos_lev, amrex::Real dt); - virtual void DepositCurrentFortran(WarpXParIter& pti, - RealVector& wp, - RealVector& uxp, - RealVector& uyp, - RealVector& uzp, - amrex::MultiFab* jx, - amrex::MultiFab* jy, - amrex::MultiFab* jz, - const long offset, - const long np_to_depose, - int thread_num, - int lev, - int depos_lev, - amrex::Real dt); - // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and // NOT before. Virtual function, overriden by derived classes. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index b34e71178..80d1caa0f 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -263,132 +263,6 @@ WarpXParticleContainer::AddNParticles (int lev, Redistribute(); } -/* \brief Current Deposition for thread thread_num using PICSAR - * \param pti : Particle iterator - * \param wp : Array of particle weights - * \param uxp uyp uzp : Array of particle - * \param jx jy jz : Full array of current density - * \param offset : Index of first particle for which current is deposited - * \param np_to_depose: Number of particles for which current is deposited. - Particles [offset,offset+np_tp_depose] deposit their - current - * \param thread_num : Thread number (if tiling) - * \param lev : Level of box that contains particles - * \param depos_lev : Level on which particles deposit (if buffers are used) - * \param dt : Time step for particle level - */ -void -WarpXParticleContainer::DepositCurrentFortran(WarpXParIter& pti, - RealVector& wp, RealVector& uxp, - RealVector& uyp, RealVector& uzp, - MultiFab* jx, MultiFab* jy, MultiFab* jz, - const long offset, const long np_to_depose, - int thread_num, int lev, int depos_lev, - Real dt) -{ - AMREX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || - (depos_lev==(lev )), - "Deposition buffers only work for lev-1"); - // If no particles, do not do anything - if (np_to_depose == 0) return; - - const long ngJ = jx->nGrow(); - const std::array<Real,3>& dx = WarpX::CellSize(std::max(depos_lev,0)); - const long lvect = 8; - int j_is_nodal = jx->is_nodal() and jy->is_nodal() and jz->is_nodal(); - - BL_PROFILE_VAR_NS("PICSAR::CurrentDeposition", blp_pxr_cd); - BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); - - // Get tile box where current is deposited. - // The tile box is different when depositing in the buffers (depos_lev<lev) - // or when depositing inside the level (depos_lev=lev) - Box tilebox; - if (lev == depos_lev) { - tilebox = pti.tilebox(); - } else { - const IntVect& ref_ratio = WarpX::RefRatio(depos_lev); - tilebox = amrex::coarsen(pti.tilebox(),ref_ratio); - } - - // Staggered tile boxes (different in each direction) - Box tbx = convert(tilebox, WarpX::jx_nodal_flag); - Box tby = convert(tilebox, WarpX::jy_nodal_flag); - Box tbz = convert(tilebox, WarpX::jz_nodal_flag); - - // Lower corner of tile box physical domain - const std::array<Real,3>& xyzmin_tile = WarpX::LowerCorner(tilebox, depos_lev); - const std::array<Real, 3>& xyzmin = xyzmin_tile; - -#ifdef AMREX_USE_GPU - // No tiling on GPU: jx_ptr points to the full - // jx array (same for jy_ptr and jz_ptr). - Real* jx_ptr = (*jx)[pti].dataPtr(); - Real* jy_ptr = (*jy)[pti].dataPtr(); - Real* jz_ptr = (*jz)[pti].dataPtr(); - - auto jxntot = (*jx)[pti].length(); - auto jyntot = (*jy)[pti].length(); - auto jzntot = (*jz)[pti].length(); -#else - // Tiling is on: jx_ptr points to local_jx[thread_num] - // (same for jy_ptr and jz_ptr) - tbx.grow(ngJ); - tby.grow(ngJ); - tbz.grow(ngJ); - - local_jx[thread_num].resize(tbx); - local_jy[thread_num].resize(tby); - local_jz[thread_num].resize(tbz); - - Real* jx_ptr = local_jx[thread_num].dataPtr(); - Real* jy_ptr = local_jy[thread_num].dataPtr(); - Real* jz_ptr = local_jz[thread_num].dataPtr(); - - // local_jx[thread_num] is set to zero - local_jx[thread_num].setVal(0.0); - local_jy[thread_num].setVal(0.0); - local_jz[thread_num].setVal(0.0); - - auto jxntot = local_jx[thread_num].length(); - auto jyntot = local_jy[thread_num].length(); - auto jzntot = local_jz[thread_num].length(); -#endif - // GPU, no tiling: deposit directly in jx - // CPU, tiling: deposit into local_jx - // (same for jx and jz) - BL_PROFILE_VAR_START(blp_pxr_cd); - warpx_current_deposition( - jx_ptr, &ngJ, jxntot.getVect(), - jy_ptr, &ngJ, jyntot.getVect(), - jz_ptr, &ngJ, jzntot.getVect(), - &np_to_depose, - m_xp[thread_num].dataPtr() + offset, - m_yp[thread_num].dataPtr() + offset, - m_zp[thread_num].dataPtr() + offset, - uxp.dataPtr() + offset, - uyp.dataPtr() + offset, - uzp.dataPtr() + offset, - m_giv[thread_num].dataPtr() + offset, - wp.dataPtr() + offset, &this->charge, - &xyzmin[0], &xyzmin[1], &xyzmin[2], - &dt, &dx[0], &dx[1], &dx[2], - &WarpX::nox,&WarpX::noy,&WarpX::noz, &j_is_nodal, - &lvect,&WarpX::current_deposition_algo); - - BL_PROFILE_VAR_STOP(blp_pxr_cd); - -#ifndef AMREX_USE_GPU - BL_PROFILE_VAR_START(blp_accumulate); - // CPU, tiling: atomicAdd local_jx into jx - // (same for jx and jz) - (*jx)[pti].atomicAdd(local_jx[thread_num], tbx, tbx, 0, 0, 1); - (*jy)[pti].atomicAdd(local_jy[thread_num], tby, tby, 0, 0, 1); - (*jz)[pti].atomicAdd(local_jz[thread_num], tbz, tbz, 0, 0, 1); - BL_PROFILE_VAR_STOP(blp_accumulate); -#endif -} - /* \brief Current Deposition for thread thread_num * \param pti : Particle iterator * \param wp : Array of particle weights @@ -429,6 +303,8 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, const Real stagger_shift = j_is_nodal ? 0.0 : 0.5; BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); + BL_PROFILE_VAR_NS("PPC::CurrentDeposition", blp_deposit); + // Get tile box where current is deposited. // The tile box is different when depositing in the buffers (depos_lev<lev) @@ -491,6 +367,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, // Better for memory? worth trying? const Dim3 lo = lbound(tilebox); + BL_PROFILE_VAR_START(blp_deposit); if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>( @@ -529,6 +406,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, stagger_shift, q); } } + BL_PROFILE_VAR_STOP(blp_deposit); #ifndef AMREX_USE_GPU BL_PROFILE_VAR_START(blp_accumulate); |