From b4c4315c77ade3ebb4b11e137ddfe354f1c95d23 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 23 Sep 2019 16:24:57 -0700 Subject: Create dedicated method for partition in buffers --- Source/Particles/Sorting/Partition.cpp | 115 +++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 Source/Particles/Sorting/Partition.cpp (limited to 'Source/Particles/Sorting/Partition.cpp') diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp new file mode 100644 index 000000000..d000a2c04 --- /dev/null +++ b/Source/Particles/Sorting/Partition.cpp @@ -0,0 +1,115 @@ +#include +#include +#include + +using namespace amrex; + +/* \brief +*/ +void +PhysicalParticleContainer::PartitionParticlesInBuffers( + long& nfine_current, long& nfine_gather, long const np, + WarpXParIter& pti, int const lev, + iMultiFab const* current_masks, + iMultiFab const* gather_masks, + RealVector& uxp, RealVector& uyp, RealVector& uzp, RealVector& wp) +{ + BL_PROFILE("PPC::Evolve::partition"); + + std::vector inexflag; + inexflag.resize(np); + + auto& aos = pti.GetArrayOfStructs(); + + // We need to partition the large buffer first + iMultiFab const* bmasks = + (WarpX::n_field_gather_buffer >= WarpX::n_current_deposition_buffer) ? + gather_masks : current_masks; + int i = 0; + const auto& msk = (*bmasks)[pti]; + for (const auto& p : aos) { + const IntVect& iv = Index(p, lev); + inexflag[i++] = msk(iv); + } + + Vector pid; + pid.resize(np); + std::iota(pid.begin(), pid.end(), 0L); + auto sep = std::stable_partition(pid.begin(), pid.end(), + [&inexflag](long id) { return inexflag[id]; }); + + if (WarpX::n_current_deposition_buffer == WarpX::n_field_gather_buffer) { + nfine_current = nfine_gather = std::distance(pid.begin(), sep); + } else if (sep != pid.end()) { + int n_buf; + if (bmasks == gather_masks) { + nfine_gather = std::distance(pid.begin(), sep); + bmasks = current_masks; + n_buf = WarpX::n_current_deposition_buffer; + } else { + nfine_current = std::distance(pid.begin(), sep); + bmasks = gather_masks; + n_buf = WarpX::n_field_gather_buffer; + } + if (n_buf > 0) + { + const auto& msk2 = (*bmasks)[pti]; + for (auto it = sep; it != pid.end(); ++it) { + const long id = *it; + const IntVect& iv = Index(aos[id], lev); + inexflag[id] = msk2(iv); + } + + auto sep2 = std::stable_partition(sep, pid.end(), + [&inexflag](long id) {return inexflag[id];}); + if (bmasks == gather_masks) { + nfine_gather = std::distance(pid.begin(), sep2); + } else { + nfine_current = std::distance(pid.begin(), sep2); + } + } + } + + // only deposit / gather to coarsest grid + if (m_deposit_on_main_grid && lev > 0) { + nfine_current = 0; + } + if (m_gather_from_main_grid && lev > 0) { + nfine_gather = 0; + } + + if (nfine_current != np || nfine_gather != np) + { + RealVector tmp; + ParticleVector particle_tmp; + + particle_tmp.resize(np); + for (long ip = 0; ip < np; ++ip) { + particle_tmp[ip] = aos[pid[ip]]; + } + std::swap(aos(), particle_tmp); + + tmp.resize(np); + + for (long ip = 0; ip < np; ++ip) { + tmp[ip] = wp[pid[ip]]; + } + std::swap(wp, tmp); + + for (long ip = 0; ip < np; ++ip) { + tmp[ip] = uxp[pid[ip]]; + } + std::swap(uxp, tmp); + + for (long ip = 0; ip < np; ++ip) { + tmp[ip] = uyp[pid[ip]]; + } + std::swap(uyp, tmp); + + for (long ip = 0; ip < np; ++ip) { + tmp[ip] = uzp[pid[ip]]; + } + std::swap(uzp, tmp); + } + +} -- cgit v1.2.3 From 37e62ed5e384256699d442fc34c9e5e3c3b9a50a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 24 Sep 2019 13:46:17 -0700 Subject: Add comments --- Source/Particles/PhysicalParticleContainer.H | 4 ++- Source/Particles/PhysicalParticleContainer.cpp | 20 +++++++++++--- Source/Particles/Sorting/Partition.cpp | 37 +++++++++++++++++++++----- 3 files changed, 51 insertions(+), 10 deletions(-) (limited to 'Source/Particles/Sorting/Partition.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index b809acc45..c0fdce983 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -111,7 +111,9 @@ public: RealVector& uxp, RealVector& uyp, RealVector& uzp, - RealVector& wp ); + RealVector& wp, + RealVector& tmp, + ParticleVector& particle_tmp ); void copy_attribs(WarpXParIter& pti,const amrex::Real* xp, const amrex::Real* yp, const amrex::Real* zp); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e1046ac7a..dae723b3f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -989,6 +989,8 @@ PhysicalParticleContainer::Evolve (int lev, FArrayBox filtered_Ex, filtered_Ey, filtered_Ez; FArrayBox filtered_Bx, filtered_By, filtered_Bz; + RealVector tmp; + ParticleVector particle_tmp; for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { @@ -1020,6 +1022,7 @@ PhysicalParticleContainer::Evolve (int lev, FArrayBox const* bzfab = &(Bz[pti]); Elixir exeli, eyeli, ezeli, bxeli, byeli, bzeli; + if (WarpX::use_fdtd_nci_corr) { #if (AMREX_SPACEDIM == 2) @@ -1079,11 +1082,22 @@ PhysicalParticleContainer::Evolve (int lev, Byp.assign(np,WarpX::B_external[1]); Bzp.assign(np,WarpX::B_external[2]); - long nfine_current = np; //! number of particles depositing to fine grid - long nfine_gather = np; //! number of particles gathering from fine grid + // Determine which particles deposit/gather in the buffer, and + // which particles deposit/gather in the fine patch + long nfine_current = np; + long nfine_gather = np; if (has_buffer && !do_not_push) { + // - Modify `nfine_current` and `nfine_gather` (in place) + // so that they correspond to the number of particles + // that deposit/gather in the fine patch respectively. + // - Reorder the particle arrays, + // so that the `nfine_current`/`nfine_gather` first particles + // deposit/gather in the fine patch + // and (thus) the `np-nfine_current`/`np-nfine_gather` last particles + // deposit/gather in the buffer PartitionParticlesInBuffers( nfine_current, nfine_gather, np, - pti, lev, current_masks, gather_masks, uxp, uyp, uzp, wp); + pti, lev, current_masks, gather_masks, uxp, uyp, uzp, wp, + tmp, particle_tmp); } const long np_current = (cjx) ? nfine_current : np; diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index d000a2c04..4c8e35f38 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -4,15 +4,43 @@ using namespace amrex; -/* \brief -*/ +/* \brief Determine which particles deposit/gather in the buffer, and + * and reorder the particle arrays accordingly + * + * More specifically: + * - Modify `nfine_current` and `nfine_gather` (in place) + * so that they correspond to the number of particles + * that deposit/gather in the fine patch respectively. + * - Reorder the particle arrays, + * so that the `nfine_current`/`nfine_gather` first particles + * deposit/gather in the fine patch + * and (thus) the `np-nfine_current`/`np-nfine_gather` last particles + * deposit/gather in the buffer + * + * \param nfine_current number of particles that deposit to the fine patch + * (modified by this function) + * \param nfine_gather number of particles that gather into the fine patch + * (modified by this function) + * \param np total number of particles in this tile + * \param pti object that holds the particle information for this tile + * \param lev current refinement level + * \param current_masks indicates, for each cell, whether that cell is + * in the deposition buffers or in the interior of the fine patch + * \param gather_masks indicates, for each cell, whether that cell is + * in the gather buffers or in the interior of the fine patch + * \param uxp, uyp, uzp, wp references to the particle momenta and weight + * (modified by this function) + * \param tmp temporary vector, used in reference swapping + * \param particle_tmp temporary vector, used in reference swapping + */ void PhysicalParticleContainer::PartitionParticlesInBuffers( long& nfine_current, long& nfine_gather, long const np, WarpXParIter& pti, int const lev, iMultiFab const* current_masks, iMultiFab const* gather_masks, - RealVector& uxp, RealVector& uyp, RealVector& uzp, RealVector& wp) + RealVector& uxp, RealVector& uyp, RealVector& uzp, RealVector& wp, + RealVector& tmp, ParticleVector& particle_tmp) { BL_PROFILE("PPC::Evolve::partition"); @@ -80,9 +108,6 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( if (nfine_current != np || nfine_gather != np) { - RealVector tmp; - ParticleVector particle_tmp; - particle_tmp.resize(np); for (long ip = 0; ip < np; ++ip) { particle_tmp[ip] = aos[pid[ip]]; -- cgit v1.2.3 From f241d5e67d3b975b39aa58851bda814662130592 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 24 Sep 2019 15:17:12 -0700 Subject: Move temporary variables into function --- Source/Particles/PhysicalParticleContainer.H | 4 +--- Source/Particles/PhysicalParticleContainer.cpp | 5 +---- Source/Particles/Sorting/Partition.cpp | 9 +++++---- 3 files changed, 7 insertions(+), 11 deletions(-) (limited to 'Source/Particles/Sorting/Partition.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index c0fdce983..b809acc45 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -111,9 +111,7 @@ public: RealVector& uxp, RealVector& uyp, RealVector& uzp, - RealVector& wp, - RealVector& tmp, - ParticleVector& particle_tmp ); + RealVector& wp ); void copy_attribs(WarpXParIter& pti,const amrex::Real* xp, const amrex::Real* yp, const amrex::Real* zp); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index dae723b3f..7e1dbe74a 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -989,8 +989,6 @@ PhysicalParticleContainer::Evolve (int lev, FArrayBox filtered_Ex, filtered_Ey, filtered_Ez; FArrayBox filtered_Bx, filtered_By, filtered_Bz; - RealVector tmp; - ParticleVector particle_tmp; for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { @@ -1096,8 +1094,7 @@ PhysicalParticleContainer::Evolve (int lev, // and (thus) the `np-nfine_current`/`np-nfine_gather` last particles // deposit/gather in the buffer PartitionParticlesInBuffers( nfine_current, nfine_gather, np, - pti, lev, current_masks, gather_masks, uxp, uyp, uzp, wp, - tmp, particle_tmp); + pti, lev, current_masks, gather_masks, uxp, uyp, uzp, wp ); } const long np_current = (cjx) ? nfine_current : np; diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 4c8e35f38..683dbbd04 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -30,8 +30,6 @@ using namespace amrex; * in the gather buffers or in the interior of the fine patch * \param uxp, uyp, uzp, wp references to the particle momenta and weight * (modified by this function) - * \param tmp temporary vector, used in reference swapping - * \param particle_tmp temporary vector, used in reference swapping */ void PhysicalParticleContainer::PartitionParticlesInBuffers( @@ -39,8 +37,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( WarpXParIter& pti, int const lev, iMultiFab const* current_masks, iMultiFab const* gather_masks, - RealVector& uxp, RealVector& uyp, RealVector& uzp, RealVector& wp, - RealVector& tmp, ParticleVector& particle_tmp) + RealVector& uxp, RealVector& uyp, RealVector& uzp, RealVector& wp) { BL_PROFILE("PPC::Evolve::partition"); @@ -108,12 +105,16 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( if (nfine_current != np || nfine_gather != np) { + + ParticleVector particle_tmp; particle_tmp.resize(np); + for (long ip = 0; ip < np; ++ip) { particle_tmp[ip] = aos[pid[ip]]; } std::swap(aos(), particle_tmp); + RealVector tmp; tmp.resize(np); for (long ip = 0; ip < np; ++ip) { -- cgit v1.2.3