From f350ce071c652bb4b53eda0d774b5c75cd7e0250 Mon Sep 17 00:00:00 2001 From: Yin-YinjianZhao Date: Mon, 7 Oct 2019 09:51:35 -0700 Subject: This is a test of git commit. --- Source/Particles/PhysicalParticleContainer.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 02dee1967..d1f25fe5d 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -157,6 +157,14 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, std::normal_distribution distz(z_m, z_rms); if (ParallelDescriptor::IOProcessor()) { + // Allocate temporary vectors on the CPU + Gpu::HostVector particle_x; + Gpu::HostVector particle_y; + Gpu::HostVector particle_z; + Gpu::HostVector particle_ux; + Gpu::HostVector particle_uy; + Gpu::HostVector particle_uz; + Gpu::HostVector particle_w; // If do_symmetrize, create 4x fewer particles, and // Replicate each particle 4 times (x,y) (-x,y) (x,-y) (-x,-y) if (do_symmetrize){ -- cgit v1.2.3 From dea53978d67c232f27da076b3ddcb0fca01bdfe6 Mon Sep 17 00:00:00 2001 From: Yin-YinjianZhao Date: Mon, 7 Oct 2019 10:09:43 -0700 Subject: Fix issue: 'AddParticles' is slow on GPU. --- Source/Particles/PhysicalParticleContainer.H | 11 ++++ Source/Particles/PhysicalParticleContainer.cpp | 72 +++++++++++++++++++++----- 2 files changed, 69 insertions(+), 14 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index b558323a3..b4575257e 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -138,6 +138,17 @@ public: void CheckAndAddParticle(amrex::Real x, amrex::Real y, amrex::Real z, std::array u, amrex::Real weight); + void CheckAndAddParticle(amrex::Real x, amrex::Real y, amrex::Real z, + std::array u, + amrex::Real weight, + amrex::Gpu::HostVector& particle_x, + amrex::Gpu::HostVector& particle_y, + amrex::Gpu::HostVector& particle_z, + amrex::Gpu::HostVector& particle_ux, + amrex::Gpu::HostVector& particle_uy, + amrex::Gpu::HostVector& particle_uz, + amrex::Gpu::HostVector& particle_w); + virtual void GetParticleSlice(const int direction, const amrex::Real z_old, const amrex::Real z_new, const amrex::Real t_boost, const amrex::Real t_lab, const amrex::Real dt, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d1f25fe5d..281345f55 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -156,15 +156,17 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, std::normal_distribution disty(y_m, y_rms); std::normal_distribution distz(z_m, z_rms); + // Allocate temporary vectors on the CPU + Gpu::HostVector particle_x; + Gpu::HostVector particle_y; + Gpu::HostVector particle_z; + Gpu::HostVector particle_ux; + Gpu::HostVector particle_uy; + Gpu::HostVector particle_uz; + Gpu::HostVector particle_w; + int np = 0; + if (ParallelDescriptor::IOProcessor()) { - // Allocate temporary vectors on the CPU - Gpu::HostVector particle_x; - Gpu::HostVector particle_y; - Gpu::HostVector particle_z; - Gpu::HostVector particle_ux; - Gpu::HostVector particle_uy; - Gpu::HostVector particle_uz; - Gpu::HostVector particle_w; // If do_symmetrize, create 4x fewer particles, and // Replicate each particle 4 times (x,y) (-x,y) (x,-y) (-x,-y) if (do_symmetrize){ @@ -189,17 +191,59 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, u.z *= PhysConst::c; if (do_symmetrize){ // Add four particles to the beam: - CheckAndAddParticle( x, y, z, { u.x, u.y, u.z}, weight/4. ); - CheckAndAddParticle( x,-y, z, { u.x,-u.y, u.z}, weight/4. ); - CheckAndAddParticle(-x, y, z, {-u.x, u.y, u.z}, weight/4. ); - CheckAndAddParticle(-x,-y, z, {-u.x,-u.y, u.z}, weight/4. ); + CheckAndAddParticle(x, y, z, { u.x, u.y, u.z}, weight/4., + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); + CheckAndAddParticle(x, -y, z, { u.x, -u.y, u.z}, weight/4., + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); + CheckAndAddParticle(-x, y, z, { -u.x, u.y, u.z}, weight/4., + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); + CheckAndAddParticle(-x, -y, z, { -u.x, -u.y, u.z}, weight/4., + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); + } else { - CheckAndAddParticle(x, y, z, {u.x,u.y,u.z}, weight); + CheckAndAddParticle(x, y, z, { u.x, u.y, u.z}, weight, + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); } } } } - Redistribute(); + // Add the temporary CPU vectors to the particle structure + np = particle_z.size(); + AddNParticles(0,np, + particle_x.dataPtr(), particle_y.dataPtr(), particle_z.dataPtr(), + particle_ux.dataPtr(), particle_uy.dataPtr(), particle_uz.dataPtr(), + 1, particle_w.dataPtr(),1); +} + +void +PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, + std::array u, + Real weight, + Gpu::HostVector& particle_x, + Gpu::HostVector& particle_y, + Gpu::HostVector& particle_z, + Gpu::HostVector& particle_ux, + Gpu::HostVector& particle_uy, + Gpu::HostVector& particle_uz, + Gpu::HostVector& particle_w) +{ + particle_x.push_back(x); + particle_y.push_back(y); + particle_z.push_back(z); + particle_ux.push_back(u[0]); + particle_uy.push_back(u[1]); + particle_uz.push_back(u[2]); + particle_w.push_back(weight); } void -- cgit v1.2.3 From 17b067bcaf982ad12f4ac604308391a806e2c1f7 Mon Sep 17 00:00:00 2001 From: Yin-YinjianZhao Date: Mon, 7 Oct 2019 10:15:53 -0700 Subject: Fix issue: 'AddParticles' is slow on GPU. --- Source/Particles/PhysicalParticleContainer.H | 2 +- Source/Particles/PhysicalParticleContainer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index b4575257e..eec516888 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -147,7 +147,7 @@ public: amrex::Gpu::HostVector& particle_ux, amrex::Gpu::HostVector& particle_uy, amrex::Gpu::HostVector& particle_uz, - amrex::Gpu::HostVector& particle_w); + amrex::Gpu::HostVector& particle_w); virtual void GetParticleSlice(const int direction, const amrex::Real z_old, const amrex::Real z_new, const amrex::Real t_boost, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 281345f55..b06e45b63 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -235,7 +235,7 @@ PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, Gpu::HostVector& particle_ux, Gpu::HostVector& particle_uy, Gpu::HostVector& particle_uz, - Gpu::HostVector& particle_w) + Gpu::HostVector& particle_w) { particle_x.push_back(x); particle_y.push_back(y); -- cgit v1.2.3 From 10d22fb4126732b4b088dce568252532da7be39f Mon Sep 17 00:00:00 2001 From: Yin-YinjianZhao Date: Mon, 7 Oct 2019 10:22:11 -0700 Subject: Fix issue: 'AddParticles' is slow on GPU + coding format change. --- Source/Particles/PhysicalParticleContainer.cpp | 47 +++++++++++++------------- 1 file changed, 23 insertions(+), 24 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b06e45b63..7c8c81eed 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -191,28 +191,27 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, u.z *= PhysConst::c; if (do_symmetrize){ // Add four particles to the beam: - CheckAndAddParticle(x, y, z, { u.x, u.y, u.z}, weight/4., - particle_x, particle_y, particle_z, - particle_ux, particle_uy, particle_uz, - particle_w); - CheckAndAddParticle(x, -y, z, { u.x, -u.y, u.z}, weight/4., - particle_x, particle_y, particle_z, - particle_ux, particle_uy, particle_uz, - particle_w); - CheckAndAddParticle(-x, y, z, { -u.x, u.y, u.z}, weight/4., - particle_x, particle_y, particle_z, - particle_ux, particle_uy, particle_uz, - particle_w); - CheckAndAddParticle(-x, -y, z, { -u.x, -u.y, u.z}, weight/4., - particle_x, particle_y, particle_z, - particle_ux, particle_uy, particle_uz, - particle_w); - + CheckAndAddParticle(x, y, z, { u.x, u.y, u.z}, weight/4., + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); + CheckAndAddParticle(x, -y, z, { u.x, -u.y, u.z}, weight/4., + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); + CheckAndAddParticle(-x, y, z, { -u.x, u.y, u.z}, weight/4., + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); + CheckAndAddParticle(-x, -y, z, { -u.x, -u.y, u.z}, weight/4., + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); } else { - CheckAndAddParticle(x, y, z, { u.x, u.y, u.z}, weight, - particle_x, particle_y, particle_z, - particle_ux, particle_uy, particle_uz, - particle_w); + CheckAndAddParticle(x, y, z, { u.x, u.y, u.z}, weight, + particle_x, particle_y, particle_z, + particle_ux, particle_uy, particle_uz, + particle_w); } } } @@ -220,9 +219,9 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, // Add the temporary CPU vectors to the particle structure np = particle_z.size(); AddNParticles(0,np, - particle_x.dataPtr(), particle_y.dataPtr(), particle_z.dataPtr(), - particle_ux.dataPtr(), particle_uy.dataPtr(), particle_uz.dataPtr(), - 1, particle_w.dataPtr(),1); + particle_x.dataPtr(), particle_y.dataPtr(), particle_z.dataPtr(), + particle_ux.dataPtr(), particle_uy.dataPtr(), particle_uz.dataPtr(), + 1, particle_w.dataPtr(),1); } void -- cgit v1.2.3 From 07795799555d9843017269415195f2226755e4a7 Mon Sep 17 00:00:00 2001 From: Yin-YinjianZhao Date: Mon, 7 Oct 2019 11:37:46 -0700 Subject: Add check 'if (WarpX::gamma_boost > 1.)' and removed unsused functions. --- Source/Particles/PhysicalParticleContainer.H | 3 -- Source/Particles/PhysicalParticleContainer.cpp | 30 ++------------------ Source/Particles/WarpXParticleContainer.H | 8 ------ Source/Particles/WarpXParticleContainer.cpp | 39 -------------------------- 4 files changed, 3 insertions(+), 77 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index eec516888..b4081e959 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -135,9 +135,6 @@ public: amrex::Real x_rms, amrex::Real y_rms, amrex::Real z_rms, amrex::Real q_tot, long npart, int do_symmetrize); - void CheckAndAddParticle(amrex::Real x, amrex::Real y, amrex::Real z, - std::array u, amrex::Real weight); - void CheckAndAddParticle(amrex::Real x, amrex::Real y, amrex::Real z, std::array u, amrex::Real weight, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 7c8c81eed..75e438454 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -236,6 +236,9 @@ PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, Gpu::HostVector& particle_uz, Gpu::HostVector& particle_w) { + if (WarpX::gamma_boost > 1.) { + MapParticletoBoostedFrame(x, y, z, u); + } particle_x.push_back(x); particle_y.push_back(y); particle_z.push_back(z); @@ -245,33 +248,6 @@ PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, particle_w.push_back(weight); } -void -PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, - std::array u, - Real weight) -{ - std::array attribs; - attribs.fill(0.0); - - // update attribs with input arguments - if (WarpX::gamma_boost > 1.) { - MapParticletoBoostedFrame(x, y, z, u); - } - attribs[PIdx::ux] = u[0]; - attribs[PIdx::uy] = u[1]; - attribs[PIdx::uz] = u[2]; - attribs[PIdx::w ] = weight; - - if ( (NumRuntimeRealComps()>0) || (NumRuntimeIntComps()>0) ) - { - // need to create old values - auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); - } - - // add particle - AddOneParticle(0, 0, 0, x, y, z, attribs); -} - void PhysicalParticleContainer::AddParticles (int lev) { diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 7b0d2d1d0..ceb88d024 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -236,14 +236,6 @@ public: const amrex::ParticleReal* vx, const amrex::ParticleReal* vy, const amrex::ParticleReal* vz, int nattr, const amrex::ParticleReal* attr, int uniqueparticles, int id=-1); - void AddOneParticle (int lev, int grid, int tile, - amrex::ParticleReal x, amrex::ParticleReal y, amrex::ParticleReal z, - std::array& attribs); - - void AddOneParticle (ParticleTileType& particle_tile, - amrex::ParticleReal x, amrex::ParticleReal y, amrex::ParticleReal z, - std::array& attribs); - virtual void ReadHeader (std::istream& is); virtual void WriteHeader (std::ostream& os) const; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 65a82f233..7fb57500d 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -134,45 +134,6 @@ WarpXParticleContainer::AllocData () resizeData(); } -void -WarpXParticleContainer::AddOneParticle (int lev, int grid, int tile, - ParticleReal x, ParticleReal y, ParticleReal z, - std::array& attribs) -{ - auto& particle_tile = DefineAndReturnParticleTile(lev, grid, tile); - AddOneParticle(particle_tile, x, y, z, attribs); -} - -void -WarpXParticleContainer::AddOneParticle (ParticleTileType& particle_tile, - ParticleReal x, ParticleReal y, ParticleReal z, - std::array& attribs) -{ - ParticleType p; - p.id() = ParticleType::NextID(); - p.cpu() = ParallelDescriptor::MyProc(); -#if (AMREX_SPACEDIM == 3) - p.pos(0) = x; - p.pos(1) = y; - p.pos(2) = z; -#elif (AMREX_SPACEDIM == 2) -#ifdef WARPX_DIM_RZ - attribs[PIdx::theta] = std::atan2(y, x); - x = std::sqrt(x*x + y*y); -#endif - p.pos(0) = x; - p.pos(1) = z; -#endif - - particle_tile.push_back(p); - particle_tile.push_back_real(attribs); - - for (int i = PIdx::nattribs; i < NumRealComps(); ++i) - { - particle_tile.push_back_real(i, 0.0); - } -} - void WarpXParticleContainer::AddNParticles (int lev, int n, const ParticleReal* x, const ParticleReal* y, const ParticleReal* z, -- cgit v1.2.3