From 10eb71343fbe101c2db5ea02605769946339c9e0 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 30 Sep 2019 09:51:36 -0700 Subject: first commit: move particle creation routine in a new file --- Source/Particles/MultiParticleContainer.cpp | 138 +--------------------------- 1 file changed, 1 insertion(+), 137 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 715c97b99..e3f0155e6 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -522,142 +522,6 @@ MultiParticleContainer::getSpeciesID (std::string product_str) namespace { - // For particle i in mfi, if is_ionized[i]=1, copy particle - // particle i from container pc_source into pc_product - void createIonizedParticles ( - int lev, const MFIter& mfi, - std::unique_ptr< WarpXParticleContainer>& pc_source, - std::unique_ptr< WarpXParticleContainer>& pc_product, - amrex::Gpu::ManagedDeviceVector& is_ionized) - { - BL_PROFILE("createIonizedParticles"); - - const int * const AMREX_RESTRICT p_is_ionized = is_ionized.dataPtr(); - - const int grid_id = mfi.index(); - const int tile_id = mfi.LocalTileIndex(); - - // Get source particle data - auto& ptile_source = pc_source->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - const int np_source = ptile_source.GetArrayOfStructs().size(); - if (np_source == 0) return; - // --- source AoS particle data - WarpXParticleContainer::ParticleType* particles_source = ptile_source.GetArrayOfStructs()().data(); - // --- source SoA particle data - auto& soa_source = ptile_source.GetStructOfArrays(); - GpuArray attribs_source; - for (int ia = 0; ia < PIdx::nattribs; ++ia) { - attribs_source[ia] = soa_source.GetRealData(ia).data(); - } - // --- source runtime attribs - GpuArray runtime_uold_source; - // Prepare arrays for boosted frame diagnostics. - runtime_uold_source[0] = soa_source.GetRealData(PIdx::ux).data(); - runtime_uold_source[1] = soa_source.GetRealData(PIdx::uy).data(); - runtime_uold_source[2] = soa_source.GetRealData(PIdx::uz).data(); - - // Indices of product particle for each ionized source particle. - // i_product[i]-1 is the location in product tile of product particle - // from source particle i. - amrex::Gpu::ManagedDeviceVector i_product; - i_product.resize(np_source); - // 0GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - // old and new (i.e., including ionized particles) number of particles - // for product species - const int np_product_old = ptile_product.GetArrayOfStructs().size(); - const int np_product_new = np_product_old + np_ionized; - // Allocate extra space in product species for ionized particles. - ptile_product.resize(np_product_new); - // --- product AoS particle data - // First element is the first newly-created product particle - WarpXParticleContainer::ParticleType* particles_product = ptile_product.GetArrayOfStructs()().data() + np_product_old; - // --- product SoA particle data - auto& soa_product = ptile_product.GetStructOfArrays(); - GpuArray attribs_product; - for (int ia = 0; ia < PIdx::nattribs; ++ia) { - // First element is the first newly-created product particle - attribs_product[ia] = soa_product.GetRealData(ia).data() + np_product_old; - } - // --- product runtime attribs - GpuArray runtime_attribs_product; - bool do_boosted_product = WarpX::do_boosted_frame_diagnostic - && pc_product->DoBoostedFrameDiags(); - if (do_boosted_product) { - std::map comps_product = pc_product->getParticleComps(); - runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; - runtime_attribs_product[1] = soa_product.GetRealData(comps_product[ "yold"]).data() + np_product_old; - runtime_attribs_product[2] = soa_product.GetRealData(comps_product[ "zold"]).data() + np_product_old; - runtime_attribs_product[3] = soa_product.GetRealData(comps_product["uxold"]).data() + np_product_old; - runtime_attribs_product[4] = soa_product.GetRealData(comps_product["uyold"]).data() + np_product_old; - runtime_attribs_product[5] = soa_product.GetRealData(comps_product["uzold"]).data() + np_product_old; - } - - int pid_product; -#pragma omp critical (doFieldIonization_nextid) - { - // ID of first newly-created product particle - pid_product = pc_product->NextID(); - // Update NextID to include particles created in this function - pc_product->setNextID(pid_product+np_ionized); - } - const int cpuid = ParallelDescriptor::MyProc(); - - // Loop over all source particles. If is_ionized, copy particle data - // to corresponding product particle. - amrex::For( - np_source, [=] AMREX_GPU_DEVICE (int is) noexcept - { - if(p_is_ionized[is]){ - // offset of 1 due to inclusive scan - int ip = p_i_product[is]-1; - // is: index of ionized particle in source species - // ip: index of corresponding new particle in product species - WarpXParticleContainer::ParticleType& p_product = particles_product[ip]; - WarpXParticleContainer::ParticleType& p_source = particles_source[is]; - // Copy particle from source to product: AoS - p_product.id() = pid_product + ip; - p_product.cpu() = cpuid; - p_product.pos(0) = p_source.pos(0); - p_product.pos(1) = p_source.pos(1); -#if (AMREX_SPACEDIM == 3) - p_product.pos(2) = p_source.pos(2); -#endif - // Copy particle from source to product: SoA - for (int ia = 0; ia < PIdx::nattribs; ++ia) { - attribs_product[ia][ip] = attribs_source[ia][is]; - } - // Update xold etc. if boosted frame diagnostics required - // for product species. Fill runtime attribs with a copy of - // current properties (xold = x etc.). - if (do_boosted_product) { - runtime_attribs_product[0][ip] = p_source.pos(0); - runtime_attribs_product[1][ip] = p_source.pos(1); - runtime_attribs_product[2][ip] = p_source.pos(2); - runtime_attribs_product[3][ip] = runtime_uold_source[0][ip]; - runtime_attribs_product[4][ip] = runtime_uold_source[1][ip]; - runtime_attribs_product[5][ip] = runtime_uold_source[2][ip]; - } - } - } - ); - } } void @@ -721,7 +585,7 @@ MultiParticleContainer::doFieldIonization () amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); // Create particles in pc_product - createIonizedParticles(lev, mfi, pc_source, pc_product, is_ionized); + createParticles(lev, mfi, pc_source, pc_product, is_ionized); } } // lev } // pc_source -- cgit v1.2.3 From ce06037bdfa6a13e11b54dd905d90a3a58d81e69 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 30 Sep 2019 14:31:25 -0700 Subject: pass functor to createParticles --- Source/Particles/MultiParticleContainer.H | 4 +++- Source/Particles/MultiParticleContainer.cpp | 3 ++- Source/Particles/ParticleCreation/CreateParticles.cpp | 14 +++++++++++++- Source/Particles/ParticleCreation/Make.package | 1 + 4 files changed, 19 insertions(+), 3 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index a4107b93f..c90a30950 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -1,6 +1,7 @@ #ifndef WARPX_ParticleContainer_H_ #define WARPX_ParticleContainer_H_ +#include "IonizationEvent.H" #include #include #include @@ -195,7 +196,8 @@ public: int lev, const amrex::MFIter& mfi, std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, - amrex::Gpu::ManagedDeviceVector& is_flagged); + amrex::Gpu::ManagedDeviceVector& is_flagged, + copyAndTransformParticle copy_and_transform_functor); protected: diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index e3f0155e6..0969dba04 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -584,8 +584,9 @@ MultiParticleContainer::doFieldIonization () // 0 if not ionized, 1 if ionized. amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); + copyAndTransformParticle copy_and_transform_functor; // Create particles in pc_product - createParticles(lev, mfi, pc_source, pc_product, is_ionized); + createParticles(lev, mfi, pc_source, pc_product, is_ionized, copy_and_transform_functor); } } // lev } // pc_source diff --git a/Source/Particles/ParticleCreation/CreateParticles.cpp b/Source/Particles/ParticleCreation/CreateParticles.cpp index 2e90676e9..5f14b6103 100644 --- a/Source/Particles/ParticleCreation/CreateParticles.cpp +++ b/Source/Particles/ParticleCreation/CreateParticles.cpp @@ -1,3 +1,4 @@ +#include "IonizationEvent.H" #include "MultiParticleContainer.H" #include "WarpX.H" @@ -9,7 +10,9 @@ void MultiParticleContainer::createParticles ( int lev, const MFIter& mfi, std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, - amrex::Gpu::ManagedDeviceVector& is_flagged) + amrex::Gpu::ManagedDeviceVector& is_flagged, + copyAndTransformParticle copy_and_transform_functor + ) { BL_PROFILE("createIonizedParticles"); @@ -100,6 +103,8 @@ void MultiParticleContainer::createParticles ( } const int cpuid = ParallelDescriptor::MyProc(); + copy_and_transform_functor(112); + // Loop over all source particles. If is_flagged, copy particle data // to corresponding product particle. amrex::For( @@ -112,6 +117,13 @@ void MultiParticleContainer::createParticles ( // ip: index of corresponding new particle in product species WarpXParticleContainer::ParticleType& p_product = particles_product[ip]; WarpXParticleContainer::ParticleType& p_source = particles_source[is]; + + copy_and_transform_functor( + p_source, + + p_product, + ); + // Copy particle from source to product: AoS p_product.id() = pid_product + ip; p_product.cpu() = cpuid; diff --git a/Source/Particles/ParticleCreation/Make.package b/Source/Particles/ParticleCreation/Make.package index 066771611..9f98cc20f 100644 --- a/Source/Particles/ParticleCreation/Make.package +++ b/Source/Particles/ParticleCreation/Make.package @@ -1,3 +1,4 @@ +CEXE_headers += IonizationEvent.H CEXE_sources += CreateParticles.cpp INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Particles/ParticleCreation/ -- cgit v1.2.3 From 956aa9c5a3d28ba2433af2789d7d7d0480e8d638 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 30 Sep 2019 16:59:59 -0700 Subject: added class particleCreationProcess. Ionization test passes --- Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 4 +- .../Particles/ParticleCreation/CreateParticles.cpp | 19 ++--- .../Particles/ParticleCreation/IonizationEvent.H | 81 +++++++++++++++++----- 4 files changed, 77 insertions(+), 29 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index c90a30950..56e0a2970 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -197,7 +197,7 @@ public: std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, amrex::Gpu::ManagedDeviceVector& is_flagged, - copyAndTransformParticle copy_and_transform_functor); + particleCreationProcess creation_process); protected: diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 0969dba04..984b7cdc9 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -584,9 +584,9 @@ MultiParticleContainer::doFieldIonization () // 0 if not ionized, 1 if ionized. amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); - copyAndTransformParticle copy_and_transform_functor; + particleCreationProcess ionization_process; // Create particles in pc_product - createParticles(lev, mfi, pc_source, pc_product, is_ionized, copy_and_transform_functor); + createParticles(lev, mfi, pc_source, pc_product, is_ionized, ionization_process); } } // lev } // pc_source diff --git a/Source/Particles/ParticleCreation/CreateParticles.cpp b/Source/Particles/ParticleCreation/CreateParticles.cpp index f447164e8..16bd1c261 100644 --- a/Source/Particles/ParticleCreation/CreateParticles.cpp +++ b/Source/Particles/ParticleCreation/CreateParticles.cpp @@ -11,8 +11,7 @@ void MultiParticleContainer::createParticles ( std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, amrex::Gpu::ManagedDeviceVector& is_flagged, - copyAndTransformParticle copy_and_transform_functor - ) + particleCreationProcess creation_process) { BL_PROFILE("createIonizedParticles"); @@ -103,6 +102,13 @@ void MultiParticleContainer::createParticles ( } const int cpuid = ParallelDescriptor::MyProc(); + copyAndTransformParticle copy_and_transform_functor = creation_process.initialize_functor( + cpuid, do_boosted_product, + &runtime_uold_source, + &attribs_source, + &attribs_product, + &runtime_attribs_product); + // Loop over all source particles. If is_flagged, copy particle data // to corresponding product particle. amrex::For( @@ -117,14 +123,9 @@ void MultiParticleContainer::createParticles ( WarpXParticleContainer::ParticleType& p_source = particles_source[is]; copy_and_transform_functor( - is, ip, pid_product, cpuid, do_boosted_product, - runtime_uold_source, + is, ip, pid_product, p_source, - p_product, - attribs_source, - attribs_product, - runtime_attribs_product); - + p_product); } } ); diff --git a/Source/Particles/ParticleCreation/IonizationEvent.H b/Source/Particles/ParticleCreation/IonizationEvent.H index d4cc203ab..f04fe7c39 100644 --- a/Source/Particles/ParticleCreation/IonizationEvent.H +++ b/Source/Particles/ParticleCreation/IonizationEvent.H @@ -5,21 +5,52 @@ class copyAndTransformParticle { + public: - void operator () ( - int is, int ip, int pid_product, int cpuid, int do_boosted_product, - const amrex::GpuArray& runtime_uold_source, - WarpXParticleContainer::ParticleType& p_source, - WarpXParticleContainer::ParticleType& p_product, - const amrex::GpuArray& attribs_source, - const amrex::GpuArray& attribs_product, - const amrex::GpuArray& runtime_attribs_product) const + + int m_cpuid; + bool m_do_boosted_product; + const amrex::GpuArray* m_runtime_uold_source; + const amrex::GpuArray* m_attribs_source; + const amrex::GpuArray* m_attribs_product; + const amrex::GpuArray* m_runtime_attribs_product; + + copyAndTransformParticle( + int cpuid, int do_boosted_product, + const amrex::GpuArray* runtime_uold_source, + const amrex::GpuArray* attribs_source, + const amrex::GpuArray* attribs_product, + const amrex::GpuArray* runtime_attribs_product) + : + m_cpuid(cpuid), + m_do_boosted_product(do_boosted_product), + m_runtime_uold_source(runtime_uold_source), + m_attribs_source(attribs_source), + m_attribs_product(attribs_product), + m_runtime_attribs_product(runtime_attribs_product) + {} +/* + { + m_cpuid = cpuid; + m_do_boosted_product = do_boosted_product; + m_runtime_uold_source = runtime_uold_source; + m_attribs_source = attribs_source; + m_attribs_product = attribs_product; + m_runtime_attribs_product = runtime_attribs_product; + } +*/ + + void operator () (int is, int ip, int pid_product, + WarpXParticleContainer::ParticleType& p_source, + WarpXParticleContainer::ParticleType& p_product + ) const { amrex::Print()<<"is = "<* runtime_uold_source, + const amrex::GpuArray* attribs_source, + const amrex::GpuArray* attribs_product, + const amrex::GpuArray* runtime_attribs_product) const noexcept + { + return copyAndTransformParticle( + cpuid, do_boosted_product, runtime_uold_source, + attribs_source, attribs_product, runtime_attribs_product); + }; +}; + #endif // IONIZATIONEVENT_H_ -- cgit v1.2.3 From 59be563f929717911bdf683ac14509cb6d7ad8c3 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 1 Oct 2019 07:43:57 -0700 Subject: some renaming for clarity --- Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 2 +- .../Particles/ParticleCreation/CreateParticles.cpp | 4 ++-- Source/Particles/ParticleCreation/IonizationEvent.H | 20 +++++++------------- 4 files changed, 11 insertions(+), 17 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 56e0a2970..b6241d814 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -197,7 +197,7 @@ public: std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, amrex::Gpu::ManagedDeviceVector& is_flagged, - particleCreationProcess creation_process); + elementaryProcess elementary_process); protected: diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 984b7cdc9..bb0af28d0 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -584,7 +584,7 @@ MultiParticleContainer::doFieldIonization () // 0 if not ionized, 1 if ionized. amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); - particleCreationProcess ionization_process; + elementaryProcess ionization_process; // Create particles in pc_product createParticles(lev, mfi, pc_source, pc_product, is_ionized, ionization_process); } diff --git a/Source/Particles/ParticleCreation/CreateParticles.cpp b/Source/Particles/ParticleCreation/CreateParticles.cpp index 16bd1c261..ab58b5331 100644 --- a/Source/Particles/ParticleCreation/CreateParticles.cpp +++ b/Source/Particles/ParticleCreation/CreateParticles.cpp @@ -11,7 +11,7 @@ void MultiParticleContainer::createParticles ( std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, amrex::Gpu::ManagedDeviceVector& is_flagged, - particleCreationProcess creation_process) + elementaryProcess elementary_process) { BL_PROFILE("createIonizedParticles"); @@ -102,7 +102,7 @@ void MultiParticleContainer::createParticles ( } const int cpuid = ParallelDescriptor::MyProc(); - copyAndTransformParticle copy_and_transform_functor = creation_process.initialize_functor( + copyAndTransformParticle copy_and_transform_functor = elementary_process.initialize_functor( cpuid, do_boosted_product, &runtime_uold_source, &attribs_source, diff --git a/Source/Particles/ParticleCreation/IonizationEvent.H b/Source/Particles/ParticleCreation/IonizationEvent.H index f04fe7c39..dd65a9cb7 100644 --- a/Source/Particles/ParticleCreation/IonizationEvent.H +++ b/Source/Particles/ParticleCreation/IonizationEvent.H @@ -3,6 +3,11 @@ #include "WarpXParticleContainer.H" +namespace +{ + +} + class copyAndTransformParticle { @@ -27,18 +32,7 @@ public: m_runtime_uold_source(runtime_uold_source), m_attribs_source(attribs_source), m_attribs_product(attribs_product), - m_runtime_attribs_product(runtime_attribs_product) - {} -/* - { - m_cpuid = cpuid; - m_do_boosted_product = do_boosted_product; - m_runtime_uold_source = runtime_uold_source; - m_attribs_source = attribs_source; - m_attribs_product = attribs_product; - m_runtime_attribs_product = runtime_attribs_product; - } -*/ + m_runtime_attribs_product(runtime_attribs_product) {} void operator () (int is, int ip, int pid_product, WarpXParticleContainer::ParticleType& p_source, @@ -74,7 +68,7 @@ public: } }; -class particleCreationProcess +class elementaryProcess { public: copyAndTransformParticle initialize_functor( -- cgit v1.2.3 From 5be8c34a9569d4a4ca9c6a86a9b65c8494a3cc6e Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 1 Oct 2019 15:16:23 -0700 Subject: one instance of class elementaryProcess per process --- Source/Particles/MultiParticleContainer.H | 8 +- Source/Particles/MultiParticleContainer.cpp | 6 +- .../Particles/ParticleCreation/IonizationEvent.H | 109 --------------------- Source/Particles/ParticleCreation/Make.package | 4 +- 4 files changed, 12 insertions(+), 115 deletions(-) delete mode 100644 Source/Particles/ParticleCreation/IonizationEvent.H (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index b6241d814..ea18f148a 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -1,7 +1,8 @@ #ifndef WARPX_ParticleContainer_H_ #define WARPX_ParticleContainer_H_ -#include "IonizationEvent.H" +#include "ElementaryProcess.H" + #include #include #include @@ -192,13 +193,16 @@ public: PhysicalParticleContainer& GetPCtmp () { return *pc_tmp; } + elementaryProcess ionization_process; + +/* void createParticles ( int lev, const amrex::MFIter& mfi, std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, amrex::Gpu::ManagedDeviceVector& is_flagged, elementaryProcess elementary_process); - +*/ protected: // Particle container types diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index bb0af28d0..58f6c5de9 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -584,9 +584,11 @@ MultiParticleContainer::doFieldIonization () // 0 if not ionized, 1 if ionized. amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); - elementaryProcess ionization_process; + // elementaryProcess ionization_process; // Create particles in pc_product - createParticles(lev, mfi, pc_source, pc_product, is_ionized, ionization_process); + bool do_boosted_product = WarpX::do_boosted_frame_diagnostic + && pc_product->DoBoostedFrameDiags(); + ionization_process.createParticles(lev, mfi, pc_source, pc_product, is_ionized, do_boosted_product); } } // lev } // pc_source diff --git a/Source/Particles/ParticleCreation/IonizationEvent.H b/Source/Particles/ParticleCreation/IonizationEvent.H deleted file mode 100644 index 12abedcf2..000000000 --- a/Source/Particles/ParticleCreation/IonizationEvent.H +++ /dev/null @@ -1,109 +0,0 @@ -#ifndef IONIZATIONEVENT_H_ -#define IONIZATIONEVENT_H_ - -#include "WarpXParticleContainer.H" - -class copyAndTransformParticle -{ - -public: - - int m_cpuid; - bool m_do_boosted_product; - const amrex::GpuArray m_runtime_uold_source; - const amrex::GpuArray m_attribs_source; - const amrex::GpuArray m_attribs_product; - const amrex::GpuArray m_runtime_attribs_product; - - //AMREX_GPU_HOST_DEVICE - //copyAndTransformParticle() {} - - AMREX_GPU_HOST_DEVICE - copyAndTransformParticle( - int cpuid, int do_boosted_product, - const amrex::GpuArray runtime_uold_source, - const amrex::GpuArray attribs_source, - const amrex::GpuArray attribs_product, - const amrex::GpuArray runtime_attribs_product) - : - m_cpuid(cpuid), - m_do_boosted_product(do_boosted_product), - m_runtime_uold_source(runtime_uold_source), - m_attribs_source(attribs_source), - m_attribs_product(attribs_product), - m_runtime_attribs_product(runtime_attribs_product) {} - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator () (int is, int ip, int pid_product, - WarpXParticleContainer::ParticleType& p_source, - WarpXParticleContainer::ParticleType& p_product) const noexcept - { - this->copyParticle(is, ip, pid_product, p_source, p_product); - this->transformSourceParticle(is, p_source); - this->transformProductParticle(is, p_product); - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void copyParticle(int is, int ip, int pid_product, - WarpXParticleContainer::ParticleType& p_source, - WarpXParticleContainer::ParticleType& p_product) const - { - // amrex::Print()<<"tata is = "< runtime_uold_source, - const amrex::GpuArray attribs_source, - const amrex::GpuArray attribs_product, - const amrex::GpuArray runtime_attribs_product) const noexcept - { - return copyAndTransformParticle( - cpuid, do_boosted_product, runtime_uold_source, - attribs_source, attribs_product, runtime_attribs_product); - }; -}; - -#endif // IONIZATIONEVENT_H_ diff --git a/Source/Particles/ParticleCreation/Make.package b/Source/Particles/ParticleCreation/Make.package index 9f98cc20f..d523a308c 100644 --- a/Source/Particles/ParticleCreation/Make.package +++ b/Source/Particles/ParticleCreation/Make.package @@ -1,5 +1,5 @@ -CEXE_headers += IonizationEvent.H -CEXE_sources += CreateParticles.cpp +CEXE_headers += ElementaryProcess.H +# CEXE_sources += CreateParticles.cpp INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Particles/ParticleCreation/ VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/ParticleCreation/ -- cgit v1.2.3 From 85aa9abe9860c018e5db5051a6e805f10ea3b7e4 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 10 Oct 2019 08:36:37 -0700 Subject: solution with derived classes without template --- Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 2 + .../Particles/ParticleCreation/ElementaryProcess.H | 43 ++-------------- .../ParticleCreation/ElementaryProcess.cpp | 4 +- .../Particles/ParticleCreation/TransformParticle.H | 60 +++++++++++++++++++--- 5 files changed, 64 insertions(+), 47 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 36efed5bc..bf35ec603 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -193,7 +193,7 @@ public: PhysicalParticleContainer& GetPCtmp () { return *pc_tmp; } - elementaryProcess ionization_process; + IonizationProcess ionization_process; protected: diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 58f6c5de9..0043a5e8a 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -47,6 +47,7 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core) nspecies_boosted_frame_diags += 1; } } + ionization_process = IonizationProcess(); } void @@ -586,6 +587,7 @@ MultiParticleContainer::doFieldIonization () pc_source->buildIonizationMask(mfi, lev, is_ionized); // elementaryProcess ionization_process; // Create particles in pc_product + // ionization_process(IonizationProcess()); bool do_boosted_product = WarpX::do_boosted_frame_diagnostic && pc_product->DoBoostedFrameDiags(); ionization_process.createParticles(lev, mfi, pc_source, pc_product, is_ionized, do_boosted_product); diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index 966789d44..0bd4acdd2 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -5,48 +5,10 @@ #include "CopyParticle.H" #include "TransformParticle.H" -/* -enum struct elementaryProcessType { Ionization }; -enum struct PType {source, product}; - - namespace transform - { - template < elementaryProcessType ElemT, PType PartT > - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void transformParticle( - int i, - WarpXParticleContainer::ParticleType& particle) - { - amrex::Print()<<"primary template\n"; - } - - template <> - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void transformParticle( - int i, - WarpXParticleContainer::ParticleType& particle) - { - amrex::Print()<<"ionization source template\n"; - } - - template <> - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void transformParticle( - int i, - WarpXParticleContainer::ParticleType& particle) - { - amrex::Print()<<"ionization product template\n"; - } - } -*/ - class elementaryProcess { - public: - elementaryProcess() {}; - copyParticle initialize_copy( int cpuid, int do_boosted_product, const amrex::GpuArray runtime_uold_source, @@ -77,4 +39,9 @@ public: copyParticle copy_functor); }; +class IonizationProcess: public elementaryProcess +{ + +}; + #endif // ELEMENTARYPROCESS_H_ diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.cpp b/Source/Particles/ParticleCreation/ElementaryProcess.cpp index 98299d5c8..406911481 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.cpp +++ b/Source/Particles/ParticleCreation/ElementaryProcess.cpp @@ -130,8 +130,8 @@ elementaryProcess::copyAndTransformParticles( WarpXParticleContainer::ParticleType& p_product = particles_product[ip]; WarpXParticleContainer::ParticleType& p_source = particles_source[is]; copy_functor(is, ip, pid_product, p_source, p_product); - transformParticle(is, p_source); - transformParticle(ip, p_product); + transformSourceParticle(is, p_source); + //transformProductParticle(ip, p_product); } } ); diff --git a/Source/Particles/ParticleCreation/TransformParticle.H b/Source/Particles/ParticleCreation/TransformParticle.H index 23ab5b9c5..5d2283b3c 100644 --- a/Source/Particles/ParticleCreation/TransformParticle.H +++ b/Source/Particles/ParticleCreation/TransformParticle.H @@ -3,11 +3,20 @@ #include "WarpXParticleContainer.H" -enum struct PType {source, product}; +enum struct elementaryProcessType { Ionization }; -template < PType PT > AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformParticle( +void transformSourceParticle( + int i, + WarpXParticleContainer::ParticleType& particle) +{ + amrex::Print()<<"primary template\n"; +} + +/* +template < elementaryProcessType ProcessT > +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void transformProductParticle( int i, WarpXParticleContainer::ParticleType& particle) { @@ -16,7 +25,7 @@ void transformParticle( template <> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformParticle < PType::source > ( +void transformSourceParticle < elementaryProcessType:Ionization > ( int i, WarpXParticleContainer::ParticleType& particle) { @@ -25,11 +34,50 @@ void transformParticle < PType::source > ( template <> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformParticle < PType::product > ( +void transformProductParticle < elementaryProcessType:Ionization> ( int i, WarpXParticleContainer::ParticleType& particle) { - amrex::Print()<<"specialized product\n"; + amrex::Print()<<"specialized source\n"; +} +*/ + + + + + + + + + +/* +enum struct PType {source, product}; + +template < elementaryProcess ProcessT, PType PT > +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void transformParticle( + int i, + WarpXParticleContainer::ParticleType& particle) +{ + amrex::Print()<<"primary template\n"; } +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void transformParticle < ProcessT, PType::source > ( + int i, + WarpXParticleContainer::ParticleType& particle) +{ + amrex::Print()<<"specialized source\n"; +} + +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void transformParticle < ProcessT, PType::product > ( + int i, + WarpXParticleContainer::ParticleType& particle) +{ + amrex::Print()<<"specialized product\n"; +} +*/ #endif // TRANSFORMPARTICLE_H_ -- cgit v1.2.3 From decbe1cd6241c2d2b89af816bbd993851a31752a Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 10 Oct 2019 09:50:00 -0700 Subject: add comments --- Source/Particles/MultiParticleContainer.cpp | 4 +- .../Particles/ParticleCreation/ElementaryProcess.H | 54 ++++++++++- .../Particles/ParticleCreation/TransformParticle.H | 103 ++++----------------- 3 files changed, 68 insertions(+), 93 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 0043a5e8a..c9a365292 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -585,11 +585,11 @@ MultiParticleContainer::doFieldIonization () // 0 if not ionized, 1 if ionized. amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); - // elementaryProcess ionization_process; // Create particles in pc_product - // ionization_process(IonizationProcess()); bool do_boosted_product = WarpX::do_boosted_frame_diagnostic && pc_product->DoBoostedFrameDiags(); + // Copy source to product particles, and increase ionization + // level of source particle ionization_process.createParticles(lev, mfi, pc_source, pc_product, is_ionized, do_boosted_product); } } // lev diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index 9d12714f6..435e31db5 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -5,11 +5,37 @@ #include "CopyParticle.H" #include "TransformParticle.H" +/** + * \brief Base class for particle creation processes + * + * Particles in a source particle container are copied to a product particle + * container if they are flagged. Both source and product particles can be + * modified. + * + * initialize_copy initializes a general copy functor + * createParticles formats the data to prepare for the copy, then + * calls copyAndTransformParticles + * copyAndTransformParticles loops over particles, performs the copy and + * transform source and product particles if needed. + * + * The class is templated on the process type. This gives flexibility + * on source and product particle transformation. + */ template class elementaryProcess { public: - elementaryProcess() {}; + + /** + * \brief initialize and return functor for particle copy + * + * \param cpuid id of MPI rank + * \param do_boosted_product whether to copy old attribs + * \param runtime_uold_source momentum of source particles + * \param attribs_source attribs of source particles + * \param attribs_product attribs of product particles + * \param runtime_attribs_product runtime attribs for product, to store old attribs + */ copyParticle initialize_copy( int cpuid, int do_boosted_product, const amrex::GpuArray runtime_uold_source, @@ -22,8 +48,19 @@ public: attribs_source, attribs_product, runtime_attribs_product); }; - // For particle i in mfi, if is_flagged[i]=1, copy particle - // particle i from container pc_source into pc_product + /** + * \brief create particles in product particle containers + * + * For particle i in mfi, if is_flagged[i]=1, copy particle + * particle i from container pc_source into pc_product + * + * \param lev MR level + * \param mfi MFIter where source particles are located + * \param pc_source source particle container + * \param pc_product product particle container + * \param is_flagged particles for which is_flagged is 1 are copied + * \param do_boosted_product whether to copy old attribs + */ void createParticles ( int lev, const amrex::MFIter& mfi, std::unique_ptr< WarpXParticleContainer>& pc_source, @@ -55,6 +92,7 @@ public: runtime_uold_source[1] = soa_source.GetRealData(PIdx::uy).data(); runtime_uold_source[2] = soa_source.GetRealData(PIdx::uz).data(); + // --- source runtime integer attribs amrex::GpuArray runtime_iattribs_source; std::map icomps_source = pc_source->getParticleiComps(); runtime_iattribs_source[0] = soa_source.GetIntData(icomps_source["ionization_level"]).data(); @@ -109,6 +147,7 @@ public: runtime_attribs_product[5] = soa_product.GetRealData(comps_product["uzold"]).data() + np_product_old; } + // --- product runtime integer attribs amrex::GpuArray runtime_iattribs_product; std::map icomps_product = pc_product->getParticleiComps(); runtime_iattribs_product[0] = soa_product.GetIntData(icomps_product["ionization_level"]).data() + np_product_old; @@ -123,6 +162,7 @@ public: } const int cpuid = amrex::ParallelDescriptor::MyProc(); + // Create instance of copy functor copyParticle copy_functor = initialize_copy( cpuid, do_boosted_product, runtime_uold_source, @@ -130,12 +170,17 @@ public: attribs_product, runtime_attribs_product); + // Loop over source particles and create product particles copyAndTransformParticles(is_flagged, i_product, np_source, pid_product, particles_product, particles_source, copy_functor, runtime_iattribs_source, runtime_iattribs_product); } - + /** + * \brief Loop over source particles and create product particles + * + * \param is_flagged particles for which is_flagged is 1 are copied + */ void copyAndTransformParticles( amrex::Gpu::ManagedDeviceVector& is_flagged, amrex::Gpu::ManagedDeviceVector& i_product, @@ -169,6 +214,7 @@ public: } }; +// Derived class for ionization process class IonizationProcess: public elementaryProcess {}; diff --git a/Source/Particles/ParticleCreation/TransformParticle.H b/Source/Particles/ParticleCreation/TransformParticle.H index 5d173599c..0601a5f63 100644 --- a/Source/Particles/ParticleCreation/TransformParticle.H +++ b/Source/Particles/ParticleCreation/TransformParticle.H @@ -5,26 +5,33 @@ enum struct elementaryProcessType { Ionization }; +/** + * \brief small modifications on source particle + * \param i index of particle + * \param particle particle struct + * \param runtime_iattribs integer attribs + */ template < elementaryProcessType ProcessT > AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void transformSourceParticle( int i, WarpXParticleContainer::ParticleType& particle, - amrex::GpuArray runtime_iattribs_source) -{ - amrex::Print()<<"primary template\n"; -} - + amrex::GpuArray runtime_iattribs) {} + +/** + * \brief small modifications on product particle + * \param i index of particle + * \param particle particle struct + * \param runtime_iattribs integer attribs + */ template < elementaryProcessType ProcessT > AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void transformProductParticle( int i, WarpXParticleContainer::ParticleType& particle, - amrex::GpuArray runtime_iattribs_product) -{ - amrex::Print()<<"primary template\n"; -} + amrex::GpuArray runtime_iattribs) {} +// For ionization, increase ionization level of source particle template <> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void transformSourceParticle < elementaryProcessType::Ionization > ( @@ -34,84 +41,6 @@ void transformSourceParticle < elementaryProcessType::Ionization > ( { // increment particle's ionization level runtime_iattribs_source[0][i] += 1; - amrex::Print()<<"specialized source\n"; -} - -template <> -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformProductParticle < elementaryProcessType::Ionization > ( - int i, - WarpXParticleContainer::ParticleType& particle, - amrex::GpuArray runtime_iattribs_product) -{ - amrex::Print()<<"specialized source\n"; -} - -/* -template < elementaryProcessType ProcessT > -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformProductParticle( - int i, - WarpXParticleContainer::ParticleType& particle) -{ - amrex::Print()<<"primary template\n"; -} - -template <> -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformSourceParticle < elementaryProcessType:Ionization > ( - int i, - WarpXParticleContainer::ParticleType& particle) -{ - amrex::Print()<<"specialized source\n"; } -template <> -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformProductParticle < elementaryProcessType:Ionization> ( - int i, - WarpXParticleContainer::ParticleType& particle) -{ - amrex::Print()<<"specialized source\n"; -} -*/ - - - - - - - - - -/* -enum struct PType {source, product}; - -template < elementaryProcess ProcessT, PType PT > -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformParticle( - int i, - WarpXParticleContainer::ParticleType& particle) -{ - amrex::Print()<<"primary template\n"; -} - -template -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformParticle < ProcessT, PType::source > ( - int i, - WarpXParticleContainer::ParticleType& particle) -{ - amrex::Print()<<"specialized source\n"; -} - -template -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformParticle < ProcessT, PType::product > ( - int i, - WarpXParticleContainer::ParticleType& particle) -{ - amrex::Print()<<"specialized product\n"; -} -*/ #endif // TRANSFORMPARTICLE_H_ -- cgit v1.2.3 From 8e989a7d1e1b7602ad12ffd30bc4cd2ef8306cd7 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 10 Oct 2019 12:43:13 -0700 Subject: add multiple product species. Does not compile --- Source/Particles/MultiParticleContainer.cpp | 12 +- Source/Particles/ParticleCreation/CopyParticle.H | 15 +- .../Particles/ParticleCreation/ElementaryProcess.H | 152 ++++++++++++--------- .../Particles/ParticleCreation/TransformParticle.H | 6 +- 4 files changed, 110 insertions(+), 75 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index c9a365292..36f16bd33 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -586,11 +586,19 @@ MultiParticleContainer::doFieldIonization () amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); // Create particles in pc_product - bool do_boosted_product = WarpX::do_boosted_frame_diagnostic + bool do_boost = WarpX::do_boosted_frame_diagnostic && pc_product->DoBoostedFrameDiags(); + const amrex::Vector v_do_boosted_product{do_boost}; + //amrex::Vector> v_pc_product{pc_product}; + // amrex::Vector v_pc_product = GetVecOfPtrs(pc_product); + const amrex::Vector v_pc_product {pc_product.get()}; + // v_pc_Prpod; + //v_pc_product.resize(1); + //v_pc_product[0] = pc_product; // Copy source to product particles, and increase ionization // level of source particle - ionization_process.createParticles(lev, mfi, pc_source, pc_product, is_ionized, do_boosted_product); + ionization_process.createParticles(lev, mfi, pc_source, v_pc_product, + is_ionized, v_do_boosted_product); } } // lev } // pc_source diff --git a/Source/Particles/ParticleCreation/CopyParticle.H b/Source/Particles/ParticleCreation/CopyParticle.H index 78c1350fb..422d96921 100644 --- a/Source/Particles/ParticleCreation/CopyParticle.H +++ b/Source/Particles/ParticleCreation/CopyParticle.H @@ -10,14 +10,14 @@ public: int m_cpuid; bool m_do_boosted_product; - const amrex::GpuArray m_runtime_uold_source; - const amrex::GpuArray m_attribs_source; - const amrex::GpuArray m_attribs_product; - const amrex::GpuArray m_runtime_attribs_product; + amrex::GpuArray m_runtime_uold_source; + amrex::GpuArray m_attribs_source; + amrex::GpuArray m_attribs_product; + amrex::GpuArray m_runtime_attribs_product; AMREX_GPU_HOST_DEVICE copyParticle( - int cpuid, int do_boosted_product, + const int cpuid, const int do_boosted_product, const amrex::GpuArray runtime_uold_source, const amrex::GpuArray attribs_source, const amrex::GpuArray attribs_product, @@ -31,9 +31,12 @@ public: m_runtime_attribs_product(runtime_attribs_product){} AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + //void operator () (int is, int ip, int pid_product, + // WarpXParticleContainer::ParticleType& p_source, + // WarpXParticleContainer::ParticleType& p_product) const noexcept void operator () (int is, int ip, int pid_product, WarpXParticleContainer::ParticleType& p_source, - WarpXParticleContainer::ParticleType& p_product) const noexcept + WarpXParticleContainer::ParticleType& p_product) { // Copy particle from source to product: AoS p_product.id() = pid_product + ip; diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index 6af7ff933..37f8a0a6d 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -37,7 +37,7 @@ public: * \param runtime_attribs_product runtime attribs for product, to store old attribs */ copyParticle initialize_copy( - int cpuid, int do_boosted_product, + const int cpuid, const int do_boosted_product, const amrex::GpuArray runtime_uold_source, const amrex::GpuArray attribs_source, const amrex::GpuArray attribs_product, @@ -64,9 +64,9 @@ public: void createParticles ( int lev, const amrex::MFIter& mfi, std::unique_ptr< WarpXParticleContainer>& pc_source, - std::unique_ptr< WarpXParticleContainer>& pc_product, + amrex::Vector v_pc_product, amrex::Gpu::ManagedDeviceVector& is_flagged, - bool do_boosted_product) + const amrex::Vector v_do_boosted_product) { BL_PROFILE("createIonizedParticles"); @@ -97,6 +97,12 @@ public: std::map icomps_source = pc_source->getParticleiComps(); runtime_iattribs_source[0] = soa_source.GetIntData(icomps_source["ionization_level"]).data(); + int nproducts = v_pc_product.size(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + v_do_boosted_product.size() == nproducts, + "v_pc_product and v_do_boosted_product must have the same size."); + + // Indices of product particle for each ionized source particle. // i_product[i]-1 is the location in product tile of product particle // from source particle i. @@ -117,63 +123,73 @@ public: int np_ionized = i_product[np_source-1]; if (np_ionized == 0) return; - // Get product particle data - auto& ptile_product = pc_product->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - // old and new (i.e., including ionized particles) number of particles - // for product species - const int np_product_old = ptile_product.GetArrayOfStructs().size(); - const int np_product_new = np_product_old + np_ionized; - // Allocate extra space in product species for ionized particles. - ptile_product.resize(np_product_new); - // --- product AoS particle data - // First element is the first newly-created product particle - WarpXParticleContainer::ParticleType* particles_product = ptile_product.GetArrayOfStructs()().data() + np_product_old; - // --- product SoA particle data - auto& soa_product = ptile_product.GetStructOfArrays(); - amrex::GpuArray attribs_product; - for (int ia = 0; ia < PIdx::nattribs; ++ia) { + amrex::Vector v_copy_functor(nproducts); + amrex::Vector v_pid_product(nproducts); + amrex::Vector v_particles_product(nproducts); + amrex::Vector> v_runtime_iattribs_product(nproducts); + for (int iproduct=0; iproductGetParticles(lev)[std::make_pair(grid_id,tile_id)]; + // old and new (i.e., including ionized particles) number of particles + // for product species + const int np_product_old = ptile_product.GetArrayOfStructs().size(); + const int np_product_new = np_product_old + np_ionized; + // Allocate extra space in product species for ionized particles. + ptile_product.resize(np_product_new); + // --- product AoS particle data // First element is the first newly-created product particle - attribs_product[ia] = soa_product.GetRealData(ia).data() + np_product_old; - } - // --- product runtime attribs - amrex::GpuArray runtime_attribs_product; - if (do_boosted_product) { - std::map comps_product = pc_product->getParticleComps(); - runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; - runtime_attribs_product[1] = soa_product.GetRealData(comps_product[ "yold"]).data() + np_product_old; - runtime_attribs_product[2] = soa_product.GetRealData(comps_product[ "zold"]).data() + np_product_old; - runtime_attribs_product[3] = soa_product.GetRealData(comps_product["uxold"]).data() + np_product_old; - runtime_attribs_product[4] = soa_product.GetRealData(comps_product["uyold"]).data() + np_product_old; - runtime_attribs_product[5] = soa_product.GetRealData(comps_product["uzold"]).data() + np_product_old; - } - - // --- product runtime integer attribs - amrex::GpuArray runtime_iattribs_product; - std::map icomps_product = pc_product->getParticleiComps(); - runtime_iattribs_product[0] = soa_product.GetIntData(icomps_product["ionization_level"]).data() + np_product_old; - - int pid_product; + v_particles_product[iproduct] = ptile_product.GetArrayOfStructs()().data() + np_product_old; + // --- product SoA particle data + auto& soa_product = ptile_product.GetStructOfArrays(); + amrex::GpuArray attribs_product; + for (int ia = 0; ia < PIdx::nattribs; ++ia) { + // First element is the first newly-created product particle + attribs_product[ia] = soa_product.GetRealData(ia).data() + np_product_old; + } + // --- product runtime attribs + amrex::GpuArray runtime_attribs_product; + int do_boost = v_do_boosted_product[iproduct]; + if (do_boost) { + std::map comps_product = pc_product->getParticleComps(); + runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; + runtime_attribs_product[1] = soa_product.GetRealData(comps_product[ "yold"]).data() + np_product_old; + runtime_attribs_product[2] = soa_product.GetRealData(comps_product[ "zold"]).data() + np_product_old; + runtime_attribs_product[3] = soa_product.GetRealData(comps_product["uxold"]).data() + np_product_old; + runtime_attribs_product[4] = soa_product.GetRealData(comps_product["uyold"]).data() + np_product_old; + runtime_attribs_product[5] = soa_product.GetRealData(comps_product["uzold"]).data() + np_product_old; + } + + // --- product runtime integer attribs + amrex::GpuArray& runtime_iattribs_product = v_runtime_iattribs_product[iproduct]; + std::map icomps_product = pc_product->getParticleiComps(); + runtime_iattribs_product[0] = soa_product.GetIntData(icomps_product["ionization_level"]).data() + np_product_old; + + int pid_product; #pragma omp critical (doFieldIonization_nextid) - { - // ID of first newly-created product particle - pid_product = pc_product->NextID(); - // Update NextID to include particles created in this function - pc_product->setNextID(pid_product+np_ionized); + { + // ID of first newly-created product particle + pid_product = pc_product->NextID(); + // Update NextID to include particles created in this function + pc_product->setNextID(pid_product+np_ionized); + } + const int cpuid = amrex::ParallelDescriptor::MyProc(); + + // Create instance of copy functor + copyParticle copy_functor = initialize_copy( + cpuid, v_do_boosted_product[iproduct], + runtime_uold_source, + attribs_source, + attribs_product, + runtime_attribs_product); + v_copy_functor[iproduct] = copy_functor; + v_pid_product[iproduct] = pid_product; } - const int cpuid = amrex::ParallelDescriptor::MyProc(); - - // Create instance of copy functor - copyParticle copy_functor = initialize_copy( - cpuid, do_boosted_product, - runtime_uold_source, - attribs_source, - attribs_product, - runtime_attribs_product); // Loop over source particles and create product particles - copyAndTransformParticles(is_flagged, i_product, np_source, pid_product, - particles_product, particles_source, copy_functor, - runtime_iattribs_source, runtime_iattribs_product); + copyAndTransformParticles(is_flagged, i_product, np_source, v_pid_product, + v_particles_product, particles_source, v_copy_functor, + runtime_iattribs_source, v_runtime_iattribs_product); } /** @@ -184,12 +200,13 @@ public: void copyAndTransformParticles( amrex::Gpu::ManagedDeviceVector& is_flagged, amrex::Gpu::ManagedDeviceVector& i_product, - int np_source, int pid_product, - WarpXParticleContainer::ParticleType* particles_product, + int np_source, + const amrex::Vector v_pid_product, + const amrex::Vector v_particles_product, WarpXParticleContainer::ParticleType* particles_source, - copyParticle copy_functor, + const amrex::Vector v_copy_functor, amrex::GpuArray runtime_iattribs_source, - amrex::GpuArray runtime_iattribs_product) + amrex::Vector> v_runtime_iattribs_product) { int const * const AMREX_RESTRICT p_is_flagged = is_flagged.dataPtr(); int const * const AMREX_RESTRICT p_i_product = i_product.dataPtr(); @@ -199,15 +216,22 @@ public: np_source, [=] AMREX_GPU_DEVICE (int is) noexcept { if(p_is_flagged[is]){ + int nproducts = v_particles_product.size(); // offset of 1 due to inclusive scan int ip = p_i_product[is]-1; - // is: index of ionized particle in source species - // ip: index of corresponding new particle in product species - WarpXParticleContainer::ParticleType& p_product = particles_product[ip]; WarpXParticleContainer::ParticleType& p_source = particles_source[is]; - copy_functor(is, ip, pid_product, p_source, p_product); + for (int iproduct=0; iproduct(is, p_source, runtime_iattribs_source); - transformProductParticle(ip, p_product, runtime_iattribs_product); + transformProductParticle(ip, v_particles_product, v_runtime_iattribs_product); } } ); diff --git a/Source/Particles/ParticleCreation/TransformParticle.H b/Source/Particles/ParticleCreation/TransformParticle.H index e562ca07e..c414ca56f 100644 --- a/Source/Particles/ParticleCreation/TransformParticle.H +++ b/Source/Particles/ParticleCreation/TransformParticle.H @@ -16,7 +16,7 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void transformSourceParticle( int i, WarpXParticleContainer::ParticleType& particle, - amrex::GpuArray runtime_iattribs) {} + amrex::GpuArray runtime_iattribs){} /** * \brief small modifications on product particle @@ -28,8 +28,8 @@ template < elementaryProcessType ProcessT > AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void transformProductParticle( int i, - WarpXParticleContainer::ParticleType& particle, - amrex::GpuArray runtime_iattribs) {} + amrex::Vector v_particle, + amrex::Vector> v_runtime_iattribs){} // For ionization, increase ionization level of source particle template <> -- cgit v1.2.3 From 2bcf51e137efbd4950649370c474d239718d61b2 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 10 Oct 2019 14:28:30 -0700 Subject: compiles and gives correct result --- Source/Particles/MultiParticleContainer.cpp | 10 +++---- Source/Particles/ParticleCreation/CopyParticle.H | 13 ++++++--- .../Particles/ParticleCreation/ElementaryProcess.H | 31 ++++++++-------------- .../Particles/ParticleCreation/TransformParticle.H | 3 +-- 4 files changed, 25 insertions(+), 32 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 36f16bd33..5d898c11a 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -586,15 +586,11 @@ MultiParticleContainer::doFieldIonization () amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); // Create particles in pc_product - bool do_boost = WarpX::do_boosted_frame_diagnostic + int do_boost = WarpX::do_boosted_frame_diagnostic && pc_product->DoBoostedFrameDiags(); - const amrex::Vector v_do_boosted_product{do_boost}; - //amrex::Vector> v_pc_product{pc_product}; - // amrex::Vector v_pc_product = GetVecOfPtrs(pc_product); + amrex::Vector v_do_boosted_product; + v_do_boosted_product.push_back(do_boost); const amrex::Vector v_pc_product {pc_product.get()}; - // v_pc_Prpod; - //v_pc_product.resize(1); - //v_pc_product[0] = pc_product; // Copy source to product particles, and increase ionization // level of source particle ionization_process.createParticles(lev, mfi, pc_source, v_pc_product, diff --git a/Source/Particles/ParticleCreation/CopyParticle.H b/Source/Particles/ParticleCreation/CopyParticle.H index 422d96921..c42dc5bf6 100644 --- a/Source/Particles/ParticleCreation/CopyParticle.H +++ b/Source/Particles/ParticleCreation/CopyParticle.H @@ -30,10 +30,17 @@ public: m_attribs_product(attribs_product), m_runtime_attribs_product(runtime_attribs_product){} + AMREX_GPU_HOST_DEVICE + copyParticle(const copyParticle& in){ + m_cpuid = in.m_cpuid; + m_do_boosted_product = in.m_do_boosted_product; + m_runtime_uold_source = in.m_runtime_uold_source; + m_attribs_source = in.m_attribs_source; + m_attribs_product = in.m_attribs_product; + m_runtime_attribs_product = in.m_runtime_attribs_product; + } + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - //void operator () (int is, int ip, int pid_product, - // WarpXParticleContainer::ParticleType& p_source, - // WarpXParticleContainer::ParticleType& p_product) const noexcept void operator () (int is, int ip, int pid_product, WarpXParticleContainer::ParticleType& p_source, WarpXParticleContainer::ParticleType& p_product) diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index 37f8a0a6d..c7909b3a1 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -66,10 +66,9 @@ public: std::unique_ptr< WarpXParticleContainer>& pc_source, amrex::Vector v_pc_product, amrex::Gpu::ManagedDeviceVector& is_flagged, - const amrex::Vector v_do_boosted_product) + const amrex::Vector v_do_boosted_product) { BL_PROFILE("createIonizedParticles"); - const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); @@ -96,7 +95,6 @@ public: amrex::GpuArray runtime_iattribs_source; std::map icomps_source = pc_source->getParticleiComps(); runtime_iattribs_source[0] = soa_source.GetIntData(icomps_source["ionization_level"]).data(); - int nproducts = v_pc_product.size(); AMREX_ALWAYS_ASSERT_WITH_MESSAGE( v_do_boosted_product.size() == nproducts, @@ -123,10 +121,10 @@ public: int np_ionized = i_product[np_source-1]; if (np_ionized == 0) return; - amrex::Vector v_copy_functor(nproducts); + // amrex::Vector v_copy_functor(nproducts); + amrex::Vector v_copy_functor; amrex::Vector v_pid_product(nproducts); amrex::Vector v_particles_product(nproducts); - amrex::Vector> v_runtime_iattribs_product(nproducts); for (int iproduct=0; iproduct runtime_attribs_product; - int do_boost = v_do_boosted_product[iproduct]; + const int do_boost = v_do_boosted_product[iproduct]; if (do_boost) { std::map comps_product = pc_product->getParticleComps(); runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; @@ -161,10 +159,6 @@ public: } // --- product runtime integer attribs - amrex::GpuArray& runtime_iattribs_product = v_runtime_iattribs_product[iproduct]; - std::map icomps_product = pc_product->getParticleiComps(); - runtime_iattribs_product[0] = soa_product.GetIntData(icomps_product["ionization_level"]).data() + np_product_old; - int pid_product; #pragma omp critical (doFieldIonization_nextid) { @@ -176,20 +170,19 @@ public: const int cpuid = amrex::ParallelDescriptor::MyProc(); // Create instance of copy functor - copyParticle copy_functor = initialize_copy( + v_copy_functor.push_back (initialize_copy( cpuid, v_do_boosted_product[iproduct], runtime_uold_source, attribs_source, attribs_product, - runtime_attribs_product); - v_copy_functor[iproduct] = copy_functor; + runtime_attribs_product) ); v_pid_product[iproduct] = pid_product; } // Loop over source particles and create product particles copyAndTransformParticles(is_flagged, i_product, np_source, v_pid_product, v_particles_product, particles_source, v_copy_functor, - runtime_iattribs_source, v_runtime_iattribs_product); + runtime_iattribs_source); } /** @@ -204,9 +197,8 @@ public: const amrex::Vector v_pid_product, const amrex::Vector v_particles_product, WarpXParticleContainer::ParticleType* particles_source, - const amrex::Vector v_copy_functor, - amrex::GpuArray runtime_iattribs_source, - amrex::Vector> v_runtime_iattribs_product) + const amrex::Vector& v_copy_functor, + amrex::GpuArray runtime_iattribs_source) { int const * const AMREX_RESTRICT p_is_flagged = is_flagged.dataPtr(); int const * const AMREX_RESTRICT p_i_product = i_product.dataPtr(); @@ -223,15 +215,14 @@ public: for (int iproduct=0; iproduct(is, p_source, runtime_iattribs_source); - transformProductParticle(ip, v_particles_product, v_runtime_iattribs_product); + transformProductParticle(ip, v_particles_product); } } ); diff --git a/Source/Particles/ParticleCreation/TransformParticle.H b/Source/Particles/ParticleCreation/TransformParticle.H index c414ca56f..090be1f26 100644 --- a/Source/Particles/ParticleCreation/TransformParticle.H +++ b/Source/Particles/ParticleCreation/TransformParticle.H @@ -28,8 +28,7 @@ template < elementaryProcessType ProcessT > AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void transformProductParticle( int i, - amrex::Vector v_particle, - amrex::Vector> v_runtime_iattribs){} + amrex::Vector v_particle){} // For ionization, increase ionization level of source particle template <> -- cgit v1.2.3 From 40a05a827bc553e99157ac5cab3fb343ea832004 Mon Sep 17 00:00:00 2001 From: Maxence Thevenet Date: Mon, 21 Oct 2019 13:55:18 -0400 Subject: use RedistributeCPU to avoid issue with runtime components on GPU --- Source/Particles/MultiParticleContainer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 5d898c11a..7d4081def 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -310,7 +310,7 @@ void MultiParticleContainer::Redistribute () { for (auto& pc : allcontainers) { - pc->Redistribute(); + pc->RedistributeCPU(); } } @@ -318,7 +318,7 @@ void MultiParticleContainer::RedistributeLocal (const int num_ghost) { for (auto& pc : allcontainers) { - pc->Redistribute(0, 0, 0, num_ghost); + pc->RedistributeCPU(0, 0, 0, num_ghost); } } -- cgit v1.2.3 From 805eaad896dc3a15bd4cccb39eba79f146a6f9ae Mon Sep 17 00:00:00 2001 From: Maxence Thevenet Date: Mon, 21 Oct 2019 20:09:38 -0400 Subject: fix ionization on GPU. Compiles and ionization test passes --- Source/Particles/MultiParticleContainer.cpp | 2 +- Source/Particles/ParticleCreation/CopyParticle.H | 6 +- .../Particles/ParticleCreation/ElementaryProcess.H | 108 ++++++++++++--------- .../Particles/ParticleCreation/TransformParticle.H | 3 +- 4 files changed, 72 insertions(+), 47 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index bd680af99..6bf153a01 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -594,7 +594,7 @@ MultiParticleContainer::doFieldIonization () // Create particles in pc_product int do_boost = WarpX::do_boosted_frame_diagnostic && pc_product->DoBoostedFrameDiags(); - amrex::Vector v_do_boosted_product; + amrex::Gpu::ManagedDeviceVector v_do_boosted_product; v_do_boosted_product.push_back(do_boost); const amrex::Vector v_pc_product {pc_product.get()}; // Copy source to product particles, and increase ionization diff --git a/Source/Particles/ParticleCreation/CopyParticle.H b/Source/Particles/ParticleCreation/CopyParticle.H index eb0f72bde..228b25546 100644 --- a/Source/Particles/ParticleCreation/CopyParticle.H +++ b/Source/Particles/ParticleCreation/CopyParticle.H @@ -48,6 +48,7 @@ public: m_runtime_attribs_product(runtime_attribs_product){} // Copy constructor. +/* AMREX_GPU_HOST_DEVICE copyParticle(const copyParticle& in){ m_cpuid = in.m_cpuid; @@ -57,6 +58,7 @@ public: m_attribs_product = in.m_attribs_product; m_runtime_attribs_product = in.m_runtime_attribs_product; } +*/ /** * \brief Overload operator () to do the copy of 1 particle @@ -70,7 +72,7 @@ public: AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator () (int is, int ip, int pid_product, WarpXParticleContainer::ParticleType& p_source, - WarpXParticleContainer::ParticleType& p_product) + WarpXParticleContainer::ParticleType& p_product) const noexcept { // Copy particle from source to product: AoS p_product.id() = pid_product + ip; @@ -87,6 +89,7 @@ public: // Update xold etc. if boosted frame diagnostics required // for product species. Fill runtime attribs with a copy of // current properties (xold = x etc.). + /* if (m_do_boosted_product) { m_runtime_attribs_product[0][ip] = p_source.pos(0); m_runtime_attribs_product[1][ip] = p_source.pos(1); @@ -95,6 +98,7 @@ public: m_runtime_attribs_product[4][ip] = m_runtime_uold_source[1][ip]; m_runtime_attribs_product[5][ip] = m_runtime_uold_source[2][ip]; } + */ } }; diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index 4c47b7f52..27505ed0c 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -67,8 +67,9 @@ public: std::unique_ptr< WarpXParticleContainer>& pc_source, amrex::Vector v_pc_product, amrex::Gpu::ManagedDeviceVector& is_flagged, - const amrex::Vector v_do_boosted_product) + amrex::Gpu::ManagedDeviceVector v_do_boosted_product) { + BL_PROFILE("createIonizedParticles"); const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); @@ -96,12 +97,12 @@ public: amrex::GpuArray runtime_iattribs_source; std::map icomps_source = pc_source->getParticleiComps(); runtime_iattribs_source[0] = soa_source.GetIntData(icomps_source["ionization_level"]).data(); + int nproducts = v_pc_product.size(); AMREX_ALWAYS_ASSERT_WITH_MESSAGE( v_do_boosted_product.size() == nproducts, "v_pc_product and v_do_boosted_product must have the same size."); - // Indices of product particle for each ionized source particle. // i_product[i]-1 is the location in product tile of product particle // from source particle i. @@ -122,9 +123,8 @@ public: int np_ionized = i_product[np_source-1]; if (np_ionized == 0) return; - // amrex::Vector v_copy_functor(nproducts); - amrex::Vector v_copy_functor; - amrex::Vector v_pid_product(nproducts); + amrex::Gpu::ManagedDeviceVector v_copy_functor; + amrex::Gpu::ManagedDeviceVector v_pid_product(nproducts); amrex::Gpu::ManagedDeviceVector v_particles_product(nproducts); for (int iproduct=0; iproduct attribs_product; @@ -149,41 +150,53 @@ public: // --- product runtime attribs amrex::GpuArray runtime_attribs_product; const int do_boost = v_do_boosted_product[iproduct]; - if (do_boost) { - std::map comps_product = pc_product->getParticleComps(); - runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; - runtime_attribs_product[1] = soa_product.GetRealData(comps_product[ "yold"]).data() + np_product_old; - runtime_attribs_product[2] = soa_product.GetRealData(comps_product[ "zold"]).data() + np_product_old; - runtime_attribs_product[3] = soa_product.GetRealData(comps_product["uxold"]).data() + np_product_old; - runtime_attribs_product[4] = soa_product.GetRealData(comps_product["uyold"]).data() + np_product_old; - runtime_attribs_product[5] = soa_product.GetRealData(comps_product["uzold"]).data() + np_product_old; - } - - // --- product runtime integer attribs - int pid_product; + if (do_boost) { + std::map comps_product = pc_product->getParticleComps(); + runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; + runtime_attribs_product[1] = soa_product.GetRealData(comps_product[ "yold"]).data() + np_product_old; + runtime_attribs_product[2] = soa_product.GetRealData(comps_product[ "zold"]).data() + np_product_old; + runtime_attribs_product[3] = soa_product.GetRealData(comps_product["uxold"]).data() + np_product_old; + runtime_attribs_product[4] = soa_product.GetRealData(comps_product["uyold"]).data() + np_product_old; + runtime_attribs_product[5] = soa_product.GetRealData(comps_product["uzold"]).data() + np_product_old; + } + + // --- product runtime integer attribs + int pid_product; #pragma omp critical (doFieldIonization_nextid) - { - // ID of first newly-created product particle - pid_product = pc_product->NextID(); - // Update NextID to include particles created in this function - pc_product->setNextID(pid_product+np_ionized); - } - const int cpuid = amrex::ParallelDescriptor::MyProc(); - - // Create instance of copy functor - v_copy_functor.push_back (initialize_copy( - cpuid, v_do_boosted_product[iproduct], - runtime_uold_source, - attribs_source, - attribs_product, - runtime_attribs_product) ); - v_pid_product[iproduct] = pid_product; - } + { + // ID of first newly-created product particle + pid_product = pc_product->NextID(); + // Update NextID to include particles created in this function + pc_product->setNextID(pid_product+np_ionized); + } + const int cpuid = amrex::ParallelDescriptor::MyProc(); + // Create instance of copy functor +/* + const copyParticle copy_functor = initialize_copy( + cpuid, do_boosted_product, + runtime_uold_source, + attribs_source, + attribs_product, + runtime_attribs_product); +*/ + v_copy_functor.push_back (initialize_copy( + cpuid, v_do_boosted_product[iproduct], + runtime_uold_source, + attribs_source, + attribs_product, + runtime_attribs_product) ); + v_pid_product[iproduct] = pid_product; + + } // Loop over source particles and create product particles copyAndTransformParticles(is_flagged, i_product, np_source, v_pid_product, v_particles_product, particles_source, v_copy_functor, runtime_iattribs_source); + + //copyAndTransformParticles(is_flagged, i_product, np_source, pid_product, + // particles_product, particles_source, copy_functor, + // runtime_iattribs_source); } /** @@ -205,15 +218,22 @@ public: amrex::Gpu::ManagedDeviceVector& is_flagged, amrex::Gpu::ManagedDeviceVector& i_product, int np_source, - const amrex::Vector v_pid_product, - const amrex::Gpu::ManagedDeviceVector v_particles_product, + amrex::Gpu::ManagedDeviceVector v_pid_product, + amrex::Gpu::ManagedDeviceVector v_particles_product, WarpXParticleContainer::ParticleType* particles_source, - const amrex::Vector& v_copy_functor, + const amrex::Gpu::ManagedDeviceVector& v_copy_functor, amrex::GpuArray runtime_iattribs_source) { + + // WarpXParticleContainer::ParticleType* particles_product, + // const copyParticle& copy_functor, + int const * const AMREX_RESTRICT p_is_flagged = is_flagged.dataPtr(); int const * const AMREX_RESTRICT p_i_product = i_product.dataPtr(); - WarpXParticleContainer::ParticleType* const * const AMREX_RESTRICT p_particles_product = v_particles_product.dataPtr(); + copyParticle const * const p_copy_functor = v_copy_functor.data(); + WarpXParticleContainer::ParticleType * * p_particles_product = v_particles_product.data(); + int const * const p_pid_product = v_pid_product.data(); + // Loop over all source particles. If is_flagged, copy particle data // to corresponding product particle. amrex::For( @@ -224,14 +244,14 @@ public: // offset of 1 due to inclusive scan int ip = p_i_product[is]-1; WarpXParticleContainer::ParticleType& p_source = particles_source[is]; + // WarpXParticleContainer::ParticleType& p_product = particles_product[ip]; for (int iproduct=0; iproduct(is, p_source, runtime_iattribs_source); diff --git a/Source/Particles/ParticleCreation/TransformParticle.H b/Source/Particles/ParticleCreation/TransformParticle.H index c202cb6d7..e16b0a4b7 100644 --- a/Source/Particles/ParticleCreation/TransformParticle.H +++ b/Source/Particles/ParticleCreation/TransformParticle.H @@ -27,7 +27,8 @@ template < elementaryProcessType ProcessT > AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void transformProductParticle( int i, - WarpXParticleContainer::ParticleType* const * const v_particle){} + WarpXParticleContainer::ParticleType* * v_particle){} +// WarpXParticleContainer::ParticleType& particle){} // For ionization, increase ionization level of source particle template <> -- cgit v1.2.3 From 42550405dbbffce12e967c73099799e54e0abb9a Mon Sep 17 00:00:00 2001 From: Maxence Thevenet Date: Tue, 22 Oct 2019 10:32:15 -0400 Subject: call RedistributeCPU if runtime components are present, even on GPU --- Source/Particles/MultiParticleContainer.cpp | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 6bf153a01..2b6b95827 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -316,18 +316,25 @@ void MultiParticleContainer::Redistribute () { for (auto& pc : allcontainers) { - pc->RedistributeCPU(); + if ( (pc->NumRuntimeRealComps()>0) || (pc->NumRuntimeIntComps()>0) ) { + pc->RedistributeCPU(); + } else { + pc->Redistribute(); + } } -} -void -MultiParticleContainer::RedistributeLocal (const int num_ghost) -{ - for (auto& pc : allcontainers) { - pc->RedistributeCPU(0, 0, 0, num_ghost); + void + MultiParticleContainer::RedistributeLocal (const int num_ghost) + { + for (auto& pc : allcontainers) { + if ( (pc->NumRuntimeRealComps()>0) || (pc->NumRuntimeIntComps()>0) ) { + pc->RedistributeCPU(0, 0, 0, num_ghost); + } else { + pc->Redistribute(0, 0, 0, num_ghost); + } + } } } - Vector MultiParticleContainer::NumberOfParticlesInGrid (int lev) const { -- cgit v1.2.3 From 2b77afdaa9ca95b0e29f5ccaf98d29c94e5206a9 Mon Sep 17 00:00:00 2001 From: Maxence Thevenet Date: Tue, 22 Oct 2019 10:42:10 -0400 Subject: comments and remove typo --- Source/Particles/MultiParticleContainer.cpp | 19 ++++++++++--------- Source/Particles/ParticleCreation/ElementaryProcess.H | 18 +++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 2b6b95827..cc48f38db 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -322,19 +322,20 @@ MultiParticleContainer::Redistribute () pc->Redistribute(); } } +} - void - MultiParticleContainer::RedistributeLocal (const int num_ghost) - { - for (auto& pc : allcontainers) { - if ( (pc->NumRuntimeRealComps()>0) || (pc->NumRuntimeIntComps()>0) ) { - pc->RedistributeCPU(0, 0, 0, num_ghost); - } else { - pc->Redistribute(0, 0, 0, num_ghost); - } +void +MultiParticleContainer::RedistributeLocal (const int num_ghost) +{ + for (auto& pc : allcontainers) { + if ( (pc->NumRuntimeRealComps()>0) || (pc->NumRuntimeIntComps()>0) ) { + pc->RedistributeCPU(0, 0, 0, num_ghost); + } else { + pc->Redistribute(0, 0, 0, num_ghost); } } } + Vector MultiParticleContainer::NumberOfParticlesInGrid (int lev) const { diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index 34e13638e..d00a81175 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -8,18 +8,18 @@ /** * \brief Base class for particle creation processes * - * Particles in a source particle container are copied to a product particle - * container if they are flagged. Both source and product particles can be + * Particles in a source particle container are copied to product particle + * containers if they are flagged. Both source and product particles can be * modified. * - * initialize_copy initializes a general copy functor - * createParticles formats the data to prepare for the copy, then - * calls copyAndTransformParticles - * copyAndTransformParticles loops over particles, performs the copy and - * transform source and product particles if needed. + * - initialize_copy initializes a general copy functor + * - createParticles formats the data to prepare for the copy, then + * calls copyAndTransformParticles + * - copyAndTransformParticles loops over particles, performs the copy and + * transform source and product particles if needed. * * The class is templated on the process type. This gives flexibility - * on source and product particle transformation. + * on source and product particle transformations. */ template class elementaryProcess @@ -52,7 +52,7 @@ public: * \brief create particles in product particle containers * * For particle i in mfi, if is_flagged[i]=1, copy particle - * particle i from container pc_source into pc_product + * particle i from container pc_source into each container in v_pc_product * * \param lev MR level * \param mfi MFIter where source particles are located -- cgit v1.2.3 From eb7564be98a2005036ed90e4cdf1b94c26fbc4c6 Mon Sep 17 00:00:00 2001 From: Maxence Thevenet Date: Tue, 22 Oct 2019 11:24:29 -0400 Subject: minor cleaning --- Source/Particles/MultiParticleContainer.cpp | 3 +-- Source/Particles/ParticleCreation/ElementaryProcess.H | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index cc48f38db..c54408b64 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -602,8 +602,7 @@ MultiParticleContainer::doFieldIonization () // Create particles in pc_product int do_boost = WarpX::do_boosted_frame_diagnostic && pc_product->DoBoostedFrameDiags(); - amrex::Gpu::ManagedDeviceVector v_do_boosted_product; - v_do_boosted_product.push_back(do_boost); + amrex::Gpu::ManagedDeviceVector v_do_boosted_product{do_boost}; const amrex::Vector v_pc_product {pc_product.get()}; // Copy source to product particles, and increase ionization // level of source particle diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index d00a81175..215574cda 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -228,13 +228,11 @@ public: // offset of 1 due to inclusive scan int ip = p_i_product[is]-1; WarpXParticleContainer::ParticleType& p_source = particles_source[is]; - // WarpXParticleContainer::ParticleType& p_product = particles_product[ip]; for (int iproduct=0; iproduct Date: Fri, 25 Oct 2019 11:56:10 -0700 Subject: replace 'boosted frame diags' with 'back-transformed diags' --- Docs/source/running_cpp/parameters.rst | 18 ++--- .../Modules/RigidInjection/inputs.BoostedFrame | 2 +- Examples/Modules/boosted_diags/inputs.2d | 2 +- Examples/Modules/boosted_diags/inputs.3d | 2 +- Examples/Modules/boosted_diags/inputs.3d.slice | 6 +- .../laser_acceleration/inputs.2d.boost | 2 +- .../plasma_acceleration/inputs.2d.boost | 2 +- .../plasma_acceleration/inputs.3d.boost | 2 +- Python/pywarpx/picmi.py | 8 +-- Source/Diagnostics/BoostedFrameDiagnostic.H | 12 ++-- Source/Diagnostics/BoostedFrameDiagnostic.cpp | 76 +++++++++++----------- Source/Diagnostics/Make.package | 4 +- Source/Evolve/WarpXEvolveEM.cpp | 6 +- Source/Initialization/WarpXInitData.cpp | 4 +- Source/Laser/LaserParticleContainer.cpp | 2 +- Source/Particles/MultiParticleContainer.H | 18 ++--- Source/Particles/MultiParticleContainer.cpp | 24 +++---- Source/Particles/PhotonParticleContainer.cpp | 2 +- Source/Particles/PhysicalParticleContainer.cpp | 10 +-- Source/Particles/WarpXParticleContainer.H | 4 +- Source/WarpX.H | 10 +-- Source/WarpX.cpp | 16 ++--- .../automated_test_4_labdiags_2ppc | 2 +- 23 files changed, 117 insertions(+), 117 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index 59bfbb14e..a2de46b03 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -349,8 +349,8 @@ Particle initialization ``.plot_vars = none`` to plot no particle data, except particle position. -* ``.do_boosted_frame_diags`` (`0` or `1` optional, default `1`) - Only used when ``warpx.do_boosted_frame_diagnostic=1``. When running in a +* ``.do_back_transformed_diagnostics`` (`0` or `1` optional, default `1`) + Only used when ``warpx.do_back_transformed_diagnostics=1``. When running in a boosted frame, whether or not to plot back-transformed diagnostics for this species. @@ -765,7 +765,7 @@ Diagnostics and output When WarpX is compiled with openPMD support, this is ``h5`` by default. ``json`` only works with serial/single-rank jobs. -* ``warpx.do_boosted_frame_diagnostic`` (`0` or `1`) +* ``warpx.do_back_transformed_diagnostics`` (`0` or `1`) Whether to use the **back-transformed diagnostics** (i.e. diagnostics that perform on-the-fly conversion to the laboratory frame, when running boosted-frame simulations) @@ -776,22 +776,22 @@ Diagnostics and output is `lab_frame_data`. * ``warpx.num_snapshots_lab`` (`integer`) - Only used when ``warpx.do_boosted_frame_diagnostic`` is ``1``. + Only used when ``warpx.do_back_transformed_diagnostics`` is ``1``. The number of lab-frame snapshots that will be written. * ``warpx.dt_snapshots_lab`` (`float`, in seconds) - Only used when ``warpx.do_boosted_frame_diagnostic`` is ``1``. + Only used when ``warpx.do_back_transformed_diagnostics`` is ``1``. The time interval inbetween the lab-frame snapshots (where this time interval is expressed in the laboratory frame). * ``warpx.dz_snapshots_lab`` (`float`, in meters) - Only used when ``warpx.do_boosted_frame_diagnostic`` is ``1``. + Only used when ``warpx.do_back_transformed_diagnostics`` is ``1``. Distance between the lab-frame snapshots (expressed in the laboratory frame). ``dt_snapshots_lab`` is then computed by ``dt_snapshots_lab = dz_snapshots_lab/c``. Either `dt_snapshots_lab` or `dz_snapshot_lab` is required. -* ``warpx.do_boosted_frame_fields`` (`0 or 1`) +* ``warpx.do_back_transformed_fields`` (`0 or 1`) Whether to use the **back-transformed diagnostics** for the fields. * ``warpx.boosted_frame_diag_fields`` (space-separated list of `string`) @@ -850,7 +850,7 @@ Diagnostics and output negative number to disable slice generation and slice data dumping. * ``slice.num_slice_snapshots_lab`` (`integer`) - Only used when ``warpx.do_boosted_frame_diagnostic`` is ``1``. + Only used when ``warpx.do_back_transformed_diagnostics`` is ``1``. The number of back-transformed field and particle data that will be written for the reduced domain defined by ``slice.dom_lo`` and ``slice.dom_hi``. Note that the 'slice' is a reduced @@ -860,7 +860,7 @@ Diagnostics and output for further details. * ``slice.dt_slice_snapshots_lab`` (`float`, in seconds) - Only used when ``warpx.do_boosted_frame_diagnostic`` is ``1``. + Only used when ``warpx.do_back_transformed_diagnostics`` is ``1``. The time interval between the back-transformed reduced diagnostics (where this time interval is expressed in the laboratory frame). diff --git a/Examples/Modules/RigidInjection/inputs.BoostedFrame b/Examples/Modules/RigidInjection/inputs.BoostedFrame index c7a60f14f..456df363d 100644 --- a/Examples/Modules/RigidInjection/inputs.BoostedFrame +++ b/Examples/Modules/RigidInjection/inputs.BoostedFrame @@ -3,7 +3,7 @@ warpx.zmax_plasma_to_compute_max_step = 50.e-6 warpx.gamma_boost = 5. warpx.boost_direction = z -warpx.do_boosted_frame_diagnostic = 1 +warpx.do_back_transformed_diagnostics = 1 warpx.num_snapshots_lab = 2 warpx.dt_snapshots_lab = 1.8679589331096515e-13 diff --git a/Examples/Modules/boosted_diags/inputs.2d b/Examples/Modules/boosted_diags/inputs.2d index 6afe6977d..7b2270f90 100644 --- a/Examples/Modules/boosted_diags/inputs.2d +++ b/Examples/Modules/boosted_diags/inputs.2d @@ -44,7 +44,7 @@ warpx.gamma_boost = 15. warpx.boost_direction = z # Diagnostics -warpx.do_boosted_frame_diagnostic = 1 +warpx.do_back_transformed_diagnostics = 1 warpx.num_snapshots_lab = 20 warpx.dt_snapshots_lab = 7.0e-14 diff --git a/Examples/Modules/boosted_diags/inputs.3d b/Examples/Modules/boosted_diags/inputs.3d index 528eb6cd9..b700a52fb 100644 --- a/Examples/Modules/boosted_diags/inputs.3d +++ b/Examples/Modules/boosted_diags/inputs.3d @@ -44,7 +44,7 @@ warpx.gamma_boost = 15. warpx.boost_direction = z # Diagnostics -warpx.do_boosted_frame_diagnostic = 1 +warpx.do_back_transformed_diagnostics = 1 warpx.num_snapshots_lab = 20; warpx.dt_snapshots_lab = 7.0e-14; diff --git a/Examples/Modules/boosted_diags/inputs.3d.slice b/Examples/Modules/boosted_diags/inputs.3d.slice index ca745f347..08f2310cd 100644 --- a/Examples/Modules/boosted_diags/inputs.3d.slice +++ b/Examples/Modules/boosted_diags/inputs.3d.slice @@ -30,7 +30,7 @@ warpx.serialize_ics = 1 warpx.gamma_boost = 10. warpx.boost_direction = z -warpx.do_boosted_frame_diagnostic = 1 +warpx.do_back_transformed_diagnostics = 1 warpx.num_snapshots_lab = 4 warpx.dz_snapshots_lab = 0.001 warpx.boosted_frame_diag_fields= Ex Ey Ez By rho @@ -53,7 +53,7 @@ electrons.zmax = .003 electrons.profile = constant electrons.density = 3.5e24 electrons.do_continuous_injection = 1 -electrons.do_boosted_frame_diags = 1 +electrons.do_back_transformed_diagnostics = 1 ions.charge = q_e ions.mass = m_p @@ -69,7 +69,7 @@ ions.zmax = .003 ions.profile = constant ions.density = 3.5e24 ions.do_continuous_injection = 1 -ions.do_boosted_frame_diags = 1 +ions.do_back_transformed_diagnostics = 1 beam.charge = -q_e beam.mass = m_e diff --git a/Examples/Physics_applications/laser_acceleration/inputs.2d.boost b/Examples/Physics_applications/laser_acceleration/inputs.2d.boost index 89bded6c0..a50b5164a 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs.2d.boost +++ b/Examples/Physics_applications/laser_acceleration/inputs.2d.boost @@ -41,7 +41,7 @@ warpx.serialize_ics = 1 ################################# warpx.gamma_boost = 10. warpx.boost_direction = z -warpx.do_boosted_frame_diagnostic = 1 +warpx.do_back_transformed_diagnostics = 1 warpx.num_snapshots_lab = 7 warpx.dt_snapshots_lab = 1.6678204759907604e-12 diff --git a/Examples/Physics_applications/plasma_acceleration/inputs.2d.boost b/Examples/Physics_applications/plasma_acceleration/inputs.2d.boost index d711815eb..af4556e25 100644 --- a/Examples/Physics_applications/plasma_acceleration/inputs.2d.boost +++ b/Examples/Physics_applications/plasma_acceleration/inputs.2d.boost @@ -39,7 +39,7 @@ interpolation.noz = 3 ################################# warpx.gamma_boost = 10.0 warpx.boost_direction = z -warpx.do_boosted_frame_diagnostic = 1 +warpx.do_back_transformed_diagnostics = 1 warpx.num_snapshots_lab = 22 warpx.dt_snapshots_lab = 3.335640951981521e-11 diff --git a/Examples/Physics_applications/plasma_acceleration/inputs.3d.boost b/Examples/Physics_applications/plasma_acceleration/inputs.3d.boost index 42446a9ac..79480c97f 100644 --- a/Examples/Physics_applications/plasma_acceleration/inputs.3d.boost +++ b/Examples/Physics_applications/plasma_acceleration/inputs.3d.boost @@ -38,7 +38,7 @@ interpolation.noz = 3 ################################# warpx.gamma_boost = 10.0 warpx.boost_direction = z -warpx.do_boosted_frame_diagnostic = 1 +warpx.do_back_transformed_diagnostics = 1 warpx.num_snapshots_lab = 22 warpx.dt_snapshots_lab = 3.335640951981521e-11 diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index c33700278..c4e6803d5 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -671,10 +671,10 @@ class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic): pywarpx.warpx.check_consistency('dt_snapshots_lab', self.dt_snapshots, 'The time between snapshots must be the same in all lab frame diagnostics') pywarpx.warpx.check_consistency('lab_data_directory', self.write_dir, 'The write directory must be the same in all lab frame diagnostics') - pywarpx.warpx.do_boosted_frame_diagnostic = 1 + pywarpx.warpx.do_back_transformed_diagnostics = 1 pywarpx.warpx.num_snapshots_lab = self.num_snapshots pywarpx.warpx.dt_snapshots_lab = self.dt_snapshots - pywarpx.warpx.do_boosted_frame_fields = 1 + pywarpx.warpx.do_back_transformed_fields = 1 pywarpx.warpx.lab_data_directory = self.write_dir @@ -685,8 +685,8 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic) pywarpx.warpx.check_consistency('dt_snapshots_lab', self.dt_snapshots, 'The time between snapshots must be the same in all lab frame diagnostics') pywarpx.warpx.check_consistency('lab_data_directory', self.write_dir, 'The write directory must be the same in all lab frame diagnostics') - pywarpx.warpx.do_boosted_frame_diagnostic = 1 + pywarpx.warpx.do_back_transformed_diagnostics = 1 pywarpx.warpx.num_snapshots_lab = self.num_snapshots pywarpx.warpx.dt_snapshots_lab = self.dt_snapshots - pywarpx.warpx.do_boosted_frame_particles = 1 + pywarpx.warpx.do_back_transformed_particles = 1 pywarpx.warpx.lab_data_directory = self.write_dir diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.H b/Source/Diagnostics/BoostedFrameDiagnostic.H index 5d95aaf7d..9e24caa1b 100644 --- a/Source/Diagnostics/BoostedFrameDiagnostic.H +++ b/Source/Diagnostics/BoostedFrameDiagnostic.H @@ -1,5 +1,5 @@ -#ifndef WARPX_BoostedFrameDiagnostic_H_ -#define WARPX_BoostedFrameDiagnostic_H_ +#ifndef WARPX_BackTransformedDiagnostic_H_ +#define WARPX_BackTransformedDiagnostic_H_ #include #include @@ -150,7 +150,7 @@ class LabFrameSlice : public LabFrameDiag { }; /** \brief - * BoostedFrameDiagnostic class handles the back-transformation of data when + * BackTransformedDiagnostic class handles the back-transformation of data when * running simulations in a boosted frame of reference to the lab-frame. * Because of the relativity of simultaneity, events that are synchronized * in the simulation boosted frame are not @@ -163,13 +163,13 @@ class LabFrameSlice : public LabFrameDiag { * to the output directory. The functions Flush() and writeLabFrameData() * are called at the end of the simulation and when the * the buffer for data storage is full, respectively. The particle data - * is collected and written only if particle.do_boosted_frame_diagnostic = 1. + * is collected and written only if particle.do_back_transformed_diagnostics = 1. */ -class BoostedFrameDiagnostic { +class BackTransformedDiagnostic { public: - BoostedFrameDiagnostic(amrex::Real zmin_lab, amrex::Real zmax_lab, + BackTransformedDiagnostic(amrex::Real zmin_lab, amrex::Real zmax_lab, amrex::Real v_window_lab, amrex::Real dt_snapshots_lab, int N_snapshots, amrex::Real dt_slice_snapshots_lab, int N_slice_snapshots, amrex::Real gamma_boost, diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.cpp b/Source/Diagnostics/BoostedFrameDiagnostic.cpp index 297b4f5be..2880b37b1 100644 --- a/Source/Diagnostics/BoostedFrameDiagnostic.cpp +++ b/Source/Diagnostics/BoostedFrameDiagnostic.cpp @@ -1,7 +1,7 @@ #include #include -#include "BoostedFrameDiagnostic.H" +#include "BackTransformedDiagnostic.H" #include "SliceDiagnostic.H" #include "WarpX_f.H" #include "WarpX.H" @@ -514,8 +514,8 @@ LorentzTransformZ(MultiFab& data, Real gamma_boost, Real beta_boost, int ncomp) } } -BoostedFrameDiagnostic:: -BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, +BackTransformedDiagnostic:: +BackTransformedDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, Real dt_snapshots_lab, int N_snapshots, Real dt_slice_snapshots_lab, int N_slice_snapshots, Real gamma_boost, Real t_boost, Real dt_boost, @@ -531,10 +531,10 @@ BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, { - BL_PROFILE("BoostedFrameDiagnostic::BoostedFrameDiagnostic"); + BL_PROFILE("BackTransformedDiagnostic::BackTransformedDiagnostic"); - AMREX_ALWAYS_ASSERT(WarpX::do_boosted_frame_fields or - WarpX::do_boosted_frame_particles); + AMREX_ALWAYS_ASSERT(WarpX::do_back_transformed_fields or + WarpX::do_back_transformed_particles); inv_gamma_boost_ = 1.0 / gamma_boost_; beta_boost_ = std::sqrt(1.0 - inv_gamma_boost_*inv_gamma_boost_); @@ -679,9 +679,9 @@ BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, AMREX_ALWAYS_ASSERT(max_box_size_ >= num_buffer_); } -void BoostedFrameDiagnostic::Flush(const Geometry& geom) +void BackTransformedDiagnostic::Flush(const Geometry& geom) { - BL_PROFILE("BoostedFrameDiagnostic::Flush"); + BL_PROFILE("BackTransformedDiagnostic::Flush"); VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); VisMF::SetHeaderVersion(amrex::VisMF::Header::NoFabHeader_v1); @@ -696,7 +696,7 @@ void BoostedFrameDiagnostic::Flush(const Geometry& geom) int i_lab = (LabFrameDiags_[i]->current_z_lab - zmin_lab) / dz_lab_; if (LabFrameDiags_[i]->buff_counter_ != 0) { - if (WarpX::do_boosted_frame_fields) { + if (WarpX::do_back_transformed_fields) { const BoxArray& ba = LabFrameDiags_[i]->data_buffer_->boxArray(); const int hi = ba[0].bigEnd(boost_direction_); const int lo = hi - LabFrameDiags_[i]->buff_counter_ + 1; @@ -731,12 +731,12 @@ void BoostedFrameDiagnostic::Flush(const Geometry& geom) #endif } - if (WarpX::do_boosted_frame_particles) { + if (WarpX::do_back_transformed_particles) { // Loop over species to be dumped to BFD - for (int j = 0; j < mypc.nSpeciesBoostedFrameDiags(); ++j) { + for (int j = 0; j < mypc.nSpeciesBackTransformedDiagnostics(); ++j) { // Get species name std::string species_name = - species_names[mypc.mapSpeciesBoostedFrameDiags(j)]; + species_names[mypc.mapSpeciesBackTransformedDiagnostics(j)]; #ifdef WARPX_USE_HDF5 // Dump species data writeParticleDataHDF5(LabFrameDiags_[i]->particles_buffer_[j], @@ -765,12 +765,12 @@ void BoostedFrameDiagnostic::Flush(const Geometry& geom) void -BoostedFrameDiagnostic:: +BackTransformedDiagnostic:: writeLabFrameData(const MultiFab* cell_centered_data, const MultiParticleContainer& mypc, const Geometry& geom, const Real t_boost, const Real dt) { - BL_PROFILE("BoostedFrameDiagnostic::writeLabFrameData"); + BL_PROFILE("BackTransformedDiagnostic::writeLabFrameData"); VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); VisMF::SetHeaderVersion(amrex::VisMF::Header::NoFabHeader_v1); @@ -808,7 +808,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, // If buffer of snapshot i is empty... if ( LabFrameDiags_[i]->buff_counter_ == 0) { // ... reset fields buffer data_buffer_ - if (WarpX::do_boosted_frame_fields) { + if (WarpX::do_back_transformed_fields) { LabFrameDiags_[i]->buff_box_.setSmall(boost_direction_, i_lab - num_buffer_ + 1); LabFrameDiags_[i]->buff_box_.setBig(boost_direction_, i_lab); @@ -820,12 +820,12 @@ writeLabFrameData(const MultiFab* cell_centered_data, buff_dm, ncomp_to_dump, 0) ); } // ... reset particle buffer particles_buffer_[i] - if (WarpX::do_boosted_frame_particles) + if (WarpX::do_back_transformed_particles) LabFrameDiags_[i]->particles_buffer_.resize( - mypc.nSpeciesBoostedFrameDiags()); + mypc.nSpeciesBackTransformedDiagnostics()); } - if (WarpX::do_boosted_frame_fields) { + if (WarpX::do_back_transformed_fields) { const int ncomp = cell_centered_data->nComp(); const int start_comp = 0; const bool interpolate = true; @@ -873,7 +873,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, tmp_slice_ptr.reset(nullptr); } - if (WarpX::do_boosted_frame_particles) { + if (WarpX::do_back_transformed_particles) { if (LabFrameDiags_[i]->t_lab != prev_t_lab ) { if (tmp_particle_buffer.size()>0) @@ -881,7 +881,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, tmp_particle_buffer.clear(); tmp_particle_buffer.shrink_to_fit(); } - tmp_particle_buffer.resize(mypc.nSpeciesBoostedFrameDiags()); + tmp_particle_buffer.resize(mypc.nSpeciesBackTransformedDiagnostics()); mypc.GetLabFrameData( LabFrameDiags_[i]->file_name, i_lab, boost_direction_, old_z_boost, LabFrameDiags_[i]->current_z_boost, @@ -889,7 +889,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, tmp_particle_buffer); } LabFrameDiags_[i]->AddPartDataToParticleBuffer(tmp_particle_buffer, - mypc.nSpeciesBoostedFrameDiags()); + mypc.nSpeciesBackTransformedDiagnostics()); } ++LabFrameDiags_[i]->buff_counter_; @@ -898,7 +898,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, // If buffer full, write to disk. if ( LabFrameDiags_[i]->buff_counter_ == num_buffer_) { - if (WarpX::do_boosted_frame_fields) { + if (WarpX::do_back_transformed_fields) { #ifdef WARPX_USE_HDF5 Box buff_box = LabFrameDiags_[i]->buff_box_; @@ -916,12 +916,12 @@ writeLabFrameData(const MultiFab* cell_centered_data, #endif } - if (WarpX::do_boosted_frame_particles) { + if (WarpX::do_back_transformed_particles) { // Loop over species to be dumped to BFD - for (int j = 0; j < mypc.nSpeciesBoostedFrameDiags(); ++j) { + for (int j = 0; j < mypc.nSpeciesBackTransformedDiagnostics(); ++j) { // Get species name const std::string species_name = species_names[ - mypc.mapSpeciesBoostedFrameDiags(j)]; + mypc.mapSpeciesBackTransformedDiagnostics(j)]; #ifdef WARPX_USE_HDF5 // Write data to disk (HDF5) writeParticleDataHDF5(LabFrameDiags_[i]->particles_buffer_[j], @@ -949,7 +949,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, #ifdef WARPX_USE_HDF5 void -BoostedFrameDiagnostic:: +BackTransformedDiagnostic:: writeParticleDataHDF5(const WarpXParticleContainer::DiagnosticParticleData& pdata, const std::string& name, const std::string& species_name) { @@ -997,11 +997,11 @@ writeParticleDataHDF5(const WarpXParticleContainer::DiagnosticParticleData& pdat #endif void -BoostedFrameDiagnostic:: +BackTransformedDiagnostic:: writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, const std::string& name, const int i_lab) { - BL_PROFILE("BoostedFrameDiagnostic::writeParticleData"); + BL_PROFILE("BackTransformedDiagnostic::writeParticleData"); std::string field_name; std::ofstream ofs; @@ -1047,10 +1047,10 @@ writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, } void -BoostedFrameDiagnostic:: +BackTransformedDiagnostic:: writeMetaData () { - BL_PROFILE("BoostedFrameDiagnostic::writeMetaData"); + BL_PROFILE("BackTransformedDiagnostic::writeMetaData"); if (ParallelDescriptor::IOProcessor()) { const std::string fullpath = WarpX::lab_data_directory + "/snapshots"; @@ -1134,7 +1134,7 @@ LabFrameSnapShot(Real t_lab_in, Real t_boost, Real inv_gamma_boost_in, file_num, 5); createLabFrameDirectories(); buff_counter_ = 0; - if (WarpX::do_boosted_frame_fields) data_buffer_.reset(nullptr); + if (WarpX::do_back_transformed_fields) data_buffer_.reset(nullptr); } void @@ -1158,7 +1158,7 @@ createLabFrameDirectories() { if (ParallelDescriptor::IOProcessor()) { - if (WarpX::do_boosted_frame_fields) + if (WarpX::do_back_transformed_fields) { const auto lo = lbound(buff_box_); for (int comp = 0; comp < ncomp_to_dump_; ++comp) { @@ -1176,15 +1176,15 @@ createLabFrameDirectories() { ParallelDescriptor::Barrier(); - if (WarpX::do_boosted_frame_particles){ + if (WarpX::do_back_transformed_particles){ auto & mypc = WarpX::GetInstance().GetPartContainer(); const std::vector species_names = mypc.GetSpeciesNames(); // Loop over species to be dumped to BFD - for (int j = 0; j < mypc.nSpeciesBoostedFrameDiags(); ++j) + for (int j = 0; j < mypc.nSpeciesBackTransformedDiagnostics(); ++j) { // Loop over species to be dumped to BFD std::string species_name = - species_names[mypc.mapSpeciesBoostedFrameDiags(j)]; + species_names[mypc.mapSpeciesBackTransformedDiagnostics(j)]; output_create_species_group(file_name, species_name); for (int k = 0; k < static_cast(particle_field_names.size()); ++k) { @@ -1211,10 +1211,10 @@ createLabFrameDirectories() { const std::string particles_prefix = "particle"; // Loop over species to be dumped to BFD - for(int i = 0; i < mypc.nSpeciesBoostedFrameDiags(); ++i) { + for(int i = 0; i < mypc.nSpeciesBackTransformedDiagnostics(); ++i) { // Get species name std::string species_name = - species_names[mypc.mapSpeciesBoostedFrameDiags(i)]; + species_names[mypc.mapSpeciesBackTransformedDiagnostics(i)]; const std::string fullpath = file_name + "/" + species_name; if (!UtilCreateDirectory(fullpath, 0755)) CreateDirectoryFailed(fullpath); @@ -1302,7 +1302,7 @@ LabFrameSlice(Real t_lab_in, Real t_boost, Real inv_gamma_boost_in, dx_ = cell_dx; dy_ = cell_dy; - if (WarpX::do_boosted_frame_fields) data_buffer_.reset(nullptr); + if (WarpX::do_back_transformed_fields) data_buffer_.reset(nullptr); } void diff --git a/Source/Diagnostics/Make.package b/Source/Diagnostics/Make.package index dfd947d53..b624d6ebe 100644 --- a/Source/Diagnostics/Make.package +++ b/Source/Diagnostics/Make.package @@ -1,9 +1,9 @@ CEXE_sources += WarpXIO.cpp -CEXE_sources += BoostedFrameDiagnostic.cpp +CEXE_sources += BackTransformedDiagnostic.cpp CEXE_sources += ParticleIO.cpp CEXE_sources += FieldIO.cpp CEXE_headers += FieldIO.H -CEXE_headers += BoostedFrameDiagnostic.H +CEXE_headers += BackTransformedDiagnostic.H CEXE_headers += ElectrostaticIO.cpp CEXE_headers += SliceDiagnostic.H CEXE_sources += SliceDiagnostic.cpp diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 7a3262703..b5fd52bdc 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -141,9 +141,9 @@ WarpX::EvolveEM (int numsteps) bool do_insitu = ((step+1) >= insitu_start) && (insitu_int > 0) && ((step+1) % insitu_int == 0); - if (do_boosted_frame_diagnostic) { + if (do_back_transformed_diagnostics) { std::unique_ptr cell_centered_data = nullptr; - if (WarpX::do_boosted_frame_fields) { + if (WarpX::do_back_transformed_fields) { cell_centered_data = GetCellCenteredData(); } myBFD->writeLabFrameData(cell_centered_data.get(), *mypc, geom[0], cur_time, dt[0]); @@ -261,7 +261,7 @@ WarpX::EvolveEM (int numsteps) WriteCheckPointFile(); } - if (do_boosted_frame_diagnostic) { + if (do_back_transformed_diagnostics) { myBFD->Flush(geom[0]); } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 78eaebfc5..6224d5348 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -82,14 +82,14 @@ WarpX::InitData () void WarpX::InitDiagnostics () { - if (do_boosted_frame_diagnostic) { + if (do_back_transformed_diagnostics) { const Real* current_lo = geom[0].ProbLo(); const Real* current_hi = geom[0].ProbHi(); Real dt_boost = dt[0]; // Find the positions of the lab-frame box that corresponds to the boosted-frame box at t=0 Real zmin_lab = current_lo[moving_window_dir]/( (1.+beta_boost)*gamma_boost ); Real zmax_lab = current_hi[moving_window_dir]/( (1.+beta_boost)*gamma_boost ); - myBFD.reset(new BoostedFrameDiagnostic(zmin_lab, + myBFD.reset(new BackTransformedDiagnostic(zmin_lab, zmax_lab, moving_window_v, dt_snapshots_lab, num_snapshots_lab, diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 8571c74ad..b9ab20197 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -26,7 +26,7 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, { charge = 1.0; mass = std::numeric_limits::max(); - do_boosted_frame_diags = 0; + do_back_transformed_diagnostics = 0; ParmParse pp(laser_name); diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 949173052..30f7354d0 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -162,9 +162,9 @@ public: int nSpecies() const {return nspecies;} - int nSpeciesBoostedFrameDiags() const {return nspecies_boosted_frame_diags;} - int mapSpeciesBoostedFrameDiags(int i) const {return map_species_boosted_frame_diags[i];} - int doBoostedFrameDiags() const {return do_boosted_frame_diags;} + int nSpeciesBackTransformedDiagnostics() const {return nspecies_back_transformed_diagnostics;} + int mapSpeciesBackTransformedDiagnostics(int i) const {return map_species_back_transformed_diagnostics[i];} + int doBackTransformedDiagnostics() const {return do_back_transformed_diagnostics;} int nSpeciesDepositOnMainGrid () const { bool const onMainGrid = true; @@ -236,12 +236,12 @@ private: void mapSpeciesProduct (); int getSpeciesID (std::string product_str); - // Number of species dumped in BoostedFrameDiagnostics - int nspecies_boosted_frame_diags = 0; - // map_species_boosted_frame_diags[i] is the species ID in - // MultiParticleContainer for 0 map_species_boosted_frame_diags; - int do_boosted_frame_diags = 0; + // Number of species dumped in BackTransformedDiagnostics + int nspecies_back_transformed_diagnostics = 0; + // map_species_back_transformed_diagnostics[i] is the species ID in + // MultiParticleContainer for 0 map_species_back_transformed_diagnostics; + int do_back_transformed_diagnostics = 0; // runtime parameters int nlasers = 0; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index f4c00404b..63aa500e9 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -38,14 +38,14 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core) // Compute the number of species for which lab-frame data is dumped // nspecies_lab_frame_diags, and map their ID to MultiParticleContainer // particle IDs in map_species_lab_diags. - map_species_boosted_frame_diags.resize(nspecies); - nspecies_boosted_frame_diags = 0; + map_species_back_transformed_diagnostics.resize(nspecies); + nspecies_back_transformed_diagnostics = 0; for (int i=0; ido_boosted_frame_diags){ - map_species_boosted_frame_diags[nspecies_boosted_frame_diags] = i; - do_boosted_frame_diags = 1; - nspecies_boosted_frame_diags += 1; + if (pc->do_back_transformed_diagnostics){ + map_species_back_transformed_diagnostics[nspecies_back_transformed_diagnostics] = i; + do_back_transformed_diagnostics = 1; + nspecies_back_transformed_diagnostics += 1; } } } @@ -387,8 +387,8 @@ MultiParticleContainer BL_PROFILE("MultiParticleContainer::GetLabFrameData"); // Loop over particle species - for (int i = 0; i < nspecies_boosted_frame_diags; ++i){ - int isp = map_species_boosted_frame_diags[i]; + for (int i = 0; i < nspecies_back_transformed_diagnostics; ++i){ + int isp = map_species_back_transformed_diagnostics[i]; WarpXParticleContainer* pc = allcontainers[isp].get(); WarpXParticleContainer::DiagnosticParticles diagnostic_particles; pc->GetParticleSlice(direction, z_old, z_new, t_boost, t_lab, dt, diagnostic_particles); @@ -603,9 +603,9 @@ namespace } // --- product runtime attribs GpuArray runtime_attribs_product; - bool do_boosted_product = WarpX::do_boosted_frame_diagnostic - && pc_product->DoBoostedFrameDiags(); - if (do_boosted_product) { + bool do_back_transformed_product = WarpX::do_back_transformed_diagnostics + && pc_product->doBackTransformedDiagnostics(); + if (do_back_transformed_product) { std::map comps_product = pc_product->getParticleComps(); runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; runtime_attribs_product[1] = soa_product.GetRealData(comps_product[ "yold"]).data() + np_product_old; @@ -652,7 +652,7 @@ namespace // Update xold etc. if boosted frame diagnostics required // for product species. Fill runtime attribs with a copy of // current properties (xold = x etc.). - if (do_boosted_product) { + if (do_back_transformed_product) { runtime_attribs_product[0][ip] = p_source.pos(0); runtime_attribs_product[1][ip] = p_source.pos(1); runtime_attribs_product[2][ip] = p_source.pos(2); diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index cf2ffbec4..612da01ca 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -73,7 +73,7 @@ PhotonParticleContainer::PushPX(WarpXParIter& pti, const ParticleReal* const AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); const ParticleReal* const AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + if (WarpX::do_back_transformed_diagnostics && do_back_transformed_diagnostics) { copy_attribs(pti, x, y, z); } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d12a4dbff..d451ed32b 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -43,7 +43,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp.query("do_continuous_injection", do_continuous_injection); // Whether to plot back-transformed (lab-frame) diagnostics // for this species. - pp.query("do_boosted_frame_diags", do_boosted_frame_diags); + pp.query("do_back_transformed_diagnostics", do_back_transformed_diagnostics); pp.query("do_field_ionization", do_field_ionization); @@ -86,7 +86,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp //variable to set plot_flags size int plot_flag_size = PIdx::nattribs; - if(WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + if(WarpX::do_back_transformed_diagnostics && do_back_transformed_diagnostics) plot_flag_size += 6; #ifdef WARPX_QED @@ -1078,7 +1078,7 @@ PhysicalParticleContainer::Evolve (int lev, bool has_buffer = cEx || cjx; - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + if (WarpX::do_back_transformed_diagnostics && do_back_transformed_diagnostics) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { @@ -1585,7 +1585,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, const ParticleReal* const AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); const ParticleReal* const AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags && (a_dt_type!=DtType::SecondHalf)) + if (WarpX::do_back_transformed_diagnostics && do_back_transformed_diagnostics && (a_dt_type!=DtType::SecondHalf)) { copy_attribs(pti, x, y, z); } @@ -1842,7 +1842,7 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real // Note the the slice should always move in the negative boost direction. AMREX_ALWAYS_ASSERT(z_new < z_old); - AMREX_ALWAYS_ASSERT(do_boosted_frame_diags == 1); + AMREX_ALWAYS_ASSERT(do_back_transformed_diagnostics == 1); const int nlevs = std::max(0, finestLevel()+1); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 879f4782e..8930cb176 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -273,7 +273,7 @@ public: AddIntComp(comm); } - int DoBoostedFrameDiags () const { return do_boosted_frame_diags; } + int doBackTransformedDiagnostics () const { return do_back_transformed_diagnostics; } virtual void buildIonizationMask (const amrex::MFIter& mfi, const int lev, amrex::Gpu::ManagedDeviceVector& ionization_mask) @@ -316,7 +316,7 @@ protected: amrex::Gpu::ManagedVector adk_exp_prefactor; std::string physical_element; - int do_boosted_frame_diags = 1; + int do_back_transformed_diagnostics = 1; #ifdef WARPX_QED bool do_qed = false; diff --git a/Source/WarpX.H b/Source/WarpX.H index 6ecc584a2..0084ba64c 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -4,7 +4,7 @@ #include "WarpXDtType.H" #include #include -#include +#include #include #include @@ -100,12 +100,12 @@ public: static bool serialize_ics; // Back transformation diagnostic - static bool do_boosted_frame_diagnostic; + static bool do_back_transformed_diagnostics; static std::string lab_data_directory; static int num_snapshots_lab; static amrex::Real dt_snapshots_lab; - static bool do_boosted_frame_fields; - static bool do_boosted_frame_particles; + static bool do_back_transformed_fields; + static bool do_back_transformed_particles; // Boosted frame parameters static amrex::Real gamma_boost; @@ -478,7 +478,7 @@ private: std::unique_ptr mypc; // Boosted Frame Diagnostics - std::unique_ptr myBFD; + std::unique_ptr myBFD; // // Fields: First array for level, second for direction diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 9ba0741bb..b329f5252 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -61,12 +61,12 @@ int WarpX::num_mirrors = 0; int WarpX::sort_int = -1; -bool WarpX::do_boosted_frame_diagnostic = false; +bool WarpX::do_back_transformed_diagnostics = false; std::string WarpX::lab_data_directory = "lab_frame_data"; int WarpX::num_snapshots_lab = std::numeric_limits::lowest(); Real WarpX::dt_snapshots_lab = std::numeric_limits::lowest(); -bool WarpX::do_boosted_frame_fields = true; -bool WarpX::do_boosted_frame_particles = true; +bool WarpX::do_back_transformed_fields = true; +bool WarpX::do_back_transformed_particles = true; int WarpX::num_slice_snapshots_lab = 0; Real WarpX::dt_slice_snapshots_lab; @@ -168,7 +168,7 @@ WarpX::WarpX () current_injection_position = geom[0].ProbLo(moving_window_dir); } } - do_boosted_frame_particles = mypc->doBoostedFrameDiags(); + do_back_transformed_particles = mypc->doBackTransformedDiagnostics(); Efield_aux.resize(nlevs_max); Bfield_aux.resize(nlevs_max); @@ -324,8 +324,8 @@ WarpX::ReadParameters () moving_window_v *= PhysConst::c; } - pp.query("do_boosted_frame_diagnostic", do_boosted_frame_diagnostic); - if (do_boosted_frame_diagnostic) { + pp.query("do_back_transformed_diagnostics", do_back_transformed_diagnostics); + if (do_back_transformed_diagnostics) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(gamma_boost > 1.0, "gamma_boost must be > 1 to use the boosted frame diagnostic."); @@ -353,7 +353,7 @@ WarpX::ReadParameters () pp.get("gamma_boost", gamma_boost); - pp.query("do_boosted_frame_fields", do_boosted_frame_fields); + pp.query("do_back_transformed_fields", do_back_transformed_fields); AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_moving_window, "The moving window should be on if using the boosted frame diagnostic."); @@ -600,7 +600,7 @@ WarpX::ReadParameters () } } - if (do_boosted_frame_diagnostic) { + if (do_back_transformed_diagnostics) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(gamma_boost > 1.0, "gamma_boost must be > 1 to use the boost frame diagnostic"); pp.query("num_slice_snapshots_lab", num_slice_snapshots_lab); diff --git a/Tools/performance_tests/automated_test_4_labdiags_2ppc b/Tools/performance_tests/automated_test_4_labdiags_2ppc index 282c93331..bac071f63 100644 --- a/Tools/performance_tests/automated_test_4_labdiags_2ppc +++ b/Tools/performance_tests/automated_test_4_labdiags_2ppc @@ -35,7 +35,7 @@ warpx.gamma_boost = 15. warpx.boost_direction = z # Diagnostics -warpx.do_boosted_frame_diagnostic = 1 +warpx.do_back_transformed_diagnostics = 1 warpx.num_snapshots_lab = 20 warpx.dt_snapshots_lab = 7.0e-14 -- cgit v1.2.3 From edaca2ba13564df6227dd399851c1f26b47d69bf Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 28 Oct 2019 16:47:25 -0700 Subject: replace 'boosted frame' with 'back-transformed' diags --- Source/Particles/MultiParticleContainer.cpp | 8 ++++---- Source/Particles/ParticleCreation/CopyParticle.H | 8 ++++---- Source/Particles/ParticleCreation/ElementaryProcess.H | 18 +++++++++--------- 3 files changed, 17 insertions(+), 17 deletions(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 4e5f12420..9cf535131 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -596,14 +596,14 @@ MultiParticleContainer::doFieldIonization () amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); // Create particles in pc_product - int do_boost = WarpX::do_boosted_frame_diagnostic - && pc_product->DoBoostedFrameDiags(); - amrex::Gpu::ManagedDeviceVector v_do_boosted_product{do_boost}; + int do_boost = WarpX::do_back_transformed_diagnostics + && pc_product->doBackTransformedDiagnostics(); + amrex::Gpu::ManagedDeviceVector v_do_back_transformed_product{do_boost}; const amrex::Vector v_pc_product {pc_product.get()}; // Copy source to product particles, and increase ionization // level of source particle ionization_process.createParticles(lev, mfi, pc_source, v_pc_product, - is_ionized, v_do_boosted_product); + is_ionized, v_do_back_transformed_product); } } // lev } // pc_source diff --git a/Source/Particles/ParticleCreation/CopyParticle.H b/Source/Particles/ParticleCreation/CopyParticle.H index fb3eca86b..bd2d530fb 100644 --- a/Source/Particles/ParticleCreation/CopyParticle.H +++ b/Source/Particles/ParticleCreation/CopyParticle.H @@ -21,7 +21,7 @@ public: // ID of MPI rank int m_cpuid; // If true, will copy old attribs for back-transforme diagnostics - bool m_do_boosted_product; + bool m_do_back_transformed_product; // Source old (runtime) attribs for back-transformed diagnostics amrex::GpuArray m_runtime_uold_source; // Source attribs @@ -34,14 +34,14 @@ public: // Simple constructor AMREX_GPU_HOST_DEVICE copyParticle( - const int cpuid, const int do_boosted_product, + const int cpuid, const int do_back_transformed_product, const amrex::GpuArray runtime_uold_source, const amrex::GpuArray attribs_source, const amrex::GpuArray attribs_product, const amrex::GpuArray runtime_attribs_product) : m_cpuid(cpuid), - m_do_boosted_product(do_boosted_product), + m_do_back_transformed_product(do_back_transformed_product), m_runtime_uold_source(runtime_uold_source), m_attribs_source(attribs_source), m_attribs_product(attribs_product), @@ -76,7 +76,7 @@ public: // Update xold etc. if boosted frame diagnostics required // for product species. Fill runtime attribs with a copy of // current properties (xold = x etc.). - if (m_do_boosted_product) { + if (m_do_back_transformed_product) { m_runtime_attribs_product[0][ip] = p_source.pos(0); m_runtime_attribs_product[1][ip] = p_source.pos(1); m_runtime_attribs_product[2][ip] = p_source.pos(2); diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index e9143ed5a..bca3f3679 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -30,21 +30,21 @@ public: * \brief initialize and return functor for particle copy * * \param cpuid id of MPI rank - * \param do_boosted_product whether to copy old attribs + * \param do_back_transformed_product whether to copy old attribs * \param runtime_uold_source momentum of source particles * \param attribs_source attribs of source particles * \param attribs_product attribs of product particles * \param runtime_attribs_product runtime attribs for product, to store old attribs */ copyParticle initialize_copy( - const int cpuid, const int do_boosted_product, + const int cpuid, const int do_back_transformed_product, const amrex::GpuArray runtime_uold_source, const amrex::GpuArray attribs_source, const amrex::GpuArray attribs_product, const amrex::GpuArray runtime_attribs_product) const noexcept { return copyParticle ( - cpuid, do_boosted_product, runtime_uold_source, + cpuid, do_back_transformed_product, runtime_uold_source, attribs_source, attribs_product, runtime_attribs_product); }; @@ -59,7 +59,7 @@ public: * \param pc_source source particle container * \param v_pc_product vector of product particle containers * \param is_flagged particles for which is_flagged is 1 are copied - * \param v_do_boosted_product vector: whether to copy old attribs for + * \param v_do_back_transformed_product vector: whether to copy old attribs for * each product container */ void createParticles ( @@ -67,7 +67,7 @@ public: std::unique_ptr< WarpXParticleContainer>& pc_source, amrex::Vector v_pc_product, amrex::Gpu::ManagedDeviceVector& is_flagged, - amrex::Gpu::ManagedDeviceVector v_do_boosted_product) + amrex::Gpu::ManagedDeviceVector v_do_back_transformed_product) { BL_PROFILE("createParticles"); @@ -100,8 +100,8 @@ public: int nproducts = v_pc_product.size(); AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - v_do_boosted_product.size() == nproducts, - "v_pc_product and v_do_boosted_product must have the same size."); + v_do_back_transformed_product.size() == nproducts, + "v_pc_product and v_do_back_transformed_product must have the same size."); // Indices of product particle for each source particle. // i_product[i]-1 is the location in product tile of product particle @@ -149,7 +149,7 @@ public: } // --- product runtime attribs amrex::GpuArray runtime_attribs_product; - const int do_boost = v_do_boosted_product[iproduct]; + const int do_boost = v_do_back_transformed_product[iproduct]; if (do_boost) { std::map comps_product = pc_product->getParticleComps(); runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; @@ -173,7 +173,7 @@ public: // Create instance of copy functor, and add it to the vector v_copy_functor.push_back (initialize_copy( - cpuid, v_do_boosted_product[iproduct], + cpuid, v_do_back_transformed_product[iproduct], runtime_uold_source, attribs_source, attribs_product, -- cgit v1.2.3 From 02a5a2f105d612508df817d98781a5d628624aea Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 29 Oct 2019 10:50:30 -0700 Subject: call StreamSynchronize at the end of functions where arrays are allocated on GPU --- Source/Particles/MultiParticleContainer.cpp | 1 + Source/Particles/ParticleCreation/ElementaryProcess.H | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 9cf535131..0dbc55676 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -604,6 +604,7 @@ MultiParticleContainer::doFieldIonization () // level of source particle ionization_process.createParticles(lev, mfi, pc_source, v_pc_product, is_ionized, v_do_back_transformed_product); + Gpu::streamSynchronize(); } } // lev } // pc_source diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index bca3f3679..cfd9d1d62 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -185,6 +185,7 @@ public: copyAndTransformParticles(is_flagged, i_product, np_source, v_pid_product, v_particles_product, particles_source, v_copy_functor, runtime_iattribs_source); + amrex::Gpu::streamSynchronize(); } /** @@ -244,7 +245,7 @@ public: } } ); - amrex::Gpu::streamSynchronize(); + } }; -- cgit v1.2.3 From 102a6a070ab0c068b8cb9a4704b421772edcbe9e Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 29 Oct 2019 12:05:07 -0700 Subject: add comments to explain cudaSynchroniwe --- Source/Particles/MultiParticleContainer.cpp | 2 ++ Source/Particles/ParticleCreation/ElementaryProcess.H | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'Source/Particles/MultiParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 0dbc55676..de2b4583d 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -604,6 +604,8 @@ MultiParticleContainer::doFieldIonization () // level of source particle ionization_process.createParticles(lev, mfi, pc_source, v_pc_product, is_ionized, v_do_back_transformed_product); + // Synchronize to prevent the destruction of temporary arrays (at the + // end of the function call) before the kernel executes. Gpu::streamSynchronize(); } } // lev diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index cfd9d1d62..fa791c244 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -185,6 +185,8 @@ public: copyAndTransformParticles(is_flagged, i_product, np_source, v_pid_product, v_particles_product, particles_source, v_copy_functor, runtime_iattribs_source); + // Synchronize to prevent the destruction of temporary arrays (at the + // end of the function call) before the kernel executes. amrex::Gpu::streamSynchronize(); } @@ -245,7 +247,6 @@ public: } } ); - } }; -- cgit v1.2.3