diff options
Diffstat (limited to 'Source')
| -rw-r--r-- | Source/Diagnostics/BoundaryScrapingDiagnostics.H | 63 | ||||
| -rw-r--r-- | Source/Diagnostics/BoundaryScrapingDiagnostics.cpp | 167 | ||||
| -rw-r--r-- | Source/Diagnostics/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | Source/Diagnostics/Diagnostics.H | 8 | ||||
| -rw-r--r-- | Source/Diagnostics/Make.package | 1 | ||||
| -rw-r--r-- | Source/Diagnostics/MultiDiagnostics.H | 2 | ||||
| -rw-r--r-- | Source/Diagnostics/MultiDiagnostics.cpp | 4 | ||||
| -rw-r--r-- | Source/Diagnostics/WarpXOpenPMD.cpp | 33 | ||||
| -rw-r--r-- | Source/Particles/ParticleBoundaryBuffer.H | 2 | ||||
| -rw-r--r-- | Source/Particles/ParticleBoundaryBuffer.cpp | 14 |
10 files changed, 278 insertions, 17 deletions
diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.H b/Source/Diagnostics/BoundaryScrapingDiagnostics.H new file mode 100644 index 000000000..cb321b54b --- /dev/null +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.H @@ -0,0 +1,63 @@ +/* Copyright 2022 Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_BOUNDARYSCRAPINGDIAGNOSTICS_H_ +#define WARPX_BOUNDARYSCRAPINGDIAGNOSTICS_H_ + +#include "Diagnostics.H" + +#include <string> + +/** collect the particles that are absorbed at the embedded boundary, throughout the simulation + */ +class +BoundaryScrapingDiagnostics final : public Diagnostics +{ +public: + + /** Constructor + * + * @param i index of diagnostics in MultiDiagnostics::alldiags + * @param name diagnostics name in the inputs file + */ + BoundaryScrapingDiagnostics (int i, std::string name); + +private: + /** Read relevant parameters for BoundaryScraping */ + void ReadParameters (); + /** \brief Flush data to file. */ + void Flush (int i_buffer) override; + /** \brief Return whether to dump data to file at this time step. + * (i.e. whether to call Flush) + * + * \param[in] step current time step + * \param[in] i_buffer used only for BackTransformedDiagnostics. + * For BoundaryScrapingDiagnostics, this is always 0 + * \param[in] force_flush if true, return true for any step + */ + bool DoDump (int step, int i_buffer, bool force_flush=false) override; + /** Return whether to pack field data in output buffers at this time step + * + * This is not used for BoundaryScrapingDiagnostics: no field to output + */ + bool DoComputeAndPack (int step, bool force_flush=false) override; + /** Initialize buffers that contain field data + * + * This is not used for BoundaryScrapingDiagnostics: no field to output + */ + void InitializeBufferData (int i_buffer, int lev) override; + /** Initialize functors that point to the fields requested by the user. + * + * This is not used for BoundaryScrapingDiagnostics: no field to output + */ + void InitializeFieldFunctors (int lev) override; + /** Initialize the diagnostics by pointing to the `ParticleBoundaryBuffer` + * that correspond to the species requested by the user. + */ + void InitializeParticleBuffer () override; + +}; +#endif // WARPX_BOUNDARYSCRAPINGDIAGNOSTICS_H_ diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp new file mode 100644 index 000000000..19beec6ab --- /dev/null +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -0,0 +1,167 @@ +/* Copyright 2022 Remi Lehe + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "BoundaryScrapingDiagnostics.H" +#include "ComputeDiagFunctors/ComputeDiagFunctor.H" +#include "Diagnostics/Diagnostics.H" +#include "Diagnostics/FlushFormats/FlushFormat.H" +#include "Particles/ParticleBoundaryBuffer.H" +#include "WarpX.H" + +#include <AMReX.H> + +#include <set> +#include <string> + +using namespace amrex::literals; + +BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, std::string name) + : Diagnostics(i, name) +{ + ReadParameters(); +} + +void +BoundaryScrapingDiagnostics::ReadParameters () +{ + BaseReadParameters(); + + // Modify some of the quantities that were initialized by default + // in the function `BaseReadParameters` + m_varnames_fields = {}; // No fields in boundary scraping diagnostics + m_varnames = {}; // No fields in boundary scraping diagnostics + + // Number of buffers = 1 for BoundaryScrapingDiagnostics. + // (buffers are used in BTDiagnostics, and correspond to different snapshots) + m_num_buffers = 1; + + // Do a few checks +#ifndef AMREX_USE_EB + amrex::Abort("You need to compile WarpX with Embedded Boundary (EB) support, in order to use BoundaryScrapingDiagnostic: -DWarpX_EB=ON"); +#endif +#ifndef WARPX_USE_OPENPMD + amrex::Abort("You need to compile WarpX with openPMD support, in order to use BoundaryScrapingDiagnostic: -DWarpX_OPENPMD=ON"); +#endif + + // Check that saving at EB has been activated for each requested species + std::set<std::string> particle_saving_activated; + for (auto const& species_name : m_output_species_names){ + amrex::ParmParse pp(species_name); + bool save_particles_at_eb; + pp.query("save_particles_at_eb", save_particles_at_eb); + if (save_particles_at_eb == false) particle_saving_activated.insert(species_name); + } + std::string error_string = "You need to set " + "you need to set:\n"; + for (auto const& species_name : particle_saving_activated){ + error_string + .append(" ") + .append(species_name) + .append("save_particles_at_eb=1\n"); + } + error_string.append("in order to use for the BoundaryScrapingDiagnostic."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + particle_saving_activated.size() == 0u, + error_string); + + // Check that the output format is openPMD + error_string = std::string("You need to set `") + .append(m_diag_name) + .append(".format=openpmd` for the BoundaryScrapingDiagnostic."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_format == "openpmd", + error_string); +} + +void +BoundaryScrapingDiagnostics::InitializeFieldFunctors (int /*lev*/) +{ + // This function is usually used for field output + // Nothing to do here for boundary scraping output, + // since it only outputs particles +} + +void +BoundaryScrapingDiagnostics::InitializeBufferData (int /*i_buffer*/, int /*lev*/) +{ + // This function is usually used for field output + // Nothing to do here for boundary scraping output, + // since it only outputs particles +} + +void +BoundaryScrapingDiagnostics::InitializeParticleBuffer () +{ + auto & warpx = WarpX::GetInstance(); + const MultiParticleContainer& mpc = warpx.GetPartContainer(); + + // If the user does not specify any species, dump all species + if (m_output_species_names.empty()) { + m_output_species_names = mpc.GetSpeciesNames(); + } + + // Initialize one ParticleDiag per species requested + ParticleBoundaryBuffer& particle_buffer = warpx.GetParticleBoundaryBuffer(); + for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { + for (auto const& species_name : m_output_species_names){ + // `particle_buffer` contains buffers for all boundaries + // here we select the one for the EB (index: AMREX_SPACEDIM*2) + WarpXParticleContainer* pc = &mpc.GetParticleContainerFromName(species_name); + PinnedMemoryParticleContainer* eb_buffer = particle_buffer.getParticleBufferPointer(species_name, AMREX_SPACEDIM*2); + m_output_species[i_buffer].push_back(ParticleDiag(m_diag_name, species_name, pc, eb_buffer)); + } + } + // Initialize total number of particles flushed + m_totalParticles_flushed_already.resize(m_num_buffers); + for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { + int const n_species = m_output_species_names.size(); + m_totalParticles_flushed_already[i_buffer].resize(n_species); + for (int i_species=0; i_species<n_species; i_species++) { + m_totalParticles_flushed_already[i_buffer][i_species] = 0; + } + } +} + +bool +BoundaryScrapingDiagnostics::DoComputeAndPack (int /*step*/, bool /*force_flush*/) +{ + return false; +} + +bool +BoundaryScrapingDiagnostics::DoDump (int /*step*/, int /*i_buffer*/, bool force_flush) +{ + if (force_flush) { + return true; + } else { + return false; + } +} + +void +BoundaryScrapingDiagnostics::Flush (int i_buffer) +{ + auto & warpx = WarpX::GetInstance(); + + // This is not a backtransform diagnostics, but we still set the flag `isBTD` + // This enables: + // - writing the data that was accumulated in a PinnedMemoryParticleContainer + // - writing repeatedly to the same file + bool const isBTD = true; + // For now, because this function is currently only called at the very end + // of the simulation for BoundaryScrapingDiagnostics, we always set `isLastBTD`. + // This tells WarpX to write all the metadata (and not purely the particle data) + bool const isLastBTD = true; + const amrex::Geometry& geom = warpx.Geom(0); // For compatibility with `WriteToFile` ; not used + + m_flush_format->WriteToFile( + m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), + 0., m_output_species[i_buffer], nlev_output, m_file_prefix, + m_file_min_digits, false, false, isBTD, i_buffer, geom, + isLastBTD, m_totalParticles_flushed_already[i_buffer]); + +} diff --git a/Source/Diagnostics/CMakeLists.txt b/Source/Diagnostics/CMakeLists.txt index e3f76df6a..4df0f68f8 100644 --- a/Source/Diagnostics/CMakeLists.txt +++ b/Source/Diagnostics/CMakeLists.txt @@ -10,6 +10,7 @@ target_sources(WarpX WarpXIO.cpp WarpXOpenPMD.cpp BTDiagnostics.cpp + BoundaryScrapingDiagnostics.cpp BTD_Plotfile_Header_Impl.cpp ) diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 2b849099f..e57447fea 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -30,14 +30,22 @@ class Diagnostics { public: + /** Constructor + * + * @param i index of diagnostics in MultiDiagnostics::alldiags + * @param name diagnostics name in the inputs file + */ Diagnostics (int i, std::string name); + /** Virtual Destructor to handle clean destruction of derived classes */ virtual ~Diagnostics (); + /** Pack (stack) all fields in the cell-centered output MultiFab m_mf_output. * * Fields are computed (e.g., cell-centered or back-transformed) on-the-fly using a functor. */ void ComputeAndPack (); + /** \brief Flush particle and field buffers to file using the FlushFormat member variable. * * This function should belong to class Diagnostics and not be virtual, as it flushes diff --git a/Source/Diagnostics/Make.package b/Source/Diagnostics/Make.package index 7c5bf822c..f8bf69f98 100644 --- a/Source/Diagnostics/Make.package +++ b/Source/Diagnostics/Make.package @@ -7,6 +7,7 @@ CEXE_sources += ParticleIO.cpp CEXE_sources += FieldIO.cpp CEXE_sources += SliceDiagnostic.cpp CEXE_sources += BTDiagnostics.cpp +CEXE_sources += BoundaryScrapingDiagnostics.cpp CEXE_sources += BTD_Plotfile_Header_Impl.cpp ifeq ($(USE_OPENPMD), TRUE) diff --git a/Source/Diagnostics/MultiDiagnostics.H b/Source/Diagnostics/MultiDiagnostics.H index d5ccf2113..06a2b87b4 100644 --- a/Source/Diagnostics/MultiDiagnostics.H +++ b/Source/Diagnostics/MultiDiagnostics.H @@ -12,7 +12,7 @@ #include <vector> /** All types of diagnostics. */ -enum struct DiagTypes {Full, BackTransformed}; +enum struct DiagTypes {Full, BackTransformed, BoundaryScraping}; /** * \brief This class contains a vector of all diagnostics in the simulation. diff --git a/Source/Diagnostics/MultiDiagnostics.cpp b/Source/Diagnostics/MultiDiagnostics.cpp index da7bf2b00..6644f7b66 100644 --- a/Source/Diagnostics/MultiDiagnostics.cpp +++ b/Source/Diagnostics/MultiDiagnostics.cpp @@ -2,6 +2,7 @@ #include "Diagnostics/BTDiagnostics.H" #include "Diagnostics/FullDiagnostics.H" +#include "Diagnostics/BoundaryScrapingDiagnostics.H" #include "Utils/TextMsg.H" #include <AMReX_ParmParse.H> @@ -28,6 +29,8 @@ MultiDiagnostics::MultiDiagnostics () #else alldiags[i] = std::make_unique<BTDiagnostics>(i, diags_names[i]); #endif + } else if ( diags_types[i] == DiagTypes::BoundaryScraping ){ + alldiags[i] = std::make_unique<BoundaryScrapingDiagnostics>(i, diags_names[i]); } else { amrex::Abort(Utils::TextMsg::Err("Unknown diagnostic type")); } @@ -70,6 +73,7 @@ MultiDiagnostics::ReadParameters () pp_diag_name.get("diag_type", diag_type_str); if (diag_type_str == "Full") diags_types[i] = DiagTypes::Full; if (diag_type_str == "BackTransformed") diags_types[i] = DiagTypes::BackTransformed; + if (diag_type_str == "BoundaryScraping") diags_types[i] = DiagTypes::BoundaryScraping; } } diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index c43e7a2f3..c5d882083 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -556,8 +556,15 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector<ParticleDiag>& part WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDParticles()"); for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { + WarpXParticleContainer* pc = particle_diags[i].getParticleContainer(); - auto tmp = pc->make_alike<amrex::PinnedArenaAllocator>(); + PinnedMemoryParticleContainer* pinned_pc = particle_diags[i].getPinnedParticleContainer(); + PinnedMemoryParticleContainer tmp; + if (! isBTD) { + tmp = pc->make_alike<amrex::PinnedArenaAllocator>(); + } else { + tmp = pinned_pc->make_alike<amrex::PinnedArenaAllocator>(); + } // names of amrex::Real and int particle attributes in SoA data amrex::Vector<std::string> real_names; amrex::Vector<std::string> int_names; @@ -568,40 +575,34 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector<ParticleDiag>& part // note: an underscore separates the record name from its component // for non-scalar records real_names.push_back("weighting"); - real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); - #ifdef WARPX_DIM_RZ real_names.push_back("theta"); #endif - // get the names of the real comps - real_names.resize(pc->NumRealComps()); - auto runtime_rnames = pc->getParticleRuntimeComps(); + real_names.resize(tmp.NumRealComps()); + auto runtime_rnames = tmp.getParticleRuntimeComps(); for (auto const& x : runtime_rnames) { real_names[x.second+PIdx::nattribs] = detail::snakeToCamel(x.first); } - // plot any "extra" fields by default real_flags = particle_diags[i].plot_flags; - real_flags.resize(pc->NumRealComps(), 1); - + real_flags.resize(tmp.NumRealComps(), 1); // and the names - int_names.resize(pc->NumIntComps()); - auto runtime_inames = pc->getParticleRuntimeiComps(); + int_names.resize(tmp.NumIntComps()); + auto runtime_inames = tmp.getParticleRuntimeiComps(); for (auto const& x : runtime_inames) { int_names[x.second+0] = detail::snakeToCamel(x.first); } - // plot by default - int_flags.resize(pc->NumIntComps(), 1); + int_flags.resize(tmp.NumIntComps(), 1); - pc->ConvertUnits(ConvertDirection::WarpX_to_SI); + pc->ConvertUnits(ConvertDirection::WarpX_to_SI); RandomFilter const random_filter(particle_diags[i].m_do_random_filter, particle_diags[i].m_random_fraction); UniformFilter const uniform_filter(particle_diags[i].m_do_uniform_filter, @@ -624,7 +625,6 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector<ParticleDiag>& part * parser_filter(p, engine) * geometry_filter(p, engine); }, true); } else if (isBTD) { - PinnedMemoryParticleContainer* pinned_pc = particle_diags[i].getPinnedParticleContainer(); tmp.SetParticleGeometry(0,pinned_pc->Geom(0)); tmp.SetParticleBoxArray(0,pinned_pc->ParticleBoxArray(0)); tmp.SetParticleDistributionMap(0, pinned_pc->ParticleDistributionMap(0)); @@ -1321,6 +1321,9 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename, series_iteration.setTime( time ); } + // If there are no fields to be written, interrupt the function here + if ( varnames.size()==0 ) return; + // loop over levels up to output_levels // note: this is usually the finestLevel, not the maxLevel for (int lev=0; lev < output_levels; lev++) { diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index f85b34044..a880512ea 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -46,6 +46,8 @@ public: PinnedMemoryParticleContainer& getParticleBuffer(const std::string species_name, int boundary); + PinnedMemoryParticleContainer* getParticleBufferPointer(const std::string species_name, int boundary); + static constexpr int numBoundaries () { return AMREX_SPACEDIM*2 #ifdef AMREX_USE_EB diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index f831b31be..86475aa05 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -172,7 +172,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, for (int i = 0; i < numSpecies(); ++i) { if (!m_do_boundary_buffer[2*idim+iside][i]) continue; - const auto& pc = mypc.GetParticleContainer(i); + const WarpXParticleContainer& pc = mypc.GetParticleContainer(i); if (!buffer[i].isDefined()) { buffer[i] = pc.make_alike<amrex::PinnedArenaAllocator>(); @@ -325,3 +325,15 @@ ParticleBoundaryBuffer::getParticleBuffer(const std::string species_name, int bo return buffer[index]; } + +PinnedMemoryParticleContainer * +ParticleBoundaryBuffer::getParticleBufferPointer(const std::string species_name, int boundary) { + + auto& buffer = m_particle_containers[boundary]; + auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_do_boundary_buffer[boundary][index], + "Attempted to get particle buffer for boundary " + + std::to_string(boundary) + ", which is not used!"); + return &buffer[index]; +} |