aboutsummaryrefslogtreecommitdiff
path: root/Source
diff options
context:
space:
mode:
Diffstat (limited to 'Source')
-rw-r--r--Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp2
-rw-r--r--Source/Diagnostics/BTDiagnostics.H7
-rw-r--r--Source/Diagnostics/BTDiagnostics.cpp50
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormat.H5
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatAscent.H5
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp3
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H5
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp5
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H5
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp17
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H5
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp4
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatSensei.H5
-rw-r--r--Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp4
-rw-r--r--Source/Diagnostics/WarpXOpenPMD.H11
-rw-r--r--Source/Diagnostics/WarpXOpenPMD.cpp209
16 files changed, 223 insertions, 119 deletions
diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp
index 9d2a6facc..90a0c8317 100644
--- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp
+++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp
@@ -106,7 +106,7 @@ BTDPlotfileHeaderImpl::WriteHeader ()
HeaderFile.precision(17);
- // Genetic Plotfile type name
+ // Generic Plotfile type name
HeaderFile << m_file_version << '\n';
// number of components
HeaderFile << m_varnames.size() << '\n';
diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H
index b1ca0ebd3..8eb9bf2af 100644
--- a/Source/Diagnostics/BTDiagnostics.H
+++ b/Source/Diagnostics/BTDiagnostics.H
@@ -164,6 +164,11 @@ private:
* stored in the buffer multifab is flushed out and the counter is reset is zero.
*/
amrex::Vector<int> m_buffer_counter;
+ /** Vector of maximum number of buffer multifabs that need to be flushed to
+ * generate each lab-frame snapshot. At the final flush, m_buffer_flush_counter
+ * will be equal to the predicted m_max_buffer_multifabs for each snapshot.
+ */
+ amrex::Vector<int> m_max_buffer_multifabs;
/** Vector of counters tracking number of times the buffer of multifab is
* flushed out and emptied before being refilled again for each snapshot */
amrex::Vector<int> m_buffer_flush_counter;
@@ -292,7 +297,7 @@ private:
"jx", "jy", "jz", "rho"};
/** Temporarily clear species output for BTD until particle buffer is added */
- void TMP_ClearSpeciesDataForBTD();
+ void TMP_ClearSpeciesDataForBTD() override;
void MergeBuffersForPlotfile (int i_snapshot);
void InterleaveBufferAndSnapshotHeader ( std::string buffer_Header,
diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp
index ff8d59427..3739bf21d 100644
--- a/Source/Diagnostics/BTDiagnostics.cpp
+++ b/Source/Diagnostics/BTDiagnostics.cpp
@@ -63,7 +63,11 @@ void BTDiagnostics::DerivedInitData ()
m_cell_centered_data.resize(nmax_lev);
// allocate vector of cell-center functors for nlevels
m_cell_center_functors.resize(nmax_lev);
- // allocate vector to counter number of times the buffer has been refilled
+ // allocate vector to estimate maximum number of buffer multifabs needed to
+ // obtain the lab-frame snapshot.
+ m_max_buffer_multifabs.resize(m_num_buffers);
+ // allocate vector to count number of times the buffer multifab
+ // has been flushed and refilled
m_buffer_flush_counter.resize(m_num_buffers);
// allocate vector of geometry objects corresponding to each snapshot
m_geom_snapshot.resize( m_num_buffers );
@@ -110,7 +114,11 @@ BTDiagnostics::ReadParameters ()
// Read list of back-transform diag parameters requested by the user //
amrex::ParmParse pp(m_diag_name);
- m_file_prefix = "diags/lab_frame_data/" + m_diag_name;
+ if (m_format == "openpmd") {
+ m_file_prefix = "diags/" + m_diag_name;
+ } else {
+ m_file_prefix = "diags/lab_frame_data/" + m_diag_name;
+ }
pp.query("file_prefix", m_file_prefix);
pp.query("do_back_transformed_fields", m_do_back_transformed_fields);
pp.query("do_back_transformed_particles", m_do_back_transformed_particles);
@@ -568,8 +576,26 @@ BTDiagnostics::DefineSnapshotGeometry (const int i_buffer, const int lev)
const int k_lab = k_index_zlab (i_buffer, lev);
// Box covering the extent of the user-defined diag in the back-transformed frame
// for the ith snapshot
- m_snapshot_box[i_buffer].setSmall( m_moving_window_dir, k_lab - m_snapshot_ncells_lab[i_buffer][m_moving_window_dir]);
+ // estimating the maximum number of buffer multifabs needed to obtain the
+ // full lab-frame snapshot
+ m_max_buffer_multifabs[i_buffer] = static_cast<int>( ceil (
+ amrex::Real(m_snapshot_ncells_lab[i_buffer][m_moving_window_dir]) /
+ amrex::Real(m_buffer_size) ) );
+ // number of cells in z is modified since each buffer multifab always
+ // contains a minimum m_buffer_size=256 cells
+ int num_z_cells_in_snapshot = m_max_buffer_multifabs[i_buffer] * m_buffer_size;
+ // Modify the domain indices according to the buffers that are flushed out
+ m_snapshot_box[i_buffer].setSmall( m_moving_window_dir,
+ k_lab - (num_z_cells_in_snapshot-1) );
m_snapshot_box[i_buffer].setBig( m_moving_window_dir, k_lab);
+
+ // Modifying the physical coordinates of the lab-frame snapshot to be
+ // consistent with the above modified domain-indices in m_snapshot_box.
+ amrex::IntVect ref_ratio = amrex::IntVect(1);
+ amrex::Real new_lo = m_snapshot_domain_lab[i_buffer].hi(m_moving_window_dir) -
+ num_z_cells_in_snapshot *
+ dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]);
+ m_snapshot_domain_lab[i_buffer].setLo(m_moving_window_dir, new_lo);
if (lev == 0) {
// The extent of the physical domain covered by the ith snapshot
// Default non-periodic geometry for diags
@@ -660,12 +686,20 @@ void
BTDiagnostics::Flush (int i_buffer)
{
auto & warpx = WarpX::GetInstance();
- std::string tmp_file_name = amrex::Concatenate(m_file_prefix +"/snapshots_plotfile/snapshot",i_buffer,5);
- tmp_file_name = tmp_file_name+"/buffer";
+ std::string file_name = m_file_prefix;
+ if (m_format=="plotfile") {
+ file_name = amrex::Concatenate(m_file_prefix +"/snapshots_plotfile/snapshot",i_buffer,5);
+ file_name = file_name+"/buffer";
+ }
+ bool isLastBTDFlush = ( ( m_max_buffer_multifabs[i_buffer]
+ - m_buffer_flush_counter[i_buffer]) == 1) ? true : false;
+ bool const isBTD = true;
+ double const labtime = m_t_lab[i_buffer];
m_flush_format->WriteToFile(
m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(),
- warpx.gett_new(0), m_output_species, nlev_output, tmp_file_name,
- m_plot_raw_fields, m_plot_raw_fields_guards, m_plot_raw_rho, m_plot_raw_F);
+ labtime, m_output_species, nlev_output, file_name,
+ m_plot_raw_fields, m_plot_raw_fields_guards, m_plot_raw_rho, m_plot_raw_F,
+ isBTD, i_buffer, m_geom_snapshot[i_buffer][0], isLastBTDFlush);
if (m_format == "plotfile") {
MergeBuffersForPlotfile(i_buffer);
@@ -715,7 +749,6 @@ void BTDiagnostics::TMP_ClearSpeciesDataForBTD ()
{
m_output_species.clear();
m_output_species_names.clear();
-
}
void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot)
@@ -854,4 +887,5 @@ BTDiagnostics::InterleaveFabArrayHeader(std::string Buffer_FabHeader_path,
}
snapshot_FabHeader.WriteMultiFabHeader();
+
}
diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H
index 826c56d03..800c99160 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormat.H
+++ b/Source/Diagnostics/FlushFormats/FlushFormat.H
@@ -17,7 +17,10 @@ public:
const std::string prefix,
bool plot_raw_fields,
bool plot_raw_fields_guards,
- bool plot_raw_rho, bool plot_raw_F) const = 0;
+ bool plot_raw_rho, bool plot_raw_F,
+ bool isBTD = false, int snapshotID = -1,
+ const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
+ bool isLastBTDFlush = false) const = 0;
virtual ~FlushFormat() {}
};
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H
index 357b9357a..de8bec091 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H
+++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H
@@ -26,7 +26,10 @@ public:
const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
bool plot_raw_fields,
bool plot_raw_fields_guards,
- bool plot_raw_rho, bool plot_raw_F) const override;
+ bool plot_raw_rho, bool plot_raw_F,
+ bool isBTD = false, int snapshotID = -1,
+ const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
+ bool isLastBTDFlush = false) const override;
/** \brief Do in-situ visualization for particle data.
* \param[in] particle_diags Each element of this vector handles output of 1 species.
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp
index 214655a53..15a67f036 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp
+++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp
@@ -13,7 +13,8 @@ FlushFormatAscent::WriteToFile (
const amrex::Vector<int> iteration, const double time,
const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
const std::string prefix, bool plot_raw_fields,
- bool plot_raw_fields_guards, bool /*plot_raw_rho*/, bool plot_raw_F) const
+ bool plot_raw_fields_guards, bool /*plot_raw_rho*/, bool plot_raw_F,
+ bool /*isBTD*/, int /*snapshotID*/, const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const
{
#ifdef AMREX_USE_ASCENT
auto & warpx = WarpX::GetInstance();
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H
index 9eeb3b7c8..aa2dccdfd 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H
+++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H
@@ -14,7 +14,10 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile
const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
bool plot_raw_fields,
bool plot_raw_fields_guards,
- bool plot_raw_rho, bool plot_raw_F) const override final;
+ bool plot_raw_rho, bool plot_raw_F,
+ bool isBTD = false, int snapshotID = -1,
+ const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
+ bool isLastBTDFlush = false) const override final;
void CheckpointParticles(const std::string& dir,
const amrex::Vector<ParticleDiag>& particle_diags) const;
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp
index 2c29bae42..9a570ce32 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp
+++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp
@@ -20,7 +20,10 @@ FlushFormatCheckpoint::WriteToFile (
const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
bool /*plot_raw_fields*/,
bool /*plot_raw_fields_guards*/,
- bool /*plot_raw_rho*/, bool /*plot_raw_F*/) const
+ bool /*plot_raw_rho*/, bool /*plot_raw_F*/,
+ bool /*isBTD*/, int /*snapshotID*/,
+ const amrex::Geometry& /*full_BTD_snapshot*/,
+ bool /*isLastBTDFlush*/) const
{
WARPX_PROFILE("FlushFormatCheckpoint::WriteToFile()");
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H
index baf8728dc..ac79428b4 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H
+++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H
@@ -25,7 +25,10 @@ public:
const amrex::Vector<ParticleDiag>& particle_diags, int /*nlev*/, const std::string prefix,
bool plot_raw_fields,
bool plot_raw_fields_guards,
- bool plot_raw_rho, bool plot_raw_F) const override final;
+ bool plot_raw_rho, bool plot_raw_F,
+ bool isBTD = false, int snapshotID = -1,
+ const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
+ bool isLastBTDFlush = false) const override final;
~FlushFormatOpenPMD ();
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp
index dea1c37ac..c63c3a0ad 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp
+++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp
@@ -29,7 +29,9 @@ FlushFormatOpenPMD::WriteToFile (
const amrex::Vector<int> iteration, const double time,
const amrex::Vector<ParticleDiag>& particle_diags, int /*nlev*/,
const std::string prefix, bool plot_raw_fields,
- bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F) const
+ bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F,
+ bool isBTD, int snapshotID, const amrex::Geometry& full_BTD_snapshot,
+ bool isLastBTDFlush) const
{
WARPX_PROFILE("FlushFormatOpenPMD::WriteToFile()");
@@ -37,18 +39,25 @@ FlushFormatOpenPMD::WriteToFile (
!plot_raw_fields && !plot_raw_fields_guards && !plot_raw_rho && !plot_raw_F,
"Cannot plot raw data with OpenPMD output format. Use plotfile instead.");
+ // we output at full steps of the coarsest level
+ int output_iteration = iteration[0];
+ // in backtransformed diagnostics (BTD), we dump into a series of labframe
+ // snapshots
+ if( isBTD )
+ output_iteration = snapshotID;
+
// Set step and output directory name.
- m_OpenPMDPlotWriter->SetStep(iteration[0], prefix);
+ m_OpenPMDPlotWriter->SetStep(output_iteration, prefix, isBTD);
// fields: only dumped for coarse level
m_OpenPMDPlotWriter->WriteOpenPMDFields(
- varnames, mf[0], geom[0], iteration[0], time);
+ varnames, mf[0], geom[0], output_iteration, time, isBTD, full_BTD_snapshot);
// particles: all (reside only on locally finest level)
m_OpenPMDPlotWriter->WriteOpenPMDParticles(particle_diags);
// signal that no further updates will be written to this iteration
- m_OpenPMDPlotWriter->CloseStep();
+ m_OpenPMDPlotWriter->CloseStep(isBTD, isLastBTDFlush);
}
FlushFormatOpenPMD::~FlushFormatOpenPMD (){
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H
index 15caabc59..e8f87f884 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H
+++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H
@@ -21,7 +21,10 @@ public:
const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
bool plot_raw_fields,
bool plot_raw_fields_guards,
- bool plot_raw_rho, bool plot_raw_F) const override;
+ bool plot_raw_rho, bool plot_raw_F,
+ bool isBTD = false, int snapshotID = -1,
+ const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
+ bool isLastBTDFlush = false) const override;
/** Write general info of the run into the plotfile */
void WriteJobInfo(const std::string& dir) const;
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
index 1462cfe29..8428b4ffd 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
+++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
@@ -21,7 +21,9 @@ FlushFormatPlotfile::WriteToFile (
const amrex::Vector<int> iteration, const double time,
const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
const std::string prefix, bool plot_raw_fields,
- bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F) const
+ bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F,
+ bool /*isBTD*/, int /*snapshotID*/, const amrex::Geometry& /*full_BTD_snapshot*/,
+ bool /*isLastBTDFlush*/) const
{
WARPX_PROFILE("FlushFormatPlotfile::WriteToFile()");
auto & warpx = WarpX::GetInstance();
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H
index eed95371e..163087fba 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H
+++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H
@@ -53,7 +53,10 @@ public:
const amrex::Vector<ParticleDiag>& particle_diags, int nlev, const std::string prefix,
bool plot_raw_fields,
bool plot_raw_fields_guards,
- bool plot_raw_rho, bool plot_raw_F) const override;
+ bool plot_raw_rho, bool plot_raw_F,
+ bool isBTD = false, int snapshotID = -1,
+ const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(),
+ bool isLastBTDFlush = false) const override;
/** \brief Do in-situ visualization for particle data.
* \param[in] particle_diags Each element of this vector handles output of 1 species.
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp
index ed58f120a..2871e210d 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp
+++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp
@@ -53,7 +53,9 @@ FlushFormatSensei::WriteToFile (
const amrex::Vector<int> iteration, const double time,
const amrex::Vector<ParticleDiag>& particle_diags, int nlev,
const std::string prefix, bool plot_raw_fields,
- bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F) const
+ bool plot_raw_fields_guards, bool plot_raw_rho, bool plot_raw_F,
+ bool /*isBTD*/, int /*snapshotID*/,
+ const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const
{
#ifndef BL_USE_SENSEI_INSITU
(void)varnames;
diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H
index d61bb3e28..07c4e4e17 100644
--- a/Source/Diagnostics/WarpXOpenPMD.H
+++ b/Source/Diagnostics/WarpXOpenPMD.H
@@ -106,13 +106,14 @@ public:
* @note If an iteration has been written, then it will give a warning
*
*/
- void SetStep (int ts, const std::string& filePrefix);
+ void SetStep (int ts, const std::string& filePrefix,
+ bool isBTD=false);
/** Close the step
*
* Signal that no further updates will be written for the step.
*/
- void CloseStep ();
+ void CloseStep (bool isBTD = false, bool isLastBTDFlush = false);
void WriteOpenPMDParticles (const amrex::Vector<ParticleDiag>& particle_diags);
@@ -120,11 +121,13 @@ public:
const std::vector<std::string>& varnames,
const amrex::MultiFab& mf,
const amrex::Geometry& geom,
- const int iteration, const double time ) const;
+ const int iteration, const double time,
+ bool isBTD = false,
+ const amrex::Geometry& full_BTD_snapshot=amrex::Geometry() ) const;
private:
- void Init (openPMD::Access access, const std::string& filePrefix);
+ void Init (openPMD::Access access, const std::string& filePrefix, bool isBTD);
/** This function sets up the entries for storing the particle positions, global IDs, and constant records (charge, mass)
*
diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp
index db2def7c7..6bef287a6 100644
--- a/Source/Diagnostics/WarpXOpenPMD.cpp
+++ b/Source/Diagnostics/WarpXOpenPMD.cpp
@@ -238,32 +238,43 @@ void WarpXOpenPMDPlot::GetFileName(std::string& filename)
}
-void WarpXOpenPMDPlot::SetStep (int ts, const std::string& filePrefix)
+void WarpXOpenPMDPlot::SetStep (int ts, const std::string& filePrefix,
+ bool isBTD)
{
- AMREX_ALWAYS_ASSERT_WITH_MESSAGE(ts >= 0 , "openPMD iterations are unsigned");
-
- if (m_CurrentStep >= ts) {
- // note m_Series is reset in Init(), so using m_Series->iterations.contains(ts) is only able to check the
- // last written step in m_Series's life time, but not other earlier written steps by other m_Series
- std::string warnMsg = " Warning from openPMD writer: Already written iteration:"+std::to_string(ts);
- std::cout<<warnMsg<<std::endl;
- amrex::Warning(warnMsg);
- }
-
- m_CurrentStep = ts;
- Init(openPMD::Access::CREATE, filePrefix);
-
+ AMREX_ALWAYS_ASSERT_WITH_MESSAGE(ts >= 0 , "openPMD iterations are unsigned");
+
+ if( ! isBTD ) {
+ if (m_CurrentStep >= ts) {
+ // note m_Series is reset in Init(), so using m_Series->iterations.contains(ts) is only able to check the
+ // last written step in m_Series's life time, but not other earlier written steps by other m_Series
+ std::string warnMsg =
+ " Warning from openPMD writer: Already written iteration:" + std::to_string(ts);
+ std::cout << warnMsg << std::endl;
+ amrex::Warning(warnMsg);
+ }
+ }
+ m_CurrentStep = ts;
+ Init(openPMD::Access::CREATE, filePrefix, isBTD);
}
-void WarpXOpenPMDPlot::CloseStep ()
+void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush)
{
- if (m_Series)
- m_Series->iterations[m_CurrentStep].close();
+ // default close is true
+ bool callClose = true;
+ // close BTD file only when isLastBTDFlush is true
+ if (isBTD and !isLastBTDFlush) callClose = false;
+ if (callClose) {
+ if (m_Series)
+ m_Series->iterations[m_CurrentStep].close();
+ }
}
void
-WarpXOpenPMDPlot::Init (openPMD::Access access, const std::string& filePrefix)
+WarpXOpenPMDPlot::Init (openPMD::Access access, const std::string& filePrefix, bool isBTD)
{
+ if( isBTD && m_Series != nullptr )
+ return; // already open for this snapshot (aka timestep in lab frame)
+
// either for the next ts file,
// or init a single file for all ts
std::string filename = filePrefix;
@@ -273,8 +284,7 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, const std::string& filePrefix)
// see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686
m_Series = nullptr;
- if( amrex::ParallelDescriptor::NProcs() > 1 )
- {
+ if (amrex::ParallelDescriptor::NProcs() > 1) {
#if defined(AMREX_USE_MPI)
m_Series = std::make_unique<openPMD::Series>(
filename, access,
@@ -285,9 +295,7 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, const std::string& filePrefix)
#else
amrex::Abort("openPMD-api not built with MPI support!");
#endif
- }
- else
- {
+ } else {
m_Series = std::make_unique<openPMD::Series>(filename, access);
m_MPISize = 1;
m_MPIRank = 1;
@@ -746,30 +754,39 @@ WarpXOpenPMDPlot::SetupPos(
// this is originally copied from FieldIO.cpp
//
void
-WarpXOpenPMDPlot::WriteOpenPMDFields( //const std::string& filename,
+WarpXOpenPMDPlot::WriteOpenPMDFields ( //const std::string& filename,
const std::vector<std::string>& varnames,
const amrex::MultiFab& mf,
- const amrex::Geometry& geom,
+ const amrex::Geometry& geom, // geometry of the mf/Fab
const int iteration,
- const double time ) const
+ const double time, bool isBTD,
+ const amrex::Geometry& full_BTD_snapshot ) const
{
//This is AMReX's tiny profiler. Possibly will apply it later
WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDFields()");
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD series must be initialized");
+ amrex::Geometry full_geom = geom;
+ if( isBTD )
+ full_geom = full_BTD_snapshot;
+
+ // is this either a regular write (true) or the first write in a
+ // backtransformed diagnostic (BTD):
+ bool const first_write_to_iteration = ! m_Series->iterations.contains( iteration );
+
int const ncomp = mf.nComp();
// Create a few vectors that store info on the global domain
// Swap the indices for each of them, since AMReX data is Fortran order
// and since the openPMD API assumes contiguous C order
// - Size of the box, in integer number of cells
- amrex::Box const & global_box = geom.Domain();
+ amrex::Box const & global_box = full_geom.Domain();
auto const global_size = getReversedVec(global_box.size());
// - Grid spacing
- std::vector<double> const grid_spacing = getReversedVec(geom.CellSize());
+ std::vector<double> const grid_spacing = getReversedVec(full_geom.CellSize());
// - Global offset
- std::vector<double> const global_offset = getReversedVec(geom.ProbLo());
+ std::vector<double> const global_offset = getReversedVec(full_geom.ProbLo());
// - AxisLabels
std::vector<std::string> axis_labels = detail::getFieldAxisLabels();
@@ -779,64 +796,70 @@ WarpXOpenPMDPlot::WriteOpenPMDFields( //const std::string& filename,
// meta data
auto series_iteration = m_Series->iterations[iteration];
- series_iteration.setTime( time );
-
- // meta data for ED-PIC extension
- auto const period = geom.periodicity(); // TODO double-check: is this the proper global bound or of some level?
- std::vector< std::string > fieldBoundary( 6, "reflecting" );
- std::vector< std::string > particleBoundary( 6, "absorbing" );
-#if AMREX_SPACEDIM!=3
- fieldBoundary.resize(4);
- particleBoundary.resize(4);
+ auto meshes = series_iteration.meshes;
+ if( first_write_to_iteration ) {
+ series_iteration.setTime( time );
+
+ // meta data for ED-PIC extension
+ auto const period = full_geom.periodicity(); // TODO double-check: is this the proper global bound or of some level?
+ std::vector<std::string> fieldBoundary(6, "reflecting");
+ std::vector<std::string> particleBoundary(6, "absorbing");
+#if AMREX_SPACEDIM != 3
+ fieldBoundary.resize(4);
+ particleBoundary.resize(4);
#endif
- for( auto i = 0u; i < fieldBoundary.size() / 2u; ++i )
- if( m_fieldPMLdirections.at( i ) )
- fieldBoundary.at( i ) = "open";
-
- for( auto i = 0u; i < fieldBoundary.size() / 2u; ++i )
- if( period.isPeriodic( i ) ) {
- fieldBoundary.at(2u*i ) = "periodic";
- fieldBoundary.at(2u*i + 1u) = "periodic";
- particleBoundary.at(2u*i ) = "periodic";
- particleBoundary.at(2u*i + 1u) = "periodic";
- }
-
- auto meshes = series_iteration.meshes;
- meshes.setAttribute( "fieldSolver", [](){
- switch( WarpX::maxwell_solver_id ) {
- case MaxwellSolverAlgo::Yee : return "Yee";
- case MaxwellSolverAlgo::CKC : return "CK";
- case MaxwellSolverAlgo::PSATD : return "PSATD";
- default: return "other";
- }
- }() );
- meshes.setAttribute( "fieldBoundary", fieldBoundary );
- meshes.setAttribute( "particleBoundary", particleBoundary );
- meshes.setAttribute( "currentSmoothing", [](){
- if( WarpX::use_filter ) return "Binomial";
+ for (auto i = 0u; i < fieldBoundary.size() / 2u; ++i)
+ if (m_fieldPMLdirections.at(i))
+ fieldBoundary.at(i) = "open";
+
+ for (auto i = 0u; i < fieldBoundary.size() / 2u; ++i)
+ if (period.isPeriodic(i)) {
+ fieldBoundary.at(2u * i) = "periodic";
+ fieldBoundary.at(2u * i + 1u) = "periodic";
+ particleBoundary.at(2u * i) = "periodic";
+ particleBoundary.at(2u * i + 1u) = "periodic";
+ }
+
+ meshes.setAttribute("fieldSolver", []() {
+ switch (WarpX::maxwell_solver_id) {
+ case MaxwellSolverAlgo::Yee :
+ return "Yee";
+ case MaxwellSolverAlgo::CKC :
+ return "CK";
+ case MaxwellSolverAlgo::PSATD :
+ return "PSATD";
+ default:
+ return "other";
+ }
+ }());
+ meshes.setAttribute("fieldBoundary", fieldBoundary);
+ meshes.setAttribute("particleBoundary", particleBoundary);
+ meshes.setAttribute("currentSmoothing", []() {
+ if (WarpX::use_filter) return "Binomial";
else return "none";
- }() );
- if( WarpX::use_filter )
- meshes.setAttribute( "currentSmoothingParameters", [](){
- std::stringstream ss;
- ss << "period=1;compensator=false";
- ss << ";numPasses_x=" << WarpX::filter_npass_each_dir[0];
+ }());
+ if (WarpX::use_filter)
+ meshes.setAttribute("currentSmoothingParameters", []() {
+ std::stringstream ss;
+ ss << "period=1;compensator=false";
+ ss << ";numPasses_x=" << WarpX::filter_npass_each_dir[0];
#if (AMREX_SPACEDIM == 3)
- ss << ";numPasses_y=" << WarpX::filter_npass_each_dir[1];
- ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[2];
+ ss << ";numPasses_y=" << WarpX::filter_npass_each_dir[1];
+ ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[2];
#else
- ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[1];
+ ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[1];
#endif
- std::string currentSmoothingParameters = ss.str();
- return currentSmoothingParameters;
- }() );
- meshes.setAttribute("chargeCorrection", [](){
- if( WarpX::do_dive_cleaning ) return "hyperbolic"; // TODO or "spectral" or something? double-check
- else return "none";
- }() );
- if( WarpX::do_dive_cleaning )
- meshes.setAttribute("chargeCorrectionParameters", "period=1");
+ std::string currentSmoothingParameters = ss.str();
+ return currentSmoothingParameters;
+ }());
+ meshes.setAttribute("chargeCorrection", []() {
+ if (WarpX::do_dive_cleaning) return "hyperbolic"; // TODO or "spectral" or something? double-check
+ else return "none";
+ }());
+ if (WarpX::do_dive_cleaning)
+ meshes.setAttribute("chargeCorrectionParameters", "period=1");
+ }
// Loop through the different components, i.e. different fields stored in mf
for (int icomp=0; icomp<ncomp; icomp++){
@@ -871,20 +894,24 @@ WarpXOpenPMDPlot::WriteOpenPMDFields( //const std::string& filename,
// we invert (only) meta-data arrays to assign labels and offsets in the
// order: slowest to fastest varying index when accessing the mesh
// contiguously (as 1D flattened logical memory)
- mesh.setDataOrder( openPMD::Mesh::DataOrder::C );
- mesh.setAxisLabels( axis_labels );
- mesh.setGridSpacing( grid_spacing );
- mesh.setGridGlobalOffset( global_offset );
- mesh.setAttribute( "fieldSmoothing", "none" );
- detail::setOpenPMDUnit( mesh, field_name );
+ if( first_write_to_iteration ) {
+ mesh.setDataOrder(openPMD::Mesh::DataOrder::C);
+ mesh.setAxisLabels(axis_labels);
+ mesh.setGridSpacing(grid_spacing);
+ mesh.setGridGlobalOffset(global_offset);
+ mesh.setAttribute("fieldSmoothing", "none");
+ detail::setOpenPMDUnit(mesh, field_name);
+ }
// Create a new mesh record component, and store the associated metadata
auto mesh_comp = mesh[comp_name];
- mesh_comp.resetDataset( dataset );
+ if( first_write_to_iteration ) {
+ mesh_comp.resetDataset(dataset);
- auto relative_cell_pos = utils::getRelativeCellPosition( mf ); // AMReX Fortran index order
- std::reverse( relative_cell_pos.begin(), relative_cell_pos.end() ); // now in C order
- mesh_comp.setPosition( relative_cell_pos );
+ auto relative_cell_pos = utils::getRelativeCellPosition(mf); // AMReX Fortran index order
+ std::reverse(relative_cell_pos.begin(), relative_cell_pos.end()); // now in C order
+ mesh_comp.setPosition(relative_cell_pos);
+ }
// Loop through the multifab, and store each box as a chunk,
// in the openPMD file.
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-09 09:10:19 +0000