Move sensei to new diagnostics (#1040)

* move ascent to new diagnostics

* move Sensei to the new diagnostics

* minor cleaning in new diagnostics

* oops, had forgotten some files

* fix typo

* another typo introduced when merging master into this branch

* FlushFormat::WriteToFile takes Vector<MultiFab>& instead of Vector<const MultiFab*>

* IO output multifab has 1 guard cell when using sensei

* updates to FlushFormatSensei

* add some documentation to the class
* add refrence to AmrMesh instance needed by the adaptor
* add some error output in the case WarpX was not compiled w/ SENSEI
* tested with SENSEI 3.2.0

Co-authored-by: Burlen Loring <bloring@lbl.gov>

1 files changed, 16 insertions, 4 deletions

diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp
index 0e13c256c..5a0f29113 100644
--- a/Source/Diagnostics/Diagnostics.cpp
+++ b/Source/Diagnostics/Diagnostics.cpp
@@ -7,6 +7,7 @@
 #include "FlushFormats/FlushFormatPlotfile.H"
 #include "FlushFormats/FlushFormatCheckpoint.H"
 #include "FlushFormats/FlushFormatAscent.H"
+#include "FlushFormats/FlushFormatSensei.H"
 #ifdef WARPX_USE_OPENPMD
 #   include "FlushFormats/FlushFormatOpenPMD.H"
 #endif
@@ -39,8 +40,10 @@ Diagnostics::ReadParameters ()
     m_intervals = IntervalsParser(period_string);
     pp.query("format", m_format);
     AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
-        m_format == "plotfile" || m_format == "openpmd" || m_format == "checkpoint" || m_format == "ascent",
-        "<diag>.format must be plotfile or openpmd or checkpoint or ascent");
+        m_format == "plotfile" || m_format == "openpmd" ||
+        m_format == "checkpoint" || m_format == "ascent" ||
+        m_format == "sensei",
+        "<diag>.format must be plotfile or openpmd or checkpoint or ascent or sensei");
     bool raw_specified = pp.query("plot_raw_fields", m_plot_raw_fields);
     raw_specified += pp.query("plot_raw_fields_guards", m_plot_raw_fields_guards);
     bool varnames_specified = pp.queryarr("fields_to_plot", m_varnames);
@@ -148,6 +151,14 @@ Diagnostics::InitData ()
         m_flush_format = new FlushFormatCheckpoint;
     } else if (m_format == "ascent"){
         m_flush_format = new FlushFormatAscent;
+    } else if (m_format == "sensei"){
+#ifdef BL_USE_SENSEI_INSITU
+        m_flush_format = new FlushFormatSensei(
+            dynamic_cast<amrex::AmrMesh*>(const_cast<WarpX*>(&warpx)),
+            m_diag_name);
+#else
+        amrex::Abort("To use SENSEI in situ, compile with USE_SENSEI=TRUE");
+#endif
     } else if (m_format == "openpmd"){
 #ifdef WARPX_USE_OPENPMD
         m_flush_format = new FlushFormatOpenPMD(m_diag_name);
@@ -195,7 +206,7 @@ Diagnostics::Flush ()
 {
     auto & warpx = WarpX::GetInstance();
     m_flush_format->WriteToFile(
-        m_varnames, GetVecOfConstPtrs(m_mf_output), warpx.Geom(), warpx.getistep(),
+        m_varnames, m_mf_output, warpx.Geom(), warpx.getistep(),
         warpx.gett_new(0), m_all_species, nlev, m_file_prefix,
         m_plot_raw_fields, m_plot_raw_fields_guards, m_plot_raw_rho, m_plot_raw_F);
 }
@@ -401,7 +412,8 @@ Diagnostics::DefineDiagMultiFab ( int lev ) {
     // is different from the lo and hi physical co-ordinates of the simulation domain.
     if (use_warpxba == false) dmap = DistributionMapping{ba};
     // Allocate output MultiFab for diagnostics. The data will be stored at cell-centers.
-    m_mf_output[lev] = MultiFab(ba, dmap, m_varnames.size(), 0);
+    int ngrow = (m_format == "sensei") ? 1 : 0;
+    m_mf_output[lev] = MultiFab(ba, dmap, m_varnames.size(), ngrow);
 }