openPMD: Add ADIOS2 Engine Parameter Control (#2872)

* Adds support for ADIOS engines #2866

Input file can now have lines like
diag1.adios2_engine.parameters.NumAggregators=2

* Docs for ADIOS engine type and parameters #2866

* Aesthetic edit in adios2 engine documentation #2866

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Removed debug print statement #2866

* Style Updates

Co-authored-by: Mehta, Kshitij V <kshitij-v-mehta@github.com>
Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

4 files changed, 100 insertions, 17 deletions

diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst
index 0b6d65436..c40dc8bea 100644
--- a/Docs/source/usage/parameters.rst
+++ b/Docs/source/usage/parameters.rst
@@ -1892,6 +1892,21 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a
         <diag_name>.adios2_operator.type = zfp
         <diag_name>.adios2_operator.parameters.precision = 3
 
+* ``<diag_name>.adios2_engine.type`` (``bp4``, ``sst``, ``ssc``, ``dataman``) optional,
+    `ADIOS2 Engine type <https://openpmd-api.readthedocs.io/en/0.14.0/details/backendconfig.html#adios2>`__ for `openPMD <https://www.openPMD.org>`_ data dumps.
+    See full list of engines at `ADIOS2 readthedocs <https://adios2.readthedocs.io/en/latest/engines/engines.html>`__
+
+* ``<diag_name>.adios2_engine.parameters.*`` optional,
+    `ADIOS2 Engine parameters <https://openpmd-api.readthedocs.io/en/0.14.0/details/backendconfig.html#adios2>`__ for `openPMD <https://www.openPMD.org>`_ data dumps.
+
+    An example for parameters for the BP engine are setting the number of writers (``NumAggregators``), transparently redirecting data to burst buffers etc.
+    A detailed list of engine-specific parameters are available at the official `ADIOS2 documentation <https://adios2.readthedocs.io/en/latest/engines/engines.html>`__
+
+    .. code-block:: text
+
+        <diag_name>.adios2_engine.parameter.NumAggregators = 2048
+        <diag_name>.adios2_engine.parameters.BurstBufferPath="/mnt/bb/username"
+
 * ``<diag_name>.fields_to_plot`` (list of `strings`, optional)
     Fields written to output.
     Possible values: ``Ex`` ``Ey`` ``Ez`` ``Bx`` ``By`` ``Bz`` ``jx`` ``jy`` ``jz`` ``part_per_cell`` ``rho`` ``phi`` ``F`` ``part_per_grid`` ``divE`` ``divB`` and ``rho_<species_name>``, where ``<species_name>`` must match the name of one of the available particle species. Note that ``phi`` will only be written out when do_electrostatic==labframe.
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp
index 401d0e184..a3f549718 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp
+++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp
@@ -76,10 +76,27 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name)
     operator_parameters.insert({k, v});
   }
 
+  // ADIOS2 engine type & parameters
+  std::string engine_type;
+  pp_diag_name.query("adios2_engine.type", engine_type);
+  std::string const engine_prefix = diag_name + ".adios2_engine.parameters";
+  ParmParse ppe;
+  auto eng_entr = ppe.getEntries(engine_prefix);
+
+  std::map< std::string, std::string > engine_parameters;
+  auto const prefixlen = engine_prefix.size() + 1;
+  for (std::string k : eng_entr) {
+    std::string v;
+    ppe.get(k.c_str(), v);
+    k.erase(0, prefixlen);
+    engine_parameters.insert({k, v});
+  }
+
   auto & warpx = WarpX::GetInstance();
   m_OpenPMDPlotWriter = std::make_unique<WarpXOpenPMDPlot>(
     encoding, openpmd_backend,
     operator_type, operator_parameters,
+    engine_type, engine_parameters,
     warpx.getPMLdirections()
   );
 
diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H
index 08b81b075..807cfdc53 100644
--- a/Source/Diagnostics/WarpXOpenPMD.H
+++ b/Source/Diagnostics/WarpXOpenPMD.H
@@ -110,6 +110,8 @@ public:
                     std::string filetype,
                     std::string operator_type,
                     std::map< std::string, std::string > operator_parameters,
+                    std::string engine_type,
+                    std::map< std::string, std::string > engine_parameters,
                     std::vector<bool> fieldPMLdirections);
 
   ~WarpXOpenPMDPlot ();
diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp
index ada56da36..87a588b9d 100644
--- a/Source/Diagnostics/WarpXOpenPMD.cpp
+++ b/Source/Diagnostics/WarpXOpenPMD.cpp
@@ -122,9 +122,18 @@ namespace detail
      */
     inline std::string
     getSeriesOptions (std::string const & operator_type,
-                      std::map< std::string, std::string > const & operator_parameters)
+                      std::map< std::string, std::string > const & operator_parameters,
+                      std::string const & engine_type,
+                      std::map< std::string, std::string > const & engine_parameters)
     {
+        if (operator_type.empty() && engine_type.empty())
+            return "{}";
+
         std::string options;
+        std::string top_block;
+        std::string end_block;
+        std::string op_block;
+        std::string en_block;
 
         std::string op_parameters;
         for (const auto& kv : operator_parameters) {
@@ -133,31 +142,68 @@ namespace detail
                     .append("\"").append(kv.first).append("\": ")    /* key */
                     .append("\"").append(kv.second).append("\""); /* value (as string) */
         }
-        if (!operator_type.empty()) {
-            options = R"END(
+
+        std::string en_parameters;
+        for (const auto& kv : engine_parameters) {
+            if (!en_parameters.empty()) en_parameters.append(",\n");
+            en_parameters.append(std::string(12, ' '))         /* just pretty alignment */
+                    .append("\"").append(kv.first).append("\": ")    /* key */
+                    .append("\"").append(kv.second).append("\""); /* value (as string) */
+        }
+
+        // create the outer-level blocks
+        top_block = R"END(
 {
-  "adios2": {
+  "adios2": {)END";
+
+        end_block = R"END(
+  }
+})END";
+
+        // add the operator string block
+        if (!operator_type.empty()) {
+            op_block = R"END(
     "dataset": {
       "operators": [
         {
           "type": ")END";
-            options += operator_type + "\"";
-        }
-        if (!operator_type.empty() && !op_parameters.empty()) {
-            options += R"END(
-         ,"parameters": {
+            op_block += operator_type + "\"";
+
+            if (!op_parameters.empty()) {
+                op_block += R"END(,
+          "parameters": {
 )END";
-            options += op_parameters + "}";
+            op_block += op_parameters + "}";
         }
-        if (!operator_type.empty())
-            options += R"END(
+            op_block += R"END(
         }
       ]
-    }
-  }
-}
+    })END";
+        if (!engine_type.empty())
+            op_block += ",";
+
+        }  // end operator string block
+
+        // add the engine string block
+        if (!engine_type.empty()) {
+            en_block = R"END(
+    "engine": {
+      "type": ")END";
+            en_block += engine_type + "\"";
+
+            if (!en_parameters.empty()) {
+                en_block += R"END(,
+      "parameters": {
 )END";
-        if (options.empty()) options = "{}";
+            en_block += en_parameters + "}";
+            }
+
+            en_block += R"END(
+    })END";
+
+        }  // end engine string block
+
+        options = top_block + op_block + en_block + end_block;
         return options;
     }
 
@@ -337,6 +383,8 @@ WarpXOpenPMDPlot::WarpXOpenPMDPlot (
     std::string openPMDFileType,
     std::string operator_type,
     std::map< std::string, std::string > operator_parameters,
+    std::string engine_type,
+    std::map< std::string, std::string > engine_parameters,
     std::vector<bool> fieldPMLdirections)
   :m_Series(nullptr),
    m_Encoding(ie),
@@ -355,7 +403,8 @@ WarpXOpenPMDPlot::WarpXOpenPMDPlot (
     m_OpenPMDFileType = "json";
 #endif
 
-    m_OpenPMDoptions = detail::getSeriesOptions(operator_type, operator_parameters);
+    m_OpenPMDoptions = detail::getSeriesOptions(operator_type, operator_parameters,
+                                                engine_type, engine_parameters);
 }
 
 WarpXOpenPMDPlot::~WarpXOpenPMDPlot ()