diff options
| -rw-r--r-- | Docs/source/usage/parameters.rst | 64 | ||||
| -rw-r--r-- | Docs/source/usage/workflows/debugging.rst | 1 | ||||
| -rw-r--r-- | Regression/Checksum/benchmarks_json/pml_psatd_dive_divb_cleaning.json | 12 | ||||
| -rw-r--r-- | Regression/WarpX-tests.ini | 2 | ||||
| -rw-r--r-- | Source/Parallelization/WarpXComm.cpp | 24 | ||||
| -rw-r--r-- | Source/WarpX.H | 24 | ||||
| -rw-r--r-- | Source/ablastr/utils/Communication.H | 5 | ||||
| -rw-r--r-- | Source/ablastr/utils/Communication.cpp | 68 |
8 files changed, 110 insertions, 90 deletions
diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 917cd06e5..b3906ed14 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -57,21 +57,6 @@ Overall simulation parameters printed to standard output. Currently only works if the Lorentz boost and the moving window are along the z direction. -* ``warpx.verbose`` (``0`` or ``1``; default is ``1`` for true) - Controls how much information is printed to the terminal, when running WarpX. - -* ``warpx.always_warn_immediately`` (``0`` or ``1``; default is ``0`` for false) - If set to ``1``, WarpX immediately prints every warning message as soon as - it is generated. It is mainly intended for debug purposes, in case a simulation - crashes before a global warning report can be printed. - -* ``warpx.abort_on_warning_threshold`` (string: ``low``, ``medium`` or ``high``) optional - Optional threshold to abort as soon as a warning is raised. - If the threshold is set, warning messages with priority greater than or - equal to the threshold trigger an immediate abort. - It is mainly intended for debug purposes, and is best used with - ``warpx.always_warn_immediately=1``. - * ``warpx.random_seed`` (`string` or `int` > 0) optional If provided ``warpx.random_seed = random``, the random seed will be determined using `std::random_device` and `std::clock()`, @@ -147,10 +132,6 @@ Overall simulation parameters Note that even with this set to ``1`` WarpX will not catch all out-of-memory events yet when operating close to maximum device memory. `Please also see the documentation in AMReX <https://amrex-codes.github.io/amrex/docs_html/GPU.html#inputs-parameters>`_. -* ``amrex.abort_on_unused_inputs`` (``0`` or ``1``; default is ``0`` for false) - When set to ``1``, this option causes simulation to fail *after* its completion if there were unused parameters. - It is mainly intended for continuous integration and automated testing to check that all tests and inputs are adapted to API changes. - Signal Handling ^^^^^^^^^^^^^^^ @@ -497,9 +478,6 @@ Distribution across MPI ranks and parallelization * ``warpx.do_dynamic_scheduling`` (`0` or `1`) optional (default `1`) Whether to activate OpenMP dynamic scheduling. -* ``warpx.safe_guard_cells`` (`0` or `1`) optional (default `0`) - For developers: run in safe mode, exchanging more guard cells, and more often in the PIC loop (for debugging). - .. _running-cpp-parameters-parser: Math parser and user-defined constants @@ -982,10 +960,6 @@ Particle initialization boosted frame, whether or not to plot back-transformed diagnostics for this species. -* ``warpx.serialize_initial_conditions`` (`0` or `1`) optional (default `0`) - Serialize the initial conditions for reproducible testing. - Mainly whether or not to use OpenMP threading for particle initialization. - * ``<species>.do_field_ionization`` (`0` or `1`) optional (default `0`) Do field ionization for this species (using the ADK theory). @@ -2850,3 +2824,41 @@ This is essentially the python slicing syntax except that the stop is inclusive Note that if a given period is zero or negative, the corresponding slice is disregarded. For example, ``something_intervals = -1`` deactivates ``something`` and ``something_intervals = ::-1,100:1000:25`` is equivalent to ``something_intervals = 100:1000:25``. + + +.. _running-cpp-parameters-test-debug: + +Testing and Debugging +--------------------- + +When developing, testing and :ref:`debugging WarpX <debugging_warpx>`, the following options can be considered. + +* ``warpx.verbose`` (``0`` or ``1``; default is ``1`` for true) + Controls how much information is printed to the terminal, when running WarpX. + +* ``warpx.always_warn_immediately`` (``0`` or ``1``; default is ``0`` for false) + If set to ``1``, WarpX immediately prints every warning message as soon as + it is generated. It is mainly intended for debug purposes, in case a simulation + crashes before a global warning report can be printed. + +* ``warpx.abort_on_warning_threshold`` (string: ``low``, ``medium`` or ``high``) optional + Optional threshold to abort as soon as a warning is raised. + If the threshold is set, warning messages with priority greater than or + equal to the threshold trigger an immediate abort. + It is mainly intended for debug purposes, and is best used with + ``warpx.always_warn_immediately=1``. + +* ``amrex.abort_on_unused_inputs`` (``0`` or ``1``; default is ``0`` for false) + When set to ``1``, this option causes simulation to fail *after* its completion if there were unused parameters. + It is mainly intended for continuous integration and automated testing to check that all tests and inputs are adapted to API changes. + +* ``warpx.serialize_initial_conditions`` (`0` or `1`) optional (default `0`) + Serialize the initial conditions for reproducible testing, e.g, in our continuous integration tests. + Mainly whether or not to use OpenMP threading for particle initialization. + +* ``warpx.safe_guard_cells`` (`0` or `1`) optional (default `0`) + Run in safe mode, exchanging more guard cells, and more often in the PIC loop (for debugging). + +* ``ablastr.fillboundary_always_sync`` (`0` or `1`) optional (default `0`) + Run all ``FillBoundary`` operations on ``MultiFab`` to force-synchronize shared nodal points. + This slightly increases communication cost and can help to spot missing ``nodal_sync`` flags in these operations. diff --git a/Docs/source/usage/workflows/debugging.rst b/Docs/source/usage/workflows/debugging.rst index 2b85523b8..9a7d7efac 100644 --- a/Docs/source/usage/workflows/debugging.rst +++ b/Docs/source/usage/workflows/debugging.rst @@ -26,6 +26,7 @@ Try the following steps to debug a simulation: #. Try to make the reproducible scenario as small as possible by modifying the inputs file. Reduce number of cells, particles and MPI processes to something as small and as quick to execute as possible. The next steps in debugging will increase runtime, so you will benefit from a fast reproducer. +#. Consider adding :ref:`runtime debug options <running-cpp-parameters-test-debug>` that can narrow down typical causes in numerical implementations. #. In case of a crash, Backtraces can be more detailed if you :ref:`re-compile <install-developers>` with debug flags: for example, try compiling with ``-DCMAKE_BUILD_TYPE=RelWithDebInfo`` (some slowdown) or even ``-DCMAKE_BUILD_TYPE=Debug`` (this will make the simulation way slower) and rerun. #. If debug builds are too costly, try instead compiling with ``-DAMReX_ASSERTIONS=ON`` to activate more checks and rerun. #. If the problem looks like a memory violation, this could be from an invalid field or particle index access. diff --git a/Regression/Checksum/benchmarks_json/pml_psatd_dive_divb_cleaning.json b/Regression/Checksum/benchmarks_json/pml_psatd_dive_divb_cleaning.json index 12f562180..f0aaaddce 100644 --- a/Regression/Checksum/benchmarks_json/pml_psatd_dive_divb_cleaning.json +++ b/Regression/Checksum/benchmarks_json/pml_psatd_dive_divb_cleaning.json @@ -1,11 +1,11 @@ { "lev=0": { - "Bx": 1.482006352778953e-07, - "By": 1.48205157883426e-07, - "Bz": 1.4954704195856524e-07, - "Ex": 11.789793626679334, - "Ey": 11.78688532983594, - "Ez": 11.770112090435557, + "Bx": 1.4820063339458393e-07, + "By": 1.4820515850615877e-07, + "Bz": 1.495470407235984e-07, + "Ex": 11.789793389856982, + "Ey": 11.786884764477099, + "Ez": 11.770110814468563, "rho": 4.903696256562049e-05 } }
\ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 8de0b8245..90f937f01 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -121,7 +121,7 @@ analysisRoutine = Examples/Tests/PML/analysis_pml_psatd.py [pml_psatd_dive_divb_cleaning] buildDir = . inputFile = Examples/Tests/PML/inputs_3d -runtime_params = warpx.do_similar_dm_pml=0 warpx.abort_on_warning_threshold=medium +runtime_params = warpx.do_similar_dm_pml=0 warpx.abort_on_warning_threshold=medium ablastr.fillboundary_always_sync=1 dim = 3 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index cee53031c..396b0f24b 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -473,7 +473,7 @@ void WarpX::UpdateCurrentNodalToStag (amrex::MultiFab& dst, amrex::MultiFab cons } void -WarpX::FillBoundaryB (IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryB (IntVect ng, std::optional<bool> nodal_sync) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -482,7 +482,7 @@ WarpX::FillBoundaryB (IntVect ng, const bool nodal_sync) } void -WarpX::FillBoundaryE (IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryE (IntVect ng, std::optional<bool> nodal_sync) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -491,7 +491,7 @@ WarpX::FillBoundaryE (IntVect ng, const bool nodal_sync) } void -WarpX::FillBoundaryF (IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryF (IntVect ng, std::optional<bool> nodal_sync) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -500,7 +500,7 @@ WarpX::FillBoundaryF (IntVect ng, const bool nodal_sync) } void -WarpX::FillBoundaryG (IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryG (IntVect ng, std::optional<bool> nodal_sync) { for (int lev = 0; lev <= finest_level; ++lev) { @@ -528,14 +528,14 @@ WarpX::FillBoundaryE_avg (IntVect ng) void -WarpX::FillBoundaryE (int lev, IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryE (int lev, IntVect ng, std::optional<bool> nodal_sync) { FillBoundaryE(lev, PatchType::fine, ng, nodal_sync); if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync); } void -WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::IntVect ng, std::optional<bool> nodal_sync) { std::array<amrex::MultiFab*,3> mf; amrex::Periodicity period; @@ -585,14 +585,14 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In } void -WarpX::FillBoundaryB (int lev, IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryB (int lev, IntVect ng, std::optional<bool> nodal_sync) { FillBoundaryB(lev, PatchType::fine, ng, nodal_sync); if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync); } void -WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::IntVect ng, std::optional<bool> nodal_sync) { std::array<amrex::MultiFab*,3> mf; amrex::Periodicity period; @@ -747,14 +747,14 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) } void -WarpX::FillBoundaryF (int lev, IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryF (int lev, IntVect ng, std::optional<bool> nodal_sync) { FillBoundaryF(lev, PatchType::fine, ng, nodal_sync); if (lev > 0) FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync); } void -WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, const bool nodal_sync) +WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optional<bool> nodal_sync) { if (patch_type == PatchType::fine) { @@ -788,7 +788,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, const bool noda } } -void WarpX::FillBoundaryG (int lev, IntVect ng, const bool nodal_sync) +void WarpX::FillBoundaryG (int lev, IntVect ng, std::optional<bool> nodal_sync) { FillBoundaryG(lev, PatchType::fine, ng, nodal_sync); @@ -798,7 +798,7 @@ void WarpX::FillBoundaryG (int lev, IntVect ng, const bool nodal_sync) } } -void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, const bool nodal_sync) +void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optional<bool> nodal_sync) { if (patch_type == PatchType::fine) { diff --git a/Source/WarpX.H b/Source/WarpX.H index 4431be57e..0f2a986e7 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -636,21 +636,21 @@ public: void UpdateCurrentNodalToStag (amrex::MultiFab& dst, amrex::MultiFab const& src); // Fill boundary cells including coarse/fine boundaries - void FillBoundaryB (amrex::IntVect ng, const bool nodal_sync = false); - void FillBoundaryE (amrex::IntVect ng, const bool nodal_sync = false); + void FillBoundaryB (amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); + void FillBoundaryE (amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); void FillBoundaryB_avg (amrex::IntVect ng); void FillBoundaryE_avg (amrex::IntVect ng); - void FillBoundaryF (amrex::IntVect ng, const bool nodal_sync = false); - void FillBoundaryG (amrex::IntVect ng, const bool nodal_sync = false); + void FillBoundaryF (amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); + void FillBoundaryG (amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); void FillBoundaryAux (amrex::IntVect ng); - void FillBoundaryE (int lev, amrex::IntVect ng, const bool nodal_sync = false); - void FillBoundaryB (int lev, amrex::IntVect ng, const bool nodal_sync = false); + void FillBoundaryE (int lev, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); + void FillBoundaryB (int lev, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); void FillBoundaryE_avg (int lev, amrex::IntVect ng); void FillBoundaryB_avg (int lev, amrex::IntVect ng); - void FillBoundaryF (int lev, amrex::IntVect ng, const bool nodal_sync = false); - void FillBoundaryG (int lev, amrex::IntVect ng, const bool nodal_sync = false); + void FillBoundaryF (int lev, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); + void FillBoundaryG (int lev, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); void FillBoundaryAux (int lev, amrex::IntVect ng); /** @@ -957,10 +957,10 @@ private: /// void EvolveEM(int numsteps); - void FillBoundaryB (const int lev, const PatchType patch_type, const amrex::IntVect ng, const bool nodal_sync = false); - void FillBoundaryE (const int lev, const PatchType patch_type, const amrex::IntVect ng, const bool nodal_sync = false); - void FillBoundaryF (int lev, PatchType patch_type, amrex::IntVect ng, const bool nodal_sync = false); - void FillBoundaryG (int lev, PatchType patch_type, amrex::IntVect ng, const bool nodal_sync = false); + void FillBoundaryB (const int lev, const PatchType patch_type, const amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); + void FillBoundaryE (const int lev, const PatchType patch_type, const amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); + void FillBoundaryF (int lev, PatchType patch_type, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); + void FillBoundaryG (int lev, PatchType patch_type, amrex::IntVect ng, std::optional<bool> nodal_sync = std::nullopt); void FillBoundaryB_avg (int lev, PatchType patch_type, amrex::IntVect ng); void FillBoundaryE_avg (int lev, PatchType patch_type, amrex::IntVect ng); diff --git a/Source/ablastr/utils/Communication.H b/Source/ablastr/utils/Communication.H index 9f58096b9..2c8794604 100644 --- a/Source/ablastr/utils/Communication.H +++ b/Source/ablastr/utils/Communication.H @@ -18,6 +18,9 @@ #include "WarpX.H" +#include <optional> + + namespace ablastr::utils::communication { @@ -66,7 +69,7 @@ void FillBoundary (amrex::MultiFab &mf, amrex::IntVect ng, bool do_single_precision_comms, const amrex::Periodicity &period = amrex::Periodicity::NonPeriodic(), - const bool nodal_sync = false); + std::optional<bool> nodal_sync = std::nullopt); void FillBoundary (amrex::iMultiFab &mf, const amrex::Periodicity &period = amrex::Periodicity::NonPeriodic()); diff --git a/Source/ablastr/utils/Communication.cpp b/Source/ablastr/utils/Communication.cpp index c7bf00c8e..71cd83802 100644 --- a/Source/ablastr/utils/Communication.cpp +++ b/Source/ablastr/utils/Communication.cpp @@ -13,6 +13,8 @@ #include <AMReX_FabArray.H> #include <AMReX_MultiFab.H> #include <AMReX_iMultiFab.H> +#include <AMReX_ParmParse.H> + namespace ablastr::utils::communication { @@ -60,27 +62,24 @@ void ParallelAdd(amrex::MultiFab &dst, const amrex::MultiFab &src, int src_comp, do_single_precision_comms, period, amrex::FabArrayBase::ADD); } -void FillBoundary (amrex::MultiFab &mf, bool do_single_precision_comms, const amrex::Periodicity &period) +void FillBoundary (amrex::MultiFab &mf, + amrex::IntVect ng, + bool do_single_precision_comms, + const amrex::Periodicity &period, + std::optional<bool> nodal_sync) { BL_PROFILE("ablastr::utils::communication::FillBoundary"); - if (do_single_precision_comms) - { - mf.FillBoundary<comm_float_type>(period); - } - else - { - mf.FillBoundary(period); - } -} + // allow developers to always enforce nodal sync, independent of the + // nodal_sync argument + bool do_nodal_sync_arg = nodal_sync.value_or(false); -void FillBoundary(amrex::MultiFab &mf, - amrex::IntVect ng, - bool do_single_precision_comms, - const amrex::Periodicity &period, - const bool nodal_sync) -{ - BL_PROFILE("ablastr::utils::communication::FillBoundary"); + amrex::ParmParse pp_ablastr("ablastr"); + bool do_nodal_sync_input = false; + pp_ablastr.query("fillboundary_always_sync", do_nodal_sync_input); + + // logic: inputs overwrite argument unless argument is true + bool const do_nodal_sync = do_nodal_sync_arg || do_nodal_sync_input; if (do_single_precision_comms) { @@ -91,7 +90,7 @@ void FillBoundary(amrex::MultiFab &mf, mixedCopy(mf_tmp, mf, 0, 0, mf.nComp(), mf.nGrowVect()); - if (nodal_sync) { + if (do_nodal_sync) { mf_tmp.FillBoundaryAndSync(0, mf.nComp(), ng, period); } else { mf_tmp.FillBoundary(ng, period); @@ -101,8 +100,7 @@ void FillBoundary(amrex::MultiFab &mf, } else { - - if (nodal_sync) { + if (do_nodal_sync) { mf.FillBoundaryAndSync(0, mf.nComp(), ng, period); } else { mf.FillBoundary(ng, period); @@ -110,9 +108,24 @@ void FillBoundary(amrex::MultiFab &mf, } } -void FillBoundary(amrex::iMultiFab &imf, const amrex::Periodicity &period) +void FillBoundary (amrex::MultiFab &mf, bool do_single_precision_comms, const amrex::Periodicity &period) { - BL_PROFILE("ablastr::utils::communication::FillBoundary"); + amrex::IntVect const ng = mf.n_grow; + FillBoundary(mf, ng, do_single_precision_comms, period); +} + +void +FillBoundary (amrex::Vector<amrex::MultiFab *> const &mf, bool do_single_precision_comms, + const amrex::Periodicity &period) +{ + for (auto x : mf) { + ablastr::utils::communication::FillBoundary(*x, do_single_precision_comms, period); + } +} + +void FillBoundary (amrex::iMultiFab &imf, const amrex::Periodicity &period) +{ + BL_PROFILE("ablastr::utils::communication::FillBoundary::iMultiFab"); imf.FillBoundary(period); } @@ -121,20 +134,11 @@ void FillBoundary (amrex::iMultiFab& imf, amrex::IntVect ng, const amrex::Periodicity& period) { - BL_PROFILE("ablastr::utils::communication::FillBoundary"); + BL_PROFILE("ablastr::utils::communication::FillBoundary::iMultiFab"); imf.FillBoundary(ng, period); } -void -FillBoundary(amrex::Vector<amrex::MultiFab *> const &mf, bool do_single_precision_comms, - const amrex::Periodicity &period) -{ - for (auto x : mf) { - ablastr::utils::communication::FillBoundary(*x, do_single_precision_comms, period); - } -} - void SumBoundary (amrex::MultiFab &mf, bool do_single_precision_comms, const amrex::Periodicity &period) { BL_PROFILE("ablastr::utils::communication::SumBoundary"); |