6 files changed, 79 insertions, 6 deletions
diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst
index 4859f6034..ff08967d7 100644
--- a/Docs/source/running_cpp/parameters.rst
+++ b/Docs/source/running_cpp/parameters.rst
@@ -45,7 +45,7 @@ Overall simulation parameters
     printed to standard output. Currently only works if the Lorentz boost and
     the moving window are along the z direction.
 
-* ``warpx.verbose`` (`0` or `1`)
+* ``warpx.verbose`` (``0`` or ``1``; default is ``1`` for true)
     Controls how much information is printed to the terminal, when running WarpX.
 
 * ``warpx.random_seed`` (`string` or `int` > 0) optional
@@ -61,7 +61,7 @@ Overall simulation parameters
     one should not expect to obtain the same random numbers,
     even if a fixed ``warpx.random_seed`` is provided.
 
-* ``warpx.do_electrostatic`` (`0` or `1`; default is `0`)
+* ``warpx.do_electrostatic`` (``0`` or ``1``; default is ``0`` for false)
     Run WarpX in electrostatic mode. Instead of updating the fields
     at each iteration with the full Maxwell equations, the fields are
     instead recomputed at each iteration from the (relativistic) Poisson
@@ -69,6 +69,12 @@ Overall simulation parameters
     electromagnetic effects (e.g. propagation of radiation, lasers, etc.)
     are not captured.
 
+* ``amrex.abort_on_out_of_gpu_memory``  (``0`` or ``1``; default is ``1`` for true)
+    When running on GPUs, memory that does not fit on the device will be automatically swapped to host memory when this option is set to ``0``.
+    This will cause severe performance drops.
+    Note that even with this set to ``1`` WarpX will not catch all out-of-memory events yet when operating close to maximum device memory.
+    `Please also see the documentation in AMReX <https://amrex-codes.github.io/amrex/docs_html/GPU.html#inputs-parameters>`_.
+
 .. _running-cpp-parameters-box:
 
 Setting up the field mesh
diff --git a/Source/Initialization/CMakeLists.txt b/Source/Initialization/CMakeLists.txt
index b6cab076c..b9f0ae345 100644
--- a/Source/Initialization/CMakeLists.txt
+++ b/Source/Initialization/CMakeLists.txt
@@ -1,5 +1,6 @@
 target_sources(WarpX
   PRIVATE
+    WarpXAMReXInit.cpp
     InjectorDensity.cpp
     InjectorMomentum.cpp
     PlasmaInjector.cpp
diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package
index c90ed7a49..f2730c6bc 100644
--- a/Source/Initialization/Make.package
+++ b/Source/Initialization/Make.package
@@ -1,3 +1,4 @@
+CEXE_sources += WarpXAMReXInit.cpp
 CEXE_sources += WarpXInitData.cpp
 CEXE_sources += PlasmaInjector.cpp
 CEXE_sources += InjectorDensity.cpp
diff --git a/Source/Initialization/WarpXAMReXInit.H b/Source/Initialization/WarpXAMReXInit.H
new file mode 100644
index 000000000..6977bdcd5
--- /dev/null
+++ b/Source/Initialization/WarpXAMReXInit.H
@@ -0,0 +1,23 @@
+/* Copyright 2020 Axel Huebl
+ *
+ * This file is part of WarpX.
+ *
+ * License: BSD-3-Clause-LBNL
+ */
+#ifndef WARPX_AMREX_INIT_H_
+#define WARPX_AMREX_INIT_H_
+
+#include <AMReX.H>
+
+/** Call amrex::Initialize
+ *
+ * This function calls amrex::Initialize and overwrites AMReX' defaults.
+ *
+ * @param[in] argc number of arguments from main()
+ * @param[in] argv argument strings from main()
+ * @returns pointer to an AMReX* object, forwarded from amrex::Initialize
+ */
+amrex::AMReX*
+warpx_amrex_init(int& argc, char**& argv);
+
+#endif
diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp
new file mode 100644
index 000000000..5b0b366fa
--- /dev/null
+++ b/Source/Initialization/WarpXAMReXInit.cpp
@@ -0,0 +1,44 @@
+/* Copyright 2020 Axel Huebl
+ *
+ * This file is part of WarpX.
+ *
+ * License: BSD-3-Clause-LBNL
+ */
+
+#include "Initialization/WarpXAMReXInit.H"
+
+#include <AMReX_ParmParse.H>
+
+
+namespace {
+    /** Overwrite defaults in AMReX Inputs
+     *
+     * This overwrites defaults in amrex::ParamParse for inputs.
+     */
+    void
+    overwrite_amrex_parser_defaults()
+    {
+        amrex::ParmParse pp("amrex");
+
+        // https://amrex-codes.github.io/amrex/docs_html/GPU.html#inputs-parameters
+        bool abort_on_out_of_gpu_memory = true; // AMReX' default: false
+        pp.query("abort_on_out_of_gpu_memory", abort_on_out_of_gpu_memory);
+        pp.add("abort_on_out_of_gpu_memory", abort_on_out_of_gpu_memory);
+    }
+}
+
+amrex::AMReX*
+warpx_amrex_init(int& argc, char**& argv)
+{
+    // note: AMReX defines a placeholder/"mock-up" for MPI_COMM_WORLD in serial builds
+    bool const build_parm_parse = true; // AMReX' default
+    MPI_Comm const mpi_comm = MPI_COMM_WORLD; // AMReX' default
+
+    return amrex::Initialize(
+        argc,
+        argv,
+        build_parm_parse,
+        mpi_comm,
+        overwrite_amrex_parser_defaults
+    );
+}
diff --git a/Source/main.cpp b/Source/main.cpp
index 4bbdce2da..75169bbb5 100644
--- a/Source/main.cpp
+++ b/Source/main.cpp
@@ -7,16 +7,14 @@
  * License: BSD-3-Clause-LBNL
  */
 #include "WarpX.H"
+#include "Initialization/WarpXAMReXInit.H"
 #include "Utils/WarpXUtil.H"
 #include "Utils/WarpXProfilerWrapper.H"
 
 #include <AMReX.H>
-#include <AMReX_ParmParse.H>
 #include <AMReX_BLProfiler.H>
 #include <AMReX_ParallelDescriptor.H>
 
-#include <iostream>
-
 
 int main(int argc, char* argv[])
 {
@@ -32,7 +30,7 @@ int main(int argc, char* argv[])
 #   endif
 #endif
 
-    amrex::Initialize(argc,argv);
+    warpx_amrex_init(argc, argv);
 
     // in Debug mode, we need a larger stack limit than usual bc of the parser.
 #if defined(AMREX_USE_CUDA) && defined(AMREX_DEBUG)