From 02d59e100674803542a3f99b38d4d25d5b34de9a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 16 Jul 2020 09:28:58 -0700 Subject: Default: abort_on_out_of_gpu_memory = 1 (#1164) * Default: abort_on_out_of_gpu_memory = 1 Change the default input parameter from AMReX `amrex.abort_on_out_of_gpu_memory` from false (`0`) to true (`1`). We set this by default to avoid that users experience super-slow GPU runs when exceeding GPU memory. In such a case, users should explicitly set this to option. In my optinion, this is only an intermediate solution since what we actually want on out-of-GPU memory events should be: - finish current simulation step and cause a load balance or - trigger a checkpoint and shut down cleanly - then the user can manually restart with more resources We want to address the opposite case, user under-utilizes a GPU, with a warning for now. Ref.: - https://amrex-codes.github.io/amrex/docs_html/GPU.html#inputs-parameters * abort_on_out_of_gpu_memory: review Add review comments. Co-authored-by: L. Diana Amorim Co-authored-by: L. Diana Amorim --- Source/Initialization/WarpXAMReXInit.cpp | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 Source/Initialization/WarpXAMReXInit.cpp (limited to 'Source/Initialization/WarpXAMReXInit.cpp') diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp new file mode 100644 index 000000000..5b0b366fa --- /dev/null +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -0,0 +1,44 @@ +/* Copyright 2020 Axel Huebl + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "Initialization/WarpXAMReXInit.H" + +#include + + +namespace { + /** Overwrite defaults in AMReX Inputs + * + * This overwrites defaults in amrex::ParamParse for inputs. + */ + void + overwrite_amrex_parser_defaults() + { + amrex::ParmParse pp("amrex"); + + // https://amrex-codes.github.io/amrex/docs_html/GPU.html#inputs-parameters + bool abort_on_out_of_gpu_memory = true; // AMReX' default: false + pp.query("abort_on_out_of_gpu_memory", abort_on_out_of_gpu_memory); + pp.add("abort_on_out_of_gpu_memory", abort_on_out_of_gpu_memory); + } +} + +amrex::AMReX* +warpx_amrex_init(int& argc, char**& argv) +{ + // note: AMReX defines a placeholder/"mock-up" for MPI_COMM_WORLD in serial builds + bool const build_parm_parse = true; // AMReX' default + MPI_Comm const mpi_comm = MPI_COMM_WORLD; // AMReX' default + + return amrex::Initialize( + argc, + argv, + build_parm_parse, + mpi_comm, + overwrite_amrex_parser_defaults + ); +} -- cgit v1.2.3