From 3fde18912506bbfeeeaacc255f0c8a66ab2e2a05 Mon Sep 17 00:00:00 2001
From: Axel Huebl <axel.huebl@plasma.ninja>
Date: Fri, 11 Dec 2020 09:16:54 -0800
Subject: PSATD Runtime Control (#1300)

* Docs: PSATD Runtime Option

* Tests: PSATD Runtime Option

Add new runtime option to PSATD regression test matrix.

* PICMI: PSATD runtime option

* Source: PSATD Runtime Option
---
 Source/Parallelization/GuardCellManager.cpp | 80 ++++++++++++++---------------
 1 file changed, 39 insertions(+), 41 deletions(-)

(limited to 'Source/Parallelization/GuardCellManager.cpp')

diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp
index 85490f2e0..74e3bff24 100644
--- a/Source/Parallelization/GuardCellManager.cpp
+++ b/Source/Parallelization/GuardCellManager.cpp
@@ -8,14 +8,14 @@
 #include "Filter/NCIGodfreyFilter.H"
 #include "Utils/WarpXAlgorithmSelection.H"
 
-#include <AMReX_Print.H>
 #include <AMReX_ParmParse.H>
 #include <AMReX.H>
 
+
 using namespace amrex;
 
 void
-guardCellManager::Init(
+guardCellManager::Init (
     const bool do_subcycling,
     const bool do_fdtd_nci_corr,
     const bool do_nodal,
@@ -105,23 +105,23 @@ guardCellManager::Init(
     if (maxwell_solver_id == MaxwellSolverAlgo::CKC) ng_alloc_F_int = std::max( ng_alloc_F_int, 1 );
     ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int));
 
-#ifdef WARPX_USE_PSATD
-    // All boxes should have the same number of guard cells
-    // (to avoid temporary parallel copies)
-    // Thus take the max of the required number of guards for each field
-    // Also: the number of guard cell should be enough to contain
-    // the stencil of the FFT solver. Here, this number (`ngFFT`)
-    // is determined *empirically* to be the order of the solver
-    // for nodal, and half the order of the solver for staggered.
+    if (maxwell_solver_id == MaxwellSolverAlgo::PSATD) {
+        // All boxes should have the same number of guard cells
+        // (to avoid temporary parallel copies)
+        // Thus take the max of the required number of guards for each field
+        // Also: the number of guard cell should be enough to contain
+        // the stencil of the FFT solver. Here, this number (`ngFFT`)
+        // is determined *empirically* to be the order of the solver
+        // for nodal, and half the order of the solver for staggered.
 
-    int ngFFt_x = do_nodal ? nox_fft : nox_fft/2;
-    int ngFFt_y = do_nodal ? noy_fft : noy_fft/2;
-    int ngFFt_z = do_nodal ? noz_fft : noz_fft/2;
+        int ngFFt_x = do_nodal ? nox_fft : nox_fft / 2;
+        int ngFFt_y = do_nodal ? noy_fft : noy_fft / 2;
+        int ngFFt_z = do_nodal ? noz_fft : noz_fft / 2;
 
-    ParmParse pp("psatd");
-    pp.query("nx_guard", ngFFt_x);
-    pp.query("ny_guard", ngFFt_y);
-    pp.query("nz_guard", ngFFt_z);
+        ParmParse pp("psatd");
+        pp.query("nx_guard", ngFFt_x);
+        pp.query("ny_guard", ngFFt_y);
+        pp.query("nz_guard", ngFFt_z);
 
 #if (AMREX_SPACEDIM == 3)
         IntVect ngFFT = IntVect(ngFFt_x, ngFFt_y, ngFFt_z);
@@ -129,35 +129,33 @@ guardCellManager::Init(
         IntVect ngFFT = IntVect(ngFFt_x, ngFFt_z);
 #endif
 
-    for (int i_dim=0; i_dim<AMREX_SPACEDIM; i_dim++ ){
-        int ng_required = ngFFT[i_dim];
-        // Get the max
-        ng_required = std::max( ng_required, ng_alloc_EB[i_dim] );
-        ng_required = std::max( ng_required, ng_alloc_J[i_dim] );
-        ng_required = std::max( ng_required, ng_alloc_Rho[i_dim] );
-        ng_required = std::max( ng_required, ng_alloc_F[i_dim] );
-        // Set the guard cells to this max
-        ng_alloc_EB[i_dim] = ng_required;
-        ng_alloc_J[i_dim] = ng_required;
-        ng_alloc_F[i_dim] = ng_required;
-        ng_alloc_Rho[i_dim] = ng_required;
-        ng_alloc_F_int = ng_required;
+        for (int i_dim = 0; i_dim < AMREX_SPACEDIM; i_dim++) {
+            int ng_required = ngFFT[i_dim];
+            // Get the max
+            ng_required = std::max(ng_required, ng_alloc_EB[i_dim]);
+            ng_required = std::max(ng_required, ng_alloc_J[i_dim]);
+            ng_required = std::max(ng_required, ng_alloc_Rho[i_dim]);
+            ng_required = std::max(ng_required, ng_alloc_F[i_dim]);
+            // Set the guard cells to this max
+            ng_alloc_EB[i_dim] = ng_required;
+            ng_alloc_J[i_dim] = ng_required;
+            ng_alloc_F[i_dim] = ng_required;
+            ng_alloc_Rho[i_dim] = ng_required;
+            ng_alloc_F_int = ng_required;
+        }
+        ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int));
     }
-    ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int));
-#else
-    ignore_unused(nox_fft, noy_fft, noz_fft);
-#endif
 
     ng_Extra = IntVect(static_cast<int>(aux_is_nodal and !do_nodal));
 
     // Compute number of cells required for Field Solver
-#ifdef WARPX_USE_PSATD
-    ng_FieldSolver = ng_alloc_EB;
-    ng_FieldSolverF = ng_alloc_EB;
-#else
-    ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1));
-    ng_FieldSolverF = IntVect(AMREX_D_DECL(1,1,1));
-#endif
+    if (maxwell_solver_id == MaxwellSolverAlgo::PSATD) {
+        ng_FieldSolver = ng_alloc_EB;
+        ng_FieldSolverF = ng_alloc_EB;
+    } else {
+        ng_FieldSolver = IntVect(AMREX_D_DECL(1, 1, 1));
+        ng_FieldSolverF = IntVect(AMREX_D_DECL(1, 1, 1));
+    }
 
     if (safe_guard_cells){
         // Run in safe mode: exchange all allocated guard cells at each
-- 
cgit v1.2.3