Merge pull request #504 from MaxThevenet/comm

Reduce number of guard cells exchanged in FillBoundary

1 files changed, 169 insertions, 0 deletions

diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp
new file mode 100644
index 000000000..7f72553b6
--- /dev/null
+++ b/Source/Parallelization/GuardCellManager.cpp
@@ -0,0 +1,169 @@
+#include "GuardCellManager.H"
+#include "NCIGodfreyFilter.H"
+#include <AMReX_Print.H>
+
+using namespace amrex;
+
+void
+guardCellManager::Init(
+    const bool do_subcycling,
+    const bool do_fdtd_nci_corr,
+    const bool do_nodal,
+    const bool do_moving_window,
+    const bool do_fft_mpi_dec,
+    const bool aux_is_nodal,
+    const int moving_window_dir,
+    const int nox,
+    const int nox_fft, const int noy_fft, const int noz_fft,
+    const int nci_corr_stencil,
+    const int maxwell_fdtd_solver_id,
+    const int max_level,
+    const bool exchange_all_guard_cells)
+{
+    // When using subcycling, the particles on the fine level perform two pushes
+    // before being redistributed ; therefore, we need one extra guard cell
+    // (the particles may move by 2*c*dt)
+    const int ngx_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox;
+    const int ngy_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox;
+    const int ngz_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox;
+
+    // Ex, Ey, Ez, Bx, By, and Bz have the same number of ghost cells.
+    // jx, jy, jz and rho have the same number of ghost cells.
+    // E and B have the same number of ghost cells as j and rho if NCI filter is not used,
+    // but different number of ghost cells in z-direction if NCI filter is used.
+    // The number of cells should be even, in order to easily perform the
+    // interpolation from coarse grid to fine grid.
+    int ngx = (ngx_tmp % 2) ? ngx_tmp+1 : ngx_tmp;  // Always even number
+    int ngy = (ngy_tmp % 2) ? ngy_tmp+1 : ngy_tmp;  // Always even number
+    int ngz_nonci = (ngz_tmp % 2) ? ngz_tmp+1 : ngz_tmp;  // Always even number
+    int ngz;
+    if (do_fdtd_nci_corr) {
+        int ng = ngz_tmp + nci_corr_stencil;
+        ngz = (ng % 2) ? ng+1 : ng;
+    } else {
+        ngz = ngz_nonci;
+    }
+
+    // J is only interpolated from fine to coarse (not coarse to fine)
+    // and therefore does not need to be even.
+    int ngJx = ngx_tmp;
+    int ngJy = ngy_tmp;
+    int ngJz = ngz_tmp;
+
+    // When calling the moving window (with one level of refinement),  we shift
+    // the fine grid by 2 cells ; therefore, we need at least 2 guard cells
+    // on level 1. This may not be necessary for level 0.
+    if (do_moving_window) {
+        ngx = std::max(ngx,2);
+        ngy = std::max(ngy,2);
+        ngz = std::max(ngz,2);
+        ngJx = std::max(ngJx,2);
+        ngJy = std::max(ngJy,2);
+        ngJz = std::max(ngJz,2);
+    }
+
+#if (AMREX_SPACEDIM == 3)
+    ng_alloc_EB = IntVect(ngx,ngy,ngz);
+    ng_alloc_J = IntVect(ngJx,ngJy,ngJz);
+#elif (AMREX_SPACEDIM == 2)
+    ng_alloc_EB = IntVect(ngx,ngz);
+    ng_alloc_J = IntVect(ngJx,ngJz);
+#endif
+
+    ng_alloc_Rho = ng_alloc_J+1; //One extra ghost cell, so that it's safe to deposit charge density
+    // after pushing particle.
+    int ng_alloc_F_int = (do_moving_window) ? 2 : 0;
+    // CKC solver requires one additional guard cell
+    if (maxwell_fdtd_solver_id == 1) ng_alloc_F_int = std::max( ng_alloc_F_int, 1 );
+    ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int));
+
+#ifdef WARPX_USE_PSATD
+    if (do_fft_mpi_dec == false){
+        // All boxes should have the same number of guard cells
+        // (to avoid temporary parallel copies)
+        // Thus take the max of the required number of guards for each field
+        // Also: the number of guard cell should be enough to contain
+        // the stencil of the FFT solver. Here, this number (`ngFFT`)
+        // is determined *empirically* to be the order of the solver
+        // for nodal, and half the order of the solver for staggered.
+        IntVect ngFFT;
+        if (do_nodal) {
+            ngFFT = IntVect(AMREX_D_DECL(nox_fft, noy_fft, noz_fft));
+        } else {
+            ngFFT = IntVect(AMREX_D_DECL(nox_fft/2, noy_fft/2, noz_fft/2));
+        }
+        for (int i_dim=0; i_dim<AMREX_SPACEDIM; i_dim++ ){
+            int ng_required = ngFFT[i_dim];
+            // Get the max
+            ng_required = std::max( ng_required, ng_alloc_EB[i_dim] );
+            ng_required = std::max( ng_required, ng_alloc_J[i_dim] );
+            ng_required = std::max( ng_required, ng_alloc_Rho[i_dim] );
+            ng_required = std::max( ng_required, ng_alloc_F[i_dim] );
+            // Set the guard cells to this max
+            ng_alloc_EB[i_dim] = ng_required;
+            ng_alloc_J[i_dim] = ng_required;
+            ng_alloc_F[i_dim] = ng_required;
+            ng_alloc_Rho[i_dim] = ng_required;
+            ng_alloc_F_int = ng_required;
+        }
+    }
+    ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int));
+#endif
+
+    ng_Extra = IntVect(static_cast<int>(aux_is_nodal and !do_nodal));
+
+    // Compute number of cells required for Field Solver
+#ifdef WARPX_USE_PSATD
+    ng_FieldSolver = ng_alloc_EB;
+    ng_FieldSolverF = ng_alloc_EB;
+#else
+    ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1));
+    ng_FieldSolverF = IntVect(AMREX_D_DECL(1,1,1));
+#endif
+
+    if (exchange_all_guard_cells){
+        // Run in safe mode: exchange all allocated guard cells at each
+        // call of FillBoundary
+        ng_FieldSolver = ng_alloc_EB;
+        ng_FieldSolverF = ng_alloc_F;
+        ng_FieldGather = ng_alloc_EB;
+        ng_UpdateAux = ng_alloc_EB;
+        ng_MovingWindow = ng_alloc_EB;
+    } else {
+
+        ng_FieldSolver = ng_FieldSolver.min(ng_alloc_EB);
+
+        // Compute number of cells required for Field Gather
+        int FGcell[4] = {0,1,1,2}; // Index is nox
+        IntVect ng_FieldGather_noNCI = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox]));
+        // Add one cell if momentum_conserving gather in a staggered-field simulation
+        ng_FieldGather_noNCI += ng_Extra;
+        // Not sure why, but need one extra guard cell when using MR
+        if (max_level >= 1) ng_FieldGather_noNCI += ng_Extra;
+        ng_FieldGather_noNCI = ng_FieldGather_noNCI.min(ng_alloc_EB);
+        // If NCI filter, add guard cells in the z direction
+        IntVect ng_NCIFilter = IntVect::TheZeroVector();
+        if (do_fdtd_nci_corr)
+            ng_NCIFilter[AMREX_SPACEDIM-1] = NCIGodfreyFilter::m_stencil_width;
+        // Note: communications of guard cells for bilinear filter are handled
+        // separately.
+        ng_FieldGather = ng_FieldGather_noNCI + ng_NCIFilter;
+
+        // Guard cells for auxiliary grid.
+        // Not sure why there is a 2* here...
+        ng_UpdateAux = 2*ng_FieldGather_noNCI + ng_NCIFilter;
+
+        // Make sure we do not exchange more guard cells than allocated.
+        ng_FieldGather = ng_FieldGather.min(ng_alloc_EB);
+        ng_UpdateAux = ng_UpdateAux.min(ng_alloc_EB);
+        ng_FieldSolverF = ng_FieldSolverF.min(ng_alloc_F);
+        // Only FillBoundary(ng_FieldGather) is called between consecutive
+        // field solves. So ng_FieldGather must have enough cells
+        // for the field solve too.
+        ng_FieldGather = ng_FieldGather.max(ng_FieldSolver);
+
+        if (do_moving_window){
+            ng_MovingWindow[moving_window_dir] = 1;
+        }
+    }
+}