From 74260bda9f4a8bc74fdafebcf649245c44c4d166 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sun, 27 Oct 2019 13:38:48 -0700 Subject: forgot to add two files --- Source/Parallelization/GuardCellManager.cpp | 118 ++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 Source/Parallelization/GuardCellManager.cpp (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp new file mode 100644 index 000000000..400bde28f --- /dev/null +++ b/Source/Parallelization/GuardCellManager.cpp @@ -0,0 +1,118 @@ +#include "GuardCellManager.H" + +using namespace amrex; + +void +guardCellManager::Init( + const bool do_subcycling, + const bool do_fdtd_nci_corr, + const bool do_nodal, + const bool do_moving_window, + const bool do_fft_mpi_dec, + const bool aux_is_nodal, + const int moving_window_dir, + const int nox, + const int nox_fft, const int noy_fft, const int noz_fft, + const int nci_corr_stencil, + const int maxwell_fdtd_solver_id, + const int max_level) +{ + // When using subcycling, the particles on the fine level perform two pushes + // before being redistributed ; therefore, we need one extra guard cell + // (the particles may move by 2*c*dt) + const int ngx_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; + const int ngy_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; + const int ngz_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; + + // Ex, Ey, Ez, Bx, By, and Bz have the same number of ghost cells. + // jx, jy, jz and rho have the same number of ghost cells. + // E and B have the same number of ghost cells as j and rho if NCI filter is not used, + // but different number of ghost cells in z-direction if NCI filter is used. + // The number of cells should be even, in order to easily perform the + // interpolation from coarse grid to fine grid. + int ngx = (ngx_tmp % 2) ? ngx_tmp+1 : ngx_tmp; // Always even number + int ngy = (ngy_tmp % 2) ? ngy_tmp+1 : ngy_tmp; // Always even number + int ngz_nonci = (ngz_tmp % 2) ? ngz_tmp+1 : ngz_tmp; // Always even number + int ngz; + if (do_fdtd_nci_corr) { + int ng = ngz_tmp + nci_corr_stencil; + ngz = (ng % 2) ? ng+1 : ng; + } else { + ngz = ngz_nonci; + } + + // J is only interpolated from fine to coarse (not coarse to fine) + // and therefore does not need to be even. + int ngJx = ngx_tmp; + int ngJy = ngy_tmp; + int ngJz = ngz_tmp; + + // When calling the moving window (with one level of refinement), we shift + // the fine grid by 2 cells ; therefore, we need at least 2 guard cells + // on level 1. This may not be necessary for level 0. + if (do_moving_window) { + ngx = std::max(ngx,2); + ngy = std::max(ngy,2); + ngz = std::max(ngz,2); + ngJx = std::max(ngJx,2); + ngJy = std::max(ngJy,2); + ngJz = std::max(ngJz,2); + } + +#if (AMREX_SPACEDIM == 3) + IntVect ngE(ngx,ngy,ngz); + IntVect ngJ(ngJx,ngJy,ngJz); +#elif (AMREX_SPACEDIM == 2) + IntVect ngE(ngx,ngz); + IntVect ngJ(ngJx,ngJz); +#endif + + IntVect ngRho = ngJ+1; //One extra ghost cell, so that it's safe to deposit charge density + // after pushing particle. + int ngF = (do_moving_window) ? 2 : 0; + // CKC solver requires one additional guard cell + if (maxwell_fdtd_solver_id == 1) ngF = std::max( ngF, 1 ); + +#ifdef WARPX_USE_PSATD + if (do_fft_mpi_dec == false){ + // All boxes should have the same number of guard cells + // (to avoid temporary parallel copies) + // Thus take the max of the required number of guards for each field + // Also: the number of guard cell should be enough to contain + // the stencil of the FFT solver. Here, this number (`ngFFT`) + // is determined *empirically* to be the order of the solver + // for nodal, and half the order of the solver for staggered. + IntVect ngFFT; + if (do_nodal) { + ngFFT = IntVect(AMREX_D_DECL(nox_fft, noy_fft, noz_fft)); + } else { + ngFFT = IntVect(AMREX_D_DECL(nox_fft/2, noy_fft/2, noz_fft/2)); + } + for (int i_dim=0; i_dim(aux_is_nodal and !do_nodal)); + + // Guard cells for field solver + ngB_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); + ngE_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); + ng_MovingWindow = IntVect(AMREX_D_DECL(0,0,0)); // Multiplied by number of cells moved at each timestep + ng_MovingWindow[moving_window_dir] = 1; + int FGcell[4] = {0,1,1,2}; // Index is nox + ng_FieldGather = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); + ngJ_CurrentDepo = ng_FieldGather; + ng_NCIFilter = IntVect(AMREX_D_DECL(0,0,4)); +} -- cgit v1.2.3 From 30abf69638c200cd4fdfb9305a9b02075c0df0b0 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sun, 27 Oct 2019 14:28:21 -0700 Subject: avoid redeclaring variables --- Source/Parallelization/GuardCellManager.cpp | 14 ++++++++++++++ Source/WarpX.cpp | 6 ++++++ 2 files changed, 20 insertions(+) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 400bde28f..88dab9fd1 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -59,6 +59,7 @@ guardCellManager::Init( ngJz = std::max(ngJz,2); } +/* #if (AMREX_SPACEDIM == 3) IntVect ngE(ngx,ngy,ngz); IntVect ngJ(ngJx,ngJy,ngJz); @@ -70,6 +71,19 @@ guardCellManager::Init( IntVect ngRho = ngJ+1; //One extra ghost cell, so that it's safe to deposit charge density // after pushing particle. int ngF = (do_moving_window) ? 2 : 0; +*/ + +#if (AMREX_SPACEDIM == 3) + ngE = IntVect(ngx,ngy,ngz); + ngJ = IntVect(ngJx,ngJy,ngJz); +#elif (AMREX_SPACEDIM == 2) + ngE = IntVect(ngx,ngz); + ngJ = IntVect(ngJx,ngJz); +#endif + + ngRho = ngJ+1; //One extra ghost cell, so that it's safe to deposit charge density + // after pushing particle. + ngF = (do_moving_window) ? 2 : 0; // CKC solver requires one additional guard cell if (maxwell_fdtd_solver_id == 1) ngF = std::max( ngF, 1 ); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index beb4d3bc8..6a36ebb3e 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -717,6 +717,12 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d int ngF = guard_cells.ngF; IntVect ngextra = guard_cells.ngExtra; + Print()<<"ngE "<nSpeciesDepositOnMainGrid() && n_current_deposition_buffer == 0) { n_current_deposition_buffer = 1; // This forces the allocation of buffers and allows the code associated -- cgit v1.2.3 From e63229421c47538283375f1601d2d38bb25246b9 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sun, 27 Oct 2019 16:59:48 -0700 Subject: FillBoundaryE/B/F take mandatory argument nguard --- Source/Evolve/WarpXEvolveEM.cpp | 78 ++++++++++++++--------------- Source/Parallelization/GuardCellManager.H | 3 +- Source/Parallelization/GuardCellManager.cpp | 10 ++-- Source/Parallelization/WarpXComm.cpp | 70 +++++++++++++++----------- Source/Python/WarpXWrappers.cpp | 4 +- Source/WarpX.H | 21 ++++---- Source/WarpX.cpp | 6 +-- 7 files changed, 106 insertions(+), 86 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 7a3262703..d2832cb5f 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -75,8 +75,8 @@ WarpX::EvolveEM (int numsteps) // Particles have p^{n} and x^{n}. // is_synchronized is true. if (is_synchronized) { - FillBoundaryE(); - FillBoundaryB(); + FillBoundaryE(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE); UpdateAuxilaryData(); // on first step, push p by -0.5*dt for (int lev = 0; lev <= finest_level; ++lev) { @@ -89,8 +89,8 @@ WarpX::EvolveEM (int numsteps) } else { // Beyond one step, we have E^{n} and B^{n}. // Particles have p^{n-1/2} and x^{n}. - FillBoundaryE(); - FillBoundaryB(); + FillBoundaryE(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE); UpdateAuxilaryData(); } @@ -185,8 +185,8 @@ WarpX::EvolveEM (int numsteps) // slice gen // if (to_make_plot || do_insitu || to_make_slice_plot) { - FillBoundaryE(); - FillBoundaryB(); + FillBoundaryE(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE); UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -237,8 +237,8 @@ WarpX::EvolveEM (int numsteps) if (write_plot_file || do_insitu) { - FillBoundaryE(); - FillBoundaryB(); + FillBoundaryE(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE); UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -308,23 +308,23 @@ WarpX::OneStep_nosub (Real cur_time) #ifdef WARPX_USE_PSATD PushPSATD(dt[0]); if (do_pml) DampPML(); - FillBoundaryE(); - FillBoundaryB(); + FillBoundaryE(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE); #else EvolveF(0.5*dt[0], DtType::FirstHalf); - FillBoundaryF(); + FillBoundaryF(guard_cells.ngF); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} - FillBoundaryB(); + FillBoundaryB(guard_cells.ngE); EvolveE(dt[0]); // We now have E^{n+1} - FillBoundaryE(); + FillBoundaryE(guard_cells.ngE); EvolveF(0.5*dt[0], DtType::SecondHalf); EvolveB(0.5*dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); - FillBoundaryE(); + FillBoundaryE(guard_cells.ngE); } - FillBoundaryB(); + FillBoundaryB(guard_cells.ngE); #endif } @@ -368,21 +368,21 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::FirstHalf); - FillBoundaryB(fine_lev, PatchType::fine); - FillBoundaryF(fine_lev, PatchType::fine); + FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ngF); EvolveE(fine_lev, PatchType::fine, dt[fine_lev]); - FillBoundaryE(fine_lev, PatchType::fine); + FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ngE); EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::SecondHalf); if (do_pml) { DampPML(fine_lev, PatchType::fine); - FillBoundaryE(fine_lev, PatchType::fine); + FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ngE); } - FillBoundaryB(fine_lev, PatchType::fine); + FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ngE); // ii) Push particles on the coarse patch and mother grid. // Push the fields on the coarse patch and mother grid @@ -394,19 +394,19 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::coarse, dt[fine_lev]); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); - FillBoundaryB(fine_lev, PatchType::coarse); - FillBoundaryF(fine_lev, PatchType::coarse); + FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ngE); + FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ngF); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); - FillBoundaryE(fine_lev, PatchType::coarse); + FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ngE); EvolveB(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); EvolveF(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev], DtType::FirstHalf); - FillBoundaryB(coarse_lev, PatchType::fine); - FillBoundaryF(coarse_lev, PatchType::fine); + FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryF(coarse_lev, PatchType::fine, guard_cells.ngF); EvolveE(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); - FillBoundaryE(coarse_lev, PatchType::fine); + FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ngE); // iii) Get auxiliary fields on the fine grid, at dt[fine_lev] UpdateAuxilaryData(); @@ -422,22 +422,22 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::FirstHalf); - FillBoundaryB(fine_lev, PatchType::fine); - FillBoundaryF(fine_lev, PatchType::fine); + FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ngF); EvolveE(fine_lev, PatchType::fine, dt[fine_lev]); - FillBoundaryE(fine_lev, PatchType::fine); + FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ngE); EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::SecondHalf); if (do_pml) { DampPML(fine_lev, PatchType::fine); - FillBoundaryE(fine_lev, PatchType::fine); + FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ngE); } - FillBoundaryB(fine_lev, PatchType::fine); - FillBoundaryF(fine_lev, PatchType::fine); + FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ngF); // v) Push the fields on the coarse patch and mother grid // by only half a coarse step (second half) @@ -446,7 +446,7 @@ WarpX::OneStep_sub1 (Real curtime) AddRhoFromFineLevelandSumBoundary(coarse_lev, ncomps, ncomps); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); - FillBoundaryE(fine_lev, PatchType::coarse); + FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ngE); EvolveB(fine_lev, PatchType::coarse, dt[fine_lev]); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::SecondHalf); @@ -454,24 +454,24 @@ WarpX::OneStep_sub1 (Real curtime) if (do_pml) { DampPML(fine_lev, PatchType::coarse); // do it twice DampPML(fine_lev, PatchType::coarse); - FillBoundaryE(fine_lev, PatchType::coarse); + FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ngE); } - FillBoundaryB(fine_lev, PatchType::coarse); - FillBoundaryF(fine_lev, PatchType::coarse); + FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ngE); + FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ngF); EvolveE(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); - FillBoundaryE(coarse_lev, PatchType::fine); + FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ngE); EvolveB(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); EvolveF(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev], DtType::SecondHalf); if (do_pml) { DampPML(coarse_lev, PatchType::fine); - FillBoundaryE(coarse_lev, PatchType::fine); + FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ngE); } - FillBoundaryB(coarse_lev, PatchType::fine); + FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ngE); } void diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index c1ba48b9b..4b85a4332 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -26,7 +26,8 @@ public: amrex::IntVect ngE; amrex::IntVect ngJ; amrex::IntVect ngRho; - int ngF; + amrex::IntVect ngF; + int ngF_int; // Guard cells to exchange data amrex::IntVect ngB_FieldSolver; diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 88dab9fd1..54e5444af 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -83,9 +83,10 @@ guardCellManager::Init( ngRho = ngJ+1; //One extra ghost cell, so that it's safe to deposit charge density // after pushing particle. - ngF = (do_moving_window) ? 2 : 0; + ngF_int = (do_moving_window) ? 2 : 0; // CKC solver requires one additional guard cell - if (maxwell_fdtd_solver_id == 1) ngF = std::max( ngF, 1 ); + if (maxwell_fdtd_solver_id == 1) ngF_int = std::max( ngF_int, 1 ); + ngF = IntVect(AMREX_D_DECL(ngF_int, ngF_int, ngF_int)); #ifdef WARPX_USE_PSATD if (do_fft_mpi_dec == false){ @@ -108,12 +109,13 @@ guardCellManager::Init( ng_required = std::max( ng_required, ngE[i_dim] ); ng_required = std::max( ng_required, ngJ[i_dim] ); ng_required = std::max( ng_required, ngRho[i_dim] ); - ng_required = std::max( ng_required, ngF ); + ng_required = std::max( ng_required, ngF[i_dim] ); // Set the guard cells to this max ngE[i_dim] = ng_required; ngJ[i_dim] = ng_required; + ngF[i_dim] = ng_required; ngRho[i_dim] = ng_required; - ngF = ng_required; + ngF_int = ng_required; } } #endif diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 52df3dc25..0dae38e2e 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -321,41 +321,41 @@ WarpX::UpdateAuxilaryDataSameType () } void -WarpX::FillBoundaryB () +WarpX::FillBoundaryB (IntVect ng) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryB(lev); + FillBoundaryB(lev, ng); } } void -WarpX::FillBoundaryE () +WarpX::FillBoundaryE (IntVect ng) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryE(lev); + FillBoundaryE(lev, ng); } } void -WarpX::FillBoundaryF () +WarpX::FillBoundaryF (IntVect ng) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryF(lev); + FillBoundaryF(lev, ng); } } void -WarpX::FillBoundaryE(int lev) +WarpX::FillBoundaryE(int lev, IntVect ng) { - FillBoundaryE(lev, PatchType::fine); - if (lev > 0) FillBoundaryE(lev, PatchType::coarse); + FillBoundaryE(lev, PatchType::fine, ng); + if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng); } void -WarpX::FillBoundaryE (int lev, PatchType patch_type) +WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { @@ -370,8 +370,11 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type) } const auto& period = Geom(lev).periodicity(); - Vector mf{Efield_fp[lev][0].get(),Efield_fp[lev][1].get(),Efield_fp[lev][2].get()}; - amrex::FillBoundary(mf, period); + Efield_fp[lev][0]->FillBoundary(0, Efield_fp[lev][0]->nComp(), ng, period); + Efield_fp[lev][1]->FillBoundary(0, Efield_fp[lev][1]->nComp(), ng, period); + Efield_fp[lev][2]->FillBoundary(0, Efield_fp[lev][2]->nComp(), ng, period); + // Vector mf{Efield_fp[lev][0].get(),Efield_fp[lev][1].get(),Efield_fp[lev][2].get()}; + // amrex::FillBoundary(mf, period); } else if (patch_type == PatchType::coarse) { @@ -386,20 +389,23 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type) } const auto& cperiod = Geom(lev-1).periodicity(); - Vector mf{Efield_cp[lev][0].get(),Efield_cp[lev][1].get(),Efield_cp[lev][2].get()}; - amrex::FillBoundary(mf, cperiod); + Efield_cp[lev][0]->FillBoundary(0, Efield_cp[lev][0]->nComp(), ng, cperiod); + Efield_cp[lev][1]->FillBoundary(0, Efield_cp[lev][1]->nComp(), ng, cperiod); + Efield_cp[lev][2]->FillBoundary(0, Efield_cp[lev][2]->nComp(), ng, cperiod); + // Vector mf{Efield_cp[lev][0].get(),Efield_cp[lev][1].get(),Efield_cp[lev][2].get()}; + // amrex::FillBoundary(mf, cperiod); } } void -WarpX::FillBoundaryB (int lev) +WarpX::FillBoundaryB (int lev, IntVect ng) { - FillBoundaryB(lev, PatchType::fine); - if (lev > 0) FillBoundaryB(lev, PatchType::coarse); + FillBoundaryB(lev, PatchType::fine, ng); + if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng); } void -WarpX::FillBoundaryB (int lev, PatchType patch_type) +WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { @@ -413,8 +419,11 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type) pml[lev]->FillBoundaryB(patch_type); } const auto& period = Geom(lev).periodicity(); - Vector mf{Bfield_fp[lev][0].get(),Bfield_fp[lev][1].get(),Bfield_fp[lev][2].get()}; - amrex::FillBoundary(mf, period); + Bfield_fp[lev][0]->FillBoundary(0, Bfield_fp[lev][0]->nComp(), ng, period); + Bfield_fp[lev][1]->FillBoundary(0, Bfield_fp[lev][1]->nComp(), ng, period); + Bfield_fp[lev][2]->FillBoundary(0, Bfield_fp[lev][2]->nComp(), ng, period); + // Vector mf{Bfield_fp[lev][0].get(),Bfield_fp[lev][1].get(),Bfield_fp[lev][2].get()}; + // amrex::FillBoundary(mf, period); } else if (patch_type == PatchType::coarse) { @@ -428,20 +437,23 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type) pml[lev]->FillBoundaryB(patch_type); } const auto& cperiod = Geom(lev-1).periodicity(); - Vector mf{Bfield_cp[lev][0].get(),Bfield_cp[lev][1].get(),Bfield_cp[lev][2].get()}; - amrex::FillBoundary(mf, cperiod); + Bfield_cp[lev][0]->FillBoundary(0, Bfield_cp[lev][0]->nComp(), ng, cperiod); + Bfield_cp[lev][1]->FillBoundary(0, Bfield_cp[lev][1]->nComp(), ng, cperiod); + Bfield_cp[lev][2]->FillBoundary(0, Bfield_cp[lev][2]->nComp(), ng, cperiod); + // Vector mf{Bfield_cp[lev][0].get(),Bfield_cp[lev][1].get(),Bfield_cp[lev][2].get()}; + // amrex::FillBoundary(mf, cperiod); } } void -WarpX::FillBoundaryF (int lev) +WarpX::FillBoundaryF (int lev, IntVect ng) { - FillBoundaryF(lev, PatchType::fine); - if (lev > 0) FillBoundaryF(lev, PatchType::coarse); + FillBoundaryF(lev, PatchType::fine, ng); + if (lev > 0) FillBoundaryF(lev, PatchType::coarse, ng); } void -WarpX::FillBoundaryF (int lev, PatchType patch_type) +WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine && F_fp[lev]) { @@ -453,7 +465,8 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type) } const auto& period = Geom(lev).periodicity(); - F_fp[lev]->FillBoundary(period); + F_fp[lev]->FillBoundary(0, F_fp[lev]->nComp(), ng, period); + // F_fp[lev]->FillBoundary(period); } else if (patch_type == PatchType::coarse && F_cp[lev]) { @@ -465,7 +478,8 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type) } const auto& cperiod = Geom(lev-1).periodicity(); - F_cp[lev]->FillBoundary(cperiod); + F_cp[lev]->FillBoundary(0, F_cp[lev]->nComp(), ng, cperiod); + // F_cp[lev]->FillBoundary(cperiod); } } diff --git a/Source/Python/WarpXWrappers.cpp b/Source/Python/WarpXWrappers.cpp index be9ec9519..3635bcd45 100644 --- a/Source/Python/WarpXWrappers.cpp +++ b/Source/Python/WarpXWrappers.cpp @@ -378,11 +378,11 @@ extern "C" } void warpx_FillBoundaryE () { WarpX& warpx = WarpX::GetInstance(); - warpx.FillBoundaryE (); + warpx.FillBoundaryE (warpx.getngE()); } void warpx_FillBoundaryB () { WarpX& warpx = WarpX::GetInstance(); - warpx.FillBoundaryB (); + warpx.FillBoundaryB (warpx.getngE()); } void warpx_SyncCurrent () { WarpX& warpx = WarpX::GetInstance(); diff --git a/Source/WarpX.H b/Source/WarpX.H index 4e91d41e8..426609831 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -210,12 +210,12 @@ public: void UpdateAuxilaryDataSameType (); // Fill boundary cells including coarse/fine boundaries - void FillBoundaryB (); - void FillBoundaryE (); - void FillBoundaryF (); - void FillBoundaryE (int lev); - void FillBoundaryB (int lev); - void FillBoundaryF (int lev); + void FillBoundaryB (amrex::IntVect ng); + void FillBoundaryE (amrex::IntVect ng); + void FillBoundaryF (amrex::IntVect ng); + void FillBoundaryE (int lev, amrex::IntVect ng); + void FillBoundaryB (int lev, amrex::IntVect ng); + void FillBoundaryF (int lev, amrex::IntVect ng); void SyncCurrent (); void SyncRho (); @@ -295,6 +295,9 @@ public: const std::array& B, const std::array& dx, int ngrow); + const amrex::IntVect getngE() const { return guard_cells.ngE; }; + const amrex::IntVect getngF() const { return guard_cells.ngF; }; + protected: //! Tagging cells for refinement @@ -332,9 +335,9 @@ private: /// void EvolveEM(int numsteps); - void FillBoundaryB (int lev, PatchType patch_type); - void FillBoundaryE (int lev, PatchType patch_type); - void FillBoundaryF (int lev, PatchType patch_type); + void FillBoundaryB (int lev, PatchType patch_type, amrex::IntVect ng); + void FillBoundaryE (int lev, PatchType patch_type, amrex::IntVect ng); + void FillBoundaryF (int lev, PatchType patch_type, amrex::IntVect ng); void OneStep_nosub (amrex::Real t); void OneStep_sub1 (amrex::Real t); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c97c43cd9..0217e0cca 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -714,13 +714,13 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d IntVect ngE = guard_cells.ngE; IntVect ngJ = guard_cells.ngJ; IntVect ngRho = guard_cells.ngRho; - int ngF = guard_cells.ngF; + int ngF_int = guard_cells.ngF_int; IntVect ngextra = guard_cells.ngExtra; Print()<<"ngE "<nSpeciesDepositOnMainGrid() && n_current_deposition_buffer == 0) { @@ -739,7 +739,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d n_field_gather_buffer = n_current_deposition_buffer + 1; } - AllocLevelMFs(lev, ba, dm, ngE, ngJ, ngRho, ngF, ngextra, aux_is_nodal); + AllocLevelMFs(lev, ba, dm, ngE, ngJ, ngRho, ngF_int, ngextra, aux_is_nodal); } void -- cgit v1.2.3 From ca4f2ce00e8cd3e687d803787bf24f9dd2f555e5 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 28 Oct 2019 10:37:26 -0700 Subject: Pass fewer guard cells in a number of FillBoundary calls --- Source/Evolve/WarpXEvolveEM.cpp | 16 ++++++++++------ Source/Parallelization/GuardCellManager.cpp | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index d2832cb5f..8c000077a 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -75,8 +75,8 @@ WarpX::EvolveEM (int numsteps) // Particles have p^{n} and x^{n}. // is_synchronized is true. if (is_synchronized) { - FillBoundaryE(guard_cells.ngE); - FillBoundaryB(guard_cells.ngE); + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); UpdateAuxilaryData(); // on first step, push p by -0.5*dt for (int lev = 0; lev <= finest_level; ++lev) { @@ -89,7 +89,9 @@ WarpX::EvolveEM (int numsteps) } else { // Beyond one step, we have E^{n} and B^{n}. // Particles have p^{n-1/2} and x^{n}. + // This is probably overkill FillBoundaryE(guard_cells.ngE); + // This is probably overkill FillBoundaryB(guard_cells.ngE); UpdateAuxilaryData(); @@ -185,7 +187,9 @@ WarpX::EvolveEM (int numsteps) // slice gen // if (to_make_plot || do_insitu || to_make_slice_plot) { + // This is probably overkill FillBoundaryE(guard_cells.ngE); + // This is probably overkill FillBoundaryB(guard_cells.ngE); UpdateAuxilaryData(); @@ -237,7 +241,9 @@ WarpX::EvolveEM (int numsteps) if (write_plot_file || do_insitu) { + // This is probably overkill FillBoundaryE(guard_cells.ngE); + // This is probably overkill FillBoundaryB(guard_cells.ngE); UpdateAuxilaryData(); @@ -314,17 +320,15 @@ WarpX::OneStep_nosub (Real cur_time) EvolveF(0.5*dt[0], DtType::FirstHalf); FillBoundaryF(guard_cells.ngF); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} - FillBoundaryB(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE_FieldSolver); EvolveE(dt[0]); // We now have E^{n+1} - FillBoundaryE(guard_cells.ngE); + FillBoundaryE(guard_cells.ngE_FieldSolver); EvolveF(0.5*dt[0], DtType::SecondHalf); EvolveB(0.5*dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); - FillBoundaryE(guard_cells.ngE); } - FillBoundaryB(guard_cells.ngE); #endif } diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 54e5444af..c790c3472 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -118,6 +118,7 @@ guardCellManager::Init( ngF_int = ng_required; } } + ngF = IntVect(AMREX_D_DECL(ngF_int, ngF_int, ngF_int)); #endif ngExtra = IntVect(static_cast(aux_is_nodal and !do_nodal)); -- cgit v1.2.3 From 0fe1905133033f128b235a14b04135c064800521 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 29 Oct 2019 20:46:45 -0700 Subject: Reduce number of guard cells exchanged in Moving window and EvolveEM --- Source/Evolve/WarpXEvolveEM.cpp | 17 +++++++-- Source/Parallelization/GuardCellManager.H | 25 ++++++------- Source/Parallelization/GuardCellManager.cpp | 7 +++- Source/Parallelization/WarpXComm.cpp | 21 +++++++++++ Source/Utils/WarpXMovingWindow.cpp | 54 ++++++++++++++++++----------- Source/WarpX.H | 16 +++++---- 6 files changed, 96 insertions(+), 44 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 8ad3b2401..c98a4688e 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -90,15 +90,22 @@ WarpX::EvolveEM (int numsteps) // Beyond one step, we have E^{n} and B^{n}. // Particles have p^{n-1/2} and x^{n}. // This is probably overkill - FillBoundaryE(guard_cells.ngE); + // FillBoundaryE(guard_cells.ngE); // This is probably overkill - FillBoundaryB(guard_cells.ngE); + // FillBoundaryB(guard_cells.ngE); + IntVect my_nc; + my_nc = guard_cells.ng_FieldGather+guard_cells.ng_NCIFilter; + FillBoundaryE(my_nc); + FillBoundaryB(my_nc); + FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); - } if (do_subcycling == 0 || finest_level == 0) { OneStep_nosub(cur_time); + // E : guard cells are up-to-date + // B : guard cells are NOT up-to-date + // F : guard cells are NOT up-to-date } else if (do_subcycling == 1 && finest_level == 1) { OneStep_sub1(cur_time); } else { @@ -108,6 +115,7 @@ WarpX::EvolveEM (int numsteps) if (num_mirrors>0){ applyMirrors(cur_time); + // E : guard cells are NOT up-to-date } #ifdef WARPX_USE_PY @@ -191,6 +199,7 @@ WarpX::EvolveEM (int numsteps) FillBoundaryE(guard_cells.ngE); // This is probably overkill FillBoundaryB(guard_cells.ngE); + FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -245,6 +254,7 @@ WarpX::EvolveEM (int numsteps) FillBoundaryE(guard_cells.ngE); // This is probably overkill FillBoundaryB(guard_cells.ngE); + FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -325,6 +335,7 @@ WarpX::OneStep_nosub (Real cur_time) EvolveE(dt[0]); // We now have E^{n+1} FillBoundaryE(guard_cells.ngE_FieldSolver); EvolveF(0.5*dt[0], DtType::SecondHalf); + FillBoundaryF(guard_cells.ngF); EvolveB(0.5*dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 4b85a4332..706b5df79 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -22,20 +22,21 @@ public: const int max_level); // Guard cells to initialize multifabs - amrex::IntVect ngExtra; - amrex::IntVect ngE; - amrex::IntVect ngJ; - amrex::IntVect ngRho; - amrex::IntVect ngF; - int ngF_int; + amrex::IntVect ngExtra = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngE = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngJ = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngRho = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngF = amrex::IntVect::TheZeroVector(); + int ngF_int = 0; // Guard cells to exchange data - amrex::IntVect ngB_FieldSolver; - amrex::IntVect ngE_FieldSolver; - amrex::IntVect ng_FieldGather; - amrex::IntVect ngJ_CurrentDepo; - amrex::IntVect ng_MovingWindow; - amrex::IntVect ng_NCIFilter; + amrex::IntVect ngB_FieldSolver = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngE_FieldSolver = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngJ_CurrentDepo = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_NCIFilter = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_Aux = amrex::IntVect::TheZeroVector(); }; #endif // GUARDCELLMANAGER_H_ diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index c790c3472..166f0d58d 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -131,5 +131,10 @@ guardCellManager::Init( int FGcell[4] = {0,1,1,2}; // Index is nox ng_FieldGather = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); ngJ_CurrentDepo = ng_FieldGather; - ng_NCIFilter = IntVect(AMREX_D_DECL(0,0,4)); + if (do_fdtd_nci_corr){ + ng_NCIFilter = IntVect::TheZeroVector(); + ng_NCIFilter[AMREX_SPACEDIM-1] = 4; + } + ng_Aux = 2*ng_FieldGather+ng_NCIFilter; + ng_Aux = ng_Aux.min(ngE); } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 0dae38e2e..7d2473f30 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -483,6 +483,27 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) } } +void +WarpX::FillBoundaryAux (IntVect ng) +{ + for (int lev = 0; lev <= finest_level-1; ++lev) + { + FillBoundaryAux(lev, ng); + } +} + +void +WarpX::FillBoundaryAux (int lev, IntVect ng) +{ + const auto& period = Geom(lev).periodicity(); + Efield_aux[lev][0]->FillBoundary(0, Efield_aux[lev][0]->nComp(), ng, period); + Efield_aux[lev][1]->FillBoundary(0, Efield_aux[lev][1]->nComp(), ng, period); + Efield_aux[lev][2]->FillBoundary(0, Efield_aux[lev][2]->nComp(), ng, period); + Bfield_aux[lev][0]->FillBoundary(0, Bfield_aux[lev][0]->nComp(), ng, period); + Bfield_aux[lev][1]->FillBoundary(0, Bfield_aux[lev][1]->nComp(), ng, period); + Bfield_aux[lev][2]->FillBoundary(0, Bfield_aux[lev][2]->nComp(), ng, period); +} + void WarpX::SyncCurrent () { diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index c577da7f3..59810d817 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -99,32 +99,34 @@ WarpX::MoveWindow (bool move_j) for (int dim = 0; dim < 3; ++dim) { // Fine grid - shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, B_external_grid[dim]); - shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, E_external_grid[dim]); + shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE, B_external_grid[dim]); + shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE, E_external_grid[dim]); if (move_j) { - shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir); + shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, guard_cells.ngJ); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_fp(); const std::array& pml_E = pml[lev]->GetE_fp(); - shiftMF(*pml_B[dim], geom[lev], num_shift, dir); - shiftMF(*pml_E[dim], geom[lev], num_shift, dir); + IntVect ng_exchange = pml_B[dim]->nGrowVect(); + shiftMF(*pml_B[dim], geom[lev], num_shift, dir, ng_exchange); + shiftMF(*pml_E[dim], geom[lev], num_shift, dir, ng_exchange); } if (lev > 0) { // Coarse grid - shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, B_external_grid[dim]); - shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, E_external_grid[dim]); - shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir); - shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir); + shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, guard_cells.ngE, B_external_grid[dim]); + shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, guard_cells.ngE, E_external_grid[dim]); + shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE); + shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE); if (move_j) { - shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, guard_cells.ngJ); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_cp(); const std::array& pml_E = pml[lev]->GetE_cp(); - shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir); - shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir); + IntVect ng_exchange = pml_B[dim]->nGrowVect(); + shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir, ng_exchange); + shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir, ng_exchange); } } } @@ -132,19 +134,21 @@ WarpX::MoveWindow (bool move_j) // Shift scalar component F for dive cleaning if (do_dive_cleaning) { // Fine grid - shiftMF(*F_fp[lev], geom[lev], num_shift, dir); + shiftMF(*F_fp[lev], geom[lev], num_shift, dir, guard_cells.ngF); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_fp(); - shiftMF(*pml_F, geom[lev], num_shift, dir); + IntVect ng_exchange = pml_F->nGrowVect(); + shiftMF(*pml_F, geom[lev], num_shift, dir, ng_exchange); } if (lev > 0) { // Coarse grid - shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir, guard_cells.ngF); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_cp(); - shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir); + IntVect ng_exchange = pml_F->nGrowVect(); + shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir, ng_exchange); } - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, guard_cells.ngRho); } } @@ -152,10 +156,10 @@ WarpX::MoveWindow (bool move_j) if (move_j) { if (rho_fp[lev]){ // Fine grid - shiftMF(*rho_fp[lev], geom[lev], num_shift, dir); + shiftMF(*rho_fp[lev], geom[lev], num_shift, dir, guard_cells.ngRho); if (lev > 0){ // Coarse grid - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, guard_cells.ngRho); } } } @@ -204,7 +208,7 @@ WarpX::MoveWindow (bool move_j) void WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, - amrex::Real external_field) + IntVect ng_exchange, amrex::Real external_field) { BL_PROFILE("WarpX::shiftMF()"); const BoxArray& ba = mf.boxArray(); @@ -216,7 +220,15 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, MultiFab tmpmf(ba, dm, nc, ng); MultiFab::Copy(tmpmf, mf, 0, 0, nc, ng); - tmpmf.FillBoundary(geom.periodicity()); + + // Not sure why this is needed, but it is... + ng_exchange[0] = 1; + ng_exchange[1] = num_shift; // 2 + Print()<<"ng_exchange "< Date: Wed, 30 Oct 2019 11:40:55 -0700 Subject: no need to pass ng in ShiftMF --- Source/Evolve/WarpXEvolveEM.cpp | 13 ++++---- Source/Parallelization/GuardCellManager.H | 3 +- Source/Parallelization/GuardCellManager.cpp | 9 +++-- Source/Utils/WarpXMovingWindow.cpp | 52 +++++++++++++---------------- Source/WarpX.H | 2 +- 5 files changed, 39 insertions(+), 40 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index c98a4688e..8707b9fde 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -93,10 +93,9 @@ WarpX::EvolveEM (int numsteps) // FillBoundaryE(guard_cells.ngE); // This is probably overkill // FillBoundaryB(guard_cells.ngE); - IntVect my_nc; - my_nc = guard_cells.ng_FieldGather+guard_cells.ng_NCIFilter; - FillBoundaryE(my_nc); - FillBoundaryB(my_nc); + IntVect ng_gather = guard_cells.ng_FieldGather+guard_cells.ng_NCIFilter; + FillBoundaryE(ng_gather); + FillBoundaryB(ng_gather); FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); } @@ -328,12 +327,12 @@ WarpX::OneStep_nosub (Real cur_time) FillBoundaryB(guard_cells.ngE); #else EvolveF(0.5*dt[0], DtType::FirstHalf); - FillBoundaryF(guard_cells.ngF); + FillBoundaryF(guard_cells.ng_FieldSolver); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} - FillBoundaryB(guard_cells.ngE_FieldSolver); + FillBoundaryB(guard_cells.ng_FieldSolver); EvolveE(dt[0]); // We now have E^{n+1} - FillBoundaryE(guard_cells.ngE_FieldSolver); + FillBoundaryE(guard_cells.ng_FieldSolver); EvolveF(0.5*dt[0], DtType::SecondHalf); FillBoundaryF(guard_cells.ngF); EvolveB(0.5*dt[0]); // We now have B^{n+1} diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 706b5df79..e241eed75 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -30,8 +30,7 @@ public: int ngF_int = 0; // Guard cells to exchange data - amrex::IntVect ngB_FieldSolver = amrex::IntVect::TheZeroVector(); - amrex::IntVect ngE_FieldSolver = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); amrex::IntVect ngJ_CurrentDepo = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 166f0d58d..a67e15eb7 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -1,4 +1,5 @@ #include "GuardCellManager.H" +#include using namespace amrex; @@ -121,11 +122,15 @@ guardCellManager::Init( ngF = IntVect(AMREX_D_DECL(ngF_int, ngF_int, ngF_int)); #endif + Print()<<"rrr ngE : "<(aux_is_nodal and !do_nodal)); // Guard cells for field solver - ngB_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); - ngE_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); + ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); ng_MovingWindow = IntVect(AMREX_D_DECL(0,0,0)); // Multiplied by number of cells moved at each timestep ng_MovingWindow[moving_window_dir] = 1; int FGcell[4] = {0,1,1,2}; // Index is nox diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 59810d817..db58ad53c 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -99,34 +99,32 @@ WarpX::MoveWindow (bool move_j) for (int dim = 0; dim < 3; ++dim) { // Fine grid - shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE, B_external_grid[dim]); - shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE, E_external_grid[dim]); + shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, B_external_grid[dim]); + shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, E_external_grid[dim]); if (move_j) { - shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, guard_cells.ngJ); + shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_fp(); const std::array& pml_E = pml[lev]->GetE_fp(); - IntVect ng_exchange = pml_B[dim]->nGrowVect(); - shiftMF(*pml_B[dim], geom[lev], num_shift, dir, ng_exchange); - shiftMF(*pml_E[dim], geom[lev], num_shift, dir, ng_exchange); + shiftMF(*pml_B[dim], geom[lev], num_shift, dir); + shiftMF(*pml_E[dim], geom[lev], num_shift, dir); } if (lev > 0) { // Coarse grid - shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, guard_cells.ngE, B_external_grid[dim]); - shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, guard_cells.ngE, E_external_grid[dim]); - shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE); - shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE); + shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, B_external_grid[dim]); + shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, E_external_grid[dim]); + shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir); + shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir); if (move_j) { - shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, guard_cells.ngJ); + shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_cp(); const std::array& pml_E = pml[lev]->GetE_cp(); - IntVect ng_exchange = pml_B[dim]->nGrowVect(); - shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir, ng_exchange); - shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir, ng_exchange); + shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir); } } } @@ -134,21 +132,19 @@ WarpX::MoveWindow (bool move_j) // Shift scalar component F for dive cleaning if (do_dive_cleaning) { // Fine grid - shiftMF(*F_fp[lev], geom[lev], num_shift, dir, guard_cells.ngF); + shiftMF(*F_fp[lev], geom[lev], num_shift, dir); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_fp(); - IntVect ng_exchange = pml_F->nGrowVect(); - shiftMF(*pml_F, geom[lev], num_shift, dir, ng_exchange); + shiftMF(*pml_F, geom[lev], num_shift, dir); } if (lev > 0) { // Coarse grid - shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir, guard_cells.ngF); + shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_cp(); - IntVect ng_exchange = pml_F->nGrowVect(); - shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir, ng_exchange); + shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir); } - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, guard_cells.ngRho); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); } } @@ -156,10 +152,10 @@ WarpX::MoveWindow (bool move_j) if (move_j) { if (rho_fp[lev]){ // Fine grid - shiftMF(*rho_fp[lev], geom[lev], num_shift, dir, guard_cells.ngRho); + shiftMF(*rho_fp[lev], geom[lev], num_shift, dir); if (lev > 0){ // Coarse grid - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, guard_cells.ngRho); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); } } } @@ -208,7 +204,7 @@ WarpX::MoveWindow (bool move_j) void WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, - IntVect ng_exchange, amrex::Real external_field) + amrex::Real external_field) { BL_PROFILE("WarpX::shiftMF()"); const BoxArray& ba = mf.boxArray(); @@ -222,11 +218,11 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, MultiFab::Copy(tmpmf, mf, 0, 0, nc, ng); // Not sure why this is needed, but it is... - ng_exchange[0] = 1; - ng_exchange[1] = num_shift; // 2 - Print()<<"ng_exchange "< Date: Thu, 31 Oct 2019 16:40:10 -0700 Subject: Each call to FillBoundary in regular PIC loop (FDTD) uses fewer guard cells --- Source/Evolve/WarpXEvolveEM.cpp | 39 +++++++++++++---------- Source/Parallelization/GuardCellManager.H | 1 + Source/Parallelization/GuardCellManager.cpp | 27 +++++++++++----- Source/Parallelization/WarpXComm.cpp | 49 ++++++++++++++++++----------- Source/Utils/WarpXMovingWindow.cpp | 48 +++++++++++++++------------- Source/WarpX.H | 10 +++--- 6 files changed, 104 insertions(+), 70 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 8707b9fde..3247ca64f 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -75,8 +75,10 @@ WarpX::EvolveEM (int numsteps) // Particles have p^{n} and x^{n}. // is_synchronized is true. if (is_synchronized) { - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); + // FillBoundaryE(guard_cells.ng_FieldGather, guard_cells.ngExtra); + // FillBoundaryB(guard_cells.ng_FieldGather, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); UpdateAuxilaryData(); // on first step, push p by -0.5*dt for (int lev = 0; lev <= finest_level; ++lev) { @@ -90,12 +92,14 @@ WarpX::EvolveEM (int numsteps) // Beyond one step, we have E^{n} and B^{n}. // Particles have p^{n-1/2} and x^{n}. // This is probably overkill - // FillBoundaryE(guard_cells.ngE); // This is probably overkill - // FillBoundaryB(guard_cells.ngE); - IntVect ng_gather = guard_cells.ng_FieldGather+guard_cells.ng_NCIFilter; - FillBoundaryE(ng_gather); - FillBoundaryB(ng_gather); + // IntVect ng_gather = guard_cells.ng_FieldGather+guard_cells.ng_NCIFilter; + // FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); + // FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); + // FillBoundaryE(ng_gather, guard_cells.ngExtra); + // FillBoundaryB(ng_gather, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ng_FieldGatherAndNCIFilter, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ng_FieldGatherAndNCIFilter, guard_cells.ngExtra); FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); } @@ -195,9 +199,9 @@ WarpX::EvolveEM (int numsteps) if (to_make_plot || do_insitu || to_make_slice_plot) { // This is probably overkill - FillBoundaryE(guard_cells.ngE); + FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); // This is probably overkill - FillBoundaryB(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); @@ -250,9 +254,9 @@ WarpX::EvolveEM (int numsteps) if (write_plot_file || do_insitu) { // This is probably overkill - FillBoundaryE(guard_cells.ngE); + FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); // This is probably overkill - FillBoundaryB(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); @@ -323,23 +327,24 @@ WarpX::OneStep_nosub (Real cur_time) #ifdef WARPX_USE_PSATD PushPSATD(dt[0]); if (do_pml) DampPML(); - FillBoundaryE(guard_cells.ngE); - FillBoundaryB(guard_cells.ngE); + FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); #else + Print()<<"Push fields\n"; EvolveF(0.5*dt[0], DtType::FirstHalf); FillBoundaryF(guard_cells.ng_FieldSolver); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} - FillBoundaryB(guard_cells.ng_FieldSolver); - + // FillBoundaryB(guard_cells.ng_FieldSolver, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); EvolveE(dt[0]); // We now have E^{n+1} - FillBoundaryE(guard_cells.ng_FieldSolver); + // FillBoundaryE(guard_cells.ng_FieldSolver, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); EvolveF(0.5*dt[0], DtType::SecondHalf); FillBoundaryF(guard_cells.ngF); EvolveB(0.5*dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); } - #endif } diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index e241eed75..93f1e9fd1 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -32,6 +32,7 @@ public: // Guard cells to exchange data amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_FieldGatherAndNCIFilter = amrex::IntVect::TheZeroVector(); amrex::IntVect ngJ_CurrentDepo = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_NCIFilter = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index a67e15eb7..9f3e5b18d 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -110,7 +110,7 @@ guardCellManager::Init( ng_required = std::max( ng_required, ngE[i_dim] ); ng_required = std::max( ng_required, ngJ[i_dim] ); ng_required = std::max( ng_required, ngRho[i_dim] ); - ng_required = std::max( ng_required, ngF[i_dim] ); +v ng_required = std::max( ng_required, ngF[i_dim] ); // Set the guard cells to this max ngE[i_dim] = ng_required; ngJ[i_dim] = ng_required; @@ -122,11 +122,6 @@ guardCellManager::Init( ngF = IntVect(AMREX_D_DECL(ngF_int, ngF_int, ngF_int)); #endif - Print()<<"rrr ngE : "<(aux_is_nodal and !do_nodal)); // Guard cells for field solver @@ -134,12 +129,28 @@ guardCellManager::Init( ng_MovingWindow = IntVect(AMREX_D_DECL(0,0,0)); // Multiplied by number of cells moved at each timestep ng_MovingWindow[moving_window_dir] = 1; int FGcell[4] = {0,1,1,2}; // Index is nox - ng_FieldGather = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); - ngJ_CurrentDepo = ng_FieldGather; + ng_FieldGather = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])) + 2*ngExtra; + + ng_FieldGather = ng_FieldGather.min(ngE); if (do_fdtd_nci_corr){ ng_NCIFilter = IntVect::TheZeroVector(); ng_NCIFilter[AMREX_SPACEDIM-1] = 4; } + + ng_FieldGatherAndNCIFilter = ng_FieldGather + ng_NCIFilter; + ng_FieldGatherAndNCIFilter = ng_FieldGatherAndNCIFilter.min(ngE); + ng_Aux = 2*ng_FieldGather+ng_NCIFilter; ng_Aux = ng_Aux.min(ngE); + + Print()<<"rrr ngE : "< #include #include +#include "WarpXAlgorithmSelection.H" #include #include @@ -321,20 +322,20 @@ WarpX::UpdateAuxilaryDataSameType () } void -WarpX::FillBoundaryB (IntVect ng) +WarpX::FillBoundaryB (IntVect ng, IntVect ng_extra_fine) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryB(lev, ng); + FillBoundaryB(lev, ng, ng_extra_fine); } } void -WarpX::FillBoundaryE (IntVect ng) +WarpX::FillBoundaryE (IntVect ng, IntVect ng_extra_fine) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryE(lev, ng); + FillBoundaryE(lev, ng, ng_extra_fine); } } @@ -348,9 +349,11 @@ WarpX::FillBoundaryF (IntVect ng) } void -WarpX::FillBoundaryE(int lev, IntVect ng) +WarpX::FillBoundaryE(int lev, IntVect ng, IntVect ng_extra_fine) { - FillBoundaryE(lev, PatchType::fine, ng); + Print()<<"FillBoundaryE ng_extra_fine "<< ng_extra_fine <<'\n'; + Print()<<"FillBoundaryE exchanges "<< ng+ng_extra_fine <<'\n'; + FillBoundaryE(lev, PatchType::fine, ng+ng_extra_fine); if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng); } @@ -370,11 +373,12 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) } const auto& period = Geom(lev).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Efield_fp[lev][0]->nGrowVect(), + "Error: in FillBoundaryE, requested more guard cells than allocated"); Efield_fp[lev][0]->FillBoundary(0, Efield_fp[lev][0]->nComp(), ng, period); Efield_fp[lev][1]->FillBoundary(0, Efield_fp[lev][1]->nComp(), ng, period); Efield_fp[lev][2]->FillBoundary(0, Efield_fp[lev][2]->nComp(), ng, period); - // Vector mf{Efield_fp[lev][0].get(),Efield_fp[lev][1].get(),Efield_fp[lev][2].get()}; - // amrex::FillBoundary(mf, period); } else if (patch_type == PatchType::coarse) { @@ -389,18 +393,21 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) } const auto& cperiod = Geom(lev-1).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Efield_cp[lev][0]->nGrowVect(), + "Error: in FillBoundaryE, requested more guard cells than allocated"); Efield_cp[lev][0]->FillBoundary(0, Efield_cp[lev][0]->nComp(), ng, cperiod); Efield_cp[lev][1]->FillBoundary(0, Efield_cp[lev][1]->nComp(), ng, cperiod); Efield_cp[lev][2]->FillBoundary(0, Efield_cp[lev][2]->nComp(), ng, cperiod); - // Vector mf{Efield_cp[lev][0].get(),Efield_cp[lev][1].get(),Efield_cp[lev][2].get()}; - // amrex::FillBoundary(mf, cperiod); } } void -WarpX::FillBoundaryB (int lev, IntVect ng) +WarpX::FillBoundaryB (int lev, IntVect ng, IntVect ng_extra_fine) { - FillBoundaryB(lev, PatchType::fine, ng); + Print()<<"FillBoundaryB ng_extra_fine "<< ng_extra_fine <<'\n'; + Print()<<"FillBoundaryB exchanges "<< ng+ng_extra_fine <<'\n'; + FillBoundaryB(lev, PatchType::fine, ng + ng_extra_fine); if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng); } @@ -419,11 +426,12 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) pml[lev]->FillBoundaryB(patch_type); } const auto& period = Geom(lev).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Bfield_fp[lev][0]->nGrowVect(), + "Error: in FillBoundaryB, requested more guard cells than allocated"); Bfield_fp[lev][0]->FillBoundary(0, Bfield_fp[lev][0]->nComp(), ng, period); Bfield_fp[lev][1]->FillBoundary(0, Bfield_fp[lev][1]->nComp(), ng, period); Bfield_fp[lev][2]->FillBoundary(0, Bfield_fp[lev][2]->nComp(), ng, period); - // Vector mf{Bfield_fp[lev][0].get(),Bfield_fp[lev][1].get(),Bfield_fp[lev][2].get()}; - // amrex::FillBoundary(mf, period); } else if (patch_type == PatchType::coarse) { @@ -437,11 +445,12 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) pml[lev]->FillBoundaryB(patch_type); } const auto& cperiod = Geom(lev-1).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Bfield_cp[lev][0]->nGrowVect(), + "Error: in FillBoundaryB, requested more guard cells than allocated"); Bfield_cp[lev][0]->FillBoundary(0, Bfield_cp[lev][0]->nComp(), ng, cperiod); Bfield_cp[lev][1]->FillBoundary(0, Bfield_cp[lev][1]->nComp(), ng, cperiod); Bfield_cp[lev][2]->FillBoundary(0, Bfield_cp[lev][2]->nComp(), ng, cperiod); - // Vector mf{Bfield_cp[lev][0].get(),Bfield_cp[lev][1].get(),Bfield_cp[lev][2].get()}; - // amrex::FillBoundary(mf, cperiod); } } @@ -465,8 +474,10 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) } const auto& period = Geom(lev).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= F_fp[lev]->nGrowVect(), + "Error: in FillBoundaryF, requested more guard cells than allocated"); F_fp[lev]->FillBoundary(0, F_fp[lev]->nComp(), ng, period); - // F_fp[lev]->FillBoundary(period); } else if (patch_type == PatchType::coarse && F_cp[lev]) { @@ -478,8 +489,10 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) } const auto& cperiod = Geom(lev-1).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= F_cp[lev]->nGrowVect(), + "Error: in FillBoundaryF, requested more guard cells than allocated"); F_cp[lev]->FillBoundary(0, F_cp[lev]->nComp(), ng, cperiod); - // F_cp[lev]->FillBoundary(cperiod); } } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index db58ad53c..8770d2059 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -1,3 +1,4 @@ +#include "GuardCellManager.H" #include #include @@ -28,6 +29,9 @@ WarpX::MoveWindow (bool move_j) { if (do_moving_window == 0) return 0; + IntVect ng_extra = guard_cells.ngExtra; + IntVect ng_zero = IntVect::TheZeroVector(); + // Update the continuous position of the moving window, // and of the plasma injection moving_window_x += moving_window_v * dt[0]; @@ -99,32 +103,32 @@ WarpX::MoveWindow (bool move_j) for (int dim = 0; dim < 3; ++dim) { // Fine grid - shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, B_external_grid[dim]); - shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, E_external_grid[dim]); + shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, ng_extra, B_external_grid[dim]); + shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, ng_extra, E_external_grid[dim]); if (move_j) { - shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir); + shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, ng_zero); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_fp(); const std::array& pml_E = pml[lev]->GetE_fp(); - shiftMF(*pml_B[dim], geom[lev], num_shift, dir); - shiftMF(*pml_E[dim], geom[lev], num_shift, dir); + shiftMF(*pml_B[dim], geom[lev], num_shift, dir, ng_extra); + shiftMF(*pml_E[dim], geom[lev], num_shift, dir, ng_extra); } if (lev > 0) { // Coarse grid - shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, B_external_grid[dim]); - shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, E_external_grid[dim]); - shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir); - shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir); + shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero, B_external_grid[dim]); + shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero, E_external_grid[dim]); + shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, ng_zero); + shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, ng_zero); if (move_j) { - shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_cp(); const std::array& pml_E = pml[lev]->GetE_cp(); - shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir); - shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir, ng_extra); + shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir, ng_extra); } } } @@ -132,19 +136,19 @@ WarpX::MoveWindow (bool move_j) // Shift scalar component F for dive cleaning if (do_dive_cleaning) { // Fine grid - shiftMF(*F_fp[lev], geom[lev], num_shift, dir); + shiftMF(*F_fp[lev], geom[lev], num_shift, dir, ng_zero); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_fp(); - shiftMF(*pml_F, geom[lev], num_shift, dir); + shiftMF(*pml_F, geom[lev], num_shift, dir, ng_extra); } if (lev > 0) { // Coarse grid - shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir, ng_zero); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_cp(); - shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir); + shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir, ng_zero); } - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, ng_zero); } } @@ -152,10 +156,10 @@ WarpX::MoveWindow (bool move_j) if (move_j) { if (rho_fp[lev]){ // Fine grid - shiftMF(*rho_fp[lev], geom[lev], num_shift, dir); + shiftMF(*rho_fp[lev], geom[lev], num_shift, dir, ng_zero); if (lev > 0){ // Coarse grid - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, ng_zero); } } } @@ -204,7 +208,7 @@ WarpX::MoveWindow (bool move_j) void WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, - amrex::Real external_field) + IntVect ng_extra, amrex::Real external_field) { BL_PROFILE("WarpX::shiftMF()"); const BoxArray& ba = mf.boxArray(); @@ -220,12 +224,12 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, // Not sure why this is needed, but it is... IntVect ng_mw = IntVect::TheUnitVector(); ng_mw[dir] = num_shift; + ng_mw += ng_extra; + ng_mw = ng_mw.min(ng); Print()<<"ng_mw "< Date: Thu, 31 Oct 2019 17:37:06 -0700 Subject: remove a bunch of unused variables, and prepare for some renaming --- Source/Evolve/WarpXEvolveEM.cpp | 17 ++++----- Source/Parallelization/GuardCellManager.H | 15 +++++--- Source/Parallelization/GuardCellManager.cpp | 56 +++++++++++++---------------- 3 files changed, 41 insertions(+), 47 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 3247ca64f..4e44aea48 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -75,8 +75,7 @@ WarpX::EvolveEM (int numsteps) // Particles have p^{n} and x^{n}. // is_synchronized is true. if (is_synchronized) { - // FillBoundaryE(guard_cells.ng_FieldGather, guard_cells.ngExtra); - // FillBoundaryB(guard_cells.ng_FieldGather, guard_cells.ngExtra); + // Not called at each iteration, so exchange all guard cells FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); UpdateAuxilaryData(); @@ -91,15 +90,11 @@ WarpX::EvolveEM (int numsteps) } else { // Beyond one step, we have E^{n} and B^{n}. // Particles have p^{n-1/2} and x^{n}. - // This is probably overkill - // This is probably overkill - // IntVect ng_gather = guard_cells.ng_FieldGather+guard_cells.ng_NCIFilter; - // FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); - // FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); - // FillBoundaryE(ng_gather, guard_cells.ngExtra); - // FillBoundaryB(ng_gather, guard_cells.ngExtra); - FillBoundaryE(guard_cells.ng_FieldGatherAndNCIFilter, guard_cells.ngExtra); - FillBoundaryB(guard_cells.ng_FieldGatherAndNCIFilter, guard_cells.ngExtra); + + // E and B are up-to-date inside the domain only + FillBoundaryE(guard_cells.ng_FieldGather, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ng_FieldGather, guard_cells.ngExtra); + // E and B are up-to-date inside the domain only FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); } diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 93f1e9fd1..19b5ac14a 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -22,21 +22,26 @@ public: const int max_level); // Guard cells to initialize multifabs - amrex::IntVect ngExtra = amrex::IntVect::TheZeroVector(); amrex::IntVect ngE = amrex::IntVect::TheZeroVector(); amrex::IntVect ngJ = amrex::IntVect::TheZeroVector(); amrex::IntVect ngRho = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngExtra = amrex::IntVect::TheZeroVector(); amrex::IntVect ngF = amrex::IntVect::TheZeroVector(); int ngF_int = 0; + + //amrex::IntVect ng_alloc_EB = amrex::IntVect::TheZeroVector(); + //amrex::IntVect ng_alloc_J = amrex::IntVect::TheZeroVector(); + //amrex::IntVect ng_alloc_Rho = amrex::IntVect::TheZeroVector(); + //amrex::IntVect ng_alloc_F = amrex::IntVect::TheZeroVector(); + //int ng_alloc_F_int = 0; // Guard cells to exchange data amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); - amrex::IntVect ng_FieldGatherAndNCIFilter = amrex::IntVect::TheZeroVector(); - amrex::IntVect ngJ_CurrentDepo = amrex::IntVect::TheZeroVector(); - amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); - amrex::IntVect ng_NCIFilter = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_Aux = amrex::IntVect::TheZeroVector(); + + // Extra guard cells for fine level of E and B + amrex::IntVect ng_Extra = amrex::IntVect::TheZeroVector(); }; #endif // GUARDCELLMANAGER_H_ diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 9f3e5b18d..3777c7fcc 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -1,4 +1,5 @@ #include "GuardCellManager.H" +#include "NCIGodfreyFilter.H" #include using namespace amrex; @@ -60,20 +61,6 @@ guardCellManager::Init( ngJz = std::max(ngJz,2); } -/* -#if (AMREX_SPACEDIM == 3) - IntVect ngE(ngx,ngy,ngz); - IntVect ngJ(ngJx,ngJy,ngJz); -#elif (AMREX_SPACEDIM == 2) - IntVect ngE(ngx,ngz); - IntVect ngJ(ngJx,ngJz); -#endif - - IntVect ngRho = ngJ+1; //One extra ghost cell, so that it's safe to deposit charge density - // after pushing particle. - int ngF = (do_moving_window) ? 2 : 0; -*/ - #if (AMREX_SPACEDIM == 3) ngE = IntVect(ngx,ngy,ngz); ngJ = IntVect(ngJx,ngJy,ngJz); @@ -110,7 +97,7 @@ guardCellManager::Init( ng_required = std::max( ng_required, ngE[i_dim] ); ng_required = std::max( ng_required, ngJ[i_dim] ); ng_required = std::max( ng_required, ngRho[i_dim] ); -v ng_required = std::max( ng_required, ngF[i_dim] ); + ng_required = std::max( ng_required, ngF[i_dim] ); // Set the guard cells to this max ngE[i_dim] = ng_required; ngJ[i_dim] = ng_required; @@ -124,23 +111,30 @@ v ng_required = std::max( ng_required, ngF[i_dim] ); ngExtra = IntVect(static_cast(aux_is_nodal and !do_nodal)); - // Guard cells for field solver + // Compute number of cells required for Field Solver ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); - ng_MovingWindow = IntVect(AMREX_D_DECL(0,0,0)); // Multiplied by number of cells moved at each timestep - ng_MovingWindow[moving_window_dir] = 1; - int FGcell[4] = {0,1,1,2}; // Index is nox - ng_FieldGather = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])) + 2*ngExtra; + // Compute number of cells required for Field Gather + int FGcell[4] = {0,1,1,2}; // Index is nox + IntVect ng_FieldGather_noNCI = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); + // Add one cell if momentum_conserving gather in a staggered-field simulation + ng_FieldGather_noNCI += ngExtra; + // Not sure why, but need one extra guard cell when using MR + if (max_level >= 1) ng_FieldGather_noNCI += ngExtra; + ng_FieldGather_noNCI = ng_FieldGather_noNCI.min(ngE); + // If NCI filter, add guard cells in the z direction + IntVect ng_NCIFilter = IntVect::TheZeroVector(); + if (do_fdtd_nci_corr) + ng_NCIFilter[AMREX_SPACEDIM-1] = NCIGodfreyFilter::m_stencil_width; + // Note: communications of guard cells for bilinear filter are handled + // separately. + ng_FieldGather = ng_FieldGather_noNCI + ng_NCIFilter; + + // Guard cells for auxiliary grid + ng_Aux = 2*ng_FieldGather_noNCI + ng_NCIFilter; + + // Make sure we do not exchange more guard cells than allocated. ng_FieldGather = ng_FieldGather.min(ngE); - if (do_fdtd_nci_corr){ - ng_NCIFilter = IntVect::TheZeroVector(); - ng_NCIFilter[AMREX_SPACEDIM-1] = 4; - } - - ng_FieldGatherAndNCIFilter = ng_FieldGather + ng_NCIFilter; - ng_FieldGatherAndNCIFilter = ng_FieldGatherAndNCIFilter.min(ngE); - - ng_Aux = 2*ng_FieldGather+ng_NCIFilter; ng_Aux = ng_Aux.min(ngE); Print()<<"rrr ngE : "< Date: Thu, 31 Oct 2019 18:00:23 -0700 Subject: more systematic names for guard cell variables --- Source/Evolve/WarpXEvolveEM.cpp | 79 +++++++++++++++-------------- Source/Parallelization/GuardCellManager.H | 20 +++----- Source/Parallelization/GuardCellManager.cpp | 61 +++++++++------------- Source/Utils/WarpXMovingWindow.cpp | 2 +- Source/WarpX.H | 4 +- Source/WarpX.cpp | 24 ++++----- 6 files changed, 84 insertions(+), 106 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 4e44aea48..2d28641d2 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -76,8 +76,8 @@ WarpX::EvolveEM (int numsteps) // is_synchronized is true. if (is_synchronized) { // Not called at each iteration, so exchange all guard cells - FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); - FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); + FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); UpdateAuxilaryData(); // on first step, push p by -0.5*dt for (int lev = 0; lev <= finest_level; ++lev) { @@ -92,10 +92,11 @@ WarpX::EvolveEM (int numsteps) // Particles have p^{n-1/2} and x^{n}. // E and B are up-to-date inside the domain only - FillBoundaryE(guard_cells.ng_FieldGather, guard_cells.ngExtra); - FillBoundaryB(guard_cells.ng_FieldGather, guard_cells.ngExtra); - // E and B are up-to-date inside the domain only - FillBoundaryAux(guard_cells.ng_Aux); + FillBoundaryE(guard_cells.ng_FieldGather, guard_cells.ng_Extra); + FillBoundaryB(guard_cells.ng_FieldGather, guard_cells.ng_Extra); + // E and B: enough guard cells to update Aux or call Field Gather in fp and cp + // Need to update Aux on lower levels, to interpolate to higher levels. + FillBoundaryAux(guard_cells.ng_UpdateAux); UpdateAuxilaryData(); } @@ -194,10 +195,10 @@ WarpX::EvolveEM (int numsteps) if (to_make_plot || do_insitu || to_make_slice_plot) { // This is probably overkill - FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); // This is probably overkill - FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); - FillBoundaryAux(guard_cells.ng_Aux); + FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); + FillBoundaryAux(guard_cells.ng_UpdateAux); UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -249,10 +250,10 @@ WarpX::EvolveEM (int numsteps) if (write_plot_file || do_insitu) { // This is probably overkill - FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); // This is probably overkill - FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); - FillBoundaryAux(guard_cells.ng_Aux); + FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); + FillBoundaryAux(guard_cells.ng_UpdateAux); UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -322,8 +323,8 @@ WarpX::OneStep_nosub (Real cur_time) #ifdef WARPX_USE_PSATD PushPSATD(dt[0]); if (do_pml) DampPML(); - FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); - FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); + FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); #else Print()<<"Push fields\n"; EvolveF(0.5*dt[0], DtType::FirstHalf); @@ -335,7 +336,7 @@ WarpX::OneStep_nosub (Real cur_time) // FillBoundaryE(guard_cells.ng_FieldSolver, guard_cells.ngExtra); FillBoundaryE(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); EvolveF(0.5*dt[0], DtType::SecondHalf); - FillBoundaryF(guard_cells.ngF); + FillBoundaryF(guard_cells.ng_alloc_F); EvolveB(0.5*dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); @@ -382,21 +383,21 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::FirstHalf); - FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ngE); - FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ngF); + FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_alloc_EB); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_alloc_F); EvolveE(fine_lev, PatchType::fine, dt[fine_lev]); - FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_alloc_EB); EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::SecondHalf); if (do_pml) { DampPML(fine_lev, PatchType::fine); - FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_alloc_EB); } - FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_alloc_EB); // ii) Push particles on the coarse patch and mother grid. // Push the fields on the coarse patch and mother grid @@ -408,19 +409,19 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::coarse, dt[fine_lev]); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); - FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ngE); - FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ngF); + FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ng_alloc_EB); + FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_alloc_F); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); - FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ngE); + FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_alloc_EB); EvolveB(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); EvolveF(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev], DtType::FirstHalf); - FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ngE); - FillBoundaryF(coarse_lev, PatchType::fine, guard_cells.ngF); + FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_alloc_EB); + FillBoundaryF(coarse_lev, PatchType::fine, guard_cells.ng_alloc_F); EvolveE(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); - FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_alloc_EB); // iii) Get auxiliary fields on the fine grid, at dt[fine_lev] UpdateAuxilaryData(); @@ -436,22 +437,22 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::FirstHalf); - FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ngE); - FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ngF); + FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_alloc_EB); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_alloc_F); EvolveE(fine_lev, PatchType::fine, dt[fine_lev]); - FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_alloc_EB); EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::SecondHalf); if (do_pml) { DampPML(fine_lev, PatchType::fine); - FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_alloc_EB); } - FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ngE); - FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ngF); + FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_alloc_EB); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_alloc_F); // v) Push the fields on the coarse patch and mother grid // by only half a coarse step (second half) @@ -460,7 +461,7 @@ WarpX::OneStep_sub1 (Real curtime) AddRhoFromFineLevelandSumBoundary(coarse_lev, ncomps, ncomps); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); - FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ngE); + FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_alloc_EB); EvolveB(fine_lev, PatchType::coarse, dt[fine_lev]); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::SecondHalf); @@ -468,24 +469,24 @@ WarpX::OneStep_sub1 (Real curtime) if (do_pml) { DampPML(fine_lev, PatchType::coarse); // do it twice DampPML(fine_lev, PatchType::coarse); - FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ngE); + FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_alloc_EB); } - FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ngE); - FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ngF); + FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ng_alloc_EB); + FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_alloc_F); EvolveE(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); - FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_alloc_EB); EvolveB(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); EvolveF(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev], DtType::SecondHalf); if (do_pml) { DampPML(coarse_lev, PatchType::fine); - FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_alloc_EB); } - FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ngE); + FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_alloc_EB); } void diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 19b5ac14a..7e6a0d9f5 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -21,24 +21,16 @@ public: const int maxwell_fdtd_solver_id, const int max_level); - // Guard cells to initialize multifabs - amrex::IntVect ngE = amrex::IntVect::TheZeroVector(); - amrex::IntVect ngJ = amrex::IntVect::TheZeroVector(); - amrex::IntVect ngRho = amrex::IntVect::TheZeroVector(); - amrex::IntVect ngExtra = amrex::IntVect::TheZeroVector(); - amrex::IntVect ngF = amrex::IntVect::TheZeroVector(); - int ngF_int = 0; - - //amrex::IntVect ng_alloc_EB = amrex::IntVect::TheZeroVector(); - //amrex::IntVect ng_alloc_J = amrex::IntVect::TheZeroVector(); - //amrex::IntVect ng_alloc_Rho = amrex::IntVect::TheZeroVector(); - //amrex::IntVect ng_alloc_F = amrex::IntVect::TheZeroVector(); - //int ng_alloc_F_int = 0; + amrex::IntVect ng_alloc_EB = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_alloc_J = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_alloc_Rho = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_alloc_F = amrex::IntVect::TheZeroVector(); + int ng_alloc_F_int = 0; // Guard cells to exchange data amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); - amrex::IntVect ng_Aux = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_UpdateAux = amrex::IntVect::TheZeroVector(); // Extra guard cells for fine level of E and B amrex::IntVect ng_Extra = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 3777c7fcc..d970cd464 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -62,19 +62,19 @@ guardCellManager::Init( } #if (AMREX_SPACEDIM == 3) - ngE = IntVect(ngx,ngy,ngz); - ngJ = IntVect(ngJx,ngJy,ngJz); + ng_alloc_EB = IntVect(ngx,ngy,ngz); + ng_alloc_J = IntVect(ngJx,ngJy,ngJz); #elif (AMREX_SPACEDIM == 2) - ngE = IntVect(ngx,ngz); - ngJ = IntVect(ngJx,ngJz); + ng_alloc_EB = IntVect(ngx,ngz); + ng_alloc_J = IntVect(ngJx,ngJz); #endif - ngRho = ngJ+1; //One extra ghost cell, so that it's safe to deposit charge density + ng_alloc_Rho = ng_alloc_J+1; //One extra ghost cell, so that it's safe to deposit charge density // after pushing particle. - ngF_int = (do_moving_window) ? 2 : 0; + ng_alloc_F_int = (do_moving_window) ? 2 : 0; // CKC solver requires one additional guard cell - if (maxwell_fdtd_solver_id == 1) ngF_int = std::max( ngF_int, 1 ); - ngF = IntVect(AMREX_D_DECL(ngF_int, ngF_int, ngF_int)); + if (maxwell_fdtd_solver_id == 1) ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); + ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int)); #ifdef WARPX_USE_PSATD if (do_fft_mpi_dec == false){ @@ -94,22 +94,22 @@ guardCellManager::Init( for (int i_dim=0; i_dim(aux_is_nodal and !do_nodal)); + ng_Extra = IntVect(static_cast(aux_is_nodal and !do_nodal)); // Compute number of cells required for Field Solver ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); @@ -118,10 +118,10 @@ guardCellManager::Init( int FGcell[4] = {0,1,1,2}; // Index is nox IntVect ng_FieldGather_noNCI = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); // Add one cell if momentum_conserving gather in a staggered-field simulation - ng_FieldGather_noNCI += ngExtra; + ng_FieldGather_noNCI += ng_Extra; // Not sure why, but need one extra guard cell when using MR - if (max_level >= 1) ng_FieldGather_noNCI += ngExtra; - ng_FieldGather_noNCI = ng_FieldGather_noNCI.min(ngE); + if (max_level >= 1) ng_FieldGather_noNCI += ng_Extra; + ng_FieldGather_noNCI = ng_FieldGather_noNCI.min(ng_alloc_EB); // If NCI filter, add guard cells in the z direction IntVect ng_NCIFilter = IntVect::TheZeroVector(); if (do_fdtd_nci_corr) @@ -131,20 +131,9 @@ guardCellManager::Init( ng_FieldGather = ng_FieldGather_noNCI + ng_NCIFilter; // Guard cells for auxiliary grid - ng_Aux = 2*ng_FieldGather_noNCI + ng_NCIFilter; + ng_UpdateAux = 2*ng_FieldGather_noNCI + ng_NCIFilter; // Make sure we do not exchange more guard cells than allocated. - ng_FieldGather = ng_FieldGather.min(ngE); - ng_Aux = ng_Aux.min(ngE); - - Print()<<"rrr ngE : "<& B, const std::array& dx, int ngrow); - const amrex::IntVect getngE() const { return guard_cells.ngE; }; - const amrex::IntVect getngF() const { return guard_cells.ngF; }; + const amrex::IntVect getngE() const { return guard_cells.ng_alloc_EB; }; + const amrex::IntVect getngF() const { return guard_cells.ng_alloc_F; }; protected: diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 1915434b6..489692b35 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -717,18 +717,6 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d maxwell_fdtd_solver_id, maxLevel()); - IntVect ngE = guard_cells.ngE; - IntVect ngJ = guard_cells.ngJ; - IntVect ngRho = guard_cells.ngRho; - int ngF_int = guard_cells.ngF_int; - IntVect ngextra = guard_cells.ngExtra; - - Print()<<"ngE "<nSpeciesDepositOnMainGrid() && n_current_deposition_buffer == 0) { n_current_deposition_buffer = 1; // This forces the allocation of buffers and allows the code associated @@ -738,14 +726,22 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d } if (n_current_deposition_buffer < 0) { - n_current_deposition_buffer = ngJ.max(); + n_current_deposition_buffer = guard_cells.ng_alloc_J.max(); } if (n_field_gather_buffer < 0) { // Field gather buffer should be larger than current deposition buffers n_field_gather_buffer = n_current_deposition_buffer + 1; } - AllocLevelMFs(lev, ba, dm, ngE, ngJ, ngRho, ngF_int, ngextra, aux_is_nodal); + //IntVect ngE = guard_cells.ng_alloc_EB; + //IntVect ngJ = guard_cells.ng_alloc_J; + //IntVect ngRho = guard_cells.ng_alloc_Rho; + //int ngF_int = guard_cells.ng_alloc_F_int; + //IntVect ngextra = guard_cells.ngExtra; + + AllocLevelMFs(lev, ba, dm, guard_cells.ng_alloc_EB, guard_cells.ng_alloc_J, + guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F_int, + guard_cells.ng_Extra, aux_is_nodal); } void -- cgit v1.2.3 From 82c791de01622ff1556203902c0782e396719629 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 31 Oct 2019 18:25:55 -0700 Subject: cleaning and use better naming --- Source/Evolve/WarpXEvolveEM.cpp | 21 ++++++++++++++------- Source/Parallelization/GuardCellManager.cpp | 10 +++++++++- Source/Utils/WarpXMovingWindow.cpp | 6 ++++-- 3 files changed, 27 insertions(+), 10 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 2d28641d2..550f7a708 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -194,10 +194,11 @@ WarpX::EvolveEM (int numsteps) // slice gen // if (to_make_plot || do_insitu || to_make_slice_plot) { - // This is probably overkill + // This is probably overkill, but it's not called often FillBoundaryE(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); - // This is probably overkill + // This is probably overkill, but it's not called often FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); + // This is probably overkill, but it's not called often FillBoundaryAux(guard_cells.ng_UpdateAux); UpdateAuxilaryData(); @@ -249,10 +250,11 @@ WarpX::EvolveEM (int numsteps) if (write_plot_file || do_insitu) { - // This is probably overkill + // This is probably overkill, but it's not called often FillBoundaryE(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); - // This is probably overkill + // This is probably overkill, but it's not called often FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); + // This is probably overkill FillBoundaryAux(guard_cells.ng_UpdateAux); UpdateAuxilaryData(); @@ -314,6 +316,10 @@ WarpX::OneStep_nosub (Real cur_time) SyncRho(); + // At this point, J is up-to-date inside the domain, and E and B are + // up-to-date including enough guard cells for first step of the field + // solve. + // For extended PML: copy J from regular grid to PML, and damp J in PML if (do_pml && pml_has_particles) CopyJPML(); if (do_pml && do_pml_j_damping) DampJPML(); @@ -326,14 +332,13 @@ WarpX::OneStep_nosub (Real cur_time) FillBoundaryE(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); #else - Print()<<"Push fields\n"; EvolveF(0.5*dt[0], DtType::FirstHalf); FillBoundaryF(guard_cells.ng_FieldSolver); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} - // FillBoundaryB(guard_cells.ng_FieldSolver, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); EvolveE(dt[0]); // We now have E^{n+1} - // FillBoundaryE(guard_cells.ng_FieldSolver, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); EvolveF(0.5*dt[0], DtType::SecondHalf); FillBoundaryF(guard_cells.ng_alloc_F); @@ -341,6 +346,8 @@ WarpX::OneStep_nosub (Real cur_time) if (do_pml) { DampPML(); } + // E and B are up-to-date in the domain, but all guard cells are + // outdated. #endif } diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index d970cd464..415a13c00 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -107,12 +107,16 @@ guardCellManager::Init( } } ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int)); -#endif +#endif ng_Extra = IntVect(static_cast(aux_is_nodal and !do_nodal)); // Compute number of cells required for Field Solver +#ifdef WARPX_USE_PSATD + ng_FieldSolver = ng_alloc_EB; +#else ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); +#endif // Compute number of cells required for Field Gather int FGcell[4] = {0,1,1,2}; // Index is nox @@ -136,4 +140,8 @@ guardCellManager::Init( // Make sure we do not exchange more guard cells than allocated. ng_FieldGather = ng_FieldGather.min(ng_alloc_EB); ng_UpdateAux = ng_UpdateAux.min(ng_alloc_EB); + // Only FillBoundary(ng_FieldGather) is called between consecutive + // field solves. So ng_FieldGather must have enough cells + // for the field solve too. + ng_FieldGather = ng_FieldGather.max(ng_FieldSolver); } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index d59d0d8b0..e3819966d 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -223,11 +223,13 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, // Not sure why this is needed, but it is... IntVect ng_mw = IntVect::TheUnitVector(); + // Enough guard cells in the MW direction ng_mw[dir] = num_shift; + // Add the extra cell (if momentum-conserving gather with staggered field solve) ng_mw += ng_extra; + // Make sure we don't exceed number of guard cells allocated ng_mw = ng_mw.min(ng); - Print()<<"ng_mw "< Date: Thu, 31 Oct 2019 18:57:09 -0700 Subject: do not update aux grids if PSATD --- Source/Evolve/WarpXEvolveEM.cpp | 6 ++++++ Source/Parallelization/GuardCellManager.cpp | 11 +++++++++++ 2 files changed, 17 insertions(+) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 550f7a708..6f935d84a 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -96,7 +96,9 @@ WarpX::EvolveEM (int numsteps) FillBoundaryB(guard_cells.ng_FieldGather, guard_cells.ng_Extra); // E and B: enough guard cells to update Aux or call Field Gather in fp and cp // Need to update Aux on lower levels, to interpolate to higher levels. +#ifndef WARPX_USE_PSATD FillBoundaryAux(guard_cells.ng_UpdateAux); +#endif UpdateAuxilaryData(); } @@ -199,7 +201,9 @@ WarpX::EvolveEM (int numsteps) // This is probably overkill, but it's not called often FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); // This is probably overkill, but it's not called often +#ifndef WARPX_USE_PSATD FillBoundaryAux(guard_cells.ng_UpdateAux); +#endif UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -255,7 +259,9 @@ WarpX::EvolveEM (int numsteps) // This is probably overkill, but it's not called often FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); // This is probably overkill +#ifndef WARPX_USE_PSATD FillBoundaryAux(guard_cells.ng_UpdateAux); +#endif UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 415a13c00..60a4a3d4f 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -144,4 +144,15 @@ guardCellManager::Init( // field solves. So ng_FieldGather must have enough cells // for the field solve too. ng_FieldGather = ng_FieldGather.max(ng_FieldSolver); + + Print()<<"ng_alloc_EB "< Date: Wed, 6 Nov 2019 16:08:55 -0800 Subject: exchange 1 extra guard cell when moving window and PMLs --- Source/Evolve/WarpXEvolveEM.cpp | 2 ++ Source/Parallelization/GuardCellManager.H | 1 + Source/Parallelization/GuardCellManager.cpp | 4 ++++ 3 files changed, 7 insertions(+) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index e08344624..f6463fbe6 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -352,6 +352,8 @@ WarpX::OneStep_nosub (Real cur_time) EvolveB(0.5*dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); +FillBoundaryE(guard_cells.ng_MovingWindow, IntVect::TheZeroVector()); +FillBoundaryB(guard_cells.ng_MovingWindow, IntVect::TheZeroVector()); } // E and B are up-to-date in the domain, but all guard cells are // outdated. diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 7e6a0d9f5..53d5ccc08 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -31,6 +31,7 @@ public: amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_UpdateAux = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); // Extra guard cells for fine level of E and B amrex::IntVect ng_Extra = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 60a4a3d4f..6deb45f0e 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -145,6 +145,9 @@ guardCellManager::Init( // for the field solve too. ng_FieldGather = ng_FieldGather.max(ng_FieldSolver); + ng_MovingWindow = IntVect::TheZeroVector(); + ng_MovingWindow[moving_window_dir] = 1; + Print()<<"ng_alloc_EB "< Date: Sun, 10 Nov 2019 14:14:42 -0800 Subject: pfew fix bug (that could lead to SEGFAULT), and indentation --- Source/Evolve/WarpXEvolveEM.cpp | 4 ++-- Source/Parallelization/GuardCellManager.cpp | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index f6463fbe6..147ea3b4c 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -352,8 +352,8 @@ WarpX::OneStep_nosub (Real cur_time) EvolveB(0.5*dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); -FillBoundaryE(guard_cells.ng_MovingWindow, IntVect::TheZeroVector()); -FillBoundaryB(guard_cells.ng_MovingWindow, IntVect::TheZeroVector()); + FillBoundaryE(guard_cells.ng_MovingWindow, IntVect::TheZeroVector()); + FillBoundaryB(guard_cells.ng_MovingWindow, IntVect::TheZeroVector()); } // E and B are up-to-date in the domain, but all guard cells are // outdated. diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 6deb45f0e..9d4a06f73 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -145,8 +145,9 @@ guardCellManager::Init( // for the field solve too. ng_FieldGather = ng_FieldGather.max(ng_FieldSolver); - ng_MovingWindow = IntVect::TheZeroVector(); - ng_MovingWindow[moving_window_dir] = 1; + if (do_moving_window){ + ng_MovingWindow[moving_window_dir] = 1; + } Print()<<"ng_alloc_EB "< Date: Mon, 11 Nov 2019 18:32:23 -0800 Subject: move FillBoundaryF call to where needed, and allow for extra guard cells --- Source/Evolve/WarpXEvolveEM.cpp | 2 +- Source/Parallelization/GuardCellManager.H | 4 +++- Source/Parallelization/GuardCellManager.cpp | 15 ++++++++++++++- Source/WarpX.H | 3 +++ Source/WarpX.cpp | 14 +++++++++++++- 5 files changed, 34 insertions(+), 4 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index db3c88aa7..7131e4305 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -471,7 +471,6 @@ WarpX::OneStep_sub1 (Real curtime) } FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); - FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); // v) Push the fields on the coarse patch and mother grid // by only half a coarse step (second half) @@ -486,6 +485,7 @@ WarpX::OneStep_sub1 (Real curtime) EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::SecondHalf); if (do_pml) { + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); DampPML(fine_lev, PatchType::coarse); // do it twice DampPML(fine_lev, PatchType::coarse); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_alloc_EB); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 53d5ccc08..19378302e 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -19,7 +19,9 @@ public: const int nox_fft, const int noy_fft, const int noz_fft, const int nci_corr_stencil, const int maxwell_fdtd_solver_id, - const int max_level); + const int max_level, + const int extra_guard_cells_alloc, + const int extra_guard_cells_exchange); amrex::IntVect ng_alloc_EB = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_alloc_J = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 9d4a06f73..6a60177c0 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -17,7 +17,9 @@ guardCellManager::Init( const int nox_fft, const int noy_fft, const int noz_fft, const int nci_corr_stencil, const int maxwell_fdtd_solver_id, - const int max_level) + const int max_level, + const int extra_guard_cells_alloc, + const int extra_guard_cells_exchange) { // When using subcycling, the particles on the fine level perform two pushes // before being redistributed ; therefore, we need one extra guard cell @@ -149,6 +151,17 @@ guardCellManager::Init( ng_MovingWindow[moving_window_dir] = 1; } + ng_alloc_EB += extra_guard_cells_alloc; + ng_alloc_J += extra_guard_cells_alloc; + ng_alloc_Rho += extra_guard_cells_alloc; + ng_alloc_F += extra_guard_cells_alloc; + ng_alloc_F_int += extra_guard_cells_alloc; + + ng_FieldSolver += extra_guard_cells_exchange; + ng_FieldGather += extra_guard_cells_exchange; + ng_UpdateAux += extra_guard_cells_exchange; + ng_MovingWindow += extra_guard_cells_exchange; + Print()<<"ng_alloc_EB "<nSpeciesDepositOnMainGrid() && n_current_deposition_buffer == 0) { n_current_deposition_buffer = 1; -- cgit v1.2.3 From 35c888f9901e9da412261556b768f7d37752d965 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 12 Nov 2019 08:21:09 -0800 Subject: Revert "move FillBoundaryF call to where needed, and allow for extra guard cells" This reverts commit 89069ca7ba61dc34f9b392c75d0d4e2f3c2e0938. --- Source/Evolve/WarpXEvolveEM.cpp | 2 +- Source/Parallelization/GuardCellManager.H | 4 +--- Source/Parallelization/GuardCellManager.cpp | 15 +-------------- Source/WarpX.H | 3 --- Source/WarpX.cpp | 14 +------------- 5 files changed, 4 insertions(+), 34 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 7131e4305..db3c88aa7 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -471,6 +471,7 @@ WarpX::OneStep_sub1 (Real curtime) } FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); // v) Push the fields on the coarse patch and mother grid // by only half a coarse step (second half) @@ -485,7 +486,6 @@ WarpX::OneStep_sub1 (Real curtime) EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::SecondHalf); if (do_pml) { - FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); DampPML(fine_lev, PatchType::coarse); // do it twice DampPML(fine_lev, PatchType::coarse); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_alloc_EB); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 19378302e..53d5ccc08 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -19,9 +19,7 @@ public: const int nox_fft, const int noy_fft, const int noz_fft, const int nci_corr_stencil, const int maxwell_fdtd_solver_id, - const int max_level, - const int extra_guard_cells_alloc, - const int extra_guard_cells_exchange); + const int max_level); amrex::IntVect ng_alloc_EB = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_alloc_J = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 6a60177c0..9d4a06f73 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -17,9 +17,7 @@ guardCellManager::Init( const int nox_fft, const int noy_fft, const int noz_fft, const int nci_corr_stencil, const int maxwell_fdtd_solver_id, - const int max_level, - const int extra_guard_cells_alloc, - const int extra_guard_cells_exchange) + const int max_level) { // When using subcycling, the particles on the fine level perform two pushes // before being redistributed ; therefore, we need one extra guard cell @@ -151,17 +149,6 @@ guardCellManager::Init( ng_MovingWindow[moving_window_dir] = 1; } - ng_alloc_EB += extra_guard_cells_alloc; - ng_alloc_J += extra_guard_cells_alloc; - ng_alloc_Rho += extra_guard_cells_alloc; - ng_alloc_F += extra_guard_cells_alloc; - ng_alloc_F_int += extra_guard_cells_alloc; - - ng_FieldSolver += extra_guard_cells_exchange; - ng_FieldGather += extra_guard_cells_exchange; - ng_UpdateAux += extra_guard_cells_exchange; - ng_MovingWindow += extra_guard_cells_exchange; - Print()<<"ng_alloc_EB "<nSpeciesDepositOnMainGrid() && n_current_deposition_buffer == 0) { n_current_deposition_buffer = 1; -- cgit v1.2.3 From 8cc9c8a153b6526dad68ee2bcdcacf912969e747 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 12 Nov 2019 10:54:38 -0800 Subject: fix psatd: problem was in buffer cells --- Source/Evolve/WarpXEvolveEM.cpp | 4 +--- Source/Parallelization/GuardCellManager.H | 2 +- Source/Parallelization/GuardCellManager.cpp | 16 ++++------------ Source/Parallelization/WarpXComm.cpp | 1 - Source/WarpX.cpp | 4 ++-- 5 files changed, 8 insertions(+), 19 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 7131e4305..19031e006 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -339,8 +339,7 @@ WarpX::OneStep_nosub (Real cur_time) FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); #else EvolveF(0.5*dt[0], DtType::FirstHalf); - // FillBoundaryF(guard_cells.ng_FieldSolver); - FillBoundaryF(guard_cells.ng_alloc_F); + FillBoundaryF(guard_cells.ng_FieldSolver); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} FillBoundaryB(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); @@ -441,7 +440,6 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldGather + guard_cells.ng_Extra); FillBoundaryAux(guard_cells.ng_UpdateAux); - // iii) Get auxiliary fields on the fine grid, at dt[fine_lev] UpdateAuxilaryData(); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 53d5ccc08..c6fdecaee 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -7,7 +7,7 @@ class guardCellManager{ public: - void Init( + int Init( const bool do_subcycling, const bool do_fdtd_nci_corr, const bool do_nodal, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 9d4a06f73..cfe6e8470 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -4,7 +4,7 @@ using namespace amrex; -void +int guardCellManager::Init( const bool do_subcycling, const bool do_fdtd_nci_corr, @@ -69,6 +69,8 @@ guardCellManager::Init( ng_alloc_J = IntVect(ngJx,ngJz); #endif + int nJ_buffer = ng_alloc_J.max(); // guard cells for J required for deposition only. + ng_alloc_Rho = ng_alloc_J+1; //One extra ghost cell, so that it's safe to deposit charge density // after pushing particle. ng_alloc_F_int = (do_moving_window) ? 2 : 0; @@ -149,15 +151,5 @@ guardCellManager::Init( ng_MovingWindow[moving_window_dir] = 1; } - Print()<<"ng_alloc_EB "< #include #include -#include "WarpXAlgorithmSelection.H" #include #include diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 10a6b731e..173a3a00f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -703,7 +703,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d bool aux_is_nodal = (field_gathering_algo == GatheringAlgo::MomentumConserving); - guard_cells.Init( + int nJ_buffer = guard_cells.Init( do_subcycling, WarpX::use_fdtd_nci_corr, do_nodal, @@ -726,7 +726,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d } if (n_current_deposition_buffer < 0) { - n_current_deposition_buffer = guard_cells.ng_alloc_J.max(); + n_current_deposition_buffer = nJ_buffer; } if (n_field_gather_buffer < 0) { // Field gather buffer should be larger than current deposition buffers -- cgit v1.2.3 From bd0735f62d784b906ce34cbda81ddf083dbde33c Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 12 Nov 2019 15:34:29 -0800 Subject: cleaning, and additional checks --- Source/Evolve/WarpXEvolveEM.cpp | 1 + Source/Parallelization/GuardCellManager.H | 3 ++- Source/Parallelization/GuardCellManager.cpp | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 19031e006..38b959808 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -117,6 +117,7 @@ WarpX::EvolveEM (int numsteps) if (num_mirrors>0){ applyMirrors(cur_time); // E : guard cells are NOT up-to-date + // B : guard cells are NOT up-to-date } #ifdef WARPX_USE_PY diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index c6fdecaee..9f2bafa19 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -21,13 +21,14 @@ public: const int maxwell_fdtd_solver_id, const int max_level); + // Guard cells allocated for each multifab amrex::IntVect ng_alloc_EB = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_alloc_J = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_alloc_Rho = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_alloc_F = amrex::IntVect::TheZeroVector(); int ng_alloc_F_int = 0; - // Guard cells to exchange data + // Guard cells exchanged for specific in the PIC loop amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_UpdateAux = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index cfe6e8470..28670fb91 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -119,6 +119,7 @@ guardCellManager::Init( #else ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); #endif + ng_FieldSolver = ng_FieldSolver.min(ng_alloc_EB); // Compute number of cells required for Field Gather int FGcell[4] = {0,1,1,2}; // Index is nox @@ -136,7 +137,8 @@ guardCellManager::Init( // separately. ng_FieldGather = ng_FieldGather_noNCI + ng_NCIFilter; - // Guard cells for auxiliary grid + // Guard cells for auxiliary grid. + // Not sure why there is a 2* here... ng_UpdateAux = 2*ng_FieldGather_noNCI + ng_NCIFilter; // Make sure we do not exchange more guard cells than allocated. -- cgit v1.2.3 From 751b6e3867a2f41db88430476891fecb2e35b053 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 12 Nov 2019 16:36:39 -0800 Subject: no guard cell exchanges for F with Yee solver. Remove print statements --- Source/Evolve/WarpXEvolveEM.cpp | 12 ++++++------ Source/Parallelization/GuardCellManager.H | 1 + Source/Parallelization/GuardCellManager.cpp | 3 +++ Source/Parallelization/WarpXComm.cpp | 4 ---- 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 38b959808..af4eb9467 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -340,7 +340,7 @@ WarpX::OneStep_nosub (Real cur_time) FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); #else EvolveF(0.5*dt[0], DtType::FirstHalf); - FillBoundaryF(guard_cells.ng_FieldSolver); + FillBoundaryF(guard_cells.ng_FieldSolverF); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} FillBoundaryB(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); @@ -427,7 +427,7 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::coarse, dt[fine_lev]); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ng_FieldGather); - FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolver); + FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolverF); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_FieldGather); @@ -435,7 +435,7 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); EvolveF(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev], DtType::FirstHalf); FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_FieldGather + guard_cells.ng_Extra); - FillBoundaryF(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver); + FillBoundaryF(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolverF); EvolveE(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldGather + guard_cells.ng_Extra); @@ -456,7 +456,7 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::FirstHalf); FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); - FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolverF); EvolveE(fine_lev, PatchType::fine, dt[fine_lev]); FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); @@ -484,7 +484,7 @@ WarpX::OneStep_sub1 (Real curtime) EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::SecondHalf); if (do_pml) { - FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolverF); DampPML(fine_lev, PatchType::coarse); // do it twice DampPML(fine_lev, PatchType::coarse); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_alloc_EB); @@ -492,7 +492,7 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolver); - FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolver); + FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolverF); EvolveE(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 9f2bafa19..2e1cebff8 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -30,6 +30,7 @@ public: // Guard cells exchanged for specific in the PIC loop amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_FieldSolverF = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_UpdateAux = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 28670fb91..34454bd7e 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -116,8 +116,10 @@ guardCellManager::Init( // Compute number of cells required for Field Solver #ifdef WARPX_USE_PSATD ng_FieldSolver = ng_alloc_EB; + ng_FieldSolverF = ng_alloc_EB; #else ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); + ng_FieldSolverF = IntVect(AMREX_D_DECL(1,1,1)); #endif ng_FieldSolver = ng_FieldSolver.min(ng_alloc_EB); @@ -144,6 +146,7 @@ guardCellManager::Init( // Make sure we do not exchange more guard cells than allocated. ng_FieldGather = ng_FieldGather.min(ng_alloc_EB); ng_UpdateAux = ng_UpdateAux.min(ng_alloc_EB); + ng_FieldSolverF = ng_FieldSolverF.min(ng_alloc_F); // Only FillBoundary(ng_FieldGather) is called between consecutive // field solves. So ng_FieldGather must have enough cells // for the field solve too. diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 5d7e44a8e..f7e9086f9 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -373,7 +373,6 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Efield_fp[lev][0]->nGrowVect(), "Error: in FillBoundaryE, requested more guard cells than allocated"); - Print()<<"FillBoundaryE exchanges "<< Efield_fp[lev][0]->nGrowVect() <<'\n'; Efield_fp[lev][0]->FillBoundary(ng, period); Efield_fp[lev][1]->FillBoundary(ng, period); Efield_fp[lev][2]->FillBoundary(ng, period); @@ -394,7 +393,6 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Efield_cp[lev][0]->nGrowVect(), "Error: in FillBoundaryE, requested more guard cells than allocated"); - Print()<<"FillBoundaryE exchanges "<< Efield_cp[lev][0]->nGrowVect() <<'\n'; Efield_cp[lev][0]->FillBoundary(ng, cperiod); Efield_cp[lev][1]->FillBoundary(ng, cperiod); Efield_cp[lev][2]->FillBoundary(ng, cperiod); @@ -426,7 +424,6 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Bfield_fp[lev][0]->nGrowVect(), "Error: in FillBoundaryB, requested more guard cells than allocated"); - Print()<<"FillBoundaryB exchanges "<< Bfield_fp[lev][0]->nGrowVect() <<'\n'; Bfield_fp[lev][0]->FillBoundary(ng, period); Bfield_fp[lev][1]->FillBoundary(ng, period); Bfield_fp[lev][2]->FillBoundary(ng, period); @@ -446,7 +443,6 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Bfield_cp[lev][0]->nGrowVect(), "Error: in FillBoundaryB, requested more guard cells than allocated"); - Print()<<"FillBoundaryB exchanges "<< Bfield_cp[lev][0]->nGrowVect() <<'\n'; Bfield_cp[lev][0]->FillBoundary(ng, cperiod); Bfield_cp[lev][1]->FillBoundary(ng, cperiod); Bfield_cp[lev][2]->FillBoundary(ng, cperiod); -- cgit v1.2.3 From 72c751d492c44c0bc4d318aeb195e05636c8c38a Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Fri, 3 Jan 2020 19:30:34 -0800 Subject: include Remi's suggestions --- Source/Evolve/WarpXEvolveEM.cpp | 1 + Source/Parallelization/GuardCellManager.H | 43 +++++++++++++++++++++++++---- Source/Parallelization/GuardCellManager.cpp | 8 ++---- Source/Utils/WarpXMovingWindow.cpp | 3 +- Source/WarpX.H | 6 ++-- Source/WarpX.cpp | 14 +++++----- 6 files changed, 52 insertions(+), 23 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 366be28d2..e313aab01 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #ifdef WARPX_USE_PY #include #endif diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 2e1cebff8..34cf549cf 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -3,11 +3,33 @@ #include +/** + * \brief This class computes and stores the number of guard cells needed for + * the allocation of the MultiFabs and required for each part of the PIC loop. + */ class guardCellManager{ public: - int Init( + /** + * \brief Initialize number of guard cells depending on the options used. + * + * \param do_subcycling bool, whether to use subcycling + * \param do_fdtd_nci_corr bool, whether to use Godfrey NCI corrector + * \param do_nodal bool, whether the field solver is nodal + * \param do_moving_window bool, whether to use moving window + * \param do_fft_mpi_dec bool, whether to do parallel FFTs for PSATD + * \param aux_is_nodal bool, true if the aux grid is nodal + * \param moving_window_dir direction of moving window + * \param nox order of current deposition + * \param nox_fft order of PSATD in x direction + * \param noy_fft order of PSATD in y direction + * \param noz_fft order of PSATD in z direction + * \param nci_corr_stencil stencil of NCI corrector + * \param maxwell_fdtd_solver_id if of Maxwell solver + * \param max_level max level of the simulation + */ + void Init( const bool do_subcycling, const bool do_fdtd_nci_corr, const bool do_nodal, @@ -21,21 +43,32 @@ public: const int maxwell_fdtd_solver_id, const int max_level); - // Guard cells allocated for each multifab + // Guard cells allocated for MultiFabs E and B amrex::IntVect ng_alloc_EB = amrex::IntVect::TheZeroVector(); + // Guard cells allocated for MultiFab J amrex::IntVect ng_alloc_J = amrex::IntVect::TheZeroVector(); + // Guard cells allocated for MultiFab Rho amrex::IntVect ng_alloc_Rho = amrex::IntVect::TheZeroVector(); + // Guard cells allocated for MultiFab F amrex::IntVect ng_alloc_F = amrex::IntVect::TheZeroVector(); - int ng_alloc_F_int = 0; - // Guard cells exchanged for specific in the PIC loop + // Guard cells exchanged for specific parts of the PIC loop + + // Number of guard cells of E and B that must exchanged before Field Solver amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); + // Number of guard cells of F that must exchanged before Field Solver amrex::IntVect ng_FieldSolverF = amrex::IntVect::TheZeroVector(); + // Number of guard cells of E and B that must exchanged before Field Gather amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); + // Number of guard cells of E and B that must exchanged before updating the Aux grid amrex::IntVect ng_UpdateAux = amrex::IntVect::TheZeroVector(); + // Number of guard cells of all MultiFabs that must exchanged before moving window amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); - // Extra guard cells for fine level of E and B + // When the auxiliary grid is nodal but the field solver is staggered + // (typically with momentum-conserving gather with FDTD Yee solver), + // An extra guard cell is needed on the fine grid to do the interpolation + // for E and B. amrex::IntVect ng_Extra = amrex::IntVect::TheZeroVector(); }; diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 34454bd7e..99feca516 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -4,7 +4,7 @@ using namespace amrex; -int +void guardCellManager::Init( const bool do_subcycling, const bool do_fdtd_nci_corr, @@ -69,11 +69,9 @@ guardCellManager::Init( ng_alloc_J = IntVect(ngJx,ngJz); #endif - int nJ_buffer = ng_alloc_J.max(); // guard cells for J required for deposition only. - ng_alloc_Rho = ng_alloc_J+1; //One extra ghost cell, so that it's safe to deposit charge density // after pushing particle. - ng_alloc_F_int = (do_moving_window) ? 2 : 0; + int ng_alloc_F_int = (do_moving_window) ? 2 : 0; // CKC solver requires one additional guard cell if (maxwell_fdtd_solver_id == 1) ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int)); @@ -155,6 +153,4 @@ guardCellManager::Init( if (do_moving_window){ ng_MovingWindow[moving_window_dir] = 1; } - - return nJ_buffer; } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 2c0c6eac9..7044d75d8 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -221,8 +221,7 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, MultiFab tmpmf(ba, dm, nc, ng); MultiFab::Copy(tmpmf, mf, 0, 0, nc, ng); - // Not sure why this is needed, but it is... - IntVect ng_mw = IntVect::TheUnitVector(); + IntVect ng_mw = IntVect::TheZeroVector(); // Enough guard cells in the MW direction ng_mw[dir] = num_shift; // Add the extra cell (if momentum-conserving gather with staggered field solve) diff --git a/Source/WarpX.H b/Source/WarpX.H index 4b2404198..80d4e3367 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -138,7 +138,7 @@ public: // do nodal static int do_nodal; - + const amrex::MultiFab& getcurrent (int lev, int direction) {return *current_fp[lev][direction];} const amrex::MultiFab& getEfield (int lev, int direction) {return *Efield_aux[lev][direction];} const amrex::MultiFab& getBfield (int lev, int direction) {return *Bfield_aux[lev][direction];} @@ -455,8 +455,8 @@ private: void AllocLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, const amrex::IntVect& ngE, const amrex::IntVect& ngJ, - const amrex::IntVect& ngRho, int ngF, const amrex::IntVect& ngextra, - const bool aux_is_nodal); + const amrex::IntVect& ngRho, const amrex::IntVect& ngF, + const amrex::IntVect& ngextra, const bool aux_is_nodal); amrex::Vector istep; // which step? amrex::Vector nsubsteps; // how many substeps on each level? diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 470c8ea7e..c9bd0c5d7 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -720,7 +720,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d bool aux_is_nodal = (field_gathering_algo == GatheringAlgo::MomentumConserving); - int nJ_buffer = guard_cells.Init( + guard_cells.Init( do_subcycling, WarpX::use_fdtd_nci_corr, do_nodal, @@ -743,7 +743,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d } if (n_current_deposition_buffer < 0) { - n_current_deposition_buffer = nJ_buffer; + n_current_deposition_buffer = guard_cells.ng_alloc_J.max(); } if (n_field_gather_buffer < 0) { // Field gather buffer should be larger than current deposition buffers @@ -751,14 +751,14 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d } AllocLevelMFs(lev, ba, dm, guard_cells.ng_alloc_EB, guard_cells.ng_alloc_J, - guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F_int, + guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F, guard_cells.ng_Extra, aux_is_nodal); } void WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm, - const IntVect& ngE, const IntVect& ngJ, const IntVect& ngRho, int ngF, - const IntVect& ngextra, const bool aux_is_nodal) + const IntVect& ngE, const IntVect& ngJ, const IntVect& ngRho, + const IntVect& ngF, const IntVect& ngextra, const bool aux_is_nodal) { #if defined WARPX_DIM_RZ @@ -799,7 +799,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (do_dive_cleaning) { - F_fp[lev].reset (new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()),dm,ncomps, ngF)); + F_fp[lev].reset (new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()),dm,ncomps, ngF.max())); } #ifdef WARPX_USE_PSATD else @@ -884,7 +884,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } if (do_dive_cleaning) { - F_cp[lev].reset (new MultiFab(amrex::convert(cba,IntVect::TheUnitVector()),dm,ncomps, ngF)); + F_cp[lev].reset (new MultiFab(amrex::convert(cba,IntVect::TheUnitVector()),dm,ncomps, ngF.max())); } #ifdef WARPX_USE_PSATD else -- cgit v1.2.3 From b2651026588db900ba9091f3af5a0d4664eb8fd4 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 8 Jan 2020 13:28:58 -0800 Subject: option to always exchange all guard cells --- Source/Parallelization/GuardCellManager.H | 3 +- Source/Parallelization/GuardCellManager.cpp | 73 +++++++++++++++++------------ Source/WarpX.H | 2 + Source/WarpX.cpp | 5 +- 4 files changed, 51 insertions(+), 32 deletions(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 34cf549cf..c57745b84 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -41,7 +41,8 @@ public: const int nox_fft, const int noy_fft, const int noz_fft, const int nci_corr_stencil, const int maxwell_fdtd_solver_id, - const int max_level); + const int max_level, + const bool exchange_all_guard_cells); // Guard cells allocated for MultiFabs E and B amrex::IntVect ng_alloc_EB = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 99feca516..41d82a6e9 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -17,7 +17,8 @@ guardCellManager::Init( const int nox_fft, const int noy_fft, const int noz_fft, const int nci_corr_stencil, const int maxwell_fdtd_solver_id, - const int max_level) + const int max_level, + const bool exchange_all_guard_cells) { // When using subcycling, the particles on the fine level perform two pushes // before being redistributed ; therefore, we need one extra guard cell @@ -119,38 +120,50 @@ guardCellManager::Init( ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); ng_FieldSolverF = IntVect(AMREX_D_DECL(1,1,1)); #endif - ng_FieldSolver = ng_FieldSolver.min(ng_alloc_EB); - // Compute number of cells required for Field Gather - int FGcell[4] = {0,1,1,2}; // Index is nox - IntVect ng_FieldGather_noNCI = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); - // Add one cell if momentum_conserving gather in a staggered-field simulation - ng_FieldGather_noNCI += ng_Extra; - // Not sure why, but need one extra guard cell when using MR - if (max_level >= 1) ng_FieldGather_noNCI += ng_Extra; - ng_FieldGather_noNCI = ng_FieldGather_noNCI.min(ng_alloc_EB); - // If NCI filter, add guard cells in the z direction - IntVect ng_NCIFilter = IntVect::TheZeroVector(); - if (do_fdtd_nci_corr) - ng_NCIFilter[AMREX_SPACEDIM-1] = NCIGodfreyFilter::m_stencil_width; - // Note: communications of guard cells for bilinear filter are handled - // separately. - ng_FieldGather = ng_FieldGather_noNCI + ng_NCIFilter; + if (exchange_all_guard_cells){ + // Run in safe mode: exchange all allocated guard cells at each + // call of FillBoundary + ng_FieldSolver = ng_alloc_EB; + ng_FieldSolverF = ng_alloc_F; + ng_FieldGather = ng_alloc_EB; + ng_UpdateAux = ng_alloc_EB; + ng_MovingWindow = ng_alloc_EB; + } else { + + ng_FieldSolver = ng_FieldSolver.min(ng_alloc_EB); + + // Compute number of cells required for Field Gather + int FGcell[4] = {0,1,1,2}; // Index is nox + IntVect ng_FieldGather_noNCI = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); + // Add one cell if momentum_conserving gather in a staggered-field simulation + ng_FieldGather_noNCI += ng_Extra; + // Not sure why, but need one extra guard cell when using MR + if (max_level >= 1) ng_FieldGather_noNCI += ng_Extra; + ng_FieldGather_noNCI = ng_FieldGather_noNCI.min(ng_alloc_EB); + // If NCI filter, add guard cells in the z direction + IntVect ng_NCIFilter = IntVect::TheZeroVector(); + if (do_fdtd_nci_corr) + ng_NCIFilter[AMREX_SPACEDIM-1] = NCIGodfreyFilter::m_stencil_width; + // Note: communications of guard cells for bilinear filter are handled + // separately. + ng_FieldGather = ng_FieldGather_noNCI + ng_NCIFilter; - // Guard cells for auxiliary grid. - // Not sure why there is a 2* here... - ng_UpdateAux = 2*ng_FieldGather_noNCI + ng_NCIFilter; + // Guard cells for auxiliary grid. + // Not sure why there is a 2* here... + ng_UpdateAux = 2*ng_FieldGather_noNCI + ng_NCIFilter; - // Make sure we do not exchange more guard cells than allocated. - ng_FieldGather = ng_FieldGather.min(ng_alloc_EB); - ng_UpdateAux = ng_UpdateAux.min(ng_alloc_EB); - ng_FieldSolverF = ng_FieldSolverF.min(ng_alloc_F); - // Only FillBoundary(ng_FieldGather) is called between consecutive - // field solves. So ng_FieldGather must have enough cells - // for the field solve too. - ng_FieldGather = ng_FieldGather.max(ng_FieldSolver); + // Make sure we do not exchange more guard cells than allocated. + ng_FieldGather = ng_FieldGather.min(ng_alloc_EB); + ng_UpdateAux = ng_UpdateAux.min(ng_alloc_EB); + ng_FieldSolverF = ng_FieldSolverF.min(ng_alloc_F); + // Only FillBoundary(ng_FieldGather) is called between consecutive + // field solves. So ng_FieldGather must have enough cells + // for the field solve too. + ng_FieldGather = ng_FieldGather.max(ng_FieldSolver); - if (do_moving_window){ - ng_MovingWindow[moving_window_dir] = 1; + if (do_moving_window){ + ng_MovingWindow[moving_window_dir] = 1; + } } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 7609e3343..24939961d 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -132,6 +132,8 @@ public: static int do_subcycling; + static bool exchange_all_guard_cells; + // buffers static int n_field_gather_buffer; //! in number of cells from the edge (identical for each dimension) static int n_current_deposition_buffer; //! in number of cells from the edge (identical for each dimension) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c9bd0c5d7..22ea37fe4 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -78,6 +78,7 @@ Real WarpX::particle_slice_width_lab = 0.0; bool WarpX::do_dynamic_scheduling = true; int WarpX::do_subcycling = 0; +bool WarpX::exchange_all_guard_cells = 0; #if (AMREX_SPACEDIM == 3) IntVect WarpX::Bx_nodal_flag(1,0,0); @@ -293,6 +294,7 @@ WarpX::ReadParameters () pp.query("verbose", verbose); pp.query("regrid_int", regrid_int); pp.query("do_subcycling", do_subcycling); + pp.query("exchange_all_guard_cells", exchange_all_guard_cells); pp.query("override_sync_int", override_sync_int); AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_subcycling != 1 || max_level <= 1, @@ -732,7 +734,8 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d nox_fft, noy_fft, noz_fft, NCIGodfreyFilter::m_stencil_width, maxwell_fdtd_solver_id, - maxLevel()); + maxLevel(), + exchange_all_guard_cells); if (mypc->nSpeciesDepositOnMainGrid() && n_current_deposition_buffer == 0) { n_current_deposition_buffer = 1; -- cgit v1.2.3 From 2d5d85d72f845713872fb190528a02da17b96f46 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 8 Jan 2020 13:30:53 -0800 Subject: delete EOL whitespace --- Source/Parallelization/GuardCellManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Source/Parallelization/GuardCellManager.cpp') diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 41d82a6e9..7f72553b6 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -132,7 +132,7 @@ guardCellManager::Init( } else { ng_FieldSolver = ng_FieldSolver.min(ng_alloc_EB); - + // Compute number of cells required for Field Gather int FGcell[4] = {0,1,1,2}; // Index is nox IntVect ng_FieldGather_noNCI = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); -- cgit v1.2.3