From e63229421c47538283375f1601d2d38bb25246b9 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sun, 27 Oct 2019 16:59:48 -0700 Subject: FillBoundaryE/B/F take mandatory argument nguard --- Source/Parallelization/WarpXComm.cpp | 70 +++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 28 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 52df3dc25..0dae38e2e 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -321,41 +321,41 @@ WarpX::UpdateAuxilaryDataSameType () } void -WarpX::FillBoundaryB () +WarpX::FillBoundaryB (IntVect ng) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryB(lev); + FillBoundaryB(lev, ng); } } void -WarpX::FillBoundaryE () +WarpX::FillBoundaryE (IntVect ng) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryE(lev); + FillBoundaryE(lev, ng); } } void -WarpX::FillBoundaryF () +WarpX::FillBoundaryF (IntVect ng) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryF(lev); + FillBoundaryF(lev, ng); } } void -WarpX::FillBoundaryE(int lev) +WarpX::FillBoundaryE(int lev, IntVect ng) { - FillBoundaryE(lev, PatchType::fine); - if (lev > 0) FillBoundaryE(lev, PatchType::coarse); + FillBoundaryE(lev, PatchType::fine, ng); + if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng); } void -WarpX::FillBoundaryE (int lev, PatchType patch_type) +WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { @@ -370,8 +370,11 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type) } const auto& period = Geom(lev).periodicity(); - Vector mf{Efield_fp[lev][0].get(),Efield_fp[lev][1].get(),Efield_fp[lev][2].get()}; - amrex::FillBoundary(mf, period); + Efield_fp[lev][0]->FillBoundary(0, Efield_fp[lev][0]->nComp(), ng, period); + Efield_fp[lev][1]->FillBoundary(0, Efield_fp[lev][1]->nComp(), ng, period); + Efield_fp[lev][2]->FillBoundary(0, Efield_fp[lev][2]->nComp(), ng, period); + // Vector mf{Efield_fp[lev][0].get(),Efield_fp[lev][1].get(),Efield_fp[lev][2].get()}; + // amrex::FillBoundary(mf, period); } else if (patch_type == PatchType::coarse) { @@ -386,20 +389,23 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type) } const auto& cperiod = Geom(lev-1).periodicity(); - Vector mf{Efield_cp[lev][0].get(),Efield_cp[lev][1].get(),Efield_cp[lev][2].get()}; - amrex::FillBoundary(mf, cperiod); + Efield_cp[lev][0]->FillBoundary(0, Efield_cp[lev][0]->nComp(), ng, cperiod); + Efield_cp[lev][1]->FillBoundary(0, Efield_cp[lev][1]->nComp(), ng, cperiod); + Efield_cp[lev][2]->FillBoundary(0, Efield_cp[lev][2]->nComp(), ng, cperiod); + // Vector mf{Efield_cp[lev][0].get(),Efield_cp[lev][1].get(),Efield_cp[lev][2].get()}; + // amrex::FillBoundary(mf, cperiod); } } void -WarpX::FillBoundaryB (int lev) +WarpX::FillBoundaryB (int lev, IntVect ng) { - FillBoundaryB(lev, PatchType::fine); - if (lev > 0) FillBoundaryB(lev, PatchType::coarse); + FillBoundaryB(lev, PatchType::fine, ng); + if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng); } void -WarpX::FillBoundaryB (int lev, PatchType patch_type) +WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { @@ -413,8 +419,11 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type) pml[lev]->FillBoundaryB(patch_type); } const auto& period = Geom(lev).periodicity(); - Vector mf{Bfield_fp[lev][0].get(),Bfield_fp[lev][1].get(),Bfield_fp[lev][2].get()}; - amrex::FillBoundary(mf, period); + Bfield_fp[lev][0]->FillBoundary(0, Bfield_fp[lev][0]->nComp(), ng, period); + Bfield_fp[lev][1]->FillBoundary(0, Bfield_fp[lev][1]->nComp(), ng, period); + Bfield_fp[lev][2]->FillBoundary(0, Bfield_fp[lev][2]->nComp(), ng, period); + // Vector mf{Bfield_fp[lev][0].get(),Bfield_fp[lev][1].get(),Bfield_fp[lev][2].get()}; + // amrex::FillBoundary(mf, period); } else if (patch_type == PatchType::coarse) { @@ -428,20 +437,23 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type) pml[lev]->FillBoundaryB(patch_type); } const auto& cperiod = Geom(lev-1).periodicity(); - Vector mf{Bfield_cp[lev][0].get(),Bfield_cp[lev][1].get(),Bfield_cp[lev][2].get()}; - amrex::FillBoundary(mf, cperiod); + Bfield_cp[lev][0]->FillBoundary(0, Bfield_cp[lev][0]->nComp(), ng, cperiod); + Bfield_cp[lev][1]->FillBoundary(0, Bfield_cp[lev][1]->nComp(), ng, cperiod); + Bfield_cp[lev][2]->FillBoundary(0, Bfield_cp[lev][2]->nComp(), ng, cperiod); + // Vector mf{Bfield_cp[lev][0].get(),Bfield_cp[lev][1].get(),Bfield_cp[lev][2].get()}; + // amrex::FillBoundary(mf, cperiod); } } void -WarpX::FillBoundaryF (int lev) +WarpX::FillBoundaryF (int lev, IntVect ng) { - FillBoundaryF(lev, PatchType::fine); - if (lev > 0) FillBoundaryF(lev, PatchType::coarse); + FillBoundaryF(lev, PatchType::fine, ng); + if (lev > 0) FillBoundaryF(lev, PatchType::coarse, ng); } void -WarpX::FillBoundaryF (int lev, PatchType patch_type) +WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine && F_fp[lev]) { @@ -453,7 +465,8 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type) } const auto& period = Geom(lev).periodicity(); - F_fp[lev]->FillBoundary(period); + F_fp[lev]->FillBoundary(0, F_fp[lev]->nComp(), ng, period); + // F_fp[lev]->FillBoundary(period); } else if (patch_type == PatchType::coarse && F_cp[lev]) { @@ -465,7 +478,8 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type) } const auto& cperiod = Geom(lev-1).periodicity(); - F_cp[lev]->FillBoundary(cperiod); + F_cp[lev]->FillBoundary(0, F_cp[lev]->nComp(), ng, cperiod); + // F_cp[lev]->FillBoundary(cperiod); } } -- cgit v1.2.3 From 0fe1905133033f128b235a14b04135c064800521 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 29 Oct 2019 20:46:45 -0700 Subject: Reduce number of guard cells exchanged in Moving window and EvolveEM --- Source/Evolve/WarpXEvolveEM.cpp | 17 +++++++-- Source/Parallelization/GuardCellManager.H | 25 ++++++------- Source/Parallelization/GuardCellManager.cpp | 7 +++- Source/Parallelization/WarpXComm.cpp | 21 +++++++++++ Source/Utils/WarpXMovingWindow.cpp | 54 ++++++++++++++++++----------- Source/WarpX.H | 16 +++++---- 6 files changed, 96 insertions(+), 44 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 8ad3b2401..c98a4688e 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -90,15 +90,22 @@ WarpX::EvolveEM (int numsteps) // Beyond one step, we have E^{n} and B^{n}. // Particles have p^{n-1/2} and x^{n}. // This is probably overkill - FillBoundaryE(guard_cells.ngE); + // FillBoundaryE(guard_cells.ngE); // This is probably overkill - FillBoundaryB(guard_cells.ngE); + // FillBoundaryB(guard_cells.ngE); + IntVect my_nc; + my_nc = guard_cells.ng_FieldGather+guard_cells.ng_NCIFilter; + FillBoundaryE(my_nc); + FillBoundaryB(my_nc); + FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); - } if (do_subcycling == 0 || finest_level == 0) { OneStep_nosub(cur_time); + // E : guard cells are up-to-date + // B : guard cells are NOT up-to-date + // F : guard cells are NOT up-to-date } else if (do_subcycling == 1 && finest_level == 1) { OneStep_sub1(cur_time); } else { @@ -108,6 +115,7 @@ WarpX::EvolveEM (int numsteps) if (num_mirrors>0){ applyMirrors(cur_time); + // E : guard cells are NOT up-to-date } #ifdef WARPX_USE_PY @@ -191,6 +199,7 @@ WarpX::EvolveEM (int numsteps) FillBoundaryE(guard_cells.ngE); // This is probably overkill FillBoundaryB(guard_cells.ngE); + FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -245,6 +254,7 @@ WarpX::EvolveEM (int numsteps) FillBoundaryE(guard_cells.ngE); // This is probably overkill FillBoundaryB(guard_cells.ngE); + FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { @@ -325,6 +335,7 @@ WarpX::OneStep_nosub (Real cur_time) EvolveE(dt[0]); // We now have E^{n+1} FillBoundaryE(guard_cells.ngE_FieldSolver); EvolveF(0.5*dt[0], DtType::SecondHalf); + FillBoundaryF(guard_cells.ngF); EvolveB(0.5*dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 4b85a4332..706b5df79 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -22,20 +22,21 @@ public: const int max_level); // Guard cells to initialize multifabs - amrex::IntVect ngExtra; - amrex::IntVect ngE; - amrex::IntVect ngJ; - amrex::IntVect ngRho; - amrex::IntVect ngF; - int ngF_int; + amrex::IntVect ngExtra = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngE = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngJ = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngRho = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngF = amrex::IntVect::TheZeroVector(); + int ngF_int = 0; // Guard cells to exchange data - amrex::IntVect ngB_FieldSolver; - amrex::IntVect ngE_FieldSolver; - amrex::IntVect ng_FieldGather; - amrex::IntVect ngJ_CurrentDepo; - amrex::IntVect ng_MovingWindow; - amrex::IntVect ng_NCIFilter; + amrex::IntVect ngB_FieldSolver = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngE_FieldSolver = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); + amrex::IntVect ngJ_CurrentDepo = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_NCIFilter = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_Aux = amrex::IntVect::TheZeroVector(); }; #endif // GUARDCELLMANAGER_H_ diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index c790c3472..166f0d58d 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -131,5 +131,10 @@ guardCellManager::Init( int FGcell[4] = {0,1,1,2}; // Index is nox ng_FieldGather = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); ngJ_CurrentDepo = ng_FieldGather; - ng_NCIFilter = IntVect(AMREX_D_DECL(0,0,4)); + if (do_fdtd_nci_corr){ + ng_NCIFilter = IntVect::TheZeroVector(); + ng_NCIFilter[AMREX_SPACEDIM-1] = 4; + } + ng_Aux = 2*ng_FieldGather+ng_NCIFilter; + ng_Aux = ng_Aux.min(ngE); } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 0dae38e2e..7d2473f30 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -483,6 +483,27 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) } } +void +WarpX::FillBoundaryAux (IntVect ng) +{ + for (int lev = 0; lev <= finest_level-1; ++lev) + { + FillBoundaryAux(lev, ng); + } +} + +void +WarpX::FillBoundaryAux (int lev, IntVect ng) +{ + const auto& period = Geom(lev).periodicity(); + Efield_aux[lev][0]->FillBoundary(0, Efield_aux[lev][0]->nComp(), ng, period); + Efield_aux[lev][1]->FillBoundary(0, Efield_aux[lev][1]->nComp(), ng, period); + Efield_aux[lev][2]->FillBoundary(0, Efield_aux[lev][2]->nComp(), ng, period); + Bfield_aux[lev][0]->FillBoundary(0, Bfield_aux[lev][0]->nComp(), ng, period); + Bfield_aux[lev][1]->FillBoundary(0, Bfield_aux[lev][1]->nComp(), ng, period); + Bfield_aux[lev][2]->FillBoundary(0, Bfield_aux[lev][2]->nComp(), ng, period); +} + void WarpX::SyncCurrent () { diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index c577da7f3..59810d817 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -99,32 +99,34 @@ WarpX::MoveWindow (bool move_j) for (int dim = 0; dim < 3; ++dim) { // Fine grid - shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, B_external_grid[dim]); - shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, E_external_grid[dim]); + shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE, B_external_grid[dim]); + shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE, E_external_grid[dim]); if (move_j) { - shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir); + shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, guard_cells.ngJ); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_fp(); const std::array& pml_E = pml[lev]->GetE_fp(); - shiftMF(*pml_B[dim], geom[lev], num_shift, dir); - shiftMF(*pml_E[dim], geom[lev], num_shift, dir); + IntVect ng_exchange = pml_B[dim]->nGrowVect(); + shiftMF(*pml_B[dim], geom[lev], num_shift, dir, ng_exchange); + shiftMF(*pml_E[dim], geom[lev], num_shift, dir, ng_exchange); } if (lev > 0) { // Coarse grid - shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, B_external_grid[dim]); - shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, E_external_grid[dim]); - shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir); - shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir); + shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, guard_cells.ngE, B_external_grid[dim]); + shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, guard_cells.ngE, E_external_grid[dim]); + shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE); + shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, guard_cells.ngE); if (move_j) { - shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, guard_cells.ngJ); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_cp(); const std::array& pml_E = pml[lev]->GetE_cp(); - shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir); - shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir); + IntVect ng_exchange = pml_B[dim]->nGrowVect(); + shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir, ng_exchange); + shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir, ng_exchange); } } } @@ -132,19 +134,21 @@ WarpX::MoveWindow (bool move_j) // Shift scalar component F for dive cleaning if (do_dive_cleaning) { // Fine grid - shiftMF(*F_fp[lev], geom[lev], num_shift, dir); + shiftMF(*F_fp[lev], geom[lev], num_shift, dir, guard_cells.ngF); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_fp(); - shiftMF(*pml_F, geom[lev], num_shift, dir); + IntVect ng_exchange = pml_F->nGrowVect(); + shiftMF(*pml_F, geom[lev], num_shift, dir, ng_exchange); } if (lev > 0) { // Coarse grid - shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir, guard_cells.ngF); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_cp(); - shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir); + IntVect ng_exchange = pml_F->nGrowVect(); + shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir, ng_exchange); } - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, guard_cells.ngRho); } } @@ -152,10 +156,10 @@ WarpX::MoveWindow (bool move_j) if (move_j) { if (rho_fp[lev]){ // Fine grid - shiftMF(*rho_fp[lev], geom[lev], num_shift, dir); + shiftMF(*rho_fp[lev], geom[lev], num_shift, dir, guard_cells.ngRho); if (lev > 0){ // Coarse grid - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, guard_cells.ngRho); } } } @@ -204,7 +208,7 @@ WarpX::MoveWindow (bool move_j) void WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, - amrex::Real external_field) + IntVect ng_exchange, amrex::Real external_field) { BL_PROFILE("WarpX::shiftMF()"); const BoxArray& ba = mf.boxArray(); @@ -216,7 +220,15 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, MultiFab tmpmf(ba, dm, nc, ng); MultiFab::Copy(tmpmf, mf, 0, 0, nc, ng); - tmpmf.FillBoundary(geom.periodicity()); + + // Not sure why this is needed, but it is... + ng_exchange[0] = 1; + ng_exchange[1] = num_shift; // 2 + Print()<<"ng_exchange "< Date: Thu, 31 Oct 2019 16:40:10 -0700 Subject: Each call to FillBoundary in regular PIC loop (FDTD) uses fewer guard cells --- Source/Evolve/WarpXEvolveEM.cpp | 39 +++++++++++++---------- Source/Parallelization/GuardCellManager.H | 1 + Source/Parallelization/GuardCellManager.cpp | 27 +++++++++++----- Source/Parallelization/WarpXComm.cpp | 49 ++++++++++++++++++----------- Source/Utils/WarpXMovingWindow.cpp | 48 +++++++++++++++------------- Source/WarpX.H | 10 +++--- 6 files changed, 104 insertions(+), 70 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 8707b9fde..3247ca64f 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -75,8 +75,10 @@ WarpX::EvolveEM (int numsteps) // Particles have p^{n} and x^{n}. // is_synchronized is true. if (is_synchronized) { - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); + // FillBoundaryE(guard_cells.ng_FieldGather, guard_cells.ngExtra); + // FillBoundaryB(guard_cells.ng_FieldGather, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); UpdateAuxilaryData(); // on first step, push p by -0.5*dt for (int lev = 0; lev <= finest_level; ++lev) { @@ -90,12 +92,14 @@ WarpX::EvolveEM (int numsteps) // Beyond one step, we have E^{n} and B^{n}. // Particles have p^{n-1/2} and x^{n}. // This is probably overkill - // FillBoundaryE(guard_cells.ngE); // This is probably overkill - // FillBoundaryB(guard_cells.ngE); - IntVect ng_gather = guard_cells.ng_FieldGather+guard_cells.ng_NCIFilter; - FillBoundaryE(ng_gather); - FillBoundaryB(ng_gather); + // IntVect ng_gather = guard_cells.ng_FieldGather+guard_cells.ng_NCIFilter; + // FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); + // FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); + // FillBoundaryE(ng_gather, guard_cells.ngExtra); + // FillBoundaryB(ng_gather, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ng_FieldGatherAndNCIFilter, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ng_FieldGatherAndNCIFilter, guard_cells.ngExtra); FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); } @@ -195,9 +199,9 @@ WarpX::EvolveEM (int numsteps) if (to_make_plot || do_insitu || to_make_slice_plot) { // This is probably overkill - FillBoundaryE(guard_cells.ngE); + FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); // This is probably overkill - FillBoundaryB(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); @@ -250,9 +254,9 @@ WarpX::EvolveEM (int numsteps) if (write_plot_file || do_insitu) { // This is probably overkill - FillBoundaryE(guard_cells.ngE); + FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); // This is probably overkill - FillBoundaryB(guard_cells.ngE); + FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); FillBoundaryAux(guard_cells.ng_Aux); UpdateAuxilaryData(); @@ -323,23 +327,24 @@ WarpX::OneStep_nosub (Real cur_time) #ifdef WARPX_USE_PSATD PushPSATD(dt[0]); if (do_pml) DampPML(); - FillBoundaryE(guard_cells.ngE); - FillBoundaryB(guard_cells.ngE); + FillBoundaryE(guard_cells.ngE, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ngE, guard_cells.ngExtra); #else + Print()<<"Push fields\n"; EvolveF(0.5*dt[0], DtType::FirstHalf); FillBoundaryF(guard_cells.ng_FieldSolver); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} - FillBoundaryB(guard_cells.ng_FieldSolver); - + // FillBoundaryB(guard_cells.ng_FieldSolver, guard_cells.ngExtra); + FillBoundaryB(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); EvolveE(dt[0]); // We now have E^{n+1} - FillBoundaryE(guard_cells.ng_FieldSolver); + // FillBoundaryE(guard_cells.ng_FieldSolver, guard_cells.ngExtra); + FillBoundaryE(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); EvolveF(0.5*dt[0], DtType::SecondHalf); FillBoundaryF(guard_cells.ngF); EvolveB(0.5*dt[0]); // We now have B^{n+1} if (do_pml) { DampPML(); } - #endif } diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index e241eed75..93f1e9fd1 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -32,6 +32,7 @@ public: // Guard cells to exchange data amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_FieldGatherAndNCIFilter = amrex::IntVect::TheZeroVector(); amrex::IntVect ngJ_CurrentDepo = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_NCIFilter = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index a67e15eb7..9f3e5b18d 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -110,7 +110,7 @@ guardCellManager::Init( ng_required = std::max( ng_required, ngE[i_dim] ); ng_required = std::max( ng_required, ngJ[i_dim] ); ng_required = std::max( ng_required, ngRho[i_dim] ); - ng_required = std::max( ng_required, ngF[i_dim] ); +v ng_required = std::max( ng_required, ngF[i_dim] ); // Set the guard cells to this max ngE[i_dim] = ng_required; ngJ[i_dim] = ng_required; @@ -122,11 +122,6 @@ guardCellManager::Init( ngF = IntVect(AMREX_D_DECL(ngF_int, ngF_int, ngF_int)); #endif - Print()<<"rrr ngE : "<(aux_is_nodal and !do_nodal)); // Guard cells for field solver @@ -134,12 +129,28 @@ guardCellManager::Init( ng_MovingWindow = IntVect(AMREX_D_DECL(0,0,0)); // Multiplied by number of cells moved at each timestep ng_MovingWindow[moving_window_dir] = 1; int FGcell[4] = {0,1,1,2}; // Index is nox - ng_FieldGather = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])); - ngJ_CurrentDepo = ng_FieldGather; + ng_FieldGather = IntVect(AMREX_D_DECL(FGcell[nox],FGcell[nox],FGcell[nox])) + 2*ngExtra; + + ng_FieldGather = ng_FieldGather.min(ngE); if (do_fdtd_nci_corr){ ng_NCIFilter = IntVect::TheZeroVector(); ng_NCIFilter[AMREX_SPACEDIM-1] = 4; } + + ng_FieldGatherAndNCIFilter = ng_FieldGather + ng_NCIFilter; + ng_FieldGatherAndNCIFilter = ng_FieldGatherAndNCIFilter.min(ngE); + ng_Aux = 2*ng_FieldGather+ng_NCIFilter; ng_Aux = ng_Aux.min(ngE); + + Print()<<"rrr ngE : "< #include #include +#include "WarpXAlgorithmSelection.H" #include #include @@ -321,20 +322,20 @@ WarpX::UpdateAuxilaryDataSameType () } void -WarpX::FillBoundaryB (IntVect ng) +WarpX::FillBoundaryB (IntVect ng, IntVect ng_extra_fine) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryB(lev, ng); + FillBoundaryB(lev, ng, ng_extra_fine); } } void -WarpX::FillBoundaryE (IntVect ng) +WarpX::FillBoundaryE (IntVect ng, IntVect ng_extra_fine) { for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryE(lev, ng); + FillBoundaryE(lev, ng, ng_extra_fine); } } @@ -348,9 +349,11 @@ WarpX::FillBoundaryF (IntVect ng) } void -WarpX::FillBoundaryE(int lev, IntVect ng) +WarpX::FillBoundaryE(int lev, IntVect ng, IntVect ng_extra_fine) { - FillBoundaryE(lev, PatchType::fine, ng); + Print()<<"FillBoundaryE ng_extra_fine "<< ng_extra_fine <<'\n'; + Print()<<"FillBoundaryE exchanges "<< ng+ng_extra_fine <<'\n'; + FillBoundaryE(lev, PatchType::fine, ng+ng_extra_fine); if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng); } @@ -370,11 +373,12 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) } const auto& period = Geom(lev).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Efield_fp[lev][0]->nGrowVect(), + "Error: in FillBoundaryE, requested more guard cells than allocated"); Efield_fp[lev][0]->FillBoundary(0, Efield_fp[lev][0]->nComp(), ng, period); Efield_fp[lev][1]->FillBoundary(0, Efield_fp[lev][1]->nComp(), ng, period); Efield_fp[lev][2]->FillBoundary(0, Efield_fp[lev][2]->nComp(), ng, period); - // Vector mf{Efield_fp[lev][0].get(),Efield_fp[lev][1].get(),Efield_fp[lev][2].get()}; - // amrex::FillBoundary(mf, period); } else if (patch_type == PatchType::coarse) { @@ -389,18 +393,21 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) } const auto& cperiod = Geom(lev-1).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Efield_cp[lev][0]->nGrowVect(), + "Error: in FillBoundaryE, requested more guard cells than allocated"); Efield_cp[lev][0]->FillBoundary(0, Efield_cp[lev][0]->nComp(), ng, cperiod); Efield_cp[lev][1]->FillBoundary(0, Efield_cp[lev][1]->nComp(), ng, cperiod); Efield_cp[lev][2]->FillBoundary(0, Efield_cp[lev][2]->nComp(), ng, cperiod); - // Vector mf{Efield_cp[lev][0].get(),Efield_cp[lev][1].get(),Efield_cp[lev][2].get()}; - // amrex::FillBoundary(mf, cperiod); } } void -WarpX::FillBoundaryB (int lev, IntVect ng) +WarpX::FillBoundaryB (int lev, IntVect ng, IntVect ng_extra_fine) { - FillBoundaryB(lev, PatchType::fine, ng); + Print()<<"FillBoundaryB ng_extra_fine "<< ng_extra_fine <<'\n'; + Print()<<"FillBoundaryB exchanges "<< ng+ng_extra_fine <<'\n'; + FillBoundaryB(lev, PatchType::fine, ng + ng_extra_fine); if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng); } @@ -419,11 +426,12 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) pml[lev]->FillBoundaryB(patch_type); } const auto& period = Geom(lev).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Bfield_fp[lev][0]->nGrowVect(), + "Error: in FillBoundaryB, requested more guard cells than allocated"); Bfield_fp[lev][0]->FillBoundary(0, Bfield_fp[lev][0]->nComp(), ng, period); Bfield_fp[lev][1]->FillBoundary(0, Bfield_fp[lev][1]->nComp(), ng, period); Bfield_fp[lev][2]->FillBoundary(0, Bfield_fp[lev][2]->nComp(), ng, period); - // Vector mf{Bfield_fp[lev][0].get(),Bfield_fp[lev][1].get(),Bfield_fp[lev][2].get()}; - // amrex::FillBoundary(mf, period); } else if (patch_type == PatchType::coarse) { @@ -437,11 +445,12 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) pml[lev]->FillBoundaryB(patch_type); } const auto& cperiod = Geom(lev-1).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= Bfield_cp[lev][0]->nGrowVect(), + "Error: in FillBoundaryB, requested more guard cells than allocated"); Bfield_cp[lev][0]->FillBoundary(0, Bfield_cp[lev][0]->nComp(), ng, cperiod); Bfield_cp[lev][1]->FillBoundary(0, Bfield_cp[lev][1]->nComp(), ng, cperiod); Bfield_cp[lev][2]->FillBoundary(0, Bfield_cp[lev][2]->nComp(), ng, cperiod); - // Vector mf{Bfield_cp[lev][0].get(),Bfield_cp[lev][1].get(),Bfield_cp[lev][2].get()}; - // amrex::FillBoundary(mf, cperiod); } } @@ -465,8 +474,10 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) } const auto& period = Geom(lev).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= F_fp[lev]->nGrowVect(), + "Error: in FillBoundaryF, requested more guard cells than allocated"); F_fp[lev]->FillBoundary(0, F_fp[lev]->nComp(), ng, period); - // F_fp[lev]->FillBoundary(period); } else if (patch_type == PatchType::coarse && F_cp[lev]) { @@ -478,8 +489,10 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) } const auto& cperiod = Geom(lev-1).periodicity(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ng <= F_cp[lev]->nGrowVect(), + "Error: in FillBoundaryF, requested more guard cells than allocated"); F_cp[lev]->FillBoundary(0, F_cp[lev]->nComp(), ng, cperiod); - // F_cp[lev]->FillBoundary(cperiod); } } diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index db58ad53c..8770d2059 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -1,3 +1,4 @@ +#include "GuardCellManager.H" #include #include @@ -28,6 +29,9 @@ WarpX::MoveWindow (bool move_j) { if (do_moving_window == 0) return 0; + IntVect ng_extra = guard_cells.ngExtra; + IntVect ng_zero = IntVect::TheZeroVector(); + // Update the continuous position of the moving window, // and of the plasma injection moving_window_x += moving_window_v * dt[0]; @@ -99,32 +103,32 @@ WarpX::MoveWindow (bool move_j) for (int dim = 0; dim < 3; ++dim) { // Fine grid - shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, B_external_grid[dim]); - shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, E_external_grid[dim]); + shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, ng_extra, B_external_grid[dim]); + shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, ng_extra, E_external_grid[dim]); if (move_j) { - shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir); + shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, ng_zero); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_fp(); const std::array& pml_E = pml[lev]->GetE_fp(); - shiftMF(*pml_B[dim], geom[lev], num_shift, dir); - shiftMF(*pml_E[dim], geom[lev], num_shift, dir); + shiftMF(*pml_B[dim], geom[lev], num_shift, dir, ng_extra); + shiftMF(*pml_E[dim], geom[lev], num_shift, dir, ng_extra); } if (lev > 0) { // Coarse grid - shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, B_external_grid[dim]); - shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, E_external_grid[dim]); - shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir); - shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir); + shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero, B_external_grid[dim]); + shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero, E_external_grid[dim]); + shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, ng_zero); + shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, ng_zero); if (move_j) { - shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, ng_zero); } if (do_pml && pml[lev]->ok()) { const std::array& pml_B = pml[lev]->GetB_cp(); const std::array& pml_E = pml[lev]->GetE_cp(); - shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir); - shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir, ng_extra); + shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir, ng_extra); } } } @@ -132,19 +136,19 @@ WarpX::MoveWindow (bool move_j) // Shift scalar component F for dive cleaning if (do_dive_cleaning) { // Fine grid - shiftMF(*F_fp[lev], geom[lev], num_shift, dir); + shiftMF(*F_fp[lev], geom[lev], num_shift, dir, ng_zero); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_fp(); - shiftMF(*pml_F, geom[lev], num_shift, dir); + shiftMF(*pml_F, geom[lev], num_shift, dir, ng_extra); } if (lev > 0) { // Coarse grid - shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir, ng_zero); if (do_pml && pml[lev]->ok()) { MultiFab* pml_F = pml[lev]->GetF_cp(); - shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir); + shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir, ng_zero); } - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, ng_zero); } } @@ -152,10 +156,10 @@ WarpX::MoveWindow (bool move_j) if (move_j) { if (rho_fp[lev]){ // Fine grid - shiftMF(*rho_fp[lev], geom[lev], num_shift, dir); + shiftMF(*rho_fp[lev], geom[lev], num_shift, dir, ng_zero); if (lev > 0){ // Coarse grid - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir, ng_zero); } } } @@ -204,7 +208,7 @@ WarpX::MoveWindow (bool move_j) void WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, - amrex::Real external_field) + IntVect ng_extra, amrex::Real external_field) { BL_PROFILE("WarpX::shiftMF()"); const BoxArray& ba = mf.boxArray(); @@ -220,12 +224,12 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, // Not sure why this is needed, but it is... IntVect ng_mw = IntVect::TheUnitVector(); ng_mw[dir] = num_shift; + ng_mw += ng_extra; + ng_mw = ng_mw.min(ng); Print()<<"ng_mw "< Date: Thu, 31 Oct 2019 19:09:40 -0700 Subject: remove print statements --- Source/Parallelization/WarpXComm.cpp | 8 ++++---- Source/WarpX.cpp | 6 ------ 2 files changed, 4 insertions(+), 10 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 60f6f3db0..ea9408953 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -351,8 +351,6 @@ WarpX::FillBoundaryF (IntVect ng) void WarpX::FillBoundaryE(int lev, IntVect ng, IntVect ng_extra_fine) { - Print()<<"FillBoundaryE ng_extra_fine "<< ng_extra_fine <<'\n'; - Print()<<"FillBoundaryE exchanges "<< ng+ng_extra_fine <<'\n'; FillBoundaryE(lev, PatchType::fine, ng+ng_extra_fine); if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng); } @@ -376,6 +374,7 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Efield_fp[lev][0]->nGrowVect(), "Error: in FillBoundaryE, requested more guard cells than allocated"); + Print()<<"FillBoundaryE exchanges "<< Efield_fp[lev][0]->nGrowVect() <<'\n'; Efield_fp[lev][0]->FillBoundary(0, Efield_fp[lev][0]->nComp(), ng, period); Efield_fp[lev][1]->FillBoundary(0, Efield_fp[lev][1]->nComp(), ng, period); Efield_fp[lev][2]->FillBoundary(0, Efield_fp[lev][2]->nComp(), ng, period); @@ -396,6 +395,7 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Efield_cp[lev][0]->nGrowVect(), "Error: in FillBoundaryE, requested more guard cells than allocated"); + Print()<<"FillBoundaryE exchanges "<< Efield_cp[lev][0]->nGrowVect() <<'\n'; Efield_cp[lev][0]->FillBoundary(0, Efield_cp[lev][0]->nComp(), ng, cperiod); Efield_cp[lev][1]->FillBoundary(0, Efield_cp[lev][1]->nComp(), ng, cperiod); Efield_cp[lev][2]->FillBoundary(0, Efield_cp[lev][2]->nComp(), ng, cperiod); @@ -405,8 +405,6 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) void WarpX::FillBoundaryB (int lev, IntVect ng, IntVect ng_extra_fine) { - Print()<<"FillBoundaryB ng_extra_fine "<< ng_extra_fine <<'\n'; - Print()<<"FillBoundaryB exchanges "<< ng+ng_extra_fine <<'\n'; FillBoundaryB(lev, PatchType::fine, ng + ng_extra_fine); if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng); } @@ -429,6 +427,7 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Bfield_fp[lev][0]->nGrowVect(), "Error: in FillBoundaryB, requested more guard cells than allocated"); + Print()<<"FillBoundaryB exchanges "<< Bfield_fp[lev][0]->nGrowVect() <<'\n'; Bfield_fp[lev][0]->FillBoundary(0, Bfield_fp[lev][0]->nComp(), ng, period); Bfield_fp[lev][1]->FillBoundary(0, Bfield_fp[lev][1]->nComp(), ng, period); Bfield_fp[lev][2]->FillBoundary(0, Bfield_fp[lev][2]->nComp(), ng, period); @@ -448,6 +447,7 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Bfield_cp[lev][0]->nGrowVect(), "Error: in FillBoundaryB, requested more guard cells than allocated"); + Print()<<"FillBoundaryB exchanges "<< Bfield_cp[lev][0]->nGrowVect() <<'\n'; Bfield_cp[lev][0]->FillBoundary(0, Bfield_cp[lev][0]->nComp(), ng, cperiod); Bfield_cp[lev][1]->FillBoundary(0, Bfield_cp[lev][1]->nComp(), ng, cperiod); Bfield_cp[lev][2]->FillBoundary(0, Bfield_cp[lev][2]->nComp(), ng, cperiod); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 489692b35..4289203cc 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -733,12 +733,6 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d n_field_gather_buffer = n_current_deposition_buffer + 1; } - //IntVect ngE = guard_cells.ng_alloc_EB; - //IntVect ngJ = guard_cells.ng_alloc_J; - //IntVect ngRho = guard_cells.ng_alloc_Rho; - //int ngF_int = guard_cells.ng_alloc_F_int; - //IntVect ngextra = guard_cells.ngExtra; - AllocLevelMFs(lev, ba, dm, guard_cells.ng_alloc_EB, guard_cells.ng_alloc_J, guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F_int, guard_cells.ng_Extra, aux_is_nodal); -- cgit v1.2.3 From e91822a76810f7a715fd1b60c2c193eb2b8f4c29 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 5 Nov 2019 09:28:35 -0800 Subject: use new version of FillBoundary from amrex --- Source/Parallelization/WarpXComm.cpp | 40 ++++++++++++++++++------------------ Source/Utils/WarpXMovingWindow.cpp | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index ea9408953..c23447e8f 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -375,9 +375,9 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) ng <= Efield_fp[lev][0]->nGrowVect(), "Error: in FillBoundaryE, requested more guard cells than allocated"); Print()<<"FillBoundaryE exchanges "<< Efield_fp[lev][0]->nGrowVect() <<'\n'; - Efield_fp[lev][0]->FillBoundary(0, Efield_fp[lev][0]->nComp(), ng, period); - Efield_fp[lev][1]->FillBoundary(0, Efield_fp[lev][1]->nComp(), ng, period); - Efield_fp[lev][2]->FillBoundary(0, Efield_fp[lev][2]->nComp(), ng, period); + Efield_fp[lev][0]->FillBoundary(ng, period); + Efield_fp[lev][1]->FillBoundary(ng, period); + Efield_fp[lev][2]->FillBoundary(ng, period); } else if (patch_type == PatchType::coarse) { @@ -396,9 +396,9 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) ng <= Efield_cp[lev][0]->nGrowVect(), "Error: in FillBoundaryE, requested more guard cells than allocated"); Print()<<"FillBoundaryE exchanges "<< Efield_cp[lev][0]->nGrowVect() <<'\n'; - Efield_cp[lev][0]->FillBoundary(0, Efield_cp[lev][0]->nComp(), ng, cperiod); - Efield_cp[lev][1]->FillBoundary(0, Efield_cp[lev][1]->nComp(), ng, cperiod); - Efield_cp[lev][2]->FillBoundary(0, Efield_cp[lev][2]->nComp(), ng, cperiod); + Efield_cp[lev][0]->FillBoundary(ng, cperiod); + Efield_cp[lev][1]->FillBoundary(ng, cperiod); + Efield_cp[lev][2]->FillBoundary(ng, cperiod); } } @@ -428,9 +428,9 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) ng <= Bfield_fp[lev][0]->nGrowVect(), "Error: in FillBoundaryB, requested more guard cells than allocated"); Print()<<"FillBoundaryB exchanges "<< Bfield_fp[lev][0]->nGrowVect() <<'\n'; - Bfield_fp[lev][0]->FillBoundary(0, Bfield_fp[lev][0]->nComp(), ng, period); - Bfield_fp[lev][1]->FillBoundary(0, Bfield_fp[lev][1]->nComp(), ng, period); - Bfield_fp[lev][2]->FillBoundary(0, Bfield_fp[lev][2]->nComp(), ng, period); + Bfield_fp[lev][0]->FillBoundary(ng, period); + Bfield_fp[lev][1]->FillBoundary(ng, period); + Bfield_fp[lev][2]->FillBoundary(ng, period); } else if (patch_type == PatchType::coarse) { @@ -448,9 +448,9 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) ng <= Bfield_cp[lev][0]->nGrowVect(), "Error: in FillBoundaryB, requested more guard cells than allocated"); Print()<<"FillBoundaryB exchanges "<< Bfield_cp[lev][0]->nGrowVect() <<'\n'; - Bfield_cp[lev][0]->FillBoundary(0, Bfield_cp[lev][0]->nComp(), ng, cperiod); - Bfield_cp[lev][1]->FillBoundary(0, Bfield_cp[lev][1]->nComp(), ng, cperiod); - Bfield_cp[lev][2]->FillBoundary(0, Bfield_cp[lev][2]->nComp(), ng, cperiod); + Bfield_cp[lev][0]->FillBoundary(ng, cperiod); + Bfield_cp[lev][1]->FillBoundary(ng, cperiod); + Bfield_cp[lev][2]->FillBoundary(ng, cperiod); } } @@ -477,7 +477,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= F_fp[lev]->nGrowVect(), "Error: in FillBoundaryF, requested more guard cells than allocated"); - F_fp[lev]->FillBoundary(0, F_fp[lev]->nComp(), ng, period); + F_fp[lev]->FillBoundary(ng, period); } else if (patch_type == PatchType::coarse && F_cp[lev]) { @@ -492,7 +492,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= F_cp[lev]->nGrowVect(), "Error: in FillBoundaryF, requested more guard cells than allocated"); - F_cp[lev]->FillBoundary(0, F_cp[lev]->nComp(), ng, cperiod); + F_cp[lev]->FillBoundary(ng, cperiod); } } @@ -509,12 +509,12 @@ void WarpX::FillBoundaryAux (int lev, IntVect ng) { const auto& period = Geom(lev).periodicity(); - Efield_aux[lev][0]->FillBoundary(0, Efield_aux[lev][0]->nComp(), ng, period); - Efield_aux[lev][1]->FillBoundary(0, Efield_aux[lev][1]->nComp(), ng, period); - Efield_aux[lev][2]->FillBoundary(0, Efield_aux[lev][2]->nComp(), ng, period); - Bfield_aux[lev][0]->FillBoundary(0, Bfield_aux[lev][0]->nComp(), ng, period); - Bfield_aux[lev][1]->FillBoundary(0, Bfield_aux[lev][1]->nComp(), ng, period); - Bfield_aux[lev][2]->FillBoundary(0, Bfield_aux[lev][2]->nComp(), ng, period); + Efield_aux[lev][0]->FillBoundary(ng, period); + Efield_aux[lev][1]->FillBoundary(ng, period); + Efield_aux[lev][2]->FillBoundary(ng, period); + Bfield_aux[lev][0]->FillBoundary(ng, period); + Bfield_aux[lev][1]->FillBoundary(ng, period); + Bfield_aux[lev][2]->FillBoundary(ng, period); } void diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index e3819966d..2c0c6eac9 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -230,7 +230,7 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir, // Make sure we don't exceed number of guard cells allocated ng_mw = ng_mw.min(ng); // Fill guard cells. - tmpmf.FillBoundary(0, tmpmf.nComp(), ng_mw, geom.periodicity()); + tmpmf.FillBoundary(ng_mw, geom.periodicity()); // Make a box that covers the region that the window moved into const IndexType& typ = ba.ixType(); -- cgit v1.2.3 From 8cc9c8a153b6526dad68ee2bcdcacf912969e747 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 12 Nov 2019 10:54:38 -0800 Subject: fix psatd: problem was in buffer cells --- Source/Evolve/WarpXEvolveEM.cpp | 4 +--- Source/Parallelization/GuardCellManager.H | 2 +- Source/Parallelization/GuardCellManager.cpp | 16 ++++------------ Source/Parallelization/WarpXComm.cpp | 1 - Source/WarpX.cpp | 4 ++-- 5 files changed, 8 insertions(+), 19 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 7131e4305..19031e006 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -339,8 +339,7 @@ WarpX::OneStep_nosub (Real cur_time) FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); #else EvolveF(0.5*dt[0], DtType::FirstHalf); - // FillBoundaryF(guard_cells.ng_FieldSolver); - FillBoundaryF(guard_cells.ng_alloc_F); + FillBoundaryF(guard_cells.ng_FieldSolver); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} FillBoundaryB(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); @@ -441,7 +440,6 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldGather + guard_cells.ng_Extra); FillBoundaryAux(guard_cells.ng_UpdateAux); - // iii) Get auxiliary fields on the fine grid, at dt[fine_lev] UpdateAuxilaryData(); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 53d5ccc08..c6fdecaee 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -7,7 +7,7 @@ class guardCellManager{ public: - void Init( + int Init( const bool do_subcycling, const bool do_fdtd_nci_corr, const bool do_nodal, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 9d4a06f73..cfe6e8470 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -4,7 +4,7 @@ using namespace amrex; -void +int guardCellManager::Init( const bool do_subcycling, const bool do_fdtd_nci_corr, @@ -69,6 +69,8 @@ guardCellManager::Init( ng_alloc_J = IntVect(ngJx,ngJz); #endif + int nJ_buffer = ng_alloc_J.max(); // guard cells for J required for deposition only. + ng_alloc_Rho = ng_alloc_J+1; //One extra ghost cell, so that it's safe to deposit charge density // after pushing particle. ng_alloc_F_int = (do_moving_window) ? 2 : 0; @@ -149,15 +151,5 @@ guardCellManager::Init( ng_MovingWindow[moving_window_dir] = 1; } - Print()<<"ng_alloc_EB "< #include #include -#include "WarpXAlgorithmSelection.H" #include #include diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 10a6b731e..173a3a00f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -703,7 +703,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d bool aux_is_nodal = (field_gathering_algo == GatheringAlgo::MomentumConserving); - guard_cells.Init( + int nJ_buffer = guard_cells.Init( do_subcycling, WarpX::use_fdtd_nci_corr, do_nodal, @@ -726,7 +726,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d } if (n_current_deposition_buffer < 0) { - n_current_deposition_buffer = guard_cells.ng_alloc_J.max(); + n_current_deposition_buffer = nJ_buffer; } if (n_field_gather_buffer < 0) { // Field gather buffer should be larger than current deposition buffers -- cgit v1.2.3 From 751b6e3867a2f41db88430476891fecb2e35b053 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 12 Nov 2019 16:36:39 -0800 Subject: no guard cell exchanges for F with Yee solver. Remove print statements --- Source/Evolve/WarpXEvolveEM.cpp | 12 ++++++------ Source/Parallelization/GuardCellManager.H | 1 + Source/Parallelization/GuardCellManager.cpp | 3 +++ Source/Parallelization/WarpXComm.cpp | 4 ---- 4 files changed, 10 insertions(+), 10 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 38b959808..af4eb9467 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -340,7 +340,7 @@ WarpX::OneStep_nosub (Real cur_time) FillBoundaryB(guard_cells.ng_alloc_EB, guard_cells.ng_Extra); #else EvolveF(0.5*dt[0], DtType::FirstHalf); - FillBoundaryF(guard_cells.ng_FieldSolver); + FillBoundaryF(guard_cells.ng_FieldSolverF); EvolveB(0.5*dt[0]); // We now have B^{n+1/2} FillBoundaryB(guard_cells.ng_FieldSolver, IntVect::TheZeroVector()); @@ -427,7 +427,7 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::coarse, dt[fine_lev]); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ng_FieldGather); - FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolver); + FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolverF); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_FieldGather); @@ -435,7 +435,7 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); EvolveF(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev], DtType::FirstHalf); FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_FieldGather + guard_cells.ng_Extra); - FillBoundaryF(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver); + FillBoundaryF(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolverF); EvolveE(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldGather + guard_cells.ng_Extra); @@ -456,7 +456,7 @@ WarpX::OneStep_sub1 (Real curtime) EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); EvolveF(fine_lev, PatchType::fine, 0.5*dt[fine_lev], DtType::FirstHalf); FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); - FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolverF); EvolveE(fine_lev, PatchType::fine, dt[fine_lev]); FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); @@ -484,7 +484,7 @@ WarpX::OneStep_sub1 (Real curtime) EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::SecondHalf); if (do_pml) { - FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); + FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolverF); DampPML(fine_lev, PatchType::coarse); // do it twice DampPML(fine_lev, PatchType::coarse); FillBoundaryE(fine_lev, PatchType::coarse, guard_cells.ng_alloc_EB); @@ -492,7 +492,7 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryB(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolver); - FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolver); + FillBoundaryF(fine_lev, PatchType::coarse, guard_cells.ng_FieldSolverF); EvolveE(coarse_lev, PatchType::fine, 0.5*dt[coarse_lev]); FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 9f2bafa19..2e1cebff8 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -30,6 +30,7 @@ public: // Guard cells exchanged for specific in the PIC loop amrex::IntVect ng_FieldSolver = amrex::IntVect::TheZeroVector(); + amrex::IntVect ng_FieldSolverF = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_FieldGather = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_UpdateAux = amrex::IntVect::TheZeroVector(); amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 28670fb91..34454bd7e 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -116,8 +116,10 @@ guardCellManager::Init( // Compute number of cells required for Field Solver #ifdef WARPX_USE_PSATD ng_FieldSolver = ng_alloc_EB; + ng_FieldSolverF = ng_alloc_EB; #else ng_FieldSolver = IntVect(AMREX_D_DECL(1,1,1)); + ng_FieldSolverF = IntVect(AMREX_D_DECL(1,1,1)); #endif ng_FieldSolver = ng_FieldSolver.min(ng_alloc_EB); @@ -144,6 +146,7 @@ guardCellManager::Init( // Make sure we do not exchange more guard cells than allocated. ng_FieldGather = ng_FieldGather.min(ng_alloc_EB); ng_UpdateAux = ng_UpdateAux.min(ng_alloc_EB); + ng_FieldSolverF = ng_FieldSolverF.min(ng_alloc_F); // Only FillBoundary(ng_FieldGather) is called between consecutive // field solves. So ng_FieldGather must have enough cells // for the field solve too. diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 5d7e44a8e..f7e9086f9 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -373,7 +373,6 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Efield_fp[lev][0]->nGrowVect(), "Error: in FillBoundaryE, requested more guard cells than allocated"); - Print()<<"FillBoundaryE exchanges "<< Efield_fp[lev][0]->nGrowVect() <<'\n'; Efield_fp[lev][0]->FillBoundary(ng, period); Efield_fp[lev][1]->FillBoundary(ng, period); Efield_fp[lev][2]->FillBoundary(ng, period); @@ -394,7 +393,6 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Efield_cp[lev][0]->nGrowVect(), "Error: in FillBoundaryE, requested more guard cells than allocated"); - Print()<<"FillBoundaryE exchanges "<< Efield_cp[lev][0]->nGrowVect() <<'\n'; Efield_cp[lev][0]->FillBoundary(ng, cperiod); Efield_cp[lev][1]->FillBoundary(ng, cperiod); Efield_cp[lev][2]->FillBoundary(ng, cperiod); @@ -426,7 +424,6 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Bfield_fp[lev][0]->nGrowVect(), "Error: in FillBoundaryB, requested more guard cells than allocated"); - Print()<<"FillBoundaryB exchanges "<< Bfield_fp[lev][0]->nGrowVect() <<'\n'; Bfield_fp[lev][0]->FillBoundary(ng, period); Bfield_fp[lev][1]->FillBoundary(ng, period); Bfield_fp[lev][2]->FillBoundary(ng, period); @@ -446,7 +443,6 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) AMREX_ALWAYS_ASSERT_WITH_MESSAGE( ng <= Bfield_cp[lev][0]->nGrowVect(), "Error: in FillBoundaryB, requested more guard cells than allocated"); - Print()<<"FillBoundaryB exchanges "<< Bfield_cp[lev][0]->nGrowVect() <<'\n'; Bfield_cp[lev][0]->FillBoundary(ng, cperiod); Bfield_cp[lev][1]->FillBoundary(ng, cperiod); Bfield_cp[lev][2]->FillBoundary(ng, cperiod); -- cgit v1.2.3