From c7c8a710cb1b99bc72343f4215011f3c0ec4f16e Mon Sep 17 00:00:00 2001 From: David Grote Date: Thu, 20 Jan 2022 15:53:23 -0800 Subject: Implement PML for the outer RZ boundary with PSATD (#2211) * Initial version of RZ PSATD PML BCs * Cleaned up some bugs * Add support of do_pml_in_domain option * Cleaned up stuff for building * Fix PMLPsatdAlgorithm macro * Removed unneeded variable from SpectralSolverRZ * Change length 3 arrays to length 2 (for 2D) * Cleanup around DampPML * Added more checks of pml[lev] * Added CI test for RZ PML * Added code to update the corner guard cells * Further updates * Added CI test * Fixed EOL space * Updated CI benchmarks, removing round off fields * Changes to CI missed on previous commit * Various fixes for clean up * More fixes for clean up * Further cleanup * Updated benchmark * Fixed benchmarks file * Minor cleanup * Added round off benchmark values * Fixed testname in analysis_pml_psatd_rz.py * Update comment in analysis file * Put pml_rz code in RZ and PSATD macro blocks * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add geometry.dims input to CI test input file, inputs_rz * Cleanup to match recent changes Co-authored-by: Remi Lehe Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Source/Parallelization/WarpXComm.cpp | 79 +++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 20 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index d0f7e352a..9e4a7de88 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -9,6 +9,9 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +# include "BoundaryConditions/PML_RZ.H" +#endif #include "Filter/BilinearFilter.H" #include "Utils/CoarsenMR.H" #include "Utils/IntervalsParser.H" @@ -548,14 +551,22 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { - if (do_pml && pml[lev]->ok()) - { - pml[lev]->ExchangeE(patch_type, - { Efield_fp[lev][0].get(), - Efield_fp[lev][1].get(), - Efield_fp[lev][2].get() }, - do_pml_in_domain); - pml[lev]->FillBoundaryE(patch_type); + if (do_pml) { + if (pml[lev] && pml[lev]->ok()) + { + pml[lev]->ExchangeE(patch_type, + { Efield_fp[lev][0].get(), + Efield_fp[lev][1].get(), + Efield_fp[lev][2].get() }, + do_pml_in_domain); + pml[lev]->FillBoundaryE(patch_type); + } +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) + if (pml_rz[lev]) + { + pml_rz[lev]->FillBoundaryE(patch_type); + } +#endif } const amrex::Periodicity& period = Geom(lev).periodicity(); @@ -610,15 +621,25 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { - if (do_pml && pml[lev]->ok()) + if (do_pml) { - pml[lev]->ExchangeB(patch_type, - { Bfield_fp[lev][0].get(), - Bfield_fp[lev][1].get(), - Bfield_fp[lev][2].get() }, - do_pml_in_domain); - pml[lev]->FillBoundaryB(patch_type); + if (pml[lev] && pml[lev]->ok()) + { + pml[lev]->ExchangeB(patch_type, + { Bfield_fp[lev][0].get(), + Bfield_fp[lev][1].get(), + Bfield_fp[lev][2].get() }, + do_pml_in_domain); + pml[lev]->FillBoundaryB(patch_type); + } +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) + if (pml_rz[lev]) + { + pml_rz[lev]->FillBoundaryB(patch_type); + } +#endif } + const amrex::Periodicity& period = Geom(lev).periodicity(); if ( safe_guard_cells ) { Vector mf{Bfield_fp[lev][0].get(),Bfield_fp[lev][1].get(),Bfield_fp[lev][2].get()}; @@ -777,7 +798,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { - if (do_pml && pml[lev]->ok()) + if (do_pml && pml[lev] && pml[lev]->ok()) { if (F_fp[lev]) pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); pml[lev]->FillBoundaryF(patch_type); @@ -792,7 +813,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng) } else if (patch_type == PatchType::coarse) { - if (do_pml && pml[lev]->ok()) + if (do_pml && pml[lev] && pml[lev]->ok()) { if (F_cp[lev]) pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); pml[lev]->FillBoundaryF(patch_type); @@ -821,7 +842,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng) { if (patch_type == PatchType::fine) { - if (do_pml && pml[lev]->ok()) + if (do_pml && pml[lev] && pml[lev]->ok()) { if (G_fp[lev]) pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); pml[lev]->FillBoundaryG(patch_type); @@ -836,7 +857,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng) } else if (patch_type == PatchType::coarse) { - if (do_pml && pml[lev]->ok()) + if (do_pml && pml[lev] && pml[lev]->ok()) { if (G_cp[lev]) pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); pml[lev]->FillBoundaryG(patch_type); @@ -1284,7 +1305,7 @@ void WarpX::NodalSyncPML (int lev) void WarpX::NodalSyncPML (int lev, PatchType patch_type) { - if (pml[lev]->ok()) + if (pml[lev] && pml[lev]->ok()) { const std::array& pml_E = (patch_type == PatchType::fine) ? pml[lev]->GetE_fp() : pml[lev]->GetE_cp(); @@ -1308,6 +1329,24 @@ void WarpX::NodalSyncPML (int lev, PatchType patch_type) WarpXCommUtil::OverrideSync(*pml_G, period); } } + +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) + if (pml_rz[lev]) + { + // This is not actually needed with RZ PSATD since the + // arrays are always cell centered. Keep for now since + // it may be useful if the PML is used with FDTD + const std::array pml_rz_E = pml_rz[lev]->GetE_fp(); + const std::array pml_rz_B = pml_rz[lev]->GetB_fp(); + + // Always synchronize nodal points + const amrex::Periodicity& period = Geom(lev).periodicity(); + pml_rz_E[0]->OverrideSync(period); + pml_rz_E[1]->OverrideSync(period); + pml_rz_B[0]->OverrideSync(period); + pml_rz_B[1]->OverrideSync(period); + } +#endif } void WarpX::NodalSync (amrex::Vector,3>>& mf_fp, -- cgit v1.2.3