aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Docs/source/running_cpp/parameters.rst7
-rw-r--r--Source/Parallelization/WarpXComm.cpp4
-rw-r--r--Source/Parallelization/WarpXRegrid.cpp5
-rw-r--r--Source/WarpX.H9
-rw-r--r--Source/WarpX.cpp28
5 files changed, 16 insertions, 37 deletions
diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst
index ea681470e..0e2866f58 100644
--- a/Docs/source/running_cpp/parameters.rst
+++ b/Docs/source/running_cpp/parameters.rst
@@ -655,6 +655,13 @@ Numerics and algorithms
See `this section of the FFTW documentation <http://www.fftw.org/fftw3_doc/Planner-Flags.html>`__
for more information.
+* ``warpx.override_sync_int`` (`integer`) optional (default `10`)
+ Number of time steps between synchronization of sources (`rho` and `J`) on
+ grid nodes at box boundaries. Since the grid nodes at the interface between
+ two neighbor boxes are duplicated in both boxes, an instability can occur
+ if they have too different values. This option makes sure that they are
+ synchronized periodically.
+
Boundary conditions
-------------------
diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp
index 88adbc147..93fc12799 100644
--- a/Source/Parallelization/WarpXComm.cpp
+++ b/Source/Parallelization/WarpXComm.cpp
@@ -714,6 +714,8 @@ WarpX::AddRhoFromFineLevelandSumBoundary(int lev, int icomp, int ncomp)
void
WarpX::NodalSyncJ (int lev, PatchType patch_type)
{
+ if (override_sync_int <= 0 or istep[0] % override_sync_int != 0) return;
+
if (patch_type == PatchType::fine)
{
const auto& period = Geom(lev).periodicity();
@@ -733,6 +735,8 @@ WarpX::NodalSyncJ (int lev, PatchType patch_type)
void
WarpX::NodalSyncRho (int lev, PatchType patch_type, int icomp, int ncomp)
{
+ if (override_sync_int <= 0 or istep[0] % override_sync_int != 0) return;
+
if (patch_type == PatchType::fine && rho_fp[lev])
{
const auto& period = Geom(lev).periodicity();
diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp
index eb119d4a2..9b3baafe1 100644
--- a/Source/Parallelization/WarpXRegrid.cpp
+++ b/Source/Parallelization/WarpXRegrid.cpp
@@ -62,7 +62,6 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa
auto pmf = std::unique_ptr<MultiFab>(new MultiFab(current_fp[lev][idim]->boxArray(),
dm, current_fp[lev][idim]->nComp(), ng));
current_fp[lev][idim] = std::move(pmf);
- current_fp_owner_masks[lev][idim] = std::move(current_fp[lev][idim]->OwnerMask(period));
}
if (current_store[lev][idim])
{
@@ -88,7 +87,6 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa
auto pmf = std::unique_ptr<MultiFab>(new MultiFab(rho_fp[lev]->boxArray(),
dm, nc, ng));
rho_fp[lev] = std::move(pmf);
- rho_fp_owner_masks[lev] = std::move(rho_fp[lev]->OwnerMask(period));
}
// Aux patch
@@ -143,8 +141,6 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa
auto pmf = std::unique_ptr<MultiFab>( new MultiFab(current_cp[lev][idim]->boxArray(),
dm, current_cp[lev][idim]->nComp(), ng));
current_cp[lev][idim] = std::move(pmf);
- current_cp_owner_masks[lev][idim] = std::move(
- current_cp[lev][idim]->OwnerMask(cperiod));
}
}
@@ -162,7 +158,6 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa
auto pmf = std::unique_ptr<MultiFab>(new MultiFab(rho_cp[lev]->boxArray(),
dm, nc, ng));
rho_cp[lev] = std::move(pmf);
- rho_cp_owner_masks[lev] = std::move(rho_cp[lev]->OwnerMask(cperiod));
}
}
diff --git a/Source/WarpX.H b/Source/WarpX.H
index 9236b8e20..4f0342c8a 100644
--- a/Source/WarpX.H
+++ b/Source/WarpX.H
@@ -496,12 +496,6 @@ private:
amrex::Vector<std::array< std::unique_ptr<amrex::MultiFab>, 3 > > current_buf;
amrex::Vector<std::unique_ptr<amrex::MultiFab> > charge_buf;
- amrex::Vector<std::array<std::unique_ptr<amrex::iMultiFab>, 3 > > current_fp_owner_masks;
- amrex::Vector<std::array<std::unique_ptr<amrex::iMultiFab>, 3 > > current_cp_owner_masks;
-
- amrex::Vector<std::unique_ptr<amrex::iMultiFab> > rho_fp_owner_masks;
- amrex::Vector<std::unique_ptr<amrex::iMultiFab> > rho_cp_owner_masks;
-
// div E cleaning
int do_dive_cleaning = 0;
@@ -533,6 +527,9 @@ private:
int load_balance_with_sfc = 0;
amrex::Real load_balance_knapsack_factor = 1.24;
+ // Override sync every ? steps
+ int override_sync_int = 10;
+
// Other runtime parameters
int verbose = 1;
diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp
index 95826c075..f5d75879b 100644
--- a/Source/WarpX.cpp
+++ b/Source/WarpX.cpp
@@ -190,11 +190,6 @@ WarpX::WarpX ()
current_buf.resize(nlevs_max);
charge_buf.resize(nlevs_max);
- current_fp_owner_masks.resize(nlevs_max);
- current_cp_owner_masks.resize(nlevs_max);
- rho_fp_owner_masks.resize(nlevs_max);
- rho_cp_owner_masks.resize(nlevs_max);
-
pml.resize(nlevs_max);
#ifdef WARPX_DO_ELECTROSTATIC
@@ -281,7 +276,8 @@ WarpX::ReadParameters ()
pp.query("cfl", cfl);
pp.query("verbose", verbose);
pp.query("regrid_int", regrid_int);
- pp.query("do_subcycling", do_subcycling);
+ pp.query("do_subcycling", do_subcycling);
+ pp.query("override_sync_int", override_sync_int);
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_subcycling != 1 || max_level <= 1,
"Subcycling method 1 only works for 2 levels.");
@@ -639,14 +635,8 @@ WarpX::ClearLevel (int lev)
Efield_cax[lev][i].reset();
Bfield_cax[lev][i].reset();
current_buf[lev][i].reset();
-
- current_fp_owner_masks[lev][i].reset();
- current_cp_owner_masks[lev][i].reset();
}
- rho_fp_owner_masks[lev].reset();
- rho_cp_owner_masks[lev].reset();
-
charge_buf[lev].reset();
current_buffer_masks[lev].reset();
@@ -814,15 +804,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm
current_fp[lev][1].reset( new MultiFab(amrex::convert(ba,jy_nodal_flag),dm,ncomps,ngJ));
current_fp[lev][2].reset( new MultiFab(amrex::convert(ba,jz_nodal_flag),dm,ncomps,ngJ));
- const auto& period = Geom(lev).periodicity();
- current_fp_owner_masks[lev][0] = std::move(current_fp[lev][0]->OwnerMask(period));
- current_fp_owner_masks[lev][1] = std::move(current_fp[lev][1]->OwnerMask(period));
- current_fp_owner_masks[lev][2] = std::move(current_fp[lev][2]->OwnerMask(period));
-
if (do_dive_cleaning || plot_rho)
{
rho_fp[lev].reset(new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()),dm,2*ncomps,ngRho));
- rho_fp_owner_masks[lev] = std::move(rho_fp[lev]->OwnerMask(period));
}
if (do_subcycling == 1 && lev == 0)
@@ -840,7 +824,6 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm
else
{
rho_fp[lev].reset(new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()),dm,2*ncomps,ngRho));
- rho_fp_owner_masks[lev] = std::move(rho_fp[lev]->OwnerMask(period));
}
if (fft_hybrid_mpi_decomposition == false){
// Allocate and initialize the spectral solver
@@ -903,14 +886,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm
current_cp[lev][1].reset( new MultiFab(amrex::convert(cba,jy_nodal_flag),dm,ncomps,ngJ));
current_cp[lev][2].reset( new MultiFab(amrex::convert(cba,jz_nodal_flag),dm,ncomps,ngJ));
- const auto& cperiod = Geom(lev-1).periodicity();
- current_cp_owner_masks[lev][0] = std::move(current_cp[lev][0]->OwnerMask(cperiod));
- current_cp_owner_masks[lev][1] = std::move(current_cp[lev][1]->OwnerMask(cperiod));
- current_cp_owner_masks[lev][2] = std::move(current_cp[lev][2]->OwnerMask(cperiod));
-
if (do_dive_cleaning || plot_rho){
rho_cp[lev].reset(new MultiFab(amrex::convert(cba,IntVect::TheUnitVector()),dm,2*ncomps,ngRho));
- rho_cp_owner_masks[lev] = std::move(rho_cp[lev]->OwnerMask(cperiod));
}
if (do_dive_cleaning)
{
@@ -920,7 +897,6 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm
else
{
rho_cp[lev].reset(new MultiFab(amrex::convert(cba,IntVect::TheUnitVector()),dm,2*ncomps,ngRho));
- rho_cp_owner_masks[lev] = std::move(rho_cp[lev]->OwnerMask(cperiod));
}
if (fft_hybrid_mpi_decomposition == false){
// Allocate and initialize the spectral solver
generated by cgit v1.2.3 (git 2.46.0) at 2025-08-13 03:24:04 +0000