From 7195cc8eecf81896f3f2cd57eaf6161dd6834f5c Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Thu, 16 May 2019 17:56:07 -0700 Subject: Full implementation of the multimode RZ solver --- Source/Parallelization/WarpXRegrid.cpp | 66 +++++++++++++++++----------------- 1 file changed, 33 insertions(+), 33 deletions(-) (limited to 'Source/Parallelization/WarpXRegrid.cpp') diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 8d7873041..eb119d4a2 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -46,21 +46,21 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa { const IntVect& ng = Bfield_fp[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(Bfield_fp[lev][idim]->boxArray(), - dm, 1, ng)); - pmf->Redistribute(*Bfield_fp[lev][idim], 0, 0, 1, ng); + dm, Bfield_fp[lev][idim]->nComp(), ng)); + pmf->Redistribute(*Bfield_fp[lev][idim], 0, 0, Bfield_fp[lev][idim]->nComp(), ng); Bfield_fp[lev][idim] = std::move(pmf); } { const IntVect& ng = Efield_fp[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(Efield_fp[lev][idim]->boxArray(), - dm, 1, ng)); - pmf->Redistribute(*Efield_fp[lev][idim], 0, 0, 1, ng); + dm, Efield_fp[lev][idim]->nComp(), ng)); + pmf->Redistribute(*Efield_fp[lev][idim], 0, 0, Efield_fp[lev][idim]->nComp(), ng); Efield_fp[lev][idim] = std::move(pmf); } { const IntVect& ng = current_fp[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(current_fp[lev][idim]->boxArray(), - dm, 1, ng)); + dm, current_fp[lev][idim]->nComp(), ng)); current_fp[lev][idim] = std::move(pmf); current_fp_owner_masks[lev][idim] = std::move(current_fp[lev][idim]->OwnerMask(period)); } @@ -68,7 +68,7 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa { const IntVect& ng = current_store[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(current_store[lev][idim]->boxArray(), - dm, 1, ng)); + dm, current_store[lev][idim]->nComp(), ng)); // no need to redistribute current_store[lev][idim] = std::move(pmf); } @@ -77,8 +77,8 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa if (F_fp[lev] != nullptr) { const IntVect& ng = F_fp[lev]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(F_fp[lev]->boxArray(), - dm, 1, ng)); - pmf->Redistribute(*F_fp[lev], 0, 0, 1, ng); + dm, F_fp[lev]->nComp(), ng)); + pmf->Redistribute(*F_fp[lev], 0, 0, F_fp[lev]->nComp(), ng); F_fp[lev] = std::move(pmf); } @@ -96,8 +96,8 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa if (lev == 0) { for (int idim = 0; idim < 3; ++idim) { - Bfield_aux[lev][idim].reset(new MultiFab(*Bfield_fp[lev][idim], amrex::make_alias, 0, 1)); - Efield_aux[lev][idim].reset(new MultiFab(*Efield_fp[lev][idim], amrex::make_alias, 0, 1)); + Bfield_aux[lev][idim].reset(new MultiFab(*Bfield_fp[lev][idim], amrex::make_alias, 0, Bfield_aux[lev][idim]->nComp())); + Efield_aux[lev][idim].reset(new MultiFab(*Efield_fp[lev][idim], amrex::make_alias, 0, Efield_aux[lev][idim]->nComp())); } } else { for (int idim=0; idim < 3; ++idim) @@ -105,15 +105,15 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa { const IntVect& ng = Bfield_aux[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(Bfield_aux[lev][idim]->boxArray(), - dm, 1, ng)); - // pmf->Redistribute(*Bfield_aux[lev][idim], 0, 0, 1, ng); + dm, Bfield_aux[lev][idim]->nComp(), ng)); + // pmf->Redistribute(*Bfield_aux[lev][idim], 0, 0, Bfield_aux[lev][idim]->nComp(), ng); Bfield_aux[lev][idim] = std::move(pmf); } { const IntVect& ng = Efield_aux[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(Efield_aux[lev][idim]->boxArray(), - dm, 1, ng)); - // pmf->Redistribute(*Efield_aux[lev][idim], 0, 0, 1, ng); + dm, Efield_aux[lev][idim]->nComp(), ng)); + // pmf->Redistribute(*Efield_aux[lev][idim], 0, 0, Efield_aux[lev][idim]->nComp(), ng); Efield_aux[lev][idim] = std::move(pmf); } } @@ -127,21 +127,21 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa { const IntVect& ng = Bfield_cp[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(Bfield_cp[lev][idim]->boxArray(), - dm, 1, ng)); - pmf->Redistribute(*Bfield_cp[lev][idim], 0, 0, 1, ng); + dm, Bfield_cp[lev][idim]->nComp(), ng)); + pmf->Redistribute(*Bfield_cp[lev][idim], 0, 0, Bfield_cp[lev][idim]->nComp(), ng); Bfield_cp[lev][idim] = std::move(pmf); } { const IntVect& ng = Efield_cp[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(Efield_cp[lev][idim]->boxArray(), - dm, 1, ng)); - pmf->Redistribute(*Efield_cp[lev][idim], 0, 0, 1, ng); + dm, Efield_cp[lev][idim]->nComp(), ng)); + pmf->Redistribute(*Efield_cp[lev][idim], 0, 0, Efield_cp[lev][idim]->nComp(), ng); Efield_cp[lev][idim] = std::move(pmf); } { const IntVect& ng = current_cp[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr( new MultiFab(current_cp[lev][idim]->boxArray(), - dm, 1, ng)); + dm, current_cp[lev][idim]->nComp(), ng)); current_cp[lev][idim] = std::move(pmf); current_cp_owner_masks[lev][idim] = std::move( current_cp[lev][idim]->OwnerMask(cperiod)); @@ -151,8 +151,8 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa if (F_cp[lev] != nullptr) { const IntVect& ng = F_cp[lev]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(F_cp[lev]->boxArray(), - dm, 1, ng)); - pmf->Redistribute(*F_cp[lev], 0, 0, 1, ng); + dm, F_cp[lev]->nComp(), ng)); + pmf->Redistribute(*F_cp[lev], 0, 0, F_cp[lev]->nComp(), ng); F_cp[lev] = std::move(pmf); } @@ -173,24 +173,24 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa { const IntVect& ng = Bfield_cax[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(Bfield_cax[lev][idim]->boxArray(), - dm, 1, ng)); - // pmf->ParallelCopy(*Bfield_cax[lev][idim], 0, 0, 1, ng, ng); + dm, Bfield_cax[lev][idim]->nComp(), ng)); + // pmf->ParallelCopy(*Bfield_cax[lev][idim], 0, 0, Bfield_cax[lev][idim]->nComp(), ng, ng); Bfield_cax[lev][idim] = std::move(pmf); } if (Efield_cax[lev][idim]) { const IntVect& ng = Efield_cax[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(Efield_cax[lev][idim]->boxArray(), - dm, 1, ng)); - // pmf->ParallelCopy(*Efield_cax[lev][idim], 0, 0, 1, ng, ng); + dm, Efield_cax[lev][idim]->nComp(), ng)); + // pmf->ParallelCopy(*Efield_cax[lev][idim], 0, 0, Efield_cax[lev][idim]->nComp(), ng, ng); Efield_cax[lev][idim] = std::move(pmf); } if (current_buf[lev][idim]) { const IntVect& ng = current_buf[lev][idim]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(current_buf[lev][idim]->boxArray(), - dm, 1, ng)); - // pmf->ParallelCopy(*current_buf[lev][idim], 0, 0, 1, ng, ng); + dm, current_buf[lev][idim]->nComp(), ng)); + // pmf->ParallelCopy(*current_buf[lev][idim], 0, 0, current_buf[lev][idim]->nComp(), ng, ng); current_buf[lev][idim] = std::move(pmf); } } @@ -198,24 +198,24 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa { const IntVect& ng = charge_buf[lev]->nGrowVect(); auto pmf = std::unique_ptr(new MultiFab(charge_buf[lev]->boxArray(), - dm, 1, ng)); - // pmf->ParallelCopy(*charge_buf[lev][idim], 0, 0, 1, ng, ng); + dm, charge_buf[lev]->nComp(), ng)); + // pmf->ParallelCopy(*charge_buf[lev][idim], 0, 0, charge_buf[lev]->nComp(), ng, ng); charge_buf[lev] = std::move(pmf); } if (current_buffer_masks[lev]) { const IntVect& ng = current_buffer_masks[lev]->nGrowVect(); auto pmf = std::unique_ptr(new iMultiFab(current_buffer_masks[lev]->boxArray(), - dm, 1, ng)); - // pmf->ParallelCopy(*current_buffer_masks[lev], 0, 0, 1, ng, ng); + dm, current_buffer_masks[lev]->nComp(), ng)); + // pmf->ParallelCopy(*current_buffer_masks[lev], 0, 0, current_buffer_masks[lev]->nComp(), ng, ng); current_buffer_masks[lev] = std::move(pmf); } if (gather_buffer_masks[lev]) { const IntVect& ng = gather_buffer_masks[lev]->nGrowVect(); auto pmf = std::unique_ptr(new iMultiFab(gather_buffer_masks[lev]->boxArray(), - dm, 1, ng)); - // pmf->ParallelCopy(*gather_buffer_masks[lev], 0, 0, 1, ng, ng); + dm, gather_buffer_masks[lev]->nComp(), ng)); + // pmf->ParallelCopy(*gather_buffer_masks[lev], 0, 0, gather_buffer_masks[lev]->nComp(), ng, ng); gather_buffer_masks[lev] = std::move(pmf); } } -- cgit v1.2.3 From 798f107335d7cd8bde4e0191c202631c97df02b8 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Wed, 4 Sep 2019 15:33:48 -0700 Subject: remove owner masks because they have not been used anymore --- Source/Parallelization/WarpXRegrid.cpp | 5 ----- Source/WarpX.H | 6 ------ Source/WarpX.cpp | 25 ------------------------- 3 files changed, 36 deletions(-) (limited to 'Source/Parallelization/WarpXRegrid.cpp') diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index eb119d4a2..9b3baafe1 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -62,7 +62,6 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa auto pmf = std::unique_ptr(new MultiFab(current_fp[lev][idim]->boxArray(), dm, current_fp[lev][idim]->nComp(), ng)); current_fp[lev][idim] = std::move(pmf); - current_fp_owner_masks[lev][idim] = std::move(current_fp[lev][idim]->OwnerMask(period)); } if (current_store[lev][idim]) { @@ -88,7 +87,6 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa auto pmf = std::unique_ptr(new MultiFab(rho_fp[lev]->boxArray(), dm, nc, ng)); rho_fp[lev] = std::move(pmf); - rho_fp_owner_masks[lev] = std::move(rho_fp[lev]->OwnerMask(period)); } // Aux patch @@ -143,8 +141,6 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa auto pmf = std::unique_ptr( new MultiFab(current_cp[lev][idim]->boxArray(), dm, current_cp[lev][idim]->nComp(), ng)); current_cp[lev][idim] = std::move(pmf); - current_cp_owner_masks[lev][idim] = std::move( - current_cp[lev][idim]->OwnerMask(cperiod)); } } @@ -162,7 +158,6 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa auto pmf = std::unique_ptr(new MultiFab(rho_cp[lev]->boxArray(), dm, nc, ng)); rho_cp[lev] = std::move(pmf); - rho_cp_owner_masks[lev] = std::move(rho_cp[lev]->OwnerMask(cperiod)); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index f819ba6ca..4f0342c8a 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -496,12 +496,6 @@ private: amrex::Vector, 3 > > current_buf; amrex::Vector > charge_buf; - amrex::Vector, 3 > > current_fp_owner_masks; - amrex::Vector, 3 > > current_cp_owner_masks; - - amrex::Vector > rho_fp_owner_masks; - amrex::Vector > rho_cp_owner_masks; - // div E cleaning int do_dive_cleaning = 0; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 7695cb91d..f5d75879b 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -190,11 +190,6 @@ WarpX::WarpX () current_buf.resize(nlevs_max); charge_buf.resize(nlevs_max); - current_fp_owner_masks.resize(nlevs_max); - current_cp_owner_masks.resize(nlevs_max); - rho_fp_owner_masks.resize(nlevs_max); - rho_cp_owner_masks.resize(nlevs_max); - pml.resize(nlevs_max); #ifdef WARPX_DO_ELECTROSTATIC @@ -640,14 +635,8 @@ WarpX::ClearLevel (int lev) Efield_cax[lev][i].reset(); Bfield_cax[lev][i].reset(); current_buf[lev][i].reset(); - - current_fp_owner_masks[lev][i].reset(); - current_cp_owner_masks[lev][i].reset(); } - rho_fp_owner_masks[lev].reset(); - rho_cp_owner_masks[lev].reset(); - charge_buf[lev].reset(); current_buffer_masks[lev].reset(); @@ -815,15 +804,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm current_fp[lev][1].reset( new MultiFab(amrex::convert(ba,jy_nodal_flag),dm,ncomps,ngJ)); current_fp[lev][2].reset( new MultiFab(amrex::convert(ba,jz_nodal_flag),dm,ncomps,ngJ)); - const auto& period = Geom(lev).periodicity(); - current_fp_owner_masks[lev][0] = std::move(current_fp[lev][0]->OwnerMask(period)); - current_fp_owner_masks[lev][1] = std::move(current_fp[lev][1]->OwnerMask(period)); - current_fp_owner_masks[lev][2] = std::move(current_fp[lev][2]->OwnerMask(period)); - if (do_dive_cleaning || plot_rho) { rho_fp[lev].reset(new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()),dm,2*ncomps,ngRho)); - rho_fp_owner_masks[lev] = std::move(rho_fp[lev]->OwnerMask(period)); } if (do_subcycling == 1 && lev == 0) @@ -841,7 +824,6 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm else { rho_fp[lev].reset(new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()),dm,2*ncomps,ngRho)); - rho_fp_owner_masks[lev] = std::move(rho_fp[lev]->OwnerMask(period)); } if (fft_hybrid_mpi_decomposition == false){ // Allocate and initialize the spectral solver @@ -904,14 +886,8 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm current_cp[lev][1].reset( new MultiFab(amrex::convert(cba,jy_nodal_flag),dm,ncomps,ngJ)); current_cp[lev][2].reset( new MultiFab(amrex::convert(cba,jz_nodal_flag),dm,ncomps,ngJ)); - const auto& cperiod = Geom(lev-1).periodicity(); - current_cp_owner_masks[lev][0] = std::move(current_cp[lev][0]->OwnerMask(cperiod)); - current_cp_owner_masks[lev][1] = std::move(current_cp[lev][1]->OwnerMask(cperiod)); - current_cp_owner_masks[lev][2] = std::move(current_cp[lev][2]->OwnerMask(cperiod)); - if (do_dive_cleaning || plot_rho){ rho_cp[lev].reset(new MultiFab(amrex::convert(cba,IntVect::TheUnitVector()),dm,2*ncomps,ngRho)); - rho_cp_owner_masks[lev] = std::move(rho_cp[lev]->OwnerMask(cperiod)); } if (do_dive_cleaning) { @@ -921,7 +897,6 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm else { rho_cp[lev].reset(new MultiFab(amrex::convert(cba,IntVect::TheUnitVector()),dm,2*ncomps,ngRho)); - rho_cp_owner_masks[lev] = std::move(rho_cp[lev]->OwnerMask(cperiod)); } if (fft_hybrid_mpi_decomposition == false){ // Allocate and initialize the spectral solver -- cgit v1.2.3